From 31027a997c3cc5de9140930560d5155bef26289b Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Wed, 14 Jul 2021 10:38:21 -0500 Subject: Add support for parallel filters to h5repack (#832) --- tools/src/h5repack/h5repack_copy.c | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index 536de69..934b4d1 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -626,6 +626,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */ hid_t ocpl_id = H5I_INVALID_HID; /* property to pass copy options */ hid_t lcpl_id = H5I_INVALID_HID; /* link creation property list */ + hid_t dxpl_id = H5I_INVALID_HID; /* dataset transfer property list */ named_dt_t * named_dt_head = NULL; /* Pointer to the stack of named datatypes copied */ size_t msize; /* size of type */ hsize_t nelmts; /* number of elements in dataset */ @@ -995,12 +996,27 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti if (need < H5TOOLS_MALLOCSIZE) buf = HDmalloc(need); + /* Set up collective write if using filters in parallel */ + { +#ifdef H5_HAVE_PARALLEL + hbool_t parallel = (H5FD_MPIO == H5Pget_driver(options->fout_fapl)); + + if (parallel && apply_s && apply_f) { + if ((dxpl_id = H5Pcreate(H5P_DATASET_XFER)) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pcreate failed"); + if (H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pset_dxpl_mpio failed"); + } + else +#endif + dxpl_id = H5P_DEFAULT; + } + if (buf != NULL) { if (H5Dread(dset_in, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dread failed"); - if (H5Dwrite(dset_out, wtype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < - 0) + if (H5Dwrite(dset_out, wtype_id, H5S_ALL, H5S_ALL, dxpl_id, buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dwrite failed"); /* Check if we have VL data in the dataset's @@ -1102,8 +1118,8 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti if (H5Dread(dset_in, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dread failed"); - if (H5Dwrite(dset_out, wtype_id, hslab_space, f_space_id, - H5P_DEFAULT, hslab_buf) < 0) + if (H5Dwrite(dset_out, wtype_id, hslab_space, f_space_id, dxpl_id, + hslab_buf) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Dwrite failed"); /* reclaim any VL memory, if necessary */ @@ -1382,6 +1398,7 @@ done: H5Pclose(dcpl_in); H5Pclose(gcpl_in); H5Pclose(gcpl_out); + H5Pclose(dxpl_id); H5Sclose(f_space_id); H5Dclose(dset_in); H5Dclose(dset_out); -- cgit v0.12 From 21c6bb38e6cf0776dbdc37841280dffd6ca79990 Mon Sep 17 00:00:00 2001 From: Jan-Willem Blokland Date: Wed, 14 Jul 2021 17:42:14 +0200 Subject: CMake: (feature) libaec (#703) * CMake: (feature) libaec Implemented better support for the compression library libaec. Especially, for the new version 1.0.5: - The CMake config files of this version 1.0.5 now set certain SZIP variables which can be used in the existing CMake structure. - Both static and shared SZIP compatible libaec library contains all required objects such it can be easily used when building HDF5 from scratch. - Introduced the USE_LIBAEC_STATIC option to make use of the static version of SZIP compatible libaec library. * CMake: (fix) libaec Implemented the fallback option for the case if version 1.0.5 of libaec could not be found. * Docs: Added description about libaec implementation. Added short description about the libaec improvement to the Bug Fixes section in the Configuration sub-section. --- CMakeFilters.cmake | 18 +++++++++++++++--- release_docs/RELEASE.txt | 15 +++++++++++++++ 2 files changed, 30 insertions(+), 3 deletions(-) diff --git a/CMakeFilters.cmake b/CMakeFilters.cmake index d5f801e..51ac61c 100644 --- a/CMakeFilters.cmake +++ b/CMakeFilters.cmake @@ -10,6 +10,7 @@ # help@hdfgroup.org. # option (USE_LIBAEC "Use AEC library as SZip Filter" OFF) +option (USE_LIBAEC_STATIC "Use static AEC library " OFF) include (ExternalProject) include (FetchContent) @@ -110,13 +111,24 @@ option (HDF5_ENABLE_SZIP_SUPPORT "Use SZip Filter" OFF) if (HDF5_ENABLE_SZIP_SUPPORT) option (HDF5_ENABLE_SZIP_ENCODING "Use SZip Encoding" OFF) if (NOT SZIP_USE_EXTERNAL) - find_package (SZIP NAMES ${SZIP_PACKAGE_NAME}${HDF_PACKAGE_EXT} COMPONENTS static shared) - if (NOT SZIP_FOUND) - find_package (SZIP) # Legacy find + set(SZIP_FOUND FALSE) + if (USE_LIBAEC) + set(libaec_USE_STATIC_LIBS ${USE_LIBAEC_STATIC}) + find_package (libaec 1.0.5 CONFIG) if (SZIP_FOUND) set (LINK_COMP_LIBS ${LINK_COMP_LIBS} ${SZIP_LIBRARIES}) endif () endif () + + if (NOT SZIP_FOUND) + find_package (SZIP NAMES ${SZIP_PACKAGE_NAME}${HDF_PACKAGE_EXT} COMPONENTS static shared) + if (NOT SZIP_FOUND) + find_package (SZIP) # Legacy find + if (SZIP_FOUND) + set (LINK_COMP_LIBS ${LINK_COMP_LIBS} ${SZIP_LIBRARIES}) + endif () + endif () + endif () endif () if (SZIP_FOUND) set (H5_HAVE_FILTER_SZIP 1) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 1e69f54..bd186d8 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1060,6 +1060,21 @@ Bug Fixes since HDF5-1.12.0 release Configuration ------------- + - Better support for libaec (open-source Szip library) in CMake + + Implemented better support for libaec 1.0.5 (or later) library. This version + of libaec contains improvements for better integration with HDF5. Furthermore, + the variable USE_LIBAEC_STATIC has been introduced to allow to make use of + static version of libaec library. Use libaec_DIR or libaec_ROOT to set + the location in which libaec can be found. + + Be aware, the Szip library of libaec 1.0.4 depends on another library within + libaec library. This dependency is not specified in the current CMake + configuration which means that one can not use the static Szip library of + libaec 1.0.4 when building HDF5. This has been resolved in libaec 1.0.5. + + (JWSB - 2021/06/22) + - Refactor CMake configure for Fortran The Fortran configure tests for KINDs reused a single output file that was -- cgit v0.12 From 9ec773bb18b8e7ddce5e347577329fcc8f7a73a9 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Wed, 14 Jul 2021 11:12:01 -0500 Subject: Changes to docs for using plugins (#814) * Changes to docs for using plugins * VS2019 corrections * Fix typo --- bin/release | 11 ++++++++--- config/cmake/HDF5_Examples.cmake.in | 2 +- config/cmake/HDF5_Examples_options.cmake | 3 +++ config/cmake/scripts/HDF5config.cmake | 6 +++--- config/cmake/scripts/HDF5options.cmake | 2 +- release_docs/README_HDF5_CMake | 17 +++++++++-------- release_docs/USING_CMake_Examples.txt | 1 + release_docs/USING_HDF5_CMake.txt | 32 +++++++++++++++++++++++++------- 8 files changed, 51 insertions(+), 23 deletions(-) diff --git a/bin/release b/bin/release index 2570b38..16e6bd2 100755 --- a/bin/release +++ b/bin/release @@ -229,11 +229,14 @@ tar2cmakezip() (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201564 -C Release -V -O hdf5.log" > build-VS2015-64.bat; chmod 755 build-VS2015-64.bat) (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS2017 -C Release -V -O hdf5.log" > build-VS2017-32.bat; chmod 755 build-VS2017-32.bat) (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201764 -C Release -V -O hdf5.log" > build-VS2017-64.bat; chmod 755 build-VS2017-64.bat) + (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS2019 -C Release -V -O hdf5.log" > build-VS2019-32.bat; chmod 755 build-VS2019-32.bat) + (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201964 -C Release -V -O hdf5.log" > build-VS2019-64.bat; chmod 755 build-VS2019-64.bat) # step 3: add LIBAEC.tar.gz, ZLib.tar.gz and cmake files cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmziptmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmziptmpsubdir - cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.1-Source.zip $cmziptmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.2-Source.zip $cmziptmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-master.zip $cmziptmpsubdir cp $cmziptmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmziptmpsubdir cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmziptmpsubdir cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmziptmpsubdir @@ -328,7 +331,8 @@ tar2cmaketgz() # step 3: add LIBAEC.tar.gz, ZLib.tar.gz and cmake files cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir - cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.1-Source.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.2-Source.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-master.tar.gz $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpsubdir @@ -411,7 +415,8 @@ tar2hpccmaketgz() # step 3: add LIBAEC.tar.gz, ZLib.tar.gz and cmake files cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir - cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.1-Source.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.2-Source.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-master.tar.gz $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir diff --git a/config/cmake/HDF5_Examples.cmake.in b/config/cmake/HDF5_Examples.cmake.in index 3eea743..5911005 100644 --- a/config/cmake/HDF5_Examples.cmake.in +++ b/config/cmake/HDF5_Examples.cmake.in @@ -77,7 +77,7 @@ set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDN #TAR_SOURCE - name of tarfile #if(NOT DEFINED TAR_SOURCE) -# set(CTEST_USE_TAR_SOURCE "HDF5Examples-1.14.1-Source") +# set(CTEST_USE_TAR_SOURCE "HDF5Examples-1.14.2-Source") #endif() ############################################################################################################### diff --git a/config/cmake/HDF5_Examples_options.cmake b/config/cmake/HDF5_Examples_options.cmake index b639b19..236a814 100644 --- a/config/cmake/HDF5_Examples_options.cmake +++ b/config/cmake/HDF5_Examples_options.cmake @@ -42,8 +42,11 @@ ### enable JAVA builds #set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_BUILD_JAVA:BOOL=ON") +############################################################################################# ### enable FILTERS builds #set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_BUILD_FILTERS:BOOL=ON") +### default HDF5_PLUGIN_PATH to where the filter libraries are located +#set(ENV{HDF5_PLUGIN_PATH} "${INSTALLDIR}/lib/plugin") ############################################################################################# ### enable parallel program builds diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index 08e38e8..8df7dd4 100644 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -11,7 +11,7 @@ # ############################################################################################# ### ${CTEST_SCRIPT_ARG} is of the form OPTION=VALUE ### -### BUILD_GENERATOR required [Unix, VS2017, VS201764, VS2015, VS201564, VS2013, VS201364] ### +### BUILD_GENERATOR required [Unix, VS2019, VS201964, VS2017, VS201764, VS2015, VS201564] ### ### ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201764 -C Release -VV -O hdf5.log ### ############################################################################################# @@ -68,7 +68,7 @@ endif () # build generator must be defined if (NOT DEFINED BUILD_GENERATOR) - message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2017, or VS201764, VS2015, VS201564, VS2013, VS201364") + message (FATAL_ERROR "BUILD_GENERATOR must be defined - Unix, VS2019, VS201964, VS2017, or VS201764, VS2015, VS201564") endif () ################################################################### @@ -163,7 +163,7 @@ if (NOT DEFINED HPC) set (SITE_COMPILER_NAME "vs2012") set (SITE_COMPILER_VERSION "11") else () - message (FATAL_ERROR "Invalid BUILD_GENERATOR must be - Unix, VS2017, or VS201764, VS2015, VS201564, VS2013, VS201364") + message (FATAL_ERROR "Invalid BUILD_GENERATOR must be - Unix, VS2019, VS201964, VS2017, or VS201764, VS2015, VS201564") endif () ## Set the following to unique id your computer ## set (CTEST_SITE "WIN7${BUILD_GENERATOR}.XXXX") diff --git a/config/cmake/scripts/HDF5options.cmake b/config/cmake/scripts/HDF5options.cmake index 7133e6c..e3c4c1a 100644 --- a/config/cmake/scripts/HDF5options.cmake +++ b/config/cmake/scripts/HDF5options.cmake @@ -69,7 +69,7 @@ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRIN #set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_ENCODING:BOOL=OFF") #### package examples #### -#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACK_EXAMPLES:BOOL=ON -DHDF5_EXAMPLES_COMPRESSED:STRING=HDF5Examples-1.14.1-Source.tar.gz -DHDF5_EXAMPLES_COMPRESSED_DIR:PATH=${CTEST_SCRIPT_DIRECTORY}") +#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACK_EXAMPLES:BOOL=ON -DHDF5_EXAMPLES_COMPRESSED:STRING=HDF5Examples-1.14.2-Source.tar.gz -DHDF5_EXAMPLES_COMPRESSED_DIR:PATH=${CTEST_SCRIPT_DIRECTORY}") ############################################################################################# ### enable parallel builds diff --git a/release_docs/README_HDF5_CMake b/release_docs/README_HDF5_CMake index cf0ab6f..1242695 100644 --- a/release_docs/README_HDF5_CMake +++ b/release_docs/README_HDF5_CMake @@ -1,15 +1,16 @@ This tar file contains - build-unix.sh script to build HDF5 with CMake on unix machines - build-unix-hpc.sh script to build HDF5 with CMake on unix machines and run - tests with batch scripts (sbatch). + build-unix.sh script to build HDF5 with CMake on unix machines + build-unix-hpc.sh script to build HDF5 with CMake on unix machines and run + tests with batch scripts (sbatch). CTestScript.cmake - HDF5config.cmake CMake scripts for building HDF5 + HDF5config.cmake CMake scripts for building HDF5 HDF5options.cmake - hdf5-1.13.0 HDF5 1.13.0 source - LIBAEC.tar.gz source for building SZIP replacement - ZLib.tar.gz source for building ZLIB - hdf5_plugins.tar.gz source for building compression plugins + hdf5-1.13.0 HDF5 1.13.0 source + LIBAEC.tar.gz source for building SZIP replacement + ZLib.tar.gz source for building ZLIB + hdf5_plugins.tar.gz source for building compression plugins + HDF5Examples-1.14.2-Source.tar.gz source for building examples For more information about building HDF5 with CMake, see USING_HDF5_CMake.txt in hdf5-1.13.0/release_docs, or diff --git a/release_docs/USING_CMake_Examples.txt b/release_docs/USING_CMake_Examples.txt index bd089a6..a12a952 100644 --- a/release_docs/USING_CMake_Examples.txt +++ b/release_docs/USING_CMake_Examples.txt @@ -83,6 +83,7 @@ III. Defaults in the HDF5_Examples_options.cmake file #### HDF_BUILD_CXX:BOOL=OFF ### #### HDF_BUILD_FORTRAN:BOOL=OFF ### #### HDF_BUILD_JAVA:BOOL=OFF ### +#### HDF_BUILD_FILTERS:BOOL=OFF ### #### BUILD_TESTING:BOOL=OFF ### #### HDF_ENABLE_PARALLEL:BOOL=OFF ### #### HDF_ENABLE_THREADSAFE:BOOL=OFF ### diff --git a/release_docs/USING_HDF5_CMake.txt b/release_docs/USING_HDF5_CMake.txt index f2d7754..792c719 100644 --- a/release_docs/USING_HDF5_CMake.txt +++ b/release_docs/USING_HDF5_CMake.txt @@ -41,13 +41,30 @@ I. Preconditions 2. You have installed the HDF5 library built with CMake, by executing the HDF Install Utility (the *.msi file in the binary package for - Windows). If you are using a Windows platform, you can obtain a - pre-built Windows binary from The HDF Group's website at + Windows). You can obtain pre-built binaries from The HDF Group's website at www.hdfgroup.org. 3. Set the environment variable HDF5_DIR to the installed location of - the config files for HDF5. On Windows: + the config files for HDF5. + On Windows: HDF5_DIR=C:/Program Files/HDF_Group/HDF5/1.13.x/cmake + On unix: + HDF5_DIR=/HDF_Group/HDF5/1.13.x/cmake + + If you are using shared libraries, you may need to add to the path + environment variable. Set the path environment variable to the + installed location of the library files for HDF5. + On Windows (*.dll): + PATH=%PATH%;C:/Program Files/HDF_Group/HDF5/1.13.x/bin + On unix (*.so): + LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/HDF_Group/HDF5/1.13.x/lib + + If you are using filter plugin libraries, you will need to set the + HDF5_PLUGIN_PATH environment variable. + On Windows: + HDF5_PLUGIN_PATH=C:/Program Files/HDF_Group/HDF5/1.13.x/lib/plugin + On unix: + HDF5_PLUGIN_PATH=/HDF_Group/HDF5/1.13.x/lib/plugin (Note there are no quote characters used on Windows and all platforms use forward slashes) @@ -99,12 +116,13 @@ These steps are described in more detail below. * MinGW Makefiles * NMake Makefiles * Unix Makefiles - * Visual Studio 12 2013 - * Visual Studio 12 2013 Win64 * Visual Studio 14 2015 * Visual Studio 14 2015 Win64 * Visual Studio 15 2017 * Visual Studio 15 2017 Win64 + * Visual Studio 16 2019 + * ... in addition VS2019 will need to set the "-A" option, + * ... [Win32, x64, ARM, ARM64] is: * BUILD_TESTING:BOOL=ON @@ -114,7 +132,7 @@ These steps are described in more detail below. 2.1 Visual CMake users, click the Configure button. If this is the first time you are running cmake-gui in this directory, you will be prompted for the - generator you wish to use (for example on Windows, Visual Studio 12 2013). + generator you wish to use (for example on Windows, Visual Studio 14 2015 Win64). CMake will read in the CMakeLists.txt files from the source directory and display options for the HDF5 project. After the first configure you can adjust the cache settings and/or specify locations of other programs. @@ -132,7 +150,7 @@ These steps are described in more detail below. 2.2 Alternative command line example on Windows in c:\MyHDFstuff\hdf5\build directory: - cmake -G "Visual Studio 12 2013" -DBUILD_TESTING:BOOL=ON .. + cmake -G "Visual Studio 14 2015 Win64" -DBUILD_TESTING:BOOL=ON .. 3. Build HDF5 Applications -- cgit v0.12 From 436727dceab2df46ef688f31fa199254e64ec34a Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Wed, 14 Jul 2021 22:59:38 -0500 Subject: Perform option arg variables (#834) * Correct option name * Undo accidental commit * Note LLVM 10 to 11 format default changes * Update format plugin * Undo clang-format version 11 changes * One more correction * Update supported platforms * Revert whitespace changes * Correct whitespace * Changes from PR#3 * HDFFV-11213 added option to control gcc10 warnings diagnostics * HDFFV-11212 Use the new references correctly in JNI utility and tests * format source * Fix typo * Add new test file * HDFFV-11212 - update test and remove unused arg * Minor non-space formatting changes * Use H5I_INVALID_ID instead of "-1" * source formatting * add missing testfile, update jni function * Undo commit of debug code * remove mislocated file * Fix h5repack test for handling of fapls and id close * Update h5diff test files usage text * HDFFV-11212 add new ref tests for JNI export dataset * src format update * Remove blank line typo * src format typo * long double requires %Lg * Another long double foramt specifer S.B. %Lg * issue with t128bit test * Windows issue with h5dump and type. * Fix review issues * refactor function nesting and fix error checks * format fixes * Remove untested functions and javadoc quiet comments * Restore TRY block. * Change string append errors to memory exception * revert to H5_JNI_FATAL_ERROR - support functions need work * Add assertion error for h5util functions * remove duplicate function * format fix * Revert HD function error handling * Update copyright comments * GH #386 java folder copyright corrections * Whitespace * GH #359 implement and fix tools 1.6 API usage * remove excessive comments * Flip inits to correct ifdef section * rework ifdef to be simpler * format issue * Reformat ifdef inits * remove static attribute * format compliance * Update names * Revert because logic relies on float not being int * Changes noticed from creating merge of #412 * Double underscore change * Correct compiler version variable used * Remove header guard underscores * Whitespace cleanup * Split format source and commit changes on repo push * remove pre-split setting * Change windows TS to use older VS. * correct window os name * HDFFV-11212 JNI export util and Javadoc * Suggested review changes * Another change found * Committing clang-format changes * HDFFV-11113 long double in tools * HDFFV-11113 add note * Disable long double tests for now * HDFFV-11228 remove arbitrary CMake warning groups. * Make each flag a string * Some Javadoc warning fixes * Updated javadoc fixes * # WARNING: head commit changed in the meantime HDFFV-11229 Fix long double usage in tools and java Changed h5dump and h5ls to just print 128-bit for long double type. Added test and file for dataset and attributes with all three float types. * Committing clang-format changes * HDFFV-11229 Add release note * HDFFV-11229 windows testfile needed * fix typo * Remove non supported message text * HDFFV-11229 - change ldouble test to check both native and general * HDFFV-11229 add second file compare * HDFFV-11229 fix reference file * HDFFV-11229 autotools check two refs * HDFFV-11229 revert back to removal of NATIVE_LDOUBLE in tools output * Committing clang-format changes * Update release note * Update attribute type of ref file * Change source of ninja for macs * try port instead of brew * Recommended is to use brew. * Undo non long double changes * remove unneeded file * Fix perl and doxygen CMake code * Add "option" command for clang options * Rework CMake add_custom to use the BYPRODUCTS argument * Add stamp files to BYPRODUCTS * Only one copy of file exists * Fix custom cmmand depends targets * Fix fortran custom command DEPENDS * Add LD_LIBRARY_PATH to tests * Add custom target for DEPENDS * Add h5detect conditionaly to generated target DEPENDS * Correct DEPENDS targets * Parallel builds need the mpi compiler for pkgconfig scripts. * install only if MPI build * Fortran target depends * Remove incorrect source attribute * doxygen adjustments * doxygen build updates * Correct version * Correct function version - function has been merged to 1.12 * Correct version string for map functions * Cleanup warnings for tools debug builds * TRILAB-227 - fix indexing for h5diff selections * Correct location of pos to index function call * TRILAB-227 Enable test * Quote subset args * Use MATCHES because of AppleClang * if blocks needed for build modes * Update list of DT platforms * VS2019 correctly displays float values * revert VS2019 change * Issue #669 remove version from pkgcfg filename * remove version from h5cc script * Java reference functions updated enabled fortran in cmake gcc action yaml file java reference test changed to correctly test refs jni reference functions that create ids changed to use jni id logging * Correct BYPRODUCTS argument * Correct more genereated files BYPRODUCTS * BYPRODUCTS must have unique locations * Fix typo * Fix fortran configure checks * Rework H5_PAC_C_MAX_REAL_PRECISION setting logic * Add note about fortran configure change * Adds a quick for for some egregious chunk_info badness (#722) * Fixes issue with ccmake that prevents building Fortran (#723) ccmake runs iteratively, and the check_fortran_source_runs macros were clobbering a single output file that did not get updated on further configure iterations * Fix conflicts with merge * Move MAX_PRECISION back to HDF5UseFortran.cmake * Use STREQUAL to test macro argument * Move C language test to ConfigureChecks from HDF5UseFortran * MAX_PRECISION defines must be defined * Organize flags and align autotools and cmake * Fix comment in no-error-general warnings files. * Flag cleanup and fix typos * Add comment * Correct VAR used to find configure time file * Set the path correctly * Update missing release note info. * Update code owners * Correct JIRA note * add known problem. * Use only core library for testing dynamic plugins. * Fix static perform tools compile for arg options Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Dana Robinson <43805+derobins@users.noreply.github.com> --- tools/test/perform/pio_standalone.c | 5 ----- tools/test/perform/pio_standalone.h | 6 +++--- tools/test/perform/sio_standalone.c | 5 ----- tools/test/perform/sio_standalone.h | 6 +++--- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/tools/test/perform/pio_standalone.c b/tools/test/perform/pio_standalone.c index db8633e..ded15a4 100644 --- a/tools/test/perform/pio_standalone.c +++ b/tools/test/perform/pio_standalone.c @@ -26,11 +26,6 @@ MPI_Info h5_io_info_g = MPI_INFO_NULL; /* MPI INFO object for IO */ /* global variables */ int nCols = 80; -/* ``get_option'' variables */ -int H5_opterr = 1; /*get_option prints errors if this is on */ -int H5_optind = 1; /*token pointer */ -const char *H5_optarg; /*flag argument (or value) */ - int get_option(int argc, const char **argv, const char *opts, const struct h5_long_options *l_opts) { diff --git a/tools/test/perform/pio_standalone.h b/tools/test/perform/pio_standalone.h index 4b5c7b2..ef6d97b 100644 --- a/tools/test/perform/pio_standalone.h +++ b/tools/test/perform/pio_standalone.h @@ -454,9 +454,9 @@ void h5_dump_info_object(MPI_Info info); /** From h5tools_utils.h **/ -extern int H5_opterr; /* getoption prints errors if this is on */ -extern int H5_optind; /* token pointer */ -extern const char *H5_optarg; /* flag argument (or value) */ +H5_DLLVAR int H5_opterr; /* getoption prints errors if this is on */ +H5_DLLVAR int H5_optind; /* token pointer */ +H5_DLLVAR const char *H5_optarg; /* flag argument (or value) */ enum h5_arg_level { no_arg = 0, /* doesn't take an argument */ diff --git a/tools/test/perform/sio_standalone.c b/tools/test/perform/sio_standalone.c index 2e793fb..7c22b53 100644 --- a/tools/test/perform/sio_standalone.c +++ b/tools/test/perform/sio_standalone.c @@ -22,11 +22,6 @@ /* global variables */ int nCols = 80; -/* ``get_option'' variables */ -int H5_opterr = 1; /*get_option prints errors if this is on */ -int H5_optind = 1; /*token pointer */ -const char *H5_optarg; /*flag argument (or value) */ - int get_option(int argc, const char **argv, const char *opts, const struct h5_long_options *l_opts) { diff --git a/tools/test/perform/sio_standalone.h b/tools/test/perform/sio_standalone.h index 098b98a..3b699dc 100644 --- a/tools/test/perform/sio_standalone.h +++ b/tools/test/perform/sio_standalone.h @@ -469,9 +469,9 @@ void h5_dump_info_object(MPI_Info info); /** From h5tools_utils.h **/ -extern int H5_opterr; /* getoption prints errors if this is on */ -extern int H5_optind; /* token pointer */ -extern const char *H5_optarg; /* flag argument (or value) */ +H5_DLLVAR int H5_opterr; /* getoption prints errors if this is on */ +H5_DLLVAR int H5_optind; /* token pointer */ +H5_DLLVAR const char *H5_optarg; /* flag argument (or value) */ enum h5_arg_level { no_arg = 0, /* doesn't take an argument */ -- cgit v0.12 From 50a37fde06457ee5e75ea65259c233cb6527bd91 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Mon, 19 Jul 2021 20:40:28 -0500 Subject: Allow parallel filters feature for comm size of 1 (#840) --- src/H5Dio.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/H5Dio.c b/src/H5Dio.c index 600f08a..d1861c4 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -810,11 +810,19 @@ H5D__ioinfo_adjust(H5D_io_info_t *io_info, const H5D_t *dset, const H5S_t *file_ io_info->io_ops.single_write = H5D__mpio_select_write; } /* end if */ else { + int comm_size = 0; + + /* Retrieve size of MPI communicator used for file */ + if ((comm_size = H5F_shared_mpi_get_size(io_info->f_sh)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get MPI communicator size") + /* Check if there are any filters in the pipeline. If there are, - * we cannot break to independent I/O if this is a write operation; - * otherwise there will be metadata inconsistencies in the file. + * we cannot break to independent I/O if this is a write operation + * with multiple ranks involved; otherwise, there will be metadata + * inconsistencies in the file. */ - if (io_info->op_type == H5D_IO_OP_WRITE && io_info->dset->shared->dcpl_cache.pline.nused > 0) { + if (comm_size > 1 && io_info->op_type == H5D_IO_OP_WRITE && + io_info->dset->shared->dcpl_cache.pline.nused > 0) { H5D_mpio_no_collective_cause_t cause; uint32_t local_no_collective_cause; uint32_t global_no_collective_cause; -- cgit v0.12 From 20dd1cedb35d95ecc21decb29352d7779d49a908 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Wed, 21 Jul 2021 11:12:39 -0500 Subject: Issue #833 fix setting return result (#843) --- config/cmake/ConfigureChecks.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 7baf77b..4db3ad4 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -261,7 +261,7 @@ macro (C_RUN FUNCTION_NAME SOURCE_CODE RETURN_VAR) if (${COMPILE_RESULT_VAR}) if (${RUN_RESULT_VAR} MATCHES 0) - set (${RUN_RESULT_VAR} 1 CACHE INTERNAL "Have C function ${FUNCTION_NAME}") + set (${RETURN_VAR} 1 CACHE INTERNAL "Have C function ${FUNCTION_NAME}") if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") message (VERBOSE "Testing C ${FUNCTION_NAME} - OK") endif () @@ -273,7 +273,7 @@ macro (C_RUN FUNCTION_NAME SOURCE_CODE RETURN_VAR) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") message (VERBOSE "Testing C ${FUNCTION_NAME} - Fail") endif () - set (${RUN_RESULT_VAR} 0 CACHE INTERNAL "Have C function ${FUNCTION_NAME}") + set (${RETURN_VAR} 0 CACHE INTERNAL "Have C function ${FUNCTION_NAME}") file (APPEND ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/CMakeError.log "Determining if the C ${FUNCTION_NAME} exists failed with the following output:\n" "${OUTPUT_VAR}\n\n") -- cgit v0.12 From 96b6872a1e7deac7673d31697dafa1f44763ede4 Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Thu, 22 Jul 2021 08:12:42 -0500 Subject: Avoid popping API context when one wasn't pushed (#848) --- src/H5private.h | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/src/H5private.h b/src/H5private.h index f29ec02..7823bfc 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -2082,6 +2082,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); /* Push the API context */ \ if (H5CX_push() < 0) \ HGOTO_ERROR(H5E_FUNC, H5E_CANTSET, err, "can't set API context") \ + else \ + api_ctx_pushed = TRUE; \ \ BEGIN_MPE_LOG @@ -2089,6 +2091,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); #define FUNC_ENTER_API(err) \ { \ { \ + hbool_t api_ctx_pushed = FALSE; \ + \ FUNC_ENTER_API_COMMON \ FUNC_ENTER_API_INIT(err); \ FUNC_ENTER_API_PUSH(err); \ @@ -2103,6 +2107,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); #define FUNC_ENTER_API_NOCLEAR(err) \ { \ { \ + hbool_t api_ctx_pushed = FALSE; \ + \ FUNC_ENTER_API_COMMON \ FUNC_ENTER_API_INIT(err); \ FUNC_ENTER_API_PUSH(err); \ @@ -2369,14 +2375,17 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); H5_API_SET_CANCEL #define FUNC_LEAVE_API_COMMON(ret_value) \ - ; \ - } /*end scope from end of FUNC_ENTER*/ \ FINISH_MPE_LOG \ H5TRACE_RETURN(ret_value); #define FUNC_LEAVE_API(ret_value) \ + ; \ + } /*end scope from end of FUNC_ENTER*/ \ FUNC_LEAVE_API_COMMON(ret_value); \ - (void)H5CX_pop(TRUE); \ + if (api_ctx_pushed) { \ + (void)H5CX_pop(TRUE); \ + api_ctx_pushed = FALSE; \ + } \ H5_POP_FUNC \ if (err_occurred) \ (void)H5E_dump_api_stack(TRUE); \ @@ -2387,6 +2396,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); /* Use this macro to match the FUNC_ENTER_API_NOINIT macro */ #define FUNC_LEAVE_API_NOINIT(ret_value) \ + ; \ + } /*end scope from end of FUNC_ENTER*/ \ FUNC_LEAVE_API_COMMON(ret_value); \ H5_POP_FUNC \ if (err_occurred) \ @@ -2399,6 +2410,8 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); /* Use this macro to match the FUNC_ENTER_API_NOINIT_NOERR_NOFS macro */ #define FUNC_LEAVE_API_NOFS(ret_value) \ + ; \ + } /*end scope from end of FUNC_ENTER*/ \ FUNC_LEAVE_API_COMMON(ret_value); \ FUNC_LEAVE_API_THREADSAFE \ return (ret_value); \ -- cgit v0.12 From 06de43332d80229dbe5f5a1ac4addf3fbc8325f5 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Thu, 22 Jul 2021 06:13:14 -0700 Subject: Fixes a couple of warnings: (#849) - warn_unused_result warnings from h5jamgentest and j5stat_gentest that were triggered by write(2) calls where we didn't check the return value - A missing callback in the NULL VOL connector - A possibly uninitialized property list pointer --- src/H5Pdxpl.c | 4 +- test/null_vol_connector.c | 1 + tools/test/h5jam/h5jamgentest.c | 384 ++++++++++++++++++++++++------------- tools/test/h5stat/h5stat_gentest.c | 145 ++++++++++---- 4 files changed, 370 insertions(+), 164 deletions(-) diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index 046b046..5f83a1d 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -2281,7 +2281,7 @@ herr_t H5Pset_dataset_io_hyperslab_selection(hid_t plist_id, unsigned rank, H5S_seloper_t op, const hsize_t start[], const hsize_t stride[], const hsize_t count[], const hsize_t block[]) { - H5P_genplist_t *plist; /* Property list pointer */ + H5P_genplist_t *plist = NULL; /* Property list pointer */ H5S_t * space; /* Dataspace to hold selection */ hbool_t space_created = FALSE; /* Whether a new dataspace has been created */ hbool_t reset_prop_on_error = FALSE; /* Whether to reset the property on failure */ @@ -2371,7 +2371,7 @@ H5Pset_dataset_io_hyperslab_selection(hid_t plist_id, unsigned rank, H5S_seloper done: /* Cleanup on failure */ if (ret_value < 0) { - if (reset_prop_on_error && H5P_poke(plist, H5D_XFER_DSET_IO_SEL_NAME, &space) < 0) + if (reset_prop_on_error && plist && H5P_poke(plist, H5D_XFER_DSET_IO_SEL_NAME, &space) < 0) HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "error setting dataset I/O selection") if (space_created && H5S_close(space) < 0) HDONE_ERROR(H5E_PLIST, H5E_CLOSEERROR, FAIL, "unable to release dataspace") diff --git a/test/null_vol_connector.c b/test/null_vol_connector.c index 2b375d8..6555763 100644 --- a/test/null_vol_connector.c +++ b/test/null_vol_connector.c @@ -116,6 +116,7 @@ static const H5VL_class_t null_vol_g = { { /* introspect_cls */ NULL, /* get_conn_cls */ + NULL, /* get_cap_flags */ NULL, /* opt_query */ }, { diff --git a/tools/test/h5jam/h5jamgentest.c b/tools/test/h5jam/h5jamgentest.c index 1da6b63..ae6c594 100644 --- a/tools/test/h5jam/h5jamgentest.c +++ b/tools/test/h5jam/h5jamgentest.c @@ -83,7 +83,7 @@ UD_traverse(const char H5_ATTR_UNUSED *link_name, hid_t H5_ATTR_UNUSED cur_group const void H5_ATTR_UNUSED *udata, size_t H5_ATTR_UNUSED udata_size, hid_t H5_ATTR_UNUSED lapl_id, hid_t H5_ATTR_UNUSED dxpl_id) { - return -1; + return H5I_INVALID_HID; } #define MY_LINKCLASS 187 @@ -116,196 +116,311 @@ g2 : dset2.1 dset2.2 udlink */ -static void +static herr_t gent_ub(const char *filename, size_t ub_size, size_t ub_fill) { - hid_t fid, group, attr, dataset, space; - hid_t create_plist; + hid_t fid = H5I_INVALID_HID; + hid_t group = H5I_INVALID_HID; + hid_t attr = H5I_INVALID_HID; + hid_t dataset = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t create_plist = H5I_INVALID_HID; hsize_t dims[2]; int data[2][2], dset1[10][10], dset2[20]; char buf[BUF_SIZE]; int i, j; size_t u; float dset2_1[10], dset2_2[3][5]; - int fd; - char * bp; + int fd = -1; if (ub_size > 0) { - create_plist = H5Pcreate(H5P_FILE_CREATE); - H5Pset_userblock(create_plist, (hsize_t)ub_size); - fid = H5Fcreate(filename, H5F_ACC_TRUNC, create_plist, H5P_DEFAULT); + if ((create_plist = H5Pcreate(H5P_FILE_CREATE)) < 0) + goto error; + if (H5Pset_userblock(create_plist, (hsize_t)ub_size) < 0) + goto error; + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, create_plist, H5P_DEFAULT)) < 0) + goto error; } else { - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; } - /* create groups */ - group = H5Gcreate2(fid, "/g1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + /* Create groups */ + if ((group = H5Gcreate2(fid, "/g1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - group = H5Gcreate2(fid, "/g2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + if ((group = H5Gcreate2(fid, "/g2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - group = H5Gcreate2(fid, "/g1/g1.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + if ((group = H5Gcreate2(fid, "/g1/g1.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - group = H5Gcreate2(fid, "/g1/g1.2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + if ((group = H5Gcreate2(fid, "/g1/g1.2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - group = H5Gcreate2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + if ((group = H5Gcreate2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - /* root attributes */ - group = H5Gopen2(fid, "/", H5P_DEFAULT); + /* Root attributes */ + if ((group = H5Gopen2(fid, "/", H5P_DEFAULT)) < 0) + goto error; dims[0] = 10; - space = H5Screate_simple(1, dims, NULL); - attr = H5Acreate2(group, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT); - HDsprintf(buf, "abcdefghi"); - H5Awrite(attr, H5T_NATIVE_SCHAR, buf); - H5Sclose(space); - H5Aclose(attr); - - dims[0] = 2; - dims[1] = 2; - space = H5Screate_simple(2, dims, NULL); - attr = H5Acreate2(group, "attr2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((attr = H5Acreate2(group, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (HDsprintf(buf, "abcdefghi") < 0) + goto error; + if (H5Awrite(attr, H5T_NATIVE_SCHAR, buf) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Aclose(attr) < 0) + goto error; + + dims[0] = 2; + dims[1] = 2; + if ((space = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + if ((attr = H5Acreate2(group, "attr2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; data[0][0] = 0; data[0][1] = 1; data[1][0] = 2; data[1][1] = 3; - H5Awrite(attr, H5T_NATIVE_INT, data); - H5Sclose(space); - H5Aclose(attr); + if (H5Awrite(attr, H5T_NATIVE_INT, data) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Aclose(attr) < 0) + goto error; - H5Gclose(group); + if (H5Gclose(group) < 0) + goto error; - group = H5Gopen2(fid, "/g1/g1.1", H5P_DEFAULT); + if ((group = H5Gopen2(fid, "/g1/g1.1", H5P_DEFAULT)) < 0) + goto error; - /* dset1.1.1 */ + /* Dataset 1.1.1 */ dims[0] = 10; dims[1] = 10; - space = H5Screate_simple(2, dims, NULL); - dataset = H5Dcreate2(group, "dset1.1.1", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + if ((dataset = + H5Dcreate2(group, "dset1.1.1", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < 10; i++) for (j = 0; j < 10; j++) dset1[i][j] = j * i; - H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset1); - H5Sclose(space); + if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset1) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; - /* attributes of dset1.1.1 */ + /* Attributes of dset1.1.1 */ dims[0] = 27; - space = H5Screate_simple(1, dims, NULL); - attr = H5Acreate2(dataset, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT); - HDsprintf(buf, "1st attribute of dset1.1.1"); - H5Awrite(attr, H5T_NATIVE_SCHAR, buf); - H5Sclose(space); - H5Aclose(attr); + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((attr = H5Acreate2(dataset, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (HDsprintf(buf, "1st attribute of dset1.1.1") < 0) + goto error; + if (H5Awrite(attr, H5T_NATIVE_SCHAR, buf) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Aclose(attr) < 0) + goto error; dims[0] = 27; - space = H5Screate_simple(1, dims, NULL); - attr = H5Acreate2(dataset, "attr2", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT); - HDsprintf(buf, "2nd attribute of dset1.1.1"); - H5Awrite(attr, H5T_NATIVE_SCHAR, buf); - H5Sclose(space); - H5Aclose(attr); - - H5Dclose(dataset); - - /* dset1.1.2 */ + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((attr = H5Acreate2(dataset, "attr2", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (HDsprintf(buf, "2nd attribute of dset1.1.1") < 0) + goto error; + if (H5Awrite(attr, H5T_NATIVE_SCHAR, buf) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Aclose(attr) < 0) + goto error; + + if (H5Dclose(dataset) < 0) + goto error; + + /* Dataset 1.1.2 */ dims[0] = 20; - space = H5Screate_simple(1, dims, NULL); - dataset = H5Dcreate2(group, "dset1.1.2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((dataset = + H5Dcreate2(group, "dset1.1.2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < 20; i++) dset2[i] = i; - H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2); - H5Sclose(space); - H5Dclose(dataset); - - H5Gclose(group); - - /* external link */ - H5Lcreate_external("somefile", "somepath", fid, "/g1/g1.2/extlink", H5P_DEFAULT, H5P_DEFAULT); - - /* soft link */ - group = H5Gopen2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT); - H5Lcreate_soft("somevalue", group, "slink", H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); - - group = H5Gopen2(fid, "/g2", H5P_DEFAULT); - - /* dset2.1 */ + if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Dclose(dataset) < 0) + goto error; + + if (H5Gclose(group) < 0) + goto error; + + /* External link */ + if (H5Lcreate_external("somefile", "somepath", fid, "/g1/g1.2/extlink", H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + + /* Soft link */ + if ((group = H5Gopen2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT)) < 0) + goto error; + if (H5Lcreate_soft("somevalue", group, "slink", H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; + + if ((group = H5Gopen2(fid, "/g2", H5P_DEFAULT)) < 0) + goto error; + + /* Dataset 2.1 */ dims[0] = 10; - space = H5Screate_simple(1, dims, NULL); - dataset = H5Dcreate2(group, "dset2.1", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((dataset = + H5Dcreate2(group, "dset2.1", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < 10; i++) dset2_1[i] = (float)((float)i * 0.1F + 1.0F); - H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_1); - H5Sclose(space); - H5Dclose(dataset); - - /* dset2.2 */ + if (H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_1) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Dclose(dataset) < 0) + goto error; + + /* Dataset 2.2 */ dims[0] = 3; dims[1] = 5; - space = H5Screate_simple(2, dims, NULL); - dataset = H5Dcreate2(group, "dset2.2", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + if ((dataset = + H5Dcreate2(group, "dset2.2", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < 3; i++) for (j = 0; j < 5; j++) dset2_2[i][j] = (float)(((float)i + 1.0F) * (float)j * 0.1F); - H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_2); - H5Sclose(space); - H5Dclose(dataset); - - H5Gclose(group); - - /* user-defined link */ - H5Lregister(UD_link_class); - H5Lcreate_ud(fid, "/g2/udlink", (H5L_type_t)MY_LINKCLASS, NULL, (size_t)0, H5P_DEFAULT, H5P_DEFAULT); - - H5Fclose(fid); + if (H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_2) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Dclose(dataset) < 0) + goto error; + + if (H5Gclose(group) < 0) + goto error; + + /* User-defined link */ + if (H5Lregister(UD_link_class) < 0) + goto error; + if (H5Lcreate_ud(fid, "/g2/udlink", (H5L_type_t)MY_LINKCLASS, NULL, (size_t)0, H5P_DEFAULT, H5P_DEFAULT) < + 0) + goto error; + + /* MUST close the file ID before the user block code or you risk tripping + * over file locking issues. + */ + if (H5Fclose(fid) < 0) + goto error; /* If a user block is being used, write to it here */ if (ub_size > 0) { - HDassert(ub_size <= BUF_SIZE); + char *bp; + + if (ub_size > BUF_SIZE) + goto error; - fd = HDopen(filename, O_RDWR); - HDassert(fd >= 0); + if ((fd = HDopen(filename, O_RDWR)) < 0) + goto error; - /* fill buf with pattern */ + /* Fill buf with pattern */ HDmemset(buf, '\0', ub_size); bp = buf; for (u = 0; u < ub_fill; u++) *bp++ = pattern[u % 10]; - (void)HDwrite(fd, buf, ub_size); + if (HDwrite(fd, buf, ub_size) < 0) + goto error; + + if (HDclose(fd) < 0) + goto error; + } + return SUCCEED; + +error: + if (fd >= 0) HDclose(fd); + + H5E_BEGIN_TRY + { + H5Fclose(fid); + H5Gclose(group); + H5Aclose(attr); + H5Dclose(dataset); + H5Sclose(space); + H5Pclose(create_plist); } + H5E_END_TRY; + + return FAIL; } -static void +/* Creates a simple (i.e., not HDF5) text file and fills it with a pattern */ +static herr_t create_textfile(const char *name, size_t size) { - char * buf; - int fd; + char * buf = NULL; + int fd = -1; size_t i; - char * bp; + char * bp = NULL; - fd = HDcreat(name, 0777); - HDassert(fd >= 0); - buf = (char *)HDcalloc(size, (size_t)1); - HDassert(buf); + if ((fd = HDcreat(name, 0777)) < 0) + goto error; + if (NULL == (buf = (char *)HDcalloc(size, 1))) + goto error; - /* fill buf with pattern */ + /* Fill buf with pattern */ bp = buf; for (i = 0; i < size; i++) *bp++ = pattern[i % 10]; - (void)HDwrite(fd, buf, size); + if (HDwrite(fd, buf, size) < 0) + goto error; HDfree(buf); - HDclose(fd); + + return SUCCEED; + +error: + HDfree(buf); + if (fd >= 0) + HDclose(fd); + + return FAIL; } #ifdef notdef @@ -318,19 +433,20 @@ create_binfile(char *name, off_t size) int i; char *bp; - fd = creat(name, 0777); + fd = HDcreat(name, 0777); HDassert(fd >= 0); buf = HDcalloc(size, 1); HDassert(buf); - /* fill buf with pattern */ + /* Fill buf with pattern */ bp = buf; for (i = 0; i < size; i++) *bp++ = (char)i & 0xff; (void)HDwrite(fd, buf, size); + HDfree(buf); HDclose(fd); } #endif @@ -344,14 +460,19 @@ create_binfile(char *name, off_t size) int main(void) { - /* create_textfile(UBTXT1, (size_t)0); */ - create_textfile(UBTXT2, (size_t)10); - create_textfile(UBTXT3, (size_t)511); - create_textfile(UBTXT4, (size_t)512); - create_textfile(UBTXT5, (size_t)513); + + if (create_textfile(UBTXT2, 10) < 0) + goto error; + if (create_textfile(UBTXT3, 511) < 0) + goto error; + if (create_textfile(UBTXT4, 512) < 0) + goto error; + if (create_textfile(UBTXT5, 513) < 0) + goto error; + /* create_textfile(UBTXT6, (size_t)1023); create_textfile(UBTXT7, (size_t)1024); @@ -365,11 +486,18 @@ main(void) create_binfile(UBBIN3, (off_t)511); create_binfile(UBBIN4, (off_t)512); create_binfile(UBBIN5, (off_t)513); - */ - gent_ub(FILE7, (size_t)0, (size_t)0); - gent_ub(FILE8, (size_t)512, HDstrlen(pattern)); - gent_ub(FILE9, (size_t)1024, (size_t)513); - return 0; + if (gent_ub(FILE7, 0, 0) < 0) + goto error; + if (gent_ub(FILE8, 512, HDstrlen(pattern)) < 0) + goto error; + if (gent_ub(FILE9, 1024, 513) < 0) + goto error; + + return EXIT_SUCCESS; + +error: + HDfprintf(stderr, "h5jam test generator FAILED\n"); + return EXIT_FAILURE; } diff --git a/tools/test/h5stat/h5stat_gentest.c b/tools/test/h5stat/h5stat_gentest.c index a9813e7..c775d6a 100644 --- a/tools/test/h5stat/h5stat_gentest.c +++ b/tools/test/h5stat/h5stat_gentest.c @@ -51,7 +51,7 @@ * Generate HDF5 file with latest format with * NUM_GRPS groups and NUM_ATTRS attributes for the dataset */ -static void +static herr_t gen_newgrat_file(const char *fname) { hid_t fcpl = H5I_INVALID_HID; /* File creation property */ @@ -117,6 +117,21 @@ gen_newgrat_file(const char *fname) } /* end for */ /* Close dataset, dataspace, datatype, file */ + if (H5Pclose(fapl) < 0) + goto error; + if (H5Pclose(fcpl) < 0) + goto error; + if (H5Dclose(did) < 0) + goto error; + if (H5Tclose(tid) < 0) + goto error; + if (H5Sclose(sid) < 0) + goto error; + if (H5Fclose(fid) < 0) + goto error; + + return SUCCEED; + error: H5E_BEGIN_TRY { @@ -130,6 +145,8 @@ error: H5Fclose(fid); } H5E_END_TRY; + + return FAIL; } /* gen_newgrat_file() */ /* @@ -139,7 +156,7 @@ error: * datasets. -a N (--numattrs=N): Set the threshold for the # of attributes when printing information for * small # of attributes. */ -static void +static herr_t gen_threshold_file(const char *fname) { hid_t fid = H5I_INVALID_HID; /* File ID */ @@ -302,6 +319,23 @@ gen_threshold_file(const char *fname) goto error; } + if (H5Gclose(gid) < 0) + goto error; + if (H5Sclose(sid0) < 0) + goto error; + if (H5Sclose(sid1) < 0) + goto error; + if (H5Sclose(sid2) < 0) + goto error; + if (H5Sclose(sid3) < 0) + goto error; + if (H5Sclose(sid4) < 0) + goto error; + if (H5Fclose(fid) < 0) + goto error; + + return SUCCEED; + error: H5E_BEGIN_TRY { @@ -317,6 +351,8 @@ error: } H5E_END_TRY; + return FAIL; + } /* gen_threshold_file() */ /* @@ -327,18 +363,21 @@ error: * one dataset: fixed dimension, chunked layout, w/ filters * */ -static void +static herr_t gen_idx_file(const char *fname) { - hid_t fapl = H5I_INVALID_HID; /* file access property id */ - hid_t fid = H5I_INVALID_HID; /* file id */ - hid_t sid = H5I_INVALID_HID; /* space id */ - hid_t dcpl = H5I_INVALID_HID; /* dataset creation property id */ - hid_t did = -1, did2 = H5I_INVALID_HID; /* dataset id */ - hsize_t dims[1] = {10}; /* dataset dimension */ - hsize_t c_dims[1] = {2}; /* chunk dimension */ - int i; /* local index variable */ - int buf[10]; /* data buffer */ + hid_t fapl = H5I_INVALID_HID; /* file access property id */ + hid_t fid = H5I_INVALID_HID; /* file id */ + hid_t sid = H5I_INVALID_HID; /* space id */ + hid_t dcpl = H5I_INVALID_HID; /* dataset creation property id */ + hid_t did = H5I_INVALID_HID; /* dataset id */ +#if defined(H5_HAVE_FILTER_DEFLATE) + hid_t did2 = H5I_INVALID_HID; /* dataset id (compressed) */ +#endif + hsize_t dims[1] = {10}; /* dataset dimension */ + hsize_t c_dims[1] = {2}; /* chunk dimension */ + int i; /* local index variable */ + int buf[10]; /* data buffer */ /* Get a copy of the file access property */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) @@ -390,6 +429,19 @@ gen_idx_file(const char *fname) #endif /* closing: dataspace, dataset, file */ + if (H5Pclose(fapl) < 0) + goto error; + if (H5Pclose(dcpl) < 0) + goto error; + if (H5Sclose(sid) < 0) + goto error; + if (H5Dclose(did) < 0) + goto error; + if (H5Fclose(fid) < 0) + goto error; + + return SUCCEED; + error: H5E_BEGIN_TRY { @@ -404,6 +456,8 @@ error: } H5E_END_TRY; + return FAIL; + } /* gen_idx_file() */ /* @@ -419,20 +473,21 @@ error: * H5O_refcount_decode in the jira issue. * */ -static void +static herr_t gen_err_refcount(const char *fname) { - hid_t fid = H5I_INVALID_HID; /* File identifier */ - hid_t sid = H5I_INVALID_HID; /* Dataspace message */ - hid_t did = H5I_INVALID_HID; /* Dataset identifier */ - hid_t gid = H5I_INVALID_HID; /* Group identifier */ - hid_t aid1 = -1, aid2 = H5I_INVALID_HID; /* Attribute identifier */ - hid_t tid = H5I_INVALID_HID; /* Datatype identifier */ - int i, n; /* Local index variables */ - int buf[10]; /* Data buffer */ - hsize_t dims[1]; /* Dimension size */ - int fd = -1; /* File descriptor */ - unsigned short val = 22; /* The refcount message ID */ + hid_t fid = H5I_INVALID_HID; /* File identifier */ + hid_t sid = H5I_INVALID_HID; /* Dataspace message */ + hid_t did = H5I_INVALID_HID; /* Dataset identifier */ + hid_t gid = H5I_INVALID_HID; /* Group identifier */ + hid_t aid1 = H5I_INVALID_HID; /* Attribute identifier */ + hid_t aid2 = H5I_INVALID_HID; /* Attribute identifier */ + hid_t tid = H5I_INVALID_HID; /* Datatype identifier */ + int i, n; /* Local index variables */ + int buf[10]; /* Data buffer */ + hsize_t dims[1]; /* Dimension size */ + int fd = -1; /* File descriptor */ + unsigned short val = 22; /* The refcount message ID */ /* Initialize data buffer */ n = 0; @@ -485,6 +540,10 @@ gen_err_refcount(const char *fname) goto error; if (H5Tclose(tid) < 0) goto error; + + /* Be sure to close this before opening the file again via open(), below, + * or you'll possibly trip over the file locking. + */ if (H5Fclose(fid) < 0) goto error; @@ -495,11 +554,16 @@ gen_err_refcount(const char *fname) with the committed datatype */ /* 24: the offset in the object header containing the version of the attribute message */ - if ((fd = HDopen(fname, O_RDWR, 0633)) >= 0) { - HDlseek(fd, 4520 + 24, SEEK_SET); - (void)HDwrite(fd, &val, 2); - HDclose(fd); - } + if ((fd = HDopen(fname, O_RDWR, 0633)) < 0) + goto error; + if (HDlseek(fd, 4520 + 24, SEEK_SET) < 0) + goto error; + if (HDwrite(fd, &val, 2) < 0) + goto error; + if (HDclose(fd) < 0) + goto error; + + return SUCCEED; error: H5E_BEGIN_TRY @@ -513,6 +577,11 @@ error: H5Fclose(fid); } H5E_END_TRY; + + if (fd >= 0) + HDclose(fd); + + return FAIL; } /* gen_err_refcount() */ /* @@ -542,14 +611,22 @@ error: int main(void) { - gen_newgrat_file(NEWGRAT_FILE); - gen_threshold_file(THRESHOLD_FILE); + if (gen_newgrat_file(NEWGRAT_FILE) < 0) + goto error; + if (gen_threshold_file(THRESHOLD_FILE) < 0) + goto error; /* Generate an HDF file to test for datasets with Fixed Array indexing */ - gen_idx_file(IDX_FILE); + if (gen_idx_file(IDX_FILE) < 0) + goto error; /* Generate a file with a refcount message ID */ - gen_err_refcount(ERR_REFCOUNT_FILE); + if (gen_err_refcount(ERR_REFCOUNT_FILE) < 0) + goto error; + + return EXIT_SUCCESS; - return 0; +error: + HDfprintf(stderr, "h5stat test generator FAILED\n"); + return EXIT_FAILURE; } -- cgit v0.12 From 725da3768b6537d0cfae59f6b220396ea9fa042d Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Thu, 22 Jul 2021 11:42:06 -0700 Subject: Removes unused commented-out code from h5jamgentest.c (#851) --- tools/test/h5jam/h5jamgentest.c | 77 ----------------------------------------- 1 file changed, 77 deletions(-) diff --git a/tools/test/h5jam/h5jamgentest.c b/tools/test/h5jam/h5jamgentest.c index ae6c594..5632cc0 100644 --- a/tools/test/h5jam/h5jamgentest.c +++ b/tools/test/h5jam/h5jamgentest.c @@ -31,24 +31,7 @@ #define UBTXT3 "u511.txt" #define UBTXT4 "u512.txt" #define UBTXT5 "u513.txt" -/* not used yet -#define UBTXT6 "u1023.txt" -#define UBTXT7 "u1024.txt" -#define UBTXT8 "u1025.txt" -#define UBTXT9 "u2047.txt" -#define UBTXT10 "u2048.txt" -#define UBTXT11 "u2049.txt" -#define UBBIN1 "u0.dat" -#define UBBIN2 "u10.dat" -#define UBBIN3 "u511.dat" -#define UBBIN4 "u512.dat" -#define UBBIN5 "u513.dat" -*/ -/* not used yet -#define FILE1 "tnull.h5" -#define FILE2 "tnullwithub.h5" -*/ /* tall is same as dumper test */ #define FILE7 "tall.h5" #define FILE8 "twithub.h5" @@ -66,17 +49,6 @@ char pattern[11] = "abcdefghij"; #define BUF_SIZE 1024 -/* Element selection information */ - -typedef enum { RED, GREEN, BLUE, WHITE, BLACK } enumtype; - -/* Compound datatype */ -typedef struct s1_t { - unsigned int a; - unsigned int b; - float c; -} s1_t; - /* A UD link traversal function. Shouldn't actually be called. */ static hid_t UD_traverse(const char H5_ATTR_UNUSED *link_name, hid_t H5_ATTR_UNUSED cur_group, @@ -115,7 +87,6 @@ g1.2.1 : slink g2 : dset2.1 dset2.2 udlink */ - static herr_t gent_ub(const char *filename, size_t ub_size, size_t ub_fill) { @@ -423,47 +394,14 @@ error: return FAIL; } -#ifdef notdef -/* not used yet */ -void -create_binfile(char *name, off_t size) -{ - char *buf; - int fd; - int i; - char *bp; - - fd = HDcreat(name, 0777); - HDassert(fd >= 0); - - buf = HDcalloc(size, 1); - HDassert(buf); - - /* Fill buf with pattern */ - bp = buf; - for (i = 0; i < size; i++) - *bp++ = (char)i & 0xff; - - (void)HDwrite(fd, buf, size); - - HDfree(buf); - HDclose(fd); -} -#endif - /*------------------------------------------------------------------------- * Function: main * *------------------------------------------------------------------------- */ - int main(void) { - /* - create_textfile(UBTXT1, (size_t)0); - */ - if (create_textfile(UBTXT2, 10) < 0) goto error; if (create_textfile(UBTXT3, 511) < 0) @@ -473,21 +411,6 @@ main(void) if (create_textfile(UBTXT5, 513) < 0) goto error; - /* - create_textfile(UBTXT6, (size_t)1023); - create_textfile(UBTXT7, (size_t)1024); - create_textfile(UBTXT8, (size_t)1025); - create_textfile(UBTXT9, (size_t)2047); - create_textfile(UBTXT10, (size_t)2048); - create_textfile(UBTXT11, (size_t)2049); - - create_binfile(UBBIN1, (off_t)0); - create_binfile(UBBIN2, (off_t)10); - create_binfile(UBBIN3, (off_t)511); - create_binfile(UBBIN4, (off_t)512); - create_binfile(UBBIN5, (off_t)513); - */ - if (gent_ub(FILE7, 0, 0) < 0) goto error; if (gent_ub(FILE8, 512, HDstrlen(pattern)) < 0) -- cgit v0.12 From f3441d54866d0736468b2697bbbee3ec81a83a5b Mon Sep 17 00:00:00 2001 From: "H. Joe Lee" Date: Mon, 26 Jul 2021 15:18:15 -0500 Subject: OESS-168:Remove clang warnings. (#835) * OESS-168:Remove implicit conversion increases floating-point precision warnings. * OESS-168:Remove clang warnings. * OESS-168:Remove clang warnings. * Committing clang-format changes * OESS-168:Remove clang warnings. * Committing clang-format changes * OESS-168:Remove clang warning. * OESS-168:Remove clang warnings. * OESS-168:Address @byrnHDF review. * Remove line added to generated hl/src/H5LTanalyze.c file. Issue #854 created for creating a fix. * Remove extra whitespace from H5LTanalyze.c. * Remove extra blank line. Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Larry Knox --- hl/test/test_ds.c | 2 ++ hl/test/test_packet.c | 8 ++++---- hl/tools/h5watch/h5watchgentest.c | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/hl/test/test_ds.c b/hl/test/test_ds.c index 8c6ef91..964e13f 100644 --- a/hl/test/test_ds.c +++ b/hl/test/test_ds.c @@ -5210,6 +5210,8 @@ test_attach_detach(void) HL_TESTING2("permutations of attaching and detaching"); + gid = var1_id = var2_id = var3_id = H5I_INVALID_HID; + if ((fid = H5Fcreate(FILE8, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) goto out; diff --git a/hl/test/test_packet.c b/hl/test/test_packet.c index 1d3a569..311f36a 100644 --- a/hl/test/test_packet.c +++ b/hl/test/test_packet.c @@ -45,10 +45,10 @@ typedef struct particle_t { * a static array of particles for writing and checking reads *------------------------------------------------------------------------- */ -static particle_t testPart[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0f}, {"one", 10, 10, 1.0f, 10.0f}, - {"two", 20, 20, 2.0f, 20.0f}, {"three", 30, 30, 3.0f, 30.0f}, - {"four", 40, 40, 4.0f, 40.0f}, {"five", 50, 50, 5.0f, 50.0f}, - {"six", 60, 60, 6.0f, 60.0f}, {"seven", 70, 70, 7.0f, 70.0f}}; +static particle_t testPart[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, + {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, + {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, + {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; /*------------------------------------------------------------------------- * function that compares one particle diff --git a/hl/tools/h5watch/h5watchgentest.c b/hl/tools/h5watch/h5watchgentest.c index c83e485..44cb0cb 100644 --- a/hl/tools/h5watch/h5watchgentest.c +++ b/hl/tools/h5watch/h5watchgentest.c @@ -224,7 +224,7 @@ main(void) one_cbuf[i].field2.b.a = 20; one_cbuf[i].field2.b.b = 40; one_cbuf[i].field2.b.c = 80; - one_cbuf[i].field3 = 3.0F; + one_cbuf[i].field3 = 3.0; one_cbuf[i].field4.a = 4; one_cbuf[i].field4.b = 8; } /* end for */ @@ -313,7 +313,7 @@ main(void) two_cbuf[i].field2.b.a = 20; two_cbuf[i].field2.b.b = 40; two_cbuf[i].field2.b.c = 80; - two_cbuf[i].field3 = 3.0F; + two_cbuf[i].field3 = 3.0; two_cbuf[i].field4.a = 4; two_cbuf[i].field4.b = 8; } /* end for */ -- cgit v0.12 From 74f1590d47acd41cd9aea8d9f6be52a5947ee5af Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Tue, 27 Jul 2021 13:59:05 -0500 Subject: Only test h5diff_830 in serial (#858) --- tools/test/h5diff/CMakeTests.cmake | 63 ++++++++++++++++++++------------------ 1 file changed, 34 insertions(+), 29 deletions(-) diff --git a/tools/test/h5diff/CMakeTests.cmake b/tools/test/h5diff/CMakeTests.cmake index 5ada730..4fcfa26 100644 --- a/tools/test/h5diff/CMakeTests.cmake +++ b/tools/test/h5diff/CMakeTests.cmake @@ -376,40 +376,44 @@ macro (ADD_H5_TEST resultfile resultcode) if (HDF5_TEST_SERIAL) - # If using memchecker add tests without using scripts - if (HDF5_ENABLE_USING_MEMCHECKER) - add_test (NAME H5DIFF-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ ${ARGN}) - set_tests_properties (H5DIFF-${resultfile} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") - if (${resultcode}) - set_tests_properties (H5DIFF-${resultfile} PROPERTIES WILL_FAIL "true") - endif () - if (last_test) - set_tests_properties (H5DIFF-${resultfile} PROPERTIES DEPENDS ${last_test}) - endif () - else () - add_test ( - NAME H5DIFF-${resultfile} - COMMAND "${CMAKE_COMMAND}" - -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}" - -D "TEST_PROGRAM=$" - -D "TEST_ARGS:STRING=${ARGN}" - -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" - -D "TEST_OUTPUT=${resultfile}.out" - -D "TEST_EXPECT=${resultcode}" - -D "TEST_REFERENCE=${resultfile}.txt" - -D "TEST_APPEND=EXIT CODE:" - -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" - ) - if (last_test) - set_tests_properties (H5DIFF-${resultfile} PROPERTIES DEPENDS ${last_test}) - endif () - endif () + ADD_SH5_TEST (${resultfile} ${resultcode} ${ARGN}) endif () if (H5_HAVE_PARALLEL AND HDF5_TEST_PARALLEL) ADD_PH5_TEST (${resultfile} ${resultcode} ${ARGN}) endif () endmacro () + macro (ADD_SH5_TEST resultfile resultcode) + # If using memchecker add tests without using scripts + if (HDF5_ENABLE_USING_MEMCHECKER) + add_test (NAME H5DIFF-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ ${ARGN}) + set_tests_properties (H5DIFF-${resultfile} PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") + if (${resultcode}) + set_tests_properties (H5DIFF-${resultfile} PROPERTIES WILL_FAIL "true") + endif () + if (last_test) + set_tests_properties (H5DIFF-${resultfile} PROPERTIES DEPENDS ${last_test}) + endif () + else () + add_test ( + NAME H5DIFF-${resultfile} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}" + -D "TEST_PROGRAM=$" + -D "TEST_ARGS:STRING=${ARGN}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${resultfile}.txt" + -D "TEST_APPEND=EXIT CODE:" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + if (last_test) + set_tests_properties (H5DIFF-${resultfile} PROPERTIES DEPENDS ${last_test}) + endif () + endif () + endmacro () + macro (ADD_PH5_TEST resultfile resultcode) # If using memchecker add tests without using scripts if (HDF5_ENABLE_USING_MEMCHECKER) @@ -1549,7 +1553,8 @@ ADD_H5_TEST (h5diff_801 1 -v ${FILE7} ${FILE8A} /g1/array /g1/array) # ############################################################################## # # dataset subsets # ############################################################################## -ADD_H5_TEST (h5diff_830 1 --enable-error-stack -v ${FILE7} ${FILE8} /g1/array3D[0,0,0;2,2,1;2,2,2;] /g1/array3D[0,0,0;2,2,1;2,2,2;]) +#serial only +ADD_SH5_TEST (h5diff_830 1 --enable-error-stack -v ${FILE7} ${FILE8} /g1/array3D[0,0,0;2,2,1;2,2,2;] /g1/array3D[0,0,0;2,2,1;2,2,2;]) # ############################################################################## # # VDS tests -- cgit v0.12 From cfcdf215187653703a21f1f99cc18e207138c39b Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Tue, 27 Jul 2021 15:44:05 -0500 Subject: Remove version-check for relnum in H5check. (#812) * Remove version-check for relnum in H5check. * Add in release exception code check * Fix typos * Fix more typos * Rework comments * format change * format whitespace * Library version must be less than or equal to headers * Need the NOT version of the compare * Enable release+1 check for PASS * Add release note * Update note --- release_docs/RELEASE.txt | 19 ++++ src/H5.c | 241 +++++++++++++++++++++++++++---------------- src/H5private.h | 6 +- test/CMakeTests.cmake | 2 +- test/testcheck_version.sh.in | 106 +++++++++---------- 5 files changed, 227 insertions(+), 147 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index bd186d8..314a713 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -463,6 +463,25 @@ New Features Library: -------- + - Change how the release part of version, in major.minor.release is checked + for compatibility + + The HDF5 library uses a function, H5check_version, to check that + the version defined in the header files, which is used to compile an + application is compatible with the version codified in the library, which + the application loads at runtime. This previously required an exact match + or the library would print a warning, dump the build settings and then + abort or continue. An environment variable controlled the logic. + + Now the function first checks that the library release version, in + major.minor.release, is not older than the version in the headers. + Secondly, if the release version is different, it checks if either + the library version or the header version is in the exception list, in + which case the release part of version, in major.minor.release, must + be exact. An environment variable still controls the logic. + + (ADB - 2021/07/27) + - gcc warning suppression macros were moved out of H5public.h The HDF5 library uses a set of macros to suppress warnings on gcc. diff --git a/src/H5.c b/src/H5.c index 71a8a30..12d226c 100644 --- a/src/H5.c +++ b/src/H5.c @@ -70,6 +70,10 @@ hbool_t H5_PKG_INIT_VAR = FALSE; /* Library Private Variables */ /*****************************/ +/* Library incompatible release versions */ +const unsigned VERS_RELEASE_EXCEPTIONS[] = {0}; +const unsigned VERS_RELEASE_EXCEPTIONS_SIZE = 0; + /* statically initialize block for pthread_once call used in initializing */ /* the first global mutex */ #ifdef H5_HAVE_THREADSAFE @@ -281,13 +285,13 @@ done: } /* end H5_init_library() */ /*------------------------------------------------------------------------- - * Function: H5_term_library + * Function: H5_term_library * - * Purpose: Terminate interfaces in a well-defined order due to - * dependencies among the interfaces, then terminate - * library-specific data. + * Purpose: Terminate interfaces in a well-defined order due to + * dependencies among the interfaces, then terminate + * library-specific data. * - * Return: void + * Return: void * *------------------------------------------------------------------------- */ @@ -505,22 +509,22 @@ done: } /* end H5_term_library() */ /*------------------------------------------------------------------------- - * Function: H5dont_atexit + * Function: H5dont_atexit * - * Purpose: Indicates that the library is not to clean up after itself - * when the application exits by calling exit() or returning - * from main(). This function must be called before any other - * HDF5 function or constant is used or it will have no effect. + * Purpose: Indicates that the library is not to clean up after itself + * when the application exits by calling exit() or returning + * from main(). This function must be called before any other + * HDF5 function or constant is used or it will have no effect. * - * If this function is used then certain memory buffers will not - * be de-allocated nor will open files be flushed automatically. - * The application may still call H5close() explicitly to - * accomplish these things. + * If this function is used then certain memory buffers will not + * be de-allocated nor will open files be flushed automatically. + * The application may still call H5close() explicitly to + * accomplish these things. * - * Return: Success: non-negative + * Return: Success: non-negative * - * Failure: negative if this function is called more than - * once or if it is called too late. + * Failure: negative if this function is called more than + * once or if it is called too late. * *------------------------------------------------------------------------- */ @@ -541,19 +545,19 @@ H5dont_atexit(void) } /* end H5dont_atexit() */ /*------------------------------------------------------------------------- - * Function: H5garbage_collect + * Function: H5garbage_collect * - * Purpose: Walks through all the garbage collection routines for the - * library, which are supposed to free any unused memory they have - * allocated. + * Purpose: Walks through all the garbage collection routines for the + * library, which are supposed to free any unused memory they have + * allocated. * * These should probably be registered dynamically in a linked list of * functions to call, but there aren't that many right now, so we * hard-wire them... * - * Return: Success: non-negative + * Return: Success: non-negative * - * Failure: negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -574,9 +578,9 @@ done: } /* end H5garbage_collect() */ /*------------------------------------------------------------------------- - * Function: H5set_free_list_limits + * Function: H5set_free_list_limits * - * Purpose: Sets limits on the different kinds of free lists. Setting a value + * Purpose: Sets limits on the different kinds of free lists. Setting a value * of -1 for a limit means no limit of that type. These limits are global * for the entire library. Each "global" limit only applies to free lists * of that type, so if an application sets a limit of 1 MB on each of the @@ -594,9 +598,9 @@ done: * int blk_global_lim; IN: The limit on all "block" free list memory used * int blk_list_lim; IN: The limit on memory used in each "block" free list * - * Return: Success: non-negative + * Return: Success: non-negative * - * Failure: negative + * Failure: negative * *------------------------------------------------------------------------- */ @@ -620,11 +624,11 @@ done: } /* end H5set_free_list_limits() */ /*------------------------------------------------------------------------- - * Function: H5get_free_list_sizes + * Function: H5get_free_list_sizes * - * Purpose: Gets the current size of the different kinds of free lists that - * the library uses to manage memory. The free list sizes can be set with - * H5set_free_list_limits and garbage collected with H5garbage_collect. + * Purpose: Gets the current size of the different kinds of free lists that + * the library uses to manage memory. The free list sizes can be set with + * H5set_free_list_limits and garbage collected with H5garbage_collect. * These lists are global for the entire library. * * Parameters: @@ -633,8 +637,8 @@ done: * size_t *blk_size; OUT: The current size of all "block" free list memory used * size_t *fac_size; OUT: The current size of all "factory" free list memory used * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * * Programmer: Quincey Koziol * Friday, March 6, 2020 @@ -659,23 +663,23 @@ done: } /* end H5get_free_list_sizes() */ /*------------------------------------------------------------------------- - * Function: H5get_alloc_stats + * Function: H5get_alloc_stats * - * Purpose: Gets the memory allocation statistics for the library, if the - * --enable-memory-alloc-sanity-check option was given when building the + * Purpose: Gets the memory allocation statistics for the library, if the + * --enable-memory-alloc-sanity-check option was given when building the * library. Applications can check whether this option was enabled by - * detecting if the 'H5_MEMORY_ALLOC_SANITY_CHECK' macro is defined. This - * option is enabled by default for debug builds of the library and - * disabled by default for non-debug builds. If the option is not enabled, - * all the values returned with be 0. These statistics are global for the - * entire library, but don't include allocations from chunked dataset I/O - * filters or non-native VOL connectors. + * detecting if the 'H5_MEMORY_ALLOC_SANITY_CHECK' macro is defined. This + * option is enabled by default for debug builds of the library and + * disabled by default for non-debug builds. If the option is not enabled, + * all the values returned with be 0. These statistics are global for the + * entire library, but don't include allocations from chunked dataset I/O + * filters or non-native VOL connectors. * * Parameters: * H5_alloc_stats_t *stats; OUT: Memory allocation statistics * - * Return: Success: non-negative - * Failure: negative + * Return: Success: non-negative + * Failure: negative * * Programmer: Quincey Koziol * Saturday, March 7, 2020 @@ -808,12 +812,12 @@ H5__debug_mask(const char *s) #ifdef H5_HAVE_PARALLEL /*------------------------------------------------------------------------- - * Function: H5__mpi_delete_cb + * Function: H5__mpi_delete_cb * - * Purpose: Callback attribute on MPI_COMM_SELF to terminate the HDF5 + * Purpose: Callback attribute on MPI_COMM_SELF to terminate the HDF5 * library when the communicator is destroyed, i.e. on MPI_Finalize. * - * Return: MPI_SUCCESS + * Return: MPI_SUCCESS * *------------------------------------------------------------------------- */ @@ -827,18 +831,18 @@ H5__mpi_delete_cb(MPI_Comm H5_ATTR_UNUSED comm, int H5_ATTR_UNUSED keyval, void #endif /*H5_HAVE_PARALLEL*/ /*------------------------------------------------------------------------- - * Function: H5get_libversion + * Function: H5get_libversion * - * Purpose: Returns the library version numbers through arguments. MAJNUM - * will be the major revision number of the library, MINNUM the - * minor revision number, and RELNUM the release revision number. + * Purpose: Returns the library version numbers through arguments. MAJNUM + * will be the major revision number of the library, MINNUM the + * minor revision number, and RELNUM the release revision number. * - * Note: When printing an HDF5 version number it should be printed as + * Note: When printing an HDF5 version number it should be printed as * - * printf("%u.%u.%u", maj, min, rel) or - * printf("version %u.%u release %u", maj, min, rel) + * printf("%u.%u.%u", maj, min, rel) or + * printf("version %u.%u release %u", maj, min, rel) * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -863,17 +867,20 @@ done: } /* end H5get_libversion() */ /*------------------------------------------------------------------------- - * Function: H5check_version + * Function: H5check_version * - * Purpose: Verifies that the arguments match the version numbers - * compiled into the library. This function is intended to be - * called from user to verify that the versions of header files - * compiled into the application match the version of the hdf5 - * library. + * Purpose: Verifies that the arguments match the version numbers + * compiled into the library. This function is intended to be + * called from user to verify that the versions of header files + * compiled into the application match the version of the hdf5 + * library. + * Within major.minor.release version, the expectation + * is that all release versions are compatible, exceptions to + * this rule must be added to the VERS_RELEASE_EXCEPTIONS list. * - * Return: Success: SUCCEED + * Return: Success: SUCCEED * - * Failure: abort() + * Failure: abort() * *------------------------------------------------------------------------- */ @@ -886,6 +893,15 @@ done: "linked with a different version of static or shared HDF5 library.\n" \ "You should recompile the application or check your shared library related\n" \ "settings such as 'LD_LIBRARY_PATH'.\n" +#define RELEASE_MISMATCH_WARNING \ + "Warning! ***HDF5 library release mismatched error***\n" \ + "The HDF5 header files used to compile this application are not compatible with\n" \ + "the version used by the HDF5 library to which this application is linked.\n" \ + "Data corruption or segmentation faults may occur if the application continues.\n" \ + "This can happen when an application was compiled by one version of HDF5 but\n" \ + "linked with an incompatible version of static or shared HDF5 library.\n" \ + "You should recompile the application or check your shared library related\n" \ + "settings such as 'LD_LIBRARY_PATH'.\n" herr_t H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) @@ -914,7 +930,8 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) disable_version_check = (unsigned int)HDstrtol(s, NULL, 0); } - if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE != relnum) { + /* H5_VERS_MAJOR and H5_VERS_MINOR must match */ + if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > relnum) { switch (disable_version_check) { case 0: HDfprintf(stderr, "%s%s", version_mismatch_warning, @@ -949,7 +966,51 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) break; } /* end switch */ - } /* end if */ + } /* end if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > relnum) */ + + /* H5_VERS_RELEASE should be compatible, we will only add checks for exceptions */ + if (H5_VERS_RELEASE != relnum) { + for (unsigned i = 0; i < VERS_RELEASE_EXCEPTIONS_SIZE; i++) { + /* Check for incompatible headers or incompatible library */ + if (VERS_RELEASE_EXCEPTIONS[i] == relnum || VERS_RELEASE_EXCEPTIONS[i] == H5_VERS_RELEASE) { + switch (disable_version_check) { + case 0: + HDfprintf( + stderr, "%s%s", version_mismatch_warning, + "You can, at your own risk, disable this warning by setting the environment\n" + "variable 'HDF5_DISABLE_VERSION_CHECK' to a value of '1'.\n" + "Setting it to 2 or higher will suppress the warning messages totally.\n"); + /* Mention the versions we are referring to */ + HDfprintf(stderr, "Headers are %u.%u.%u, library is %u.%u.%u\n", majnum, minnum, + relnum, (unsigned)H5_VERS_MAJOR, (unsigned)H5_VERS_MINOR, + (unsigned)H5_VERS_RELEASE); + + /* Bail out now. */ + HDfputs("Bye...\n", stderr); + HDabort(); + case 1: + /* continue with a warning */ + /* Note that the warning message is embedded in the format string.*/ + HDfprintf(stderr, + "%s'HDF5_DISABLE_VERSION_CHECK' " + "environment variable is set to %d, application will\n" + "continue at your own risk.\n", + version_mismatch_warning, disable_version_check); + /* Mention the versions we are referring to */ + HDfprintf(stderr, "Headers are %u.%u.%u, library is %u.%u.%u\n", majnum, minnum, + relnum, (unsigned)H5_VERS_MAJOR, (unsigned)H5_VERS_MINOR, + (unsigned)H5_VERS_RELEASE); + break; + default: + /* 2 or higher: continue silently */ + break; + } /* end switch */ + + } /* end if */ + + } /* end for */ + + } /* end if (H5_VERS_RELEASE != relnum) */ /* Indicate that the version check has been performed */ checked = 1; @@ -991,7 +1052,7 @@ done: * is failing inexplicably, then try calling this function * first. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1010,12 +1071,12 @@ done: } /* end H5open() */ /*------------------------------------------------------------------------- - * Function: H5atclose + * Function: H5atclose * - * Purpose: Register a callback for the library to invoke when it's - * closing. Callbacks are invoked in LIFO order. + * Purpose: Register a callback for the library to invoke when it's + * closing. Callbacks are invoked in LIFO order. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1049,11 +1110,11 @@ done: } /* end H5atclose() */ /*------------------------------------------------------------------------- - * Function: H5close + * Function: H5close * - * Purpose: Terminate the library and release all resources. + * Purpose: Terminate the library and release all resources. * - * Return: Non-negative on success/Negative on failure + * Return: Non-negative on success/Negative on failure * *------------------------------------------------------------------------- */ @@ -1074,9 +1135,9 @@ H5close(void) } /* end H5close() */ /*------------------------------------------------------------------------- - * Function: H5allocate_memory + * Function: H5allocate_memory * - * Purpose: Allocate a memory buffer with the semantics of malloc(). + * Purpose: Allocate a memory buffer with the semantics of malloc(). * * NOTE: This function is intended for use with filter * plugins so that all allocation and free operations @@ -1114,9 +1175,9 @@ H5allocate_memory(size_t size, hbool_t clear) } /* end H5allocate_memory() */ /*------------------------------------------------------------------------- - * Function: H5resize_memory + * Function: H5resize_memory * - * Purpose: Resize a memory buffer with the semantics of realloc(). + * Purpose: Resize a memory buffer with the semantics of realloc(). * * NOTE: This function is intended for use with filter * plugins so that all allocation and free operations @@ -1151,14 +1212,14 @@ H5resize_memory(void *mem, size_t size) } /* end H5resize_memory() */ /*------------------------------------------------------------------------- - * Function: H5free_memory + * Function: H5free_memory * - * Purpose: Frees memory allocated by the library that it is the user's + * Purpose: Frees memory allocated by the library that it is the user's * responsibility to free. Ensures that the same library * that was used to allocate the memory frees it. Passing * NULL pointers is allowed. * - * Return: SUCCEED/FAIL + * Return: SUCCEED/FAIL * *------------------------------------------------------------------------- */ @@ -1175,12 +1236,12 @@ H5free_memory(void *mem) } /* end H5free_memory() */ /*------------------------------------------------------------------------- - * Function: H5is_library_threadsafe + * Function: H5is_library_threadsafe * - * Purpose: Checks to see if the library was built with thread-safety + * Purpose: Checks to see if the library was built with thread-safety * enabled. * - * Return: SUCCEED/FAIL + * Return: SUCCEED/FAIL * *------------------------------------------------------------------------- */ @@ -1206,16 +1267,16 @@ H5is_library_threadsafe(hbool_t *is_ts /*out*/) } /* end H5is_library_threadsafe() */ /*------------------------------------------------------------------------- - * Function: H5is_library_terminating + * Function: H5is_library_terminating * - * Purpose: Checks to see if the library is shutting down. + * Purpose: Checks to see if the library is shutting down. * - * Note: Useful for plugins to detect when the library is terminating. - * For example, a VOL connector could check if a "file close" - * callback was the result of the library shutdown process, or - * an API action from the application. + * Note: Useful for plugins to detect when the library is terminating. + * For example, a VOL connector could check if a "file close" + * callback was the result of the library shutdown process, or + * an API action from the application. * - * Return: SUCCEED/FAIL + * Return: SUCCEED/FAIL * *------------------------------------------------------------------------- */ diff --git a/src/H5private.h b/src/H5private.h index 7823bfc..802dd90 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -2169,7 +2169,7 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); * Use this macro for API functions that shouldn't perform _any_ initialization * of the library or an interface, or push themselves on the function * stack, or perform tracing, etc. This macro _only_ sanity checks the - * API name itself. Examples are: H5TSmutex_acquire, + * API name itself. Examples are: H5TSmutex_acquire, * */ #define FUNC_ENTER_API_NAMECHECK_ONLY \ @@ -2265,7 +2265,7 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); * Use this macro for non-API functions that shouldn't perform _any_ initialization * of the library or an interface, or push themselves on the function * stack, or perform tracing, etc. This macro _only_ sanity checks the - * API name itself. Examples are private routines in the H5TS package. + * API name itself. Examples are private routines in the H5TS package. * */ #define FUNC_ENTER_NOAPI_NAMECHECK_ONLY \ @@ -2343,7 +2343,7 @@ H5_DLL herr_t H5CX_pop(hbool_t update_dxpl_props); * Use this macro for non-API functions that shouldn't perform _any_ initialization * of the library or an interface, or push themselves on the function * stack, or perform tracing, etc. This macro _only_ sanity checks the - * API name itself. Examples are static routines in the H5TS package. + * API name itself. Examples are static routines in the H5TS package. * */ #define FUNC_ENTER_STATIC_NAMECHECK_ONLY \ diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index e840559..bb5356d 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -649,10 +649,10 @@ set_tests_properties (H5TEST-tcheck_version-minor PROPERTIES WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST WILL_FAIL "true" ) +# release + 1 should pass add_test (NAME H5TEST-tcheck_version-release COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $ "-tr") set_tests_properties (H5TEST-tcheck_version-release PROPERTIES WORKING_DIRECTORY ${HDF5_TEST_BINARY_DIR}/H5TEST - WILL_FAIL "true" ) ############################################################################## diff --git a/test/testcheck_version.sh.in b/test/testcheck_version.sh.in index 43d1b46..6378ee5 100644 --- a/test/testcheck_version.sh.in +++ b/test/testcheck_version.sh.in @@ -15,7 +15,7 @@ # Tests for the H5check_version function. # # Programmer: Albert Cheng -# Sep 28, 2009 +# Sep 28, 2009 srcdir=@srcdir@ @@ -24,7 +24,7 @@ srcdir=@srcdir@ Shared_Lib=@enable_shared@ Static_Lib=@enable_static@ Static_exec=@STATIC_EXEC@ -h5haveexitcode=yes # default is yes +h5haveexitcode=yes # default is yes CMP='cmp -s' DIFF='diff -c' @@ -103,10 +103,10 @@ WarnMesg2(){ # mismatch). # # Expected results: -# Value of $HDF5_DISABLE_VERSION_CHECK -# unset "" -1 0 1 2 3 -# Matched OK OK OK OK OK OK OK -# Mismatched W/A W/A W/A W/A W2/OK OK W2/OK +# Value of $HDF5_DISABLE_VERSION_CHECK +# unset "" -1 0 1 2 3 +# Matched OK OK OK OK OK OK OK +# Mismatched W/A W/A W/A W/A W2/OK OK W2/OK # Result codes: # OK: No warning, exit 0. # W/A: Warning, abort and exit non-0. @@ -130,42 +130,42 @@ TESTING() { xxh5versrelease=$h5versrelease if [ "$h5DisableVersion" = unset ]; then - envcmd="" # noop + envcmd="" # noop else - envcmd="env HDF5_DISABLE_VERSION_CHECK=$h5DisableVersion" + envcmd="env HDF5_DISABLE_VERSION_CHECK=$h5DisableVersion" fi if [ "$wrongversionnumbers" = none ]; then - # OK: No warning, exit 0 - cp /dev/null $expect - expect_code=0 + # OK: No warning, exit 0 + cp /dev/null $expect + expect_code=0 else - arguments=-t"$wrongversionnumbers" - # calculate mismatched version numbers by listing. - case $wrongversionnumbers in - "M") xxh5versmajor=`expr $h5versmajor + 1` - ;; - "m") xxh5versminor=`expr $h5versminor + 1` - ;; - "r") xxh5versrelease=`expr $h5versrelease + 1` - ;; - esac - case "$h5DisableVersion" in - 1) - # W2/OK: Different Warning, exit 0. - WarnMesg2 > $expect - expect_code=0 - ;; - [2-9]|[1-9][0-9]*) - # OK: No warning, exit 0 - cp /dev/null $expect - expect_code=0 - ;; - *) # W/A: Warning, abort and exit non-0. - WarnMesg > $expect - expect_code=6 # Signal Abort exit code (128+6) - ;; - esac + arguments=-t"$wrongversionnumbers" + # calculate mismatched version numbers by listing. + case $wrongversionnumbers in + "M") xxh5versmajor=`expr $h5versmajor + 1` + ;; + "m") xxh5versminor=`expr $h5versminor + 1` + ;; + "r") xxh5versrelease=`expr $h5versrelease + 1` + ;; + esac + case "$h5DisableVersion" in + 1) + # W2/OK: Different Warning, exit 0. + WarnMesg2 > $expect + expect_code=0 + ;; + [2-9]|[1-9][0-9]*) + # OK: No warning, exit 0 + cp /dev/null $expect + expect_code=0 + ;; + *) # W/A: Warning, abort and exit non-0. + WarnMesg > $expect + expect_code=6 # Signal Abort exit code (128+6) + ;; + esac fi # Run test. @@ -177,23 +177,23 @@ TESTING() { cat $actual_err >> $actual if [ $h5haveexitcode = 'yes' -a \( $expect_code -ne $ret_code \) ]; then - echo "*FAILED*" - echo " Expected exit code ($expect_code) differs from actual code ($ret_code)" - nerrors="`expr $nerrors + 1`" + echo "*FAILED*" + echo " Expected exit code ($expect_code) differs from actual code ($ret_code)" + nerrors="`expr $nerrors + 1`" elif $CMP $expect $actual; then - echo " PASSED" + echo " PASSED" else - echo "*FAILED*" - echo " Expected result differs from actual result" - nerrors="`expr $nerrors + 1`" - test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + echo "*FAILED*" + echo " Expected result differs from actual result" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' fi # Clean up output file. # Also clean the core file generated by H5check_version's abort. if test -z "$HDF5_NOCLEANUP"; then - $RM $expect $actual $actual_err - $RM core + $RM $expect $actual $actual_err + $RM core fi } @@ -201,15 +201,15 @@ TESTING() { # Echo parameters for debugging if verbose mode is on. DEBUGPRINT() { if [ -n "$debugmode" ]; then - echo $* + echo $* fi } # MAIN Body nerrors=0 -verbose=yes # default on -debugmode= # default off +verbose=yes # default on +debugmode= # default off H5_HAVE_EMBEDDED_LIBINFO=`grep '#define H5_HAVE_EMBEDDED_LIBINFO ' ../src/H5pubconf.h` h5libsettings=../src/libhdf5.settings @@ -240,13 +240,13 @@ fi # Three Categories of tests: # Normal: where the version numbers all matched (wrong_version == none). # Mismatched version numbers (could be Major or minor version -# or release numbers or a combination of all three.) +# or release numbers or a combination of all three.) # Test all the above with different values of the environment variable, # HDF5_DISABLE_VERSION_CHECK, as unset, "", -1, 0, 1, 2, 3 for val_disable_version_check in unset "" -1 0 1 2 3; do - for wrong_version in none M m r; do - TESTING "$val_disable_version_check" "$wrong_version" + for wrong_version in none M m; do + TESTING "$val_disable_version_check" "$wrong_version" done done -- cgit v0.12 From 9bb1d6d78e305e73b5c882e8f5b3ef66a1acdf8d Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Wed, 28 Jul 2021 14:29:10 -0700 Subject: Updates a comment in H5mpi.c (#863) MPI_Type_struct() is an MPI-1 call we no longer use. --- src/H5mpi.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5mpi.c b/src/H5mpi.c index 9749721..4a8aa44 100644 --- a/src/H5mpi.c +++ b/src/H5mpi.c @@ -523,7 +523,7 @@ H5_mpio_create_large_type(hsize_t num_elements, MPI_Aint stride_bytes, MPI_Datat MPI_Type_get_extent(old_type, &unused_lb_arg, &old_extent); } - /* Set up the arguments for MPI_Type_struct constructor */ + /* Set up the arguments for MPI_Type_create_struct() */ type[0] = outer_type; type[1] = leftover_type; block_len[0] = 1; -- cgit v0.12 From 9fd356e2408b6f91bfbe6820cdb979b6fade3fb3 Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Thu, 29 Jul 2021 20:18:37 -0500 Subject: Added int variable for comparison with H5_VERS_RELEASE in H5.c to avoid (#866) warning that unsigned comparison < 0 is always false, which is known, but for later versions the comparison can possibly be true. --- src/H5.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/H5.c b/src/H5.c index 12d226c..2debc92 100644 --- a/src/H5.c +++ b/src/H5.c @@ -912,6 +912,8 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) static unsigned int disable_version_check = 0; /* Set if the version check should be disabled */ static const char * version_mismatch_warning = VERSION_MISMATCH_WARNING; herr_t ret_value = SUCCEED; /* Return value */ + int releasenum = relnum; /* To avoid warning for unsigned < 0 + comparison in first release versions */ FUNC_ENTER_API_NOINIT_NOERR_NOFS H5TRACE3("e", "IuIuIu", majnum, minnum, relnum); @@ -931,7 +933,7 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) } /* H5_VERS_MAJOR and H5_VERS_MINOR must match */ - if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > relnum) { + if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > releasenum) { switch (disable_version_check) { case 0: HDfprintf(stderr, "%s%s", version_mismatch_warning, -- cgit v0.12 From 48a05aa887ae7d18512756bf1eeb177e08d0feaf Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Sun, 1 Aug 2021 22:07:01 -0500 Subject: Unsigned comparison to 0 warning avoidance (#869) * Added int variable for comparison with H5_VERS_RELEASE in H5.c to avoid warning that unsigned comparison < 0 is always false, which is known, but for later versions the comparison can possibly be true. * Better solution for avoiding warning that unsigned comparison < 0. --- src/H5.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/H5.c b/src/H5.c index 2debc92..3ceaecd 100644 --- a/src/H5.c +++ b/src/H5.c @@ -912,8 +912,6 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) static unsigned int disable_version_check = 0; /* Set if the version check should be disabled */ static const char * version_mismatch_warning = VERSION_MISMATCH_WARNING; herr_t ret_value = SUCCEED; /* Return value */ - int releasenum = relnum; /* To avoid warning for unsigned < 0 - comparison in first release versions */ FUNC_ENTER_API_NOINIT_NOERR_NOFS H5TRACE3("e", "IuIuIu", majnum, minnum, relnum); @@ -933,7 +931,10 @@ H5check_version(unsigned majnum, unsigned minnum, unsigned relnum) } /* H5_VERS_MAJOR and H5_VERS_MINOR must match */ - if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > releasenum) { + /* Cast relnum to int to avoid warning for unsigned < 0 comparison + * in first release versions */ + if (H5_VERS_MAJOR != majnum || H5_VERS_MINOR != minnum || H5_VERS_RELEASE > (int)relnum) { + switch (disable_version_check) { case 0: HDfprintf(stderr, "%s%s", version_mismatch_warning, -- cgit v0.12 From 04f1bff6757892df07cd72520af276f7691ff43b Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Tue, 3 Aug 2021 14:04:30 -0700 Subject: Updates the RELEASE.txt file with a note about the variable-length fill (#871) value issue. A fix is in progress, but it might be a while before we have time to finish it up. --- release_docs/RELEASE.txt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 314a713..20ee467 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1455,6 +1455,10 @@ The following platforms are not supported but have been tested for this release. Known Problems ============== + Setting a variable-length dataset fill value will leak the memory allocated + for the p field of the hvl_t struct. A fix is in progress for this. + HDFFV-10840 + CMake files do not behave correctly with paths containing spaces. Do not use spaces in paths because the required escaping for handling spaces results in very complex and fragile build files. -- cgit v0.12 From 5dabec4cf6b97793e3af6df3b6a3953394bbf27d Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Tue, 3 Aug 2021 14:04:58 -0700 Subject: Removes partial long options from the tools (#872) Some of the command-line tools would allow partial long options (e.g., --datas instead of --dataset). These were inconsistently implemented, difficult to maintain, and occasionally blocked useful long options. They have been removed from all the tools. NOTE: This change should NOT be merged to 1.12 or earlier. --- release_docs/RELEASE.txt | 9 +++ tools/src/h5dump/h5dump.c | 78 ------------------------- tools/src/h5format_convert/h5format_convert.c | 23 +------- tools/src/h5jam/h5jam.c | 12 ++-- tools/src/h5jam/h5unjam.c | 10 +--- tools/src/h5stat/h5stat.c | 83 --------------------------- tools/src/misc/h5clear.c | 37 ++---------- tools/test/h5stat/CMakeTests.cmake | 2 +- tools/test/h5stat/testh5stat.sh.in | 2 +- tools/test/misc/CMakeTestsClear.cmake | 4 +- tools/test/misc/testh5clear.sh.in | 4 +- 11 files changed, 29 insertions(+), 235 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 20ee467..e12f652 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -820,6 +820,15 @@ New Features Tools: ------ + - Removed partial long exceptions + + Some of the tools accepted shortened versions of the long options + (ex: --datas instead of --dataset). These were implemented inconsistently, + are difficult to maintian, and occasionally block useful long option + names. These partial long options have been removed from all the tools. + + (DER - 2021/08/03) + - h5repack added help text for user-defined filters. Added help text line that states the valid values of the filter flag diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index 70f03df..36114ba 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -82,124 +82,46 @@ struct handler_t { /* "xxx" "yyy" into "xxxyyy". */ static const char * s_opts = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HM:N:O*RS:VX:"; static struct h5_long_options l_opts[] = {{"attribute", require_arg, 'a'}, - {"attribut", require_arg, 'a'}, - {"attribu", require_arg, 'a'}, - {"attrib", require_arg, 'a'}, - {"attri", require_arg, 'a'}, - {"attr", require_arg, 'a'}, - {"att", require_arg, 'a'}, - {"at", require_arg, 'a'}, {"binary", optional_arg, 'b'}, {"count", require_arg, 'c'}, - {"coun", require_arg, 'c'}, - {"cou", require_arg, 'c'}, - {"co", require_arg, 'c'}, {"dataset", require_arg, 'd'}, - {"datase", require_arg, 'd'}, - {"datas", require_arg, 'd'}, {"escape", no_arg, 'e'}, {"filedriver", require_arg, 'f'}, - {"filedrive", require_arg, 'f'}, - {"filedriv", require_arg, 'f'}, - {"filedri", require_arg, 'f'}, - {"filedr", require_arg, 'f'}, - {"filed", require_arg, 'f'}, - {"file", require_arg, 'f'}, - {"fil", require_arg, 'f'}, - {"fi", require_arg, 'f'}, {"group", require_arg, 'g'}, - {"grou", require_arg, 'g'}, - {"gro", require_arg, 'g'}, - {"gr", require_arg, 'g'}, {"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, {"object-ids", no_arg, 'i'}, - {"object-id", no_arg, 'i'}, - {"object-i", no_arg, 'i'}, - {"object", no_arg, 'i'}, - {"objec", no_arg, 'i'}, - {"obje", no_arg, 'i'}, - {"obj", no_arg, 'i'}, - {"ob", no_arg, 'i'}, {"block", require_arg, 'k'}, - {"bloc", require_arg, 'k'}, - {"blo", require_arg, 'k'}, - {"bl", require_arg, 'k'}, {"soft-link", require_arg, 'l'}, - {"soft-lin", require_arg, 'l'}, - {"soft-li", require_arg, 'l'}, - {"soft-l", require_arg, 'l'}, - {"soft", require_arg, 'l'}, - {"sof", require_arg, 'l'}, {"format", require_arg, 'm'}, {"contents", optional_arg, 'n'}, {"output", optional_arg, 'o'}, - {"outpu", optional_arg, 'o'}, - {"outp", optional_arg, 'o'}, - {"out", optional_arg, 'o'}, - {"ou", optional_arg, 'o'}, {"properties", no_arg, 'p'}, {"sort_by", require_arg, 'q'}, {"string", no_arg, 'r'}, - {"strin", no_arg, 'r'}, {"start", require_arg, 's'}, - {"star", require_arg, 's'}, - {"sta", require_arg, 's'}, {"datatype", require_arg, 't'}, - {"datatyp", require_arg, 't'}, - {"dataty", require_arg, 't'}, - {"datat", require_arg, 't'}, {"use-dtd", no_arg, 'u'}, - {"use-dt", no_arg, 'u'}, - {"use-d", no_arg, 'u'}, - {"use-", no_arg, 'u'}, - {"use", no_arg, 'u'}, - {"us", no_arg, 'u'}, - {"u", no_arg, 'u'}, {"vds-view-first-missing", no_arg, 'v'}, {"width", require_arg, 'w'}, - {"widt", require_arg, 'w'}, - {"wid", require_arg, 'w'}, - {"wi", require_arg, 'w'}, {"xml", no_arg, 'x'}, - {"xm", no_arg, 'x'}, {"noindex", no_arg, 'y'}, {"sort_order", require_arg, 'z'}, {"onlyattr", optional_arg, 'A'}, {"superblock", no_arg, 'B'}, {"boot-block", no_arg, 'B'}, - {"boot-bloc", no_arg, 'B'}, - {"boot-blo", no_arg, 'B'}, - {"boot-bl", no_arg, 'B'}, - {"boot-b", no_arg, 'B'}, - {"boot", no_arg, 'B'}, - {"boo", no_arg, 'B'}, - {"bo", no_arg, 'B'}, {"no-compact-subset", no_arg, 'C'}, {"xml-dtd", require_arg, 'D'}, - {"xml-dt", require_arg, 'D'}, - {"xml-d", require_arg, 'D'}, {"enable-error-stack", optional_arg, 'E'}, {"form", require_arg, 'F'}, {"vds-gap-size", require_arg, 'G'}, {"header", no_arg, 'H'}, - {"heade", no_arg, 'H'}, - {"head", no_arg, 'H'}, - {"hea", no_arg, 'H'}, {"packed-bits", require_arg, 'M'}, {"any_path", require_arg, 'N'}, {"ddl", optional_arg, 'O'}, {"region", no_arg, 'R'}, {"stride", require_arg, 'S'}, - {"strid", require_arg, 'S'}, {"version", no_arg, 'V'}, - {"versio", no_arg, 'V'}, - {"versi", no_arg, 'V'}, - {"vers", no_arg, 'V'}, - {"ver", no_arg, 'V'}, - {"ve", no_arg, 'V'}, {"xml-ns", require_arg, 'X'}, - {"xml-n", require_arg, 'X'}, {"s3-cred", require_arg, '$'}, {"hdfs-attrs", require_arg, '#'}, {"vol-value", require_arg, '1'}, diff --git a/tools/src/h5format_convert/h5format_convert.c b/tools/src/h5format_convert/h5format_convert.c index 3430c55..ddf129c 100644 --- a/tools/src/h5format_convert/h5format_convert.c +++ b/tools/src/h5format_convert/h5format_convert.c @@ -39,26 +39,9 @@ static int verbose_g = 0; * parameters. */ static const char * s_opts = "hVvd:n"; -static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, - {"version", no_arg, 'V'}, - {"version", no_arg, 'V'}, - {"versio", no_arg, 'V'}, - {"versi", no_arg, 'V'}, - {"vers", no_arg, 'V'}, - {"verbose", no_arg, 'v'}, - {"verbos", no_arg, 'v'}, - {"verbo", no_arg, 'v'}, - {"verb", no_arg, 'v'}, - {"dname", require_arg, 'd'}, - {"dnam", require_arg, 'd'}, - {"dna", require_arg, 'd'}, - {"dn", require_arg, 'd'}, - {"noop", no_arg, 'n'}, - {"noo", no_arg, 'n'}, - {"no", no_arg, 'n'}, - {"enable-error-stack", no_arg, 'E'}, +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"version", no_arg, 'V'}, + {"verbose", no_arg, 'v'}, {"dname", require_arg, 'd'}, + {"noop", no_arg, 'n'}, {"enable-error-stack", no_arg, 'E'}, {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- diff --git a/tools/src/h5jam/h5jam.c b/tools/src/h5jam/h5jam.c index 0a2ea5d..7f3385c 100644 --- a/tools/src/h5jam/h5jam.c +++ b/tools/src/h5jam/h5jam.c @@ -34,14 +34,10 @@ char *ub_file = NULL; * parameters. The long-named ones can be partially spelled. When * adding more, make sure that they don't clash with each other. */ -static const char * s_opts = "hi:u:o:c:V"; /* add more later ? */ -static struct h5_long_options l_opts[] = { - {"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, {"i", require_arg, 'i'}, /* input file */ - {"u", require_arg, 'u'}, /* user block file */ - {"o", require_arg, 'o'}, /* output file */ - {"clobber", no_arg, 'c'}, /* clobber existing UB */ - {"clobbe", no_arg, 'c'}, {"clobb", no_arg, 'c'}, {"clob", no_arg, 'c'}, - {"clo", no_arg, 'c'}, {"cl", no_arg, 'c'}, {NULL, 0, '\0'}}; +static const char * s_opts = "hi:u:o:c:V"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"i", require_arg, 'i'}, + {"u", require_arg, 'u'}, {"o", require_arg, 'o'}, + {"clobber", no_arg, 'c'}, {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage diff --git a/tools/src/h5jam/h5unjam.c b/tools/src/h5jam/h5unjam.c index 38d1a5e..fa23b06 100644 --- a/tools/src/h5jam/h5unjam.c +++ b/tools/src/h5jam/h5unjam.c @@ -36,13 +36,9 @@ char *ub_file = NULL; * adding more, make sure that they don't clash with each other. */ static const char * s_opts = "hu:i:o:d:V"; -static struct h5_long_options l_opts[] = { - {"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, {"i", require_arg, 'i'}, /* input file */ - {"u", require_arg, 'u'}, /* user block file */ - {"o", require_arg, 'o'}, /* output file */ - {"delete", no_arg, 'd'}, /* delete ub */ - {"delet", no_arg, 'd'}, {"dele", no_arg, 'd'}, {"del", no_arg, 'd'}, - {"de", no_arg, 'd'}, {NULL, 0, '\0'}}; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"i", require_arg, 'i'}, + {"u", require_arg, 'u'}, {"o", require_arg, 'o'}, + {"delete", no_arg, 'd'}, {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index 3bf641a..efc80b6 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -172,102 +172,19 @@ struct handler_t { static const char *s_opts = "Aa:Ddm:EFfhGgl:sSTO:Vw:H:"; /* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */ static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, {"filemetadata", no_arg, 'F'}, - {"filemetadat", no_arg, 'F'}, - {"filemetada", no_arg, 'F'}, - {"filemetad", no_arg, 'F'}, - {"filemeta", no_arg, 'F'}, - {"filemet", no_arg, 'F'}, - {"fileme", no_arg, 'F'}, - {"filem", no_arg, 'F'}, - {"file", no_arg, 'f'}, - {"fil", no_arg, 'f'}, - {"fi", no_arg, 'f'}, {"groupmetadata", no_arg, 'G'}, - {"groupmetadat", no_arg, 'G'}, - {"groupmetada", no_arg, 'G'}, - {"groupmetad", no_arg, 'G'}, - {"groupmeta", no_arg, 'G'}, - {"groupmet", no_arg, 'G'}, - {"groupme", no_arg, 'G'}, - {"groupm", no_arg, 'G'}, - {"group", no_arg, 'g'}, - {"grou", no_arg, 'g'}, - {"gro", no_arg, 'g'}, - {"gr", no_arg, 'g'}, {"links", require_arg, 'l'}, - {"link", require_arg, 'l'}, - {"lin", require_arg, 'l'}, - {"li", require_arg, 'l'}, {"dsetmetadata", no_arg, 'D'}, - {"dsetmetadat", no_arg, 'D'}, - {"dsetmetada", no_arg, 'D'}, - {"dsetmetad", no_arg, 'D'}, - {"dsetmeta", no_arg, 'D'}, - {"dsetmet", no_arg, 'D'}, - {"dsetme", no_arg, 'D'}, - {"dsetm", no_arg, 'D'}, - {"dset", no_arg, 'd'}, - {"dse", no_arg, 'd'}, - {"ds", no_arg, 'd'}, {"dims", require_arg, 'm'}, - {"dim", require_arg, 'm'}, - {"di", require_arg, 'm'}, {"dtypemetadata", no_arg, 'T'}, - {"dtypemetadat", no_arg, 'T'}, - {"dtypemetada", no_arg, 'T'}, - {"dtypemetad", no_arg, 'T'}, - {"dtypemeta", no_arg, 'T'}, - {"dtypemet", no_arg, 'T'}, - {"dtypeme", no_arg, 'T'}, - {"dtypem", no_arg, 'T'}, - {"dtype", no_arg, 'T'}, - {"dtyp", no_arg, 'T'}, - {"dty", no_arg, 'T'}, - {"dt", no_arg, 'T'}, {"object", require_arg, 'O'}, - {"objec", require_arg, 'O'}, - {"obje", require_arg, 'O'}, - {"obj", require_arg, 'O'}, - {"ob", require_arg, 'O'}, {"version", no_arg, 'V'}, - {"versio", no_arg, 'V'}, - {"versi", no_arg, 'V'}, - {"vers", no_arg, 'V'}, - {"ver", no_arg, 'V'}, - {"ve", no_arg, 'V'}, {"attribute", no_arg, 'A'}, - {"attribut", no_arg, 'A'}, - {"attribu", no_arg, 'A'}, - {"attrib", no_arg, 'A'}, - {"attri", no_arg, 'A'}, - {"attr", no_arg, 'A'}, - {"att", no_arg, 'A'}, - {"at", no_arg, 'A'}, {"enable-error-stack", no_arg, 'E'}, {"numattrs", require_arg, 'a'}, - {"numattr", require_arg, 'a'}, - {"numatt", require_arg, 'a'}, - {"numat", require_arg, 'a'}, - {"numa", require_arg, 'a'}, - {"num", require_arg, 'a'}, - {"nu", require_arg, 'a'}, {"freespace", no_arg, 's'}, - {"freespac", no_arg, 's'}, - {"freespa", no_arg, 's'}, - {"freesp", no_arg, 's'}, - {"frees", no_arg, 's'}, - {"free", no_arg, 's'}, - {"fre", no_arg, 's'}, - {"fr", no_arg, 's'}, {"summary", no_arg, 'S'}, - {"summar", no_arg, 'S'}, - {"summa", no_arg, 'S'}, - {"summ", no_arg, 'S'}, - {"sum", no_arg, 'S'}, - {"su", no_arg, 'S'}, {"s3-cred", require_arg, 'w'}, {"hdfs-attrs", require_arg, 'H'}, {NULL, 0, '\0'}}; diff --git a/tools/src/misc/h5clear.c b/tools/src/misc/h5clear.c index 524ad03..008dae3 100644 --- a/tools/src/misc/h5clear.c +++ b/tools/src/misc/h5clear.c @@ -45,39 +45,10 @@ static hsize_t increment = DEFAULT_INCREMENT; * Command-line options: only publicize long options */ static const char * s_opts = "hVsmzi*"; -static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, - {"version", no_arg, 'V'}, - {"version", no_arg, 'V'}, - {"versio", no_arg, 'V'}, - {"versi", no_arg, 'V'}, - {"vers", no_arg, 'V'}, - {"status", no_arg, 's'}, - {"statu", no_arg, 's'}, - {"stat", no_arg, 's'}, - {"sta", no_arg, 's'}, - {"st", no_arg, 's'}, - {"image", no_arg, 'm'}, - {"imag", no_arg, 'm'}, - {"ima", no_arg, 'm'}, - {"im", no_arg, 'm'}, - {"filesize", no_arg, 'z'}, - {"filesiz", no_arg, 'z'}, - {"filesi", no_arg, 'z'}, - {"files", no_arg, 'z'}, - {"file", no_arg, 'z'}, - {"fil", no_arg, 'z'}, - {"fi", no_arg, 'z'}, - {"increment", optional_arg, 'i'}, - {"incremen", optional_arg, 'i'}, - {"increme", optional_arg, 'i'}, - {"increm", optional_arg, 'i'}, - {"incre", optional_arg, 'i'}, - {"incr", optional_arg, 'i'}, - {"inc", optional_arg, 'i'}, - {"in", optional_arg, 'i'}, - {NULL, 0, '\0'}}; +static struct h5_long_options l_opts[] = { + {"help", no_arg, 'h'}, {"version", no_arg, 'V'}, {"status", no_arg, 's'}, + {"image", no_arg, 'm'}, {"filesize", no_arg, 'z'}, {"increment", optional_arg, 'i'}, + {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage diff --git a/tools/test/h5stat/CMakeTests.cmake b/tools/test/h5stat/CMakeTests.cmake index 79de6b9..d4238b5 100644 --- a/tools/test/h5stat/CMakeTests.cmake +++ b/tools/test/h5stat/CMakeTests.cmake @@ -207,7 +207,7 @@ # -d --di=15 ADD_H5_ERR_TEST (h5stat_err1_dims 1 -d --dims=-1 h5stat_threshold.h5) ADD_H5_TEST (h5stat_dims1 0 -gd -m 5 h5stat_threshold.h5) - ADD_H5_TEST (h5stat_dims2 0 -d --di=15 h5stat_threshold.h5) + ADD_H5_TEST (h5stat_dims2 0 -d --dims=15 h5stat_threshold.h5) # # Tests for -a option on h5stat_threshold.h5 # -a -2 (incorrect threshold value) diff --git a/tools/test/h5stat/testh5stat.sh.in b/tools/test/h5stat/testh5stat.sh.in index 470c381..647f06a 100644 --- a/tools/test/h5stat/testh5stat.sh.in +++ b/tools/test/h5stat/testh5stat.sh.in @@ -304,7 +304,7 @@ TOOLTEST h5stat_links5.ddl -g -l 40000 h5stat_newgrat.h5 # -d --di=15 TOOLTEST h5stat_err1_dims.ddl -d --dims=-1 h5stat_threshold.h5 TOOLTEST h5stat_dims1.ddl -gd -m 5 h5stat_threshold.h5 -TOOLTEST h5stat_dims2.ddl -d --di=15 h5stat_threshold.h5 +TOOLTEST h5stat_dims2.ddl -d --dims=15 h5stat_threshold.h5 # # Tests for -a option on h5stat_threshold.h5 # -a -2 (incorrect threshold value) diff --git a/tools/test/misc/CMakeTestsClear.cmake b/tools/test/misc/CMakeTestsClear.cmake index ec24a61..198a363 100644 --- a/tools/test/misc/CMakeTestsClear.cmake +++ b/tools/test/misc/CMakeTestsClear.cmake @@ -397,7 +397,7 @@ # # The following are tests to verify the expected exit code from h5clear: # "h5clear -m h5clear_mdc_image.h5" (valid option, existing file, succeed exit code) -# "h5clear --vers" (valid option, version #, succeed exit code) +# "h5clear --version" (valid option, version #, succeed exit code) # "h5clear -k" (invalid 1 option, no file, fail exit code) # "h5clear -k junk.h5" (invalid 1 option, nonexisting file, fail exit code) # "h5clear -l h5clear_sec2_v2.h5" (invalid 1 option, existing file, fail exit code) @@ -408,7 +408,7 @@ # "h5clear -m -l h5clear_sec2_v0.h5" (valid/invalid 2 options, existing file, fail exit code) # "h5clear -l -m h5clear_sec2_v0.h5" (invalid/valid 2 options, existing file, fail exit code) ADD_H5_RETTEST (h5clr_mdc_image "false" "-m" h5clear_mdc_image.h5) - ADD_H5_RETTEST (h5clr_vers "false" "--vers") + ADD_H5_RETTEST (h5clr_vers "false" "--version") ADD_H5_RETTEST (h5clr_k "true" "-k") ADD_H5_RETTEST (h5clr_k_junk "true" "-k" junk.h5) ADD_H5_RETTEST (h5clr_l_sec2 "true" "-l" h5clear_sec2_v2.h5) diff --git a/tools/test/misc/testh5clear.sh.in b/tools/test/misc/testh5clear.sh.in index c05e43d..a01fa4a 100644 --- a/tools/test/misc/testh5clear.sh.in +++ b/tools/test/misc/testh5clear.sh.in @@ -315,7 +315,7 @@ TOOLTEST_ERR orig_h5clear_sec2_v0.h5 -s -m "" h5clear_no_mdc_image.err # # The following are tests to verify the expected exit code from h5clear: # "h5clear -m h5clear_mdc_image.h5" (valid option, existing file, succeed exit code) -# "h5clear --vers" (valid option, version #, succeed exit code) +# "h5clear --version" (valid option, version #, succeed exit code) # "h5clear -k" (invalid 1 option, no file, fail exit code) # "h5clear -k junk.h5" (invalid 1 option, nonexisting file, fail exit code) # "h5clear -l h5clear_sec2_v2.h5" (invalid 1 option, existing file, fail exit code) @@ -326,7 +326,7 @@ TOOLTEST_ERR orig_h5clear_sec2_v0.h5 -s -m "" h5clear_no_mdc_image.err # "h5clear -m -l h5clear_sec2_v0.h5" (valid/invalid 2 options, existing file, fail exit code) # "h5clear -l -m h5clear_sec2_v0.h5" (invalid/valid 2 options, existing file, fail exit code) TOOLTEST h5clear_mdc_image.h5 -m "" $SUCCEED -TOOLTEST "" --vers "" $SUCCEED +TOOLTEST "" --version "" $SUCCEED TOOLTEST "" -k "" $FAIL TOOLTEST junk.h5 -k "" $FAIL TOOLTEST h5clear_sec2_v2.h5 -l "" $FAIL -- cgit v0.12 From acb186f6e5f0ac3e5bfb9501e3381c1968489a59 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Sat, 7 Aug 2021 08:55:23 -0500 Subject: Add parallel version of h5pfc script (#881) --- fortran/src/CMakeLists.txt | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/fortran/src/CMakeLists.txt b/fortran/src/CMakeLists.txt index e59a829..316d3c7 100644 --- a/fortran/src/CMakeLists.txt +++ b/fortran/src/CMakeLists.txt @@ -582,4 +582,17 @@ if (NOT WIN32 AND NOT MINGW) PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE COMPONENT fortlibraries ) + if (HDF5_ENABLE_PARALLEL AND MPI_Fortran_FOUND) + configure_file ( + ${HDF_RESOURCES_DIR}/libh5cc.in + ${HDF5_BINARY_DIR}/CMakeFiles/h5pfc + @ONLY + ) + install ( + FILES ${HDF5_BINARY_DIR}/CMakeFiles/h5pfc + DESTINATION ${HDF5_INSTALL_BIN_DIR} + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE + COMPONENT fortlibraries + ) + endif () endif () -- cgit v0.12 From 06a09a962acd30527aac7f8922ac7b6c6a571874 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Tue, 10 Aug 2021 08:57:36 -0500 Subject: Reorg tools perform to provide h5perf for installation (#884) * Changes from PR#3 * HDFFV-11213 added option to control gcc10 warnings diagnostics * HDFFV-11212 Use the new references correctly in JNI utility and tests * format source * Fix typo * Add new test file * HDFFV-11212 - update test and remove unused arg * Minor non-space formatting changes * Use H5I_INVALID_ID instead of "-1" * source formatting * add missing testfile, update jni function * Undo commit of debug code * remove mislocated file * Fix h5repack test for handling of fapls and id close * Update h5diff test files usage text * HDFFV-11212 add new ref tests for JNI export dataset * src format update * Remove blank line typo * src format typo * long double requires %Lg * Another long double foramt specifer S.B. %Lg * issue with t128bit test * Windows issue with h5dump and type. * Fix review issues * refactor function nesting and fix error checks * format fixes * Remove untested functions and javadoc quiet comments * Restore TRY block. * Change string append errors to memory exception * revert to H5_JNI_FATAL_ERROR - support functions need work * Add assertion error for h5util functions * remove duplicate function * format fix * Revert HD function error handling * Update copyright comments * GH #386 java folder copyright corrections * Whitespace * GH #359 implement and fix tools 1.6 API usage * remove excessive comments * Flip inits to correct ifdef section * rework ifdef to be simpler * format issue * Reformat ifdef inits * remove static attribute * format compliance * Update names * Revert because logic relies on float not being int * Changes noticed from creating merge of #412 * Double underscore change * Correct compiler version variable used * Remove header guard underscores * Whitespace cleanup * Split format source and commit changes on repo push * remove pre-split setting * Change windows TS to use older VS. * correct window os name * HDFFV-11212 JNI export util and Javadoc * Suggested review changes * Another change found * Committing clang-format changes * HDFFV-11113 long double in tools * HDFFV-11113 add note * Disable long double tests for now * HDFFV-11228 remove arbitrary CMake warning groups. * Make each flag a string * Some Javadoc warning fixes * Updated javadoc fixes * # WARNING: head commit changed in the meantime HDFFV-11229 Fix long double usage in tools and java Changed h5dump and h5ls to just print 128-bit for long double type. Added test and file for dataset and attributes with all three float types. * Committing clang-format changes * HDFFV-11229 Add release note * HDFFV-11229 windows testfile needed * fix typo * Remove non supported message text * HDFFV-11229 - change ldouble test to check both native and general * HDFFV-11229 add second file compare * HDFFV-11229 fix reference file * HDFFV-11229 autotools check two refs * HDFFV-11229 revert back to removal of NATIVE_LDOUBLE in tools output * Committing clang-format changes * Update release note * Update attribute type of ref file * Change source of ninja for macs * try port instead of brew * Recommended is to use brew. * Undo non long double changes * remove unneeded file * Fix perl and doxygen CMake code * Add "option" command for clang options * Rework CMake add_custom to use the BYPRODUCTS argument * Add stamp files to BYPRODUCTS * Only one copy of file exists * Fix custom cmmand depends targets * Fix fortran custom command DEPENDS * Add LD_LIBRARY_PATH to tests * Add custom target for DEPENDS * Add h5detect conditionaly to generated target DEPENDS * Correct DEPENDS targets * Parallel builds need the mpi compiler for pkgconfig scripts. * install only if MPI build * Fortran target depends * Remove incorrect source attribute * doxygen adjustments * doxygen build updates * Correct version * Correct function version - function has been merged to 1.12 * Correct version string for map functions * Cleanup warnings for tools debug builds * TRILAB-227 - fix indexing for h5diff selections * Correct location of pos to index function call * TRILAB-227 Enable test * Quote subset args * Use MATCHES because of AppleClang * if blocks needed for build modes * Update list of DT platforms * VS2019 correctly displays float values * revert VS2019 change * Issue #669 remove version from pkgcfg filename * remove version from h5cc script * Java reference functions updated enabled fortran in cmake gcc action yaml file java reference test changed to correctly test refs jni reference functions that create ids changed to use jni id logging * Correct BYPRODUCTS argument * Correct more genereated files BYPRODUCTS * BYPRODUCTS must have unique locations * Fix typo * Fix fortran configure checks * Rework H5_PAC_C_MAX_REAL_PRECISION setting logic * Add note about fortran configure change * Adds a quick for for some egregious chunk_info badness (#722) * Fixes issue with ccmake that prevents building Fortran (#723) ccmake runs iteratively, and the check_fortran_source_runs macros were clobbering a single output file that did not get updated on further configure iterations * Fix conflicts with merge * Move MAX_PRECISION back to HDF5UseFortran.cmake * Use STREQUAL to test macro argument * Move C language test to ConfigureChecks from HDF5UseFortran * MAX_PRECISION defines must be defined * Organize flags and align autotools and cmake * Fix comment in no-error-general warnings files. * Flag cleanup and fix typos * Add comment * Correct VAR used to find configure time file * Set the path correctly * Update missing release note info. * Update code owners * Correct JIRA note * add known problem. * Use only core library for testing dynamic plugins. * Reorg tools perform to provide h5perf for installation * Correct file paths * Correct path * Add new src folder to makefile list * Remove bin_PROGRAMS from TEST_PROG * Default h5perf executables to static build * Remove test lib dependency from h5perf * format adjustments * Remove test lib from autotools makefile * Add note * h5perf needs the math library Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Dana Robinson <43805+derobins@users.noreply.github.com> --- MANIFEST | 17 +- configure.ac | 1 + release_docs/RELEASE.txt | 17 +- tools/src/CMakeLists.txt | 5 +- tools/src/Makefile.am | 2 +- tools/src/h5perf/CMakeLists.txt | 103 ++ tools/src/h5perf/Makefile.am | 63 + tools/src/h5perf/perf.c | 796 ++++++++++ tools/src/h5perf/pio_engine.c | 2745 +++++++++++++++++++++++++++++++++++ tools/src/h5perf/pio_perf.c | 1834 +++++++++++++++++++++++ tools/src/h5perf/pio_perf.h | 109 ++ tools/src/h5perf/sio_engine.c | 1328 +++++++++++++++++ tools/src/h5perf/sio_perf.c | 1437 ++++++++++++++++++ tools/src/h5perf/sio_perf.h | 104 ++ tools/test/perform/CMakeLists.txt | 81 +- tools/test/perform/Makefile.am | 23 +- tools/test/perform/perf.c | 471 ------ tools/test/perform/pio_engine.c | 2745 ----------------------------------- tools/test/perform/pio_perf.c | 1699 ---------------------- tools/test/perform/pio_perf.h | 100 -- tools/test/perform/pio_standalone.c | 135 -- tools/test/perform/pio_standalone.h | 11 - tools/test/perform/sio_engine.c | 1328 ----------------- tools/test/perform/sio_perf.c | 1437 ------------------ tools/test/perform/sio_perf.h | 104 -- tools/test/perform/sio_standalone.h | 11 - 26 files changed, 8558 insertions(+), 8148 deletions(-) create mode 100644 tools/src/h5perf/CMakeLists.txt create mode 100644 tools/src/h5perf/Makefile.am create mode 100644 tools/src/h5perf/perf.c create mode 100644 tools/src/h5perf/pio_engine.c create mode 100644 tools/src/h5perf/pio_perf.c create mode 100644 tools/src/h5perf/pio_perf.h create mode 100644 tools/src/h5perf/sio_engine.c create mode 100644 tools/src/h5perf/sio_perf.c create mode 100644 tools/src/h5perf/sio_perf.h delete mode 100644 tools/test/perform/perf.c delete mode 100644 tools/test/perform/pio_engine.c delete mode 100644 tools/test/perform/pio_perf.c delete mode 100644 tools/test/perform/pio_perf.h delete mode 100644 tools/test/perform/sio_engine.c delete mode 100644 tools/test/perform/sio_perf.c delete mode 100644 tools/test/perform/sio_perf.h diff --git a/MANIFEST b/MANIFEST index 0615fd0..668ba0a 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2947,6 +2947,15 @@ ./tools/testfiles/h5mkgrp_single_p.ls ./tools/testfiles/h5mkgrp_single_l.ls +./tools/src/h5perf/Makefile.am +./tools/src/h5perf/perf.c +./tools/src/h5perf/pio_engine.c +./tools/src/h5perf/pio_perf.c +./tools/src/h5perf/pio_perf.h +./tools/src/h5perf/sio_engine.c +./tools/src/h5perf/sio_perf.c +./tools/src/h5perf/sio_perf.h + ./tools/test/perform/Makefile.am ./tools/test/perform/build_h5perf_alone.sh ./tools/test/perform/build_h5perf_serial_alone.sh @@ -2956,16 +2965,9 @@ ./tools/test/perform/gen_report.pl ./tools/test/perform/iopipe.c ./tools/test/perform/overhead.c -./tools/test/perform/perf.c ./tools/test/perform/perf_meta.c -./tools/test/perform/pio_engine.c -./tools/test/perform/pio_perf.c -./tools/test/perform/pio_perf.h ./tools/test/perform/pio_standalone.c ./tools/test/perform/pio_standalone.h -./tools/test/perform/sio_engine.c -./tools/test/perform/sio_perf.c -./tools/test/perform/sio_perf.h ./tools/test/perform/sio_standalone.c ./tools/test/perform/sio_standalone.h ./tools/test/perform/zip_perf.c @@ -3708,6 +3710,7 @@ ./tools/test/h5stat/CMakeLists.txt ./tools/test/h5stat/CMakeTests.cmake ./tools/src/misc/CMakeLists.txt +./tools/src/h5perf/CMakeLists.txt ./tools/test/misc/CMakeLists.txt ./tools/test/misc/CMakeTestsClear.cmake ./tools/test/misc/CMakeTestsMkgrp.cmake diff --git a/configure.ac b/configure.ac index 55ea12c..cbb192d 100644 --- a/configure.ac +++ b/configure.ac @@ -4019,6 +4019,7 @@ AC_CONFIG_FILES([src/libhdf5.settings tools/test/misc/vds/Makefile tools/test/h5stat/Makefile tools/test/h5stat/testh5stat.sh + tools/src/h5perf/Makefile tools/test/perform/Makefile examples/Makefile examples/run-c-ex.sh diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index e12f652..40b9175 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -820,6 +820,17 @@ New Features Tools: ------ + - Refactored the perform tools and removed depends on test library. + + Moved the perf and h5perf tools from tools/test/perform to + tools/src/h5perf so that they can be installed. This required + that the test library dependency be removed by copying the + needed functions from h5test.c. + The standalone scripts and other perform tools remain in the + tools/test/perform folder. + + (ADB - 2021/08/10) + - Removed partial long exceptions Some of the tools accepted shortened versions of the long options @@ -1089,10 +1100,10 @@ Bug Fixes since HDF5-1.12.0 release Configuration ------------- - Better support for libaec (open-source Szip library) in CMake - - Implemented better support for libaec 1.0.5 (or later) library. This version + + Implemented better support for libaec 1.0.5 (or later) library. This version of libaec contains improvements for better integration with HDF5. Furthermore, - the variable USE_LIBAEC_STATIC has been introduced to allow to make use of + the variable USE_LIBAEC_STATIC has been introduced to allow to make use of static version of libaec library. Use libaec_DIR or libaec_ROOT to set the location in which libaec can be found. diff --git a/tools/src/CMakeLists.txt b/tools/src/CMakeLists.txt index 8c3e361..e291f61 100644 --- a/tools/src/CMakeLists.txt +++ b/tools/src/CMakeLists.txt @@ -13,7 +13,7 @@ add_subdirectory (misc) #-- Add the h5import and test executables add_subdirectory (h5import) -#-- h5Repack executables +#-- h5repack executables add_subdirectory (h5repack) #-- Add the h5dump and test executables @@ -30,3 +30,6 @@ add_subdirectory (h5dump) #-- Add the h5format_convert and test executables add_subdirectory (h5format_convert) + +#-- h5perf executables +add_subdirectory (h5perf) diff --git a/tools/src/Makefile.am b/tools/src/Makefile.am index 397bd31..5af7d06 100644 --- a/tools/src/Makefile.am +++ b/tools/src/Makefile.am @@ -23,6 +23,6 @@ CONFIG=ordered # All subdirectories SUBDIRS=h5diff h5ls h5dump misc h5import h5repack h5jam h5copy \ - h5format_convert h5stat + h5format_convert h5stat h5perf include $(top_srcdir)/config/conclude.am diff --git a/tools/src/h5perf/CMakeLists.txt b/tools/src/h5perf/CMakeLists.txt new file mode 100644 index 0000000..644a5ad --- /dev/null +++ b/tools/src/h5perf/CMakeLists.txt @@ -0,0 +1,103 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TOOLS_SRC_H5PERF C) + +# -------------------------------------------------------------------- +# Add the executables +# -------------------------------------------------------------------- +#-- Adding test for h5perf_serial +set (h5perf_serial_SOURCES + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/sio_perf.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/sio_engine.c +) +add_executable (h5perf_serial ${h5perf_serial_SOURCES}) +target_include_directories (h5perf_serial PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") +if (NOT ONLY_SHARED_LIBS) + TARGET_C_PROPERTIES (h5perf_serial STATIC) + target_link_libraries (h5perf_serial PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (h5perf_serial SHARED) + target_link_libraries (h5perf_serial PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) +endif () +set_target_properties (h5perf_serial PROPERTIES FOLDER perform) +set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5perf_serial") + +set (H5_DEP_EXECUTABLES h5perf_serial) + +#----------------------------------------------------------------------------- +# Add Target to clang-format +#----------------------------------------------------------------------------- +if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TOOLS_SRC_H5PERF_h5perf_serial_FORMAT h5perf_serial) +endif () + +if (H5_HAVE_PARALLEL) + if (UNIX) + #-- Adding test for perf - only on unix systems + set (perf_SOURCES + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/perf.c + ) + add_executable (perf ${perf_SOURCES}) + target_include_directories (perf PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + if (NOT ONLY_SHARED_LIBS) + TARGET_C_PROPERTIES (perf STATIC) + target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) + else () + TARGET_C_PROPERTIES (perf SHARED) + target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) + endif () + set_target_properties (perf PROPERTIES FOLDER perform) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};perf") + + set (H5_DEP_EXECUTABLES perf) + + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TOOLS_SRC_H5PERF_perf_FORMAT perf) + endif () + endif () + + #-- Adding test for h5perf + set (h5perf_SOURCES + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/pio_perf.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/pio_engine.c + ) + add_executable (h5perf ${h5perf_SOURCES}) + target_include_directories (h5perf PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + if (NOT ONLY_SHARED_LIBS) + TARGET_C_PROPERTIES (h5perf STATIC) + target_link_libraries (h5perf PRIVATE ${LINK_LIBS} ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) + else () + TARGET_C_PROPERTIES (h5perf SHARED) + target_link_libraries (h5perf PRIVATE ${LINK_LIBS} ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) + endif () + set_target_properties (h5perf PROPERTIES FOLDER perform) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5perf") + + set (H5_DEP_EXECUTABLES h5perf) + + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TOOLS_SRC_H5PERF_h5perf_FORMAT h5perf) + endif () +endif () + +#----------------------------------------------------------------------------- +# Rules for Installation of tools using make Install target +#----------------------------------------------------------------------------- +if (HDF5_EXPORTED_TARGETS) + foreach (exec ${H5_DEP_EXECUTABLES}) + INSTALL_PROGRAM_PDB (${exec} ${HDF5_INSTALL_BIN_DIR} toolsapplications) + endforeach () + + install ( + TARGETS + ${H5_DEP_EXECUTABLES} + EXPORT + ${HDF5_EXPORTED_TARGETS} + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications + ) +endif () diff --git a/tools/src/h5perf/Makefile.am b/tools/src/h5perf/Makefile.am new file mode 100644 index 0000000..e8a9fdd --- /dev/null +++ b/tools/src/h5perf/Makefile.am @@ -0,0 +1,63 @@ +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# HDF5 Library Performance Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib + +# bin_PROGRAMS will be installed. +if BUILD_PARALLEL_CONDITIONAL + bin_PROGRAMS=h5perf_serial perf h5perf +else + bin_PROGRAMS=h5perf_serial +endif + +# Add h5perf and h5perf_serial specific linker flags here +h5perf_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) +h5perf_serial_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) + +# Some programs are not built or run by default, but can be built by hand or by +# specifying --enable-build-all at configure time. +# Also, some of these programs should only be built in parallel. +# Currently there is no such program. +if BUILD_PARALLEL_CONDITIONAL + PARA_BUILD_ALL= +endif +if BUILD_ALL_CONDITIONAL + BUILD_ALL_PROGS=$(PARA_BUILD_ALL) +endif + +# Define programs that will be run in 'make check' +# List them in the order they should be run. +# Parallel test programs. +if BUILD_PARALLEL_CONDITIONAL + TEST_PROG_PARA=h5perf perf +endif + +h5perf_SOURCES=pio_perf.c pio_engine.c +h5perf_serial_SOURCES=sio_perf.c sio_engine.c + +# All of the programs depend on the main hdf5 library, and some of them +# depend on test or tools library. +LDADD=$(LIBHDF5) +h5perf_LDADD=$(LIBH5TOOLS) $(LIBHDF5) +h5perf_serial_LDADD=$(LIBH5TOOLS) $(LIBHDF5) +perf_LDADD=$(LIBHDF5) + +include $(top_srcdir)/config/conclude.am diff --git a/tools/src/h5perf/perf.c b/tools/src/h5perf/perf.c new file mode 100644 index 0000000..83d4ab0 --- /dev/null +++ b/tools/src/h5perf/perf.c @@ -0,0 +1,796 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Author: Albert Cheng of NCSA, May 1, 2001. + * This is derived from code given to me by Robert Ross. + * + * NOTE: This code assumes that all command line arguments make it out to all + * the processes that make up the parallel job, which isn't always the case. + * So if it doesn't work on some platform, that might be why. + */ + +#include "hdf5.h" +#include "H5private.h" + +#ifdef H5_HAVE_PARALLEL + +#include +#include +#include +#include +#include +#include + +#ifdef H5_HAVE_SYS_STAT_H +#include +#endif + +#ifdef H5_HAVE_SYS_TIME_H +#include +#endif + +#ifdef H5_HAVE_SYS_TYPES_H +#include +#endif + +#ifdef H5_HAVE_UNISTD_H +#include +#endif + +#include +#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ +#include +#endif + +/* Macro definitions */ +/* Verify: + * if val is false (0), print mesg and if fatal is true (non-zero), die. + */ +#define H5FATAL 1 +#define VRFY(val, mesg, fatal) \ + do { \ + if (!val) { \ + printf("Proc %d: ", mynod); \ + printf("*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ + if (fatal) { \ + fflush(stdout); \ + goto die_jar_jar_die; \ + } \ + } \ + } while (0) +#define RANK 1 +#define MAX_PATH 1024 + +hsize_t dims[RANK]; /* dataset dim sizes */ +hsize_t block[RANK], stride[RANK], count[RANK]; +hsize_t start[RANK]; +hid_t fid; /* HDF5 file ID */ +hid_t acc_tpl; /* File access templates */ +hid_t sid; /* Dataspace ID */ +hid_t file_dataspace; /* File dataspace ID */ +hid_t mem_dataspace; /* memory dataspace ID */ +hid_t dataset; /* Dataset ID */ +hsize_t opt_alignment = 1; +hsize_t opt_threshold = 1; +int opt_split_vfd = 0; +char * meta_ext, *raw_ext; /* holds the meta and raw file extension if */ + /* opt_split_vfd is set */ + +/* DEFAULT VALUES FOR OPTIONS */ +int64_t opt_block = 1048576 * 16; +int opt_iter = 1; +int opt_stripe = -1; +int opt_correct = 0; +int amode = O_RDWR | O_CREAT; +char opt_file[256] = "perftest.out"; +char opt_pvfstab[256] = "notset"; +int opt_pvfstab_set = 0; + +const char *FILENAME[] = {opt_file, NULL}; + +/* function prototypes */ +static int parse_args(int argc, char **argv); + +#ifndef H5_HAVE_UNISTD_H +/* globals needed for getopt */ +extern char *optarg; +#endif + +#ifndef HDF5_PARAPREFIX +#define HDF5_PARAPREFIX "" +#endif +char * paraprefix = NULL; /* for command line option para-prefix */ +MPI_Info h5_io_info_g = MPI_INFO_NULL; /* MPI INFO object for IO */ + +static char *h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fullname, + size_t size, hbool_t nest_printf, hbool_t subst_for_superblock); + +int +main(int argc, char **argv) +{ + char * buf, *tmp, *buf2 = NULL, *tmp2 = NULL, *check; + int i, j, mynod = 0, nprocs = 1, my_correct = 1, correct, myerrno; + double stim, etim; + double write_tim = 0; + double read_tim = 0; + double read_bw, write_bw; + double max_read_tim, max_write_tim; + double min_read_tim, min_write_tim; + double ave_read_tim, ave_write_tim; + int64_t iter_jump = 0; + char filename[MAX_PATH]; + herr_t ret; /* Generic return value */ + + /* startup MPI and determine the rank of this process */ + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &mynod); + + /* parse the command line arguments */ + parse_args(argc, argv); + + if (mynod == 0) + printf("# Using hdf5-io calls.\n"); + +#ifdef H5_HAVE_UNISTD_H + /* Kind of a weird hack- if the location of the pvfstab file was + * specified on the command line, then spit out this location into + * the appropriate environment variable. + */ + if (opt_pvfstab_set) { + if ((setenv("PVFSTAB_FILE", opt_pvfstab, 1)) < 0) { + perror("setenv"); + goto die_jar_jar_die; + } + } +#endif + + /* this is how much of the file data is covered on each iteration of + * the test. used to help determine the seek offset on each + * iteration */ + iter_jump = nprocs * opt_block; + + /* setup a buffer of data to write */ + if (!(tmp = (char *)malloc((size_t)opt_block + 256))) { + perror("malloc"); + goto die_jar_jar_die; + } + buf = tmp + 128 - (((long)tmp) % 128); /* align buffer */ + + if (opt_correct) { + /* do the same buffer setup for verifiable data */ + if (!(tmp2 = (char *)malloc((size_t)opt_block + 256))) { + perror("malloc2"); + goto die_jar_jar_die; + } + buf2 = tmp + 128 - (((long)tmp) % 128); + } + + /* setup file access template with parallel IO access. */ + if (opt_split_vfd) { + hid_t mpio_pl; + + mpio_pl = H5Pcreate(H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "", H5FATAL); + ret = H5Pset_fapl_mpio(mpio_pl, MPI_COMM_WORLD, MPI_INFO_NULL); + VRFY((ret >= 0), "", H5FATAL); + + /* set optional allocation alignment */ + if (opt_alignment * opt_threshold != 1) { + ret = H5Pset_alignment(acc_tpl, opt_threshold, opt_alignment); + VRFY((ret >= 0), "H5Pset_alignment succeeded", !H5FATAL); + } + + /* setup file access template */ + acc_tpl = H5Pcreate(H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "", H5FATAL); + ret = H5Pset_fapl_split(acc_tpl, meta_ext, mpio_pl, raw_ext, mpio_pl); + VRFY((ret >= 0), "H5Pset_fapl_split succeeded", H5FATAL); + ret = H5Pclose(mpio_pl); + VRFY((ret >= 0), "H5Pclose mpio_pl succeeded", H5FATAL); + } + else { + /* setup file access template */ + acc_tpl = H5Pcreate(H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "", H5FATAL); + ret = H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL); + VRFY((ret >= 0), "", H5FATAL); + + /* set optional allocation alignment */ + if (opt_alignment * opt_threshold != 1) { + ret = H5Pset_alignment(acc_tpl, opt_threshold, opt_alignment); + VRFY((ret >= 0), "H5Pset_alignment succeeded", !H5FATAL); + } + } + + h5_fixname_real(FILENAME[0], acc_tpl, NULL, filename, sizeof filename, FALSE, FALSE); + + /* create the parallel file */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); + VRFY((fid >= 0), "H5Fcreate succeeded", H5FATAL); + + /* define a contiquous dataset of opt_iter*nprocs*opt_block chars */ + dims[0] = (hsize_t)opt_iter * (hsize_t)nprocs * (hsize_t)opt_block; + sid = H5Screate_simple(RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded", H5FATAL); + dataset = H5Dcreate2(fid, "Dataset1", H5T_NATIVE_CHAR, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded", H5FATAL); + + /* create the memory dataspace and the file dataspace */ + dims[0] = (hsize_t)opt_block; + mem_dataspace = H5Screate_simple(RANK, dims, NULL); + VRFY((mem_dataspace >= 0), "", H5FATAL); + file_dataspace = H5Dget_space(dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded", H5FATAL); + + /* now each process writes a block of opt_block chars in round robbin + * fashion until the whole dataset is covered. + */ + for (j = 0; j < opt_iter; j++) { + /* setup a file dataspace selection */ + start[0] = (hsize_t)((j * iter_jump) + (mynod * opt_block)); + stride[0] = block[0] = (hsize_t)opt_block; + count[0] = 1; + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded", H5FATAL); + + if (opt_correct) /* fill in buffer for iteration */ { + for (i = mynod + j, check = buf; i < opt_block; i++, check++) + *check = (char)i; + } + + /* discover the starting time of the operation */ + MPI_Barrier(MPI_COMM_WORLD); + stim = MPI_Wtime(); + + /* write data */ + ret = H5Dwrite(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded", !H5FATAL); + + /* discover the ending time of the operation */ + etim = MPI_Wtime(); + + write_tim += (etim - stim); + + /* we are done with this "write" iteration */ + } + + /* close dataset and file */ + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose succeeded", H5FATAL); + ret = H5Fclose(fid); + VRFY((ret >= 0), "H5Fclose succeeded", H5FATAL); + + /* wait for everyone to synchronize at this point */ + MPI_Barrier(MPI_COMM_WORLD); + + /* reopen the file for reading */ + fid = H5Fopen(filename, H5F_ACC_RDONLY, acc_tpl); + VRFY((fid >= 0), "", H5FATAL); + + /* open the dataset */ + dataset = H5Dopen2(fid, "Dataset1", H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen succeeded", H5FATAL); + + /* we can re-use the same mem_dataspace and file_dataspace + * the H5Dwrite used since the dimension size is the same. + */ + + /* we are going to repeat the read the same pattern the write used */ + for (j = 0; j < opt_iter; j++) { + /* setup a file dataspace selection */ + start[0] = (hsize_t)((j * iter_jump) + (mynod * opt_block)); + stride[0] = block[0] = (hsize_t)opt_block; + count[0] = 1; + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded", H5FATAL); + /* seek to the appropriate spot give the current iteration and + * rank within the MPI processes */ + + /* discover the start time */ + MPI_Barrier(MPI_COMM_WORLD); + stim = MPI_Wtime(); + + /* read in the file data */ + if (!opt_correct) { + ret = H5Dread(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf); + } + else { + ret = H5Dread(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf2); + } + myerrno = errno; + + /* discover the end time */ + etim = MPI_Wtime(); + read_tim += (etim - stim); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded", !H5FATAL); + + if (ret < 0) + HDfprintf(stderr, "node %d, read error, loc = %" PRId64 ": %s\n", mynod, mynod * opt_block, + strerror(myerrno)); + + /* if the user wanted to check correctness, compare the write + * buffer to the read buffer */ + if (opt_correct && memcmp(buf, buf2, (size_t)opt_block)) { + HDfprintf(stderr, "node %d, correctness test failed\n", mynod); + my_correct = 0; + MPI_Allreduce(&my_correct, &correct, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + } + + /* we are done with this read iteration */ + } + + /* close dataset and file */ + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose succeeded", H5FATAL); + ret = H5Fclose(fid); + VRFY((ret >= 0), "H5Fclose succeeded", H5FATAL); + ret = H5Pclose(acc_tpl); + VRFY((ret >= 0), "H5Pclose succeeded", H5FATAL); + + /* compute the read and write times */ + MPI_Allreduce(&read_tim, &max_read_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + MPI_Allreduce(&read_tim, &min_read_tim, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(&read_tim, &ave_read_tim, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + /* calculate the average from the sum */ + ave_read_tim = ave_read_tim / nprocs; + + MPI_Allreduce(&write_tim, &max_write_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + MPI_Allreduce(&write_tim, &min_write_tim, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(&write_tim, &ave_write_tim, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + /* calculate the average from the sum */ + ave_write_tim = ave_write_tim / nprocs; + + /* print out the results on one node */ + if (mynod == 0) { + read_bw = (double)((int64_t)(opt_block * nprocs * opt_iter)) / (max_read_tim * 1000000.0); + write_bw = (double)((int64_t)(opt_block * nprocs * opt_iter)) / (max_write_tim * 1000000.0); + + printf("nr_procs = %d, nr_iter = %d, blk_sz = %ld\n", nprocs, opt_iter, (long)opt_block); + + printf("# total_size = %ld\n", (long)(opt_block * nprocs * opt_iter)); + + printf("# Write: min_time = %f, max_time = %f, mean_time = %f\n", min_write_tim, max_write_tim, + ave_write_tim); + printf("# Read: min_time = %f, max_time = %f, mean_time = %f\n", min_read_tim, max_read_tim, + ave_read_tim); + + printf("Write bandwidth = %f Mbytes/sec\n", write_bw); + printf("Read bandwidth = %f Mbytes/sec\n", read_bw); + + if (opt_correct) { + printf("Correctness test %s.\n", correct ? "passed" : "failed"); + } + } + +die_jar_jar_die: + +#ifdef H5_HAVE_UNISTD + /* Clear the environment variable if it was set earlier */ + if (opt_pvfstab_set) { + unsetenv("PVFSTAB_FILE"); + } +#endif + + free(tmp); + if (opt_correct) + free(tmp2); + + MPI_Finalize(); + + return (0); +} + +static int +parse_args(int argc, char **argv) +{ + int c; + + while ((c = getopt(argc, argv, "s:b:i:f:p:a:2:c")) != EOF) { + switch (c) { + case 's': /* stripe */ + opt_stripe = atoi(optarg); + break; + case 'b': /* block size */ + opt_block = atoi(optarg); + break; + case 'i': /* iterations */ + opt_iter = atoi(optarg); + break; + case 'f': /* filename */ + strncpy(opt_file, optarg, 255); + FILENAME[0] = opt_file; + break; + case 'p': /* pvfstab file */ + strncpy(opt_pvfstab, optarg, 255); + opt_pvfstab_set = 1; + break; + case 'a': /* aligned allocation. + * syntax: -a/ + * e.g., -a4096/512 allocate at 4096 bytes + * boundary if request size >= 512. + */ + { + char *p; + + opt_alignment = (hsize_t)HDatoi(optarg); + if (NULL != (p = (char *)HDstrchr(optarg, '/'))) + opt_threshold = (hsize_t)HDatoi(p + 1); + } + HDfprintf(stdout, "alignment/threshold=%" PRIuHSIZE "/%" PRIuHSIZE "\n", opt_alignment, + opt_threshold); + break; + case '2': /* use 2-files, i.e., split file driver */ + opt_split_vfd = 1; + /* get meta and raw file extension. */ + /* syntax is , */ + meta_ext = raw_ext = optarg; + while (*raw_ext != '\0') { + if (*raw_ext == ',') { + *raw_ext = '\0'; + raw_ext++; + break; + } + raw_ext++; + } + printf("split-file-vfd used: %s,%s\n", meta_ext, raw_ext); + break; + case 'c': /* correctness */ + opt_correct = 1; + break; + case '?': /* unknown */ + default: + break; + } + } + + return (0); +} +/*------------------------------------------------------------------------- + * Function: getenv_all + * + * Purpose: Used to get the environment that the root MPI task has. + * name specifies which environment variable to look for + * val is the string to which the value of that environment + * variable will be copied. + * + * NOTE: The pointer returned by this function is only + * valid until the next call to getenv_all and the data + * stored there must be copied somewhere else before any + * further calls to getenv_all take place. + * + * Return: pointer to a string containing the value of the environment variable + * NULL if the varialbe doesn't exist in task 'root's environment. + * + * Programmer: Leon Arber + * 4/4/05 + * + * Modifications: + * Use original getenv if MPI is not initialized. This happens + * one uses the PHDF5 library to build a serial nature code. + * Albert 2006/04/07 + * + *------------------------------------------------------------------------- + */ +char * +getenv_all(MPI_Comm comm, int root, const char *name) +{ + int mpi_size, mpi_rank, mpi_initialized, mpi_finalized; + int len; + static char *env = NULL; + + HDassert(name); + + MPI_Initialized(&mpi_initialized); + MPI_Finalized(&mpi_finalized); + + if (mpi_initialized && !mpi_finalized) { + MPI_Comm_rank(comm, &mpi_rank); + MPI_Comm_size(comm, &mpi_size); + HDassert(root < mpi_size); + + /* The root task does the getenv call + * and sends the result to the other tasks */ + if (mpi_rank == root) { + env = HDgetenv(name); + if (env) { + len = (int)HDstrlen(env); + MPI_Bcast(&len, 1, MPI_INT, root, comm); + MPI_Bcast(env, len, MPI_CHAR, root, comm); + } + else { + /* len -1 indicates that the variable was not in the environment */ + len = -1; + MPI_Bcast(&len, 1, MPI_INT, root, comm); + } + } + else { + MPI_Bcast(&len, 1, MPI_INT, root, comm); + if (len >= 0) { + if (env == NULL) + env = (char *)HDmalloc((size_t)len + 1); + else if (HDstrlen(env) < (size_t)len) + env = (char *)HDrealloc(env, (size_t)len + 1); + + MPI_Bcast(env, len, MPI_CHAR, root, comm); + env[len] = '\0'; + } + else { + if (env) + HDfree(env); + env = NULL; + } + } +#ifndef NDEBUG + MPI_Barrier(comm); +#endif + } + else { + /* use original getenv */ + if (env) + HDfree(env); + env = HDgetenv(name); + } /* end if */ + + return env; +} + +/*------------------------------------------------------------------------- + * Function: h5_fixname_real + * + * Purpose: Create a file name from a file base name like `test' and + * return it through the FULLNAME (at most SIZE characters + * counting the null terminator). The full name is created by + * prepending the contents of HDF5_PREFIX (separated from the + * base name by a slash) and appending a file extension based on + * the driver supplied, resulting in something like + * `ufs:/u/matzke/test.h5'. + * + * Return: Success: The FULLNAME pointer. + * + * Failure: NULL if BASENAME or FULLNAME is the null + * pointer or if FULLNAME isn't large enough for + * the result. + * + * Programmer: Robb Matzke + * Thursday, November 19, 1998 + * + *------------------------------------------------------------------------- + */ +static char * +h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fullname, size_t size, + hbool_t nest_printf, hbool_t subst_for_superblock) +{ + const char *prefix = NULL; + const char *env = NULL; /* HDF5_DRIVER environment variable */ + char * ptr, last = '\0'; + const char *suffix = _suffix; + size_t i, j; + hid_t driver = -1; + int isppdriver = 0; /* if the driver is MPI parallel */ + + if (!base_name || !fullname || size < 1) + return NULL; + + HDmemset(fullname, 0, size); + + /* figure out the suffix */ + if (H5P_DEFAULT != fapl) { + if ((driver = H5Pget_driver(fapl)) < 0) + return NULL; + + if (suffix) { + if (H5FD_FAMILY == driver) { + if (subst_for_superblock) + suffix = "00000.h5"; + else + suffix = nest_printf ? "%%05d.h5" : "%05d.h5"; + } + else if (H5FD_MULTI == driver) { + + /* Get the environment variable, if it exists, in case + * we are using the split driver since both of those + * use the multi VFD under the hood. + */ + env = HDgetenv("HDF5_DRIVER"); +#ifdef HDF5_DRIVER + /* Use the environment variable, then the compile-time constant */ + if (!env) + env = HDF5_DRIVER; +#endif + if (env && !HDstrcmp(env, "split")) { + /* split VFD */ + if (subst_for_superblock) + suffix = "-m.h5"; + else + suffix = NULL; + } + else { + /* multi VFD */ + if (subst_for_superblock) + suffix = "-s.h5"; + else + suffix = NULL; + } + } + } + } + + /* Must first check fapl is not H5P_DEFAULT (-1) because H5FD_XXX + * could be of value -1 if it is not defined. + */ + isppdriver = H5P_DEFAULT != fapl && (H5FD_MPIO == driver); + + /* Check what prefix to use for test files. Process HDF5_PARAPREFIX and + * HDF5_PREFIX. + * Use different ones depending on parallel or serial driver used. + * (The #ifdef is needed to prevent compile failure in case MPI is not + * configured.) + */ + if (isppdriver) { + /* + * For parallel: + * First use command line option, then the environment + * variable, then try the constant + */ + static int explained = 0; + + prefix = (paraprefix ? paraprefix : getenv_all(MPI_COMM_WORLD, 0, "HDF5_PARAPREFIX")); + + if (!prefix && !explained) { + /* print hint by process 0 once. */ + int mpi_rank; + + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + if (mpi_rank == 0) + HDprintf("*** Hint ***\n" + "You can use environment variable HDF5_PARAPREFIX to " + "run parallel test files in a\n" + "different directory or to add file type prefix. e.g.,\n" + " HDF5_PARAPREFIX=pfs:/PFS/user/me\n" + " export HDF5_PARAPREFIX\n" + "*** End of Hint ***\n"); + + explained = TRUE; +#ifdef HDF5_PARAPREFIX + prefix = HDF5_PARAPREFIX; +#endif /* HDF5_PARAPREFIX */ + } + } + else { + /* + * For serial: + * First use the environment variable, then try the constant + */ + prefix = HDgetenv("HDF5_PREFIX"); + +#ifdef HDF5_PREFIX + if (!prefix) + prefix = HDF5_PREFIX; +#endif /* HDF5_PREFIX */ + } + + /* Prepend the prefix value to the base name */ + if (prefix && *prefix) { + if (isppdriver) { + /* This is a parallel system */ + char *subdir; + + if (!HDstrcmp(prefix, HDF5_PARAPREFIX)) { + /* + * If the prefix specifies the HDF5_PARAPREFIX directory, then + * default to using the "/tmp/$USER" or "/tmp/$LOGIN" + * directory instead. + */ + char *user, *login; + + user = HDgetenv("USER"); + login = HDgetenv("LOGIN"); + subdir = (user ? user : login); + + if (subdir) { + for (i = 0; i < size && prefix[i]; i++) + fullname[i] = prefix[i]; + + fullname[i++] = '/'; + + for (j = 0; i < size && subdir[j]; ++i, ++j) + fullname[i] = subdir[j]; + } + } + + if (!fullname[0]) { + /* We didn't append the prefix yet */ + HDstrncpy(fullname, prefix, size); + fullname[size - 1] = '\0'; + } + + if (HDstrlen(fullname) + HDstrlen(base_name) + 1 < size) { + /* + * Append the base_name with a slash first. Multiple + * slashes are handled below. + */ + h5_stat_t buf; + + if (HDstat(fullname, &buf) < 0) + /* The directory doesn't exist just yet */ + if (HDmkdir(fullname, (mode_t)0755) < 0 && errno != EEXIST) + /* + * We couldn't make the "/tmp/${USER,LOGIN}" + * subdirectory. Default to PREFIX's original + * prefix value. + */ + HDstrcpy(fullname, prefix); + + HDstrcat(fullname, "/"); + HDstrcat(fullname, base_name); + } + else { + /* Buffer is too small */ + return NULL; + } + } + else { + if (HDsnprintf(fullname, size, "%s/%s", prefix, base_name) == (int)size) + /* Buffer is too small */ + return NULL; + } + } + else if (HDstrlen(base_name) >= size) { + /* Buffer is too small */ + return NULL; + } + else { + HDstrcpy(fullname, base_name); + } + + /* Append a suffix */ + if (suffix) { + if (HDstrlen(fullname) + HDstrlen(suffix) >= size) + return NULL; + + HDstrcat(fullname, suffix); + } + + /* Remove any double slashes in the filename */ + for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) { + if (*ptr != '/' || last != '/') + fullname[j++] = *ptr; + + last = *ptr; + } + + return fullname; +} + +/* + * Local variables: + * c-indent-level: 3 + * c-basic-offset: 3 + * tab-width: 3 + * End: + */ + +#else /* H5_HAVE_PARALLEL */ +/* dummy program since H5_HAVE_PARALLEL is not configured in */ +int +main(int H5_ATTR_UNUSED argc, char H5_ATTR_UNUSED **argv) +{ + printf("No parallel performance because parallel is not configured in\n"); + return (0); +} +#endif /* H5_HAVE_PARALLEL */ diff --git a/tools/src/h5perf/pio_engine.c b/tools/src/h5perf/pio_engine.c new file mode 100644 index 0000000..cac36d7 --- /dev/null +++ b/tools/src/h5perf/pio_engine.c @@ -0,0 +1,2745 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Author: Albert Cheng of NCSA, Oct 24, 2001. + */ + +#include "hdf5.h" + +#include +#include +#include +#include + +#ifdef H5_HAVE_UNISTD_H +#include +#include +#endif + +#ifdef H5_HAVE_SYS_STAT_H +#include +#endif + +#ifdef H5_HAVE_PARALLEL + +#include + +#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ +#include +#endif /* !MPI_FILE_NULL */ + +#include "pio_perf.h" + +/* Macro definitions */ + +#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 6 +#define H5DCREATE(fd, name, type, space, dcpl) H5Dcreate(fd, name, type, space, dcpl) +#define H5DOPEN(fd, name) H5Dopen(fd, name) +#else +#define H5DCREATE(fd, name, type, space, dcpl) \ + H5Dcreate2(fd, name, type, space, H5P_DEFAULT, dcpl, H5P_DEFAULT) +#define H5DOPEN(fd, name) H5Dopen2(fd, name, H5P_DEFAULT) +#endif + +/* sizes of various items. these sizes won't change during program execution */ +/* The following three must have the same type */ +#define ELMT_H5_TYPE H5T_NATIVE_UCHAR + +#define GOTOERROR(errcode) \ + { \ + ret_code = errcode; \ + goto done; \ + } +#define ERRMSG(mesg) \ + { \ + HDfprintf(stderr, "Proc %d: ", pio_mpi_rank_g); \ + HDfprintf(stderr, "*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ + } + +/* verify: if val is false (0), print mesg. */ +#define VRFY(val, mesg) \ + do { \ + if (!val) { \ + ERRMSG(mesg); \ + GOTOERROR(FAIL); \ + } \ + } while (0) + +/* POSIX I/O macros */ +#ifdef H5_HAVE_WIN32_API +/* Can't link against the library, so this test will use the older, non-Unicode + * _open() call on Windows. + */ +#define HDopen(S, F, ...) _open(S, F | _O_BINARY, __VA_ARGS__) +#endif /* H5_HAVE_WIN32_API */ +#define POSIXCREATE(fn) HDopen(fn, O_CREAT | O_TRUNC | O_RDWR, 0600) +#define POSIXOPEN(fn, F) HDopen(fn, F, 0600) +#define POSIXCLOSE(F) HDclose(F) +#define POSIXSEEK(F, L) HDlseek(F, L, SEEK_SET) +#define POSIXWRITE(F, B, S) HDwrite(F, B, S) +#define POSIXREAD(F, B, S) HDread(F, B, S) + +enum { PIO_CREATE = 1, PIO_WRITE = 2, PIO_READ = 4 }; + +/* Global variables */ +static int clean_file_g = -1; /*whether to cleanup temporary test */ +/*files. -1 is not defined; */ +/*0 is no cleanup; 1 is do cleanup */ + +/* + * In a parallel machine, the filesystem suitable for compiling is + * unlikely a parallel file system that is suitable for parallel I/O. + * There is no standard pathname for the parallel file system. /tmp + * is about the best guess. + */ +#ifndef HDF5_PARAPREFIX +#define HDF5_PARAPREFIX "" +#endif /* !HDF5_PARAPREFIX */ + +#ifndef MIN +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif /* !MIN */ + +/* the different types of file descriptors we can expect */ +typedef union _file_descr { + int posixfd; /* POSIX file handle*/ + MPI_File mpifd; /* MPI file */ + hid_t h5fd; /* HDF5 file */ +} file_descr; + +/* local functions */ +static char * pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size); +static herr_t do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nelmts, + size_t buf_size, void *buffer); +static herr_t do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nelmts, + size_t buf_size, void *buffer /*out*/); +static herr_t do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags); +static herr_t do_fclose(iotype iot, file_descr *fd); +static void do_cleanupfile(iotype iot, char *fname); +static off_t sqrto(off_t); + +/* + * Function: do_pio + * Purpose: PIO Engine where Parallel IO are executed. + * Return: results + * Programmer: Albert Cheng, Bill Wendling 2001/12/12 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +results +do_pio(parameters param) +{ + /* return codes */ + herr_t ret_code = 0; /*return code */ + results res; + + file_descr fd; + iotype iot; + + char fname[FILENAME_MAX]; + long nf; + long ndsets; + off_t nbytes; /*number of bytes per dataset */ + off_t snbytes; /*general dataset size */ + /*for 1D, it is the actual dataset size */ + /*for 2D, it is the size of a side of the dataset square */ + char * buffer = NULL; /*data buffer pointer */ + size_t buf_size; /*general buffer size in bytes */ + /*for 1D, it is the actual buffer size */ + /*for 2D, it is the length of the buffer rectangle */ + size_t blk_size; /*data block size in bytes */ + size_t bsize; /*actual buffer size */ + + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + + /* Sanity check parameters */ + + /* IO type */ + iot = param.io_type; + + switch (iot) { + case MPIO: + fd.mpifd = MPI_FILE_NULL; + res.timers = io_time_new(MPI_CLOCK); + break; + case POSIXIO: + fd.posixfd = -1; + res.timers = io_time_new(MPI_CLOCK); + break; + case PHDF5: + fd.h5fd = -1; + res.timers = io_time_new(MPI_CLOCK); + break; + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", iot); + GOTOERROR(FAIL); + } + + ndsets = param.num_dsets; /* number of datasets per file */ + nbytes = param.num_bytes; /* number of bytes per dataset */ + buf_size = param.buf_size; + blk_size = param.blk_size; + + if (!param.dim2d) { + snbytes = nbytes; /* General dataset size */ + bsize = buf_size; /* Actual buffer size */ + } + else { + snbytes = sqrto(nbytes); /* General dataset size */ + bsize = buf_size * blk_size; /* Actual buffer size */ + } + + if (param.num_files < 0) { + HDfprintf(stderr, "number of files must be >= 0 (%ld)\n", param.num_files); + GOTOERROR(FAIL); + } + + if (ndsets < 0) { + HDfprintf(stderr, "number of datasets per file must be >= 0 (%ld)\n", ndsets); + GOTOERROR(FAIL); + } + + if (param.num_procs <= 0) { + HDfprintf(stderr, "maximum number of process to use must be > 0 (%d)\n", param.num_procs); + GOTOERROR(FAIL); + } + + /* Validate transfer buffer size & block size*/ + if (blk_size <= 0) { + HDfprintf(stderr, "Transfer block size (%zu) must be > 0\n", blk_size); + GOTOERROR(FAIL); + } + if (buf_size <= 0) { + HDfprintf(stderr, "Transfer buffer size (%zu) must be > 0\n", buf_size); + GOTOERROR(FAIL); + } + if ((buf_size % blk_size) != 0) { + HDfprintf(stderr, + "Transfer buffer size (%zu) must be a multiple of the " + "interleaved I/O block size (%zu)\n", + buf_size, blk_size); + GOTOERROR(FAIL); + } + if ((snbytes % pio_mpi_nprocs_g) != 0) { + HDfprintf(stderr, + "Dataset size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " + "number of processes (%d)\n", + (long long)snbytes, pio_mpi_nprocs_g); + GOTOERROR(FAIL); + } + + if (!param.dim2d) { + if (((size_t)(snbytes / pio_mpi_nprocs_g) % buf_size) != 0) { + HDfprintf(stderr, + "Dataset size/process (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " + "trasfer buffer size (%zu)\n", + (long long)(snbytes / pio_mpi_nprocs_g), buf_size); + GOTOERROR(FAIL); + } + } + else { + if (((size_t)snbytes % buf_size) != 0) { + HDfprintf(stderr, + "Dataset side size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " + "trasfer buffer size (%zu)\n", + (long long)snbytes, buf_size); + GOTOERROR(FAIL); + } + } + + /* Allocate transfer buffer */ + if ((buffer = malloc(bsize)) == NULL) { + HDfprintf(stderr, "malloc for transfer buffer size (%zu) failed\n", bsize); + GOTOERROR(FAIL); + } + + if (pio_debug_level >= 4) { + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + + /* output all of the times for all iterations */ + if (myrank == 0) + HDfprintf(output, "Timer details:\n"); + } + + for (nf = 1; nf <= param.num_files; nf++) { + /* + * Write performance measurement + */ + /* Open file for write */ + char base_name[256]; + + HDsprintf(base_name, "#pio_tmp_%lu", nf); + pio_create_filename(iot, base_name, fname, sizeof(fname)); + if (pio_debug_level > 0) + HDfprintf(output, "rank %d: data filename=%s\n", pio_mpi_rank_g, fname); + + /* Need barrier to make sure everyone starts at the same time */ + MPI_Barrier(pio_comm_g); + + io_time_set(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTART); + hrc = do_fopen(¶m, fname, &fd, PIO_CREATE | PIO_WRITE); + + VRFY((hrc == SUCCESS), "do_fopen failed"); + + io_time_set(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTART); + hrc = do_write(&res, &fd, ¶m, ndsets, nbytes, buf_size, buffer); + io_time_set(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTOP); + + VRFY((hrc == SUCCESS), "do_write failed"); + + /* Close file for write */ + hrc = do_fclose(iot, &fd); + + io_time_set(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_fclose failed"); + + if (!param.h5_write_only) { + /* + * Read performance measurement + */ + /* Need barrier to make sure everyone is done writing and has + * closed the file. Also to make sure everyone starts reading + * at the same time. + */ + MPI_Barrier(pio_comm_g); + + /* Open file for read */ + io_time_set(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTART); + hrc = do_fopen(¶m, fname, &fd, PIO_READ); + + VRFY((hrc == SUCCESS), "do_fopen failed"); + + io_time_set(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTART); + hrc = do_read(&res, &fd, ¶m, ndsets, nbytes, buf_size, buffer); + io_time_set(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_read failed"); + + /* Close file for read */ + hrc = do_fclose(iot, &fd); + + io_time_set(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_fclose failed"); + } + + /* Need barrier to make sure everyone is done with the file */ + /* before it may be removed by do_cleanupfile */ + MPI_Barrier(pio_comm_g); + do_cleanupfile(iot, fname); + } + +done: + /* clean up */ + /* release HDF5 objects */ + + /* close any opened files */ + /* no remove(fname) because that should have happened normally. */ + switch (iot) { + case POSIXIO: + if (fd.posixfd != -1) + hrc = do_fclose(iot, &fd); + break; + case MPIO: + if (fd.mpifd != MPI_FILE_NULL) + hrc = do_fclose(iot, &fd); + break; + case PHDF5: + if (fd.h5fd != -1) + hrc = do_fclose(iot, &fd); + break; + default: + break; + } + + /* release generic resources */ + if (buffer) + HDfree(buffer); + res.ret_code = ret_code; + return res; +} + +/* + * Function: pio_create_filename + * Purpose: Create a new filename to write to. Determine the correct + * suffix to append to the filename by the type of I/O we're + * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if + * USER or LOGIN are specified in the environment. + * Return: Pointer to filename or NULL + * Programmer: Bill Wendling, 21. November 2001 + * Modifications: + */ +static char * +pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size) +{ + const char *prefix, *suffix = ""; + char * ptr, last = '\0'; + size_t i, j; + + if (!base_name || !fullname || size < 1) + return NULL; + + HDmemset(fullname, 0, size); + + switch (iot) { + case POSIXIO: + suffix = ".posix"; + break; + case MPIO: + suffix = ".mpio"; + break; + case PHDF5: + suffix = ".h5"; + break; + default: + break; + } + + /* First use the environment variable and then try the constant */ + prefix = HDgetenv("HDF5_PARAPREFIX"); + +#ifdef HDF5_PARAPREFIX + if (!prefix) + prefix = HDF5_PARAPREFIX; +#endif /* HDF5_PARAPREFIX */ + + /* Prepend the prefix value to the base name */ + if (prefix && *prefix) { + /* If the prefix specifies the HDF5_PARAPREFIX directory, then + * default to using the "/tmp/$USER" or "/tmp/$LOGIN" + * directory instead. */ + register char *user, *login, *subdir; + + user = HDgetenv("USER"); + login = HDgetenv("LOGIN"); + subdir = (user ? user : login); + + if (subdir) { + for (i = 0; i < size - 1 && prefix[i]; i++) + fullname[i] = prefix[i]; + + fullname[i++] = '/'; + + for (j = 0; i < size && subdir[j]; i++, j++) + fullname[i] = subdir[j]; + } + else { + /* We didn't append the prefix yet */ + HDstrncpy(fullname, prefix, size); + fullname[size - 1] = '\0'; + } + + if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) { + /* Append the base_name with a slash first. Multiple slashes are + * handled below. */ + h5_stat_t buf; + + if (HDstat(fullname, &buf) < 0) + /* The directory doesn't exist just yet */ + if (HDmkdir(fullname, (mode_t)0755) < 0 && errno != EEXIST) { + /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. + * Default to PREFIX's original prefix value. */ + HDstrcpy(fullname, prefix); + } + + HDstrcat(fullname, "/"); + HDstrcat(fullname, base_name); + } + else { + /* Buffer is too small */ + return NULL; + } + } + else if (HDstrlen(base_name) >= size) { + /* Buffer is too small */ + return NULL; + } + else { + HDstrcpy(fullname, base_name); + } + + /* Append a suffix */ + if (suffix) { + if (HDstrlen(fullname) + HDstrlen(suffix) >= size) + return NULL; + + HDstrcat(fullname, suffix); + } + + /* Remove any double slashes in the filename */ + for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) { + if (*ptr != '/' || last != '/') + fullname[j++] = *ptr; + + last = *ptr; + } + + return fullname; +} + +/* + * Function: do_write + * Purpose: Write the required amount of data to the file. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static herr_t +do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbytes, size_t buf_size, + void *buffer) +{ + int ret_code = SUCCESS; + int rc; /*routine return code */ + long ndset; + size_t blk_size; /* The block size to subdivide the xfer buffer into */ + off_t nbytes_xfer; /* Total number of bytes transferred so far */ + size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */ + size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */ + char dname[64]; + off_t dset_offset = 0; /*dataset offset in a file */ + off_t bytes_begin[2]; /*first elmt this process transfer */ + off_t bytes_count; /*number of elmts this process transfer */ + off_t snbytes = 0; /*size of a side of the dataset square */ + unsigned char *buf_p; /* Current buffer pointer */ + + /* POSIX variables */ + off_t file_offset; /* File offset of the next transfer */ + off_t file_offset_advance; /* File offset advance after each I/O operation */ + off_t posix_file_offset; /* Base file offset of the next transfer */ + + /* MPI variables */ + MPI_Offset mpi_file_offset; /* Base file offset of the next transfer*/ + MPI_Offset mpi_offset; /* Offset in MPI file */ + MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */ + MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */ + MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */ + MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */ + MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */ + MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */ + MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */ + MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */ + MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */ + MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */ + MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */ + MPI_Status mpi_status; + int mrc; /* MPI return code */ + + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + hsize_t h5dims[2]; /*dataset dim sizes */ + hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ + hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ + hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ + hsize_t h5block[2]; /*dataspace selection */ + hsize_t h5stride[2]; + hsize_t h5count[2]; + hsize_t h5start[2]; + hssize_t h5offset[2]; /* Selection offset within dataspace */ + hid_t h5dcpl = H5I_INVALID_HID; /* Dataset creation property list */ + hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ + + /* Get the parameters from the parameter block */ + blk_size = parms->blk_size; + + /* There are two kinds of transfer patterns, contiguous and interleaved. + * Let 0,1,2,...,n be data accessed by process 0,1,2,...,n + * where n is rank of the last process. + * In contiguous pattern, data are accessed as + * 000...111...222...nnn... + * In interleaved pattern, data are accessed as + * 012...n012...n... + * These are all in the scope of one dataset. + */ + + /* 1D dataspace */ + if (!parms->dim2d) { + /* Contiguous Pattern: */ + if (!parms->interleaved) { + bytes_begin[0] = (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); + } /* end if */ + /* Interleaved Pattern: */ + else { + bytes_begin[0] = (off_t)(blk_size * (size_t)pio_mpi_rank_g); + } /* end else */ + + /* Prepare buffer for verifying data */ + if (parms->verify) + memset(buffer, pio_mpi_rank_g + 1, buf_size); + } /* end if */ + /* 2D dataspace */ + else { + /* nbytes is always the number of bytes per dataset (1D or 2D). If the + dataspace is 2D, snbytes is the size of a side of the dataset square. + */ + snbytes = sqrto(nbytes); + + /* Contiguous Pattern: */ + if (!parms->interleaved) { + bytes_begin[0] = (off_t)((double)snbytes * pio_mpi_rank_g / pio_mpi_nprocs_g); + bytes_begin[1] = 0; + } /* end if */ + /* Interleaved Pattern: */ + else { + bytes_begin[0] = 0; + + if (!parms->h5_use_chunks || parms->io_type == PHDF5) + bytes_begin[1] = (off_t)(blk_size * (size_t)pio_mpi_rank_g); + else + bytes_begin[1] = (off_t)(blk_size * blk_size * (size_t)pio_mpi_rank_g); + } /* end else */ + + /* Prepare buffer for verifying data */ + if (parms->verify) + HDmemset(buffer, pio_mpi_rank_g + 1, buf_size * blk_size); + } /* end else */ + + /* Calculate the total number of bytes (bytes_count) to be + * transferred by this process. It may be different for different + * transfer pattern due to rounding to integral values. + */ + /* + * Calculate the beginning bytes of this process and the next. + * bytes_count is the difference between these two beginnings. + * This way, it eliminates any rounding errors. + * (This is tricky, don't mess with the formula, rounding errors + * can easily get introduced) */ + bytes_count = (off_t)(((double)nbytes * (pio_mpi_rank_g + 1)) / pio_mpi_nprocs_g) - + (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); + + /* debug */ + if (pio_debug_level >= 4) { + HDprint_rank(output); + if (!parms->dim2d) { + HDfprintf(output, + "Debug(do_write): " + "buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH + "d\n", + buf_size, (long long)bytes_begin[0], (long long)bytes_count); + } + else { + HDfprintf(output, + "Debug(do_write): " + "linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH + "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n", + buf_size * blk_size, (long long)bytes_begin[0], (long long)bytes_begin[1], + (long long)bytes_count); + } + } + + /* I/O Access specific setup */ + switch (parms->io_type) { + case POSIXIO: + /* No extra setup */ + break; + + case MPIO: /* MPI-I/O setup */ + /* 1D dataspace */ + if (!parms->dim2d) { + /* Build block's derived type */ + mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Build file's derived type */ + mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)pio_mpi_nprocs_g, mpi_blk_type, + &mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit file type */ + mrc = MPI_Type_commit(&mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Commit buffer type */ + mrc = MPI_Type_commit(&mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + } /* end if */ + /* 2D dataspace */ + else { + /* Build partial buffer derived type for contiguous access */ + + mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE, &mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit partial buffer derived type */ + mrc = MPI_Type_commit(&mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build contiguous file's derived type */ + mrc = MPI_Type_vector((int)blk_size, (int)1, (int)((size_t)snbytes / buf_size), + mpi_partial_buffer_cont, &mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit contiguous file type */ + mrc = MPI_Type_commit(&mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build partial buffer derived type for interleaved access */ + mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit partial buffer derived type */ + mrc = MPI_Type_commit(&mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build interleaved file's derived type */ + mrc = MPI_Type_vector((int)buf_size, (int)1, (int)((size_t)snbytes / blk_size), + mpi_partial_buffer_inter, &mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit interleaved file type */ + mrc = MPI_Type_commit(&mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build full buffer derived type */ + mrc = MPI_Type_contiguous((int)(blk_size * buf_size), MPI_BYTE, &mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit full buffer derived type */ + mrc = MPI_Type_commit(&mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build full chunk derived type */ + mrc = MPI_Type_contiguous((int)(blk_size * blk_size), MPI_BYTE, &mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit full chunk derived type */ + mrc = MPI_Type_commit(&mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build chunk interleaved file's derived type */ + mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)((size_t)snbytes / blk_size), + mpi_full_chunk, &mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit chunk interleaved file type */ + mrc = MPI_Type_commit(&mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + } /* end else */ + break; + + case PHDF5: /* HDF5 setup */ + /* 1D dataspace */ + if (!parms->dim2d) { + if (nbytes > 0) { + /* define a contiguous dataset of nbytes native bytes */ + h5dims[0] = (hsize_t)nbytes; + h5dset_space_id = H5Screate_simple(1, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + /* Set up the file dset space id to select the pattern to access */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5stride[0] = h5block[0] = blk_size; + h5count[0] = buf_size / blk_size; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5stride[0] = blk_size * (size_t)pio_mpi_nprocs_g; + h5block[0] = blk_size; + h5count[0] = buf_size / blk_size; + } /* end else */ + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, + h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + } /* end if */ + else { + h5dset_space_id = H5Screate(H5S_SCALAR); + VRFY((h5dset_space_id >= 0), "H5Screate"); + } /* end else */ + + /* Create the memory dataspace that corresponds to the xfer buffer */ + if (buf_size > 0) { + h5dims[0] = buf_size; + h5mem_space_id = H5Screate_simple(1, h5dims, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + } /* end if */ + else { + h5mem_space_id = H5Screate(H5S_SCALAR); + VRFY((h5mem_space_id >= 0), "H5Screate"); + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + if (nbytes > 0) { + /* define a contiguous dataset of nbytes native bytes */ + h5dims[0] = (hsize_t)snbytes; + h5dims[1] = (hsize_t)snbytes; + h5dset_space_id = H5Screate_simple(2, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + /* Set up the file dset space id to select the pattern to access */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5start[1] = (hsize_t)bytes_begin[1]; + h5stride[0] = 1; + h5stride[1] = h5block[0] = h5block[1] = blk_size; + h5count[0] = 1; + h5count[1] = buf_size / blk_size; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5start[1] = (hsize_t)bytes_begin[1]; + h5stride[0] = blk_size; + h5stride[1] = blk_size * (size_t)pio_mpi_nprocs_g; + h5block[0] = h5block[1] = blk_size; + h5count[0] = buf_size / blk_size; + h5count[1] = 1; + } /* end else */ + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, + h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + } /* end if */ + else { + h5dset_space_id = H5Screate(H5S_SCALAR); + VRFY((h5dset_space_id >= 0), "H5Screate"); + } /* end else */ + + /* Create the memory dataspace that corresponds to the xfer buffer */ + if (buf_size > 0) { + if (!parms->interleaved) { + h5dims[0] = blk_size; + h5dims[1] = buf_size; + } + else { + h5dims[0] = buf_size; + h5dims[1] = blk_size; + } + h5mem_space_id = H5Screate_simple(2, h5dims, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + } /* end if */ + else { + h5mem_space_id = H5Screate(H5S_SCALAR); + VRFY((h5mem_space_id >= 0), "H5Screate"); + } /* end else */ + } /* end else */ + + /* Create the dataset transfer property list */ + h5dxpl = H5Pcreate(H5P_DATASET_XFER); + if (h5dxpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + /* Change to collective I/O, if asked */ + if (parms->collective) { + hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + break; + + default: + break; + } /* end switch */ + + for (ndset = 1; ndset <= ndsets; ++ndset) { + + /* Calculate dataset offset within a file */ + + /* create dataset */ + switch (parms->io_type) { + case POSIXIO: + case MPIO: + /* both posix and mpi io just need dataset offset in file*/ + dset_offset = (ndset - 1) * nbytes; + break; + + case PHDF5: + h5dcpl = H5Pcreate(H5P_DATASET_CREATE); + if (h5dcpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + /* 1D dataspace */ + if (!parms->dim2d) { + /* Make the dataset chunked if asked */ + if (parms->h5_use_chunks) { + /* Set the chunk size to be the same as the buffer size */ + h5dims[0] = blk_size; + hrc = H5Pset_chunk(h5dcpl, 1, h5dims); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + } /* end if */ + else { + /* 2D dataspace */ + if (parms->h5_use_chunks) { + /* Set the chunk size to be the same as the block size */ + h5dims[0] = blk_size; + h5dims[1] = blk_size; + hrc = H5Pset_chunk(h5dcpl, 2, h5dims); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + } /* end else */ + + HDsprintf(dname, "Dataset_%ld", ndset); + h5ds_id = H5DCREATE(fd->h5fd, dname, ELMT_H5_TYPE, h5dset_space_id, h5dcpl); + + if (h5ds_id < 0) { + HDfprintf(stderr, "HDF5 Dataset Create failed\n"); + GOTOERROR(FAIL); + } + + hrc = H5Pclose(h5dcpl); + /* verifying the close of the dcpl */ + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Close failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + break; + } + + /* The task is to transfer bytes_count bytes, starting at + * bytes_begin position, using transfer buffer of buf_size bytes. + * If interleaved, select buf_size at a time, in round robin + * fashion, according to number of process. Otherwise, select + * all bytes_count in contiguous. + */ + nbytes_xfer = 0; + + /* 1D dataspace */ + if (!parms->dim2d) { + /* Set base file offset for all I/O patterns and POSIX access */ + posix_file_offset = dset_offset + bytes_begin[0]; + + /* Set base file offset for all I/O patterns and MPI access */ + mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]); + } /* end if */ + else { + /* Set base file offset for all I/O patterns and POSIX access */ + posix_file_offset = dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]; + + /* Set base file offset for all I/O patterns and MPI access */ + mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]); + } /* end else */ + + /* Start "raw data" write timer */ + io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART); + + while (nbytes_xfer < bytes_count) { + /* Write */ + /* Calculate offset of write within a dataset/file */ + switch (parms->io_type) { + case POSIXIO: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Contiguous pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + (off_t)nbytes_xfer; + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are written */ + rc = ((ssize_t)buf_size == POSIXWRITE(fd->posixfd, buffer, buf_size)); + VRFY((rc != 0), "POSIXWRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size; + + /* Loop over the buffers to write */ + while (nbytes_toxfer > 0) { + /* Skip offset over blocks of other processes */ + file_offset = posix_file_offset + (off_t)(nbytes_xfer * pio_mpi_nprocs_g); + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are written */ + rc = ((ssize_t)blk_size == POSIXWRITE(fd->posixfd, buf_p, blk_size)); + VRFY((rc != 0), "POSIXWRITE"); + + /* Advance location in buffer */ + buf_p += blk_size; + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)blk_size; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= blk_size; + } /* end while */ + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* Contiguous storage */ + if (!parms->h5_use_chunks) { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + + (off_t)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * + (blk_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / blk_size) % (size_t)snbytes)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = buf_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = (off_t)snbytes; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute file offset */ + file_offset = + posix_file_offset + + (off_t)(((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / + (size_t)snbytes) * + (buf_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % + (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = (off_t)snbytes; + } /* end else */ + } /* end if */ + /* Chunked storage */ + else { + /*Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + (off_t)nbytes_xfer; + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * buf_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = 0; + } /* end if */ + /*Interleaved access pattern */ + else { + /* Compute file offset */ + /* Before simplification */ + /* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size) + *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size + *snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size)) + *pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */ + + file_offset = posix_file_offset + + (off_t)((((size_t)nbytes_xfer / (buf_size / blk_size) * + (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * blk_size)) * + (buf_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / (buf_size / blk_size)) * + (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * blk_size)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * blk_size; + + /* Global offset advance after each I/O operation */ + /* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */ + file_offset_advance = (off_t)snbytes * (off_t)blk_size; + } /* end else */ + } /* end else */ + + /* Common code for file access */ + + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size * blk_size; + + /* Loop over portions of the buffer to write */ + while (nbytes_toxfer > 0) { + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are written */ + rc = ((ssize_t)nbytes_xfer_advance == + POSIXWRITE(fd->posixfd, buf_p, nbytes_xfer_advance)); + VRFY((rc != 0), "POSIXWRITE"); + + /* Advance location in buffer */ + buf_p += nbytes_xfer_advance; + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)nbytes_xfer_advance; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= nbytes_xfer_advance; + + /* Partially advance file offset */ + file_offset += file_offset_advance; + } /* end while */ + + } /* end else */ + + break; + + case MPIO: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Independent file access */ + if (!parms->collective) { + /* Contiguous pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Perform independent write */ + mrc = + MPI_File_write_at(fd->mpifd, mpi_offset, buffer, + (int)(buf_size / blk_size), mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size; + + /* Loop over the buffers to write */ + while (nbytes_toxfer > 0) { + /* Skip offset over blocks of other processes */ + mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); + + /* Perform independent write */ + mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p, (int)1, + mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance location in buffer */ + buf_p += blk_size; + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)blk_size; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= blk_size; + } /* end while */ + } /* end else */ + } /* end if */ + /* Collective file access */ + else { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Perform independent write */ + mrc = MPI_File_write_at_all(fd->mpifd, mpi_offset, buffer, + (int)(buf_size / blk_size), mpi_blk_type, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); + + /* Set the file view */ + mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, + (char *)"native", h5_io_info_g); + VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); + + /* Perform write */ + mrc = MPI_File_write_at_all(fd->mpifd, 0, buffer, (int)(buf_size / blk_size), + mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)buf_size; + } /* end else */ + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* Contiguous storage */ + if (!parms->h5_use_chunks) { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = + mpi_file_offset + + (MPI_Offset)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * + (blk_size * (size_t)snbytes)) + + (MPI_Offset)(((size_t)nbytes_xfer / blk_size) % (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = buf_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = snbytes; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_cont_type; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute offset in file */ + mpi_offset = + mpi_file_offset + + (MPI_Offset)( + ((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / + (size_t)snbytes) * + (buf_size * (size_t)snbytes)) + + (MPI_Offset)( + (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % + (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = snbytes; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_inter_type; + } /* end else */ + } /* end if */ + /* Chunked storage */ + else { + /*Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * buf_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = 0; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_full_buffer; + } /* end if */ + /*Interleaved access pattern */ + else { + /* Compute offset in file */ + /* Before simplification */ + /* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size) + *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))* + (buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+ + ((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes + /blk_size*(blk_size*blk_size)); */ + mpi_offset = mpi_file_offset + + (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size) * + (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * blk_size)) * + (buf_size * (size_t)snbytes)) + + (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size)) * + (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * blk_size)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * blk_size; + + /* Global offset advance after each I/O operation */ + /* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */ + mpi_offset_advance = (MPI_Offset)((size_t)snbytes * blk_size); + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_chunk_inter_type; + } /* end else */ + } /* end else */ + + /* Common code for independent file access */ + if (!parms->collective) { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size * blk_size; + + /* Loop over portions of the buffer to write */ + while (nbytes_toxfer > 0) { + /* Perform independent write */ + mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p, + (int)nbytes_xfer_advance, MPI_BYTE, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance location in buffer */ + buf_p += nbytes_xfer_advance; + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)nbytes_xfer_advance; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= nbytes_xfer_advance; + + /* Partially advance global offset in dataset */ + mpi_offset += mpi_offset_advance; + } /* end while */ + } /* end if */ + + /* Common code for collective file access */ + else { + /* Set the file view */ + mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, + (char *)"native", h5_io_info_g); + VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); + + /* Perform write */ + MPI_File_write_at_all(fd->mpifd, 0, buffer, (int)(buf_size * blk_size), MPI_BYTE, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size * (off_t)blk_size; + } /* end else */ + + } /* end else */ + + break; + + case PHDF5: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Set up the file dset space id to move the selection to process */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5offset[0] = nbytes_xfer; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5offset[0] = (nbytes_xfer * pio_mpi_nprocs_g); + } /* end else */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Write the buffer out */ + hrc = + H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dwrite"); + + /* Increment number of bytes transferred */ + nbytes_xfer += (ssize_t)buf_size; + } /* end if */ + /* 2D dataspace */ + else { + /* Set up the file dset space id to move the selection to process */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5offset[0] = + (hssize_t)(((size_t)nbytes_xfer / ((size_t)snbytes * blk_size)) * blk_size); + h5offset[1] = + (hssize_t)(((size_t)nbytes_xfer % ((size_t)snbytes * blk_size)) / blk_size); + + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5offset[0] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * buf_size)) * + buf_size); + h5offset[1] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * buf_size)) / + buf_size); + + } /* end else */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Write the buffer out */ + hrc = + H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dwrite"); + + /* Increment number of bytes transferred */ + nbytes_xfer += (off_t)buf_size * (off_t)blk_size; + + } /* end else */ + + break; + + default: + break; + } /* switch (parms->io_type) */ + } /* end while */ + + /* Stop "raw data" write timer */ + io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP); + + /* Calculate write time */ + + /* Close dataset. Only HDF5 needs to do an explicit close. */ + if (parms->io_type == PHDF5) { + hrc = H5Dclose(h5ds_id); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Close failed\n"); + GOTOERROR(FAIL); + } + + h5ds_id = H5I_INVALID_HID; + } /* end if */ + } /* end for */ + +done: + /* release MPI-I/O objects */ + if (parms->io_type == MPIO) { + /* 1D dataspace */ + if (!parms->dim2d) { + /* Free file type */ + mrc = MPI_Type_free(&mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free buffer type */ + mrc = MPI_Type_free(&mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + } /* end if */ + /* 2D dataspace */ + else { + /* Free partial buffer type for contiguous access */ + mrc = MPI_Type_free(&mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free contiguous file type */ + mrc = MPI_Type_free(&mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free partial buffer type for interleaved access */ + mrc = MPI_Type_free(&mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free interleaved file type */ + mrc = MPI_Type_free(&mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free full buffer type */ + mrc = MPI_Type_free(&mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free full chunk type */ + mrc = MPI_Type_free(&mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free chunk interleaved file type */ + mrc = MPI_Type_free(&mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + } /* end else */ + } /* end if */ + + /* release HDF5 objects */ + if (h5dset_space_id != -1) { + hrc = H5Sclose(h5dset_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); + ret_code = FAIL; + } + else { + h5dset_space_id = H5I_INVALID_HID; + } + } + + if (h5mem_space_id != -1) { + hrc = H5Sclose(h5mem_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); + ret_code = FAIL; + } + else { + h5mem_space_id = H5I_INVALID_HID; + } + } + + if (h5dxpl != -1) { + hrc = H5Pclose(h5dxpl); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); + ret_code = FAIL; + } + else { + h5dxpl = H5I_INVALID_HID; + } + } + + return ret_code; +} + +static off_t +sqrto(off_t x) +{ + double root_x = sqrt((double)x); + return (off_t)root_x; +} + +/* + * Function: do_read + * Purpose: read the required amount of data from the file. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng 2001/12/13 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static herr_t +do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbytes, size_t buf_size, + void *buffer /*out*/) +{ + int ret_code = SUCCESS; + int rc; /*routine return code */ + long ndset; + size_t blk_size; /* The block size to subdivide the xfer buffer into */ + size_t bsize; /* Size of the actual buffer */ + off_t nbytes_xfer; /* Total number of bytes transferred so far */ + size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */ + size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */ + char dname[64]; + off_t dset_offset = 0; /*dataset offset in a file */ + off_t bytes_begin[2]; /*first elmt this process transfer */ + off_t bytes_count; /*number of elmts this process transfer */ + off_t snbytes = 0; /*size of a side of the dataset square */ + unsigned char *buf_p; /* Current buffer pointer */ + + /* POSIX variables */ + off_t file_offset; /* File offset of the next transfer */ + off_t file_offset_advance; /* File offset advance after each I/O operation */ + off_t posix_file_offset; /* Base file offset of the next transfer */ + + /* MPI variables */ + MPI_Offset mpi_file_offset; /* Base file offset of the next transfer*/ + MPI_Offset mpi_offset; /* Offset in MPI file */ + MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */ + MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */ + MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */ + MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */ + MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */ + MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */ + MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */ + MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */ + MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */ + MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */ + MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */ + MPI_Status mpi_status; + int mrc; /* MPI return code */ + + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + hsize_t h5dims[2]; /*dataset dim sizes */ + hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ + hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ + hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ + hsize_t h5block[2]; /*dataspace selection */ + hsize_t h5stride[2]; + hsize_t h5count[2]; + hsize_t h5start[2]; + hssize_t h5offset[2]; /* Selection offset within dataspace */ + hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ + + /* Get the parameters from the parameter block */ + blk_size = parms->blk_size; + + /* There are two kinds of transfer patterns, contiguous and interleaved. + * Let 0,1,2,...,n be data accessed by process 0,1,2,...,n + * where n is rank of the last process. + * In contiguous pattern, data are accessed as + * 000...111...222...nnn... + * In interleaved pattern, data are accessed as + * 012...n012...n... + * These are all in the scope of one dataset. + */ + + /* 1D dataspace */ + if (!parms->dim2d) { + bsize = buf_size; + /* Contiguous Pattern: */ + if (!parms->interleaved) { + bytes_begin[0] = (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); + } /* end if */ + /* Interleaved Pattern: */ + else { + bytes_begin[0] = (off_t)blk_size * (off_t)pio_mpi_rank_g; + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* nbytes is always the number of bytes per dataset (1D or 2D). If the + dataspace is 2D, snbytes is the size of a side of the 'dataset square'. + */ + snbytes = sqrto(nbytes); + + bsize = buf_size * blk_size; + + /* Contiguous Pattern: */ + if (!parms->interleaved) { + bytes_begin[0] = (off_t)((double)snbytes * pio_mpi_rank_g / pio_mpi_nprocs_g); + bytes_begin[1] = 0; + } /* end if */ + /* Interleaved Pattern: */ + else { + bytes_begin[0] = 0; + + if (!parms->h5_use_chunks || parms->io_type == PHDF5) + bytes_begin[1] = (off_t)blk_size * (off_t)pio_mpi_rank_g; + else + bytes_begin[1] = (off_t)blk_size * (off_t)blk_size * (off_t)pio_mpi_rank_g; + } /* end else */ + } /* end else */ + + /* Calculate the total number of bytes (bytes_count) to be + * transferred by this process. It may be different for different + * transfer pattern due to rounding to integral values. + */ + /* + * Calculate the beginning bytes of this process and the next. + * bytes_count is the difference between these two beginnings. + * This way, it eliminates any rounding errors. + * (This is tricky, don't mess with the formula, rounding errors + * can easily get introduced) */ + bytes_count = (off_t)(((double)nbytes * (pio_mpi_rank_g + 1)) / pio_mpi_nprocs_g) - + (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); + + /* debug */ + if (pio_debug_level >= 4) { + HDprint_rank(output); + if (!parms->dim2d) { + HDfprintf(output, + "Debug(do_write): " + "buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH + "d\n", + buf_size, (long long)bytes_begin[0], (long long)bytes_count); + } + else { + HDfprintf(output, + "Debug(do_write): " + "linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH + "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n", + buf_size * blk_size, (long long)bytes_begin[0], (long long)bytes_begin[1], + (long long)bytes_count); + } + } + + /* I/O Access specific setup */ + switch (parms->io_type) { + case POSIXIO: + /* No extra setup */ + break; + + case MPIO: /* MPI-I/O setup */ + /* 1D dataspace */ + if (!parms->dim2d) { + /* Build block's derived type */ + mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Build file's derived type */ + mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)pio_mpi_nprocs_g, mpi_blk_type, + &mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit file type */ + mrc = MPI_Type_commit(&mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Commit buffer type */ + mrc = MPI_Type_commit(&mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + } /* end if */ + /* 2D dataspace */ + else { + /* Build partial buffer derived type for contiguous access */ + mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE, &mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit partial buffer derived type */ + mrc = MPI_Type_commit(&mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build contiguous file's derived type */ + mrc = MPI_Type_vector((int)blk_size, (int)1, (int)((size_t)snbytes / buf_size), + mpi_partial_buffer_cont, &mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit contiguous file type */ + mrc = MPI_Type_commit(&mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build partial buffer derived type for interleaved access */ + mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit partial buffer derived type */ + mrc = MPI_Type_commit(&mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build interleaved file's derived type */ + mrc = MPI_Type_vector((int)buf_size, (int)1, (int)((size_t)snbytes / blk_size), + mpi_partial_buffer_inter, &mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit interleaved file type */ + mrc = MPI_Type_commit(&mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build full buffer derived type */ + mrc = MPI_Type_contiguous((int)(blk_size * buf_size), MPI_BYTE, &mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit full buffer derived type */ + mrc = MPI_Type_commit(&mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build full chunk derived type */ + mrc = MPI_Type_contiguous((int)(blk_size * blk_size), MPI_BYTE, &mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit full chunk derived type */ + mrc = MPI_Type_commit(&mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build chunk interleaved file's derived type */ + mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)((size_t)snbytes / blk_size), + mpi_full_chunk, &mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit chunk interleaved file type */ + mrc = MPI_Type_commit(&mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + } /* end else */ + break; + + case PHDF5: /* HDF5 setup */ + /* 1D dataspace */ + if (!parms->dim2d) { + if (nbytes > 0) { + /* define a contiguous dataset of nbytes native bytes */ + h5dims[0] = (hsize_t)nbytes; + h5dset_space_id = H5Screate_simple(1, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + /* Set up the file dset space id to select the pattern to access */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5stride[0] = h5block[0] = blk_size; + h5count[0] = buf_size / blk_size; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5stride[0] = blk_size * (size_t)pio_mpi_nprocs_g; + h5block[0] = blk_size; + h5count[0] = buf_size / blk_size; + } /* end else */ + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, + h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + } /* end if */ + else { + h5dset_space_id = H5Screate(H5S_SCALAR); + VRFY((h5dset_space_id >= 0), "H5Screate"); + } /* end else */ + + /* Create the memory dataspace that corresponds to the xfer buffer */ + if (buf_size > 0) { + h5dims[0] = buf_size; + h5mem_space_id = H5Screate_simple(1, h5dims, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + } /* end if */ + else { + h5mem_space_id = H5Screate(H5S_SCALAR); + VRFY((h5mem_space_id >= 0), "H5Screate"); + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + if (nbytes > 0) { + /* define a contiguous dataset of nbytes native bytes */ + h5dims[0] = (hsize_t)snbytes; + h5dims[1] = (hsize_t)snbytes; + h5dset_space_id = H5Screate_simple(2, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + /* Set up the file dset space id to select the pattern to access */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5start[1] = (hsize_t)bytes_begin[1]; + h5stride[0] = 1; + h5stride[1] = h5block[0] = h5block[1] = blk_size; + h5count[0] = 1; + h5count[1] = buf_size / blk_size; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5start[1] = (hsize_t)bytes_begin[1]; + h5stride[0] = blk_size; + h5stride[1] = blk_size * (size_t)pio_mpi_nprocs_g; + h5block[0] = h5block[1] = blk_size; + h5count[0] = buf_size / blk_size; + h5count[1] = 1; + } /* end else */ + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, + h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + } /* end if */ + else { + h5dset_space_id = H5Screate(H5S_SCALAR); + VRFY((h5dset_space_id >= 0), "H5Screate"); + } /* end else */ + + /* Create the memory dataspace that corresponds to the xfer buffer */ + if (buf_size > 0) { + if (!parms->interleaved) { + h5dims[0] = blk_size; + h5dims[1] = buf_size; + } + else { + h5dims[0] = buf_size; + h5dims[1] = blk_size; + } + h5mem_space_id = H5Screate_simple(2, h5dims, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + } /* end if */ + else { + h5mem_space_id = H5Screate(H5S_SCALAR); + VRFY((h5mem_space_id >= 0), "H5Screate"); + } /* end else */ + } /* end else */ + + /* Create the dataset transfer property list */ + h5dxpl = H5Pcreate(H5P_DATASET_XFER); + if (h5dxpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + /* Change to collective I/O, if asked */ + if (parms->collective) { + hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + break; + + default: + break; + } /* end switch */ + + for (ndset = 1; ndset <= ndsets; ++ndset) { + + /* Calculate dataset offset within a file */ + + /* create dataset */ + switch (parms->io_type) { + case POSIXIO: + case MPIO: + /* both posix and mpi io just need dataset offset in file*/ + dset_offset = (ndset - 1) * nbytes; + break; + + case PHDF5: + HDsprintf(dname, "Dataset_%ld", ndset); + h5ds_id = H5DOPEN(fd->h5fd, dname); + if (h5ds_id < 0) { + HDfprintf(stderr, "HDF5 Dataset open failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + break; + } + + /* The task is to transfer bytes_count bytes, starting at + * bytes_begin position, using transfer buffer of buf_size bytes. + * If interleaved, select buf_size at a time, in round robin + * fashion, according to number of process. Otherwise, select + * all bytes_count in contiguous. + */ + nbytes_xfer = 0; + + /* 1D dataspace */ + if (!parms->dim2d) { + /* Set base file offset for all I/O patterns and POSIX access */ + posix_file_offset = dset_offset + bytes_begin[0]; + + /* Set base file offset for all I/O patterns and MPI access */ + mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]); + } /* end if */ + else { + /* Set base file offset for all I/O patterns and POSIX access */ + posix_file_offset = dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]; + + /* Set base file offset for all I/O patterns and MPI access */ + mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]); + } /* end else */ + + /* Start "raw data" read timer */ + io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTART); + + while (nbytes_xfer < bytes_count) { + /* Read */ + /* Calculate offset of read within a dataset/file */ + switch (parms->io_type) { + case POSIXIO: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Contiguous pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + (off_t)nbytes_xfer; + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are read */ + rc = ((ssize_t)buf_size == POSIXREAD(fd->posixfd, buffer, buf_size)); + VRFY((rc != 0), "POSIXREAD"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size; + + /* Loop over the buffers to read */ + while (nbytes_toxfer > 0) { + /* Skip offset over blocks of other processes */ + file_offset = posix_file_offset + (off_t)(nbytes_xfer * pio_mpi_nprocs_g); + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are read */ + rc = ((ssize_t)blk_size == POSIXREAD(fd->posixfd, buf_p, blk_size)); + VRFY((rc != 0), "POSIXREAD"); + + /* Advance location in buffer */ + buf_p += blk_size; + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)blk_size; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= blk_size; + } /* end while */ + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* Contiguous storage */ + if (!parms->h5_use_chunks) { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + + (off_t)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * + (blk_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / blk_size) % (size_t)snbytes)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = buf_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = (off_t)snbytes; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute file offset */ + file_offset = + posix_file_offset + + (off_t)(((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / + (size_t)snbytes) * + (buf_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % + (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = (off_t)snbytes; + } /* end else */ + } /* end if */ + /* Chunked storage */ + else { + /*Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + (off_t)nbytes_xfer; + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * buf_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = 0; + } /* end if */ + /*Interleaved access pattern */ + else { + /* Compute file offset */ + /* Before simplification */ + /* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size) + *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size + *snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size)) + *pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */ + + file_offset = posix_file_offset + + (off_t)((((size_t)nbytes_xfer / (buf_size / blk_size) * + (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * blk_size)) * + (buf_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / (buf_size / blk_size)) * + (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * blk_size)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * blk_size; + + /* Global offset advance after each I/O operation */ + /* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */ + file_offset_advance = (off_t)((size_t)snbytes * blk_size); + } /* end else */ + } /* end else */ + + /* Common code for file access */ + + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size * blk_size; + + /* Loop over portions of the buffer to read */ + while (nbytes_toxfer > 0) { + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are read */ + rc = ((ssize_t)nbytes_xfer_advance == + POSIXREAD(fd->posixfd, buf_p, nbytes_xfer_advance)); + VRFY((rc != 0), "POSIXREAD"); + + /* Advance location in buffer */ + buf_p += nbytes_xfer_advance; + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)nbytes_xfer_advance; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= nbytes_xfer_advance; + + /* Partially advance file offset */ + file_offset += file_offset_advance; + } /* end while */ + + } /* end else */ + break; + + case MPIO: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Independent file access */ + if (!parms->collective) { + /* Contiguous pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Perform independent read */ + mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buffer, + (int)(buf_size / blk_size), mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size; + + /* Loop over the buffers to read */ + while (nbytes_toxfer > 0) { + /* Skip offset over blocks of other processes */ + mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); + + /* Perform independent read */ + mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p, (int)1, mpi_blk_type, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance location in buffer */ + buf_p += blk_size; + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)blk_size; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= blk_size; + } /* end while */ + } /* end else */ + } /* end if */ + /* Collective file access */ + else { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Perform collective read */ + mrc = MPI_File_read_at_all(fd->mpifd, mpi_offset, buffer, + (int)(buf_size / blk_size), mpi_blk_type, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); + + /* Set the file view */ + mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, + (char *)"native", h5_io_info_g); + VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); + + /* Perform collective read */ + mrc = MPI_File_read_at_all(fd->mpifd, 0, buffer, (int)(buf_size / blk_size), + mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size; + } /* end else */ + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* Contiguous storage */ + if (!parms->h5_use_chunks) { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = + mpi_file_offset + + (MPI_Offset)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * + (blk_size * (size_t)snbytes)) + + (MPI_Offset)(((size_t)nbytes_xfer / blk_size) % (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = buf_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = snbytes; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_cont_type; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute offset in file */ + mpi_offset = + mpi_file_offset + + (MPI_Offset)( + ((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / + (size_t)snbytes) * + (buf_size * (size_t)snbytes)) + + (MPI_Offset)( + (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % + (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = snbytes; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_inter_type; + } /* end else */ + } /* end if */ + /* Chunked storage */ + else { + /*Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * buf_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = 0; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_full_buffer; + } /* end if */ + /*Interleaved access pattern */ + else { + /* Compute offset in file */ + /* Before simplification */ + /* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size) + *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))* + (buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+ + ((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes + /blk_size*(blk_size*blk_size)); */ + mpi_offset = mpi_file_offset + + (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size) * + (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * blk_size)) * + (buf_size * (size_t)snbytes)) + + (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size)) * + (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * blk_size)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * blk_size; + + /* Global offset advance after each I/O operation */ + /* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */ + mpi_offset_advance = (MPI_Offset)((size_t)snbytes * blk_size); + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_chunk_inter_type; + } /* end else */ + } /* end else */ + + /* Common code for independent file access */ + if (!parms->collective) { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size * blk_size; + + /* Loop over portions of the buffer to read */ + while (nbytes_toxfer > 0) { + /* Perform independent read */ + mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p, (int)nbytes_xfer_advance, + MPI_BYTE, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance location in buffer */ + buf_p += nbytes_xfer_advance; + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)nbytes_xfer_advance; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= nbytes_xfer_advance; + + /* Partially advance global offset in dataset */ + mpi_offset += mpi_offset_advance; + } /* end while */ + } /* end if */ + + /* Common code for collective file access */ + else { + /* Set the file view */ + mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, + (char *)"native", h5_io_info_g); + VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); + + /* Perform read */ + MPI_File_read_at_all(fd->mpifd, 0, buffer, (int)(buf_size * blk_size), MPI_BYTE, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size * (off_t)blk_size; + } /* end else */ + + } /* end else */ + break; + + case PHDF5: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Set up the file dset space id to move the selection to process */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5offset[0] = nbytes_xfer; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5offset[0] = (nbytes_xfer * pio_mpi_nprocs_g); + } /* end else */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Read the buffer in */ + hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dread"); + + /* Increment number of bytes transferred */ + nbytes_xfer += (off_t)buf_size; + } /* end if */ + /* 2D dataspace */ + else { + /* Set up the file dset space id to move the selection to process */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5offset[0] = + (hssize_t)(((size_t)nbytes_xfer / ((size_t)snbytes * blk_size)) * blk_size); + h5offset[1] = + (hssize_t)(((size_t)nbytes_xfer % ((size_t)snbytes * blk_size)) / blk_size); + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5offset[0] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * buf_size)) * + buf_size); + h5offset[1] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * buf_size)) / + buf_size); + + } /* end else */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Write the buffer out */ + hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dread"); + + /* Increment number of bytes transferred */ + nbytes_xfer += (off_t)buf_size * (off_t)blk_size; + + } /* end else */ + break; + + default: + break; + } /* switch (parms->io_type) */ + + /* Verify raw data, if asked */ + if (parms->verify) { + /* Verify data read */ + unsigned char *ucharptr = (unsigned char *)buffer; + size_t i; + int nerror = 0; + + for (i = 0; i < bsize; ++i) { + if (*ucharptr++ != pio_mpi_rank_g + 1) { + if (++nerror < 20) { + /* report at most 20 errors */ + HDprint_rank(output); + HDfprintf(output, + "read data error, expected (%d), " + "got (%d)\n", + pio_mpi_rank_g + 1, (int)*(ucharptr - 1)); + } /* end if */ + } /* end if */ + } /* end for */ + if (nerror >= 20) { + HDprint_rank(output); + HDfprintf(output, "..."); + HDfprintf(output, "total read data errors=%d\n", nerror); + } /* end if */ + } /* if (parms->verify) */ + + } /* end while */ + + /* Stop "raw data" read timer */ + io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTOP); + + /* Calculate read time */ + + /* Close dataset. Only HDF5 needs to do an explicit close. */ + if (parms->io_type == PHDF5) { + hrc = H5Dclose(h5ds_id); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Close failed\n"); + GOTOERROR(FAIL); + } + + h5ds_id = H5I_INVALID_HID; + } /* end if */ + } /* end for */ + +done: + /* release MPI-I/O objects */ + if (parms->io_type == MPIO) { + /* 1D dataspace */ + if (!parms->dim2d) { + /* Free file type */ + mrc = MPI_Type_free(&mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free buffer type */ + mrc = MPI_Type_free(&mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + } /* end if */ + /* 2D dataspace */ + else { + /* Free partial buffer type for contiguous access */ + mrc = MPI_Type_free(&mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free contiguous file type */ + mrc = MPI_Type_free(&mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free partial buffer type for interleaved access */ + mrc = MPI_Type_free(&mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free interleaved file type */ + mrc = MPI_Type_free(&mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free full buffer type */ + mrc = MPI_Type_free(&mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free full chunk type */ + mrc = MPI_Type_free(&mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free chunk interleaved file type */ + mrc = MPI_Type_free(&mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + } /* end else */ + } /* end if */ + + /* release HDF5 objects */ + if (h5dset_space_id != -1) { + hrc = H5Sclose(h5dset_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); + ret_code = FAIL; + } + else { + h5dset_space_id = H5I_INVALID_HID; + } + } + + if (h5mem_space_id != -1) { + hrc = H5Sclose(h5mem_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); + ret_code = FAIL; + } + else { + h5mem_space_id = H5I_INVALID_HID; + } + } + + if (h5dxpl != -1) { + hrc = H5Pclose(h5dxpl); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); + ret_code = FAIL; + } + else { + h5dxpl = H5I_INVALID_HID; + } + } + + return ret_code; +} + +/* + * Function: do_fopen + * Purpose: Open the specified file. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: + */ +static herr_t +do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) +{ + int ret_code = SUCCESS, mrc; + hid_t acc_tpl = H5I_INVALID_HID; /* file access templates */ + + switch (param->io_type) { + case POSIXIO: + if (flags & (PIO_CREATE | PIO_WRITE)) + fd->posixfd = POSIXCREATE(fname); + else + fd->posixfd = POSIXOPEN(fname, O_RDONLY); + + if (fd->posixfd < 0) { + HDfprintf(stderr, "POSIX File Open failed(%s)\n", fname); + GOTOERROR(FAIL); + } + + /* The perils of POSIX I/O in a parallel environment. The problem is: + * + * - Process n opens a file with truncation and then starts + * writing to the file. + * - Process m also opens the file with truncation, but after + * process n has already started to write to the file. Thus, + * all of the stuff process n wrote is now lost. + */ + MPI_Barrier(pio_comm_g); + + break; + + case MPIO: + if (flags & (PIO_CREATE | PIO_WRITE)) { + MPI_File_delete(fname, h5_io_info_g); + mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_CREATE | MPI_MODE_RDWR, h5_io_info_g, + &fd->mpifd); + + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI File Open failed(%s)\n", fname); + GOTOERROR(FAIL); + } + + /*since MPI_File_open with MPI_MODE_CREATE does not truncate */ + /*filesize , set size to 0 explicitedly. */ + mrc = MPI_File_set_size(fd->mpifd, (MPI_Offset)0); + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI_File_set_size failed\n"); + GOTOERROR(FAIL); + } + } + else { + mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_RDONLY, h5_io_info_g, &fd->mpifd); + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI File Open failed(%s)\n", fname); + GOTOERROR(FAIL); + } + } + + break; + + case PHDF5: + if ((acc_tpl = H5Pcreate(H5P_FILE_ACCESS)) < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + /* Set the file driver to the MPI-IO driver */ + if (H5Pset_fapl_mpio(acc_tpl, pio_comm_g, h5_io_info_g) < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } + + /* Set the alignment of objects in HDF5 file */ + if (H5Pset_alignment(acc_tpl, param->h5_thresh, param->h5_align) < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } + + /* create the parallel file */ + if (flags & (PIO_CREATE | PIO_WRITE)) + fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); + else + fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, acc_tpl); + if (fd->h5fd < 0) { + HDfprintf(stderr, "HDF5 File Create failed(%s)\n", fname); + GOTOERROR(FAIL); + } + + /* verifying the close of the acc_tpl */ + if (H5Pclose(acc_tpl) < 0) { + HDfprintf(stderr, "HDF5 Property List Close failed\n"); + GOTOERROR(FAIL); + } + + break; + + default: + break; + } + +done: + return ret_code; +} + +/* + * Function: do_fclose + * Purpose: Close the specified file descriptor. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: + */ +static herr_t +do_fclose(iotype iot, file_descr *fd /*out*/) +{ + herr_t ret_code = SUCCESS, hrc; + int mrc = 0, rc = 0; + + switch (iot) { + case POSIXIO: + rc = POSIXCLOSE(fd->posixfd); + + if (rc != 0) { + HDfprintf(stderr, "POSIX File Close failed\n"); + GOTOERROR(FAIL); + } + + fd->posixfd = -1; + break; + + case MPIO: + mrc = MPI_File_close(&fd->mpifd); + + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI File close failed\n"); + GOTOERROR(FAIL); + } + + fd->mpifd = MPI_FILE_NULL; + break; + + case PHDF5: + hrc = H5Fclose(fd->h5fd); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 File Close failed\n"); + GOTOERROR(FAIL); + } + + fd->h5fd = -1; + break; + + default: + break; + } + +done: + return ret_code; +} + +/* + * Function: do_fclose + * Purpose: Cleanup temporary file unless HDF5_NOCLEANUP is set. + * Only Proc 0 of the PIO communicator will do the cleanup. + * Other processes just return. + * Return: void + * Programmer: Albert Cheng 2001/12/12 + * Modifications: + */ +static void +do_cleanupfile(iotype iot, char *fname) +{ + if (pio_mpi_rank_g != 0) + return; + + if (clean_file_g == -1) + clean_file_g = (getenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; + + if (clean_file_g) { + switch (iot) { + case POSIXIO: + HDremove(fname); + break; + case MPIO: + case PHDF5: + MPI_File_delete(fname, h5_io_info_g); + break; + default: + break; + } + } +} + +#ifdef TIME_MPI +/* instrument the MPI_File_wrirte_xxx and read_xxx calls to measure + * pure time spent in MPI_File code. + */ +int +MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, + MPI_Status *status) +{ + int err; + io_time_set(timer_g, HDF5_MPI_READ, TSTART); + err = PMPI_File_read_at(fh, offset, buf, count, datatype, status); + io_time_set(timer_g, HDF5_MPI_READ, TSTOP); + return err; +} + +int +MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, + MPI_Status *status) +{ + int err; + io_time_set(timer_g, HDF5_MPI_READ, TSTART); + err = PMPI_File_read_at_all(fh, offset, buf, count, datatype, status); + io_time_set(timer_g, HDF5_MPI_READ, TSTOP); + return err; +} + +int +MPI_File_write_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, + MPI_Status *status) +{ + int err; + io_time_set(timer_g, HDF5_MPI_WRITE, TSTART); + err = PMPI_File_write_at(fh, offset, buf, count, datatype, status); + io_time_set(timer_g, HDF5_MPI_WRITE, TSTOP); + return err; +} + +int +MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, + MPI_Status *status) +{ + int err; + io_time_set(timer_g, HDF5_MPI_WRITE, TSTART); + err = PMPI_File_write_at_all(fh, offset, buf, count, datatype, status); + io_time_set(timer_g, HDF5_MPI_WRITE, TSTOP); + return err; +} + +#endif /* TIME_MPI */ +#endif /* H5_HAVE_PARALLEL */ diff --git a/tools/src/h5perf/pio_perf.c b/tools/src/h5perf/pio_perf.c new file mode 100644 index 0000000..96cba2d --- /dev/null +++ b/tools/src/h5perf/pio_perf.c @@ -0,0 +1,1834 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Parallel HDF5 Performance Testing Code + * -------------------------------------- + * + * Portable code to test performance on the different platforms we support. + * This is what the report should look like: + * + * nprocs = Max#Procs + * IO API = POSIXIO + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * IO API = MPIO + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * IO API = PHDF5 + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * nprocs = Max#Procs / 2 + * + * . . . + * + */ + +/* system header files */ +#include +#include +#include + +#include "hdf5.h" + +#ifdef H5_HAVE_PARALLEL + +/* library header files */ +#include + +/* our header files */ +#include "pio_perf.h" + +/* useful macros */ +#define TAB_SPACE 4 + +#define ONE_KB 1024 +#define ONE_MB (ONE_KB * ONE_KB) +#define ONE_GB (ONE_MB * ONE_KB) + +#define PIO_POSIX 0x1 +#define PIO_MPI 0x2 +#define PIO_HDF5 0x4 + +#ifdef STANDALONE +#define DBL_EPSILON 2.2204460492503131e-16 +#define H5_DBL_ABS_EQUAL(X, Y) (fabs((X) - (Y)) < DBL_EPSILON) +#endif + +/* report 0.0 in case t is zero too */ +#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL((t), 0.0) ? 0.0 : ((((double)bytes) / ONE_MB) / (t))) + +#ifndef TRUE +#define TRUE 1 +#endif /* TRUE */ +#ifndef FALSE +#define FALSE (!TRUE) +#endif /* FALSE */ + +/* global variables */ +FILE * output; /* output file */ +int comm_world_rank_g; /* my rank in MPI_COMM_RANK */ +int comm_world_nprocs_g; /* num. of processes of MPI_COMM_WORLD */ +MPI_Comm pio_comm_g; /* Communicator to run the PIO */ +int pio_mpi_rank_g; /* MPI rank of pio_comm_g */ +int pio_mpi_nprocs_g; /* Number of processes of pio_comm_g */ +int pio_debug_level = 0; /* The debug level: + * 0 - Off + * 1 - Minimal + * 2 - Some more + * 3 - Maximal + * 4 - Maximal & then some + */ + +/* local variables */ +static const char *progname = "h5perf"; + +#ifndef HDF5_PARAPREFIX +#define HDF5_PARAPREFIX "" +#endif +char * paraprefix = NULL; /* for command line option para-prefix */ +MPI_Info h5_io_info_g = MPI_INFO_NULL; /* MPI INFO object for IO */ + +/* + * Command-line options: The user can specify short or long-named + * parameters. The long-named ones can be partially spelled. When + * adding more, make sure that they don't clash with each other. + */ +#if 1 +static const char *s_opts = "a:A:B:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; +#else +static const char *s_opts = "a:A:bB:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; +#endif /* 1 */ +static struct h5_long_options l_opts[] = {{"align", require_arg, 'a'}, + {"alig", require_arg, 'a'}, + {"ali", require_arg, 'a'}, + {"al", require_arg, 'a'}, + {"api", require_arg, 'A'}, + {"ap", require_arg, 'A'}, +#if 0 + /* a sighting of the elusive binary option */ + { "binary", no_arg, 'b' }, + { "binar", no_arg, 'b' }, + { "bina", no_arg, 'b' }, + { "bin", no_arg, 'b' }, + { "bi", no_arg, 'b' }, +#endif /* 0 */ + {"block-size", require_arg, 'B'}, + {"block-siz", require_arg, 'B'}, + {"block-si", require_arg, 'B'}, + {"block-s", require_arg, 'B'}, + {"block-", require_arg, 'B'}, + {"block", require_arg, 'B'}, + {"bloc", require_arg, 'B'}, + {"blo", require_arg, 'B'}, + {"bl", require_arg, 'B'}, + {"chunk", no_arg, 'c'}, + {"chun", no_arg, 'c'}, + {"chu", no_arg, 'c'}, + {"ch", no_arg, 'c'}, + {"collective", no_arg, 'C'}, + {"collectiv", no_arg, 'C'}, + {"collecti", no_arg, 'C'}, + {"collect", no_arg, 'C'}, + {"collec", no_arg, 'C'}, + {"colle", no_arg, 'C'}, + {"coll", no_arg, 'C'}, + {"col", no_arg, 'C'}, + {"co", no_arg, 'C'}, + {"debug", require_arg, 'D'}, + {"debu", require_arg, 'D'}, + {"deb", require_arg, 'D'}, + {"de", require_arg, 'D'}, + {"geometry", no_arg, 'g'}, + {"geometr", no_arg, 'g'}, + {"geomet", no_arg, 'g'}, + {"geome", no_arg, 'g'}, + {"geom", no_arg, 'g'}, + {"geo", no_arg, 'g'}, + {"ge", no_arg, 'g'}, + {"help", no_arg, 'h'}, + {"hel", no_arg, 'h'}, + {"he", no_arg, 'h'}, + {"interleaved", require_arg, 'I'}, + {"interleave", require_arg, 'I'}, + {"interleav", require_arg, 'I'}, + {"interlea", require_arg, 'I'}, + {"interle", require_arg, 'I'}, + {"interl", require_arg, 'I'}, + {"inter", require_arg, 'I'}, + {"inte", require_arg, 'I'}, + {"int", require_arg, 'I'}, + {"in", require_arg, 'I'}, + {"max-num-processes", require_arg, 'P'}, + {"max-num-processe", require_arg, 'P'}, + {"max-num-process", require_arg, 'P'}, + {"max-num-proces", require_arg, 'P'}, + {"max-num-proce", require_arg, 'P'}, + {"max-num-proc", require_arg, 'P'}, + {"max-num-pro", require_arg, 'P'}, + {"max-num-pr", require_arg, 'P'}, + {"max-num-p", require_arg, 'P'}, + {"min-num-processes", require_arg, 'p'}, + {"min-num-processe", require_arg, 'p'}, + {"min-num-process", require_arg, 'p'}, + {"min-num-proces", require_arg, 'p'}, + {"min-num-proce", require_arg, 'p'}, + {"min-num-proc", require_arg, 'p'}, + {"min-num-pro", require_arg, 'p'}, + {"min-num-pr", require_arg, 'p'}, + {"min-num-p", require_arg, 'p'}, + {"max-xfer-size", require_arg, 'X'}, + {"max-xfer-siz", require_arg, 'X'}, + {"max-xfer-si", require_arg, 'X'}, + {"max-xfer-s", require_arg, 'X'}, + {"max-xfer", require_arg, 'X'}, + {"max-xfe", require_arg, 'X'}, + {"max-xf", require_arg, 'X'}, + {"max-x", require_arg, 'X'}, + {"min-xfer-size", require_arg, 'x'}, + {"min-xfer-siz", require_arg, 'x'}, + {"min-xfer-si", require_arg, 'x'}, + {"min-xfer-s", require_arg, 'x'}, + {"min-xfer", require_arg, 'x'}, + {"min-xfe", require_arg, 'x'}, + {"min-xf", require_arg, 'x'}, + {"min-x", require_arg, 'x'}, + {"num-bytes", require_arg, 'e'}, + {"num-byte", require_arg, 'e'}, + {"num-byt", require_arg, 'e'}, + {"num-by", require_arg, 'e'}, + {"num-b", require_arg, 'e'}, + {"num-dsets", require_arg, 'd'}, + {"num-dset", require_arg, 'd'}, + {"num-dse", require_arg, 'd'}, + {"num-ds", require_arg, 'd'}, + {"num-d", require_arg, 'd'}, + {"num-files", require_arg, 'F'}, + {"num-file", require_arg, 'F'}, + {"num-fil", require_arg, 'F'}, + {"num-fi", require_arg, 'F'}, + {"num-f", require_arg, 'F'}, + {"num-iterations", require_arg, 'i'}, + {"num-iteration", require_arg, 'i'}, + {"num-iteratio", require_arg, 'i'}, + {"num-iterati", require_arg, 'i'}, + {"num-iterat", require_arg, 'i'}, + {"num-itera", require_arg, 'i'}, + {"num-iter", require_arg, 'i'}, + {"num-ite", require_arg, 'i'}, + {"num-it", require_arg, 'i'}, + {"num-i", require_arg, 'i'}, + {"output", require_arg, 'o'}, + {"outpu", require_arg, 'o'}, + {"outp", require_arg, 'o'}, + {"out", require_arg, 'o'}, + {"ou", require_arg, 'o'}, + {"threshold", require_arg, 'T'}, + {"threshol", require_arg, 'T'}, + {"thresho", require_arg, 'T'}, + {"thresh", require_arg, 'T'}, + {"thres", require_arg, 'T'}, + {"thre", require_arg, 'T'}, + {"thr", require_arg, 'T'}, + {"th", require_arg, 'T'}, + {"write-only", require_arg, 'w'}, + {"write-onl", require_arg, 'w'}, + {"write-on", require_arg, 'w'}, + {"write-o", require_arg, 'w'}, + {"write", require_arg, 'w'}, + {"writ", require_arg, 'w'}, + {"wri", require_arg, 'w'}, + {"wr", require_arg, 'w'}, + {NULL, 0, '\0'}}; + +struct options { + long io_types; /* bitmask of which I/O types to test */ + const char *output_file; /* file to print report to */ + long num_dsets; /* number of datasets */ + long num_files; /* number of files */ + off_t num_bpp; /* number of bytes per proc per dset */ + int num_iters; /* number of iterations */ + int max_num_procs; /* maximum number of processes to use */ + int min_num_procs; /* minimum number of processes to use */ + size_t max_xfer_size; /* maximum transfer buffer size */ + size_t min_xfer_size; /* minimum transfer buffer size */ + size_t blk_size; /* Block size */ + unsigned interleaved; /* Interleaved vs. contiguous blocks */ + unsigned collective; /* Collective vs. independent I/O */ + unsigned dim2d; /* 1D vs. 2D geometry */ + int print_times; /* print times as well as throughputs */ + int print_raw; /* print raw data throughput info */ + off_t h5_alignment; /* alignment in HDF5 file */ + off_t h5_threshold; /* threshold for alignment in HDF5 file */ + int h5_use_chunks; /* Make HDF5 dataset chunked */ + int h5_write_only; /* Perform the write tests only */ + int verify; /* Verify data correctness */ +}; + +typedef struct _minmax { + double min; + double max; + double sum; + int num; +} minmax; + +/* local functions */ +static off_t parse_size_directive(const char *size); +static struct options *parse_command_line(int argc, char *argv[]); +static void run_test_loop(struct options *options); +static int run_test(iotype iot, parameters parms, struct options *opts); +static void output_all_info(minmax *mm, int count, int indent_level); +static void get_minmax(minmax *mm, double val); +static minmax accumulate_minmax_stuff(minmax *mm, int count); +static int create_comm_world(int num_procs, int *doing_pio); +static int destroy_comm_world(void); +static void output_results(const struct options *options, const char *name, minmax *table, int table_size, + off_t data_size); +static void output_times(const struct options *options, const char *name, minmax *table, int table_size); +static void output_report(const char *fmt, ...); +static void print_indent(register int indent); +static void usage(const char *prog); +static void report_parameters(struct options *opts); +static off_t squareo(off_t); + +/* + * Function: main + * Purpose: Start things up. Initialize MPI and then call the test looping + * function. + * Return: EXIT_SUCCESS or EXIT_FAILURE + * Programmer: Bill Wendling, 30. October 2001 + * Modifications: + */ +int +main(int argc, char *argv[]) +{ + int ret; + int exit_value = EXIT_SUCCESS; + struct options *opts = NULL; + +#ifndef STANDALONE + /* Initialize h5tools lib */ + h5tools_init(); +#endif + + output = stdout; + + /* initialize MPI and get the maximum num of processors we started with */ + MPI_Init(&argc, &argv); + ret = MPI_Comm_size(MPI_COMM_WORLD, &comm_world_nprocs_g); + + if (ret != MPI_SUCCESS) { + HDfprintf(stderr, "%s: MPI_Comm_size call failed\n", progname); + + if (ret == MPI_ERR_COMM) + HDfprintf(stderr, "invalid MPI communicator\n"); + else + HDfprintf(stderr, "invalid argument\n"); + + exit_value = EXIT_FAILURE; + goto finish; + } + + ret = MPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank_g); + + if (ret != MPI_SUCCESS) { + HDfprintf(stderr, "%s: MPI_Comm_rank call failed\n", progname); + + if (ret == MPI_ERR_COMM) + HDfprintf(stderr, "invalid MPI communicator\n"); + else + HDfprintf(stderr, "invalid argument\n"); + + exit_value = EXIT_FAILURE; + goto finish; + } + + pio_comm_g = MPI_COMM_WORLD; + + h5_set_info_object(); + opts = parse_command_line(argc, argv); + + if (!opts) { + exit_value = EXIT_FAILURE; + goto finish; + } + + if (opts->output_file) { + if ((output = HDfopen(opts->output_file, "w")) == NULL) { + HDfprintf(stderr, "%s: cannot open output file\n", progname); + perror(opts->output_file); + goto finish; + } + } + + if ((pio_debug_level == 0 && comm_world_rank_g == 0) || pio_debug_level > 0) + report_parameters(opts); + + run_test_loop(opts); + +finish: + MPI_Finalize(); + free(opts); + return exit_value; +} + +off_t +squareo(off_t x) +{ + return x * x; +} + +/* + * Function: run_test_loop + * Purpose: Run the I/O tests. Write the results to OUTPUT. + * + * - The slowest changing part of the test is the number of + * processors to use. For each loop iteration, we divide that + * number by 2 and rerun the test. + * + * - The second slowest is what type of IO API to perform. We have + * three choices: POSIXIO, MPI-IO, and PHDF5. + * + * - Then we change the size of the buffer. This information is + * inferred from the number of datasets to create and the number + * of integers to put into each dataset. The backend code figures + * this out. + * + * Return: Nothing + * Programmer: Bill Wendling, 30. October 2001 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static void +run_test_loop(struct options *opts) +{ + parameters parms; + int num_procs; + int doing_pio; /* if this process is doing PIO */ + + parms.num_files = opts->num_files; + parms.num_dsets = opts->num_dsets; + parms.num_iters = opts->num_iters; + parms.blk_size = opts->blk_size; + parms.interleaved = opts->interleaved; + parms.collective = opts->collective; + parms.dim2d = opts->dim2d; + parms.h5_align = (hsize_t)opts->h5_alignment; + parms.h5_thresh = (hsize_t)opts->h5_threshold; + parms.h5_use_chunks = opts->h5_use_chunks; + parms.h5_write_only = opts->h5_write_only; + parms.verify = opts->verify; + + /* start with max_num_procs and decrement it by half for each loop. */ + /* if performance needs restart, fewer processes may be needed. */ + for (num_procs = opts->max_num_procs; num_procs >= opts->min_num_procs; num_procs >>= 1) { + register size_t buf_size; + + parms.num_procs = num_procs; + + if (create_comm_world(parms.num_procs, &doing_pio) != SUCCESS) { + /* do something harsh */ + } + + /* only processes doing PIO will run the tests */ + if (doing_pio) { + output_report("Number of processors = %ld\n", parms.num_procs); + + /* multiply the xfer buffer size by 2 for each loop iteration */ + for (buf_size = opts->min_xfer_size; buf_size <= opts->max_xfer_size; buf_size <<= 1) { + parms.buf_size = buf_size; + + if (parms.dim2d) { + parms.num_bytes = squareo(opts->num_bpp * parms.num_procs); + if (parms.interleaved) + output_report("Transfer Buffer Size: %ldx%ld bytes, File size: %.2f MB\n", buf_size, + opts->blk_size, + ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); + else + output_report("Transfer Buffer Size: %ldx%ld bytes, File size: %.2f MB\n", + opts->blk_size, buf_size, + ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); + + print_indent(1); + output_report(" # of files: %ld, # of datasets: %ld, dataset size: %.2fx%.2f KB\n", + parms.num_files, parms.num_dsets, + (double)(opts->num_bpp * parms.num_procs) / ONE_KB, + (double)(opts->num_bpp * parms.num_procs) / ONE_KB); + } + else { + parms.num_bytes = (off_t)opts->num_bpp * parms.num_procs; + output_report("Transfer Buffer Size: %ld bytes, File size: %.2f MB\n", buf_size, + ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); + + print_indent(1); + output_report(" # of files: %ld, # of datasets: %ld, dataset size: %.2f MB\n", + parms.num_files, parms.num_dsets, + (double)(opts->num_bpp * parms.num_procs) / ONE_MB); + } + + if (opts->io_types & PIO_POSIX) + run_test(POSIXIO, parms, opts); + + if (opts->io_types & PIO_MPI) + run_test(MPIO, parms, opts); + + if (opts->io_types & PIO_HDF5) + run_test(PHDF5, parms, opts); + + /* Run the tests once if buf_size==0, but then break out */ + if (buf_size == 0) + break; + } + + if (destroy_comm_world() != SUCCESS) { + /* do something harsh */ + } + } + } +} + +/* + * Function: run_test + * Purpose: Inner loop call to actually run the I/O test. + * Return: Nothing + * Programmer: Bill Wendling, 18. December 2001 + * Modifications: + */ +static int +run_test(iotype iot, parameters parms, struct options *opts) +{ + results res; + register int i, ret_value = SUCCESS; + int comm_size; + off_t raw_size; + minmax * write_mpi_mm_table = NULL; + minmax * write_mm_table = NULL; + minmax * write_gross_mm_table = NULL; + minmax * write_raw_mm_table = NULL; + minmax * read_mpi_mm_table = NULL; + minmax * read_mm_table = NULL; + minmax * read_gross_mm_table = NULL; + minmax * read_raw_mm_table = NULL; + minmax * read_open_mm_table = NULL; + minmax * read_close_mm_table = NULL; + minmax * write_open_mm_table = NULL; + minmax * write_close_mm_table = NULL; + minmax write_mpi_mm = {0.0, 0.0, 0.0, 0}; + minmax write_mm = {0.0, 0.0, 0.0, 0}; + minmax write_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax write_raw_mm = {0.0, 0.0, 0.0, 0}; + minmax read_mpi_mm = {0.0, 0.0, 0.0, 0}; + minmax read_mm = {0.0, 0.0, 0.0, 0}; + minmax read_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax read_raw_mm = {0.0, 0.0, 0.0, 0}; + minmax read_open_mm = {0.0, 0.0, 0.0, 0}; + minmax read_close_mm = {0.0, 0.0, 0.0, 0}; + minmax write_open_mm = {0.0, 0.0, 0.0, 0}; + minmax write_close_mm = {0.0, 0.0, 0.0, 0}; + + raw_size = parms.num_files * (off_t)parms.num_dsets * (off_t)parms.num_bytes; + parms.io_type = iot; + print_indent(2); + output_report("IO API = "); + + switch (iot) { + case POSIXIO: + output_report("POSIX\n"); + break; + case MPIO: + output_report("MPIO\n"); + break; + case PHDF5: + output_report("PHDF5 (w/MPI-IO driver)\n"); + break; + default: + break; + } + + MPI_Comm_size(pio_comm_g, &comm_size); + + /* allocate space for tables minmax and that it is sufficient */ + /* to initialize all elements to zeros by calloc. */ + write_mpi_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_gross_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_raw_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_open_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_close_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + if (!parms.h5_write_only) { + read_mpi_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_gross_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_raw_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_open_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_close_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + } + + /* Do IO iteration times, collecting statistics each time */ + for (i = 0; i < parms.num_iters; ++i) { + double t; + + MPI_Barrier(pio_comm_g); + res = do_pio(parms); + + /* gather all of the "mpi write" times */ + t = io_time_get(res.timers, HDF5_MPI_WRITE); + get_minmax(&write_mpi_mm, t); + + write_mpi_mm_table[i] = write_mpi_mm; + + /* gather all of the "write" times */ + t = io_time_get(res.timers, HDF5_FINE_WRITE_FIXED_DIMS); + get_minmax(&write_mm, t); + + write_mm_table[i] = write_mm; + + /* gather all of the "write" times from open to close */ + t = io_time_get(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS); + get_minmax(&write_gross_mm, t); + + write_gross_mm_table[i] = write_gross_mm; + + /* gather all of the raw "write" times */ + t = io_time_get(res.timers, HDF5_RAW_WRITE_FIXED_DIMS); + get_minmax(&write_raw_mm, t); + + write_raw_mm_table[i] = write_raw_mm; + + /* gather all of the file open times (time from open to first write) */ + t = io_time_get(res.timers, HDF5_FILE_WRITE_OPEN); + get_minmax(&write_open_mm, t); + + write_open_mm_table[i] = write_open_mm; + + /* gather all of the file close times (time from last write to close) */ + t = io_time_get(res.timers, HDF5_FILE_WRITE_CLOSE); + get_minmax(&write_close_mm, t); + + write_close_mm_table[i] = write_close_mm; + + if (!parms.h5_write_only) { + /* gather all of the "mpi read" times */ + t = io_time_get(res.timers, HDF5_MPI_READ); + get_minmax(&read_mpi_mm, t); + + read_mpi_mm_table[i] = read_mpi_mm; + + /* gather all of the "read" times */ + t = io_time_get(res.timers, HDF5_FINE_READ_FIXED_DIMS); + get_minmax(&read_mm, t); + + read_mm_table[i] = read_mm; + + /* gather all of the "read" times from open to close */ + t = io_time_get(res.timers, HDF5_GROSS_READ_FIXED_DIMS); + get_minmax(&read_gross_mm, t); + + read_gross_mm_table[i] = read_gross_mm; + + /* gather all of the raw "read" times */ + t = io_time_get(res.timers, HDF5_RAW_READ_FIXED_DIMS); + get_minmax(&read_raw_mm, t); + + read_raw_mm_table[i] = read_raw_mm; + + /* gather all of the file open times (time from open to first read) */ + t = io_time_get(res.timers, HDF5_FILE_READ_OPEN); + get_minmax(&read_open_mm, t); + + read_open_mm_table[i] = read_open_mm; + + /* gather all of the file close times (time from last read to close) */ + t = io_time_get(res.timers, HDF5_FILE_READ_CLOSE); + get_minmax(&read_close_mm, t); + + read_close_mm_table[i] = read_close_mm; + } + + io_time_destroy(res.timers); + } + + /* + * Show various statistics + */ + /* Write statistics */ + /* Print the raw data throughput if desired */ + if (opts->print_raw) { + /* accumulate and output the max, min, and average "raw write" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Raw Data Write details:\n"); + output_all_info(write_raw_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); + } /* end if */ + + /* show mpi write statics */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("MPI Write details:\n"); + output_all_info(write_mpi_mm_table, parms.num_iters, 4); + } + + /* We don't currently output the MPI write results */ + + /* accumulate and output the max, min, and average "write" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write details:\n"); + output_all_info(write_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Write", write_mm_table, parms.num_iters, raw_size); + + /* accumulate and output the max, min, and average "gross write" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write Open-Close details:\n"); + output_all_info(write_gross_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Write Open-Close", write_gross_mm_table, parms.num_iters, raw_size); + + if (opts->print_times) { + output_times(opts, "Write File Open", write_open_mm_table, parms.num_iters); + output_times(opts, "Write File Close", write_close_mm_table, parms.num_iters); + } + + /* Print out time from open to first write */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write file open details:\n"); + output_all_info(write_open_mm_table, parms.num_iters, 4); + } + + /* Print out time from last write to close */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write file close details:\n"); + output_all_info(write_close_mm_table, parms.num_iters, 4); + } + + if (!parms.h5_write_only) { + /* Read statistics */ + /* Print the raw data throughput if desired */ + if (opts->print_raw) { + /* accumulate and output the max, min, and average "raw read" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Raw Data Read details:\n"); + output_all_info(read_raw_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); + } /* end if */ + + /* show mpi read statics */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("MPI Read details:\n"); + output_all_info(read_mpi_mm_table, parms.num_iters, 4); + } + + /* We don't currently output the MPI read results */ + + /* accumulate and output the max, min, and average "read" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read details:\n"); + output_all_info(read_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Read", read_mm_table, parms.num_iters, raw_size); + + /* accumulate and output the max, min, and average "gross read" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read Open-Close details:\n"); + output_all_info(read_gross_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Read Open-Close", read_gross_mm_table, parms.num_iters, raw_size); + + if (opts->print_times) { + output_times(opts, "Read File Open", read_open_mm_table, parms.num_iters); + output_times(opts, "Read File Close", read_close_mm_table, parms.num_iters); + } + + /* Print out time from open to first read */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read file open details:\n"); + output_all_info(read_open_mm_table, parms.num_iters, 4); + } + + /* Print out time from last read to close */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read file close details:\n"); + output_all_info(read_close_mm_table, parms.num_iters, 4); + } + } + + /* clean up our mess */ + free(write_mpi_mm_table); + free(write_mm_table); + free(write_gross_mm_table); + free(write_raw_mm_table); + free(write_open_mm_table); + free(write_close_mm_table); + + if (!parms.h5_write_only) { + free(read_mpi_mm_table); + free(read_mm_table); + free(read_gross_mm_table); + free(read_raw_mm_table); + free(read_open_mm_table); + free(read_close_mm_table); + } + + return ret_value; +} + +/* + * Function: output_all_info + * Purpose: + * Return: Nothing + * Programmer: Bill Wendling, 29. January 2002 + * Modifications: + */ +static void +output_all_info(minmax *mm, int count, int indent_level) +{ + int i; + + for (i = 0; i < count; ++i) { + print_indent(indent_level); + output_report("Iteration %d:\n", i + 1); + print_indent(indent_level + 1); + output_report("Minimum Time: %.2fs\n", mm[i].min); + print_indent(indent_level + 1); + output_report("Maximum Time: %.2fs\n", mm[i].max); + } +} + +/* + * Function: h5_set_info_object + * Purpose: Process environment variables setting to set up MPI Info + * object. + * Return: 0 if all is fine; otherwise non-zero. + * Programmer: Albert Cheng, 2002/05/21. + * Modifications: + * Bill Wendling, 2002/05/31 + * Modified so that the HDF5_MPI_INFO environment variable can + * be a semicolon separated list of "key=value" pairings. Most + * of the code is to remove any whitespaces which might be + * surrounding the "key=value" pairs. + */ +int +h5_set_info_object(void) +{ + char *envp; /* environment pointer */ + int ret_value = 0; + + /* handle any MPI INFO hints via $HDF5_MPI_INFO */ + if ((envp = HDgetenv("HDF5_MPI_INFO")) != NULL) { + char *next, *valp; + + valp = envp = next = HDstrdup(envp); + + if (!valp) + return 0; + + /* create an INFO object if not created yet */ + if (h5_io_info_g == MPI_INFO_NULL) + MPI_Info_create(&h5_io_info_g); + + do { + size_t len; + char * key_val, *endp, *namep; + + if (*valp == ';') + valp++; + + /* copy key/value pair into temporary buffer */ + len = strcspn(valp, ";"); + next = &valp[len]; + key_val = (char *)HDcalloc(1, len + 1); + + /* increment the next pointer past the terminating semicolon */ + if (*next == ';') + ++next; + + namep = HDstrncpy(key_val, valp, len); + + /* pass up any beginning whitespaces */ + while (*namep && (*namep == ' ' || *namep == '\t')) + namep++; + + if (!*namep) + continue; /* was all white space, so move to next k/v pair */ + + /* eat up any ending white spaces */ + endp = &namep[HDstrlen(namep) - 1]; + + while (endp && (*endp == ' ' || *endp == '\t')) + *endp-- = '\0'; + + /* find the '=' */ + valp = HDstrchr(namep, '='); + + if (valp != NULL) { /* it's a valid key/value pairing */ + char *tmp_val = valp + 1; + + /* change '=' to \0, move valp down one */ + *valp-- = '\0'; + + /* eat up ending whitespace on the "key" part */ + while (*valp == ' ' || *valp == '\t') + *valp-- = '\0'; + + valp = tmp_val; + + /* eat up beginning whitespace on the "value" part */ + while (*valp == ' ' || *valp == '\t') + *valp++ = '\0'; + + /* actually set the darned thing */ + if (MPI_SUCCESS != MPI_Info_set(h5_io_info_g, namep, valp)) { + HDprintf("MPI_Info_set failed\n"); + ret_value = -1; + } + } + + valp = next; + HDfree(key_val); + } while (next && *next); + + HDfree(envp); + } + + return ret_value; +} + +/* + * Function: h5_dump_info_object + * Purpose: Display content of an MPI Info object + * Return: void + * Programmer: Albert Cheng 2002/05/21 + * Modifications: + */ +void +h5_dump_info_object(MPI_Info info) +{ + char key[MPI_MAX_INFO_KEY + 1]; + char value[MPI_MAX_INFO_VAL + 1]; + int flag; + int i, nkeys; + + HDprintf("Dumping MPI Info Object (up to %d bytes per item):\n", MPI_MAX_INFO_VAL); + if (info == MPI_INFO_NULL) { + HDprintf("object is MPI_INFO_NULL\n"); + } + else { + MPI_Info_get_nkeys(info, &nkeys); + HDprintf("object has %d items\n", nkeys); + for (i = 0; i < nkeys; i++) { + MPI_Info_get_nthkey(info, i, key); + MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); + HDprintf("%s=%s\n", key, value); + } + } +} + +/* + * Function: get_minmax + * Purpose: Gather all the min, max and total of val. + * Return: Nothing + * Programmer: Bill Wendling, 21. December 2001 + * Modifications: + * Use MPI_Allreduce to do it. -akc, 2002/01/11 + */ +static void +get_minmax(minmax *mm, double val) +{ + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + MPI_Comm_size(pio_comm_g, &mm->num); + + MPI_Allreduce(&val, &mm->max, 1, MPI_DOUBLE, MPI_MAX, pio_comm_g); + MPI_Allreduce(&val, &mm->min, 1, MPI_DOUBLE, MPI_MIN, pio_comm_g); + MPI_Allreduce(&val, &mm->sum, 1, MPI_DOUBLE, MPI_SUM, pio_comm_g); +} + +/* + * Function: accumulate_minmax_stuff + * Purpose: Accumulate the minimum, maximum, and average of the times + * across all processes. + * Return: TOTAL_MM - the total of all of these. + * Programmer: Bill Wendling, 21. December 2001 + * Modifications: + * Changed to use seconds instead of MB/s - QAK, 5/9/02 + */ +static minmax +accumulate_minmax_stuff(minmax *mm, int count) +{ + int i; + minmax total_mm; + + total_mm.sum = 0.0f; + total_mm.max = -DBL_MAX; + total_mm.min = DBL_MAX; + total_mm.num = count; + + for (i = 0; i < count; ++i) { + double m = mm[i].max; + + total_mm.sum += m; + + if (m < total_mm.min) + total_mm.min = m; + + if (m > total_mm.max) + total_mm.max = m; + } + + return total_mm; +} + +/* + * Function: create_comm_world + * Purpose: Create an MPI Comm world and store it in pio_comm_g, which + * is a global variable. + * Return: SUCCESS on success. + * FAIL otherwise. + * Programmer: Bill Wendling, 19. December 2001 + * Modifications: + */ +static int +create_comm_world(int num_procs, int *doing_pio) +{ + /* MPI variables */ + int mrc; /* return values */ + int color; /* for communicator creation */ + int myrank, nprocs; + + pio_comm_g = MPI_COMM_NULL; + + /* + * Create a sub communicator for this PIO run. Easier to use the first N + * processes. + */ + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + if (num_procs > nprocs) { + HDfprintf(stderr, "number of process(%d) must be <= number of processes in MPI_COMM_WORLD(%d)\n", + num_procs, nprocs); + goto error_done; + } + + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + color = (myrank < num_procs); + mrc = MPI_Comm_split(MPI_COMM_WORLD, color, myrank, &pio_comm_g); + + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI_Comm_split failed\n"); + goto error_done; + } + + if (!color) { + /* not involved in this run */ + mrc = destroy_comm_world(); + goto done; + } + + /* determine the MPI rank in the PIO communicator */ + MPI_Comm_size(pio_comm_g, &pio_mpi_nprocs_g); + MPI_Comm_rank(pio_comm_g, &pio_mpi_rank_g); + +done: + *doing_pio = color; + return SUCCESS; + +error_done: + destroy_comm_world(); + return FAIL; +} + +/* + * Function: destroy_comm_world + * Purpose: Destroy the created MPI Comm world which is stored in the + * pio_comm_g global variable. + * Return: SUCCESS on success. + * FAIL otherwise. + * Programmer: Bill Wendling, 19. December 2001 + * Modifications: + */ +static int +destroy_comm_world(void) +{ + int mrc = SUCCESS; /* return code */ + + /* release MPI resources */ + if (pio_comm_g != MPI_COMM_NULL) + mrc = (MPI_Comm_free(&pio_comm_g) == MPI_SUCCESS ? SUCCESS : FAIL); + + return mrc; +} + +/* + * Function: output_results + * Purpose: Print information about the time & bandwidth for a given + * minmax & # of iterations. + * Return: Nothing + * Programmer: Quincey Koziol, 9. May 2002 + * Modifications: + */ +static void +output_results(const struct options *opts, const char *name, minmax *table, int table_size, off_t data_size) +{ + minmax total_mm; + + total_mm = accumulate_minmax_stuff(table, table_size); + + print_indent(3); + output_report("%s (%d iteration(s)):\n", name, table_size); + + /* Note: The maximum throughput uses the minimum amount of time & vice versa */ + + print_indent(4); + output_report("Maximum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.min)); + if (opts->print_times) + output_report(" (%7.3f s)\n", total_mm.min); + else + output_report("\n"); + + print_indent(4); + output_report("Average Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.sum / total_mm.num)); + if (opts->print_times) + output_report(" (%7.3f s)\n", (total_mm.sum / total_mm.num)); + else + output_report("\n"); + + print_indent(4); + output_report("Minimum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.max)); + if (opts->print_times) + output_report(" (%7.3f s)\n", total_mm.max); + else + output_report("\n"); +} + +static void +output_times(const struct options *opts, const char *name, minmax *table, int table_size) +{ + minmax total_mm; + + total_mm = accumulate_minmax_stuff(table, table_size); + + print_indent(3); + output_report("%s (%d iteration(s)):\n", name, table_size); + + /* Note: The maximum throughput uses the minimum amount of time & vice versa */ + + print_indent(4); + output_report("Minimum Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, (total_mm.min)); + + print_indent(4); + output_report("Average Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, + (total_mm.sum / total_mm.num)); + + print_indent(4); + output_report("Maximum Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, (total_mm.max)); +} + +/* + * Function: output_report + * Purpose: Print a line of the report. Only do so if I'm the 0 process. + * Return: Nothing + * Programmer: Bill Wendling, 19. December 2001 + * Modifications: + */ +static void +output_report(const char *fmt, ...) +{ + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + + if (myrank == 0) { + va_list ap; + + HDva_start(ap, fmt); + HDvfprintf(output, fmt, ap); + HDva_end(ap); + } +} + +/* + * Function: print_indent + * Purpose: Print spaces to indent a new line of text for pretty printing + * things. + * Return: Nothing + * Programmer: Bill Wendling, 29. October 2001 + * Modifications: + */ +static void +print_indent(register int indent) +{ + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + + if (myrank == 0) { + indent *= TAB_SPACE; + + for (; indent > 0; --indent) + HDfputc(' ', output); + } +} + +static void +recover_size_and_print(long long val, const char *end) +{ + if (val >= ONE_KB && (val % ONE_KB) == 0) { + if (val >= ONE_MB && (val % ONE_MB) == 0) { + if (val >= ONE_GB && (val % ONE_GB) == 0) + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "GB%s", + val / ONE_GB, end); + else + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "MB%s", + val / ONE_MB, end); + } + else { + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "KB%s", + val / ONE_KB, end); + } + } + else { + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "%s", + val, end); + } +} + +static void +print_io_api(long io_types) +{ + if (io_types & PIO_POSIX) + HDfprintf(output, "posix "); + if (io_types & PIO_MPI) + HDfprintf(output, "mpiio "); + if (io_types & PIO_HDF5) + HDfprintf(output, "phdf5 "); + HDfprintf(output, "\n"); +} + +static void +report_parameters(struct options *opts) +{ + int rank = comm_world_rank_g; + + print_version("HDF5 Library"); /* print library version */ + HDfprintf(output, "rank %d: ==== Parameters ====\n", rank); + + HDfprintf(output, "rank %d: IO API=", rank); + print_io_api(opts->io_types); + + HDfprintf(output, "rank %d: Number of files=%ld\n", rank, opts->num_files); + HDfprintf(output, "rank %d: Number of datasets=%ld\n", rank, opts->num_dsets); + HDfprintf(output, "rank %d: Number of iterations=%d\n", rank, opts->num_iters); + HDfprintf(output, "rank %d: Number of processes=%d:%d\n", rank, opts->min_num_procs, opts->max_num_procs); + + if (opts->dim2d) { + HDfprintf(output, "rank %d: Number of bytes per process per dataset=", rank); + recover_size_and_print((long long)(opts->num_bpp * opts->num_bpp * opts->min_num_procs), ":"); + recover_size_and_print((long long)(opts->num_bpp * opts->num_bpp * opts->max_num_procs), "\n"); + + HDfprintf(output, "rank %d: Size of dataset(s)=", rank); + recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), "x"); + recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), ":"); + recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "x"); + recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "\n"); + + HDfprintf(output, "rank %d: File size=", rank); + recover_size_and_print((long long)(squareo(opts->num_bpp * opts->min_num_procs) * opts->num_dsets), + ":"); + recover_size_and_print((long long)(squareo(opts->num_bpp * opts->max_num_procs) * opts->num_dsets), + "\n"); + + HDfprintf(output, "rank %d: Transfer buffer size=", rank); + if (opts->interleaved) { + recover_size_and_print((long long)opts->min_xfer_size, "x"); + recover_size_and_print((long long)opts->blk_size, ":"); + recover_size_and_print((long long)opts->max_xfer_size, "x"); + recover_size_and_print((long long)opts->blk_size, "\n"); + } + else { + recover_size_and_print((long long)opts->blk_size, "x"); + recover_size_and_print((long long)opts->min_xfer_size, ":"); + recover_size_and_print((long long)opts->blk_size, "x"); + recover_size_and_print((long long)opts->max_xfer_size, "\n"); + } + HDfprintf(output, "rank %d: Block size=", rank); + recover_size_and_print((long long)opts->blk_size, "x"); + recover_size_and_print((long long)opts->blk_size, "\n"); + } + else { + HDfprintf(output, "rank %d: Number of bytes per process per dataset=", rank); + recover_size_and_print((long long)opts->num_bpp, "\n"); + + HDfprintf(output, "rank %d: Size of dataset(s)=", rank); + recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), ":"); + recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "\n"); + + HDfprintf(output, "rank %d: File size=", rank); + recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs * opts->num_dsets), ":"); + recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs * opts->num_dsets), "\n"); + + HDfprintf(output, "rank %d: Transfer buffer size=", rank); + recover_size_and_print((long long)opts->min_xfer_size, ":"); + recover_size_and_print((long long)opts->max_xfer_size, "\n"); + HDfprintf(output, "rank %d: Block size=", rank); + recover_size_and_print((long long)opts->blk_size, "\n"); + } + + HDfprintf(output, "rank %d: Block Pattern in Dataset=", rank); + if (opts->interleaved) + HDfprintf(output, "Interleaved\n"); + else + HDfprintf(output, "Contiguous\n"); + + HDfprintf(output, "rank %d: I/O Method for MPI and HDF5=", rank); + if (opts->collective) + HDfprintf(output, "Collective\n"); + else + HDfprintf(output, "Independent\n"); + + HDfprintf(output, "rank %d: Geometry=", rank); + if (opts->dim2d) + HDfprintf(output, "2D\n"); + else + HDfprintf(output, "1D\n"); + + HDfprintf(output, "rank %d: VFL used for HDF5 I/O=%s\n", rank, "MPI-IO driver"); + + HDfprintf(output, "rank %d: Data storage method in HDF5=", rank); + if (opts->h5_use_chunks) + HDfprintf(output, "Chunked\n"); + else + HDfprintf(output, "Contiguous\n"); + + { + char *prefix = HDgetenv("HDF5_PARAPREFIX"); + + HDfprintf(output, "rank %d: Env HDF5_PARAPREFIX=%s\n", rank, (prefix ? prefix : "not set")); + } + + HDfprintf(output, "rank %d: ", rank); + h5_dump_info_object(h5_io_info_g); + + HDfprintf(output, "rank %d: ==== End of Parameters ====\n", rank); + HDfprintf(output, "\n"); +} + +/* + * Function: parse_command_line + * Purpose: Parse the command line options and return a STRUCT OPTIONS + * structure which will need to be freed by the calling function. + * Return: Pointer to an OPTIONS structure + * Programmer: Bill Wendling, 31. October 2001 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static struct options * +parse_command_line(int argc, char *argv[]) +{ + register int opt; + struct options *cl_opts; + + cl_opts = (struct options *)malloc(sizeof(struct options)); + + cl_opts->output_file = NULL; + cl_opts->io_types = 0; /* will set default after parsing options */ + cl_opts->num_dsets = 1; + cl_opts->num_files = 1; + cl_opts->num_bpp = 0; + cl_opts->num_iters = 1; + cl_opts->max_num_procs = comm_world_nprocs_g; + cl_opts->min_num_procs = 1; + cl_opts->max_xfer_size = 0; + cl_opts->min_xfer_size = 0; + cl_opts->blk_size = 0; + cl_opts->interleaved = 0; /* Default to contiguous blocks in dataset */ + cl_opts->collective = 0; /* Default to independent I/O access */ + cl_opts->dim2d = 0; /* Default to 1D */ + cl_opts->print_times = FALSE; /* Printing times is off by default */ + cl_opts->print_raw = FALSE; /* Printing raw data throughput is off by default */ + cl_opts->h5_alignment = 1; /* No alignment for HDF5 objects by default */ + cl_opts->h5_threshold = 1; /* No threshold for aligning HDF5 objects by default */ + cl_opts->h5_use_chunks = FALSE; /* Don't chunk the HDF5 dataset by default */ + cl_opts->h5_write_only = FALSE; /* Do both read and write by default */ + cl_opts->verify = FALSE; /* No Verify data correctness by default */ + + while ((opt = H5_get_option(argc, (const char **)argv, s_opts, l_opts)) != EOF) { + switch ((char)opt) { + case 'a': + cl_opts->h5_alignment = parse_size_directive(H5_optarg); + break; + case 'A': { + const char *end = H5_optarg; + + while (end && *end != '\0') { + char buf[10]; + int i; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (isalnum(*end) && i < 10) + buf[i++] = *end; + + if (!HDstrcasecmp(buf, "phdf5")) { + cl_opts->io_types |= PIO_HDF5; + } + else if (!HDstrcasecmp(buf, "mpiio")) { + cl_opts->io_types |= PIO_MPI; + } + else if (!HDstrcasecmp(buf, "posix")) { + cl_opts->io_types |= PIO_POSIX; + } + else { + HDfprintf(stderr, "pio_perf: invalid --api option %s\n", buf); + HDexit(EXIT_FAILURE); + } + + if (*end == '\0') + break; + + end++; + } + } + + break; +#if 0 + case 'b': + /* the future "binary" option */ + break; +#endif /* 0 */ + case 'B': + cl_opts->blk_size = (size_t)parse_size_directive(H5_optarg); + break; + case 'c': + /* Turn on chunked HDF5 dataset creation */ + cl_opts->h5_use_chunks = TRUE; + break; + case 'C': + cl_opts->collective = 1; + break; + case 'd': + cl_opts->num_dsets = atoi(H5_optarg); + break; + case 'D': { + const char *end = H5_optarg; + + while (end && *end != '\0') { + char buf[10]; + int i; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + if (HDstrlen(buf) > 1 || HDisdigit(buf[0])) { + size_t j; + + for (j = 0; j < 10 && buf[j] != '\0'; ++j) + if (!isdigit(buf[j])) { + HDfprintf(stderr, "pio_perf: invalid --debug option %s\n", buf); + HDexit(EXIT_FAILURE); + } + + pio_debug_level = atoi(buf); + + if (pio_debug_level > 4) + pio_debug_level = 4; + else if (pio_debug_level < 0) + pio_debug_level = 0; + } + else { + switch (*buf) { + case 'r': + /* Turn on raw data throughput info */ + cl_opts->print_raw = TRUE; + break; + case 't': + /* Turn on time printing */ + cl_opts->print_times = TRUE; + break; + case 'v': + /* Turn on verify data correctness*/ + cl_opts->verify = TRUE; + break; + default: + HDfprintf(stderr, "pio_perf: invalid --debug option %s\n", buf); + HDexit(EXIT_FAILURE); + } + } + + if (*end == '\0') + break; + + end++; + } + } + + break; + case 'e': + cl_opts->num_bpp = parse_size_directive(H5_optarg); + break; + case 'F': + cl_opts->num_files = HDatoi(H5_optarg); + break; + case 'g': + cl_opts->dim2d = 1; + break; + case 'i': + cl_opts->num_iters = HDatoi(H5_optarg); + break; + case 'I': + cl_opts->interleaved = 1; + break; + case 'o': + cl_opts->output_file = H5_optarg; + break; + case 'p': + cl_opts->min_num_procs = HDatoi(H5_optarg); + break; + case 'P': + cl_opts->max_num_procs = HDatoi(H5_optarg); + break; + case 'T': + cl_opts->h5_threshold = parse_size_directive(H5_optarg); + break; + case 'w': + cl_opts->h5_write_only = TRUE; + break; + case 'x': + cl_opts->min_xfer_size = (size_t)parse_size_directive(H5_optarg); + break; + case 'X': + cl_opts->max_xfer_size = (size_t)parse_size_directive(H5_optarg); + break; + case 'h': + case '?': + default: + usage(progname); + HDfree(cl_opts); + return NULL; + } + } + + if (cl_opts->num_bpp == 0) { + if (cl_opts->dim2d == 0) + cl_opts->num_bpp = 256 * ONE_KB; + else + cl_opts->num_bpp = 8 * ONE_KB; + } + + if (cl_opts->max_xfer_size == 0) + cl_opts->max_xfer_size = (size_t)cl_opts->num_bpp; + + if (cl_opts->min_xfer_size == 0) + cl_opts->min_xfer_size = (size_t)(cl_opts->num_bpp) / 2; + + if (cl_opts->blk_size == 0) + cl_opts->blk_size = (size_t)(cl_opts->num_bpp) / 2; + + /* set default if none specified yet */ + if (!cl_opts->io_types) + cl_opts->io_types = PIO_HDF5 | PIO_MPI | PIO_POSIX; /* run all API */ + + /* verify parameters sanity. Adjust if needed. */ + /* cap xfer_size with bytes per process */ + if (!cl_opts->dim2d) { + if (cl_opts->min_xfer_size > (size_t)cl_opts->num_bpp) + cl_opts->min_xfer_size = (size_t)cl_opts->num_bpp; + if (cl_opts->max_xfer_size > (size_t)cl_opts->num_bpp) + cl_opts->max_xfer_size = (size_t)cl_opts->num_bpp; + } + if (cl_opts->min_xfer_size > cl_opts->max_xfer_size) + cl_opts->min_xfer_size = cl_opts->max_xfer_size; + if (cl_opts->blk_size > (size_t)cl_opts->num_bpp) + cl_opts->blk_size = (size_t)cl_opts->num_bpp; + /* check range of number of processes */ + if (cl_opts->min_num_procs <= 0) + cl_opts->min_num_procs = 1; + if (cl_opts->max_num_procs <= 0) + cl_opts->max_num_procs = 1; + if (cl_opts->min_num_procs > cl_opts->max_num_procs) + cl_opts->min_num_procs = cl_opts->max_num_procs; + /* check iteration */ + if (cl_opts->num_iters <= 0) + cl_opts->num_iters = 1; + + return cl_opts; +} + +/* + * Function: parse_size_directive + * Purpose: Parse the size directive passed on the commandline. The size + * directive is an integer followed by a size indicator: + * + * K, k - Kilobyte + * M, m - Megabyte + * G, g - Gigabyte + * + * Return: The size as a off_t because this is related to file size. + * If an unknown size indicator is used, then the program will + * exit with EXIT_FAILURE as the return value. + * Programmer: Bill Wendling, 18. December 2001 + * Modifications: + */ +static off_t +parse_size_directive(const char *size) +{ + off_t s; + char *endptr; + + s = HDstrtol(size, &endptr, 10); + + if (endptr && *endptr) { + while (*endptr != '\0' && (*endptr == ' ' || *endptr == '\t')) + ++endptr; + + switch (*endptr) { + case 'K': + case 'k': + s *= ONE_KB; + break; + case 'M': + case 'm': + s *= ONE_MB; + break; + case 'G': + case 'g': + s *= ONE_GB; + break; + default: + HDfprintf(stderr, "Illegal size specifier '%c'\n", *endptr); + HDexit(EXIT_FAILURE); + } + } + + return s; +} + +/* + * Function: usage + * Purpose: Print a usage message and then exit. + * Return: Nothing + * Programmer: Bill Wendling, 31. October 2001 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static void +usage(const char *prog) +{ + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + + if (myrank == 0) { + print_version(prog); + HDprintf("usage: %s [OPTIONS]\n", prog); + HDprintf(" OPTIONS\n"); + HDprintf(" -h, --help Print a usage message and exit\n"); + HDprintf(" -a S, --align=S Alignment of objects in HDF5 file [default: 1]\n"); + HDprintf(" -A AL, --api=AL Which APIs to test [default: all of them]\n"); +#if 0 + HDprintf(" -b, --binary The elusive binary option\n"); +#endif /* 0 */ + HDprintf(" -B S, --block-size=S Block size within transfer buffer\n"); + HDprintf(" (see below for description)\n"); + HDprintf(" [default: half the number of bytes per process\n"); + HDprintf(" per dataset]\n"); + HDprintf(" -c, --chunk Create HDF5 datasets using chunked storage\n"); + HDprintf(" [default: contiguous storage]\n"); + HDprintf(" -C, --collective Use collective I/O for MPI and HDF5 APIs\n"); + HDprintf(" [default: independent I/O)\n"); + HDprintf(" -d N, --num-dsets=N Number of datasets per file [default: 1]\n"); + HDprintf(" -D DL, --debug=DL Indicate the debugging level\n"); + HDprintf(" [default: no debugging]\n"); + HDprintf(" -e S, --num-bytes=S Number of bytes per process per dataset\n"); + HDprintf(" (see below for description)\n"); + HDprintf(" [default: 256K for 1D, 8K for 2D]\n"); + HDprintf(" -F N, --num-files=N Number of files [default: 1]\n"); + HDprintf(" -g, --geometry Use 2D geometry [default: 1D geometry]\n"); + HDprintf(" -i N, --num-iterations=N Number of iterations to perform [default: 1]\n"); + HDprintf(" -I, --interleaved Interleaved access pattern\n"); + HDprintf(" (see below for example)\n"); + HDprintf(" [default: Contiguous access pattern]\n"); + HDprintf(" -o F, --output=F Output raw data into file F [default: none]\n"); + HDprintf(" -p N, --min-num-processes=N Minimum number of processes to use [default: 1]\n"); + HDprintf(" -P N, --max-num-processes=N Maximum number of processes to use\n"); + HDprintf(" [default: all MPI_COMM_WORLD processes ]\n"); + HDprintf(" -T S, --threshold=S Threshold for alignment of objects in HDF5 file\n"); + HDprintf(" [default: 1]\n"); + HDprintf(" -w, --write-only Perform write tests not the read tests\n"); + HDprintf(" -x S, --min-xfer-size=S Minimum transfer buffer size\n"); + HDprintf(" (see below for description)\n"); + HDprintf(" [default: half the number of bytes per process\n"); + HDprintf(" per dataset]\n"); + HDprintf(" -X S, --max-xfer-size=S Maximum transfer buffer size\n"); + HDprintf(" [default: the number of bytes per process per\n"); + HDprintf(" dataset]\n"); + HDprintf("\n"); + HDprintf(" F - is a filename.\n"); + HDprintf(" N - is an integer >=0.\n"); + HDprintf(" S - is a size specifier, an integer >=0 followed by a size indicator:\n"); + HDprintf(" K - Kilobyte (%d)\n", ONE_KB); + HDprintf(" M - Megabyte (%d)\n", ONE_MB); + HDprintf(" G - Gigabyte (%d)\n", ONE_GB); + HDprintf("\n"); + HDprintf(" Example: '37M' is 37 megabytes or %d bytes\n", 37 * ONE_MB); + HDprintf("\n"); + HDprintf(" AL - is an API list. Valid values are:\n"); + HDprintf(" phdf5 - Parallel HDF5\n"); + HDprintf(" mpiio - MPI-I/O\n"); + HDprintf(" posix - POSIX\n"); + HDprintf("\n"); + HDprintf(" Example: --api=mpiio,phdf5\n"); + HDprintf("\n"); + HDprintf(" Dataset size:\n"); + HDprintf(" Depending on the selected geometry, each test dataset is either a linear\n"); + HDprintf(" array of size bytes-per-process * num-processes, or a square array of size\n"); + HDprintf(" (bytes-per-process * num-processes) x (bytes-per-process * num-processes).\n"); + HDprintf("\n"); + HDprintf(" Block size vs. Transfer buffer size:\n"); + HDprintf(" buffer-size controls the size of the memory buffer, which is broken into\n"); + HDprintf(" blocks and written to the file. Depending on the selected geometry, each\n"); + HDprintf(" block can be a linear array of size block-size or a square array of size\n"); + HDprintf(" block-size x block-size. The arrangement in which blocks are written is\n"); + HDprintf(" determined by the access pattern.\n"); + HDprintf("\n"); + HDprintf(" In 1D geometry, the transfer buffer is a linear array of size buffer-size.\n"); + HDprintf(" In 2D geometry, it is a rectangular array of size block-size x buffer-size\n"); + HDprintf(" or buffer-size x block-size if interleaved pattern is selected.\n"); + HDprintf("\n"); + HDprintf(" Interleaved and Contiguous patterns in 1D geometry:\n"); + HDprintf(" When contiguous access pattern is chosen, the dataset is evenly divided\n"); + HDprintf(" into num-processes regions and each process writes data to its own region.\n"); + HDprintf(" When interleaved blocks are written to a dataset, space for the first\n"); + HDprintf(" block of the first process is allocated in the dataset, then space is\n"); + HDprintf(" allocated for the first block of the second process, etc. until space is\n"); + HDprintf(" allocated for the first block of each process, then space is allocated for\n"); + HDprintf(" the second block of the first process, the second block of the second\n"); + HDprintf(" process, etc.\n"); + HDprintf("\n"); + HDprintf(" For example, with a 3 process run, 512KB bytes-per-process, 256KB transfer\n"); + HDprintf(" buffer size, and 64KB block size, each process must issue 2 transfer\n"); + HDprintf(" requests to complete access to the dataset.\n"); + HDprintf(" Contiguous blocks of the first transfer request are written like so:\n"); + HDprintf(" 1111----2222----3333----\n"); + HDprintf(" Interleaved blocks of the first transfer request are written like so:\n"); + HDprintf(" 123123123123------------\n"); + HDprintf(" The actual number of I/O operations involved in a transfer request\n"); + HDprintf(" depends on the access pattern and communication mode.\n"); + HDprintf(" When using independent I/O with interleaved pattern, each process\n"); + HDprintf(" performs 4 small non-contiguous I/O operations per transfer request.\n"); + HDprintf(" If collective I/O is turned on, the combined content of the buffers of\n"); + HDprintf(" the 3 processes will be written using one collective I/O operation\n"); + HDprintf(" per transfer request.\n"); + HDprintf("\n"); + HDprintf(" For information about access patterns in 2D geometry, please refer to the\n"); + HDprintf(" HDF5 Reference Manual.\n"); + HDprintf("\n"); + HDprintf(" DL - is a list of debugging flags. Valid values are:\n"); + HDprintf(" 1 - Minimal\n"); + HDprintf(" 2 - Not quite everything\n"); + HDprintf(" 3 - Everything\n"); + HDprintf(" 4 - The kitchen sink\n"); + HDprintf(" r - Raw data I/O throughput information\n"); + HDprintf(" t - Times as well as throughputs\n"); + HDprintf(" v - Verify data correctness\n"); + HDprintf("\n"); + HDprintf(" Example: --debug=2,r,t\n"); + HDprintf("\n"); + HDprintf(" Environment variables:\n"); + HDprintf(" HDF5_NOCLEANUP Do not remove data files if set [default remove]\n"); + HDprintf(" HDF5_MPI_INFO MPI INFO object key=value separated by ;\n"); + HDprintf(" HDF5_PARAPREFIX Paralllel data files prefix\n"); + fflush(stdout); + } /* end if */ +} /* end usage() */ + +#else /* H5_HAVE_PARALLEL */ + +/* + * Function: main + * Purpose: Dummy main() function for if HDF5 was configured without + * parallel stuff. + * Return: EXIT_SUCCESS + * Programmer: Bill Wendling, 14. November 2001 + */ +int +main(void) +{ + HDprintf("No parallel IO performance because parallel is not configured\n"); + return EXIT_SUCCESS; +} /* end main */ + +#endif /* !H5_HAVE_PARALLEL */ diff --git a/tools/src/h5perf/pio_perf.h b/tools/src/h5perf/pio_perf.h new file mode 100644 index 0000000..8924c20 --- /dev/null +++ b/tools/src/h5perf/pio_perf.h @@ -0,0 +1,109 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef PIO_PERF_H +#define PIO_PERF_H + +#ifndef STANDALONE +#include "io_timer.h" +#include "H5private.h" +#include "h5tools.h" +#include "h5tools_utils.h" +#else +#include "io_timer.h" +#include "pio_standalone.h" +#endif + +#ifdef H5_HAVE_PARALLEL +extern MPI_Info h5_io_info_g; /* MPI INFO object for IO */ +#endif + +#ifdef H5_HAVE_PARALLEL +int h5_set_info_object(void); +void h5_dump_info_object(MPI_Info info); +#endif + +/* setup the dataset no fill option if this is v1.5 or more */ +#if H5_VERS_MAJOR > 1 || H5_VERS_MINOR > 4 +#define H5_HAVE_NOFILL 1 +#endif + +typedef enum iotype_ { + POSIXIO, + MPIO, + PHDF5 + /*NUM_TYPES*/ +} iotype; + +typedef struct parameters_ { + iotype io_type; /* The type of IO test to perform */ + int num_procs; /* Maximum number of processes to use */ + long num_files; /* Number of files to create */ + long num_dsets; /* Number of datasets to create */ + off_t num_bytes; /* Number of bytes in each dset */ + int num_iters; /* Number of times to loop doing the IO */ + size_t buf_size; /* Buffer size */ + size_t blk_size; /* Block size */ + unsigned interleaved; /* Interleaved vs. contiguous blocks */ + unsigned collective; /* Collective vs. independent I/O */ + unsigned dim2d; /* 1D vs. 2D */ + hsize_t h5_align; /* HDF5 object alignment */ + hsize_t h5_thresh; /* HDF5 object alignment threshold */ + int h5_use_chunks; /* Make HDF5 dataset chunked */ + int h5_write_only; /* Perform the write tests only */ + int verify; /* Verify data correctness */ +} parameters; + +typedef struct results_ { + herr_t ret_code; + io_time_t *timers; +} results; + +#ifndef SUCCESS +#define SUCCESS 0 +#endif /* !SUCCESS */ + +#ifndef FAIL +#define FAIL -1 +#endif /* !FAIL */ + +extern FILE * output; /* output file */ +extern io_time_t *timer_g; /* timer: global for stub functions */ +extern int comm_world_rank_g; /* my rank in MPI_COMM_RANK */ +extern int comm_world_nprocs_g; /* num. of processes of MPI_COMM_WORLD */ +extern MPI_Comm pio_comm_g; /* Communicator to run the PIO */ +extern int pio_mpi_rank_g; /* MPI rank of pio_comm_g */ +extern int pio_mpi_nprocs_g; /* number of processes of pio_comm_g */ +extern int pio_debug_level; /* The debug level: + * 0 - Off + * 1 - Minimal + * 2 - Some more + * 3 - Maximal + * 4 - Even More Debugging (timer stuff) + */ + +#define HDprint_rank(f) /* print rank in MPI_COMM_WORLD */ HDfprintf(f, "%d: ", comm_world_rank_g); +#define HDprint_size(f) /* print size of MPI_COMM_WORLD */ HDfprintf(f, "%d", comm_world_nprocs_g); +#define HDprint_rank_size(f) /* print rank/size of MPI_COMM_WORLD */ \ + HDfprintf(f, "%d/%d: ", comm_world_rank_g, comm_world_nprocs_g); + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +extern results do_pio(parameters param); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* PIO_PERF_H */ diff --git a/tools/src/h5perf/sio_engine.c b/tools/src/h5perf/sio_engine.c new file mode 100644 index 0000000..1af2318 --- /dev/null +++ b/tools/src/h5perf/sio_engine.c @@ -0,0 +1,1328 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Author: Christian Chilan, April 2008 + */ + +#include "hdf5.h" + +#include +#include +#include +#include + +#ifdef H5_HAVE_UNISTD_H +#include +#include +#endif + +#ifdef H5_HAVE_SYS_STAT_H +#include +#endif + +#include "sio_perf.h" + +/* Macro definitions */ + +/* sizes of various items. these sizes won't change during program execution */ +#define ELMT_H5_TYPE H5T_NATIVE_UCHAR + +#define GOTOERROR(errcode) \ + { \ + ret_code = errcode; \ + goto done; \ + } +#define ERRMSG(mesg) \ + { \ + HDfprintf(stderr, "*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ + } + +/* verify: if val is false (0), print mesg. */ +#define VRFY(val, mesg) \ + do { \ + if (!val) { \ + ERRMSG(mesg); \ + GOTOERROR(FAIL); \ + } \ + } while (0) + +/* POSIX I/O macros */ +#ifdef H5_HAVE_WIN32_API +/* Can't link against the library, so this test will use the older, non-Unicode + * _open() call on Windows. + */ +#define HDopen(S, F, ...) _open(S, F | _O_BINARY, __VA_ARGS__) +#endif /* H5_HAVE_WIN32_API */ +#define POSIXCREATE(fn) HDopen(fn, O_CREAT | O_TRUNC | O_RDWR, 0600) +#define POSIXOPEN(fn, F) HDopen(fn, F, 0600) +#define POSIXCLOSE(F) HDclose(F) +#define POSIXSEEK(F, L) HDlseek(F, L, SEEK_SET) +#define POSIXWRITE(F, B, S) HDwrite(F, B, S) +#define POSIXREAD(F, B, S) HDread(F, B, S) + +enum { SIO_CREATE = 1, SIO_WRITE = 2, SIO_READ = 4 }; + +/* Global variables */ +static int clean_file_g = -1; /*whether to cleanup temporary test */ +/*files. -1 is not defined; */ +/*0 is no cleanup; 1 is do cleanup */ + +/* the different types of file descriptors we can expect */ +typedef union { + int posixfd; /* POSIX file handle*/ + hid_t h5fd; /* HDF5 file */ +} file_descr; + +/* local functions */ +static char * sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size, + parameters *param); +static herr_t do_write(results *res, file_descr *fd, parameters *parms, void *buffer); +static herr_t do_read(results *res, file_descr *fd, parameters *parms, void *buffer); +static herr_t dset_write(int local_dim, file_descr *fd, parameters *parms, void *buffer); +static herr_t posix_buffer_write(int local_dim, file_descr *fd, parameters *parms, void *buffer); +static herr_t dset_read(int localrank, file_descr *fd, parameters *parms, void *buffer, const char *buffer2); +static herr_t posix_buffer_read(int local_dim, file_descr *fd, parameters *parms, void *buffer); +static herr_t do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags); +hid_t set_vfd(parameters *param); +static herr_t do_fclose(iotype iot, file_descr *fd); +static void do_cleanupfile(iotype iot, char *fname); + +/* global variables */ +static HDoff_t offset[MAX_DIMS]; /* dataset size in bytes */ +static size_t buf_offset[MAX_DIMS]; /* dataset size in bytes */ +static int order[MAX_DIMS]; /* dimension access order */ +static size_t linear_buf_size; /* linear buffer size */ +static int cont_dim; /* lowest dimension for contiguous POSIX + access */ +static size_t cont_size; /* size of contiguous POSIX access */ +static hid_t fapl; /* file access list */ +static unsigned char *buf_p; /* buffer pointer */ +static const char * multi_letters = "msbrglo"; /* string for multi driver */ + +/* HDF5 global variables */ +static hsize_t h5count[MAX_DIMS]; /*selection count */ +static hssize_t h5offset[MAX_DIMS]; /* Selection offset within dataspace */ +static hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ +static hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ +static hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ +static hid_t h5dcpl = H5I_INVALID_HID; /* Dataset creation property list */ +static hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ + +/* + * Function: do_sio + * Purpose: SIO Engine where IO are executed. + * Return: results + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ +void +do_sio(parameters param, results *res) +{ + char * buffer = NULL; /*data buffer pointer */ + size_t buf_size[MAX_DIMS]; /* general buffer size in bytes */ + file_descr fd; /* file handles */ + iotype iot; /* API type */ + char base_name[256]; /* test file base name */ + /* return codes */ + herr_t ret_code = 0; /*return code */ + + char fname[FILENAME_MAX]; /* test file name */ + int i; + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + + /* Sanity check parameters */ + + /* IO type */ + iot = param.io_type; + + switch (iot) { + case POSIXIO: + fd.posixfd = -1; + res->timers = io_time_new(SYS_CLOCK); + break; + case HDF5: + fd.h5fd = -1; + res->timers = io_time_new(SYS_CLOCK); + break; + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + GOTOERROR(FAIL); + } + + linear_buf_size = 1; + + for (i = 0; i < param.rank; i++) { + buf_size[i] = param.buf_size[i]; + order[i] = param.order[i]; + linear_buf_size *= buf_size[i]; + buf_offset[i] = 0; + offset[i] = 0; + + /* Validate transfer buffer size */ + if (param.buf_size[i] <= 0) { + HDfprintf(stderr, "Transfer buffer size[%d] (%zu) must be > 0\n", i, buf_size[i]); + GOTOERROR(FAIL); + } + + if ((param.dset_size[i] % param.buf_size[i]) != 0) { + HDfprintf(stderr, + "Dataset size[%d] (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " + "trasfer buffer size[%d] (%zu)\n", + param.rank, (long long)param.dset_size[i], param.rank, param.buf_size[i]); + GOTOERROR(FAIL); + } + } + + /* Allocate transfer buffer */ + if ((buffer = (char *)malloc(linear_buf_size)) == NULL) { + HDfprintf(stderr, "malloc for transfer buffer size (%zu) failed\n", linear_buf_size); + GOTOERROR(FAIL); + } + + if (sio_debug_level >= 4) + + /* output all of the times for all iterations */ + HDfprintf(output, "Timer details:\n"); + + /* + * Write performance measurement + */ + /* Open file for write */ + + HDstrcpy(base_name, "#sio_tmp"); + sio_create_filename(iot, base_name, fname, sizeof(fname), ¶m); + + if (sio_debug_level > 0) + HDfprintf(output, "data filename=%s\n", fname); + + io_time_set(res->timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTART); + hrc = do_fopen(¶m, fname, &fd, SIO_CREATE | SIO_WRITE); + VRFY((hrc == SUCCESS), "do_fopen failed"); + + io_time_set(res->timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTART); + hrc = do_write(res, &fd, ¶m, buffer); + io_time_set(res->timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_write failed"); + + /* Close file for write */ + hrc = do_fclose(iot, &fd); + io_time_set(res->timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_fclose failed"); + + if (!param.h5_write_only) { + /* + * Read performance measurement + */ + + /* Open file for read */ + io_time_set(res->timers, HDF5_GROSS_READ_FIXED_DIMS, TSTART); + hrc = do_fopen(¶m, fname, &fd, SIO_READ); + VRFY((hrc == SUCCESS), "do_fopen failed"); + + io_time_set(res->timers, HDF5_FINE_READ_FIXED_DIMS, TSTART); + hrc = do_read(res, &fd, ¶m, buffer); + io_time_set(res->timers, HDF5_FINE_READ_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_read failed"); + + /* Close file for read */ + hrc = do_fclose(iot, &fd); + + io_time_set(res->timers, HDF5_GROSS_READ_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_fclose failed"); + } + + do_cleanupfile(iot, fname); + +done: + /* clean up */ + /* release HDF5 objects */ + + /* close any opened files */ + /* no remove(fname) because that should have happened normally. */ + switch (iot) { + case POSIXIO: + if (fd.posixfd != -1) + hrc = do_fclose(iot, &fd); + break; + case HDF5: + if (fd.h5fd != -1) + hrc = do_fclose(iot, &fd); + break; + default: + /* unknown request */ + HDassert(0 && "Unknown IO type"); + break; + } + + /* release generic resources */ + if (buffer) + free(buffer); + + res->ret_code = ret_code; +} + +/* + * Function: sio_create_filename + * Purpose: Create a new filename to write to. Determine the correct + * suffix to append to the filename by the type of I/O we're + * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if + * USER or LOGIN are specified in the environment. + * Return: Pointer to filename or NULL + * Programmer: Bill Wendling, 21. November 2001 + * Modifications: Support for file drivers. Christian Chilan, April, 2008 + */ +static char * +sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size, parameters *param) +{ + const char *prefix, *suffix = ""; + char * ptr, last = '\0'; + size_t i, j; + vfdtype vfd; + vfd = param->vfd; + + if (!base_name || !fullname || size < 1) + return NULL; + + memset(fullname, 0, size); + + switch (iot) { + case POSIXIO: + suffix = ".posix"; + break; + case HDF5: + suffix = ".h5"; + if (vfd == family) + suffix = "%05d.h5"; + else if (vfd == multi) + suffix = NULL; + break; + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + HDassert(0 && "Unknown IO type"); + break; + } + + /* First use the environment variable and then try the constant */ + prefix = HDgetenv("HDF5_PREFIX"); + +#ifdef HDF5_PREFIX + if (!prefix) + prefix = HDF5_PREFIX; +#endif /* HDF5_PREFIX */ + + /* Prepend the prefix value to the base name */ + if (prefix && *prefix) { + /* If the prefix specifies the HDF5_PREFIX directory, then + * default to using the "/tmp/$USER" or "/tmp/$LOGIN" + * directory instead. */ + register char *user, *login, *subdir; + + user = HDgetenv("USER"); + login = HDgetenv("LOGIN"); + subdir = (user ? user : login); + + if (subdir) { + for (i = 0; i < size - 1 && prefix[i]; i++) + fullname[i] = prefix[i]; + + fullname[i++] = '/'; + + for (j = 0; i < size && subdir[j]; i++, j++) + fullname[i] = subdir[j]; + } + else { + /* We didn't append the prefix yet */ + HDstrncpy(fullname, prefix, size); + fullname[size - 1] = '\0'; + } + + if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) { + /* Append the base_name with a slash first. Multiple slashes are + * handled below. */ + h5_stat_t buf; + + if (HDstat(fullname, &buf) < 0) + /* The directory doesn't exist just yet */ + if (HDmkdir(fullname, 0755) < 0 && errno != EEXIST) { + /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. + * Default to PREFIX's original prefix value. */ + HDstrcpy(fullname, prefix); + } + + HDstrcat(fullname, "/"); + HDstrcat(fullname, base_name); + } + else { + /* Buffer is too small */ + return NULL; + } + } + else if (strlen(base_name) >= size) { + /* Buffer is too small */ + return NULL; + } + else { + HDstrcpy(fullname, base_name); + } + + /* Append a suffix */ + if (suffix) { + if (HDstrlen(fullname) + HDstrlen(suffix) >= size) + return NULL; + + HDstrcat(fullname, suffix); + } + + /* Remove any double slashes in the filename */ + for (ptr = fullname, i = j = 0; ptr && (i < size); i++, ptr++) { + if (*ptr != '/' || last != '/') + fullname[j++] = *ptr; + + last = *ptr; + } + + return fullname; +} + +/* + * Function: do_write + * Purpose: Write the required amount of data to the file. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ +static herr_t +do_write(results *res, file_descr *fd, parameters *parms, void *buffer) +{ + int ret_code = SUCCESS; + char dname[64]; + int i; + size_t u; + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + hsize_t h5dims[MAX_DIMS]; /*dataset dim sizes */ + hsize_t h5chunk[MAX_DIMS]; /*dataset dim sizes */ + hsize_t h5block[MAX_DIMS]; /*dataspace selection */ + hsize_t h5stride[MAX_DIMS]; /*selection stride */ + hsize_t h5start[MAX_DIMS]; /*selection start */ + hsize_t h5maxdims[MAX_DIMS]; + int rank; /*rank of dataset */ + + /* Prepare buffer for verifying data */ + /* if (parms->verify) + memset(buffer,1,linear_buf_size); */ + + buf_p = (unsigned char *)buffer; + + for (u = 0; u < linear_buf_size; u++) + buf_p[u] = u % 128; + + rank = parms->rank; + + for (i = 0; i < rank; i++) + h5offset[i] = offset[i] = 0; + + /* I/O Access specific setup */ + switch (parms->io_type) { + case POSIXIO: + + /* determine lowest dimension for contiguous POSIX access */ + cont_dim = rank; + + for (i = rank - 1; i >= 0; i--) { + if (parms->buf_size[i] == parms->dset_size[i]) + cont_dim = i; + else + break; + } + + /* determine size of the contiguous POSIX access */ + cont_size = (!cont_dim) ? 1 : parms->buf_size[cont_dim - 1]; + for (i = cont_dim; i < rank; i++) + cont_size *= parms->buf_size[i]; + + break; + + case HDF5: /* HDF5 setup */ + + for (i = 0; i < rank; i++) { + h5dims[i] = parms->dset_size[i]; + h5start[i] = 0; + h5stride[i] = 1; + h5block[i] = 1; + h5count[i] = parms->buf_size[i]; + h5chunk[i] = parms->chk_size[i]; + h5maxdims[i] = H5S_UNLIMITED; + } + + if (parms->h5_use_chunks && parms->h5_extendable) { + h5dset_space_id = H5Screate_simple(rank, h5count, h5maxdims); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + } + else { + h5dset_space_id = H5Screate_simple(rank, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + } + + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + + /* Create the memory dataspace that corresponds to the xfer buffer */ + h5mem_space_id = H5Screate_simple(rank, h5count, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + + /* Create the dataset transfer property list */ + h5dxpl = H5Pcreate(H5P_DATASET_XFER); + if (h5dxpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + break; + + default: + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + GOTOERROR(FAIL); + break; + } /* end switch */ + + /* create dataset */ + switch (parms->io_type) { + case POSIXIO: + break; + + case HDF5: + h5dcpl = H5Pcreate(H5P_DATASET_CREATE); + + if (h5dcpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + if (parms->h5_use_chunks) { + /* Set the chunk size to be the same as the buffer size */ + hrc = H5Pset_chunk(h5dcpl, rank, h5chunk); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + + HDsprintf(dname, "Dataset_%ld", (unsigned long)parms->num_bytes); + h5ds_id = + H5Dcreate2(fd->h5fd, dname, ELMT_H5_TYPE, h5dset_space_id, H5P_DEFAULT, h5dcpl, H5P_DEFAULT); + + if (h5ds_id < 0) { + HDfprintf(stderr, "HDF5 Dataset Create failed\n"); + GOTOERROR(FAIL); + } + + hrc = H5Pclose(h5dcpl); + /* verifying the close of the dcpl */ + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Close failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + GOTOERROR(FAIL); + break; + } + + /* Start "raw data" write timer */ + io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART); + + /* Perform write */ + hrc = dset_write(rank - 1, fd, parms, buffer); + + if (hrc < 0) { + HDfprintf(stderr, "Error in dataset write\n"); + GOTOERROR(FAIL); + } + + /* Stop "raw data" write timer */ + io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP); + + /* Calculate write time */ + + /* Close dataset. Only HDF5 needs to do an explicit close. */ + if (parms->io_type == HDF5) { + hrc = H5Dclose(h5ds_id); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Close failed\n"); + GOTOERROR(FAIL); + } + + h5ds_id = H5I_INVALID_HID; + } /* end if */ + +done: + + /* release HDF5 objects */ + if (h5dset_space_id != -1) { + hrc = H5Sclose(h5dset_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); + ret_code = FAIL; + } + else { + h5dset_space_id = H5I_INVALID_HID; + } + } + + if (h5mem_space_id != -1) { + hrc = H5Sclose(h5mem_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); + ret_code = FAIL; + } + else { + h5mem_space_id = H5I_INVALID_HID; + } + } + + if (h5dxpl != -1) { + hrc = H5Pclose(h5dxpl); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); + ret_code = FAIL; + } + else { + h5dxpl = H5I_INVALID_HID; + } + } + + return ret_code; +} + +/* + * Function: dset_write + * Purpose: Write buffer into the dataset. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ +static herr_t +dset_write(int local_dim, file_descr *fd, parameters *parms, void *buffer) +{ + int cur_dim = order[local_dim] - 1; + int ret_code = SUCCESS; + int k; + hsize_t dims[MAX_DIMS], maxdims[MAX_DIMS]; + hsize_t i; + int j; + herr_t hrc; + + /* iterates according to the dimensions in order array */ + for (i = 0; i < parms->dset_size[cur_dim]; i += parms->buf_size[cur_dim]) { + + h5offset[cur_dim] = (hssize_t)i; + offset[cur_dim] = (HDoff_t)i; + + if (local_dim > 0) { + + dset_write(local_dim - 1, fd, parms, buffer); + } + else { + + switch (parms->io_type) { + + case POSIXIO: + /* initialize POSIX offset in the buffer */ + for (j = 0; j < parms->rank; j++) + buf_offset[j] = 0; + buf_p = (unsigned char *)buffer; + /* write POSIX buffer */ + posix_buffer_write(0, fd, parms, buffer); + break; + + case HDF5: + /* if dimensions are extendable, extend them as needed during access */ + if (parms->h5_use_chunks && parms->h5_extendable) { + + hrc = H5Sget_simple_extent_dims(h5dset_space_id, dims, maxdims); + VRFY((hrc >= 0), "H5Sget_simple_extent_dims"); + + for (k = 0; k < parms->rank; k++) { + + HDassert(h5offset[k] >= 0); + if (dims[k] <= (hsize_t)h5offset[k]) { + dims[k] = dims[k] + h5count[k]; + hrc = H5Sset_extent_simple(h5dset_space_id, parms->rank, dims, maxdims); + VRFY((hrc >= 0), "H5Sset_extent_simple"); + hrc = H5Dset_extent(h5ds_id, dims); + VRFY((hrc >= 0), "H5Dextend"); + } + } + } + /* applies offset */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Write the buffer out */ + hrc = H5Sget_simple_extent_dims(h5dset_space_id, dims, maxdims); + hrc = H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dwrite"); + + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + HDassert(0 && "Unknown IO type"); + break; + } /* switch (parms->io_type) */ + } + } +done: + return ret_code; +} + +/* + * Function: posix_buffer_write + * Purpose: Write buffer into the POSIX file considering contiguity. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ + +static herr_t +posix_buffer_write(int local_dim, file_descr *fd, parameters *parms, void *buffer) +{ + int ret_code = SUCCESS; + + /* if dimension is not contiguous, call recursively */ + if (local_dim < parms->rank - 1 && local_dim != cont_dim) { + size_t u; + + for (u = 0; u < parms->buf_size[local_dim]; u++) { + buf_offset[local_dim] = u; + posix_buffer_write(local_dim + 1, fd, parms, buffer); + + /* if next dimension is cont_dim, it will fill out the buffer + traversing the entire dimension local_dim without the need + of performing iteration */ + if (local_dim + 1 == cont_dim) + break; + } + /* otherwise, perform contiguous POSIX access */ + } + else { + HDoff_t d_offset; + HDoff_t linear_dset_offset = 0; + int i, j, rc; + + buf_offset[local_dim] = 0; + + /* determine offset in the buffer */ + for (i = 0; i < parms->rank; i++) { + d_offset = 1; + + for (j = i + 1; j < parms->rank; j++) + d_offset *= (HDoff_t)parms->dset_size[j]; + + linear_dset_offset += (offset[i] + (HDoff_t)buf_offset[i]) * d_offset; + } + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, linear_dset_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + /* check if all bytes are written */ + rc = ((ssize_t)cont_size == POSIXWRITE(fd->posixfd, buf_p, cont_size)); + VRFY((rc != 0), "POSIXWRITE"); + + /* Advance location in buffer */ + buf_p += cont_size; + } + +done: + return ret_code; +} + +/* + * Function: do_read + * Purpose: Read the required amount of data to the file. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ +static herr_t +do_read(results *res, file_descr *fd, parameters *parms, void *buffer) +{ + char * buffer2 = NULL; /* Buffer for data verification */ + int ret_code = SUCCESS; + char dname[64]; + int i; + size_t u; + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + hsize_t h5dims[MAX_DIMS]; /*dataset dim sizes */ + hsize_t h5block[MAX_DIMS]; /*dataspace selection */ + hsize_t h5stride[MAX_DIMS]; /*selection stride */ + hsize_t h5start[MAX_DIMS]; /*selection start */ + int rank; + + /* Allocate data verification buffer */ + if (NULL == (buffer2 = (char *)malloc(linear_buf_size))) { + HDfprintf(stderr, "malloc for data verification buffer size (%zu) failed\n", linear_buf_size); + GOTOERROR(FAIL); + } /* end if */ + + /* Prepare buffer for verifying data */ + for (u = 0; u < linear_buf_size; u++) + buffer2[u] = (char)(u % 128); + + rank = parms->rank; + for (i = 0; i < rank; i++) + h5offset[i] = offset[i] = 0; + + /* I/O Access specific setup */ + switch (parms->io_type) { + case POSIXIO: + cont_dim = rank; + + for (i = rank - 1; i >= 0; i--) { + if (parms->buf_size[i] == parms->dset_size[i]) + cont_dim = i; + else + break; + } + cont_size = (!cont_dim) ? 1 : parms->buf_size[cont_dim - 1]; + for (i = cont_dim; i < rank; i++) + cont_size *= parms->buf_size[i]; + + break; + + case HDF5: /* HDF5 setup */ + for (i = 0; i < rank; i++) { + h5dims[i] = parms->dset_size[i]; + h5start[i] = 0; + h5stride[i] = 1; + h5block[i] = 1; + h5count[i] = parms->buf_size[i]; + } + + h5dset_space_id = H5Screate_simple(rank, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + + /* Create the memory dataspace that corresponds to the xfer buffer */ + h5mem_space_id = H5Screate_simple(rank, h5count, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + + /* Create the dataset transfer property list */ + h5dxpl = H5Pcreate(H5P_DATASET_XFER); + if (h5dxpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + GOTOERROR(FAIL); + break; + } /* end switch */ + + /* create dataset */ + switch (parms->io_type) { + case POSIXIO: + break; + + case HDF5: + HDsprintf(dname, "Dataset_%ld", (long)parms->num_bytes); + h5ds_id = H5Dopen2(fd->h5fd, dname, H5P_DEFAULT); + if (h5ds_id < 0) { + HDfprintf(stderr, "HDF5 Dataset open failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + GOTOERROR(FAIL); + break; + } /* end switch */ + + /* Start "raw data" read timer */ + io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTART); + hrc = dset_read(rank - 1, fd, parms, buffer, buffer2); + + if (hrc < 0) { + HDfprintf(stderr, "Error in dataset read\n"); + GOTOERROR(FAIL); + } + + /* Stop "raw data" read timer */ + io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTOP); + + /* Calculate read time */ + + /* Close dataset. Only HDF5 needs to do an explicit close. */ + if (parms->io_type == HDF5) { + hrc = H5Dclose(h5ds_id); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Close failed\n"); + GOTOERROR(FAIL); + } + + h5ds_id = H5I_INVALID_HID; + } /* end if */ + +done: + + /* release HDF5 objects */ + if (h5dset_space_id != -1) { + hrc = H5Sclose(h5dset_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); + ret_code = FAIL; + } + else { + h5dset_space_id = H5I_INVALID_HID; + } + } + + if (h5mem_space_id != -1) { + hrc = H5Sclose(h5mem_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); + ret_code = FAIL; + } + else { + h5mem_space_id = H5I_INVALID_HID; + } + } + + if (h5dxpl != -1) { + hrc = H5Pclose(h5dxpl); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); + ret_code = FAIL; + } + else { + h5dxpl = H5I_INVALID_HID; + } + } + + /* release generic resources */ + if (buffer2) + free(buffer2); + + return ret_code; +} + +/* + * Function: dset_read + * Purpose: Read buffer into the dataset. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ + +static herr_t +dset_read(int local_dim, file_descr *fd, parameters *parms, void *buffer, const char *buffer2) +{ + int cur_dim = order[local_dim] - 1; + hsize_t i; + int j; + herr_t hrc; + int ret_code = SUCCESS; + + /* iterate on the current dimension */ + for (i = 0; i < parms->dset_size[cur_dim]; i += parms->buf_size[cur_dim]) { + + h5offset[cur_dim] = (hssize_t)i; + offset[cur_dim] = (HDoff_t)i; + + /* if traverse in order array is incomplete, recurse */ + if (local_dim > 0) { + + ret_code = dset_read(local_dim - 1, fd, parms, buffer, buffer2); + + /* otherwise, write buffer into dataset */ + } + else { + + switch (parms->io_type) { + + case POSIXIO: + for (j = 0; j < parms->rank; j++) { + buf_offset[j] = 0; + } + buf_p = (unsigned char *)buffer; + posix_buffer_read(0, fd, parms, buffer); + break; + + case HDF5: + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + /* Read the buffer out */ + hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dread"); + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + HDassert(0 && "Unknown IO type"); + break; + } /* switch (parms->io_type) */ + } + } +done: + return ret_code; +} + +/* + * Function: posix_buffer_read + * Purpose: Read buffer into the POSIX file considering contiguity. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ + +static herr_t +posix_buffer_read(int local_dim, file_descr *fd, parameters *parms, void *buffer) +{ + int ret_code = SUCCESS; + + /* if local dimension is not contiguous, recurse */ + if (local_dim < parms->rank - 1 && local_dim != cont_dim) { + size_t u; + + for (u = 0; u < parms->buf_size[local_dim]; u++) { + buf_offset[local_dim] = u; + ret_code = posix_buffer_read(local_dim + 1, fd, parms, buffer); + if (local_dim + 1 == cont_dim) + break; + } + /* otherwise, perform contiguous POSIX access */ + } + else { + HDoff_t d_offset; + HDoff_t linear_dset_offset = 0; + int i, j, rc; + + buf_offset[local_dim] = 0; + /* determine offset in buffer */ + for (i = 0; i < parms->rank; i++) { + d_offset = 1; + + for (j = i + 1; j < parms->rank; j++) + d_offset *= (HDoff_t)parms->dset_size[j]; + + linear_dset_offset += (offset[i] + (HDoff_t)buf_offset[i]) * d_offset; + } + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, linear_dset_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + /* check if all bytes are read */ + rc = ((ssize_t)cont_size == POSIXREAD(fd->posixfd, buf_p, cont_size)); + VRFY((rc != 0), "POSIXREAD"); + + /* Advance location in buffer */ + buf_p += cont_size; + } +done: + return ret_code; +} + +/* + * Function: do_fopen + * Purpose: Open the specified file. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: Support for file drivers, Christian Chilan, April, 2008 + */ +static herr_t +do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) +{ + int ret_code = SUCCESS; + hid_t fcpl; + + switch (param->io_type) { + case POSIXIO: + if (flags & (SIO_CREATE | SIO_WRITE)) + fd->posixfd = POSIXCREATE(fname); + else + fd->posixfd = POSIXOPEN(fname, O_RDONLY); + + if (fd->posixfd < 0) { + HDfprintf(stderr, "POSIX File Open failed(%s)\n", fname); + GOTOERROR(FAIL); + } + + break; + + case HDF5: + + fapl = set_vfd(param); + + if (fapl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + fcpl = H5Pcreate(H5P_FILE_CREATE); + if (param->page_size) { + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1); + H5Pset_file_space_page_size(fcpl, param->page_size); + if (param->page_buffer_size) + H5Pset_page_buffer_size(fapl, param->page_buffer_size, 0, 0); + } + + /* create the parallel file */ + if (flags & (SIO_CREATE | SIO_WRITE)) { + fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, fcpl, fapl); + } + else { + fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, fapl); + } + + if (fd->h5fd < 0) { + HDfprintf(stderr, "HDF5 File Create failed(%s)\n", fname); + GOTOERROR(FAIL); + } + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)param->io_type); + GOTOERROR(FAIL); + break; + } + +done: + return ret_code; +} + +/* + * Function: set_vfd + * Purpose: Sets file driver. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ + +hid_t +set_vfd(parameters *param) +{ + hid_t my_fapl = H5I_INVALID_HID; + vfdtype vfd; + + vfd = param->vfd; + + if ((my_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + return -1; + + if (vfd == sec2) { + /* Unix read() and write() system calls */ + if (H5Pset_fapl_sec2(my_fapl) < 0) + return -1; + } + else if (vfd == stdio) { + /* Standard C fread() and fwrite() system calls */ + if (H5Pset_fapl_stdio(my_fapl) < 0) + return -1; + } + else if (vfd == core) { + /* In-core temporary file with 1MB increment */ + if (H5Pset_fapl_core(my_fapl, (size_t)1024 * 1024, TRUE) < 0) + return -1; + } + else if (vfd == split) { + /* Split meta data and raw data each using default driver */ + if (H5Pset_fapl_split(my_fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0) + return -1; + } + else if (vfd == multi) { + /* Multi-file driver, general case of the split driver */ + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl[H5FD_MEM_NTYPES]; + const char *memb_name[H5FD_MEM_NTYPES]; + char sv[H5FD_MEM_NTYPES][1024]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + H5FD_mem_t mt; + + HDmemset(memb_map, 0, sizeof memb_map); + HDmemset(memb_fapl, 0, sizeof memb_fapl); + HDmemset(memb_name, 0, sizeof memb_name); + HDmemset(memb_addr, 0, sizeof memb_addr); + + HDassert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { + memb_fapl[mt] = H5P_DEFAULT; + HDsprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]); + memb_name[mt] = sv[mt]; + memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10); + } + + if (H5Pset_fapl_multi(my_fapl, memb_map, memb_fapl, memb_name, memb_addr, FALSE) < 0) { + return -1; + } + } + else if (vfd == family) { + hsize_t fam_size = 1 * 1024 * 1024; /*100 MB*/ + + /* Family of files, each 1MB and using the default driver */ + /* if ((val=HDstrtok(NULL, " \t\n\r"))) + fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024); */ + if (H5Pset_fapl_family(my_fapl, fam_size, H5P_DEFAULT) < 0) + return -1; + } + else if (vfd == direct) { +#ifdef H5_HAVE_DIRECT + /* Linux direct read() and write() system calls. Set memory boundary, file block size, + * and copy buffer size to the default values. */ + if (H5Pset_fapl_direct(my_fapl, 1024, 4096, 8 * 4096) < 0) + return -1; +#endif + } + else { + /* Unknown driver */ + return -1; + } + + return my_fapl; +} + +/* + * Function: do_fclose + * Purpose: Close the specified file descriptor. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: + */ +static herr_t +do_fclose(iotype iot, file_descr *fd /*out*/) +{ + herr_t ret_code = SUCCESS, hrc; + int rc = 0; + + switch (iot) { + case POSIXIO: + rc = POSIXCLOSE(fd->posixfd); + + if (rc != 0) { + HDfprintf(stderr, "POSIX File Close failed\n"); + GOTOERROR(FAIL); + } + + fd->posixfd = -1; + break; + + case HDF5: + hrc = H5Fclose(fd->h5fd); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 File Close failed\n"); + GOTOERROR(FAIL); + } + + fd->h5fd = -1; + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + GOTOERROR(FAIL); + break; + } + +done: + return ret_code; +} + +/* + * Function: do_cleanupfile + * Purpose: Cleanup temporary file unless HDF5_NOCLEANUP is set. + * Return: void + * Programmer: Albert Cheng 2001/12/12 + * Modifications: Support for file drivers. Christian Chilan, April, 2008 + */ +static void +do_cleanupfile(iotype iot, char *filename) +{ + char temp[2048]; + int j; + hid_t driver; + + if (clean_file_g == -1) + clean_file_g = (HDgetenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; + + if (clean_file_g) { + + switch (iot) { + case POSIXIO: + HDremove(filename); + break; + + case HDF5: + driver = H5Pget_driver(fapl); + + if (driver == H5FD_FAMILY) { + for (j = 0; /*void*/; j++) { + HDsnprintf(temp, sizeof temp, filename, j); + + if (HDaccess(temp, F_OK) < 0) + break; + + HDremove(temp); + } + } + else if (driver == H5FD_CORE) { + hbool_t backing; /* Whether the core file has backing store */ + + H5Pget_fapl_core(fapl, NULL, &backing); + + /* If the file was stored to disk with bacing store, remove it */ + if (backing) + HDremove(filename); + } + else if (driver == H5FD_MULTI) { + H5FD_mem_t mt; + assert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); + + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { + HDsnprintf(temp, sizeof temp, "%s-%c.h5", filename, multi_letters[mt]); + HDremove(temp); /*don't care if it fails*/ + } + } + else { + HDremove(filename); + } + H5Pclose(fapl); + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + HDassert(0 && "Unknown IO type"); + break; + } + } +} diff --git a/tools/src/h5perf/sio_perf.c b/tools/src/h5perf/sio_perf.c new file mode 100644 index 0000000..51a7825 --- /dev/null +++ b/tools/src/h5perf/sio_perf.c @@ -0,0 +1,1437 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Serial HDF5 Performance Testing Code + * -------------------------------------- + * + * Portable code to test performance on the different platforms we support. + * This is what the report should look like: + * + * nprocs = Max#Procs + * IO API = POSIXIO + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * + * IO API = HDF5 + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * + * . . . + * + */ + +/* system header files */ +#include +#include +#include + +#include "hdf5.h" + +/* our header files */ +#include "sio_perf.h" + +/* useful macros */ +#define TAB_SPACE 4 + +#define ONE_KB 1024 +#define ONE_MB (ONE_KB * ONE_KB) +#define ONE_GB (ONE_MB * ONE_KB) + +#define SIO_POSIX 0x1 +#define SIO_HDF5 0x4 + +/* report 0.0 in case t is zero too */ +#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL(t, 0.0) ? 0.0 : ((((double)bytes) / (double)ONE_MB) / (t))) + +#ifndef TRUE +#define TRUE 1 +#endif /* TRUE */ +#ifndef FALSE +#define FALSE (!TRUE) +#endif /* FALSE */ + +/* global variables */ +FILE *output; /* output file */ +int sio_debug_level = 0; /* The debug level: + * 0 - Off + * 1 - Minimal + * 2 - Some more + * 3 - Maximal + * 4 - Maximal & then some + */ + +/* local variables */ +static const char *progname = "h5perf_serial"; + +/* + * Command-line options: The user can specify short or long-named + * parameters. The long-named ones can be partially spelled. When + * adding more, make sure that they don't clash with each other. + */ + +/* + * It seems that only the options that accept additional information + * such as dataset size (-e) require the colon next to it. + */ +static const char * s_opts = "a:A:B:c:Cd:D:e:F:ghi:Imno:p:P:r:stT:v:wx:X:"; +static struct h5_long_options l_opts[] = {{"align", require_arg, 'a'}, + {"alig", require_arg, 'a'}, + {"ali", require_arg, 'a'}, + {"al", require_arg, 'a'}, + {"api", require_arg, 'A'}, + {"ap", require_arg, 'A'}, +#if 0 + /* a sighting of the elusive binary option */ + { "binary", no_arg, 'b' }, + { "binar", no_arg, 'b' }, + { "bina", no_arg, 'b' }, + { "bin", no_arg, 'b' }, + { "bi", no_arg, 'b' }, +#endif /* 0 */ + {"block-size", require_arg, 'B'}, + {"block-siz", require_arg, 'B'}, + {"block-si", require_arg, 'B'}, + {"block-s", require_arg, 'B'}, + {"block-", require_arg, 'B'}, + {"block", require_arg, 'B'}, + {"bloc", require_arg, 'B'}, + {"blo", require_arg, 'B'}, + {"bl", require_arg, 'B'}, + {"chunk", no_arg, 'c'}, + {"chun", no_arg, 'c'}, + {"chu", no_arg, 'c'}, + {"ch", no_arg, 'c'}, + {"collective", no_arg, 'C'}, + {"collectiv", no_arg, 'C'}, + {"collecti", no_arg, 'C'}, + {"collect", no_arg, 'C'}, + {"collec", no_arg, 'C'}, + {"colle", no_arg, 'C'}, + {"coll", no_arg, 'C'}, + {"col", no_arg, 'C'}, + {"co", no_arg, 'C'}, + {"debug", require_arg, 'D'}, + {"debu", require_arg, 'D'}, + {"deb", require_arg, 'D'}, + {"de", require_arg, 'D'}, + {"file-driver", require_arg, 'v'}, + {"file-drive", require_arg, 'v'}, + {"file-driv", require_arg, 'v'}, + {"file-dri", require_arg, 'v'}, + {"file-dr", require_arg, 'v'}, + {"file-d", require_arg, 'v'}, + {"file-", require_arg, 'v'}, + {"file", require_arg, 'v'}, + {"fil", require_arg, 'v'}, + {"fi", require_arg, 'v'}, + {"geometry", no_arg, 'g'}, + {"geometr", no_arg, 'g'}, + {"geomet", no_arg, 'g'}, + {"geome", no_arg, 'g'}, + {"geom", no_arg, 'g'}, + {"geo", no_arg, 'g'}, + {"ge", no_arg, 'g'}, + {"help", no_arg, 'h'}, + {"hel", no_arg, 'h'}, + {"he", no_arg, 'h'}, + {"interleaved", require_arg, 'I'}, + {"interleave", require_arg, 'I'}, + {"interleav", require_arg, 'I'}, + {"interlea", require_arg, 'I'}, + {"interle", require_arg, 'I'}, + {"interl", require_arg, 'I'}, + {"inter", require_arg, 'I'}, + {"inte", require_arg, 'I'}, + {"int", require_arg, 'I'}, + {"in", require_arg, 'I'}, + {"max-num-processes", require_arg, 'P'}, + {"max-num-processe", require_arg, 'P'}, + {"max-num-process", require_arg, 'P'}, + {"max-num-proces", require_arg, 'P'}, + {"max-num-proce", require_arg, 'P'}, + {"max-num-proc", require_arg, 'P'}, + {"max-num-pro", require_arg, 'P'}, + {"max-num-pr", require_arg, 'P'}, + {"max-num-p", require_arg, 'P'}, + {"min-num-processes", require_arg, 'p'}, + {"min-num-processe", require_arg, 'p'}, + {"min-num-process", require_arg, 'p'}, + {"min-num-proces", require_arg, 'p'}, + {"min-num-proce", require_arg, 'p'}, + {"min-num-proc", require_arg, 'p'}, + {"min-num-pro", require_arg, 'p'}, + {"min-num-pr", require_arg, 'p'}, + {"min-num-p", require_arg, 'p'}, + {"max-xfer-size", require_arg, 'X'}, + {"max-xfer-siz", require_arg, 'X'}, + {"max-xfer-si", require_arg, 'X'}, + {"max-xfer-s", require_arg, 'X'}, + {"max-xfer", require_arg, 'X'}, + {"max-xfe", require_arg, 'X'}, + {"max-xf", require_arg, 'X'}, + {"max-x", require_arg, 'X'}, + {"min-xfer-size", require_arg, 'x'}, + {"min-xfer-siz", require_arg, 'x'}, + {"min-xfer-si", require_arg, 'x'}, + {"min-xfer-s", require_arg, 'x'}, + {"min-xfer", require_arg, 'x'}, + {"min-xfe", require_arg, 'x'}, + {"min-xf", require_arg, 'x'}, + {"min-x", require_arg, 'x'}, + {"num-bytes", require_arg, 'e'}, + {"num-byte", require_arg, 'e'}, + {"num-byt", require_arg, 'e'}, + {"num-by", require_arg, 'e'}, + {"num-b", require_arg, 'e'}, + {"num-dsets", require_arg, 'd'}, + {"num-dset", require_arg, 'd'}, + {"num-dse", require_arg, 'd'}, + {"num-ds", require_arg, 'd'}, + {"num-d", require_arg, 'd'}, + {"num-files", require_arg, 'F'}, + {"num-file", require_arg, 'F'}, + {"num-fil", require_arg, 'F'}, + {"num-fi", require_arg, 'F'}, + {"num-f", require_arg, 'F'}, + {"num-iterations", require_arg, 'i'}, + {"num-iteration", require_arg, 'i'}, + {"num-iteratio", require_arg, 'i'}, + {"num-iterati", require_arg, 'i'}, + {"num-iterat", require_arg, 'i'}, + {"num-itera", require_arg, 'i'}, + {"num-iter", require_arg, 'i'}, + {"num-ite", require_arg, 'i'}, + {"num-it", require_arg, 'i'}, + {"num-i", require_arg, 'i'}, + {"order", require_arg, 'r'}, + {"orde", require_arg, 'r'}, + {"ord", require_arg, 'r'}, + {"or", require_arg, 'r'}, + {"output", require_arg, 'o'}, + {"outpu", require_arg, 'o'}, + {"outp", require_arg, 'o'}, + {"out", require_arg, 'o'}, + {"ou", require_arg, 'o'}, + {"extendable", no_arg, 't'}, + {"extendabl", no_arg, 't'}, + {"extendab", no_arg, 't'}, + {"extenda", no_arg, 't'}, + {"extend", no_arg, 't'}, + {"exten", no_arg, 't'}, + {"exte", no_arg, 't'}, + {"ext", no_arg, 't'}, + {"ex", no_arg, 't'}, + {"threshold", require_arg, 'T'}, + {"threshol", require_arg, 'T'}, + {"thresho", require_arg, 'T'}, + {"thresh", require_arg, 'T'}, + {"thres", require_arg, 'T'}, + {"thre", require_arg, 'T'}, + {"thr", require_arg, 'T'}, + {"th", require_arg, 'T'}, + {"write-only", require_arg, 'w'}, + {"write-onl", require_arg, 'w'}, + {"write-on", require_arg, 'w'}, + {"write-o", require_arg, 'w'}, + {"write", require_arg, 'w'}, + {"writ", require_arg, 'w'}, + {"wri", require_arg, 'w'}, + {"wr", require_arg, 'w'}, + {NULL, 0, '\0'}}; + +struct options { + long io_types; /* bitmask of which I/O types to test */ + const char *output_file; /* file to print report to */ + long num_dsets; /* number of datasets */ + long num_files; /* number of files */ + off_t num_bpp; /* number of bytes per proc per dset */ + int num_iters; /* number of iterations */ + hsize_t dset_size[MAX_DIMS]; /* Dataset size */ + size_t buf_size[MAX_DIMS]; /* Buffer size */ + size_t chk_size[MAX_DIMS]; /* Chunk size */ + int order[MAX_DIMS]; /* Dimension access order */ + int dset_rank; /* Rank */ + int buf_rank; /* Rank */ + int order_rank; /* Rank */ + int chk_rank; /* Rank */ + int print_times; /* print times as well as throughputs */ + int print_raw; /* print raw data throughput info */ + hsize_t h5_alignment; /* alignment in HDF5 file */ + hsize_t h5_threshold; /* threshold for alignment in HDF5 file */ + int h5_use_chunks; /* Make HDF5 dataset chunked */ + int h5_write_only; /* Perform the write tests only */ + int h5_extendable; /* Perform the write tests only */ + int verify; /* Verify data correctness */ + vfdtype vfd; /* File driver */ + size_t page_buffer_size; + size_t page_size; +}; + +typedef struct { + double min; + double max; + double sum; + int num; +} minmax; + +/* local functions */ +static hsize_t parse_size_directive(const char *size); +static struct options *parse_command_line(int argc, const char *argv[]); +static void run_test_loop(struct options *options); +static int run_test(iotype iot, parameters parms, struct options *opts); +static void output_all_info(minmax *mm, int count, int indent_level); +static void get_minmax(minmax *mm, double val); +static void accumulate_minmax_stuff(const minmax *mm, int count, minmax *total_mm); +static void output_results(const struct options *options, const char *name, minmax *table, int table_size, + off_t data_size); +static void output_report(const char *fmt, ...); +static void print_indent(register int indent); +static void usage(const char *prog); +static void report_parameters(struct options *opts); + +/* + * Function: main + * Purpose: Start things up. + * Return: EXIT_SUCCESS or EXIT_FAILURE + * Programmer: Bill Wendling, 30. October 2001 + * Modifications: + */ +int +main(int argc, const char *argv[]) +{ + int exit_value = EXIT_SUCCESS; + struct options *opts = NULL; + +#ifndef STANDALONE + /* Initialize h5tools lib */ + h5tools_init(); +#endif + + output = stdout; + + opts = parse_command_line(argc, argv); + + if (!opts) { + exit_value = EXIT_FAILURE; + goto finish; + } + + if (opts->output_file) { + if ((output = HDfopen(opts->output_file, "w")) == NULL) { + HDfprintf(stderr, "%s: cannot open output file\n", progname); + HDperror(opts->output_file); + goto finish; + } + } + + report_parameters(opts); + + run_test_loop(opts); + +finish: + HDfree(opts); + return exit_value; +} + +/* + * Function: run_test_loop + * Purpose: Run the I/O tests. Write the results to OUTPUT. + * + * - The slowest changing part of the test is the number of + * processors to use. For each loop iteration, we divide that + * number by 2 and rerun the test. + * + * - The second slowest is what type of IO API to perform. We have + * three choices: POSIXIO, and HDF5. + * + * - Then we change the size of the buffer. This information is + * inferred from the number of datasets to create and the number + * of integers to put into each dataset. The backend code figures + * this out. + * + * Return: Nothing + * Programmer: Bill Wendling, 30. October 2001 + * Modifications: + * Added multidimensional testing (Christian Chilan, April, 2008) + */ +static void +run_test_loop(struct options *opts) +{ + parameters parms; + int i; + size_t buf_bytes; + + /* load options into parameter structure */ + parms.num_files = opts->num_files; + parms.num_dsets = opts->num_dsets; + parms.num_iters = opts->num_iters; + parms.rank = opts->dset_rank; + parms.h5_align = opts->h5_alignment; + parms.h5_thresh = opts->h5_threshold; + parms.h5_use_chunks = opts->h5_use_chunks; + parms.h5_extendable = opts->h5_extendable; + parms.h5_write_only = opts->h5_write_only; + parms.verify = opts->verify; + parms.vfd = opts->vfd; + parms.page_buffer_size = opts->page_buffer_size; + parms.page_size = opts->page_size; + + /* load multidimensional options */ + parms.num_bytes = 1; + buf_bytes = 1; + for (i = 0; i < parms.rank; i++) { + parms.buf_size[i] = opts->buf_size[i]; + parms.dset_size[i] = opts->dset_size[i]; + parms.chk_size[i] = opts->chk_size[i]; + parms.order[i] = opts->order[i]; + parms.num_bytes *= opts->dset_size[i]; + buf_bytes *= opts->buf_size[i]; + } + + /* print size information */ + output_report("Transfer Buffer Size (bytes): %d\n", buf_bytes); + output_report("File Size(MB): %.2f\n", ((double)parms.num_bytes) / ONE_MB); + + print_indent(0); + if (opts->io_types & SIO_POSIX) + run_test(POSIXIO, parms, opts); + + print_indent(0); + if (opts->io_types & SIO_HDF5) + run_test(HDF5, parms, opts); +} + +/* + * Function: run_test + * Purpose: Inner loop call to actually run the I/O test. + * Return: Nothing + * Programmer: Bill Wendling, 18. December 2001 + * Modifications: + */ +static int +run_test(iotype iot, parameters parms, struct options *opts) +{ + results res; + register int i, ret_value = SUCCESS; + off_t raw_size; + minmax * write_sys_mm_table = NULL; + minmax * write_mm_table = NULL; + minmax * write_gross_mm_table = NULL; + minmax * write_raw_mm_table = NULL; + minmax * read_sys_mm_table = NULL; + minmax * read_mm_table = NULL; + minmax * read_gross_mm_table = NULL; + minmax * read_raw_mm_table = NULL; + minmax write_sys_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax write_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax write_gross_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax write_raw_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax read_sys_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax read_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax read_gross_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax read_raw_mm = {0.0F, 0.0F, 0.0F, 0}; + + raw_size = (off_t)parms.num_bytes; + parms.io_type = iot; + print_indent(2); + output_report("IO API = "); + + switch (iot) { + case POSIXIO: + output_report("POSIX\n"); + break; + case HDF5: + output_report("HDF5\n"); + break; + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + HDassert(0 && "Unknown IO tpe"); + break; + } + + /* allocate space for tables minmax and that it is sufficient */ + /* to initialize all elements to zeros by calloc. */ + write_sys_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + write_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + write_gross_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + write_raw_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + + if (!parms.h5_write_only) { + read_sys_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + read_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + read_gross_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + read_raw_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + } + + /* Do IO iteration times, collecting statistics each time */ + for (i = 0; i < parms.num_iters; ++i) { + double t; + + do_sio(parms, &res); + + /* gather all of the "sys write" times */ + t = io_time_get(res.timers, HDF5_MPI_WRITE); + get_minmax(&write_sys_mm, t); + + write_sys_mm_table[i] = write_sys_mm; + + /* gather all of the "write" times */ + t = io_time_get(res.timers, HDF5_FINE_WRITE_FIXED_DIMS); + get_minmax(&write_mm, t); + + write_mm_table[i] = write_mm; + + /* gather all of the "write" times from open to close */ + t = io_time_get(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS); + get_minmax(&write_gross_mm, t); + + write_gross_mm_table[i] = write_gross_mm; + + /* gather all of the raw "write" times */ + t = io_time_get(res.timers, HDF5_RAW_WRITE_FIXED_DIMS); + get_minmax(&write_raw_mm, t); + + write_raw_mm_table[i] = write_raw_mm; + + if (!parms.h5_write_only) { + /* gather all of the "mpi read" times */ + t = io_time_get(res.timers, HDF5_MPI_READ); + get_minmax(&read_sys_mm, t); + + read_sys_mm_table[i] = read_sys_mm; + + /* gather all of the "read" times */ + t = io_time_get(res.timers, HDF5_FINE_READ_FIXED_DIMS); + get_minmax(&read_mm, t); + + read_mm_table[i] = read_mm; + + /* gather all of the "read" times from open to close */ + t = io_time_get(res.timers, HDF5_GROSS_READ_FIXED_DIMS); + get_minmax(&read_gross_mm, t); + + read_gross_mm_table[i] = read_gross_mm; + + /* gather all of the raw "read" times */ + t = io_time_get(res.timers, HDF5_RAW_READ_FIXED_DIMS); + get_minmax(&read_raw_mm, t); + + read_raw_mm_table[i] = read_gross_mm; + } + io_time_destroy(res.timers); + } + + /* + * Show various statistics + */ + /* Write statistics */ + /* Print the raw data throughput if desired */ + if (opts->print_raw) { + /* accumulate and output the max, min, and average "raw write" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Raw Data Write details:\n"); + output_all_info(write_raw_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); + } /* end if */ + + /* show sys write statics */ +#if 0 + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("MPI Write details:\n"); + output_all_info(write_sys_mm_table, parms.num_iters, 4); + } +#endif + /* We don't currently output the MPI write results */ + + /* accumulate and output the max, min, and average "write" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write details:\n"); + output_all_info(write_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Write", write_mm_table, parms.num_iters, raw_size); + + /* accumulate and output the max, min, and average "gross write" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write Open-Close details:\n"); + output_all_info(write_gross_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Write Open-Close", write_gross_mm_table, parms.num_iters, raw_size); + + if (!parms.h5_write_only) { + /* Read statistics */ + /* Print the raw data throughput if desired */ + if (opts->print_raw) { + /* accumulate and output the max, min, and average "raw read" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Raw Data Read details:\n"); + output_all_info(read_raw_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); + } /* end if */ + + /* show mpi read statics */ +#if 0 + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("MPI Read details:\n"); + output_all_info(read_sys_mm_table, parms.num_iters, 4); + } +#endif + /* We don't currently output the MPI read results */ + + /* accumulate and output the max, min, and average "read" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read details:\n"); + output_all_info(read_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Read", read_mm_table, parms.num_iters, raw_size); + + /* accumulate and output the max, min, and average "gross read" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read Open-Close details:\n"); + output_all_info(read_gross_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Read Open-Close", read_gross_mm_table, parms.num_iters, raw_size); + } + + /* clean up our mess */ + HDfree(write_sys_mm_table); + HDfree(write_mm_table); + HDfree(write_gross_mm_table); + HDfree(write_raw_mm_table); + + if (!parms.h5_write_only) { + HDfree(read_sys_mm_table); + HDfree(read_mm_table); + HDfree(read_gross_mm_table); + HDfree(read_raw_mm_table); + } + + return ret_value; +} + +/* + * Function: output_all_info + * Purpose: + * Return: Nothing + * Programmer: Bill Wendling, 29. January 2002 + * Modifications: + */ +static void +output_all_info(minmax *mm, int count, int indent_level) +{ + int i; + + for (i = 0; i < count; ++i) { + print_indent(indent_level); + output_report("Iteration %d:\n", i + 1); + print_indent(indent_level + 1); + output_report("Minimum Time: %.2fs\n", mm[i].min); + print_indent(indent_level + 1); + output_report("Maximum Time: %.2fs\n", mm[i].max); + } +} + +/* + * Function: get_minmax + * Purpose: Gather all the min, max and total of val. + * Return: Nothing + * Programmer: Bill Wendling, 21. December 2001 + * Modifications: + * Use MPI_Allreduce to do it. -akc, 2002/01/11 + */ + +static void +get_minmax(minmax *mm, double val) +{ + mm->max = val; + mm->min = val; + mm->sum = val; +} + +/* + * Function: accumulate_minmax_stuff + * Purpose: Accumulate the minimum, maximum, and average of the times + * across all processes. + * Return: TOTAL_MM - the total of all of these. + * Programmer: Bill Wendling, 21. December 2001 + * Modifications: + * Changed to use seconds instead of MB/s - QAK, 5/9/02 + */ +static void +accumulate_minmax_stuff(const minmax *mm, int count, minmax *total_mm) +{ + int i; + + total_mm->sum = 0.0F; + total_mm->max = -DBL_MAX; + total_mm->min = DBL_MAX; + total_mm->num = count; + + for (i = 0; i < count; ++i) { + double m = mm[i].max; + + total_mm->sum += m; + + if (m < total_mm->min) + total_mm->min = m; + + if (m > total_mm->max) + total_mm->max = m; + } +} + +/* + * Function: output_results + * Purpose: Print information about the time & bandwidth for a given + * minmax & # of iterations. + * Return: Nothing + * Programmer: Quincey Koziol, 9. May 2002 + * Modifications: + */ +static void +output_results(const struct options *opts, const char *name, minmax *table, int table_size, off_t data_size) +{ + minmax total_mm; + + accumulate_minmax_stuff(table, table_size, &total_mm); + + print_indent(3); + output_report("%s (%d iteration(s)):\n", name, table_size); + + /* Note: The maximum throughput uses the minimum amount of time & vice versa */ + + print_indent(4); + output_report("Maximum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.min)); + if (opts->print_times) + output_report(" (%7.3f s)\n", total_mm.min); + else + output_report("\n"); + + print_indent(4); + output_report("Average Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.sum / total_mm.num)); + if (opts->print_times) + output_report(" (%7.3f s)\n", (total_mm.sum / total_mm.num)); + else + output_report("\n"); + + print_indent(4); + output_report("Minimum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.max)); + if (opts->print_times) + output_report(" (%7.3f s)\n", total_mm.max); + else + output_report("\n"); +} + +/* + * Function: output_report + * Purpose: Print a line of the report. Only do so if I'm the 0 process. + * Return: Nothing + * Programmer: Bill Wendling, 19. December 2001 + * Modifications: + */ +static void +output_report(const char *fmt, ...) +{ + va_list ap; + + HDva_start(ap, fmt); + HDvfprintf(output, fmt, ap); + HDva_end(ap); +} + +/* + * Function: print_indent + * Purpose: Print spaces to indent a new line of text for pretty printing + * things. + * Return: Nothing + * Programmer: Bill Wendling, 29. October 2001 + * Modifications: + */ +static void +print_indent(register int indent) +{ + indent *= TAB_SPACE; + + for (; indent > 0; --indent) + HDfputc(' ', output); +} + +static void +recover_size_and_print(long long val, const char *end) +{ + if (val >= ONE_KB && (val % ONE_KB) == 0) { + if (val >= ONE_MB && (val % ONE_MB) == 0) { + if (val >= ONE_GB && (val % ONE_GB) == 0) + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "GB%s", + val / ONE_GB, end); + else + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "MB%s", + val / ONE_MB, end); + } + else { + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "KB%s", + val / ONE_KB, end); + } + } + else { + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "%s", + val, end); + } +} + +static void +print_io_api(long io_types) +{ + if (io_types & SIO_POSIX) + HDfprintf(output, "posix "); + if (io_types & SIO_HDF5) + HDfprintf(output, "hdf5 "); + HDfprintf(output, "\n"); +} + +static void +report_parameters(struct options *opts) +{ + int i, rank; + rank = opts->dset_rank; + + print_version("HDF5 Library"); /* print library version */ + HDfprintf(output, "==== Parameters ====\n"); + + HDfprintf(output, "IO API="); + print_io_api(opts->io_types); + + HDfprintf(output, "Number of iterations=%d\n", opts->num_iters); + + HDfprintf(output, "Dataset size="); + + for (i = 0; i < rank; i++) + recover_size_and_print((long long)opts->dset_size[i], " "); + HDfprintf(output, "\n"); + + HDfprintf(output, "Transfer buffer size="); + for (i = 0; i < rank; i++) + recover_size_and_print((long long)opts->buf_size[i], " "); + HDfprintf(output, "\n"); + + if (opts->page_size) { + HDfprintf(output, "Page Aggregation Enabled. Page size = %zu\n", opts->page_size); + if (opts->page_buffer_size) + HDfprintf(output, "Page Buffering Enabled. Page Buffer size = %zu\n", opts->page_buffer_size); + else + HDfprintf(output, "Page Buffering Disabled\n"); + } + else + HDfprintf(output, "Page Aggregation Disabled\n"); + + HDfprintf(output, "Dimension access order="); + for (i = 0; i < rank; i++) + recover_size_and_print((long long)opts->order[i], " "); + HDfprintf(output, "\n"); + + if (opts->io_types & SIO_HDF5) { + + HDfprintf(output, "HDF5 data storage method="); + + if (opts->h5_use_chunks) { + + HDfprintf(output, "Chunked\n"); + HDfprintf(output, "HDF5 chunk size="); + for (i = 0; i < rank; i++) + recover_size_and_print((long long)opts->chk_size[i], " "); + HDfprintf(output, "\n"); + + HDfprintf(output, "HDF5 dataset dimensions="); + if (opts->h5_extendable) { + HDfprintf(output, "Extendable\n"); + } + else { + HDfprintf(output, "Fixed\n"); + } + } + else { + HDfprintf(output, "Contiguous\n"); + } + + HDfprintf(output, "HDF5 file driver="); + if (opts->vfd == sec2) { + HDfprintf(output, "sec2\n"); + } + else if (opts->vfd == stdio) { + HDfprintf(output, "stdio\n"); + } + else if (opts->vfd == core) { + HDfprintf(output, "core\n"); + } + else if (opts->vfd == split) { + HDfprintf(output, "split\n"); + } + else if (opts->vfd == multi) { + HDfprintf(output, "multi\n"); + } + else if (opts->vfd == family) { + HDfprintf(output, "family\n"); + } + else if (opts->vfd == direct) { + HDfprintf(output, "direct\n"); + } + } + + { + char *prefix = HDgetenv("HDF5_PREFIX"); + + HDfprintf(output, "Env HDF5_PREFIX=%s\n", (prefix ? prefix : "not set")); + } + + HDfprintf(output, "==== End of Parameters ====\n"); + HDfprintf(output, "\n"); +} + +/* + * Function: parse_command_line + * Purpose: Parse the command line options and return a STRUCT OPTIONS + * structure which will need to be freed by the calling function. + * Return: Pointer to an OPTIONS structure + * Programmer: Bill Wendling, 31. October 2001 + * Modifications: + * Added multidimensional testing (Christian Chilan, April, 2008) + */ +static struct options * +parse_command_line(int argc, const char *argv[]) +{ + int opt; + struct options *cl_opts; + int i, default_rank, actual_rank, ranks[4]; + + cl_opts = (struct options *)HDmalloc(sizeof(struct options)); + + cl_opts->page_buffer_size = 0; + cl_opts->page_size = 0; + + cl_opts->output_file = NULL; + cl_opts->io_types = 0; /* will set default after parsing options */ + cl_opts->num_iters = 1; + + default_rank = 2; + + cl_opts->dset_rank = 0; + cl_opts->buf_rank = 0; + cl_opts->chk_rank = 0; + cl_opts->order_rank = 0; + + for (i = 0; i < MAX_DIMS; i++) { + cl_opts->buf_size[i] = (size_t)((i + 1) * 10); + cl_opts->dset_size[i] = (hsize_t)((i + 1) * 100); + cl_opts->chk_size[i] = (size_t)((i + 1) * 10); + cl_opts->order[i] = i + 1; + } + + cl_opts->vfd = sec2; + + cl_opts->print_times = FALSE; /* Printing times is off by default */ + cl_opts->print_raw = FALSE; /* Printing raw data throughput is off by default */ + cl_opts->h5_alignment = 1; /* No alignment for HDF5 objects by default */ + cl_opts->h5_threshold = 1; /* No threshold for aligning HDF5 objects by default */ + cl_opts->h5_use_chunks = FALSE; /* Don't chunk the HDF5 dataset by default */ + cl_opts->h5_write_only = FALSE; /* Do both read and write by default */ + cl_opts->h5_extendable = FALSE; /* Use extendable dataset */ + cl_opts->verify = FALSE; /* No Verify data correctness by default */ + + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { + switch ((char)opt) { + case 'a': + cl_opts->h5_alignment = parse_size_directive(H5_optarg); + break; + case 'G': + cl_opts->page_size = parse_size_directive(H5_optarg); + break; + case 'b': + cl_opts->page_buffer_size = parse_size_directive(H5_optarg); + break; + case 'A': { + const char *end = H5_optarg; + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + if (!HDstrcasecmp(buf, "hdf5")) { + cl_opts->io_types |= SIO_HDF5; + } + else if (!HDstrcasecmp(buf, "posix")) { + cl_opts->io_types |= SIO_POSIX; + } + else { + HDfprintf(stderr, "sio_perf: invalid --api option %s\n", buf); + HDexit(EXIT_FAILURE); + } + + if (*end == '\0') + break; + + end++; + } + } + + break; +#if 0 + case 'b': + /* the future "binary" option */ + break; +#endif /* 0 */ + case 'c': + /* Turn on chunked HDF5 dataset creation */ + cl_opts->h5_use_chunks = 1; + { + const char *end = H5_optarg; + int j = 0; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + cl_opts->chk_size[j] = parse_size_directive(buf); + + j++; + + if (*end == '\0') + break; + + end++; + } + cl_opts->chk_rank = j; + } + + break; + + case 'D': { + const char *end = H5_optarg; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + if (HDstrlen(buf) > 1 || HDisdigit(buf[0])) { + size_t j; + + for (j = 0; j < 10 && buf[j] != '\0'; ++j) + if (!HDisdigit(buf[j])) { + HDfprintf(stderr, "sio_perf: invalid --debug option %s\n", buf); + HDexit(EXIT_FAILURE); + } + + sio_debug_level = atoi(buf); + + if (sio_debug_level > 4) + sio_debug_level = 4; + else if (sio_debug_level < 0) + sio_debug_level = 0; + } + else { + switch (*buf) { + case 'r': + /* Turn on raw data throughput info */ + cl_opts->print_raw = TRUE; + break; + case 't': + /* Turn on time printing */ + cl_opts->print_times = TRUE; + break; + case 'v': + /* Turn on verify data correctness*/ + cl_opts->verify = TRUE; + break; + default: + HDfprintf(stderr, "sio_perf: invalid --debug option %s\n", buf); + HDexit(EXIT_FAILURE); + } + } + + if (*end == '\0') + break; + + end++; + } + } + + break; + case 'e': { + const char *end = H5_optarg; + int j = 0; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + cl_opts->dset_size[j] = parse_size_directive(buf); + + j++; + + if (*end == '\0') + break; + + end++; + } + cl_opts->dset_rank = j; + } + + break; + + case 'i': + cl_opts->num_iters = HDatoi(H5_optarg); + break; + case 'o': + cl_opts->output_file = H5_optarg; + break; + case 'T': + cl_opts->h5_threshold = parse_size_directive(H5_optarg); + break; + case 'v': + if (!HDstrcasecmp(H5_optarg, "sec2")) { + cl_opts->vfd = sec2; + } + else if (!HDstrcasecmp(H5_optarg, "stdio")) { + cl_opts->vfd = stdio; + } + else if (!HDstrcasecmp(H5_optarg, "core")) { + cl_opts->vfd = core; + } + else if (!HDstrcasecmp(H5_optarg, "split")) { + cl_opts->vfd = split; + } + else if (!HDstrcasecmp(H5_optarg, "multi")) { + cl_opts->vfd = multi; + } + else if (!HDstrcasecmp(H5_optarg, "family")) { + cl_opts->vfd = family; + } + else if (!HDstrcasecmp(H5_optarg, "direct")) { + cl_opts->vfd = direct; + } + else { + HDfprintf(stderr, "sio_perf: invalid --api option %s\n", H5_optarg); + HDexit(EXIT_FAILURE); + } + break; + case 'w': + cl_opts->h5_write_only = TRUE; + break; + case 't': + cl_opts->h5_extendable = TRUE; + break; + case 'x': { + const char *end = H5_optarg; + int j = 0; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + cl_opts->buf_size[j] = parse_size_directive(buf); + + j++; + + if (*end == '\0') + break; + + end++; + } + cl_opts->buf_rank = j; + } + + break; + + case 'r': { + const char *end = H5_optarg; + int j = 0; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + cl_opts->order[j] = (int)parse_size_directive(buf); + + j++; + + if (*end == '\0') + break; + + end++; + } + + cl_opts->order_rank = j; + } + + break; + + case 'h': + case '?': + default: + usage(progname); + HDfree(cl_opts); + return NULL; + } + } + + /* perform rank consistency analysis */ + actual_rank = 0; + + ranks[0] = cl_opts->dset_rank; + ranks[1] = cl_opts->buf_rank; + ranks[2] = cl_opts->order_rank; + ranks[3] = cl_opts->chk_rank; + + for (i = 0; i < 4; i++) { + if (ranks[i] > 0) { + if (!actual_rank) { + actual_rank = ranks[i]; + } + else { + if (actual_rank != ranks[i]) + exit(EXIT_FAILURE); + } + } + } + + if (!actual_rank) + actual_rank = default_rank; + + cl_opts->dset_rank = actual_rank; + cl_opts->buf_rank = actual_rank; + cl_opts->order_rank = actual_rank; + cl_opts->chk_rank = actual_rank; + + for (i = 0; i < actual_rank; i++) { + if (cl_opts->order[i] > actual_rank) { + exit(EXIT_FAILURE); + } + } + + /* set default if none specified yet */ + if (!cl_opts->io_types) + cl_opts->io_types = SIO_HDF5 | SIO_POSIX; /* run all API */ + + /* verify parameters sanity. Adjust if needed. */ + /* cap xfer_size with bytes per process */ + if (cl_opts->num_iters <= 0) + cl_opts->num_iters = 1; + + return cl_opts; +} + +/* + * Function: parse_size_directive + * Purpose: Parse the size directive passed on the commandline. The size + * directive is an integer followed by a size indicator: + * + * K, k - Kilobyte + * M, m - Megabyte + * G, g - Gigabyte + * + * Return: The size as a off_t because this is related to file size. + * If an unknown size indicator is used, then the program will + * exit with EXIT_FAILURE as the return value. + * Programmer: Bill Wendling, 18. December 2001 + * Modifications: + */ + +static hsize_t +parse_size_directive(const char *size) +{ + hsize_t s; + char * endptr; + + s = HDstrtoull(size, &endptr, 10); + + if (endptr && *endptr) { + while (*endptr != '\0' && (*endptr == ' ' || *endptr == '\t')) + ++endptr; + + switch (*endptr) { + case 'K': + case 'k': + s *= ONE_KB; + break; + + case 'M': + case 'm': + s *= ONE_MB; + break; + + case 'G': + case 'g': + s *= ONE_GB; + break; + + default: + HDfprintf(stderr, "Illegal size specifier '%c'\n", *endptr); + HDexit(EXIT_FAILURE); + } + } + + return s; +} + +/* + * Function: usage + * Purpose: Print a usage message and then exit. + * Return: Nothing + * Programmer: Bill Wendling, 31. October 2001 + * Modifications: + */ +static void +usage(const char *prog) +{ + print_version(prog); + HDprintf("usage: %s [OPTIONS]\n", prog); + HDprintf(" OPTIONS\n"); + HDprintf(" -h Print an usage message and exit\n"); + HDprintf(" -A AL Which APIs to test\n"); + HDprintf(" [default: all of them]\n"); + HDprintf(" -c SL Selects chunked storage and defines chunks dimensions\n"); + HDprintf(" and sizes\n"); + HDprintf(" [default: Off]\n"); + HDprintf(" -e SL Dimensions and sizes of dataset\n"); + HDprintf(" [default: 100,200]\n"); + HDprintf(" -i N Number of iterations to perform\n"); + HDprintf(" [default: 1]\n"); + HDprintf(" -r NL Dimension access order (see below for description)\n"); + HDprintf(" [default: 1,2]\n"); + HDprintf(" -t Selects extendable dimensions for HDF5 dataset\n"); + HDprintf(" [default: Off]\n"); + HDprintf(" -v VFD Selects file driver for HDF5 access\n"); + HDprintf(" [default: sec2]\n"); + HDprintf(" -w Perform write tests, not the read tests\n"); + HDprintf(" [default: Off]\n"); + HDprintf(" -x SL Dimensions and sizes of the transfer buffer\n"); + HDprintf(" [default: 10,20]\n"); + HDprintf("\n"); + HDprintf(" N - is an integer > 0.\n"); + HDprintf("\n"); + HDprintf(" S - is a size specifier, an integer > 0 followed by a size indicator:\n"); + HDprintf(" K - Kilobyte (%d)\n", ONE_KB); + HDprintf(" M - Megabyte (%d)\n", ONE_MB); + HDprintf(" G - Gigabyte (%d)\n", ONE_GB); + HDprintf("\n"); + HDprintf(" Example: '37M' is 37 megabytes or %d bytes\n", 37 * ONE_MB); + HDprintf("\n"); + HDprintf(" AL - is an API list. Valid values are:\n"); + HDprintf(" hdf5 - HDF5\n"); + HDprintf(" posix - POSIX\n"); + HDprintf("\n"); + HDprintf(" Example: -A posix,hdf5\n"); + HDprintf("\n"); + HDprintf(" NL - is list of integers (N) separated by commas.\n"); + HDprintf("\n"); + HDprintf(" Example: 1,2,3\n"); + HDprintf("\n"); + HDprintf(" SL - is list of size specifiers (S) separated by commas.\n"); + HDprintf("\n"); + HDprintf(" Example: 2K,2K,3K\n"); + HDprintf("\n"); + HDprintf(" The example defines an object (dataset, tranfer buffer) with three\n"); + HDprintf(" dimensions. Be aware that as the number of dimensions increases, the\n"); + HDprintf(" the total size of the object increases exponentially.\n"); + HDprintf("\n"); + HDprintf(" VFD - is an HDF5 file driver specifier. Valid values are:\n"); + HDprintf(" sec2, stdio, core, split, multi, family, direct\n"); + HDprintf("\n"); + HDprintf(" Dimension access order:\n"); + HDprintf(" Data access starts at the cardinal origin of the dataset using the\n"); + HDprintf(" transfer buffer. The next access occurs on a dataset region next to\n"); + HDprintf(" the previous one. For a multidimensional dataset, there are several\n"); + HDprintf(" directions as to where to proceed. This can be specified in the dimension\n"); + HDprintf(" access order. For example, -r 1,2 states that the tool should traverse\n"); + HDprintf(" dimension 1 first, and then dimension 2.\n"); + HDprintf("\n"); + HDprintf(" Environment variables:\n"); + HDprintf(" HDF5_NOCLEANUP Do not remove data files if set [default remove]\n"); + HDprintf(" HDF5_PREFIX Data file prefix\n"); + HDprintf("\n"); + HDfflush(stdout); +} /* end usage() */ diff --git a/tools/src/h5perf/sio_perf.h b/tools/src/h5perf/sio_perf.h new file mode 100644 index 0000000..d998377 --- /dev/null +++ b/tools/src/h5perf/sio_perf.h @@ -0,0 +1,104 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef SIO_PERF_H +#define SIO_PERF_H + +#ifndef STANDALONE +#include "io_timer.h" +#include "H5private.h" +#include "h5tools.h" +#include "h5tools_utils.h" +#else +#include "io_timer.h" +#include "sio_standalone.h" +#endif + +/* setup the dataset no fill option if this is v1.5 or more */ +#if H5_VERS_MAJOR > 1 || H5_VERS_MINOR > 4 +#define H5_HAVE_NOFILL 1 +#endif + +#define MAX_DIMS 32 + +typedef enum iotype_ { + POSIXIO, + HDF5 + /*NUM_TYPES*/ +} iotype; + +typedef enum vfdtype_ { + sec2, + stdio, + core, + split, + multi, + family, + direct + /*NUM_TYPES*/ +} vfdtype; + +typedef struct parameters_ { + iotype io_type; /* The type of IO test to perform */ + vfdtype vfd; + long num_files; /* Number of files to create */ + long num_dsets; /* Number of datasets to create */ + hsize_t num_bytes; /* Number of bytes in each dset */ + int num_iters; /* Number of times to loop doing the IO */ + int rank; /* Rank of dataset */ + hsize_t dset_size[MAX_DIMS]; /* Dataset size */ + size_t buf_size[MAX_DIMS]; /* Buffer size */ + size_t chk_size[MAX_DIMS]; /* Chunk size */ + int order[MAX_DIMS]; /* Buffer size */ + hsize_t h5_align; /* HDF5 object alignment */ + hsize_t h5_thresh; /* HDF5 object alignment threshold */ + int h5_use_chunks; /* Make HDF5 dataset chunked */ + int h5_extendable; /* Make HDF5 dataset chunked */ + int h5_write_only; /* Perform the write tests only */ + int verify; /* Verify data correctness */ + size_t page_size; + size_t page_buffer_size; +} parameters; + +typedef struct results_ { + herr_t ret_code; + io_time_t *timers; +} results; + +#ifndef SUCCESS +#define SUCCESS 0 +#endif /* !SUCCESS */ + +#ifndef FAIL +#define FAIL -1 +#endif /* !FAIL */ + +extern FILE * output; /* output file */ +extern io_time_t *timer_g; /* timer: global for stub functions */ +extern int sio_debug_level; /* The debug level: + * 0 - Off + * 1 - Minimal + * 2 - Some more + * 3 - Maximal + * 4 - Even More Debugging (timer stuff) + */ +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +extern void do_sio(parameters param, results *res); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SIO_PERF_H */ diff --git a/tools/test/perform/CMakeLists.txt b/tools/test/perform/CMakeLists.txt index 7399723..7bf79c5 100644 --- a/tools/test/perform/CMakeLists.txt +++ b/tools/test/perform/CMakeLists.txt @@ -4,39 +4,17 @@ project (HDF5_TOOLS_TEST_PERFORM C) # -------------------------------------------------------------------- # Add the executables # -------------------------------------------------------------------- -#-- Adding test for h5perf_serial -set (h5perf_serial_SOURCES - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_perf.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_engine.c -) -add_executable (h5perf_serial ${h5perf_serial_SOURCES}) -target_include_directories (h5perf_serial PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") -if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (h5perf_serial STATIC) - target_link_libraries (h5perf_serial PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) -else () - TARGET_C_PROPERTIES (h5perf_serial SHARED) - target_link_libraries (h5perf_serial PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) -endif () -set_target_properties (h5perf_serial PROPERTIES FOLDER perform) - -#----------------------------------------------------------------------------- -# Add Target to clang-format -#----------------------------------------------------------------------------- -if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_TOOLS_TEST_PERFORM_h5perf_serial_FORMAT h5perf_serial) -endif () if (HDF5_BUILD_PERFORM_STANDALONE) #-- Adding test for h5perf_serial_alone - io_timer.c includes set (h5perf_serial_alone_SOURCES ${HDF5_TOOLS_DIR}/lib/io_timer.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_perf.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_engine.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/sio_perf.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/sio_engine.c ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_standalone.c ) add_executable (h5perf_serial_alone ${h5perf_serial_alone_SOURCES}) - target_include_directories (h5perf_serial_alone PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TOOLS_DIR}/lib;$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5perf_serial_alone PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TOOLS_DIR}/lib;${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR};${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (h5perf_serial_alone STATIC) target_link_libraries (h5perf_serial_alone PRIVATE ${HDF5_LIB_TARGET} "$<$:${MPI_C_LIBRARIES}>") @@ -188,63 +166,16 @@ if (HDF5_ENABLE_FORMATTERS) endif () if (H5_HAVE_PARALLEL AND HDF5_TEST_PARALLEL) - if (UNIX) - #-- Adding test for perf - only on unix systems - set (perf_SOURCES - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/perf.c - ) - add_executable (perf ${perf_SOURCES}) - target_include_directories (perf PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") - if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (perf STATIC) - target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_TEST_LIB_TARGET} ${HDF5_LIB_TARGET}) - else () - TARGET_C_PROPERTIES (perf SHARED) - target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_TEST_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) - endif () - set_target_properties (perf PROPERTIES FOLDER perform) - - #----------------------------------------------------------------------------- - # Add Target to clang-format - #----------------------------------------------------------------------------- - if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_TOOLS_TEST_PERFORM_perf_FORMAT perf) - endif () - endif () - - #-- Adding test for h5perf - set (h5perf_SOURCES - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_perf.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_engine.c - ) - add_executable (h5perf ${h5perf_SOURCES}) - target_include_directories (h5perf PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") - if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (h5perf STATIC) - target_link_libraries (h5perf PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_TEST_LIB_TARGET} ${HDF5_LIB_TARGET}) - else () - TARGET_C_PROPERTIES (h5perf SHARED) - target_link_libraries (h5perf PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_TEST_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) - endif () - set_target_properties (h5perf PROPERTIES FOLDER perform) - - #----------------------------------------------------------------------------- - # Add Target to clang-format - #----------------------------------------------------------------------------- - if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_TOOLS_TEST_PERFORM_h5perf_FORMAT h5perf) - endif () - if (HDF5_BUILD_PERFORM_STANDALONE) #-- Adding test for h5perf set (h5perf_alone_SOURCES ${HDF5_TOOLS_DIR}/lib/io_timer.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_perf.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_engine.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/pio_perf.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/pio_engine.c ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_standalone.c ) add_executable (h5perf_alone ${h5perf_alone_SOURCES}) - target_include_directories (h5perf_alone PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TOOLS_DIR}/lib;$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5perf_alone PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TOOLS_DIR}/lib;${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR};${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (h5perf_alone STATIC) target_link_libraries (h5perf_alone PRIVATE ${HDF5_LIB_TARGET} ${LINK_LIBS} "$<$:${MPI_C_LIBRARIES}>") diff --git a/tools/test/perform/Makefile.am b/tools/test/perform/Makefile.am index 244ef3a..10b13fe 100644 --- a/tools/test/perform/Makefile.am +++ b/tools/test/perform/Makefile.am @@ -21,17 +21,6 @@ include $(top_srcdir)/config/commence.am AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -# bin_PROGRAMS will be installed. -if BUILD_PARALLEL_CONDITIONAL - bin_PROGRAMS=h5perf_serial h5perf -else - bin_PROGRAMS=h5perf_serial -endif - -# Add h5perf and h5perf_serial specific linker flags here -h5perf_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) -h5perf_serial_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) - # Some programs are not built or run by default, but can be built by hand or by # specifying --enable-build-all at configure time. # Also, some of these programs should only be built in parallel. @@ -47,18 +36,15 @@ endif # List them in the order they should be run. # Parallel test programs. if BUILD_PARALLEL_CONDITIONAL - TEST_PROG_PARA=h5perf perf + TEST_PROG_PARA= endif # Serial test programs. -TEST_PROG = iopipe chunk chunk_cache overhead zip_perf perf_meta h5perf_serial $(BUILD_ALL_PROGS) +TEST_PROG = iopipe chunk chunk_cache overhead zip_perf perf_meta $(BUILD_ALL_PROGS) # check_PROGRAMS will be built but not installed. Do not any executable # that is in bin_PROGRAMS already. Otherwise, it will be removed twice in # "make clean" and some systems, e.g., AIX, do not like it. -check_PROGRAMS= iopipe chunk chunk_cache overhead zip_perf perf_meta $(BUILD_ALL_PROGS) perf - -h5perf_SOURCES=pio_perf.c pio_engine.c -h5perf_serial_SOURCES=sio_perf.c sio_engine.c +check_PROGRAMS= $(TEST_PROG) $(BUILD_ALL_PROGS) # These are the files that `make clean' (and derivatives) will remove from # this directory. @@ -67,9 +53,6 @@ CLEANFILES=*.h5 *.raw *.dat x-gnuplot perftest.out # All of the programs depend on the main hdf5 library, and some of them # depend on test or tools library. LDADD=$(LIBHDF5) -h5perf_LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) -h5perf_serial_LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) -perf_LDADD=$(LIBH5TEST) $(LIBHDF5) iopipe_LDADD=$(LIBH5TEST) $(LIBHDF5) zip_perf_LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) perf_meta_LDADD=$(LIBH5TEST) $(LIBHDF5) diff --git a/tools/test/perform/perf.c b/tools/test/perform/perf.c deleted file mode 100644 index 875f932..0000000 --- a/tools/test/perform/perf.c +++ /dev/null @@ -1,471 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Author: Albert Cheng of NCSA, May 1, 2001. - * This is derived from code given to me by Robert Ross. - * - * NOTE: This code assumes that all command line arguments make it out to all - * the processes that make up the parallel job, which isn't always the case. - * So if it doesn't work on some platform, that might be why. - */ - -#include "hdf5.h" -#include "H5private.h" -#include "h5test.h" - -#ifdef H5_HAVE_PARALLEL - -#include -#include -#include -#include -#include -#include - -#ifdef H5_HAVE_SYS_STAT_H -#include -#endif - -#ifdef H5_HAVE_SYS_TIME_H -#include -#endif - -#ifdef H5_HAVE_SYS_TYPES_H -#include -#endif - -#ifdef H5_HAVE_UNISTD_H -#include -#endif - -#include -#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ -#include -#endif - -/* Macro definitions */ -/* Verify: - * if val is false (0), print mesg and if fatal is true (non-zero), die. - */ -#define H5FATAL 1 -#define VRFY(val, mesg, fatal) \ - do { \ - if (!val) { \ - printf("Proc %d: ", mynod); \ - printf("*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ - if (fatal) { \ - fflush(stdout); \ - goto die_jar_jar_die; \ - } \ - } \ - } while (0) -#define RANK 1 -#define MAX_PATH 1024 - -hsize_t dims[RANK]; /* dataset dim sizes */ -hsize_t block[RANK], stride[RANK], count[RANK]; -hsize_t start[RANK]; -hid_t fid; /* HDF5 file ID */ -hid_t acc_tpl; /* File access templates */ -hid_t sid; /* Dataspace ID */ -hid_t file_dataspace; /* File dataspace ID */ -hid_t mem_dataspace; /* memory dataspace ID */ -hid_t dataset; /* Dataset ID */ -hsize_t opt_alignment = 1; -hsize_t opt_threshold = 1; -int opt_split_vfd = 0; -char * meta_ext, *raw_ext; /* holds the meta and raw file extension if */ - /* opt_split_vfd is set */ - -/* DEFAULT VALUES FOR OPTIONS */ -int64_t opt_block = 1048576 * 16; -int opt_iter = 1; -int opt_stripe = -1; -int opt_correct = 0; -int amode = O_RDWR | O_CREAT; -char opt_file[256] = "perftest.out"; -char opt_pvfstab[256] = "notset"; -int opt_pvfstab_set = 0; - -const char *FILENAME[] = {opt_file, NULL}; - -/* function prototypes */ -static int parse_args(int argc, char **argv); - -#ifndef H5_HAVE_UNISTD_H -/* globals needed for getopt */ -extern char *optarg; -#endif - -int -main(int argc, char **argv) -{ - char * buf, *tmp, *buf2 = NULL, *tmp2 = NULL, *check; - int i, j, mynod = 0, nprocs = 1, my_correct = 1, correct, myerrno; - double stim, etim; - double write_tim = 0; - double read_tim = 0; - double read_bw, write_bw; - double max_read_tim, max_write_tim; - double min_read_tim, min_write_tim; - double ave_read_tim, ave_write_tim; - int64_t iter_jump = 0; - char filename[MAX_PATH]; - herr_t ret; /* Generic return value */ - - /* startup MPI and determine the rank of this process */ - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &mynod); - - /* parse the command line arguments */ - parse_args(argc, argv); - - if (mynod == 0) - printf("# Using hdf5-io calls.\n"); - -#ifdef H5_HAVE_UNISTD_H - /* Kind of a weird hack- if the location of the pvfstab file was - * specified on the command line, then spit out this location into - * the appropriate environment variable. - */ - if (opt_pvfstab_set) { - if ((setenv("PVFSTAB_FILE", opt_pvfstab, 1)) < 0) { - perror("setenv"); - goto die_jar_jar_die; - } - } -#endif - - /* this is how much of the file data is covered on each iteration of - * the test. used to help determine the seek offset on each - * iteration */ - iter_jump = nprocs * opt_block; - - /* setup a buffer of data to write */ - if (!(tmp = (char *)malloc((size_t)opt_block + 256))) { - perror("malloc"); - goto die_jar_jar_die; - } - buf = tmp + 128 - (((long)tmp) % 128); /* align buffer */ - - if (opt_correct) { - /* do the same buffer setup for verifiable data */ - if (!(tmp2 = (char *)malloc((size_t)opt_block + 256))) { - perror("malloc2"); - goto die_jar_jar_die; - } - buf2 = tmp + 128 - (((long)tmp) % 128); - } - - /* setup file access template with parallel IO access. */ - if (opt_split_vfd) { - hid_t mpio_pl; - - mpio_pl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((acc_tpl >= 0), "", H5FATAL); - ret = H5Pset_fapl_mpio(mpio_pl, MPI_COMM_WORLD, MPI_INFO_NULL); - VRFY((ret >= 0), "", H5FATAL); - - /* set optional allocation alignment */ - if (opt_alignment * opt_threshold != 1) { - ret = H5Pset_alignment(acc_tpl, opt_threshold, opt_alignment); - VRFY((ret >= 0), "H5Pset_alignment succeeded", !H5FATAL); - } - - /* setup file access template */ - acc_tpl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((acc_tpl >= 0), "", H5FATAL); - ret = H5Pset_fapl_split(acc_tpl, meta_ext, mpio_pl, raw_ext, mpio_pl); - VRFY((ret >= 0), "H5Pset_fapl_split succeeded", H5FATAL); - ret = H5Pclose(mpio_pl); - VRFY((ret >= 0), "H5Pclose mpio_pl succeeded", H5FATAL); - } - else { - /* setup file access template */ - acc_tpl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((acc_tpl >= 0), "", H5FATAL); - ret = H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL); - VRFY((ret >= 0), "", H5FATAL); - - /* set optional allocation alignment */ - if (opt_alignment * opt_threshold != 1) { - ret = H5Pset_alignment(acc_tpl, opt_threshold, opt_alignment); - VRFY((ret >= 0), "H5Pset_alignment succeeded", !H5FATAL); - } - } - - h5_fixname_no_suffix(FILENAME[0], acc_tpl, filename, sizeof filename); - - /* create the parallel file */ - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); - VRFY((fid >= 0), "H5Fcreate succeeded", H5FATAL); - - /* define a contiquous dataset of opt_iter*nprocs*opt_block chars */ - dims[0] = (hsize_t)opt_iter * (hsize_t)nprocs * (hsize_t)opt_block; - sid = H5Screate_simple(RANK, dims, NULL); - VRFY((sid >= 0), "H5Screate_simple succeeded", H5FATAL); - dataset = H5Dcreate2(fid, "Dataset1", H5T_NATIVE_CHAR, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - VRFY((dataset >= 0), "H5Dcreate2 succeeded", H5FATAL); - - /* create the memory dataspace and the file dataspace */ - dims[0] = (hsize_t)opt_block; - mem_dataspace = H5Screate_simple(RANK, dims, NULL); - VRFY((mem_dataspace >= 0), "", H5FATAL); - file_dataspace = H5Dget_space(dataset); - VRFY((file_dataspace >= 0), "H5Dget_space succeeded", H5FATAL); - - /* now each process writes a block of opt_block chars in round robbin - * fashion until the whole dataset is covered. - */ - for (j = 0; j < opt_iter; j++) { - /* setup a file dataspace selection */ - start[0] = (hsize_t)((j * iter_jump) + (mynod * opt_block)); - stride[0] = block[0] = (hsize_t)opt_block; - count[0] = 1; - ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); - VRFY((ret >= 0), "H5Sset_hyperslab succeeded", H5FATAL); - - if (opt_correct) /* fill in buffer for iteration */ { - for (i = mynod + j, check = buf; i < opt_block; i++, check++) - *check = (char)i; - } - - /* discover the starting time of the operation */ - MPI_Barrier(MPI_COMM_WORLD); - stim = MPI_Wtime(); - - /* write data */ - ret = H5Dwrite(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf); - VRFY((ret >= 0), "H5Dwrite dataset1 succeeded", !H5FATAL); - - /* discover the ending time of the operation */ - etim = MPI_Wtime(); - - write_tim += (etim - stim); - - /* we are done with this "write" iteration */ - } - - /* close dataset and file */ - ret = H5Dclose(dataset); - VRFY((ret >= 0), "H5Dclose succeeded", H5FATAL); - ret = H5Fclose(fid); - VRFY((ret >= 0), "H5Fclose succeeded", H5FATAL); - - /* wait for everyone to synchronize at this point */ - MPI_Barrier(MPI_COMM_WORLD); - - /* reopen the file for reading */ - fid = H5Fopen(filename, H5F_ACC_RDONLY, acc_tpl); - VRFY((fid >= 0), "", H5FATAL); - - /* open the dataset */ - dataset = H5Dopen2(fid, "Dataset1", H5P_DEFAULT); - VRFY((dataset >= 0), "H5Dopen succeeded", H5FATAL); - - /* we can re-use the same mem_dataspace and file_dataspace - * the H5Dwrite used since the dimension size is the same. - */ - - /* we are going to repeat the read the same pattern the write used */ - for (j = 0; j < opt_iter; j++) { - /* setup a file dataspace selection */ - start[0] = (hsize_t)((j * iter_jump) + (mynod * opt_block)); - stride[0] = block[0] = (hsize_t)opt_block; - count[0] = 1; - ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); - VRFY((ret >= 0), "H5Sset_hyperslab succeeded", H5FATAL); - /* seek to the appropriate spot give the current iteration and - * rank within the MPI processes */ - - /* discover the start time */ - MPI_Barrier(MPI_COMM_WORLD); - stim = MPI_Wtime(); - - /* read in the file data */ - if (!opt_correct) { - ret = H5Dread(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf); - } - else { - ret = H5Dread(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf2); - } - myerrno = errno; - - /* discover the end time */ - etim = MPI_Wtime(); - read_tim += (etim - stim); - VRFY((ret >= 0), "H5Dwrite dataset1 succeeded", !H5FATAL); - - if (ret < 0) - HDfprintf(stderr, "node %d, read error, loc = %" PRId64 ": %s\n", mynod, mynod * opt_block, - strerror(myerrno)); - - /* if the user wanted to check correctness, compare the write - * buffer to the read buffer */ - if (opt_correct && memcmp(buf, buf2, (size_t)opt_block)) { - HDfprintf(stderr, "node %d, correctness test failed\n", mynod); - my_correct = 0; - MPI_Allreduce(&my_correct, &correct, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); - } - - /* we are done with this read iteration */ - } - - /* close dataset and file */ - ret = H5Dclose(dataset); - VRFY((ret >= 0), "H5Dclose succeeded", H5FATAL); - ret = H5Fclose(fid); - VRFY((ret >= 0), "H5Fclose succeeded", H5FATAL); - ret = H5Pclose(acc_tpl); - VRFY((ret >= 0), "H5Pclose succeeded", H5FATAL); - - /* compute the read and write times */ - MPI_Allreduce(&read_tim, &max_read_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - MPI_Allreduce(&read_tim, &min_read_tim, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); - MPI_Allreduce(&read_tim, &ave_read_tim, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - - /* calculate the average from the sum */ - ave_read_tim = ave_read_tim / nprocs; - - MPI_Allreduce(&write_tim, &max_write_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - MPI_Allreduce(&write_tim, &min_write_tim, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); - MPI_Allreduce(&write_tim, &ave_write_tim, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - - /* calculate the average from the sum */ - ave_write_tim = ave_write_tim / nprocs; - - /* print out the results on one node */ - if (mynod == 0) { - read_bw = (double)((int64_t)(opt_block * nprocs * opt_iter)) / (max_read_tim * 1000000.0); - write_bw = (double)((int64_t)(opt_block * nprocs * opt_iter)) / (max_write_tim * 1000000.0); - - printf("nr_procs = %d, nr_iter = %d, blk_sz = %ld\n", nprocs, opt_iter, (long)opt_block); - - printf("# total_size = %ld\n", (long)(opt_block * nprocs * opt_iter)); - - printf("# Write: min_time = %f, max_time = %f, mean_time = %f\n", min_write_tim, max_write_tim, - ave_write_tim); - printf("# Read: min_time = %f, max_time = %f, mean_time = %f\n", min_read_tim, max_read_tim, - ave_read_tim); - - printf("Write bandwidth = %f Mbytes/sec\n", write_bw); - printf("Read bandwidth = %f Mbytes/sec\n", read_bw); - - if (opt_correct) { - printf("Correctness test %s.\n", correct ? "passed" : "failed"); - } - } - -die_jar_jar_die: - -#ifdef H5_HAVE_UNISTD - /* Clear the environment variable if it was set earlier */ - if (opt_pvfstab_set) { - unsetenv("PVFSTAB_FILE"); - } -#endif - - free(tmp); - if (opt_correct) - free(tmp2); - - MPI_Finalize(); - - return (0); -} - -static int -parse_args(int argc, char **argv) -{ - int c; - - while ((c = getopt(argc, argv, "s:b:i:f:p:a:2:c")) != EOF) { - switch (c) { - case 's': /* stripe */ - opt_stripe = atoi(optarg); - break; - case 'b': /* block size */ - opt_block = atoi(optarg); - break; - case 'i': /* iterations */ - opt_iter = atoi(optarg); - break; - case 'f': /* filename */ - strncpy(opt_file, optarg, 255); - FILENAME[0] = opt_file; - break; - case 'p': /* pvfstab file */ - strncpy(opt_pvfstab, optarg, 255); - opt_pvfstab_set = 1; - break; - case 'a': /* aligned allocation. - * syntax: -a/ - * e.g., -a4096/512 allocate at 4096 bytes - * boundary if request size >= 512. - */ - { - char *p; - - opt_alignment = (hsize_t)HDatoi(optarg); - if (NULL != (p = (char *)HDstrchr(optarg, '/'))) - opt_threshold = (hsize_t)HDatoi(p + 1); - } - HDfprintf(stdout, "alignment/threshold=%" PRIuHSIZE "/%" PRIuHSIZE "\n", opt_alignment, - opt_threshold); - break; - case '2': /* use 2-files, i.e., split file driver */ - opt_split_vfd = 1; - /* get meta and raw file extension. */ - /* syntax is , */ - meta_ext = raw_ext = optarg; - while (*raw_ext != '\0') { - if (*raw_ext == ',') { - *raw_ext = '\0'; - raw_ext++; - break; - } - raw_ext++; - } - printf("split-file-vfd used: %s,%s\n", meta_ext, raw_ext); - break; - case 'c': /* correctness */ - opt_correct = 1; - break; - case '?': /* unknown */ - default: - break; - } - } - - return (0); -} - -/* - * Local variables: - * c-indent-level: 3 - * c-basic-offset: 3 - * tab-width: 3 - * End: - */ - -#else /* H5_HAVE_PARALLEL */ -/* dummy program since H5_HAVE_PARALLEL is not configured in */ -int -main(int H5_ATTR_UNUSED argc, char H5_ATTR_UNUSED **argv) -{ - printf("No parallel performance because parallel is not configured in\n"); - return (0); -} -#endif /* H5_HAVE_PARALLEL */ diff --git a/tools/test/perform/pio_engine.c b/tools/test/perform/pio_engine.c deleted file mode 100644 index cac36d7..0000000 --- a/tools/test/perform/pio_engine.c +++ /dev/null @@ -1,2745 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Author: Albert Cheng of NCSA, Oct 24, 2001. - */ - -#include "hdf5.h" - -#include -#include -#include -#include - -#ifdef H5_HAVE_UNISTD_H -#include -#include -#endif - -#ifdef H5_HAVE_SYS_STAT_H -#include -#endif - -#ifdef H5_HAVE_PARALLEL - -#include - -#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ -#include -#endif /* !MPI_FILE_NULL */ - -#include "pio_perf.h" - -/* Macro definitions */ - -#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 6 -#define H5DCREATE(fd, name, type, space, dcpl) H5Dcreate(fd, name, type, space, dcpl) -#define H5DOPEN(fd, name) H5Dopen(fd, name) -#else -#define H5DCREATE(fd, name, type, space, dcpl) \ - H5Dcreate2(fd, name, type, space, H5P_DEFAULT, dcpl, H5P_DEFAULT) -#define H5DOPEN(fd, name) H5Dopen2(fd, name, H5P_DEFAULT) -#endif - -/* sizes of various items. these sizes won't change during program execution */ -/* The following three must have the same type */ -#define ELMT_H5_TYPE H5T_NATIVE_UCHAR - -#define GOTOERROR(errcode) \ - { \ - ret_code = errcode; \ - goto done; \ - } -#define ERRMSG(mesg) \ - { \ - HDfprintf(stderr, "Proc %d: ", pio_mpi_rank_g); \ - HDfprintf(stderr, "*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ - } - -/* verify: if val is false (0), print mesg. */ -#define VRFY(val, mesg) \ - do { \ - if (!val) { \ - ERRMSG(mesg); \ - GOTOERROR(FAIL); \ - } \ - } while (0) - -/* POSIX I/O macros */ -#ifdef H5_HAVE_WIN32_API -/* Can't link against the library, so this test will use the older, non-Unicode - * _open() call on Windows. - */ -#define HDopen(S, F, ...) _open(S, F | _O_BINARY, __VA_ARGS__) -#endif /* H5_HAVE_WIN32_API */ -#define POSIXCREATE(fn) HDopen(fn, O_CREAT | O_TRUNC | O_RDWR, 0600) -#define POSIXOPEN(fn, F) HDopen(fn, F, 0600) -#define POSIXCLOSE(F) HDclose(F) -#define POSIXSEEK(F, L) HDlseek(F, L, SEEK_SET) -#define POSIXWRITE(F, B, S) HDwrite(F, B, S) -#define POSIXREAD(F, B, S) HDread(F, B, S) - -enum { PIO_CREATE = 1, PIO_WRITE = 2, PIO_READ = 4 }; - -/* Global variables */ -static int clean_file_g = -1; /*whether to cleanup temporary test */ -/*files. -1 is not defined; */ -/*0 is no cleanup; 1 is do cleanup */ - -/* - * In a parallel machine, the filesystem suitable for compiling is - * unlikely a parallel file system that is suitable for parallel I/O. - * There is no standard pathname for the parallel file system. /tmp - * is about the best guess. - */ -#ifndef HDF5_PARAPREFIX -#define HDF5_PARAPREFIX "" -#endif /* !HDF5_PARAPREFIX */ - -#ifndef MIN -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#endif /* !MIN */ - -/* the different types of file descriptors we can expect */ -typedef union _file_descr { - int posixfd; /* POSIX file handle*/ - MPI_File mpifd; /* MPI file */ - hid_t h5fd; /* HDF5 file */ -} file_descr; - -/* local functions */ -static char * pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size); -static herr_t do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nelmts, - size_t buf_size, void *buffer); -static herr_t do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nelmts, - size_t buf_size, void *buffer /*out*/); -static herr_t do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags); -static herr_t do_fclose(iotype iot, file_descr *fd); -static void do_cleanupfile(iotype iot, char *fname); -static off_t sqrto(off_t); - -/* - * Function: do_pio - * Purpose: PIO Engine where Parallel IO are executed. - * Return: results - * Programmer: Albert Cheng, Bill Wendling 2001/12/12 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -results -do_pio(parameters param) -{ - /* return codes */ - herr_t ret_code = 0; /*return code */ - results res; - - file_descr fd; - iotype iot; - - char fname[FILENAME_MAX]; - long nf; - long ndsets; - off_t nbytes; /*number of bytes per dataset */ - off_t snbytes; /*general dataset size */ - /*for 1D, it is the actual dataset size */ - /*for 2D, it is the size of a side of the dataset square */ - char * buffer = NULL; /*data buffer pointer */ - size_t buf_size; /*general buffer size in bytes */ - /*for 1D, it is the actual buffer size */ - /*for 2D, it is the length of the buffer rectangle */ - size_t blk_size; /*data block size in bytes */ - size_t bsize; /*actual buffer size */ - - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - - /* Sanity check parameters */ - - /* IO type */ - iot = param.io_type; - - switch (iot) { - case MPIO: - fd.mpifd = MPI_FILE_NULL; - res.timers = io_time_new(MPI_CLOCK); - break; - case POSIXIO: - fd.posixfd = -1; - res.timers = io_time_new(MPI_CLOCK); - break; - case PHDF5: - fd.h5fd = -1; - res.timers = io_time_new(MPI_CLOCK); - break; - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", iot); - GOTOERROR(FAIL); - } - - ndsets = param.num_dsets; /* number of datasets per file */ - nbytes = param.num_bytes; /* number of bytes per dataset */ - buf_size = param.buf_size; - blk_size = param.blk_size; - - if (!param.dim2d) { - snbytes = nbytes; /* General dataset size */ - bsize = buf_size; /* Actual buffer size */ - } - else { - snbytes = sqrto(nbytes); /* General dataset size */ - bsize = buf_size * blk_size; /* Actual buffer size */ - } - - if (param.num_files < 0) { - HDfprintf(stderr, "number of files must be >= 0 (%ld)\n", param.num_files); - GOTOERROR(FAIL); - } - - if (ndsets < 0) { - HDfprintf(stderr, "number of datasets per file must be >= 0 (%ld)\n", ndsets); - GOTOERROR(FAIL); - } - - if (param.num_procs <= 0) { - HDfprintf(stderr, "maximum number of process to use must be > 0 (%d)\n", param.num_procs); - GOTOERROR(FAIL); - } - - /* Validate transfer buffer size & block size*/ - if (blk_size <= 0) { - HDfprintf(stderr, "Transfer block size (%zu) must be > 0\n", blk_size); - GOTOERROR(FAIL); - } - if (buf_size <= 0) { - HDfprintf(stderr, "Transfer buffer size (%zu) must be > 0\n", buf_size); - GOTOERROR(FAIL); - } - if ((buf_size % blk_size) != 0) { - HDfprintf(stderr, - "Transfer buffer size (%zu) must be a multiple of the " - "interleaved I/O block size (%zu)\n", - buf_size, blk_size); - GOTOERROR(FAIL); - } - if ((snbytes % pio_mpi_nprocs_g) != 0) { - HDfprintf(stderr, - "Dataset size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "number of processes (%d)\n", - (long long)snbytes, pio_mpi_nprocs_g); - GOTOERROR(FAIL); - } - - if (!param.dim2d) { - if (((size_t)(snbytes / pio_mpi_nprocs_g) % buf_size) != 0) { - HDfprintf(stderr, - "Dataset size/process (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "trasfer buffer size (%zu)\n", - (long long)(snbytes / pio_mpi_nprocs_g), buf_size); - GOTOERROR(FAIL); - } - } - else { - if (((size_t)snbytes % buf_size) != 0) { - HDfprintf(stderr, - "Dataset side size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "trasfer buffer size (%zu)\n", - (long long)snbytes, buf_size); - GOTOERROR(FAIL); - } - } - - /* Allocate transfer buffer */ - if ((buffer = malloc(bsize)) == NULL) { - HDfprintf(stderr, "malloc for transfer buffer size (%zu) failed\n", bsize); - GOTOERROR(FAIL); - } - - if (pio_debug_level >= 4) { - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - - /* output all of the times for all iterations */ - if (myrank == 0) - HDfprintf(output, "Timer details:\n"); - } - - for (nf = 1; nf <= param.num_files; nf++) { - /* - * Write performance measurement - */ - /* Open file for write */ - char base_name[256]; - - HDsprintf(base_name, "#pio_tmp_%lu", nf); - pio_create_filename(iot, base_name, fname, sizeof(fname)); - if (pio_debug_level > 0) - HDfprintf(output, "rank %d: data filename=%s\n", pio_mpi_rank_g, fname); - - /* Need barrier to make sure everyone starts at the same time */ - MPI_Barrier(pio_comm_g); - - io_time_set(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTART); - hrc = do_fopen(¶m, fname, &fd, PIO_CREATE | PIO_WRITE); - - VRFY((hrc == SUCCESS), "do_fopen failed"); - - io_time_set(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTART); - hrc = do_write(&res, &fd, ¶m, ndsets, nbytes, buf_size, buffer); - io_time_set(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTOP); - - VRFY((hrc == SUCCESS), "do_write failed"); - - /* Close file for write */ - hrc = do_fclose(iot, &fd); - - io_time_set(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_fclose failed"); - - if (!param.h5_write_only) { - /* - * Read performance measurement - */ - /* Need barrier to make sure everyone is done writing and has - * closed the file. Also to make sure everyone starts reading - * at the same time. - */ - MPI_Barrier(pio_comm_g); - - /* Open file for read */ - io_time_set(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTART); - hrc = do_fopen(¶m, fname, &fd, PIO_READ); - - VRFY((hrc == SUCCESS), "do_fopen failed"); - - io_time_set(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTART); - hrc = do_read(&res, &fd, ¶m, ndsets, nbytes, buf_size, buffer); - io_time_set(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_read failed"); - - /* Close file for read */ - hrc = do_fclose(iot, &fd); - - io_time_set(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_fclose failed"); - } - - /* Need barrier to make sure everyone is done with the file */ - /* before it may be removed by do_cleanupfile */ - MPI_Barrier(pio_comm_g); - do_cleanupfile(iot, fname); - } - -done: - /* clean up */ - /* release HDF5 objects */ - - /* close any opened files */ - /* no remove(fname) because that should have happened normally. */ - switch (iot) { - case POSIXIO: - if (fd.posixfd != -1) - hrc = do_fclose(iot, &fd); - break; - case MPIO: - if (fd.mpifd != MPI_FILE_NULL) - hrc = do_fclose(iot, &fd); - break; - case PHDF5: - if (fd.h5fd != -1) - hrc = do_fclose(iot, &fd); - break; - default: - break; - } - - /* release generic resources */ - if (buffer) - HDfree(buffer); - res.ret_code = ret_code; - return res; -} - -/* - * Function: pio_create_filename - * Purpose: Create a new filename to write to. Determine the correct - * suffix to append to the filename by the type of I/O we're - * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if - * USER or LOGIN are specified in the environment. - * Return: Pointer to filename or NULL - * Programmer: Bill Wendling, 21. November 2001 - * Modifications: - */ -static char * -pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size) -{ - const char *prefix, *suffix = ""; - char * ptr, last = '\0'; - size_t i, j; - - if (!base_name || !fullname || size < 1) - return NULL; - - HDmemset(fullname, 0, size); - - switch (iot) { - case POSIXIO: - suffix = ".posix"; - break; - case MPIO: - suffix = ".mpio"; - break; - case PHDF5: - suffix = ".h5"; - break; - default: - break; - } - - /* First use the environment variable and then try the constant */ - prefix = HDgetenv("HDF5_PARAPREFIX"); - -#ifdef HDF5_PARAPREFIX - if (!prefix) - prefix = HDF5_PARAPREFIX; -#endif /* HDF5_PARAPREFIX */ - - /* Prepend the prefix value to the base name */ - if (prefix && *prefix) { - /* If the prefix specifies the HDF5_PARAPREFIX directory, then - * default to using the "/tmp/$USER" or "/tmp/$LOGIN" - * directory instead. */ - register char *user, *login, *subdir; - - user = HDgetenv("USER"); - login = HDgetenv("LOGIN"); - subdir = (user ? user : login); - - if (subdir) { - for (i = 0; i < size - 1 && prefix[i]; i++) - fullname[i] = prefix[i]; - - fullname[i++] = '/'; - - for (j = 0; i < size && subdir[j]; i++, j++) - fullname[i] = subdir[j]; - } - else { - /* We didn't append the prefix yet */ - HDstrncpy(fullname, prefix, size); - fullname[size - 1] = '\0'; - } - - if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) { - /* Append the base_name with a slash first. Multiple slashes are - * handled below. */ - h5_stat_t buf; - - if (HDstat(fullname, &buf) < 0) - /* The directory doesn't exist just yet */ - if (HDmkdir(fullname, (mode_t)0755) < 0 && errno != EEXIST) { - /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. - * Default to PREFIX's original prefix value. */ - HDstrcpy(fullname, prefix); - } - - HDstrcat(fullname, "/"); - HDstrcat(fullname, base_name); - } - else { - /* Buffer is too small */ - return NULL; - } - } - else if (HDstrlen(base_name) >= size) { - /* Buffer is too small */ - return NULL; - } - else { - HDstrcpy(fullname, base_name); - } - - /* Append a suffix */ - if (suffix) { - if (HDstrlen(fullname) + HDstrlen(suffix) >= size) - return NULL; - - HDstrcat(fullname, suffix); - } - - /* Remove any double slashes in the filename */ - for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) { - if (*ptr != '/' || last != '/') - fullname[j++] = *ptr; - - last = *ptr; - } - - return fullname; -} - -/* - * Function: do_write - * Purpose: Write the required amount of data to the file. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static herr_t -do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbytes, size_t buf_size, - void *buffer) -{ - int ret_code = SUCCESS; - int rc; /*routine return code */ - long ndset; - size_t blk_size; /* The block size to subdivide the xfer buffer into */ - off_t nbytes_xfer; /* Total number of bytes transferred so far */ - size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */ - size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */ - char dname[64]; - off_t dset_offset = 0; /*dataset offset in a file */ - off_t bytes_begin[2]; /*first elmt this process transfer */ - off_t bytes_count; /*number of elmts this process transfer */ - off_t snbytes = 0; /*size of a side of the dataset square */ - unsigned char *buf_p; /* Current buffer pointer */ - - /* POSIX variables */ - off_t file_offset; /* File offset of the next transfer */ - off_t file_offset_advance; /* File offset advance after each I/O operation */ - off_t posix_file_offset; /* Base file offset of the next transfer */ - - /* MPI variables */ - MPI_Offset mpi_file_offset; /* Base file offset of the next transfer*/ - MPI_Offset mpi_offset; /* Offset in MPI file */ - MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */ - MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */ - MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */ - MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */ - MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */ - MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */ - MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */ - MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */ - MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */ - MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */ - MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */ - MPI_Status mpi_status; - int mrc; /* MPI return code */ - - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - hsize_t h5dims[2]; /*dataset dim sizes */ - hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ - hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ - hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ - hsize_t h5block[2]; /*dataspace selection */ - hsize_t h5stride[2]; - hsize_t h5count[2]; - hsize_t h5start[2]; - hssize_t h5offset[2]; /* Selection offset within dataspace */ - hid_t h5dcpl = H5I_INVALID_HID; /* Dataset creation property list */ - hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ - - /* Get the parameters from the parameter block */ - blk_size = parms->blk_size; - - /* There are two kinds of transfer patterns, contiguous and interleaved. - * Let 0,1,2,...,n be data accessed by process 0,1,2,...,n - * where n is rank of the last process. - * In contiguous pattern, data are accessed as - * 000...111...222...nnn... - * In interleaved pattern, data are accessed as - * 012...n012...n... - * These are all in the scope of one dataset. - */ - - /* 1D dataspace */ - if (!parms->dim2d) { - /* Contiguous Pattern: */ - if (!parms->interleaved) { - bytes_begin[0] = (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); - } /* end if */ - /* Interleaved Pattern: */ - else { - bytes_begin[0] = (off_t)(blk_size * (size_t)pio_mpi_rank_g); - } /* end else */ - - /* Prepare buffer for verifying data */ - if (parms->verify) - memset(buffer, pio_mpi_rank_g + 1, buf_size); - } /* end if */ - /* 2D dataspace */ - else { - /* nbytes is always the number of bytes per dataset (1D or 2D). If the - dataspace is 2D, snbytes is the size of a side of the dataset square. - */ - snbytes = sqrto(nbytes); - - /* Contiguous Pattern: */ - if (!parms->interleaved) { - bytes_begin[0] = (off_t)((double)snbytes * pio_mpi_rank_g / pio_mpi_nprocs_g); - bytes_begin[1] = 0; - } /* end if */ - /* Interleaved Pattern: */ - else { - bytes_begin[0] = 0; - - if (!parms->h5_use_chunks || parms->io_type == PHDF5) - bytes_begin[1] = (off_t)(blk_size * (size_t)pio_mpi_rank_g); - else - bytes_begin[1] = (off_t)(blk_size * blk_size * (size_t)pio_mpi_rank_g); - } /* end else */ - - /* Prepare buffer for verifying data */ - if (parms->verify) - HDmemset(buffer, pio_mpi_rank_g + 1, buf_size * blk_size); - } /* end else */ - - /* Calculate the total number of bytes (bytes_count) to be - * transferred by this process. It may be different for different - * transfer pattern due to rounding to integral values. - */ - /* - * Calculate the beginning bytes of this process and the next. - * bytes_count is the difference between these two beginnings. - * This way, it eliminates any rounding errors. - * (This is tricky, don't mess with the formula, rounding errors - * can easily get introduced) */ - bytes_count = (off_t)(((double)nbytes * (pio_mpi_rank_g + 1)) / pio_mpi_nprocs_g) - - (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); - - /* debug */ - if (pio_debug_level >= 4) { - HDprint_rank(output); - if (!parms->dim2d) { - HDfprintf(output, - "Debug(do_write): " - "buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH - "d\n", - buf_size, (long long)bytes_begin[0], (long long)bytes_count); - } - else { - HDfprintf(output, - "Debug(do_write): " - "linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH - "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n", - buf_size * blk_size, (long long)bytes_begin[0], (long long)bytes_begin[1], - (long long)bytes_count); - } - } - - /* I/O Access specific setup */ - switch (parms->io_type) { - case POSIXIO: - /* No extra setup */ - break; - - case MPIO: /* MPI-I/O setup */ - /* 1D dataspace */ - if (!parms->dim2d) { - /* Build block's derived type */ - mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Build file's derived type */ - mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)pio_mpi_nprocs_g, mpi_blk_type, - &mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit file type */ - mrc = MPI_Type_commit(&mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Commit buffer type */ - mrc = MPI_Type_commit(&mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - } /* end if */ - /* 2D dataspace */ - else { - /* Build partial buffer derived type for contiguous access */ - - mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE, &mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit partial buffer derived type */ - mrc = MPI_Type_commit(&mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build contiguous file's derived type */ - mrc = MPI_Type_vector((int)blk_size, (int)1, (int)((size_t)snbytes / buf_size), - mpi_partial_buffer_cont, &mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit contiguous file type */ - mrc = MPI_Type_commit(&mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build partial buffer derived type for interleaved access */ - mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit partial buffer derived type */ - mrc = MPI_Type_commit(&mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build interleaved file's derived type */ - mrc = MPI_Type_vector((int)buf_size, (int)1, (int)((size_t)snbytes / blk_size), - mpi_partial_buffer_inter, &mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit interleaved file type */ - mrc = MPI_Type_commit(&mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build full buffer derived type */ - mrc = MPI_Type_contiguous((int)(blk_size * buf_size), MPI_BYTE, &mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit full buffer derived type */ - mrc = MPI_Type_commit(&mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build full chunk derived type */ - mrc = MPI_Type_contiguous((int)(blk_size * blk_size), MPI_BYTE, &mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit full chunk derived type */ - mrc = MPI_Type_commit(&mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build chunk interleaved file's derived type */ - mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)((size_t)snbytes / blk_size), - mpi_full_chunk, &mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit chunk interleaved file type */ - mrc = MPI_Type_commit(&mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - } /* end else */ - break; - - case PHDF5: /* HDF5 setup */ - /* 1D dataspace */ - if (!parms->dim2d) { - if (nbytes > 0) { - /* define a contiguous dataset of nbytes native bytes */ - h5dims[0] = (hsize_t)nbytes; - h5dset_space_id = H5Screate_simple(1, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - /* Set up the file dset space id to select the pattern to access */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5stride[0] = h5block[0] = blk_size; - h5count[0] = buf_size / blk_size; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5stride[0] = blk_size * (size_t)pio_mpi_nprocs_g; - h5block[0] = blk_size; - h5count[0] = buf_size / blk_size; - } /* end else */ - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, - h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - } /* end if */ - else { - h5dset_space_id = H5Screate(H5S_SCALAR); - VRFY((h5dset_space_id >= 0), "H5Screate"); - } /* end else */ - - /* Create the memory dataspace that corresponds to the xfer buffer */ - if (buf_size > 0) { - h5dims[0] = buf_size; - h5mem_space_id = H5Screate_simple(1, h5dims, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - } /* end if */ - else { - h5mem_space_id = H5Screate(H5S_SCALAR); - VRFY((h5mem_space_id >= 0), "H5Screate"); - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - if (nbytes > 0) { - /* define a contiguous dataset of nbytes native bytes */ - h5dims[0] = (hsize_t)snbytes; - h5dims[1] = (hsize_t)snbytes; - h5dset_space_id = H5Screate_simple(2, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - /* Set up the file dset space id to select the pattern to access */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5start[1] = (hsize_t)bytes_begin[1]; - h5stride[0] = 1; - h5stride[1] = h5block[0] = h5block[1] = blk_size; - h5count[0] = 1; - h5count[1] = buf_size / blk_size; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5start[1] = (hsize_t)bytes_begin[1]; - h5stride[0] = blk_size; - h5stride[1] = blk_size * (size_t)pio_mpi_nprocs_g; - h5block[0] = h5block[1] = blk_size; - h5count[0] = buf_size / blk_size; - h5count[1] = 1; - } /* end else */ - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, - h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - } /* end if */ - else { - h5dset_space_id = H5Screate(H5S_SCALAR); - VRFY((h5dset_space_id >= 0), "H5Screate"); - } /* end else */ - - /* Create the memory dataspace that corresponds to the xfer buffer */ - if (buf_size > 0) { - if (!parms->interleaved) { - h5dims[0] = blk_size; - h5dims[1] = buf_size; - } - else { - h5dims[0] = buf_size; - h5dims[1] = blk_size; - } - h5mem_space_id = H5Screate_simple(2, h5dims, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - } /* end if */ - else { - h5mem_space_id = H5Screate(H5S_SCALAR); - VRFY((h5mem_space_id >= 0), "H5Screate"); - } /* end else */ - } /* end else */ - - /* Create the dataset transfer property list */ - h5dxpl = H5Pcreate(H5P_DATASET_XFER); - if (h5dxpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - /* Change to collective I/O, if asked */ - if (parms->collective) { - hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - break; - - default: - break; - } /* end switch */ - - for (ndset = 1; ndset <= ndsets; ++ndset) { - - /* Calculate dataset offset within a file */ - - /* create dataset */ - switch (parms->io_type) { - case POSIXIO: - case MPIO: - /* both posix and mpi io just need dataset offset in file*/ - dset_offset = (ndset - 1) * nbytes; - break; - - case PHDF5: - h5dcpl = H5Pcreate(H5P_DATASET_CREATE); - if (h5dcpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - /* 1D dataspace */ - if (!parms->dim2d) { - /* Make the dataset chunked if asked */ - if (parms->h5_use_chunks) { - /* Set the chunk size to be the same as the buffer size */ - h5dims[0] = blk_size; - hrc = H5Pset_chunk(h5dcpl, 1, h5dims); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - } /* end if */ - else { - /* 2D dataspace */ - if (parms->h5_use_chunks) { - /* Set the chunk size to be the same as the block size */ - h5dims[0] = blk_size; - h5dims[1] = blk_size; - hrc = H5Pset_chunk(h5dcpl, 2, h5dims); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - } /* end else */ - - HDsprintf(dname, "Dataset_%ld", ndset); - h5ds_id = H5DCREATE(fd->h5fd, dname, ELMT_H5_TYPE, h5dset_space_id, h5dcpl); - - if (h5ds_id < 0) { - HDfprintf(stderr, "HDF5 Dataset Create failed\n"); - GOTOERROR(FAIL); - } - - hrc = H5Pclose(h5dcpl); - /* verifying the close of the dcpl */ - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Close failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - break; - } - - /* The task is to transfer bytes_count bytes, starting at - * bytes_begin position, using transfer buffer of buf_size bytes. - * If interleaved, select buf_size at a time, in round robin - * fashion, according to number of process. Otherwise, select - * all bytes_count in contiguous. - */ - nbytes_xfer = 0; - - /* 1D dataspace */ - if (!parms->dim2d) { - /* Set base file offset for all I/O patterns and POSIX access */ - posix_file_offset = dset_offset + bytes_begin[0]; - - /* Set base file offset for all I/O patterns and MPI access */ - mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]); - } /* end if */ - else { - /* Set base file offset for all I/O patterns and POSIX access */ - posix_file_offset = dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]; - - /* Set base file offset for all I/O patterns and MPI access */ - mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]); - } /* end else */ - - /* Start "raw data" write timer */ - io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART); - - while (nbytes_xfer < bytes_count) { - /* Write */ - /* Calculate offset of write within a dataset/file */ - switch (parms->io_type) { - case POSIXIO: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Contiguous pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + (off_t)nbytes_xfer; - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are written */ - rc = ((ssize_t)buf_size == POSIXWRITE(fd->posixfd, buffer, buf_size)); - VRFY((rc != 0), "POSIXWRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size; - - /* Loop over the buffers to write */ - while (nbytes_toxfer > 0) { - /* Skip offset over blocks of other processes */ - file_offset = posix_file_offset + (off_t)(nbytes_xfer * pio_mpi_nprocs_g); - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are written */ - rc = ((ssize_t)blk_size == POSIXWRITE(fd->posixfd, buf_p, blk_size)); - VRFY((rc != 0), "POSIXWRITE"); - - /* Advance location in buffer */ - buf_p += blk_size; - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)blk_size; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= blk_size; - } /* end while */ - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* Contiguous storage */ - if (!parms->h5_use_chunks) { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + - (off_t)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * - (blk_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / blk_size) % (size_t)snbytes)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = buf_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = (off_t)snbytes; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute file offset */ - file_offset = - posix_file_offset + - (off_t)(((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / - (size_t)snbytes) * - (buf_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % - (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = (off_t)snbytes; - } /* end else */ - } /* end if */ - /* Chunked storage */ - else { - /*Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + (off_t)nbytes_xfer; - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * buf_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = 0; - } /* end if */ - /*Interleaved access pattern */ - else { - /* Compute file offset */ - /* Before simplification */ - /* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size) - *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size - *snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size)) - *pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */ - - file_offset = posix_file_offset + - (off_t)((((size_t)nbytes_xfer / (buf_size / blk_size) * - (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * blk_size)) * - (buf_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / (buf_size / blk_size)) * - (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * blk_size)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * blk_size; - - /* Global offset advance after each I/O operation */ - /* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */ - file_offset_advance = (off_t)snbytes * (off_t)blk_size; - } /* end else */ - } /* end else */ - - /* Common code for file access */ - - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size * blk_size; - - /* Loop over portions of the buffer to write */ - while (nbytes_toxfer > 0) { - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are written */ - rc = ((ssize_t)nbytes_xfer_advance == - POSIXWRITE(fd->posixfd, buf_p, nbytes_xfer_advance)); - VRFY((rc != 0), "POSIXWRITE"); - - /* Advance location in buffer */ - buf_p += nbytes_xfer_advance; - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)nbytes_xfer_advance; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= nbytes_xfer_advance; - - /* Partially advance file offset */ - file_offset += file_offset_advance; - } /* end while */ - - } /* end else */ - - break; - - case MPIO: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Independent file access */ - if (!parms->collective) { - /* Contiguous pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Perform independent write */ - mrc = - MPI_File_write_at(fd->mpifd, mpi_offset, buffer, - (int)(buf_size / blk_size), mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size; - - /* Loop over the buffers to write */ - while (nbytes_toxfer > 0) { - /* Skip offset over blocks of other processes */ - mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); - - /* Perform independent write */ - mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p, (int)1, - mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance location in buffer */ - buf_p += blk_size; - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)blk_size; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= blk_size; - } /* end while */ - } /* end else */ - } /* end if */ - /* Collective file access */ - else { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Perform independent write */ - mrc = MPI_File_write_at_all(fd->mpifd, mpi_offset, buffer, - (int)(buf_size / blk_size), mpi_blk_type, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); - - /* Set the file view */ - mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, - (char *)"native", h5_io_info_g); - VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); - - /* Perform write */ - mrc = MPI_File_write_at_all(fd->mpifd, 0, buffer, (int)(buf_size / blk_size), - mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)buf_size; - } /* end else */ - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* Contiguous storage */ - if (!parms->h5_use_chunks) { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = - mpi_file_offset + - (MPI_Offset)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * - (blk_size * (size_t)snbytes)) + - (MPI_Offset)(((size_t)nbytes_xfer / blk_size) % (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = buf_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = snbytes; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_cont_type; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute offset in file */ - mpi_offset = - mpi_file_offset + - (MPI_Offset)( - ((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / - (size_t)snbytes) * - (buf_size * (size_t)snbytes)) + - (MPI_Offset)( - (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % - (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = snbytes; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_inter_type; - } /* end else */ - } /* end if */ - /* Chunked storage */ - else { - /*Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * buf_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = 0; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_full_buffer; - } /* end if */ - /*Interleaved access pattern */ - else { - /* Compute offset in file */ - /* Before simplification */ - /* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size) - *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))* - (buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+ - ((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes - /blk_size*(blk_size*blk_size)); */ - mpi_offset = mpi_file_offset + - (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size) * - (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * blk_size)) * - (buf_size * (size_t)snbytes)) + - (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size)) * - (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * blk_size)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * blk_size; - - /* Global offset advance after each I/O operation */ - /* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */ - mpi_offset_advance = (MPI_Offset)((size_t)snbytes * blk_size); - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_chunk_inter_type; - } /* end else */ - } /* end else */ - - /* Common code for independent file access */ - if (!parms->collective) { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size * blk_size; - - /* Loop over portions of the buffer to write */ - while (nbytes_toxfer > 0) { - /* Perform independent write */ - mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p, - (int)nbytes_xfer_advance, MPI_BYTE, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance location in buffer */ - buf_p += nbytes_xfer_advance; - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)nbytes_xfer_advance; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= nbytes_xfer_advance; - - /* Partially advance global offset in dataset */ - mpi_offset += mpi_offset_advance; - } /* end while */ - } /* end if */ - - /* Common code for collective file access */ - else { - /* Set the file view */ - mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, - (char *)"native", h5_io_info_g); - VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); - - /* Perform write */ - MPI_File_write_at_all(fd->mpifd, 0, buffer, (int)(buf_size * blk_size), MPI_BYTE, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size * (off_t)blk_size; - } /* end else */ - - } /* end else */ - - break; - - case PHDF5: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Set up the file dset space id to move the selection to process */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5offset[0] = nbytes_xfer; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5offset[0] = (nbytes_xfer * pio_mpi_nprocs_g); - } /* end else */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Write the buffer out */ - hrc = - H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dwrite"); - - /* Increment number of bytes transferred */ - nbytes_xfer += (ssize_t)buf_size; - } /* end if */ - /* 2D dataspace */ - else { - /* Set up the file dset space id to move the selection to process */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5offset[0] = - (hssize_t)(((size_t)nbytes_xfer / ((size_t)snbytes * blk_size)) * blk_size); - h5offset[1] = - (hssize_t)(((size_t)nbytes_xfer % ((size_t)snbytes * blk_size)) / blk_size); - - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5offset[0] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * buf_size)) * - buf_size); - h5offset[1] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * buf_size)) / - buf_size); - - } /* end else */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Write the buffer out */ - hrc = - H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dwrite"); - - /* Increment number of bytes transferred */ - nbytes_xfer += (off_t)buf_size * (off_t)blk_size; - - } /* end else */ - - break; - - default: - break; - } /* switch (parms->io_type) */ - } /* end while */ - - /* Stop "raw data" write timer */ - io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP); - - /* Calculate write time */ - - /* Close dataset. Only HDF5 needs to do an explicit close. */ - if (parms->io_type == PHDF5) { - hrc = H5Dclose(h5ds_id); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Close failed\n"); - GOTOERROR(FAIL); - } - - h5ds_id = H5I_INVALID_HID; - } /* end if */ - } /* end for */ - -done: - /* release MPI-I/O objects */ - if (parms->io_type == MPIO) { - /* 1D dataspace */ - if (!parms->dim2d) { - /* Free file type */ - mrc = MPI_Type_free(&mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free buffer type */ - mrc = MPI_Type_free(&mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - } /* end if */ - /* 2D dataspace */ - else { - /* Free partial buffer type for contiguous access */ - mrc = MPI_Type_free(&mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free contiguous file type */ - mrc = MPI_Type_free(&mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free partial buffer type for interleaved access */ - mrc = MPI_Type_free(&mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free interleaved file type */ - mrc = MPI_Type_free(&mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free full buffer type */ - mrc = MPI_Type_free(&mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free full chunk type */ - mrc = MPI_Type_free(&mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free chunk interleaved file type */ - mrc = MPI_Type_free(&mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - } /* end else */ - } /* end if */ - - /* release HDF5 objects */ - if (h5dset_space_id != -1) { - hrc = H5Sclose(h5dset_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); - ret_code = FAIL; - } - else { - h5dset_space_id = H5I_INVALID_HID; - } - } - - if (h5mem_space_id != -1) { - hrc = H5Sclose(h5mem_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); - ret_code = FAIL; - } - else { - h5mem_space_id = H5I_INVALID_HID; - } - } - - if (h5dxpl != -1) { - hrc = H5Pclose(h5dxpl); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); - ret_code = FAIL; - } - else { - h5dxpl = H5I_INVALID_HID; - } - } - - return ret_code; -} - -static off_t -sqrto(off_t x) -{ - double root_x = sqrt((double)x); - return (off_t)root_x; -} - -/* - * Function: do_read - * Purpose: read the required amount of data from the file. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng 2001/12/13 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static herr_t -do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbytes, size_t buf_size, - void *buffer /*out*/) -{ - int ret_code = SUCCESS; - int rc; /*routine return code */ - long ndset; - size_t blk_size; /* The block size to subdivide the xfer buffer into */ - size_t bsize; /* Size of the actual buffer */ - off_t nbytes_xfer; /* Total number of bytes transferred so far */ - size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */ - size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */ - char dname[64]; - off_t dset_offset = 0; /*dataset offset in a file */ - off_t bytes_begin[2]; /*first elmt this process transfer */ - off_t bytes_count; /*number of elmts this process transfer */ - off_t snbytes = 0; /*size of a side of the dataset square */ - unsigned char *buf_p; /* Current buffer pointer */ - - /* POSIX variables */ - off_t file_offset; /* File offset of the next transfer */ - off_t file_offset_advance; /* File offset advance after each I/O operation */ - off_t posix_file_offset; /* Base file offset of the next transfer */ - - /* MPI variables */ - MPI_Offset mpi_file_offset; /* Base file offset of the next transfer*/ - MPI_Offset mpi_offset; /* Offset in MPI file */ - MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */ - MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */ - MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */ - MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */ - MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */ - MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */ - MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */ - MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */ - MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */ - MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */ - MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */ - MPI_Status mpi_status; - int mrc; /* MPI return code */ - - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - hsize_t h5dims[2]; /*dataset dim sizes */ - hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ - hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ - hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ - hsize_t h5block[2]; /*dataspace selection */ - hsize_t h5stride[2]; - hsize_t h5count[2]; - hsize_t h5start[2]; - hssize_t h5offset[2]; /* Selection offset within dataspace */ - hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ - - /* Get the parameters from the parameter block */ - blk_size = parms->blk_size; - - /* There are two kinds of transfer patterns, contiguous and interleaved. - * Let 0,1,2,...,n be data accessed by process 0,1,2,...,n - * where n is rank of the last process. - * In contiguous pattern, data are accessed as - * 000...111...222...nnn... - * In interleaved pattern, data are accessed as - * 012...n012...n... - * These are all in the scope of one dataset. - */ - - /* 1D dataspace */ - if (!parms->dim2d) { - bsize = buf_size; - /* Contiguous Pattern: */ - if (!parms->interleaved) { - bytes_begin[0] = (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); - } /* end if */ - /* Interleaved Pattern: */ - else { - bytes_begin[0] = (off_t)blk_size * (off_t)pio_mpi_rank_g; - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* nbytes is always the number of bytes per dataset (1D or 2D). If the - dataspace is 2D, snbytes is the size of a side of the 'dataset square'. - */ - snbytes = sqrto(nbytes); - - bsize = buf_size * blk_size; - - /* Contiguous Pattern: */ - if (!parms->interleaved) { - bytes_begin[0] = (off_t)((double)snbytes * pio_mpi_rank_g / pio_mpi_nprocs_g); - bytes_begin[1] = 0; - } /* end if */ - /* Interleaved Pattern: */ - else { - bytes_begin[0] = 0; - - if (!parms->h5_use_chunks || parms->io_type == PHDF5) - bytes_begin[1] = (off_t)blk_size * (off_t)pio_mpi_rank_g; - else - bytes_begin[1] = (off_t)blk_size * (off_t)blk_size * (off_t)pio_mpi_rank_g; - } /* end else */ - } /* end else */ - - /* Calculate the total number of bytes (bytes_count) to be - * transferred by this process. It may be different for different - * transfer pattern due to rounding to integral values. - */ - /* - * Calculate the beginning bytes of this process and the next. - * bytes_count is the difference between these two beginnings. - * This way, it eliminates any rounding errors. - * (This is tricky, don't mess with the formula, rounding errors - * can easily get introduced) */ - bytes_count = (off_t)(((double)nbytes * (pio_mpi_rank_g + 1)) / pio_mpi_nprocs_g) - - (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); - - /* debug */ - if (pio_debug_level >= 4) { - HDprint_rank(output); - if (!parms->dim2d) { - HDfprintf(output, - "Debug(do_write): " - "buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH - "d\n", - buf_size, (long long)bytes_begin[0], (long long)bytes_count); - } - else { - HDfprintf(output, - "Debug(do_write): " - "linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH - "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n", - buf_size * blk_size, (long long)bytes_begin[0], (long long)bytes_begin[1], - (long long)bytes_count); - } - } - - /* I/O Access specific setup */ - switch (parms->io_type) { - case POSIXIO: - /* No extra setup */ - break; - - case MPIO: /* MPI-I/O setup */ - /* 1D dataspace */ - if (!parms->dim2d) { - /* Build block's derived type */ - mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Build file's derived type */ - mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)pio_mpi_nprocs_g, mpi_blk_type, - &mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit file type */ - mrc = MPI_Type_commit(&mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Commit buffer type */ - mrc = MPI_Type_commit(&mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - } /* end if */ - /* 2D dataspace */ - else { - /* Build partial buffer derived type for contiguous access */ - mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE, &mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit partial buffer derived type */ - mrc = MPI_Type_commit(&mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build contiguous file's derived type */ - mrc = MPI_Type_vector((int)blk_size, (int)1, (int)((size_t)snbytes / buf_size), - mpi_partial_buffer_cont, &mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit contiguous file type */ - mrc = MPI_Type_commit(&mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build partial buffer derived type for interleaved access */ - mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit partial buffer derived type */ - mrc = MPI_Type_commit(&mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build interleaved file's derived type */ - mrc = MPI_Type_vector((int)buf_size, (int)1, (int)((size_t)snbytes / blk_size), - mpi_partial_buffer_inter, &mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit interleaved file type */ - mrc = MPI_Type_commit(&mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build full buffer derived type */ - mrc = MPI_Type_contiguous((int)(blk_size * buf_size), MPI_BYTE, &mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit full buffer derived type */ - mrc = MPI_Type_commit(&mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build full chunk derived type */ - mrc = MPI_Type_contiguous((int)(blk_size * blk_size), MPI_BYTE, &mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit full chunk derived type */ - mrc = MPI_Type_commit(&mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build chunk interleaved file's derived type */ - mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)((size_t)snbytes / blk_size), - mpi_full_chunk, &mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit chunk interleaved file type */ - mrc = MPI_Type_commit(&mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - } /* end else */ - break; - - case PHDF5: /* HDF5 setup */ - /* 1D dataspace */ - if (!parms->dim2d) { - if (nbytes > 0) { - /* define a contiguous dataset of nbytes native bytes */ - h5dims[0] = (hsize_t)nbytes; - h5dset_space_id = H5Screate_simple(1, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - /* Set up the file dset space id to select the pattern to access */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5stride[0] = h5block[0] = blk_size; - h5count[0] = buf_size / blk_size; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5stride[0] = blk_size * (size_t)pio_mpi_nprocs_g; - h5block[0] = blk_size; - h5count[0] = buf_size / blk_size; - } /* end else */ - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, - h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - } /* end if */ - else { - h5dset_space_id = H5Screate(H5S_SCALAR); - VRFY((h5dset_space_id >= 0), "H5Screate"); - } /* end else */ - - /* Create the memory dataspace that corresponds to the xfer buffer */ - if (buf_size > 0) { - h5dims[0] = buf_size; - h5mem_space_id = H5Screate_simple(1, h5dims, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - } /* end if */ - else { - h5mem_space_id = H5Screate(H5S_SCALAR); - VRFY((h5mem_space_id >= 0), "H5Screate"); - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - if (nbytes > 0) { - /* define a contiguous dataset of nbytes native bytes */ - h5dims[0] = (hsize_t)snbytes; - h5dims[1] = (hsize_t)snbytes; - h5dset_space_id = H5Screate_simple(2, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - /* Set up the file dset space id to select the pattern to access */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5start[1] = (hsize_t)bytes_begin[1]; - h5stride[0] = 1; - h5stride[1] = h5block[0] = h5block[1] = blk_size; - h5count[0] = 1; - h5count[1] = buf_size / blk_size; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5start[1] = (hsize_t)bytes_begin[1]; - h5stride[0] = blk_size; - h5stride[1] = blk_size * (size_t)pio_mpi_nprocs_g; - h5block[0] = h5block[1] = blk_size; - h5count[0] = buf_size / blk_size; - h5count[1] = 1; - } /* end else */ - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, - h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - } /* end if */ - else { - h5dset_space_id = H5Screate(H5S_SCALAR); - VRFY((h5dset_space_id >= 0), "H5Screate"); - } /* end else */ - - /* Create the memory dataspace that corresponds to the xfer buffer */ - if (buf_size > 0) { - if (!parms->interleaved) { - h5dims[0] = blk_size; - h5dims[1] = buf_size; - } - else { - h5dims[0] = buf_size; - h5dims[1] = blk_size; - } - h5mem_space_id = H5Screate_simple(2, h5dims, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - } /* end if */ - else { - h5mem_space_id = H5Screate(H5S_SCALAR); - VRFY((h5mem_space_id >= 0), "H5Screate"); - } /* end else */ - } /* end else */ - - /* Create the dataset transfer property list */ - h5dxpl = H5Pcreate(H5P_DATASET_XFER); - if (h5dxpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - /* Change to collective I/O, if asked */ - if (parms->collective) { - hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - break; - - default: - break; - } /* end switch */ - - for (ndset = 1; ndset <= ndsets; ++ndset) { - - /* Calculate dataset offset within a file */ - - /* create dataset */ - switch (parms->io_type) { - case POSIXIO: - case MPIO: - /* both posix and mpi io just need dataset offset in file*/ - dset_offset = (ndset - 1) * nbytes; - break; - - case PHDF5: - HDsprintf(dname, "Dataset_%ld", ndset); - h5ds_id = H5DOPEN(fd->h5fd, dname); - if (h5ds_id < 0) { - HDfprintf(stderr, "HDF5 Dataset open failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - break; - } - - /* The task is to transfer bytes_count bytes, starting at - * bytes_begin position, using transfer buffer of buf_size bytes. - * If interleaved, select buf_size at a time, in round robin - * fashion, according to number of process. Otherwise, select - * all bytes_count in contiguous. - */ - nbytes_xfer = 0; - - /* 1D dataspace */ - if (!parms->dim2d) { - /* Set base file offset for all I/O patterns and POSIX access */ - posix_file_offset = dset_offset + bytes_begin[0]; - - /* Set base file offset for all I/O patterns and MPI access */ - mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]); - } /* end if */ - else { - /* Set base file offset for all I/O patterns and POSIX access */ - posix_file_offset = dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]; - - /* Set base file offset for all I/O patterns and MPI access */ - mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]); - } /* end else */ - - /* Start "raw data" read timer */ - io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTART); - - while (nbytes_xfer < bytes_count) { - /* Read */ - /* Calculate offset of read within a dataset/file */ - switch (parms->io_type) { - case POSIXIO: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Contiguous pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + (off_t)nbytes_xfer; - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are read */ - rc = ((ssize_t)buf_size == POSIXREAD(fd->posixfd, buffer, buf_size)); - VRFY((rc != 0), "POSIXREAD"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size; - - /* Loop over the buffers to read */ - while (nbytes_toxfer > 0) { - /* Skip offset over blocks of other processes */ - file_offset = posix_file_offset + (off_t)(nbytes_xfer * pio_mpi_nprocs_g); - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are read */ - rc = ((ssize_t)blk_size == POSIXREAD(fd->posixfd, buf_p, blk_size)); - VRFY((rc != 0), "POSIXREAD"); - - /* Advance location in buffer */ - buf_p += blk_size; - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)blk_size; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= blk_size; - } /* end while */ - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* Contiguous storage */ - if (!parms->h5_use_chunks) { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + - (off_t)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * - (blk_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / blk_size) % (size_t)snbytes)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = buf_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = (off_t)snbytes; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute file offset */ - file_offset = - posix_file_offset + - (off_t)(((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / - (size_t)snbytes) * - (buf_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % - (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = (off_t)snbytes; - } /* end else */ - } /* end if */ - /* Chunked storage */ - else { - /*Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + (off_t)nbytes_xfer; - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * buf_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = 0; - } /* end if */ - /*Interleaved access pattern */ - else { - /* Compute file offset */ - /* Before simplification */ - /* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size) - *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size - *snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size)) - *pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */ - - file_offset = posix_file_offset + - (off_t)((((size_t)nbytes_xfer / (buf_size / blk_size) * - (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * blk_size)) * - (buf_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / (buf_size / blk_size)) * - (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * blk_size)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * blk_size; - - /* Global offset advance after each I/O operation */ - /* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */ - file_offset_advance = (off_t)((size_t)snbytes * blk_size); - } /* end else */ - } /* end else */ - - /* Common code for file access */ - - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size * blk_size; - - /* Loop over portions of the buffer to read */ - while (nbytes_toxfer > 0) { - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are read */ - rc = ((ssize_t)nbytes_xfer_advance == - POSIXREAD(fd->posixfd, buf_p, nbytes_xfer_advance)); - VRFY((rc != 0), "POSIXREAD"); - - /* Advance location in buffer */ - buf_p += nbytes_xfer_advance; - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)nbytes_xfer_advance; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= nbytes_xfer_advance; - - /* Partially advance file offset */ - file_offset += file_offset_advance; - } /* end while */ - - } /* end else */ - break; - - case MPIO: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Independent file access */ - if (!parms->collective) { - /* Contiguous pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Perform independent read */ - mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buffer, - (int)(buf_size / blk_size), mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size; - - /* Loop over the buffers to read */ - while (nbytes_toxfer > 0) { - /* Skip offset over blocks of other processes */ - mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); - - /* Perform independent read */ - mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p, (int)1, mpi_blk_type, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance location in buffer */ - buf_p += blk_size; - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)blk_size; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= blk_size; - } /* end while */ - } /* end else */ - } /* end if */ - /* Collective file access */ - else { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Perform collective read */ - mrc = MPI_File_read_at_all(fd->mpifd, mpi_offset, buffer, - (int)(buf_size / blk_size), mpi_blk_type, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); - - /* Set the file view */ - mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, - (char *)"native", h5_io_info_g); - VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); - - /* Perform collective read */ - mrc = MPI_File_read_at_all(fd->mpifd, 0, buffer, (int)(buf_size / blk_size), - mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size; - } /* end else */ - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* Contiguous storage */ - if (!parms->h5_use_chunks) { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = - mpi_file_offset + - (MPI_Offset)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * - (blk_size * (size_t)snbytes)) + - (MPI_Offset)(((size_t)nbytes_xfer / blk_size) % (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = buf_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = snbytes; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_cont_type; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute offset in file */ - mpi_offset = - mpi_file_offset + - (MPI_Offset)( - ((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / - (size_t)snbytes) * - (buf_size * (size_t)snbytes)) + - (MPI_Offset)( - (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % - (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = snbytes; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_inter_type; - } /* end else */ - } /* end if */ - /* Chunked storage */ - else { - /*Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * buf_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = 0; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_full_buffer; - } /* end if */ - /*Interleaved access pattern */ - else { - /* Compute offset in file */ - /* Before simplification */ - /* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size) - *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))* - (buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+ - ((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes - /blk_size*(blk_size*blk_size)); */ - mpi_offset = mpi_file_offset + - (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size) * - (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * blk_size)) * - (buf_size * (size_t)snbytes)) + - (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size)) * - (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * blk_size)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * blk_size; - - /* Global offset advance after each I/O operation */ - /* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */ - mpi_offset_advance = (MPI_Offset)((size_t)snbytes * blk_size); - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_chunk_inter_type; - } /* end else */ - } /* end else */ - - /* Common code for independent file access */ - if (!parms->collective) { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size * blk_size; - - /* Loop over portions of the buffer to read */ - while (nbytes_toxfer > 0) { - /* Perform independent read */ - mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p, (int)nbytes_xfer_advance, - MPI_BYTE, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance location in buffer */ - buf_p += nbytes_xfer_advance; - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)nbytes_xfer_advance; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= nbytes_xfer_advance; - - /* Partially advance global offset in dataset */ - mpi_offset += mpi_offset_advance; - } /* end while */ - } /* end if */ - - /* Common code for collective file access */ - else { - /* Set the file view */ - mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, - (char *)"native", h5_io_info_g); - VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); - - /* Perform read */ - MPI_File_read_at_all(fd->mpifd, 0, buffer, (int)(buf_size * blk_size), MPI_BYTE, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size * (off_t)blk_size; - } /* end else */ - - } /* end else */ - break; - - case PHDF5: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Set up the file dset space id to move the selection to process */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5offset[0] = nbytes_xfer; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5offset[0] = (nbytes_xfer * pio_mpi_nprocs_g); - } /* end else */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Read the buffer in */ - hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dread"); - - /* Increment number of bytes transferred */ - nbytes_xfer += (off_t)buf_size; - } /* end if */ - /* 2D dataspace */ - else { - /* Set up the file dset space id to move the selection to process */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5offset[0] = - (hssize_t)(((size_t)nbytes_xfer / ((size_t)snbytes * blk_size)) * blk_size); - h5offset[1] = - (hssize_t)(((size_t)nbytes_xfer % ((size_t)snbytes * blk_size)) / blk_size); - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5offset[0] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * buf_size)) * - buf_size); - h5offset[1] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * buf_size)) / - buf_size); - - } /* end else */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Write the buffer out */ - hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dread"); - - /* Increment number of bytes transferred */ - nbytes_xfer += (off_t)buf_size * (off_t)blk_size; - - } /* end else */ - break; - - default: - break; - } /* switch (parms->io_type) */ - - /* Verify raw data, if asked */ - if (parms->verify) { - /* Verify data read */ - unsigned char *ucharptr = (unsigned char *)buffer; - size_t i; - int nerror = 0; - - for (i = 0; i < bsize; ++i) { - if (*ucharptr++ != pio_mpi_rank_g + 1) { - if (++nerror < 20) { - /* report at most 20 errors */ - HDprint_rank(output); - HDfprintf(output, - "read data error, expected (%d), " - "got (%d)\n", - pio_mpi_rank_g + 1, (int)*(ucharptr - 1)); - } /* end if */ - } /* end if */ - } /* end for */ - if (nerror >= 20) { - HDprint_rank(output); - HDfprintf(output, "..."); - HDfprintf(output, "total read data errors=%d\n", nerror); - } /* end if */ - } /* if (parms->verify) */ - - } /* end while */ - - /* Stop "raw data" read timer */ - io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTOP); - - /* Calculate read time */ - - /* Close dataset. Only HDF5 needs to do an explicit close. */ - if (parms->io_type == PHDF5) { - hrc = H5Dclose(h5ds_id); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Close failed\n"); - GOTOERROR(FAIL); - } - - h5ds_id = H5I_INVALID_HID; - } /* end if */ - } /* end for */ - -done: - /* release MPI-I/O objects */ - if (parms->io_type == MPIO) { - /* 1D dataspace */ - if (!parms->dim2d) { - /* Free file type */ - mrc = MPI_Type_free(&mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free buffer type */ - mrc = MPI_Type_free(&mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - } /* end if */ - /* 2D dataspace */ - else { - /* Free partial buffer type for contiguous access */ - mrc = MPI_Type_free(&mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free contiguous file type */ - mrc = MPI_Type_free(&mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free partial buffer type for interleaved access */ - mrc = MPI_Type_free(&mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free interleaved file type */ - mrc = MPI_Type_free(&mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free full buffer type */ - mrc = MPI_Type_free(&mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free full chunk type */ - mrc = MPI_Type_free(&mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free chunk interleaved file type */ - mrc = MPI_Type_free(&mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - } /* end else */ - } /* end if */ - - /* release HDF5 objects */ - if (h5dset_space_id != -1) { - hrc = H5Sclose(h5dset_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); - ret_code = FAIL; - } - else { - h5dset_space_id = H5I_INVALID_HID; - } - } - - if (h5mem_space_id != -1) { - hrc = H5Sclose(h5mem_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); - ret_code = FAIL; - } - else { - h5mem_space_id = H5I_INVALID_HID; - } - } - - if (h5dxpl != -1) { - hrc = H5Pclose(h5dxpl); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); - ret_code = FAIL; - } - else { - h5dxpl = H5I_INVALID_HID; - } - } - - return ret_code; -} - -/* - * Function: do_fopen - * Purpose: Open the specified file. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: - */ -static herr_t -do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) -{ - int ret_code = SUCCESS, mrc; - hid_t acc_tpl = H5I_INVALID_HID; /* file access templates */ - - switch (param->io_type) { - case POSIXIO: - if (flags & (PIO_CREATE | PIO_WRITE)) - fd->posixfd = POSIXCREATE(fname); - else - fd->posixfd = POSIXOPEN(fname, O_RDONLY); - - if (fd->posixfd < 0) { - HDfprintf(stderr, "POSIX File Open failed(%s)\n", fname); - GOTOERROR(FAIL); - } - - /* The perils of POSIX I/O in a parallel environment. The problem is: - * - * - Process n opens a file with truncation and then starts - * writing to the file. - * - Process m also opens the file with truncation, but after - * process n has already started to write to the file. Thus, - * all of the stuff process n wrote is now lost. - */ - MPI_Barrier(pio_comm_g); - - break; - - case MPIO: - if (flags & (PIO_CREATE | PIO_WRITE)) { - MPI_File_delete(fname, h5_io_info_g); - mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_CREATE | MPI_MODE_RDWR, h5_io_info_g, - &fd->mpifd); - - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI File Open failed(%s)\n", fname); - GOTOERROR(FAIL); - } - - /*since MPI_File_open with MPI_MODE_CREATE does not truncate */ - /*filesize , set size to 0 explicitedly. */ - mrc = MPI_File_set_size(fd->mpifd, (MPI_Offset)0); - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI_File_set_size failed\n"); - GOTOERROR(FAIL); - } - } - else { - mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_RDONLY, h5_io_info_g, &fd->mpifd); - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI File Open failed(%s)\n", fname); - GOTOERROR(FAIL); - } - } - - break; - - case PHDF5: - if ((acc_tpl = H5Pcreate(H5P_FILE_ACCESS)) < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - /* Set the file driver to the MPI-IO driver */ - if (H5Pset_fapl_mpio(acc_tpl, pio_comm_g, h5_io_info_g) < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } - - /* Set the alignment of objects in HDF5 file */ - if (H5Pset_alignment(acc_tpl, param->h5_thresh, param->h5_align) < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } - - /* create the parallel file */ - if (flags & (PIO_CREATE | PIO_WRITE)) - fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); - else - fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, acc_tpl); - if (fd->h5fd < 0) { - HDfprintf(stderr, "HDF5 File Create failed(%s)\n", fname); - GOTOERROR(FAIL); - } - - /* verifying the close of the acc_tpl */ - if (H5Pclose(acc_tpl) < 0) { - HDfprintf(stderr, "HDF5 Property List Close failed\n"); - GOTOERROR(FAIL); - } - - break; - - default: - break; - } - -done: - return ret_code; -} - -/* - * Function: do_fclose - * Purpose: Close the specified file descriptor. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: - */ -static herr_t -do_fclose(iotype iot, file_descr *fd /*out*/) -{ - herr_t ret_code = SUCCESS, hrc; - int mrc = 0, rc = 0; - - switch (iot) { - case POSIXIO: - rc = POSIXCLOSE(fd->posixfd); - - if (rc != 0) { - HDfprintf(stderr, "POSIX File Close failed\n"); - GOTOERROR(FAIL); - } - - fd->posixfd = -1; - break; - - case MPIO: - mrc = MPI_File_close(&fd->mpifd); - - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI File close failed\n"); - GOTOERROR(FAIL); - } - - fd->mpifd = MPI_FILE_NULL; - break; - - case PHDF5: - hrc = H5Fclose(fd->h5fd); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 File Close failed\n"); - GOTOERROR(FAIL); - } - - fd->h5fd = -1; - break; - - default: - break; - } - -done: - return ret_code; -} - -/* - * Function: do_fclose - * Purpose: Cleanup temporary file unless HDF5_NOCLEANUP is set. - * Only Proc 0 of the PIO communicator will do the cleanup. - * Other processes just return. - * Return: void - * Programmer: Albert Cheng 2001/12/12 - * Modifications: - */ -static void -do_cleanupfile(iotype iot, char *fname) -{ - if (pio_mpi_rank_g != 0) - return; - - if (clean_file_g == -1) - clean_file_g = (getenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; - - if (clean_file_g) { - switch (iot) { - case POSIXIO: - HDremove(fname); - break; - case MPIO: - case PHDF5: - MPI_File_delete(fname, h5_io_info_g); - break; - default: - break; - } - } -} - -#ifdef TIME_MPI -/* instrument the MPI_File_wrirte_xxx and read_xxx calls to measure - * pure time spent in MPI_File code. - */ -int -MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status) -{ - int err; - io_time_set(timer_g, HDF5_MPI_READ, TSTART); - err = PMPI_File_read_at(fh, offset, buf, count, datatype, status); - io_time_set(timer_g, HDF5_MPI_READ, TSTOP); - return err; -} - -int -MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status) -{ - int err; - io_time_set(timer_g, HDF5_MPI_READ, TSTART); - err = PMPI_File_read_at_all(fh, offset, buf, count, datatype, status); - io_time_set(timer_g, HDF5_MPI_READ, TSTOP); - return err; -} - -int -MPI_File_write_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status) -{ - int err; - io_time_set(timer_g, HDF5_MPI_WRITE, TSTART); - err = PMPI_File_write_at(fh, offset, buf, count, datatype, status); - io_time_set(timer_g, HDF5_MPI_WRITE, TSTOP); - return err; -} - -int -MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status) -{ - int err; - io_time_set(timer_g, HDF5_MPI_WRITE, TSTART); - err = PMPI_File_write_at_all(fh, offset, buf, count, datatype, status); - io_time_set(timer_g, HDF5_MPI_WRITE, TSTOP); - return err; -} - -#endif /* TIME_MPI */ -#endif /* H5_HAVE_PARALLEL */ diff --git a/tools/test/perform/pio_perf.c b/tools/test/perform/pio_perf.c deleted file mode 100644 index d38c574..0000000 --- a/tools/test/perform/pio_perf.c +++ /dev/null @@ -1,1699 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Parallel HDF5 Performance Testing Code - * -------------------------------------- - * - * Portable code to test performance on the different platforms we support. - * This is what the report should look like: - * - * nprocs = Max#Procs - * IO API = POSIXIO - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * IO API = MPIO - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * IO API = PHDF5 - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * nprocs = Max#Procs / 2 - * - * . . . - * - */ - -/* system header files */ -#include -#include -#include - -#include "hdf5.h" - -#ifdef H5_HAVE_PARALLEL - -/* library header files */ -#include - -/* our header files */ -#include "pio_perf.h" - -/* useful macros */ -#define TAB_SPACE 4 - -#define ONE_KB 1024 -#define ONE_MB (ONE_KB * ONE_KB) -#define ONE_GB (ONE_MB * ONE_KB) - -#define PIO_POSIX 0x1 -#define PIO_MPI 0x2 -#define PIO_HDF5 0x4 - -#ifdef STANDALONE -#define DBL_EPSILON 2.2204460492503131e-16 -#define H5_DBL_ABS_EQUAL(X, Y) (fabs((X) - (Y)) < DBL_EPSILON) -#endif - -/* report 0.0 in case t is zero too */ -#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL((t), 0.0) ? 0.0 : ((((double)bytes) / ONE_MB) / (t))) - -#ifndef TRUE -#define TRUE 1 -#endif /* TRUE */ -#ifndef FALSE -#define FALSE (!TRUE) -#endif /* FALSE */ - -/* global variables */ -FILE * output; /* output file */ -int comm_world_rank_g; /* my rank in MPI_COMM_RANK */ -int comm_world_nprocs_g; /* num. of processes of MPI_COMM_WORLD */ -MPI_Comm pio_comm_g; /* Communicator to run the PIO */ -int pio_mpi_rank_g; /* MPI rank of pio_comm_g */ -int pio_mpi_nprocs_g; /* Number of processes of pio_comm_g */ -int pio_debug_level = 0; /* The debug level: - * 0 - Off - * 1 - Minimal - * 2 - Some more - * 3 - Maximal - * 4 - Maximal & then some - */ - -/* local variables */ -static const char *progname = "h5perf"; - -/* - * Command-line options: The user can specify short or long-named - * parameters. The long-named ones can be partially spelled. When - * adding more, make sure that they don't clash with each other. - */ -#if 1 -static const char *s_opts = "a:A:B:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; -#else -static const char *s_opts = "a:A:bB:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; -#endif /* 1 */ -static struct h5_long_options l_opts[] = {{"align", require_arg, 'a'}, - {"alig", require_arg, 'a'}, - {"ali", require_arg, 'a'}, - {"al", require_arg, 'a'}, - {"api", require_arg, 'A'}, - {"ap", require_arg, 'A'}, -#if 0 - /* a sighting of the elusive binary option */ - { "binary", no_arg, 'b' }, - { "binar", no_arg, 'b' }, - { "bina", no_arg, 'b' }, - { "bin", no_arg, 'b' }, - { "bi", no_arg, 'b' }, -#endif /* 0 */ - {"block-size", require_arg, 'B'}, - {"block-siz", require_arg, 'B'}, - {"block-si", require_arg, 'B'}, - {"block-s", require_arg, 'B'}, - {"block-", require_arg, 'B'}, - {"block", require_arg, 'B'}, - {"bloc", require_arg, 'B'}, - {"blo", require_arg, 'B'}, - {"bl", require_arg, 'B'}, - {"chunk", no_arg, 'c'}, - {"chun", no_arg, 'c'}, - {"chu", no_arg, 'c'}, - {"ch", no_arg, 'c'}, - {"collective", no_arg, 'C'}, - {"collectiv", no_arg, 'C'}, - {"collecti", no_arg, 'C'}, - {"collect", no_arg, 'C'}, - {"collec", no_arg, 'C'}, - {"colle", no_arg, 'C'}, - {"coll", no_arg, 'C'}, - {"col", no_arg, 'C'}, - {"co", no_arg, 'C'}, - {"debug", require_arg, 'D'}, - {"debu", require_arg, 'D'}, - {"deb", require_arg, 'D'}, - {"de", require_arg, 'D'}, - {"geometry", no_arg, 'g'}, - {"geometr", no_arg, 'g'}, - {"geomet", no_arg, 'g'}, - {"geome", no_arg, 'g'}, - {"geom", no_arg, 'g'}, - {"geo", no_arg, 'g'}, - {"ge", no_arg, 'g'}, - {"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, - {"interleaved", require_arg, 'I'}, - {"interleave", require_arg, 'I'}, - {"interleav", require_arg, 'I'}, - {"interlea", require_arg, 'I'}, - {"interle", require_arg, 'I'}, - {"interl", require_arg, 'I'}, - {"inter", require_arg, 'I'}, - {"inte", require_arg, 'I'}, - {"int", require_arg, 'I'}, - {"in", require_arg, 'I'}, - {"max-num-processes", require_arg, 'P'}, - {"max-num-processe", require_arg, 'P'}, - {"max-num-process", require_arg, 'P'}, - {"max-num-proces", require_arg, 'P'}, - {"max-num-proce", require_arg, 'P'}, - {"max-num-proc", require_arg, 'P'}, - {"max-num-pro", require_arg, 'P'}, - {"max-num-pr", require_arg, 'P'}, - {"max-num-p", require_arg, 'P'}, - {"min-num-processes", require_arg, 'p'}, - {"min-num-processe", require_arg, 'p'}, - {"min-num-process", require_arg, 'p'}, - {"min-num-proces", require_arg, 'p'}, - {"min-num-proce", require_arg, 'p'}, - {"min-num-proc", require_arg, 'p'}, - {"min-num-pro", require_arg, 'p'}, - {"min-num-pr", require_arg, 'p'}, - {"min-num-p", require_arg, 'p'}, - {"max-xfer-size", require_arg, 'X'}, - {"max-xfer-siz", require_arg, 'X'}, - {"max-xfer-si", require_arg, 'X'}, - {"max-xfer-s", require_arg, 'X'}, - {"max-xfer", require_arg, 'X'}, - {"max-xfe", require_arg, 'X'}, - {"max-xf", require_arg, 'X'}, - {"max-x", require_arg, 'X'}, - {"min-xfer-size", require_arg, 'x'}, - {"min-xfer-siz", require_arg, 'x'}, - {"min-xfer-si", require_arg, 'x'}, - {"min-xfer-s", require_arg, 'x'}, - {"min-xfer", require_arg, 'x'}, - {"min-xfe", require_arg, 'x'}, - {"min-xf", require_arg, 'x'}, - {"min-x", require_arg, 'x'}, - {"num-bytes", require_arg, 'e'}, - {"num-byte", require_arg, 'e'}, - {"num-byt", require_arg, 'e'}, - {"num-by", require_arg, 'e'}, - {"num-b", require_arg, 'e'}, - {"num-dsets", require_arg, 'd'}, - {"num-dset", require_arg, 'd'}, - {"num-dse", require_arg, 'd'}, - {"num-ds", require_arg, 'd'}, - {"num-d", require_arg, 'd'}, - {"num-files", require_arg, 'F'}, - {"num-file", require_arg, 'F'}, - {"num-fil", require_arg, 'F'}, - {"num-fi", require_arg, 'F'}, - {"num-f", require_arg, 'F'}, - {"num-iterations", require_arg, 'i'}, - {"num-iteration", require_arg, 'i'}, - {"num-iteratio", require_arg, 'i'}, - {"num-iterati", require_arg, 'i'}, - {"num-iterat", require_arg, 'i'}, - {"num-itera", require_arg, 'i'}, - {"num-iter", require_arg, 'i'}, - {"num-ite", require_arg, 'i'}, - {"num-it", require_arg, 'i'}, - {"num-i", require_arg, 'i'}, - {"output", require_arg, 'o'}, - {"outpu", require_arg, 'o'}, - {"outp", require_arg, 'o'}, - {"out", require_arg, 'o'}, - {"ou", require_arg, 'o'}, - {"threshold", require_arg, 'T'}, - {"threshol", require_arg, 'T'}, - {"thresho", require_arg, 'T'}, - {"thresh", require_arg, 'T'}, - {"thres", require_arg, 'T'}, - {"thre", require_arg, 'T'}, - {"thr", require_arg, 'T'}, - {"th", require_arg, 'T'}, - {"write-only", require_arg, 'w'}, - {"write-onl", require_arg, 'w'}, - {"write-on", require_arg, 'w'}, - {"write-o", require_arg, 'w'}, - {"write", require_arg, 'w'}, - {"writ", require_arg, 'w'}, - {"wri", require_arg, 'w'}, - {"wr", require_arg, 'w'}, - {NULL, 0, '\0'}}; - -struct options { - long io_types; /* bitmask of which I/O types to test */ - const char *output_file; /* file to print report to */ - long num_dsets; /* number of datasets */ - long num_files; /* number of files */ - off_t num_bpp; /* number of bytes per proc per dset */ - int num_iters; /* number of iterations */ - int max_num_procs; /* maximum number of processes to use */ - int min_num_procs; /* minimum number of processes to use */ - size_t max_xfer_size; /* maximum transfer buffer size */ - size_t min_xfer_size; /* minimum transfer buffer size */ - size_t blk_size; /* Block size */ - unsigned interleaved; /* Interleaved vs. contiguous blocks */ - unsigned collective; /* Collective vs. independent I/O */ - unsigned dim2d; /* 1D vs. 2D geometry */ - int print_times; /* print times as well as throughputs */ - int print_raw; /* print raw data throughput info */ - off_t h5_alignment; /* alignment in HDF5 file */ - off_t h5_threshold; /* threshold for alignment in HDF5 file */ - int h5_use_chunks; /* Make HDF5 dataset chunked */ - int h5_write_only; /* Perform the write tests only */ - int verify; /* Verify data correctness */ -}; - -typedef struct _minmax { - double min; - double max; - double sum; - int num; -} minmax; - -/* local functions */ -static off_t parse_size_directive(const char *size); -static struct options *parse_command_line(int argc, char *argv[]); -static void run_test_loop(struct options *options); -static int run_test(iotype iot, parameters parms, struct options *opts); -static void output_all_info(minmax *mm, int count, int indent_level); -static void get_minmax(minmax *mm, double val); -static minmax accumulate_minmax_stuff(minmax *mm, int count); -static int create_comm_world(int num_procs, int *doing_pio); -static int destroy_comm_world(void); -static void output_results(const struct options *options, const char *name, minmax *table, int table_size, - off_t data_size); -static void output_times(const struct options *options, const char *name, minmax *table, int table_size); -static void output_report(const char *fmt, ...); -static void print_indent(register int indent); -static void usage(const char *prog); -static void report_parameters(struct options *opts); -static off_t squareo(off_t); - -/* - * Function: main - * Purpose: Start things up. Initialize MPI and then call the test looping - * function. - * Return: EXIT_SUCCESS or EXIT_FAILURE - * Programmer: Bill Wendling, 30. October 2001 - * Modifications: - */ -int -main(int argc, char *argv[]) -{ - int ret; - int exit_value = EXIT_SUCCESS; - struct options *opts = NULL; - -#ifndef STANDALONE - /* Initialize h5tools lib */ - h5tools_init(); -#endif - - output = stdout; - - /* initialize MPI and get the maximum num of processors we started with */ - MPI_Init(&argc, &argv); - ret = MPI_Comm_size(MPI_COMM_WORLD, &comm_world_nprocs_g); - - if (ret != MPI_SUCCESS) { - HDfprintf(stderr, "%s: MPI_Comm_size call failed\n", progname); - - if (ret == MPI_ERR_COMM) - HDfprintf(stderr, "invalid MPI communicator\n"); - else - HDfprintf(stderr, "invalid argument\n"); - - exit_value = EXIT_FAILURE; - goto finish; - } - - ret = MPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank_g); - - if (ret != MPI_SUCCESS) { - HDfprintf(stderr, "%s: MPI_Comm_rank call failed\n", progname); - - if (ret == MPI_ERR_COMM) - HDfprintf(stderr, "invalid MPI communicator\n"); - else - HDfprintf(stderr, "invalid argument\n"); - - exit_value = EXIT_FAILURE; - goto finish; - } - - pio_comm_g = MPI_COMM_WORLD; - - h5_set_info_object(); - opts = parse_command_line(argc, argv); - - if (!opts) { - exit_value = EXIT_FAILURE; - goto finish; - } - - if (opts->output_file) { - if ((output = HDfopen(opts->output_file, "w")) == NULL) { - HDfprintf(stderr, "%s: cannot open output file\n", progname); - perror(opts->output_file); - goto finish; - } - } - - if ((pio_debug_level == 0 && comm_world_rank_g == 0) || pio_debug_level > 0) - report_parameters(opts); - - run_test_loop(opts); - -finish: - MPI_Finalize(); - free(opts); - return exit_value; -} - -off_t -squareo(off_t x) -{ - return x * x; -} - -/* - * Function: run_test_loop - * Purpose: Run the I/O tests. Write the results to OUTPUT. - * - * - The slowest changing part of the test is the number of - * processors to use. For each loop iteration, we divide that - * number by 2 and rerun the test. - * - * - The second slowest is what type of IO API to perform. We have - * three choices: POSIXIO, MPI-IO, and PHDF5. - * - * - Then we change the size of the buffer. This information is - * inferred from the number of datasets to create and the number - * of integers to put into each dataset. The backend code figures - * this out. - * - * Return: Nothing - * Programmer: Bill Wendling, 30. October 2001 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static void -run_test_loop(struct options *opts) -{ - parameters parms; - int num_procs; - int doing_pio; /* if this process is doing PIO */ - - parms.num_files = opts->num_files; - parms.num_dsets = opts->num_dsets; - parms.num_iters = opts->num_iters; - parms.blk_size = opts->blk_size; - parms.interleaved = opts->interleaved; - parms.collective = opts->collective; - parms.dim2d = opts->dim2d; - parms.h5_align = (hsize_t)opts->h5_alignment; - parms.h5_thresh = (hsize_t)opts->h5_threshold; - parms.h5_use_chunks = opts->h5_use_chunks; - parms.h5_write_only = opts->h5_write_only; - parms.verify = opts->verify; - - /* start with max_num_procs and decrement it by half for each loop. */ - /* if performance needs restart, fewer processes may be needed. */ - for (num_procs = opts->max_num_procs; num_procs >= opts->min_num_procs; num_procs >>= 1) { - register size_t buf_size; - - parms.num_procs = num_procs; - - if (create_comm_world(parms.num_procs, &doing_pio) != SUCCESS) { - /* do something harsh */ - } - - /* only processes doing PIO will run the tests */ - if (doing_pio) { - output_report("Number of processors = %ld\n", parms.num_procs); - - /* multiply the xfer buffer size by 2 for each loop iteration */ - for (buf_size = opts->min_xfer_size; buf_size <= opts->max_xfer_size; buf_size <<= 1) { - parms.buf_size = buf_size; - - if (parms.dim2d) { - parms.num_bytes = squareo(opts->num_bpp * parms.num_procs); - if (parms.interleaved) - output_report("Transfer Buffer Size: %ldx%ld bytes, File size: %.2f MB\n", buf_size, - opts->blk_size, - ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); - else - output_report("Transfer Buffer Size: %ldx%ld bytes, File size: %.2f MB\n", - opts->blk_size, buf_size, - ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); - - print_indent(1); - output_report(" # of files: %ld, # of datasets: %ld, dataset size: %.2fx%.2f KB\n", - parms.num_files, parms.num_dsets, - (double)(opts->num_bpp * parms.num_procs) / ONE_KB, - (double)(opts->num_bpp * parms.num_procs) / ONE_KB); - } - else { - parms.num_bytes = (off_t)opts->num_bpp * parms.num_procs; - output_report("Transfer Buffer Size: %ld bytes, File size: %.2f MB\n", buf_size, - ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); - - print_indent(1); - output_report(" # of files: %ld, # of datasets: %ld, dataset size: %.2f MB\n", - parms.num_files, parms.num_dsets, - (double)(opts->num_bpp * parms.num_procs) / ONE_MB); - } - - if (opts->io_types & PIO_POSIX) - run_test(POSIXIO, parms, opts); - - if (opts->io_types & PIO_MPI) - run_test(MPIO, parms, opts); - - if (opts->io_types & PIO_HDF5) - run_test(PHDF5, parms, opts); - - /* Run the tests once if buf_size==0, but then break out */ - if (buf_size == 0) - break; - } - - if (destroy_comm_world() != SUCCESS) { - /* do something harsh */ - } - } - } -} - -/* - * Function: run_test - * Purpose: Inner loop call to actually run the I/O test. - * Return: Nothing - * Programmer: Bill Wendling, 18. December 2001 - * Modifications: - */ -static int -run_test(iotype iot, parameters parms, struct options *opts) -{ - results res; - register int i, ret_value = SUCCESS; - int comm_size; - off_t raw_size; - minmax * write_mpi_mm_table = NULL; - minmax * write_mm_table = NULL; - minmax * write_gross_mm_table = NULL; - minmax * write_raw_mm_table = NULL; - minmax * read_mpi_mm_table = NULL; - minmax * read_mm_table = NULL; - minmax * read_gross_mm_table = NULL; - minmax * read_raw_mm_table = NULL; - minmax * read_open_mm_table = NULL; - minmax * read_close_mm_table = NULL; - minmax * write_open_mm_table = NULL; - minmax * write_close_mm_table = NULL; - minmax write_mpi_mm = {0.0, 0.0, 0.0, 0}; - minmax write_mm = {0.0, 0.0, 0.0, 0}; - minmax write_gross_mm = {0.0, 0.0, 0.0, 0}; - minmax write_raw_mm = {0.0, 0.0, 0.0, 0}; - minmax read_mpi_mm = {0.0, 0.0, 0.0, 0}; - minmax read_mm = {0.0, 0.0, 0.0, 0}; - minmax read_gross_mm = {0.0, 0.0, 0.0, 0}; - minmax read_raw_mm = {0.0, 0.0, 0.0, 0}; - minmax read_open_mm = {0.0, 0.0, 0.0, 0}; - minmax read_close_mm = {0.0, 0.0, 0.0, 0}; - minmax write_open_mm = {0.0, 0.0, 0.0, 0}; - minmax write_close_mm = {0.0, 0.0, 0.0, 0}; - - raw_size = parms.num_files * (off_t)parms.num_dsets * (off_t)parms.num_bytes; - parms.io_type = iot; - print_indent(2); - output_report("IO API = "); - - switch (iot) { - case POSIXIO: - output_report("POSIX\n"); - break; - case MPIO: - output_report("MPIO\n"); - break; - case PHDF5: - output_report("PHDF5 (w/MPI-IO driver)\n"); - break; - default: - break; - } - - MPI_Comm_size(pio_comm_g, &comm_size); - - /* allocate space for tables minmax and that it is sufficient */ - /* to initialize all elements to zeros by calloc. */ - write_mpi_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_gross_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_raw_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_open_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_close_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - if (!parms.h5_write_only) { - read_mpi_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_gross_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_raw_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_open_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_close_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - } - - /* Do IO iteration times, collecting statistics each time */ - for (i = 0; i < parms.num_iters; ++i) { - double t; - - MPI_Barrier(pio_comm_g); - res = do_pio(parms); - - /* gather all of the "mpi write" times */ - t = io_time_get(res.timers, HDF5_MPI_WRITE); - get_minmax(&write_mpi_mm, t); - - write_mpi_mm_table[i] = write_mpi_mm; - - /* gather all of the "write" times */ - t = io_time_get(res.timers, HDF5_FINE_WRITE_FIXED_DIMS); - get_minmax(&write_mm, t); - - write_mm_table[i] = write_mm; - - /* gather all of the "write" times from open to close */ - t = io_time_get(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS); - get_minmax(&write_gross_mm, t); - - write_gross_mm_table[i] = write_gross_mm; - - /* gather all of the raw "write" times */ - t = io_time_get(res.timers, HDF5_RAW_WRITE_FIXED_DIMS); - get_minmax(&write_raw_mm, t); - - write_raw_mm_table[i] = write_raw_mm; - - /* gather all of the file open times (time from open to first write) */ - t = io_time_get(res.timers, HDF5_FILE_WRITE_OPEN); - get_minmax(&write_open_mm, t); - - write_open_mm_table[i] = write_open_mm; - - /* gather all of the file close times (time from last write to close) */ - t = io_time_get(res.timers, HDF5_FILE_WRITE_CLOSE); - get_minmax(&write_close_mm, t); - - write_close_mm_table[i] = write_close_mm; - - if (!parms.h5_write_only) { - /* gather all of the "mpi read" times */ - t = io_time_get(res.timers, HDF5_MPI_READ); - get_minmax(&read_mpi_mm, t); - - read_mpi_mm_table[i] = read_mpi_mm; - - /* gather all of the "read" times */ - t = io_time_get(res.timers, HDF5_FINE_READ_FIXED_DIMS); - get_minmax(&read_mm, t); - - read_mm_table[i] = read_mm; - - /* gather all of the "read" times from open to close */ - t = io_time_get(res.timers, HDF5_GROSS_READ_FIXED_DIMS); - get_minmax(&read_gross_mm, t); - - read_gross_mm_table[i] = read_gross_mm; - - /* gather all of the raw "read" times */ - t = io_time_get(res.timers, HDF5_RAW_READ_FIXED_DIMS); - get_minmax(&read_raw_mm, t); - - read_raw_mm_table[i] = read_raw_mm; - - /* gather all of the file open times (time from open to first read) */ - t = io_time_get(res.timers, HDF5_FILE_READ_OPEN); - get_minmax(&read_open_mm, t); - - read_open_mm_table[i] = read_open_mm; - - /* gather all of the file close times (time from last read to close) */ - t = io_time_get(res.timers, HDF5_FILE_READ_CLOSE); - get_minmax(&read_close_mm, t); - - read_close_mm_table[i] = read_close_mm; - } - - io_time_destroy(res.timers); - } - - /* - * Show various statistics - */ - /* Write statistics */ - /* Print the raw data throughput if desired */ - if (opts->print_raw) { - /* accumulate and output the max, min, and average "raw write" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Raw Data Write details:\n"); - output_all_info(write_raw_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); - } /* end if */ - - /* show mpi write statics */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("MPI Write details:\n"); - output_all_info(write_mpi_mm_table, parms.num_iters, 4); - } - - /* We don't currently output the MPI write results */ - - /* accumulate and output the max, min, and average "write" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write details:\n"); - output_all_info(write_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Write", write_mm_table, parms.num_iters, raw_size); - - /* accumulate and output the max, min, and average "gross write" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write Open-Close details:\n"); - output_all_info(write_gross_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Write Open-Close", write_gross_mm_table, parms.num_iters, raw_size); - - if (opts->print_times) { - output_times(opts, "Write File Open", write_open_mm_table, parms.num_iters); - output_times(opts, "Write File Close", write_close_mm_table, parms.num_iters); - } - - /* Print out time from open to first write */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write file open details:\n"); - output_all_info(write_open_mm_table, parms.num_iters, 4); - } - - /* Print out time from last write to close */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write file close details:\n"); - output_all_info(write_close_mm_table, parms.num_iters, 4); - } - - if (!parms.h5_write_only) { - /* Read statistics */ - /* Print the raw data throughput if desired */ - if (opts->print_raw) { - /* accumulate and output the max, min, and average "raw read" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Raw Data Read details:\n"); - output_all_info(read_raw_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); - } /* end if */ - - /* show mpi read statics */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("MPI Read details:\n"); - output_all_info(read_mpi_mm_table, parms.num_iters, 4); - } - - /* We don't currently output the MPI read results */ - - /* accumulate and output the max, min, and average "read" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read details:\n"); - output_all_info(read_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Read", read_mm_table, parms.num_iters, raw_size); - - /* accumulate and output the max, min, and average "gross read" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read Open-Close details:\n"); - output_all_info(read_gross_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Read Open-Close", read_gross_mm_table, parms.num_iters, raw_size); - - if (opts->print_times) { - output_times(opts, "Read File Open", read_open_mm_table, parms.num_iters); - output_times(opts, "Read File Close", read_close_mm_table, parms.num_iters); - } - - /* Print out time from open to first read */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read file open details:\n"); - output_all_info(read_open_mm_table, parms.num_iters, 4); - } - - /* Print out time from last read to close */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read file close details:\n"); - output_all_info(read_close_mm_table, parms.num_iters, 4); - } - } - - /* clean up our mess */ - free(write_mpi_mm_table); - free(write_mm_table); - free(write_gross_mm_table); - free(write_raw_mm_table); - free(write_open_mm_table); - free(write_close_mm_table); - - if (!parms.h5_write_only) { - free(read_mpi_mm_table); - free(read_mm_table); - free(read_gross_mm_table); - free(read_raw_mm_table); - free(read_open_mm_table); - free(read_close_mm_table); - } - - return ret_value; -} - -/* - * Function: output_all_info - * Purpose: - * Return: Nothing - * Programmer: Bill Wendling, 29. January 2002 - * Modifications: - */ -static void -output_all_info(minmax *mm, int count, int indent_level) -{ - int i; - - for (i = 0; i < count; ++i) { - print_indent(indent_level); - output_report("Iteration %d:\n", i + 1); - print_indent(indent_level + 1); - output_report("Minimum Time: %.2fs\n", mm[i].min); - print_indent(indent_level + 1); - output_report("Maximum Time: %.2fs\n", mm[i].max); - } -} - -/* - * Function: get_minmax - * Purpose: Gather all the min, max and total of val. - * Return: Nothing - * Programmer: Bill Wendling, 21. December 2001 - * Modifications: - * Use MPI_Allreduce to do it. -akc, 2002/01/11 - */ -static void -get_minmax(minmax *mm, double val) -{ - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - MPI_Comm_size(pio_comm_g, &mm->num); - - MPI_Allreduce(&val, &mm->max, 1, MPI_DOUBLE, MPI_MAX, pio_comm_g); - MPI_Allreduce(&val, &mm->min, 1, MPI_DOUBLE, MPI_MIN, pio_comm_g); - MPI_Allreduce(&val, &mm->sum, 1, MPI_DOUBLE, MPI_SUM, pio_comm_g); -} - -/* - * Function: accumulate_minmax_stuff - * Purpose: Accumulate the minimum, maximum, and average of the times - * across all processes. - * Return: TOTAL_MM - the total of all of these. - * Programmer: Bill Wendling, 21. December 2001 - * Modifications: - * Changed to use seconds instead of MB/s - QAK, 5/9/02 - */ -static minmax -accumulate_minmax_stuff(minmax *mm, int count) -{ - int i; - minmax total_mm; - - total_mm.sum = 0.0f; - total_mm.max = -DBL_MAX; - total_mm.min = DBL_MAX; - total_mm.num = count; - - for (i = 0; i < count; ++i) { - double m = mm[i].max; - - total_mm.sum += m; - - if (m < total_mm.min) - total_mm.min = m; - - if (m > total_mm.max) - total_mm.max = m; - } - - return total_mm; -} - -/* - * Function: create_comm_world - * Purpose: Create an MPI Comm world and store it in pio_comm_g, which - * is a global variable. - * Return: SUCCESS on success. - * FAIL otherwise. - * Programmer: Bill Wendling, 19. December 2001 - * Modifications: - */ -static int -create_comm_world(int num_procs, int *doing_pio) -{ - /* MPI variables */ - int mrc; /* return values */ - int color; /* for communicator creation */ - int myrank, nprocs; - - pio_comm_g = MPI_COMM_NULL; - - /* - * Create a sub communicator for this PIO run. Easier to use the first N - * processes. - */ - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - if (num_procs > nprocs) { - HDfprintf(stderr, "number of process(%d) must be <= number of processes in MPI_COMM_WORLD(%d)\n", - num_procs, nprocs); - goto error_done; - } - - MPI_Comm_rank(MPI_COMM_WORLD, &myrank); - color = (myrank < num_procs); - mrc = MPI_Comm_split(MPI_COMM_WORLD, color, myrank, &pio_comm_g); - - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI_Comm_split failed\n"); - goto error_done; - } - - if (!color) { - /* not involved in this run */ - mrc = destroy_comm_world(); - goto done; - } - - /* determine the MPI rank in the PIO communicator */ - MPI_Comm_size(pio_comm_g, &pio_mpi_nprocs_g); - MPI_Comm_rank(pio_comm_g, &pio_mpi_rank_g); - -done: - *doing_pio = color; - return SUCCESS; - -error_done: - destroy_comm_world(); - return FAIL; -} - -/* - * Function: destroy_comm_world - * Purpose: Destroy the created MPI Comm world which is stored in the - * pio_comm_g global variable. - * Return: SUCCESS on success. - * FAIL otherwise. - * Programmer: Bill Wendling, 19. December 2001 - * Modifications: - */ -static int -destroy_comm_world(void) -{ - int mrc = SUCCESS; /* return code */ - - /* release MPI resources */ - if (pio_comm_g != MPI_COMM_NULL) - mrc = (MPI_Comm_free(&pio_comm_g) == MPI_SUCCESS ? SUCCESS : FAIL); - - return mrc; -} - -/* - * Function: output_results - * Purpose: Print information about the time & bandwidth for a given - * minmax & # of iterations. - * Return: Nothing - * Programmer: Quincey Koziol, 9. May 2002 - * Modifications: - */ -static void -output_results(const struct options *opts, const char *name, minmax *table, int table_size, off_t data_size) -{ - minmax total_mm; - - total_mm = accumulate_minmax_stuff(table, table_size); - - print_indent(3); - output_report("%s (%d iteration(s)):\n", name, table_size); - - /* Note: The maximum throughput uses the minimum amount of time & vice versa */ - - print_indent(4); - output_report("Maximum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.min)); - if (opts->print_times) - output_report(" (%7.3f s)\n", total_mm.min); - else - output_report("\n"); - - print_indent(4); - output_report("Average Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.sum / total_mm.num)); - if (opts->print_times) - output_report(" (%7.3f s)\n", (total_mm.sum / total_mm.num)); - else - output_report("\n"); - - print_indent(4); - output_report("Minimum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.max)); - if (opts->print_times) - output_report(" (%7.3f s)\n", total_mm.max); - else - output_report("\n"); -} - -static void -output_times(const struct options *opts, const char *name, minmax *table, int table_size) -{ - minmax total_mm; - - total_mm = accumulate_minmax_stuff(table, table_size); - - print_indent(3); - output_report("%s (%d iteration(s)):\n", name, table_size); - - /* Note: The maximum throughput uses the minimum amount of time & vice versa */ - - print_indent(4); - output_report("Minimum Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, (total_mm.min)); - - print_indent(4); - output_report("Average Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, - (total_mm.sum / total_mm.num)); - - print_indent(4); - output_report("Maximum Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, (total_mm.max)); -} - -/* - * Function: output_report - * Purpose: Print a line of the report. Only do so if I'm the 0 process. - * Return: Nothing - * Programmer: Bill Wendling, 19. December 2001 - * Modifications: - */ -static void -output_report(const char *fmt, ...) -{ - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - - if (myrank == 0) { - va_list ap; - - HDva_start(ap, fmt); - HDvfprintf(output, fmt, ap); - HDva_end(ap); - } -} - -/* - * Function: print_indent - * Purpose: Print spaces to indent a new line of text for pretty printing - * things. - * Return: Nothing - * Programmer: Bill Wendling, 29. October 2001 - * Modifications: - */ -static void -print_indent(register int indent) -{ - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - - if (myrank == 0) { - indent *= TAB_SPACE; - - for (; indent > 0; --indent) - HDfputc(' ', output); - } -} - -static void -recover_size_and_print(long long val, const char *end) -{ - if (val >= ONE_KB && (val % ONE_KB) == 0) { - if (val >= ONE_MB && (val % ONE_MB) == 0) { - if (val >= ONE_GB && (val % ONE_GB) == 0) - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "GB%s", - val / ONE_GB, end); - else - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "MB%s", - val / ONE_MB, end); - } - else { - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "KB%s", - val / ONE_KB, end); - } - } - else { - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "%s", - val, end); - } -} - -static void -print_io_api(long io_types) -{ - if (io_types & PIO_POSIX) - HDfprintf(output, "posix "); - if (io_types & PIO_MPI) - HDfprintf(output, "mpiio "); - if (io_types & PIO_HDF5) - HDfprintf(output, "phdf5 "); - HDfprintf(output, "\n"); -} - -static void -report_parameters(struct options *opts) -{ - int rank = comm_world_rank_g; - - print_version("HDF5 Library"); /* print library version */ - HDfprintf(output, "rank %d: ==== Parameters ====\n", rank); - - HDfprintf(output, "rank %d: IO API=", rank); - print_io_api(opts->io_types); - - HDfprintf(output, "rank %d: Number of files=%ld\n", rank, opts->num_files); - HDfprintf(output, "rank %d: Number of datasets=%ld\n", rank, opts->num_dsets); - HDfprintf(output, "rank %d: Number of iterations=%d\n", rank, opts->num_iters); - HDfprintf(output, "rank %d: Number of processes=%d:%d\n", rank, opts->min_num_procs, opts->max_num_procs); - - if (opts->dim2d) { - HDfprintf(output, "rank %d: Number of bytes per process per dataset=", rank); - recover_size_and_print((long long)(opts->num_bpp * opts->num_bpp * opts->min_num_procs), ":"); - recover_size_and_print((long long)(opts->num_bpp * opts->num_bpp * opts->max_num_procs), "\n"); - - HDfprintf(output, "rank %d: Size of dataset(s)=", rank); - recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), "x"); - recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), ":"); - recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "x"); - recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "\n"); - - HDfprintf(output, "rank %d: File size=", rank); - recover_size_and_print((long long)(squareo(opts->num_bpp * opts->min_num_procs) * opts->num_dsets), - ":"); - recover_size_and_print((long long)(squareo(opts->num_bpp * opts->max_num_procs) * opts->num_dsets), - "\n"); - - HDfprintf(output, "rank %d: Transfer buffer size=", rank); - if (opts->interleaved) { - recover_size_and_print((long long)opts->min_xfer_size, "x"); - recover_size_and_print((long long)opts->blk_size, ":"); - recover_size_and_print((long long)opts->max_xfer_size, "x"); - recover_size_and_print((long long)opts->blk_size, "\n"); - } - else { - recover_size_and_print((long long)opts->blk_size, "x"); - recover_size_and_print((long long)opts->min_xfer_size, ":"); - recover_size_and_print((long long)opts->blk_size, "x"); - recover_size_and_print((long long)opts->max_xfer_size, "\n"); - } - HDfprintf(output, "rank %d: Block size=", rank); - recover_size_and_print((long long)opts->blk_size, "x"); - recover_size_and_print((long long)opts->blk_size, "\n"); - } - else { - HDfprintf(output, "rank %d: Number of bytes per process per dataset=", rank); - recover_size_and_print((long long)opts->num_bpp, "\n"); - - HDfprintf(output, "rank %d: Size of dataset(s)=", rank); - recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), ":"); - recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "\n"); - - HDfprintf(output, "rank %d: File size=", rank); - recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs * opts->num_dsets), ":"); - recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs * opts->num_dsets), "\n"); - - HDfprintf(output, "rank %d: Transfer buffer size=", rank); - recover_size_and_print((long long)opts->min_xfer_size, ":"); - recover_size_and_print((long long)opts->max_xfer_size, "\n"); - HDfprintf(output, "rank %d: Block size=", rank); - recover_size_and_print((long long)opts->blk_size, "\n"); - } - - HDfprintf(output, "rank %d: Block Pattern in Dataset=", rank); - if (opts->interleaved) - HDfprintf(output, "Interleaved\n"); - else - HDfprintf(output, "Contiguous\n"); - - HDfprintf(output, "rank %d: I/O Method for MPI and HDF5=", rank); - if (opts->collective) - HDfprintf(output, "Collective\n"); - else - HDfprintf(output, "Independent\n"); - - HDfprintf(output, "rank %d: Geometry=", rank); - if (opts->dim2d) - HDfprintf(output, "2D\n"); - else - HDfprintf(output, "1D\n"); - - HDfprintf(output, "rank %d: VFL used for HDF5 I/O=%s\n", rank, "MPI-IO driver"); - - HDfprintf(output, "rank %d: Data storage method in HDF5=", rank); - if (opts->h5_use_chunks) - HDfprintf(output, "Chunked\n"); - else - HDfprintf(output, "Contiguous\n"); - - { - char *prefix = HDgetenv("HDF5_PARAPREFIX"); - - HDfprintf(output, "rank %d: Env HDF5_PARAPREFIX=%s\n", rank, (prefix ? prefix : "not set")); - } - - HDfprintf(output, "rank %d: ", rank); - h5_dump_info_object(h5_io_info_g); - - HDfprintf(output, "rank %d: ==== End of Parameters ====\n", rank); - HDfprintf(output, "\n"); -} - -/* - * Function: parse_command_line - * Purpose: Parse the command line options and return a STRUCT OPTIONS - * structure which will need to be freed by the calling function. - * Return: Pointer to an OPTIONS structure - * Programmer: Bill Wendling, 31. October 2001 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static struct options * -parse_command_line(int argc, char *argv[]) -{ - register int opt; - struct options *cl_opts; - - cl_opts = (struct options *)malloc(sizeof(struct options)); - - cl_opts->output_file = NULL; - cl_opts->io_types = 0; /* will set default after parsing options */ - cl_opts->num_dsets = 1; - cl_opts->num_files = 1; - cl_opts->num_bpp = 0; - cl_opts->num_iters = 1; - cl_opts->max_num_procs = comm_world_nprocs_g; - cl_opts->min_num_procs = 1; - cl_opts->max_xfer_size = 0; - cl_opts->min_xfer_size = 0; - cl_opts->blk_size = 0; - cl_opts->interleaved = 0; /* Default to contiguous blocks in dataset */ - cl_opts->collective = 0; /* Default to independent I/O access */ - cl_opts->dim2d = 0; /* Default to 1D */ - cl_opts->print_times = FALSE; /* Printing times is off by default */ - cl_opts->print_raw = FALSE; /* Printing raw data throughput is off by default */ - cl_opts->h5_alignment = 1; /* No alignment for HDF5 objects by default */ - cl_opts->h5_threshold = 1; /* No threshold for aligning HDF5 objects by default */ - cl_opts->h5_use_chunks = FALSE; /* Don't chunk the HDF5 dataset by default */ - cl_opts->h5_write_only = FALSE; /* Do both read and write by default */ - cl_opts->verify = FALSE; /* No Verify data correctness by default */ - - while ((opt = H5_get_option(argc, (const char **)argv, s_opts, l_opts)) != EOF) { - switch ((char)opt) { - case 'a': - cl_opts->h5_alignment = parse_size_directive(H5_optarg); - break; - case 'A': { - const char *end = H5_optarg; - - while (end && *end != '\0') { - char buf[10]; - int i; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (isalnum(*end) && i < 10) - buf[i++] = *end; - - if (!HDstrcasecmp(buf, "phdf5")) { - cl_opts->io_types |= PIO_HDF5; - } - else if (!HDstrcasecmp(buf, "mpiio")) { - cl_opts->io_types |= PIO_MPI; - } - else if (!HDstrcasecmp(buf, "posix")) { - cl_opts->io_types |= PIO_POSIX; - } - else { - HDfprintf(stderr, "pio_perf: invalid --api option %s\n", buf); - HDexit(EXIT_FAILURE); - } - - if (*end == '\0') - break; - - end++; - } - } - - break; -#if 0 - case 'b': - /* the future "binary" option */ - break; -#endif /* 0 */ - case 'B': - cl_opts->blk_size = (size_t)parse_size_directive(H5_optarg); - break; - case 'c': - /* Turn on chunked HDF5 dataset creation */ - cl_opts->h5_use_chunks = TRUE; - break; - case 'C': - cl_opts->collective = 1; - break; - case 'd': - cl_opts->num_dsets = atoi(H5_optarg); - break; - case 'D': { - const char *end = H5_optarg; - - while (end && *end != '\0') { - char buf[10]; - int i; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - if (HDstrlen(buf) > 1 || HDisdigit(buf[0])) { - size_t j; - - for (j = 0; j < 10 && buf[j] != '\0'; ++j) - if (!isdigit(buf[j])) { - HDfprintf(stderr, "pio_perf: invalid --debug option %s\n", buf); - HDexit(EXIT_FAILURE); - } - - pio_debug_level = atoi(buf); - - if (pio_debug_level > 4) - pio_debug_level = 4; - else if (pio_debug_level < 0) - pio_debug_level = 0; - } - else { - switch (*buf) { - case 'r': - /* Turn on raw data throughput info */ - cl_opts->print_raw = TRUE; - break; - case 't': - /* Turn on time printing */ - cl_opts->print_times = TRUE; - break; - case 'v': - /* Turn on verify data correctness*/ - cl_opts->verify = TRUE; - break; - default: - HDfprintf(stderr, "pio_perf: invalid --debug option %s\n", buf); - HDexit(EXIT_FAILURE); - } - } - - if (*end == '\0') - break; - - end++; - } - } - - break; - case 'e': - cl_opts->num_bpp = parse_size_directive(H5_optarg); - break; - case 'F': - cl_opts->num_files = HDatoi(H5_optarg); - break; - case 'g': - cl_opts->dim2d = 1; - break; - case 'i': - cl_opts->num_iters = HDatoi(H5_optarg); - break; - case 'I': - cl_opts->interleaved = 1; - break; - case 'o': - cl_opts->output_file = H5_optarg; - break; - case 'p': - cl_opts->min_num_procs = HDatoi(H5_optarg); - break; - case 'P': - cl_opts->max_num_procs = HDatoi(H5_optarg); - break; - case 'T': - cl_opts->h5_threshold = parse_size_directive(H5_optarg); - break; - case 'w': - cl_opts->h5_write_only = TRUE; - break; - case 'x': - cl_opts->min_xfer_size = (size_t)parse_size_directive(H5_optarg); - break; - case 'X': - cl_opts->max_xfer_size = (size_t)parse_size_directive(H5_optarg); - break; - case 'h': - case '?': - default: - usage(progname); - HDfree(cl_opts); - return NULL; - } - } - - if (cl_opts->num_bpp == 0) { - if (cl_opts->dim2d == 0) - cl_opts->num_bpp = 256 * ONE_KB; - else - cl_opts->num_bpp = 8 * ONE_KB; - } - - if (cl_opts->max_xfer_size == 0) - cl_opts->max_xfer_size = (size_t)cl_opts->num_bpp; - - if (cl_opts->min_xfer_size == 0) - cl_opts->min_xfer_size = (size_t)(cl_opts->num_bpp) / 2; - - if (cl_opts->blk_size == 0) - cl_opts->blk_size = (size_t)(cl_opts->num_bpp) / 2; - - /* set default if none specified yet */ - if (!cl_opts->io_types) - cl_opts->io_types = PIO_HDF5 | PIO_MPI | PIO_POSIX; /* run all API */ - - /* verify parameters sanity. Adjust if needed. */ - /* cap xfer_size with bytes per process */ - if (!cl_opts->dim2d) { - if (cl_opts->min_xfer_size > (size_t)cl_opts->num_bpp) - cl_opts->min_xfer_size = (size_t)cl_opts->num_bpp; - if (cl_opts->max_xfer_size > (size_t)cl_opts->num_bpp) - cl_opts->max_xfer_size = (size_t)cl_opts->num_bpp; - } - if (cl_opts->min_xfer_size > cl_opts->max_xfer_size) - cl_opts->min_xfer_size = cl_opts->max_xfer_size; - if (cl_opts->blk_size > (size_t)cl_opts->num_bpp) - cl_opts->blk_size = (size_t)cl_opts->num_bpp; - /* check range of number of processes */ - if (cl_opts->min_num_procs <= 0) - cl_opts->min_num_procs = 1; - if (cl_opts->max_num_procs <= 0) - cl_opts->max_num_procs = 1; - if (cl_opts->min_num_procs > cl_opts->max_num_procs) - cl_opts->min_num_procs = cl_opts->max_num_procs; - /* check iteration */ - if (cl_opts->num_iters <= 0) - cl_opts->num_iters = 1; - - return cl_opts; -} - -/* - * Function: parse_size_directive - * Purpose: Parse the size directive passed on the commandline. The size - * directive is an integer followed by a size indicator: - * - * K, k - Kilobyte - * M, m - Megabyte - * G, g - Gigabyte - * - * Return: The size as a off_t because this is related to file size. - * If an unknown size indicator is used, then the program will - * exit with EXIT_FAILURE as the return value. - * Programmer: Bill Wendling, 18. December 2001 - * Modifications: - */ -static off_t -parse_size_directive(const char *size) -{ - off_t s; - char *endptr; - - s = HDstrtol(size, &endptr, 10); - - if (endptr && *endptr) { - while (*endptr != '\0' && (*endptr == ' ' || *endptr == '\t')) - ++endptr; - - switch (*endptr) { - case 'K': - case 'k': - s *= ONE_KB; - break; - case 'M': - case 'm': - s *= ONE_MB; - break; - case 'G': - case 'g': - s *= ONE_GB; - break; - default: - HDfprintf(stderr, "Illegal size specifier '%c'\n", *endptr); - HDexit(EXIT_FAILURE); - } - } - - return s; -} - -/* - * Function: usage - * Purpose: Print a usage message and then exit. - * Return: Nothing - * Programmer: Bill Wendling, 31. October 2001 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static void -usage(const char *prog) -{ - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - - if (myrank == 0) { - print_version(prog); - HDprintf("usage: %s [OPTIONS]\n", prog); - HDprintf(" OPTIONS\n"); - HDprintf(" -h, --help Print a usage message and exit\n"); - HDprintf(" -a S, --align=S Alignment of objects in HDF5 file [default: 1]\n"); - HDprintf(" -A AL, --api=AL Which APIs to test [default: all of them]\n"); -#if 0 - HDprintf(" -b, --binary The elusive binary option\n"); -#endif /* 0 */ - HDprintf(" -B S, --block-size=S Block size within transfer buffer\n"); - HDprintf(" (see below for description)\n"); - HDprintf(" [default: half the number of bytes per process\n"); - HDprintf(" per dataset]\n"); - HDprintf(" -c, --chunk Create HDF5 datasets using chunked storage\n"); - HDprintf(" [default: contiguous storage]\n"); - HDprintf(" -C, --collective Use collective I/O for MPI and HDF5 APIs\n"); - HDprintf(" [default: independent I/O)\n"); - HDprintf(" -d N, --num-dsets=N Number of datasets per file [default: 1]\n"); - HDprintf(" -D DL, --debug=DL Indicate the debugging level\n"); - HDprintf(" [default: no debugging]\n"); - HDprintf(" -e S, --num-bytes=S Number of bytes per process per dataset\n"); - HDprintf(" (see below for description)\n"); - HDprintf(" [default: 256K for 1D, 8K for 2D]\n"); - HDprintf(" -F N, --num-files=N Number of files [default: 1]\n"); - HDprintf(" -g, --geometry Use 2D geometry [default: 1D geometry]\n"); - HDprintf(" -i N, --num-iterations=N Number of iterations to perform [default: 1]\n"); - HDprintf(" -I, --interleaved Interleaved access pattern\n"); - HDprintf(" (see below for example)\n"); - HDprintf(" [default: Contiguous access pattern]\n"); - HDprintf(" -o F, --output=F Output raw data into file F [default: none]\n"); - HDprintf(" -p N, --min-num-processes=N Minimum number of processes to use [default: 1]\n"); - HDprintf(" -P N, --max-num-processes=N Maximum number of processes to use\n"); - HDprintf(" [default: all MPI_COMM_WORLD processes ]\n"); - HDprintf(" -T S, --threshold=S Threshold for alignment of objects in HDF5 file\n"); - HDprintf(" [default: 1]\n"); - HDprintf(" -w, --write-only Perform write tests not the read tests\n"); - HDprintf(" -x S, --min-xfer-size=S Minimum transfer buffer size\n"); - HDprintf(" (see below for description)\n"); - HDprintf(" [default: half the number of bytes per process\n"); - HDprintf(" per dataset]\n"); - HDprintf(" -X S, --max-xfer-size=S Maximum transfer buffer size\n"); - HDprintf(" [default: the number of bytes per process per\n"); - HDprintf(" dataset]\n"); - HDprintf("\n"); - HDprintf(" F - is a filename.\n"); - HDprintf(" N - is an integer >=0.\n"); - HDprintf(" S - is a size specifier, an integer >=0 followed by a size indicator:\n"); - HDprintf(" K - Kilobyte (%d)\n", ONE_KB); - HDprintf(" M - Megabyte (%d)\n", ONE_MB); - HDprintf(" G - Gigabyte (%d)\n", ONE_GB); - HDprintf("\n"); - HDprintf(" Example: '37M' is 37 megabytes or %d bytes\n", 37 * ONE_MB); - HDprintf("\n"); - HDprintf(" AL - is an API list. Valid values are:\n"); - HDprintf(" phdf5 - Parallel HDF5\n"); - HDprintf(" mpiio - MPI-I/O\n"); - HDprintf(" posix - POSIX\n"); - HDprintf("\n"); - HDprintf(" Example: --api=mpiio,phdf5\n"); - HDprintf("\n"); - HDprintf(" Dataset size:\n"); - HDprintf(" Depending on the selected geometry, each test dataset is either a linear\n"); - HDprintf(" array of size bytes-per-process * num-processes, or a square array of size\n"); - HDprintf(" (bytes-per-process * num-processes) x (bytes-per-process * num-processes).\n"); - HDprintf("\n"); - HDprintf(" Block size vs. Transfer buffer size:\n"); - HDprintf(" buffer-size controls the size of the memory buffer, which is broken into\n"); - HDprintf(" blocks and written to the file. Depending on the selected geometry, each\n"); - HDprintf(" block can be a linear array of size block-size or a square array of size\n"); - HDprintf(" block-size x block-size. The arrangement in which blocks are written is\n"); - HDprintf(" determined by the access pattern.\n"); - HDprintf("\n"); - HDprintf(" In 1D geometry, the transfer buffer is a linear array of size buffer-size.\n"); - HDprintf(" In 2D geometry, it is a rectangular array of size block-size x buffer-size\n"); - HDprintf(" or buffer-size x block-size if interleaved pattern is selected.\n"); - HDprintf("\n"); - HDprintf(" Interleaved and Contiguous patterns in 1D geometry:\n"); - HDprintf(" When contiguous access pattern is chosen, the dataset is evenly divided\n"); - HDprintf(" into num-processes regions and each process writes data to its own region.\n"); - HDprintf(" When interleaved blocks are written to a dataset, space for the first\n"); - HDprintf(" block of the first process is allocated in the dataset, then space is\n"); - HDprintf(" allocated for the first block of the second process, etc. until space is\n"); - HDprintf(" allocated for the first block of each process, then space is allocated for\n"); - HDprintf(" the second block of the first process, the second block of the second\n"); - HDprintf(" process, etc.\n"); - HDprintf("\n"); - HDprintf(" For example, with a 3 process run, 512KB bytes-per-process, 256KB transfer\n"); - HDprintf(" buffer size, and 64KB block size, each process must issue 2 transfer\n"); - HDprintf(" requests to complete access to the dataset.\n"); - HDprintf(" Contiguous blocks of the first transfer request are written like so:\n"); - HDprintf(" 1111----2222----3333----\n"); - HDprintf(" Interleaved blocks of the first transfer request are written like so:\n"); - HDprintf(" 123123123123------------\n"); - HDprintf(" The actual number of I/O operations involved in a transfer request\n"); - HDprintf(" depends on the access pattern and communication mode.\n"); - HDprintf(" When using independent I/O with interleaved pattern, each process\n"); - HDprintf(" performs 4 small non-contiguous I/O operations per transfer request.\n"); - HDprintf(" If collective I/O is turned on, the combined content of the buffers of\n"); - HDprintf(" the 3 processes will be written using one collective I/O operation\n"); - HDprintf(" per transfer request.\n"); - HDprintf("\n"); - HDprintf(" For information about access patterns in 2D geometry, please refer to the\n"); - HDprintf(" HDF5 Reference Manual.\n"); - HDprintf("\n"); - HDprintf(" DL - is a list of debugging flags. Valid values are:\n"); - HDprintf(" 1 - Minimal\n"); - HDprintf(" 2 - Not quite everything\n"); - HDprintf(" 3 - Everything\n"); - HDprintf(" 4 - The kitchen sink\n"); - HDprintf(" r - Raw data I/O throughput information\n"); - HDprintf(" t - Times as well as throughputs\n"); - HDprintf(" v - Verify data correctness\n"); - HDprintf("\n"); - HDprintf(" Example: --debug=2,r,t\n"); - HDprintf("\n"); - HDprintf(" Environment variables:\n"); - HDprintf(" HDF5_NOCLEANUP Do not remove data files if set [default remove]\n"); - HDprintf(" HDF5_MPI_INFO MPI INFO object key=value separated by ;\n"); - HDprintf(" HDF5_PARAPREFIX Paralllel data files prefix\n"); - fflush(stdout); - } /* end if */ -} /* end usage() */ - -#else /* H5_HAVE_PARALLEL */ - -/* - * Function: main - * Purpose: Dummy main() function for if HDF5 was configured without - * parallel stuff. - * Return: EXIT_SUCCESS - * Programmer: Bill Wendling, 14. November 2001 - */ -int -main(void) -{ - HDprintf("No parallel IO performance because parallel is not configured\n"); - return EXIT_SUCCESS; -} /* end main */ - -#endif /* !H5_HAVE_PARALLEL */ diff --git a/tools/test/perform/pio_perf.h b/tools/test/perform/pio_perf.h deleted file mode 100644 index 24621da..0000000 --- a/tools/test/perform/pio_perf.h +++ /dev/null @@ -1,100 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef PIO_PERF_H -#define PIO_PERF_H - -#ifndef STANDALONE -#include "io_timer.h" -#include "h5test.h" -#include "h5tools.h" -#include "h5tools_utils.h" -#else -#include "io_timer.h" -#include "pio_standalone.h" -#endif - -/* setup the dataset no fill option if this is v1.5 or more */ -#if H5_VERS_MAJOR > 1 || H5_VERS_MINOR > 4 -#define H5_HAVE_NOFILL 1 -#endif - -typedef enum iotype_ { - POSIXIO, - MPIO, - PHDF5 - /*NUM_TYPES*/ -} iotype; - -typedef struct parameters_ { - iotype io_type; /* The type of IO test to perform */ - int num_procs; /* Maximum number of processes to use */ - long num_files; /* Number of files to create */ - long num_dsets; /* Number of datasets to create */ - off_t num_bytes; /* Number of bytes in each dset */ - int num_iters; /* Number of times to loop doing the IO */ - size_t buf_size; /* Buffer size */ - size_t blk_size; /* Block size */ - unsigned interleaved; /* Interleaved vs. contiguous blocks */ - unsigned collective; /* Collective vs. independent I/O */ - unsigned dim2d; /* 1D vs. 2D */ - hsize_t h5_align; /* HDF5 object alignment */ - hsize_t h5_thresh; /* HDF5 object alignment threshold */ - int h5_use_chunks; /* Make HDF5 dataset chunked */ - int h5_write_only; /* Perform the write tests only */ - int verify; /* Verify data correctness */ -} parameters; - -typedef struct results_ { - herr_t ret_code; - io_time_t *timers; -} results; - -#ifndef SUCCESS -#define SUCCESS 0 -#endif /* !SUCCESS */ - -#ifndef FAIL -#define FAIL -1 -#endif /* !FAIL */ - -extern FILE * output; /* output file */ -extern io_time_t *timer_g; /* timer: global for stub functions */ -extern int comm_world_rank_g; /* my rank in MPI_COMM_RANK */ -extern int comm_world_nprocs_g; /* num. of processes of MPI_COMM_WORLD */ -extern MPI_Comm pio_comm_g; /* Communicator to run the PIO */ -extern int pio_mpi_rank_g; /* MPI rank of pio_comm_g */ -extern int pio_mpi_nprocs_g; /* number of processes of pio_comm_g */ -extern int pio_debug_level; /* The debug level: - * 0 - Off - * 1 - Minimal - * 2 - Some more - * 3 - Maximal - * 4 - Even More Debugging (timer stuff) - */ - -#define HDprint_rank(f) /* print rank in MPI_COMM_WORLD */ HDfprintf(f, "%d: ", comm_world_rank_g); -#define HDprint_size(f) /* print size of MPI_COMM_WORLD */ HDfprintf(f, "%d", comm_world_nprocs_g); -#define HDprint_rank_size(f) /* print rank/size of MPI_COMM_WORLD */ \ - HDfprintf(f, "%d/%d: ", comm_world_rank_g, comm_world_nprocs_g); - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -extern results do_pio(parameters param); - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* PIO_PERF_H */ diff --git a/tools/test/perform/pio_standalone.c b/tools/test/perform/pio_standalone.c index ded15a4..032bfba 100644 --- a/tools/test/perform/pio_standalone.c +++ b/tools/test/perform/pio_standalone.c @@ -17,10 +17,6 @@ #include "pio_perf.h" -#ifdef STANDALONE -MPI_Info h5_io_info_g = MPI_INFO_NULL; /* MPI INFO object for IO */ -#endif - /** From h5tools_utils.c **/ /* global variables */ @@ -156,134 +152,3 @@ print_version(const char *progname) printf("%s: Version %u.%u.%u%s%s\n", progname, H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE, H5_VERS_SUBRELEASE[0] ? "-" : "", H5_VERS_SUBRELEASE); } - -#ifdef STANDALONE -/* - * Function: h5_set_info_object - * Purpose: Process environment variables setting to set up MPI Info - * object. - * Return: 0 if all is fine; otherwise non-zero. - * Programmer: Albert Cheng, 2002/05/21. - * Modifications: - * Bill Wendling, 2002/05/31 - * Modified so that the HDF5_MPI_INFO environment variable can - * be a semicolon separated list of "key=value" pairings. Most - * of the code is to remove any whitespaces which might be - * surrounding the "key=value" pairs. - */ -int -h5_set_info_object(void) -{ - char *envp; /* environment pointer */ - int ret_value = 0; - - /* handle any MPI INFO hints via $HDF5_MPI_INFO */ - if ((envp = HDgetenv("HDF5_MPI_INFO")) != NULL) { - char *next, *valp; - - valp = envp = next = HDstrdup(envp); - - if (!valp) - return 0; - - /* create an INFO object if not created yet */ - if (h5_io_info_g == MPI_INFO_NULL) - MPI_Info_create(&h5_io_info_g); - - do { - size_t len; - char * key_val, *endp, *namep; - - if (*valp == ';') - valp++; - - /* copy key/value pair into temporary buffer */ - len = strcspn(valp, ";"); - next = &valp[len]; - key_val = (char *)HDcalloc(1, len + 1); - - /* increment the next pointer past the terminating semicolon */ - if (*next == ';') - ++next; - - namep = HDstrncpy(key_val, valp, len); - - /* pass up any beginning whitespaces */ - while (*namep && (*namep == ' ' || *namep == '\t')) - namep++; - - if (!*namep) - continue; /* was all white space, so move to next k/v pair */ - - /* eat up any ending white spaces */ - endp = &namep[HDstrlen(namep) - 1]; - - while (endp && (*endp == ' ' || *endp == '\t')) - *endp-- = '\0'; - - /* find the '=' */ - valp = HDstrchr(namep, '='); - - if (valp != NULL) { /* it's a valid key/value pairing */ - char *tmp_val = valp + 1; - - /* change '=' to \0, move valp down one */ - *valp-- = '\0'; - - /* eat up ending whitespace on the "key" part */ - while (*valp == ' ' || *valp == '\t') - *valp-- = '\0'; - - valp = tmp_val; - - /* eat up beginning whitespace on the "value" part */ - while (*valp == ' ' || *valp == '\t') - *valp++ = '\0'; - - /* actually set the darned thing */ - if (MPI_SUCCESS != MPI_Info_set(h5_io_info_g, namep, valp)) { - HDprintf("MPI_Info_set failed\n"); - ret_value = -1; - } - } - - valp = next; - HDfree(key_val); - } while (next && *next); - - HDfree(envp); - } - - return ret_value; -} - -/* - * Function: h5_dump_info_object - * Purpose: Display content of an MPI Info object - * Return: void - * Programmer: Albert Cheng 2002/05/21 - * Modifications: - */ -void -h5_dump_info_object(MPI_Info info) -{ - char key[MPI_MAX_INFO_KEY + 1]; - char value[MPI_MAX_INFO_VAL + 1]; - int flag; - int i, nkeys; - - HDprintf("Dumping MPI Info Object (up to %d bytes per item):\n", MPI_MAX_INFO_VAL); - if (info == MPI_INFO_NULL) { - HDprintf("object is MPI_INFO_NULL\n"); - } - else { - MPI_Info_get_nkeys(info, &nkeys); - HDprintf("object has %d items\n", nkeys); - for (i = 0; i < nkeys; i++) { - MPI_Info_get_nthkey(info, i, key); - MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); - HDprintf("%s=%s\n", key, value); - } - } -} -#endif /* STANDALONE */ diff --git a/tools/test/perform/pio_standalone.h b/tools/test/perform/pio_standalone.h index ef6d97b..f1fb946 100644 --- a/tools/test/perform/pio_standalone.h +++ b/tools/test/perform/pio_standalone.h @@ -441,17 +441,6 @@ extern char * strdup(const char *s); #define TRUE true #endif -/** From h5test.h **/ - -#ifdef H5_HAVE_PARALLEL -extern MPI_Info h5_io_info_g; /* MPI INFO object for IO */ -#endif - -#ifdef H5_HAVE_PARALLEL -int h5_set_info_object(void); -void h5_dump_info_object(MPI_Info info); -#endif - /** From h5tools_utils.h **/ H5_DLLVAR int H5_opterr; /* getoption prints errors if this is on */ diff --git a/tools/test/perform/sio_engine.c b/tools/test/perform/sio_engine.c deleted file mode 100644 index 1af2318..0000000 --- a/tools/test/perform/sio_engine.c +++ /dev/null @@ -1,1328 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Author: Christian Chilan, April 2008 - */ - -#include "hdf5.h" - -#include -#include -#include -#include - -#ifdef H5_HAVE_UNISTD_H -#include -#include -#endif - -#ifdef H5_HAVE_SYS_STAT_H -#include -#endif - -#include "sio_perf.h" - -/* Macro definitions */ - -/* sizes of various items. these sizes won't change during program execution */ -#define ELMT_H5_TYPE H5T_NATIVE_UCHAR - -#define GOTOERROR(errcode) \ - { \ - ret_code = errcode; \ - goto done; \ - } -#define ERRMSG(mesg) \ - { \ - HDfprintf(stderr, "*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ - } - -/* verify: if val is false (0), print mesg. */ -#define VRFY(val, mesg) \ - do { \ - if (!val) { \ - ERRMSG(mesg); \ - GOTOERROR(FAIL); \ - } \ - } while (0) - -/* POSIX I/O macros */ -#ifdef H5_HAVE_WIN32_API -/* Can't link against the library, so this test will use the older, non-Unicode - * _open() call on Windows. - */ -#define HDopen(S, F, ...) _open(S, F | _O_BINARY, __VA_ARGS__) -#endif /* H5_HAVE_WIN32_API */ -#define POSIXCREATE(fn) HDopen(fn, O_CREAT | O_TRUNC | O_RDWR, 0600) -#define POSIXOPEN(fn, F) HDopen(fn, F, 0600) -#define POSIXCLOSE(F) HDclose(F) -#define POSIXSEEK(F, L) HDlseek(F, L, SEEK_SET) -#define POSIXWRITE(F, B, S) HDwrite(F, B, S) -#define POSIXREAD(F, B, S) HDread(F, B, S) - -enum { SIO_CREATE = 1, SIO_WRITE = 2, SIO_READ = 4 }; - -/* Global variables */ -static int clean_file_g = -1; /*whether to cleanup temporary test */ -/*files. -1 is not defined; */ -/*0 is no cleanup; 1 is do cleanup */ - -/* the different types of file descriptors we can expect */ -typedef union { - int posixfd; /* POSIX file handle*/ - hid_t h5fd; /* HDF5 file */ -} file_descr; - -/* local functions */ -static char * sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size, - parameters *param); -static herr_t do_write(results *res, file_descr *fd, parameters *parms, void *buffer); -static herr_t do_read(results *res, file_descr *fd, parameters *parms, void *buffer); -static herr_t dset_write(int local_dim, file_descr *fd, parameters *parms, void *buffer); -static herr_t posix_buffer_write(int local_dim, file_descr *fd, parameters *parms, void *buffer); -static herr_t dset_read(int localrank, file_descr *fd, parameters *parms, void *buffer, const char *buffer2); -static herr_t posix_buffer_read(int local_dim, file_descr *fd, parameters *parms, void *buffer); -static herr_t do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags); -hid_t set_vfd(parameters *param); -static herr_t do_fclose(iotype iot, file_descr *fd); -static void do_cleanupfile(iotype iot, char *fname); - -/* global variables */ -static HDoff_t offset[MAX_DIMS]; /* dataset size in bytes */ -static size_t buf_offset[MAX_DIMS]; /* dataset size in bytes */ -static int order[MAX_DIMS]; /* dimension access order */ -static size_t linear_buf_size; /* linear buffer size */ -static int cont_dim; /* lowest dimension for contiguous POSIX - access */ -static size_t cont_size; /* size of contiguous POSIX access */ -static hid_t fapl; /* file access list */ -static unsigned char *buf_p; /* buffer pointer */ -static const char * multi_letters = "msbrglo"; /* string for multi driver */ - -/* HDF5 global variables */ -static hsize_t h5count[MAX_DIMS]; /*selection count */ -static hssize_t h5offset[MAX_DIMS]; /* Selection offset within dataspace */ -static hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ -static hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ -static hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ -static hid_t h5dcpl = H5I_INVALID_HID; /* Dataset creation property list */ -static hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ - -/* - * Function: do_sio - * Purpose: SIO Engine where IO are executed. - * Return: results - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ -void -do_sio(parameters param, results *res) -{ - char * buffer = NULL; /*data buffer pointer */ - size_t buf_size[MAX_DIMS]; /* general buffer size in bytes */ - file_descr fd; /* file handles */ - iotype iot; /* API type */ - char base_name[256]; /* test file base name */ - /* return codes */ - herr_t ret_code = 0; /*return code */ - - char fname[FILENAME_MAX]; /* test file name */ - int i; - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - - /* Sanity check parameters */ - - /* IO type */ - iot = param.io_type; - - switch (iot) { - case POSIXIO: - fd.posixfd = -1; - res->timers = io_time_new(SYS_CLOCK); - break; - case HDF5: - fd.h5fd = -1; - res->timers = io_time_new(SYS_CLOCK); - break; - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - GOTOERROR(FAIL); - } - - linear_buf_size = 1; - - for (i = 0; i < param.rank; i++) { - buf_size[i] = param.buf_size[i]; - order[i] = param.order[i]; - linear_buf_size *= buf_size[i]; - buf_offset[i] = 0; - offset[i] = 0; - - /* Validate transfer buffer size */ - if (param.buf_size[i] <= 0) { - HDfprintf(stderr, "Transfer buffer size[%d] (%zu) must be > 0\n", i, buf_size[i]); - GOTOERROR(FAIL); - } - - if ((param.dset_size[i] % param.buf_size[i]) != 0) { - HDfprintf(stderr, - "Dataset size[%d] (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "trasfer buffer size[%d] (%zu)\n", - param.rank, (long long)param.dset_size[i], param.rank, param.buf_size[i]); - GOTOERROR(FAIL); - } - } - - /* Allocate transfer buffer */ - if ((buffer = (char *)malloc(linear_buf_size)) == NULL) { - HDfprintf(stderr, "malloc for transfer buffer size (%zu) failed\n", linear_buf_size); - GOTOERROR(FAIL); - } - - if (sio_debug_level >= 4) - - /* output all of the times for all iterations */ - HDfprintf(output, "Timer details:\n"); - - /* - * Write performance measurement - */ - /* Open file for write */ - - HDstrcpy(base_name, "#sio_tmp"); - sio_create_filename(iot, base_name, fname, sizeof(fname), ¶m); - - if (sio_debug_level > 0) - HDfprintf(output, "data filename=%s\n", fname); - - io_time_set(res->timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTART); - hrc = do_fopen(¶m, fname, &fd, SIO_CREATE | SIO_WRITE); - VRFY((hrc == SUCCESS), "do_fopen failed"); - - io_time_set(res->timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTART); - hrc = do_write(res, &fd, ¶m, buffer); - io_time_set(res->timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_write failed"); - - /* Close file for write */ - hrc = do_fclose(iot, &fd); - io_time_set(res->timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_fclose failed"); - - if (!param.h5_write_only) { - /* - * Read performance measurement - */ - - /* Open file for read */ - io_time_set(res->timers, HDF5_GROSS_READ_FIXED_DIMS, TSTART); - hrc = do_fopen(¶m, fname, &fd, SIO_READ); - VRFY((hrc == SUCCESS), "do_fopen failed"); - - io_time_set(res->timers, HDF5_FINE_READ_FIXED_DIMS, TSTART); - hrc = do_read(res, &fd, ¶m, buffer); - io_time_set(res->timers, HDF5_FINE_READ_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_read failed"); - - /* Close file for read */ - hrc = do_fclose(iot, &fd); - - io_time_set(res->timers, HDF5_GROSS_READ_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_fclose failed"); - } - - do_cleanupfile(iot, fname); - -done: - /* clean up */ - /* release HDF5 objects */ - - /* close any opened files */ - /* no remove(fname) because that should have happened normally. */ - switch (iot) { - case POSIXIO: - if (fd.posixfd != -1) - hrc = do_fclose(iot, &fd); - break; - case HDF5: - if (fd.h5fd != -1) - hrc = do_fclose(iot, &fd); - break; - default: - /* unknown request */ - HDassert(0 && "Unknown IO type"); - break; - } - - /* release generic resources */ - if (buffer) - free(buffer); - - res->ret_code = ret_code; -} - -/* - * Function: sio_create_filename - * Purpose: Create a new filename to write to. Determine the correct - * suffix to append to the filename by the type of I/O we're - * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if - * USER or LOGIN are specified in the environment. - * Return: Pointer to filename or NULL - * Programmer: Bill Wendling, 21. November 2001 - * Modifications: Support for file drivers. Christian Chilan, April, 2008 - */ -static char * -sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size, parameters *param) -{ - const char *prefix, *suffix = ""; - char * ptr, last = '\0'; - size_t i, j; - vfdtype vfd; - vfd = param->vfd; - - if (!base_name || !fullname || size < 1) - return NULL; - - memset(fullname, 0, size); - - switch (iot) { - case POSIXIO: - suffix = ".posix"; - break; - case HDF5: - suffix = ".h5"; - if (vfd == family) - suffix = "%05d.h5"; - else if (vfd == multi) - suffix = NULL; - break; - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - HDassert(0 && "Unknown IO type"); - break; - } - - /* First use the environment variable and then try the constant */ - prefix = HDgetenv("HDF5_PREFIX"); - -#ifdef HDF5_PREFIX - if (!prefix) - prefix = HDF5_PREFIX; -#endif /* HDF5_PREFIX */ - - /* Prepend the prefix value to the base name */ - if (prefix && *prefix) { - /* If the prefix specifies the HDF5_PREFIX directory, then - * default to using the "/tmp/$USER" or "/tmp/$LOGIN" - * directory instead. */ - register char *user, *login, *subdir; - - user = HDgetenv("USER"); - login = HDgetenv("LOGIN"); - subdir = (user ? user : login); - - if (subdir) { - for (i = 0; i < size - 1 && prefix[i]; i++) - fullname[i] = prefix[i]; - - fullname[i++] = '/'; - - for (j = 0; i < size && subdir[j]; i++, j++) - fullname[i] = subdir[j]; - } - else { - /* We didn't append the prefix yet */ - HDstrncpy(fullname, prefix, size); - fullname[size - 1] = '\0'; - } - - if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) { - /* Append the base_name with a slash first. Multiple slashes are - * handled below. */ - h5_stat_t buf; - - if (HDstat(fullname, &buf) < 0) - /* The directory doesn't exist just yet */ - if (HDmkdir(fullname, 0755) < 0 && errno != EEXIST) { - /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. - * Default to PREFIX's original prefix value. */ - HDstrcpy(fullname, prefix); - } - - HDstrcat(fullname, "/"); - HDstrcat(fullname, base_name); - } - else { - /* Buffer is too small */ - return NULL; - } - } - else if (strlen(base_name) >= size) { - /* Buffer is too small */ - return NULL; - } - else { - HDstrcpy(fullname, base_name); - } - - /* Append a suffix */ - if (suffix) { - if (HDstrlen(fullname) + HDstrlen(suffix) >= size) - return NULL; - - HDstrcat(fullname, suffix); - } - - /* Remove any double slashes in the filename */ - for (ptr = fullname, i = j = 0; ptr && (i < size); i++, ptr++) { - if (*ptr != '/' || last != '/') - fullname[j++] = *ptr; - - last = *ptr; - } - - return fullname; -} - -/* - * Function: do_write - * Purpose: Write the required amount of data to the file. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ -static herr_t -do_write(results *res, file_descr *fd, parameters *parms, void *buffer) -{ - int ret_code = SUCCESS; - char dname[64]; - int i; - size_t u; - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - hsize_t h5dims[MAX_DIMS]; /*dataset dim sizes */ - hsize_t h5chunk[MAX_DIMS]; /*dataset dim sizes */ - hsize_t h5block[MAX_DIMS]; /*dataspace selection */ - hsize_t h5stride[MAX_DIMS]; /*selection stride */ - hsize_t h5start[MAX_DIMS]; /*selection start */ - hsize_t h5maxdims[MAX_DIMS]; - int rank; /*rank of dataset */ - - /* Prepare buffer for verifying data */ - /* if (parms->verify) - memset(buffer,1,linear_buf_size); */ - - buf_p = (unsigned char *)buffer; - - for (u = 0; u < linear_buf_size; u++) - buf_p[u] = u % 128; - - rank = parms->rank; - - for (i = 0; i < rank; i++) - h5offset[i] = offset[i] = 0; - - /* I/O Access specific setup */ - switch (parms->io_type) { - case POSIXIO: - - /* determine lowest dimension for contiguous POSIX access */ - cont_dim = rank; - - for (i = rank - 1; i >= 0; i--) { - if (parms->buf_size[i] == parms->dset_size[i]) - cont_dim = i; - else - break; - } - - /* determine size of the contiguous POSIX access */ - cont_size = (!cont_dim) ? 1 : parms->buf_size[cont_dim - 1]; - for (i = cont_dim; i < rank; i++) - cont_size *= parms->buf_size[i]; - - break; - - case HDF5: /* HDF5 setup */ - - for (i = 0; i < rank; i++) { - h5dims[i] = parms->dset_size[i]; - h5start[i] = 0; - h5stride[i] = 1; - h5block[i] = 1; - h5count[i] = parms->buf_size[i]; - h5chunk[i] = parms->chk_size[i]; - h5maxdims[i] = H5S_UNLIMITED; - } - - if (parms->h5_use_chunks && parms->h5_extendable) { - h5dset_space_id = H5Screate_simple(rank, h5count, h5maxdims); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - } - else { - h5dset_space_id = H5Screate_simple(rank, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - } - - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - - /* Create the memory dataspace that corresponds to the xfer buffer */ - h5mem_space_id = H5Screate_simple(rank, h5count, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - - /* Create the dataset transfer property list */ - h5dxpl = H5Pcreate(H5P_DATASET_XFER); - if (h5dxpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - break; - - default: - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - GOTOERROR(FAIL); - break; - } /* end switch */ - - /* create dataset */ - switch (parms->io_type) { - case POSIXIO: - break; - - case HDF5: - h5dcpl = H5Pcreate(H5P_DATASET_CREATE); - - if (h5dcpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - if (parms->h5_use_chunks) { - /* Set the chunk size to be the same as the buffer size */ - hrc = H5Pset_chunk(h5dcpl, rank, h5chunk); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - - HDsprintf(dname, "Dataset_%ld", (unsigned long)parms->num_bytes); - h5ds_id = - H5Dcreate2(fd->h5fd, dname, ELMT_H5_TYPE, h5dset_space_id, H5P_DEFAULT, h5dcpl, H5P_DEFAULT); - - if (h5ds_id < 0) { - HDfprintf(stderr, "HDF5 Dataset Create failed\n"); - GOTOERROR(FAIL); - } - - hrc = H5Pclose(h5dcpl); - /* verifying the close of the dcpl */ - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Close failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - GOTOERROR(FAIL); - break; - } - - /* Start "raw data" write timer */ - io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART); - - /* Perform write */ - hrc = dset_write(rank - 1, fd, parms, buffer); - - if (hrc < 0) { - HDfprintf(stderr, "Error in dataset write\n"); - GOTOERROR(FAIL); - } - - /* Stop "raw data" write timer */ - io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP); - - /* Calculate write time */ - - /* Close dataset. Only HDF5 needs to do an explicit close. */ - if (parms->io_type == HDF5) { - hrc = H5Dclose(h5ds_id); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Close failed\n"); - GOTOERROR(FAIL); - } - - h5ds_id = H5I_INVALID_HID; - } /* end if */ - -done: - - /* release HDF5 objects */ - if (h5dset_space_id != -1) { - hrc = H5Sclose(h5dset_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); - ret_code = FAIL; - } - else { - h5dset_space_id = H5I_INVALID_HID; - } - } - - if (h5mem_space_id != -1) { - hrc = H5Sclose(h5mem_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); - ret_code = FAIL; - } - else { - h5mem_space_id = H5I_INVALID_HID; - } - } - - if (h5dxpl != -1) { - hrc = H5Pclose(h5dxpl); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); - ret_code = FAIL; - } - else { - h5dxpl = H5I_INVALID_HID; - } - } - - return ret_code; -} - -/* - * Function: dset_write - * Purpose: Write buffer into the dataset. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ -static herr_t -dset_write(int local_dim, file_descr *fd, parameters *parms, void *buffer) -{ - int cur_dim = order[local_dim] - 1; - int ret_code = SUCCESS; - int k; - hsize_t dims[MAX_DIMS], maxdims[MAX_DIMS]; - hsize_t i; - int j; - herr_t hrc; - - /* iterates according to the dimensions in order array */ - for (i = 0; i < parms->dset_size[cur_dim]; i += parms->buf_size[cur_dim]) { - - h5offset[cur_dim] = (hssize_t)i; - offset[cur_dim] = (HDoff_t)i; - - if (local_dim > 0) { - - dset_write(local_dim - 1, fd, parms, buffer); - } - else { - - switch (parms->io_type) { - - case POSIXIO: - /* initialize POSIX offset in the buffer */ - for (j = 0; j < parms->rank; j++) - buf_offset[j] = 0; - buf_p = (unsigned char *)buffer; - /* write POSIX buffer */ - posix_buffer_write(0, fd, parms, buffer); - break; - - case HDF5: - /* if dimensions are extendable, extend them as needed during access */ - if (parms->h5_use_chunks && parms->h5_extendable) { - - hrc = H5Sget_simple_extent_dims(h5dset_space_id, dims, maxdims); - VRFY((hrc >= 0), "H5Sget_simple_extent_dims"); - - for (k = 0; k < parms->rank; k++) { - - HDassert(h5offset[k] >= 0); - if (dims[k] <= (hsize_t)h5offset[k]) { - dims[k] = dims[k] + h5count[k]; - hrc = H5Sset_extent_simple(h5dset_space_id, parms->rank, dims, maxdims); - VRFY((hrc >= 0), "H5Sset_extent_simple"); - hrc = H5Dset_extent(h5ds_id, dims); - VRFY((hrc >= 0), "H5Dextend"); - } - } - } - /* applies offset */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Write the buffer out */ - hrc = H5Sget_simple_extent_dims(h5dset_space_id, dims, maxdims); - hrc = H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dwrite"); - - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - HDassert(0 && "Unknown IO type"); - break; - } /* switch (parms->io_type) */ - } - } -done: - return ret_code; -} - -/* - * Function: posix_buffer_write - * Purpose: Write buffer into the POSIX file considering contiguity. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ - -static herr_t -posix_buffer_write(int local_dim, file_descr *fd, parameters *parms, void *buffer) -{ - int ret_code = SUCCESS; - - /* if dimension is not contiguous, call recursively */ - if (local_dim < parms->rank - 1 && local_dim != cont_dim) { - size_t u; - - for (u = 0; u < parms->buf_size[local_dim]; u++) { - buf_offset[local_dim] = u; - posix_buffer_write(local_dim + 1, fd, parms, buffer); - - /* if next dimension is cont_dim, it will fill out the buffer - traversing the entire dimension local_dim without the need - of performing iteration */ - if (local_dim + 1 == cont_dim) - break; - } - /* otherwise, perform contiguous POSIX access */ - } - else { - HDoff_t d_offset; - HDoff_t linear_dset_offset = 0; - int i, j, rc; - - buf_offset[local_dim] = 0; - - /* determine offset in the buffer */ - for (i = 0; i < parms->rank; i++) { - d_offset = 1; - - for (j = i + 1; j < parms->rank; j++) - d_offset *= (HDoff_t)parms->dset_size[j]; - - linear_dset_offset += (offset[i] + (HDoff_t)buf_offset[i]) * d_offset; - } - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, linear_dset_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - /* check if all bytes are written */ - rc = ((ssize_t)cont_size == POSIXWRITE(fd->posixfd, buf_p, cont_size)); - VRFY((rc != 0), "POSIXWRITE"); - - /* Advance location in buffer */ - buf_p += cont_size; - } - -done: - return ret_code; -} - -/* - * Function: do_read - * Purpose: Read the required amount of data to the file. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ -static herr_t -do_read(results *res, file_descr *fd, parameters *parms, void *buffer) -{ - char * buffer2 = NULL; /* Buffer for data verification */ - int ret_code = SUCCESS; - char dname[64]; - int i; - size_t u; - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - hsize_t h5dims[MAX_DIMS]; /*dataset dim sizes */ - hsize_t h5block[MAX_DIMS]; /*dataspace selection */ - hsize_t h5stride[MAX_DIMS]; /*selection stride */ - hsize_t h5start[MAX_DIMS]; /*selection start */ - int rank; - - /* Allocate data verification buffer */ - if (NULL == (buffer2 = (char *)malloc(linear_buf_size))) { - HDfprintf(stderr, "malloc for data verification buffer size (%zu) failed\n", linear_buf_size); - GOTOERROR(FAIL); - } /* end if */ - - /* Prepare buffer for verifying data */ - for (u = 0; u < linear_buf_size; u++) - buffer2[u] = (char)(u % 128); - - rank = parms->rank; - for (i = 0; i < rank; i++) - h5offset[i] = offset[i] = 0; - - /* I/O Access specific setup */ - switch (parms->io_type) { - case POSIXIO: - cont_dim = rank; - - for (i = rank - 1; i >= 0; i--) { - if (parms->buf_size[i] == parms->dset_size[i]) - cont_dim = i; - else - break; - } - cont_size = (!cont_dim) ? 1 : parms->buf_size[cont_dim - 1]; - for (i = cont_dim; i < rank; i++) - cont_size *= parms->buf_size[i]; - - break; - - case HDF5: /* HDF5 setup */ - for (i = 0; i < rank; i++) { - h5dims[i] = parms->dset_size[i]; - h5start[i] = 0; - h5stride[i] = 1; - h5block[i] = 1; - h5count[i] = parms->buf_size[i]; - } - - h5dset_space_id = H5Screate_simple(rank, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - - /* Create the memory dataspace that corresponds to the xfer buffer */ - h5mem_space_id = H5Screate_simple(rank, h5count, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - - /* Create the dataset transfer property list */ - h5dxpl = H5Pcreate(H5P_DATASET_XFER); - if (h5dxpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - GOTOERROR(FAIL); - break; - } /* end switch */ - - /* create dataset */ - switch (parms->io_type) { - case POSIXIO: - break; - - case HDF5: - HDsprintf(dname, "Dataset_%ld", (long)parms->num_bytes); - h5ds_id = H5Dopen2(fd->h5fd, dname, H5P_DEFAULT); - if (h5ds_id < 0) { - HDfprintf(stderr, "HDF5 Dataset open failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - GOTOERROR(FAIL); - break; - } /* end switch */ - - /* Start "raw data" read timer */ - io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTART); - hrc = dset_read(rank - 1, fd, parms, buffer, buffer2); - - if (hrc < 0) { - HDfprintf(stderr, "Error in dataset read\n"); - GOTOERROR(FAIL); - } - - /* Stop "raw data" read timer */ - io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTOP); - - /* Calculate read time */ - - /* Close dataset. Only HDF5 needs to do an explicit close. */ - if (parms->io_type == HDF5) { - hrc = H5Dclose(h5ds_id); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Close failed\n"); - GOTOERROR(FAIL); - } - - h5ds_id = H5I_INVALID_HID; - } /* end if */ - -done: - - /* release HDF5 objects */ - if (h5dset_space_id != -1) { - hrc = H5Sclose(h5dset_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); - ret_code = FAIL; - } - else { - h5dset_space_id = H5I_INVALID_HID; - } - } - - if (h5mem_space_id != -1) { - hrc = H5Sclose(h5mem_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); - ret_code = FAIL; - } - else { - h5mem_space_id = H5I_INVALID_HID; - } - } - - if (h5dxpl != -1) { - hrc = H5Pclose(h5dxpl); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); - ret_code = FAIL; - } - else { - h5dxpl = H5I_INVALID_HID; - } - } - - /* release generic resources */ - if (buffer2) - free(buffer2); - - return ret_code; -} - -/* - * Function: dset_read - * Purpose: Read buffer into the dataset. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ - -static herr_t -dset_read(int local_dim, file_descr *fd, parameters *parms, void *buffer, const char *buffer2) -{ - int cur_dim = order[local_dim] - 1; - hsize_t i; - int j; - herr_t hrc; - int ret_code = SUCCESS; - - /* iterate on the current dimension */ - for (i = 0; i < parms->dset_size[cur_dim]; i += parms->buf_size[cur_dim]) { - - h5offset[cur_dim] = (hssize_t)i; - offset[cur_dim] = (HDoff_t)i; - - /* if traverse in order array is incomplete, recurse */ - if (local_dim > 0) { - - ret_code = dset_read(local_dim - 1, fd, parms, buffer, buffer2); - - /* otherwise, write buffer into dataset */ - } - else { - - switch (parms->io_type) { - - case POSIXIO: - for (j = 0; j < parms->rank; j++) { - buf_offset[j] = 0; - } - buf_p = (unsigned char *)buffer; - posix_buffer_read(0, fd, parms, buffer); - break; - - case HDF5: - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - /* Read the buffer out */ - hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dread"); - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - HDassert(0 && "Unknown IO type"); - break; - } /* switch (parms->io_type) */ - } - } -done: - return ret_code; -} - -/* - * Function: posix_buffer_read - * Purpose: Read buffer into the POSIX file considering contiguity. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ - -static herr_t -posix_buffer_read(int local_dim, file_descr *fd, parameters *parms, void *buffer) -{ - int ret_code = SUCCESS; - - /* if local dimension is not contiguous, recurse */ - if (local_dim < parms->rank - 1 && local_dim != cont_dim) { - size_t u; - - for (u = 0; u < parms->buf_size[local_dim]; u++) { - buf_offset[local_dim] = u; - ret_code = posix_buffer_read(local_dim + 1, fd, parms, buffer); - if (local_dim + 1 == cont_dim) - break; - } - /* otherwise, perform contiguous POSIX access */ - } - else { - HDoff_t d_offset; - HDoff_t linear_dset_offset = 0; - int i, j, rc; - - buf_offset[local_dim] = 0; - /* determine offset in buffer */ - for (i = 0; i < parms->rank; i++) { - d_offset = 1; - - for (j = i + 1; j < parms->rank; j++) - d_offset *= (HDoff_t)parms->dset_size[j]; - - linear_dset_offset += (offset[i] + (HDoff_t)buf_offset[i]) * d_offset; - } - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, linear_dset_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - /* check if all bytes are read */ - rc = ((ssize_t)cont_size == POSIXREAD(fd->posixfd, buf_p, cont_size)); - VRFY((rc != 0), "POSIXREAD"); - - /* Advance location in buffer */ - buf_p += cont_size; - } -done: - return ret_code; -} - -/* - * Function: do_fopen - * Purpose: Open the specified file. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: Support for file drivers, Christian Chilan, April, 2008 - */ -static herr_t -do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) -{ - int ret_code = SUCCESS; - hid_t fcpl; - - switch (param->io_type) { - case POSIXIO: - if (flags & (SIO_CREATE | SIO_WRITE)) - fd->posixfd = POSIXCREATE(fname); - else - fd->posixfd = POSIXOPEN(fname, O_RDONLY); - - if (fd->posixfd < 0) { - HDfprintf(stderr, "POSIX File Open failed(%s)\n", fname); - GOTOERROR(FAIL); - } - - break; - - case HDF5: - - fapl = set_vfd(param); - - if (fapl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - fcpl = H5Pcreate(H5P_FILE_CREATE); - if (param->page_size) { - H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1); - H5Pset_file_space_page_size(fcpl, param->page_size); - if (param->page_buffer_size) - H5Pset_page_buffer_size(fapl, param->page_buffer_size, 0, 0); - } - - /* create the parallel file */ - if (flags & (SIO_CREATE | SIO_WRITE)) { - fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, fcpl, fapl); - } - else { - fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, fapl); - } - - if (fd->h5fd < 0) { - HDfprintf(stderr, "HDF5 File Create failed(%s)\n", fname); - GOTOERROR(FAIL); - } - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)param->io_type); - GOTOERROR(FAIL); - break; - } - -done: - return ret_code; -} - -/* - * Function: set_vfd - * Purpose: Sets file driver. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ - -hid_t -set_vfd(parameters *param) -{ - hid_t my_fapl = H5I_INVALID_HID; - vfdtype vfd; - - vfd = param->vfd; - - if ((my_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) - return -1; - - if (vfd == sec2) { - /* Unix read() and write() system calls */ - if (H5Pset_fapl_sec2(my_fapl) < 0) - return -1; - } - else if (vfd == stdio) { - /* Standard C fread() and fwrite() system calls */ - if (H5Pset_fapl_stdio(my_fapl) < 0) - return -1; - } - else if (vfd == core) { - /* In-core temporary file with 1MB increment */ - if (H5Pset_fapl_core(my_fapl, (size_t)1024 * 1024, TRUE) < 0) - return -1; - } - else if (vfd == split) { - /* Split meta data and raw data each using default driver */ - if (H5Pset_fapl_split(my_fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0) - return -1; - } - else if (vfd == multi) { - /* Multi-file driver, general case of the split driver */ - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; - hid_t memb_fapl[H5FD_MEM_NTYPES]; - const char *memb_name[H5FD_MEM_NTYPES]; - char sv[H5FD_MEM_NTYPES][1024]; - haddr_t memb_addr[H5FD_MEM_NTYPES]; - H5FD_mem_t mt; - - HDmemset(memb_map, 0, sizeof memb_map); - HDmemset(memb_fapl, 0, sizeof memb_fapl); - HDmemset(memb_name, 0, sizeof memb_name); - HDmemset(memb_addr, 0, sizeof memb_addr); - - HDassert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { - memb_fapl[mt] = H5P_DEFAULT; - HDsprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]); - memb_name[mt] = sv[mt]; - memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10); - } - - if (H5Pset_fapl_multi(my_fapl, memb_map, memb_fapl, memb_name, memb_addr, FALSE) < 0) { - return -1; - } - } - else if (vfd == family) { - hsize_t fam_size = 1 * 1024 * 1024; /*100 MB*/ - - /* Family of files, each 1MB and using the default driver */ - /* if ((val=HDstrtok(NULL, " \t\n\r"))) - fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024); */ - if (H5Pset_fapl_family(my_fapl, fam_size, H5P_DEFAULT) < 0) - return -1; - } - else if (vfd == direct) { -#ifdef H5_HAVE_DIRECT - /* Linux direct read() and write() system calls. Set memory boundary, file block size, - * and copy buffer size to the default values. */ - if (H5Pset_fapl_direct(my_fapl, 1024, 4096, 8 * 4096) < 0) - return -1; -#endif - } - else { - /* Unknown driver */ - return -1; - } - - return my_fapl; -} - -/* - * Function: do_fclose - * Purpose: Close the specified file descriptor. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: - */ -static herr_t -do_fclose(iotype iot, file_descr *fd /*out*/) -{ - herr_t ret_code = SUCCESS, hrc; - int rc = 0; - - switch (iot) { - case POSIXIO: - rc = POSIXCLOSE(fd->posixfd); - - if (rc != 0) { - HDfprintf(stderr, "POSIX File Close failed\n"); - GOTOERROR(FAIL); - } - - fd->posixfd = -1; - break; - - case HDF5: - hrc = H5Fclose(fd->h5fd); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 File Close failed\n"); - GOTOERROR(FAIL); - } - - fd->h5fd = -1; - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - GOTOERROR(FAIL); - break; - } - -done: - return ret_code; -} - -/* - * Function: do_cleanupfile - * Purpose: Cleanup temporary file unless HDF5_NOCLEANUP is set. - * Return: void - * Programmer: Albert Cheng 2001/12/12 - * Modifications: Support for file drivers. Christian Chilan, April, 2008 - */ -static void -do_cleanupfile(iotype iot, char *filename) -{ - char temp[2048]; - int j; - hid_t driver; - - if (clean_file_g == -1) - clean_file_g = (HDgetenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; - - if (clean_file_g) { - - switch (iot) { - case POSIXIO: - HDremove(filename); - break; - - case HDF5: - driver = H5Pget_driver(fapl); - - if (driver == H5FD_FAMILY) { - for (j = 0; /*void*/; j++) { - HDsnprintf(temp, sizeof temp, filename, j); - - if (HDaccess(temp, F_OK) < 0) - break; - - HDremove(temp); - } - } - else if (driver == H5FD_CORE) { - hbool_t backing; /* Whether the core file has backing store */ - - H5Pget_fapl_core(fapl, NULL, &backing); - - /* If the file was stored to disk with bacing store, remove it */ - if (backing) - HDremove(filename); - } - else if (driver == H5FD_MULTI) { - H5FD_mem_t mt; - assert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); - - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { - HDsnprintf(temp, sizeof temp, "%s-%c.h5", filename, multi_letters[mt]); - HDremove(temp); /*don't care if it fails*/ - } - } - else { - HDremove(filename); - } - H5Pclose(fapl); - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - HDassert(0 && "Unknown IO type"); - break; - } - } -} diff --git a/tools/test/perform/sio_perf.c b/tools/test/perform/sio_perf.c deleted file mode 100644 index 51a7825..0000000 --- a/tools/test/perform/sio_perf.c +++ /dev/null @@ -1,1437 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Serial HDF5 Performance Testing Code - * -------------------------------------- - * - * Portable code to test performance on the different platforms we support. - * This is what the report should look like: - * - * nprocs = Max#Procs - * IO API = POSIXIO - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * - * IO API = HDF5 - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * - * . . . - * - */ - -/* system header files */ -#include -#include -#include - -#include "hdf5.h" - -/* our header files */ -#include "sio_perf.h" - -/* useful macros */ -#define TAB_SPACE 4 - -#define ONE_KB 1024 -#define ONE_MB (ONE_KB * ONE_KB) -#define ONE_GB (ONE_MB * ONE_KB) - -#define SIO_POSIX 0x1 -#define SIO_HDF5 0x4 - -/* report 0.0 in case t is zero too */ -#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL(t, 0.0) ? 0.0 : ((((double)bytes) / (double)ONE_MB) / (t))) - -#ifndef TRUE -#define TRUE 1 -#endif /* TRUE */ -#ifndef FALSE -#define FALSE (!TRUE) -#endif /* FALSE */ - -/* global variables */ -FILE *output; /* output file */ -int sio_debug_level = 0; /* The debug level: - * 0 - Off - * 1 - Minimal - * 2 - Some more - * 3 - Maximal - * 4 - Maximal & then some - */ - -/* local variables */ -static const char *progname = "h5perf_serial"; - -/* - * Command-line options: The user can specify short or long-named - * parameters. The long-named ones can be partially spelled. When - * adding more, make sure that they don't clash with each other. - */ - -/* - * It seems that only the options that accept additional information - * such as dataset size (-e) require the colon next to it. - */ -static const char * s_opts = "a:A:B:c:Cd:D:e:F:ghi:Imno:p:P:r:stT:v:wx:X:"; -static struct h5_long_options l_opts[] = {{"align", require_arg, 'a'}, - {"alig", require_arg, 'a'}, - {"ali", require_arg, 'a'}, - {"al", require_arg, 'a'}, - {"api", require_arg, 'A'}, - {"ap", require_arg, 'A'}, -#if 0 - /* a sighting of the elusive binary option */ - { "binary", no_arg, 'b' }, - { "binar", no_arg, 'b' }, - { "bina", no_arg, 'b' }, - { "bin", no_arg, 'b' }, - { "bi", no_arg, 'b' }, -#endif /* 0 */ - {"block-size", require_arg, 'B'}, - {"block-siz", require_arg, 'B'}, - {"block-si", require_arg, 'B'}, - {"block-s", require_arg, 'B'}, - {"block-", require_arg, 'B'}, - {"block", require_arg, 'B'}, - {"bloc", require_arg, 'B'}, - {"blo", require_arg, 'B'}, - {"bl", require_arg, 'B'}, - {"chunk", no_arg, 'c'}, - {"chun", no_arg, 'c'}, - {"chu", no_arg, 'c'}, - {"ch", no_arg, 'c'}, - {"collective", no_arg, 'C'}, - {"collectiv", no_arg, 'C'}, - {"collecti", no_arg, 'C'}, - {"collect", no_arg, 'C'}, - {"collec", no_arg, 'C'}, - {"colle", no_arg, 'C'}, - {"coll", no_arg, 'C'}, - {"col", no_arg, 'C'}, - {"co", no_arg, 'C'}, - {"debug", require_arg, 'D'}, - {"debu", require_arg, 'D'}, - {"deb", require_arg, 'D'}, - {"de", require_arg, 'D'}, - {"file-driver", require_arg, 'v'}, - {"file-drive", require_arg, 'v'}, - {"file-driv", require_arg, 'v'}, - {"file-dri", require_arg, 'v'}, - {"file-dr", require_arg, 'v'}, - {"file-d", require_arg, 'v'}, - {"file-", require_arg, 'v'}, - {"file", require_arg, 'v'}, - {"fil", require_arg, 'v'}, - {"fi", require_arg, 'v'}, - {"geometry", no_arg, 'g'}, - {"geometr", no_arg, 'g'}, - {"geomet", no_arg, 'g'}, - {"geome", no_arg, 'g'}, - {"geom", no_arg, 'g'}, - {"geo", no_arg, 'g'}, - {"ge", no_arg, 'g'}, - {"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, - {"interleaved", require_arg, 'I'}, - {"interleave", require_arg, 'I'}, - {"interleav", require_arg, 'I'}, - {"interlea", require_arg, 'I'}, - {"interle", require_arg, 'I'}, - {"interl", require_arg, 'I'}, - {"inter", require_arg, 'I'}, - {"inte", require_arg, 'I'}, - {"int", require_arg, 'I'}, - {"in", require_arg, 'I'}, - {"max-num-processes", require_arg, 'P'}, - {"max-num-processe", require_arg, 'P'}, - {"max-num-process", require_arg, 'P'}, - {"max-num-proces", require_arg, 'P'}, - {"max-num-proce", require_arg, 'P'}, - {"max-num-proc", require_arg, 'P'}, - {"max-num-pro", require_arg, 'P'}, - {"max-num-pr", require_arg, 'P'}, - {"max-num-p", require_arg, 'P'}, - {"min-num-processes", require_arg, 'p'}, - {"min-num-processe", require_arg, 'p'}, - {"min-num-process", require_arg, 'p'}, - {"min-num-proces", require_arg, 'p'}, - {"min-num-proce", require_arg, 'p'}, - {"min-num-proc", require_arg, 'p'}, - {"min-num-pro", require_arg, 'p'}, - {"min-num-pr", require_arg, 'p'}, - {"min-num-p", require_arg, 'p'}, - {"max-xfer-size", require_arg, 'X'}, - {"max-xfer-siz", require_arg, 'X'}, - {"max-xfer-si", require_arg, 'X'}, - {"max-xfer-s", require_arg, 'X'}, - {"max-xfer", require_arg, 'X'}, - {"max-xfe", require_arg, 'X'}, - {"max-xf", require_arg, 'X'}, - {"max-x", require_arg, 'X'}, - {"min-xfer-size", require_arg, 'x'}, - {"min-xfer-siz", require_arg, 'x'}, - {"min-xfer-si", require_arg, 'x'}, - {"min-xfer-s", require_arg, 'x'}, - {"min-xfer", require_arg, 'x'}, - {"min-xfe", require_arg, 'x'}, - {"min-xf", require_arg, 'x'}, - {"min-x", require_arg, 'x'}, - {"num-bytes", require_arg, 'e'}, - {"num-byte", require_arg, 'e'}, - {"num-byt", require_arg, 'e'}, - {"num-by", require_arg, 'e'}, - {"num-b", require_arg, 'e'}, - {"num-dsets", require_arg, 'd'}, - {"num-dset", require_arg, 'd'}, - {"num-dse", require_arg, 'd'}, - {"num-ds", require_arg, 'd'}, - {"num-d", require_arg, 'd'}, - {"num-files", require_arg, 'F'}, - {"num-file", require_arg, 'F'}, - {"num-fil", require_arg, 'F'}, - {"num-fi", require_arg, 'F'}, - {"num-f", require_arg, 'F'}, - {"num-iterations", require_arg, 'i'}, - {"num-iteration", require_arg, 'i'}, - {"num-iteratio", require_arg, 'i'}, - {"num-iterati", require_arg, 'i'}, - {"num-iterat", require_arg, 'i'}, - {"num-itera", require_arg, 'i'}, - {"num-iter", require_arg, 'i'}, - {"num-ite", require_arg, 'i'}, - {"num-it", require_arg, 'i'}, - {"num-i", require_arg, 'i'}, - {"order", require_arg, 'r'}, - {"orde", require_arg, 'r'}, - {"ord", require_arg, 'r'}, - {"or", require_arg, 'r'}, - {"output", require_arg, 'o'}, - {"outpu", require_arg, 'o'}, - {"outp", require_arg, 'o'}, - {"out", require_arg, 'o'}, - {"ou", require_arg, 'o'}, - {"extendable", no_arg, 't'}, - {"extendabl", no_arg, 't'}, - {"extendab", no_arg, 't'}, - {"extenda", no_arg, 't'}, - {"extend", no_arg, 't'}, - {"exten", no_arg, 't'}, - {"exte", no_arg, 't'}, - {"ext", no_arg, 't'}, - {"ex", no_arg, 't'}, - {"threshold", require_arg, 'T'}, - {"threshol", require_arg, 'T'}, - {"thresho", require_arg, 'T'}, - {"thresh", require_arg, 'T'}, - {"thres", require_arg, 'T'}, - {"thre", require_arg, 'T'}, - {"thr", require_arg, 'T'}, - {"th", require_arg, 'T'}, - {"write-only", require_arg, 'w'}, - {"write-onl", require_arg, 'w'}, - {"write-on", require_arg, 'w'}, - {"write-o", require_arg, 'w'}, - {"write", require_arg, 'w'}, - {"writ", require_arg, 'w'}, - {"wri", require_arg, 'w'}, - {"wr", require_arg, 'w'}, - {NULL, 0, '\0'}}; - -struct options { - long io_types; /* bitmask of which I/O types to test */ - const char *output_file; /* file to print report to */ - long num_dsets; /* number of datasets */ - long num_files; /* number of files */ - off_t num_bpp; /* number of bytes per proc per dset */ - int num_iters; /* number of iterations */ - hsize_t dset_size[MAX_DIMS]; /* Dataset size */ - size_t buf_size[MAX_DIMS]; /* Buffer size */ - size_t chk_size[MAX_DIMS]; /* Chunk size */ - int order[MAX_DIMS]; /* Dimension access order */ - int dset_rank; /* Rank */ - int buf_rank; /* Rank */ - int order_rank; /* Rank */ - int chk_rank; /* Rank */ - int print_times; /* print times as well as throughputs */ - int print_raw; /* print raw data throughput info */ - hsize_t h5_alignment; /* alignment in HDF5 file */ - hsize_t h5_threshold; /* threshold for alignment in HDF5 file */ - int h5_use_chunks; /* Make HDF5 dataset chunked */ - int h5_write_only; /* Perform the write tests only */ - int h5_extendable; /* Perform the write tests only */ - int verify; /* Verify data correctness */ - vfdtype vfd; /* File driver */ - size_t page_buffer_size; - size_t page_size; -}; - -typedef struct { - double min; - double max; - double sum; - int num; -} minmax; - -/* local functions */ -static hsize_t parse_size_directive(const char *size); -static struct options *parse_command_line(int argc, const char *argv[]); -static void run_test_loop(struct options *options); -static int run_test(iotype iot, parameters parms, struct options *opts); -static void output_all_info(minmax *mm, int count, int indent_level); -static void get_minmax(minmax *mm, double val); -static void accumulate_minmax_stuff(const minmax *mm, int count, minmax *total_mm); -static void output_results(const struct options *options, const char *name, minmax *table, int table_size, - off_t data_size); -static void output_report(const char *fmt, ...); -static void print_indent(register int indent); -static void usage(const char *prog); -static void report_parameters(struct options *opts); - -/* - * Function: main - * Purpose: Start things up. - * Return: EXIT_SUCCESS or EXIT_FAILURE - * Programmer: Bill Wendling, 30. October 2001 - * Modifications: - */ -int -main(int argc, const char *argv[]) -{ - int exit_value = EXIT_SUCCESS; - struct options *opts = NULL; - -#ifndef STANDALONE - /* Initialize h5tools lib */ - h5tools_init(); -#endif - - output = stdout; - - opts = parse_command_line(argc, argv); - - if (!opts) { - exit_value = EXIT_FAILURE; - goto finish; - } - - if (opts->output_file) { - if ((output = HDfopen(opts->output_file, "w")) == NULL) { - HDfprintf(stderr, "%s: cannot open output file\n", progname); - HDperror(opts->output_file); - goto finish; - } - } - - report_parameters(opts); - - run_test_loop(opts); - -finish: - HDfree(opts); - return exit_value; -} - -/* - * Function: run_test_loop - * Purpose: Run the I/O tests. Write the results to OUTPUT. - * - * - The slowest changing part of the test is the number of - * processors to use. For each loop iteration, we divide that - * number by 2 and rerun the test. - * - * - The second slowest is what type of IO API to perform. We have - * three choices: POSIXIO, and HDF5. - * - * - Then we change the size of the buffer. This information is - * inferred from the number of datasets to create and the number - * of integers to put into each dataset. The backend code figures - * this out. - * - * Return: Nothing - * Programmer: Bill Wendling, 30. October 2001 - * Modifications: - * Added multidimensional testing (Christian Chilan, April, 2008) - */ -static void -run_test_loop(struct options *opts) -{ - parameters parms; - int i; - size_t buf_bytes; - - /* load options into parameter structure */ - parms.num_files = opts->num_files; - parms.num_dsets = opts->num_dsets; - parms.num_iters = opts->num_iters; - parms.rank = opts->dset_rank; - parms.h5_align = opts->h5_alignment; - parms.h5_thresh = opts->h5_threshold; - parms.h5_use_chunks = opts->h5_use_chunks; - parms.h5_extendable = opts->h5_extendable; - parms.h5_write_only = opts->h5_write_only; - parms.verify = opts->verify; - parms.vfd = opts->vfd; - parms.page_buffer_size = opts->page_buffer_size; - parms.page_size = opts->page_size; - - /* load multidimensional options */ - parms.num_bytes = 1; - buf_bytes = 1; - for (i = 0; i < parms.rank; i++) { - parms.buf_size[i] = opts->buf_size[i]; - parms.dset_size[i] = opts->dset_size[i]; - parms.chk_size[i] = opts->chk_size[i]; - parms.order[i] = opts->order[i]; - parms.num_bytes *= opts->dset_size[i]; - buf_bytes *= opts->buf_size[i]; - } - - /* print size information */ - output_report("Transfer Buffer Size (bytes): %d\n", buf_bytes); - output_report("File Size(MB): %.2f\n", ((double)parms.num_bytes) / ONE_MB); - - print_indent(0); - if (opts->io_types & SIO_POSIX) - run_test(POSIXIO, parms, opts); - - print_indent(0); - if (opts->io_types & SIO_HDF5) - run_test(HDF5, parms, opts); -} - -/* - * Function: run_test - * Purpose: Inner loop call to actually run the I/O test. - * Return: Nothing - * Programmer: Bill Wendling, 18. December 2001 - * Modifications: - */ -static int -run_test(iotype iot, parameters parms, struct options *opts) -{ - results res; - register int i, ret_value = SUCCESS; - off_t raw_size; - minmax * write_sys_mm_table = NULL; - minmax * write_mm_table = NULL; - minmax * write_gross_mm_table = NULL; - minmax * write_raw_mm_table = NULL; - minmax * read_sys_mm_table = NULL; - minmax * read_mm_table = NULL; - minmax * read_gross_mm_table = NULL; - minmax * read_raw_mm_table = NULL; - minmax write_sys_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax write_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax write_gross_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax write_raw_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_sys_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_gross_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_raw_mm = {0.0F, 0.0F, 0.0F, 0}; - - raw_size = (off_t)parms.num_bytes; - parms.io_type = iot; - print_indent(2); - output_report("IO API = "); - - switch (iot) { - case POSIXIO: - output_report("POSIX\n"); - break; - case HDF5: - output_report("HDF5\n"); - break; - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - HDassert(0 && "Unknown IO tpe"); - break; - } - - /* allocate space for tables minmax and that it is sufficient */ - /* to initialize all elements to zeros by calloc. */ - write_sys_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - write_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - write_gross_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - write_raw_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - - if (!parms.h5_write_only) { - read_sys_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - read_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - read_gross_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - read_raw_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - } - - /* Do IO iteration times, collecting statistics each time */ - for (i = 0; i < parms.num_iters; ++i) { - double t; - - do_sio(parms, &res); - - /* gather all of the "sys write" times */ - t = io_time_get(res.timers, HDF5_MPI_WRITE); - get_minmax(&write_sys_mm, t); - - write_sys_mm_table[i] = write_sys_mm; - - /* gather all of the "write" times */ - t = io_time_get(res.timers, HDF5_FINE_WRITE_FIXED_DIMS); - get_minmax(&write_mm, t); - - write_mm_table[i] = write_mm; - - /* gather all of the "write" times from open to close */ - t = io_time_get(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS); - get_minmax(&write_gross_mm, t); - - write_gross_mm_table[i] = write_gross_mm; - - /* gather all of the raw "write" times */ - t = io_time_get(res.timers, HDF5_RAW_WRITE_FIXED_DIMS); - get_minmax(&write_raw_mm, t); - - write_raw_mm_table[i] = write_raw_mm; - - if (!parms.h5_write_only) { - /* gather all of the "mpi read" times */ - t = io_time_get(res.timers, HDF5_MPI_READ); - get_minmax(&read_sys_mm, t); - - read_sys_mm_table[i] = read_sys_mm; - - /* gather all of the "read" times */ - t = io_time_get(res.timers, HDF5_FINE_READ_FIXED_DIMS); - get_minmax(&read_mm, t); - - read_mm_table[i] = read_mm; - - /* gather all of the "read" times from open to close */ - t = io_time_get(res.timers, HDF5_GROSS_READ_FIXED_DIMS); - get_minmax(&read_gross_mm, t); - - read_gross_mm_table[i] = read_gross_mm; - - /* gather all of the raw "read" times */ - t = io_time_get(res.timers, HDF5_RAW_READ_FIXED_DIMS); - get_minmax(&read_raw_mm, t); - - read_raw_mm_table[i] = read_gross_mm; - } - io_time_destroy(res.timers); - } - - /* - * Show various statistics - */ - /* Write statistics */ - /* Print the raw data throughput if desired */ - if (opts->print_raw) { - /* accumulate and output the max, min, and average "raw write" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Raw Data Write details:\n"); - output_all_info(write_raw_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); - } /* end if */ - - /* show sys write statics */ -#if 0 - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("MPI Write details:\n"); - output_all_info(write_sys_mm_table, parms.num_iters, 4); - } -#endif - /* We don't currently output the MPI write results */ - - /* accumulate and output the max, min, and average "write" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write details:\n"); - output_all_info(write_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Write", write_mm_table, parms.num_iters, raw_size); - - /* accumulate and output the max, min, and average "gross write" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write Open-Close details:\n"); - output_all_info(write_gross_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Write Open-Close", write_gross_mm_table, parms.num_iters, raw_size); - - if (!parms.h5_write_only) { - /* Read statistics */ - /* Print the raw data throughput if desired */ - if (opts->print_raw) { - /* accumulate and output the max, min, and average "raw read" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Raw Data Read details:\n"); - output_all_info(read_raw_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); - } /* end if */ - - /* show mpi read statics */ -#if 0 - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("MPI Read details:\n"); - output_all_info(read_sys_mm_table, parms.num_iters, 4); - } -#endif - /* We don't currently output the MPI read results */ - - /* accumulate and output the max, min, and average "read" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read details:\n"); - output_all_info(read_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Read", read_mm_table, parms.num_iters, raw_size); - - /* accumulate and output the max, min, and average "gross read" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read Open-Close details:\n"); - output_all_info(read_gross_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Read Open-Close", read_gross_mm_table, parms.num_iters, raw_size); - } - - /* clean up our mess */ - HDfree(write_sys_mm_table); - HDfree(write_mm_table); - HDfree(write_gross_mm_table); - HDfree(write_raw_mm_table); - - if (!parms.h5_write_only) { - HDfree(read_sys_mm_table); - HDfree(read_mm_table); - HDfree(read_gross_mm_table); - HDfree(read_raw_mm_table); - } - - return ret_value; -} - -/* - * Function: output_all_info - * Purpose: - * Return: Nothing - * Programmer: Bill Wendling, 29. January 2002 - * Modifications: - */ -static void -output_all_info(minmax *mm, int count, int indent_level) -{ - int i; - - for (i = 0; i < count; ++i) { - print_indent(indent_level); - output_report("Iteration %d:\n", i + 1); - print_indent(indent_level + 1); - output_report("Minimum Time: %.2fs\n", mm[i].min); - print_indent(indent_level + 1); - output_report("Maximum Time: %.2fs\n", mm[i].max); - } -} - -/* - * Function: get_minmax - * Purpose: Gather all the min, max and total of val. - * Return: Nothing - * Programmer: Bill Wendling, 21. December 2001 - * Modifications: - * Use MPI_Allreduce to do it. -akc, 2002/01/11 - */ - -static void -get_minmax(minmax *mm, double val) -{ - mm->max = val; - mm->min = val; - mm->sum = val; -} - -/* - * Function: accumulate_minmax_stuff - * Purpose: Accumulate the minimum, maximum, and average of the times - * across all processes. - * Return: TOTAL_MM - the total of all of these. - * Programmer: Bill Wendling, 21. December 2001 - * Modifications: - * Changed to use seconds instead of MB/s - QAK, 5/9/02 - */ -static void -accumulate_minmax_stuff(const minmax *mm, int count, minmax *total_mm) -{ - int i; - - total_mm->sum = 0.0F; - total_mm->max = -DBL_MAX; - total_mm->min = DBL_MAX; - total_mm->num = count; - - for (i = 0; i < count; ++i) { - double m = mm[i].max; - - total_mm->sum += m; - - if (m < total_mm->min) - total_mm->min = m; - - if (m > total_mm->max) - total_mm->max = m; - } -} - -/* - * Function: output_results - * Purpose: Print information about the time & bandwidth for a given - * minmax & # of iterations. - * Return: Nothing - * Programmer: Quincey Koziol, 9. May 2002 - * Modifications: - */ -static void -output_results(const struct options *opts, const char *name, minmax *table, int table_size, off_t data_size) -{ - minmax total_mm; - - accumulate_minmax_stuff(table, table_size, &total_mm); - - print_indent(3); - output_report("%s (%d iteration(s)):\n", name, table_size); - - /* Note: The maximum throughput uses the minimum amount of time & vice versa */ - - print_indent(4); - output_report("Maximum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.min)); - if (opts->print_times) - output_report(" (%7.3f s)\n", total_mm.min); - else - output_report("\n"); - - print_indent(4); - output_report("Average Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.sum / total_mm.num)); - if (opts->print_times) - output_report(" (%7.3f s)\n", (total_mm.sum / total_mm.num)); - else - output_report("\n"); - - print_indent(4); - output_report("Minimum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.max)); - if (opts->print_times) - output_report(" (%7.3f s)\n", total_mm.max); - else - output_report("\n"); -} - -/* - * Function: output_report - * Purpose: Print a line of the report. Only do so if I'm the 0 process. - * Return: Nothing - * Programmer: Bill Wendling, 19. December 2001 - * Modifications: - */ -static void -output_report(const char *fmt, ...) -{ - va_list ap; - - HDva_start(ap, fmt); - HDvfprintf(output, fmt, ap); - HDva_end(ap); -} - -/* - * Function: print_indent - * Purpose: Print spaces to indent a new line of text for pretty printing - * things. - * Return: Nothing - * Programmer: Bill Wendling, 29. October 2001 - * Modifications: - */ -static void -print_indent(register int indent) -{ - indent *= TAB_SPACE; - - for (; indent > 0; --indent) - HDfputc(' ', output); -} - -static void -recover_size_and_print(long long val, const char *end) -{ - if (val >= ONE_KB && (val % ONE_KB) == 0) { - if (val >= ONE_MB && (val % ONE_MB) == 0) { - if (val >= ONE_GB && (val % ONE_GB) == 0) - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "GB%s", - val / ONE_GB, end); - else - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "MB%s", - val / ONE_MB, end); - } - else { - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "KB%s", - val / ONE_KB, end); - } - } - else { - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "%s", - val, end); - } -} - -static void -print_io_api(long io_types) -{ - if (io_types & SIO_POSIX) - HDfprintf(output, "posix "); - if (io_types & SIO_HDF5) - HDfprintf(output, "hdf5 "); - HDfprintf(output, "\n"); -} - -static void -report_parameters(struct options *opts) -{ - int i, rank; - rank = opts->dset_rank; - - print_version("HDF5 Library"); /* print library version */ - HDfprintf(output, "==== Parameters ====\n"); - - HDfprintf(output, "IO API="); - print_io_api(opts->io_types); - - HDfprintf(output, "Number of iterations=%d\n", opts->num_iters); - - HDfprintf(output, "Dataset size="); - - for (i = 0; i < rank; i++) - recover_size_and_print((long long)opts->dset_size[i], " "); - HDfprintf(output, "\n"); - - HDfprintf(output, "Transfer buffer size="); - for (i = 0; i < rank; i++) - recover_size_and_print((long long)opts->buf_size[i], " "); - HDfprintf(output, "\n"); - - if (opts->page_size) { - HDfprintf(output, "Page Aggregation Enabled. Page size = %zu\n", opts->page_size); - if (opts->page_buffer_size) - HDfprintf(output, "Page Buffering Enabled. Page Buffer size = %zu\n", opts->page_buffer_size); - else - HDfprintf(output, "Page Buffering Disabled\n"); - } - else - HDfprintf(output, "Page Aggregation Disabled\n"); - - HDfprintf(output, "Dimension access order="); - for (i = 0; i < rank; i++) - recover_size_and_print((long long)opts->order[i], " "); - HDfprintf(output, "\n"); - - if (opts->io_types & SIO_HDF5) { - - HDfprintf(output, "HDF5 data storage method="); - - if (opts->h5_use_chunks) { - - HDfprintf(output, "Chunked\n"); - HDfprintf(output, "HDF5 chunk size="); - for (i = 0; i < rank; i++) - recover_size_and_print((long long)opts->chk_size[i], " "); - HDfprintf(output, "\n"); - - HDfprintf(output, "HDF5 dataset dimensions="); - if (opts->h5_extendable) { - HDfprintf(output, "Extendable\n"); - } - else { - HDfprintf(output, "Fixed\n"); - } - } - else { - HDfprintf(output, "Contiguous\n"); - } - - HDfprintf(output, "HDF5 file driver="); - if (opts->vfd == sec2) { - HDfprintf(output, "sec2\n"); - } - else if (opts->vfd == stdio) { - HDfprintf(output, "stdio\n"); - } - else if (opts->vfd == core) { - HDfprintf(output, "core\n"); - } - else if (opts->vfd == split) { - HDfprintf(output, "split\n"); - } - else if (opts->vfd == multi) { - HDfprintf(output, "multi\n"); - } - else if (opts->vfd == family) { - HDfprintf(output, "family\n"); - } - else if (opts->vfd == direct) { - HDfprintf(output, "direct\n"); - } - } - - { - char *prefix = HDgetenv("HDF5_PREFIX"); - - HDfprintf(output, "Env HDF5_PREFIX=%s\n", (prefix ? prefix : "not set")); - } - - HDfprintf(output, "==== End of Parameters ====\n"); - HDfprintf(output, "\n"); -} - -/* - * Function: parse_command_line - * Purpose: Parse the command line options and return a STRUCT OPTIONS - * structure which will need to be freed by the calling function. - * Return: Pointer to an OPTIONS structure - * Programmer: Bill Wendling, 31. October 2001 - * Modifications: - * Added multidimensional testing (Christian Chilan, April, 2008) - */ -static struct options * -parse_command_line(int argc, const char *argv[]) -{ - int opt; - struct options *cl_opts; - int i, default_rank, actual_rank, ranks[4]; - - cl_opts = (struct options *)HDmalloc(sizeof(struct options)); - - cl_opts->page_buffer_size = 0; - cl_opts->page_size = 0; - - cl_opts->output_file = NULL; - cl_opts->io_types = 0; /* will set default after parsing options */ - cl_opts->num_iters = 1; - - default_rank = 2; - - cl_opts->dset_rank = 0; - cl_opts->buf_rank = 0; - cl_opts->chk_rank = 0; - cl_opts->order_rank = 0; - - for (i = 0; i < MAX_DIMS; i++) { - cl_opts->buf_size[i] = (size_t)((i + 1) * 10); - cl_opts->dset_size[i] = (hsize_t)((i + 1) * 100); - cl_opts->chk_size[i] = (size_t)((i + 1) * 10); - cl_opts->order[i] = i + 1; - } - - cl_opts->vfd = sec2; - - cl_opts->print_times = FALSE; /* Printing times is off by default */ - cl_opts->print_raw = FALSE; /* Printing raw data throughput is off by default */ - cl_opts->h5_alignment = 1; /* No alignment for HDF5 objects by default */ - cl_opts->h5_threshold = 1; /* No threshold for aligning HDF5 objects by default */ - cl_opts->h5_use_chunks = FALSE; /* Don't chunk the HDF5 dataset by default */ - cl_opts->h5_write_only = FALSE; /* Do both read and write by default */ - cl_opts->h5_extendable = FALSE; /* Use extendable dataset */ - cl_opts->verify = FALSE; /* No Verify data correctness by default */ - - while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { - switch ((char)opt) { - case 'a': - cl_opts->h5_alignment = parse_size_directive(H5_optarg); - break; - case 'G': - cl_opts->page_size = parse_size_directive(H5_optarg); - break; - case 'b': - cl_opts->page_buffer_size = parse_size_directive(H5_optarg); - break; - case 'A': { - const char *end = H5_optarg; - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - if (!HDstrcasecmp(buf, "hdf5")) { - cl_opts->io_types |= SIO_HDF5; - } - else if (!HDstrcasecmp(buf, "posix")) { - cl_opts->io_types |= SIO_POSIX; - } - else { - HDfprintf(stderr, "sio_perf: invalid --api option %s\n", buf); - HDexit(EXIT_FAILURE); - } - - if (*end == '\0') - break; - - end++; - } - } - - break; -#if 0 - case 'b': - /* the future "binary" option */ - break; -#endif /* 0 */ - case 'c': - /* Turn on chunked HDF5 dataset creation */ - cl_opts->h5_use_chunks = 1; - { - const char *end = H5_optarg; - int j = 0; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - cl_opts->chk_size[j] = parse_size_directive(buf); - - j++; - - if (*end == '\0') - break; - - end++; - } - cl_opts->chk_rank = j; - } - - break; - - case 'D': { - const char *end = H5_optarg; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - if (HDstrlen(buf) > 1 || HDisdigit(buf[0])) { - size_t j; - - for (j = 0; j < 10 && buf[j] != '\0'; ++j) - if (!HDisdigit(buf[j])) { - HDfprintf(stderr, "sio_perf: invalid --debug option %s\n", buf); - HDexit(EXIT_FAILURE); - } - - sio_debug_level = atoi(buf); - - if (sio_debug_level > 4) - sio_debug_level = 4; - else if (sio_debug_level < 0) - sio_debug_level = 0; - } - else { - switch (*buf) { - case 'r': - /* Turn on raw data throughput info */ - cl_opts->print_raw = TRUE; - break; - case 't': - /* Turn on time printing */ - cl_opts->print_times = TRUE; - break; - case 'v': - /* Turn on verify data correctness*/ - cl_opts->verify = TRUE; - break; - default: - HDfprintf(stderr, "sio_perf: invalid --debug option %s\n", buf); - HDexit(EXIT_FAILURE); - } - } - - if (*end == '\0') - break; - - end++; - } - } - - break; - case 'e': { - const char *end = H5_optarg; - int j = 0; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - cl_opts->dset_size[j] = parse_size_directive(buf); - - j++; - - if (*end == '\0') - break; - - end++; - } - cl_opts->dset_rank = j; - } - - break; - - case 'i': - cl_opts->num_iters = HDatoi(H5_optarg); - break; - case 'o': - cl_opts->output_file = H5_optarg; - break; - case 'T': - cl_opts->h5_threshold = parse_size_directive(H5_optarg); - break; - case 'v': - if (!HDstrcasecmp(H5_optarg, "sec2")) { - cl_opts->vfd = sec2; - } - else if (!HDstrcasecmp(H5_optarg, "stdio")) { - cl_opts->vfd = stdio; - } - else if (!HDstrcasecmp(H5_optarg, "core")) { - cl_opts->vfd = core; - } - else if (!HDstrcasecmp(H5_optarg, "split")) { - cl_opts->vfd = split; - } - else if (!HDstrcasecmp(H5_optarg, "multi")) { - cl_opts->vfd = multi; - } - else if (!HDstrcasecmp(H5_optarg, "family")) { - cl_opts->vfd = family; - } - else if (!HDstrcasecmp(H5_optarg, "direct")) { - cl_opts->vfd = direct; - } - else { - HDfprintf(stderr, "sio_perf: invalid --api option %s\n", H5_optarg); - HDexit(EXIT_FAILURE); - } - break; - case 'w': - cl_opts->h5_write_only = TRUE; - break; - case 't': - cl_opts->h5_extendable = TRUE; - break; - case 'x': { - const char *end = H5_optarg; - int j = 0; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - cl_opts->buf_size[j] = parse_size_directive(buf); - - j++; - - if (*end == '\0') - break; - - end++; - } - cl_opts->buf_rank = j; - } - - break; - - case 'r': { - const char *end = H5_optarg; - int j = 0; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - cl_opts->order[j] = (int)parse_size_directive(buf); - - j++; - - if (*end == '\0') - break; - - end++; - } - - cl_opts->order_rank = j; - } - - break; - - case 'h': - case '?': - default: - usage(progname); - HDfree(cl_opts); - return NULL; - } - } - - /* perform rank consistency analysis */ - actual_rank = 0; - - ranks[0] = cl_opts->dset_rank; - ranks[1] = cl_opts->buf_rank; - ranks[2] = cl_opts->order_rank; - ranks[3] = cl_opts->chk_rank; - - for (i = 0; i < 4; i++) { - if (ranks[i] > 0) { - if (!actual_rank) { - actual_rank = ranks[i]; - } - else { - if (actual_rank != ranks[i]) - exit(EXIT_FAILURE); - } - } - } - - if (!actual_rank) - actual_rank = default_rank; - - cl_opts->dset_rank = actual_rank; - cl_opts->buf_rank = actual_rank; - cl_opts->order_rank = actual_rank; - cl_opts->chk_rank = actual_rank; - - for (i = 0; i < actual_rank; i++) { - if (cl_opts->order[i] > actual_rank) { - exit(EXIT_FAILURE); - } - } - - /* set default if none specified yet */ - if (!cl_opts->io_types) - cl_opts->io_types = SIO_HDF5 | SIO_POSIX; /* run all API */ - - /* verify parameters sanity. Adjust if needed. */ - /* cap xfer_size with bytes per process */ - if (cl_opts->num_iters <= 0) - cl_opts->num_iters = 1; - - return cl_opts; -} - -/* - * Function: parse_size_directive - * Purpose: Parse the size directive passed on the commandline. The size - * directive is an integer followed by a size indicator: - * - * K, k - Kilobyte - * M, m - Megabyte - * G, g - Gigabyte - * - * Return: The size as a off_t because this is related to file size. - * If an unknown size indicator is used, then the program will - * exit with EXIT_FAILURE as the return value. - * Programmer: Bill Wendling, 18. December 2001 - * Modifications: - */ - -static hsize_t -parse_size_directive(const char *size) -{ - hsize_t s; - char * endptr; - - s = HDstrtoull(size, &endptr, 10); - - if (endptr && *endptr) { - while (*endptr != '\0' && (*endptr == ' ' || *endptr == '\t')) - ++endptr; - - switch (*endptr) { - case 'K': - case 'k': - s *= ONE_KB; - break; - - case 'M': - case 'm': - s *= ONE_MB; - break; - - case 'G': - case 'g': - s *= ONE_GB; - break; - - default: - HDfprintf(stderr, "Illegal size specifier '%c'\n", *endptr); - HDexit(EXIT_FAILURE); - } - } - - return s; -} - -/* - * Function: usage - * Purpose: Print a usage message and then exit. - * Return: Nothing - * Programmer: Bill Wendling, 31. October 2001 - * Modifications: - */ -static void -usage(const char *prog) -{ - print_version(prog); - HDprintf("usage: %s [OPTIONS]\n", prog); - HDprintf(" OPTIONS\n"); - HDprintf(" -h Print an usage message and exit\n"); - HDprintf(" -A AL Which APIs to test\n"); - HDprintf(" [default: all of them]\n"); - HDprintf(" -c SL Selects chunked storage and defines chunks dimensions\n"); - HDprintf(" and sizes\n"); - HDprintf(" [default: Off]\n"); - HDprintf(" -e SL Dimensions and sizes of dataset\n"); - HDprintf(" [default: 100,200]\n"); - HDprintf(" -i N Number of iterations to perform\n"); - HDprintf(" [default: 1]\n"); - HDprintf(" -r NL Dimension access order (see below for description)\n"); - HDprintf(" [default: 1,2]\n"); - HDprintf(" -t Selects extendable dimensions for HDF5 dataset\n"); - HDprintf(" [default: Off]\n"); - HDprintf(" -v VFD Selects file driver for HDF5 access\n"); - HDprintf(" [default: sec2]\n"); - HDprintf(" -w Perform write tests, not the read tests\n"); - HDprintf(" [default: Off]\n"); - HDprintf(" -x SL Dimensions and sizes of the transfer buffer\n"); - HDprintf(" [default: 10,20]\n"); - HDprintf("\n"); - HDprintf(" N - is an integer > 0.\n"); - HDprintf("\n"); - HDprintf(" S - is a size specifier, an integer > 0 followed by a size indicator:\n"); - HDprintf(" K - Kilobyte (%d)\n", ONE_KB); - HDprintf(" M - Megabyte (%d)\n", ONE_MB); - HDprintf(" G - Gigabyte (%d)\n", ONE_GB); - HDprintf("\n"); - HDprintf(" Example: '37M' is 37 megabytes or %d bytes\n", 37 * ONE_MB); - HDprintf("\n"); - HDprintf(" AL - is an API list. Valid values are:\n"); - HDprintf(" hdf5 - HDF5\n"); - HDprintf(" posix - POSIX\n"); - HDprintf("\n"); - HDprintf(" Example: -A posix,hdf5\n"); - HDprintf("\n"); - HDprintf(" NL - is list of integers (N) separated by commas.\n"); - HDprintf("\n"); - HDprintf(" Example: 1,2,3\n"); - HDprintf("\n"); - HDprintf(" SL - is list of size specifiers (S) separated by commas.\n"); - HDprintf("\n"); - HDprintf(" Example: 2K,2K,3K\n"); - HDprintf("\n"); - HDprintf(" The example defines an object (dataset, tranfer buffer) with three\n"); - HDprintf(" dimensions. Be aware that as the number of dimensions increases, the\n"); - HDprintf(" the total size of the object increases exponentially.\n"); - HDprintf("\n"); - HDprintf(" VFD - is an HDF5 file driver specifier. Valid values are:\n"); - HDprintf(" sec2, stdio, core, split, multi, family, direct\n"); - HDprintf("\n"); - HDprintf(" Dimension access order:\n"); - HDprintf(" Data access starts at the cardinal origin of the dataset using the\n"); - HDprintf(" transfer buffer. The next access occurs on a dataset region next to\n"); - HDprintf(" the previous one. For a multidimensional dataset, there are several\n"); - HDprintf(" directions as to where to proceed. This can be specified in the dimension\n"); - HDprintf(" access order. For example, -r 1,2 states that the tool should traverse\n"); - HDprintf(" dimension 1 first, and then dimension 2.\n"); - HDprintf("\n"); - HDprintf(" Environment variables:\n"); - HDprintf(" HDF5_NOCLEANUP Do not remove data files if set [default remove]\n"); - HDprintf(" HDF5_PREFIX Data file prefix\n"); - HDprintf("\n"); - HDfflush(stdout); -} /* end usage() */ diff --git a/tools/test/perform/sio_perf.h b/tools/test/perform/sio_perf.h deleted file mode 100644 index 6242782..0000000 --- a/tools/test/perform/sio_perf.h +++ /dev/null @@ -1,104 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef SIO_PERF_H -#define SIO_PERF_H - -#ifndef STANDALONE -#include "io_timer.h" -#include "h5test.h" -#include "h5tools.h" -#include "h5tools_utils.h" -#else -#include "io_timer.h" -#include "sio_standalone.h" -#endif - -/* setup the dataset no fill option if this is v1.5 or more */ -#if H5_VERS_MAJOR > 1 || H5_VERS_MINOR > 4 -#define H5_HAVE_NOFILL 1 -#endif - -#define MAX_DIMS 32 - -typedef enum iotype_ { - POSIXIO, - HDF5 - /*NUM_TYPES*/ -} iotype; - -typedef enum vfdtype_ { - sec2, - stdio, - core, - split, - multi, - family, - direct - /*NUM_TYPES*/ -} vfdtype; - -typedef struct parameters_ { - iotype io_type; /* The type of IO test to perform */ - vfdtype vfd; - long num_files; /* Number of files to create */ - long num_dsets; /* Number of datasets to create */ - hsize_t num_bytes; /* Number of bytes in each dset */ - int num_iters; /* Number of times to loop doing the IO */ - int rank; /* Rank of dataset */ - hsize_t dset_size[MAX_DIMS]; /* Dataset size */ - size_t buf_size[MAX_DIMS]; /* Buffer size */ - size_t chk_size[MAX_DIMS]; /* Chunk size */ - int order[MAX_DIMS]; /* Buffer size */ - hsize_t h5_align; /* HDF5 object alignment */ - hsize_t h5_thresh; /* HDF5 object alignment threshold */ - int h5_use_chunks; /* Make HDF5 dataset chunked */ - int h5_extendable; /* Make HDF5 dataset chunked */ - int h5_write_only; /* Perform the write tests only */ - int verify; /* Verify data correctness */ - size_t page_size; - size_t page_buffer_size; -} parameters; - -typedef struct results_ { - herr_t ret_code; - io_time_t *timers; -} results; - -#ifndef SUCCESS -#define SUCCESS 0 -#endif /* !SUCCESS */ - -#ifndef FAIL -#define FAIL -1 -#endif /* !FAIL */ - -extern FILE * output; /* output file */ -extern io_time_t *timer_g; /* timer: global for stub functions */ -extern int sio_debug_level; /* The debug level: - * 0 - Off - * 1 - Minimal - * 2 - Some more - * 3 - Maximal - * 4 - Even More Debugging (timer stuff) - */ -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -extern void do_sio(parameters param, results *res); - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* SIO_PERF_H */ diff --git a/tools/test/perform/sio_standalone.h b/tools/test/perform/sio_standalone.h index 3b699dc..99cca75 100644 --- a/tools/test/perform/sio_standalone.h +++ b/tools/test/perform/sio_standalone.h @@ -456,17 +456,6 @@ extern char * strdup(const char *s); #define TRUE true #endif -/** From h5test.h **/ - -#ifdef H5_HAVE_PARALLEL -extern MPI_Info h5_io_info_g; /* MPI INFO object for IO */ -#endif - -#ifdef H5_HAVE_PARALLEL -int h5_set_info_object(void); -void h5_dump_info_object(MPI_Info info); -#endif - /** From h5tools_utils.h **/ H5_DLLVAR int H5_opterr; /* getoption prints errors if this is on */ -- cgit v0.12 From baefe444942452a17f982081ca733f010557073c Mon Sep 17 00:00:00 2001 From: jhendersonHDF Date: Wed, 11 Aug 2021 07:31:49 -0500 Subject: Fix link issue with perf/h5perf in parallel library (#889) --- tools/src/h5perf/CMakeLists.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tools/src/h5perf/CMakeLists.txt b/tools/src/h5perf/CMakeLists.txt index 644a5ad..36b0b2f 100644 --- a/tools/src/h5perf/CMakeLists.txt +++ b/tools/src/h5perf/CMakeLists.txt @@ -40,10 +40,10 @@ if (H5_HAVE_PARALLEL) target_include_directories (perf PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT ONLY_SHARED_LIBS) TARGET_C_PROPERTIES (perf STATIC) - target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) + target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} "$<$:${MPI_C_LIBRARIES}>") else () TARGET_C_PROPERTIES (perf SHARED) - target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) + target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} "$<$:${MPI_C_LIBRARIES}>") endif () set_target_properties (perf PROPERTIES FOLDER perform) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};perf") @@ -67,10 +67,10 @@ if (H5_HAVE_PARALLEL) target_include_directories (h5perf PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT ONLY_SHARED_LIBS) TARGET_C_PROPERTIES (h5perf STATIC) - target_link_libraries (h5perf PRIVATE ${LINK_LIBS} ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) + target_link_libraries (h5perf PRIVATE ${LINK_LIBS} ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} "$<$:${MPI_C_LIBRARIES}>") else () TARGET_C_PROPERTIES (h5perf SHARED) - target_link_libraries (h5perf PRIVATE ${LINK_LIBS} ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) + target_link_libraries (h5perf PRIVATE ${LINK_LIBS} ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} "$<$:${MPI_C_LIBRARIES}>") endif () set_target_properties (h5perf PROPERTIES FOLDER perform) set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5perf") -- cgit v0.12 From 7c918e685fea4d58b632389999f092b1f4b33d17 Mon Sep 17 00:00:00 2001 From: David Wells Date: Thu, 12 Aug 2021 08:12:21 -0400 Subject: Guard the SKIP_MPICXX macros against redefinition. (#892) --- src/H5public.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/H5public.h b/src/H5public.h index e192de0..dbccbf8 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -56,8 +56,12 @@ #ifdef H5_HAVE_PARALLEL /* Don't link against MPI C++ bindings */ +#ifndef MPICH_SKIP_MPICXX #define MPICH_SKIP_MPICXX 1 +#endif +#ifndef OMPI_SKIP_MPICXX #define OMPI_SKIP_MPICXX 1 +#endif #include #ifndef MPI_FILE_NULL /* MPIO may be defined in mpi.h already */ #include -- cgit v0.12 From b5c66529e9709839f336d2b6f0d453139a0744b5 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Thu, 12 Aug 2021 05:21:50 -0700 Subject: Fixes a bad memory read and unfreed memory in fsinfo code (#893) * Fixes a bad memory read and unfreed memory in fsinfo code The segfaul from CVE-2020-10810 was fixed some time ago, but the illegal memory read and unfreed memory were not. This fix tracks some buffer sizes and errors out gracefully on errors, ensuring buffers are cleaned up and avoiding the H5FL infinite loop + abort on library close. * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- MANIFEST | 1 + release_docs/RELEASE.txt | 18 +++++++++++++++ src/H5Ocache.c | 21 ++++++++++-------- src/H5Ofsinfo.c | 15 ++++++++----- test/cve_2020_10810.h5 | Bin 0 -> 1808 bytes test/ohdr.c | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+), 14 deletions(-) create mode 100644 test/cve_2020_10810.h5 diff --git a/MANIFEST b/MANIFEST index 668ba0a..ed28027 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1168,6 +1168,7 @@ ./test/cork.c ./test/corrupt_stab_msg.h5 ./test/cross_read.c +./test/cve_2020_10810.h5 ./test/dangle.c ./test/deflate.h5 ./test/del_many_dense_attrs.c diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 40b9175..396629c 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -924,6 +924,24 @@ Bug Fixes since HDF5-1.12.0 release =================================== Library ------- + - Fixed an invalid read and memory leak when parsing corrupt file space + info messages + + When the corrupt file from CVE-2020-10810 was parsed by the library, + the code that imports the version 0 file space info object header + message to the version 1 struct could read past the buffer read from + the disk, causing an invalid memory read. Not catching this error would + cause downstream errors that eventually resulted in a previously + allocated buffer to be unfreed when the library shut down. In builds + where the free lists are in use, this could result in an infinite loop + and SIGABRT when the library shuts down. + + We now track the buffer size and raise an error on attempts to read + past the end of it. + + (DER - 2021/08/12, HDFFV-11053) + + - Fixed CVE-2018-14460 The tool h5repack produced a segfault when the rank in dataspace diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 62dc2f2..3aae0cb 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -78,8 +78,8 @@ static herr_t H5O__cache_chk_free_icr(void *thing); static herr_t H5O__prefix_deserialize(const uint8_t *image, H5O_cache_ud_t *udata); /* Chunk routines */ -static herr_t H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image, - H5O_common_cache_ud_t *udata, hbool_t *dirty); +static herr_t H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t *image, + size_t len, H5O_common_cache_ud_t *udata, hbool_t *dirty); static herr_t H5O__chunk_serialize(const H5F_t *f, H5O_t *oh, unsigned chunkno); /* Misc. routines */ @@ -287,7 +287,7 @@ H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) *------------------------------------------------------------------------- */ static void * -H5O__cache_deserialize(const void *image, size_t H5_ATTR_NDEBUG_UNUSED len, void *_udata, hbool_t *dirty) +H5O__cache_deserialize(const void *image, size_t len, void *_udata, hbool_t *dirty) { H5O_t * oh = NULL; /* Object header read in */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ @@ -333,7 +333,7 @@ H5O__cache_deserialize(const void *image, size_t H5_ATTR_NDEBUG_UNUSED len, void oh->proxy = NULL; /* Parse the first chunk */ - if (H5O__chunk_deserialize(oh, udata->common.addr, udata->chunk0_size, (const uint8_t *)image, + if (H5O__chunk_deserialize(oh, udata->common.addr, udata->chunk0_size, (const uint8_t *)image, len, &(udata->common), dirty) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk") @@ -736,7 +736,7 @@ H5O__cache_chk_verify_chksum(const void *_image, size_t len, void *_udata) *------------------------------------------------------------------------- */ static void * -H5O__cache_chk_deserialize(const void *image, size_t H5_ATTR_NDEBUG_UNUSED len, void *_udata, hbool_t *dirty) +H5O__cache_chk_deserialize(const void *image, size_t len, void *_udata, hbool_t *dirty) { H5O_chunk_proxy_t * chk_proxy = NULL; /* Chunk proxy object */ H5O_chk_cache_ud_t *udata = (H5O_chk_cache_ud_t *)_udata; /* User data for callback */ @@ -763,7 +763,7 @@ H5O__cache_chk_deserialize(const void *image, size_t H5_ATTR_NDEBUG_UNUSED len, HDassert(udata->common.cont_msg_info); /* Parse the chunk */ - if (H5O__chunk_deserialize(udata->oh, udata->common.addr, udata->size, (const uint8_t *)image, + if (H5O__chunk_deserialize(udata->oh, udata->common.addr, udata->size, (const uint8_t *)image, len, &(udata->common), dirty) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize object header chunk") @@ -1275,7 +1275,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image, +H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t chunk_size, const uint8_t *image, size_t len, H5O_common_cache_ud_t *udata, hbool_t *dirty) { const uint8_t *chunk_image; /* Pointer into buffer to decode */ @@ -1295,6 +1295,7 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image HDassert(oh); HDassert(H5F_addr_defined(addr)); HDassert(image); + HDassert(len); HDassert(udata->f); HDassert(udata->cont_msg_info); @@ -1315,14 +1316,16 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image oh->chunk[chunkno].addr = addr; if (chunkno == 0) /* First chunk's 'image' includes room for the object header prefix */ - oh->chunk[0].size = len + (size_t)H5O_SIZEOF_HDR(oh); + oh->chunk[0].size = chunk_size + (size_t)H5O_SIZEOF_HDR(oh); else - oh->chunk[chunkno].size = len; + oh->chunk[chunkno].size = chunk_size; if (NULL == (oh->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, oh->chunk[chunkno].size))) HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") oh->chunk[chunkno].chunk_proxy = NULL; /* Copy disk image into chunk's image */ + if (len < oh->chunk[chunkno].size) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "attempted to copy too many disk image bytes into buffer") H5MM_memcpy(oh->chunk[chunkno].image, image, oh->chunk[chunkno].size); /* Point into chunk image to decode */ diff --git a/src/H5Ofsinfo.c b/src/H5Ofsinfo.c index 44c4985..b60f589 100644 --- a/src/H5Ofsinfo.c +++ b/src/H5Ofsinfo.c @@ -91,11 +91,12 @@ H5FL_DEFINE_STATIC(H5O_fsinfo_t); */ static void * H5O__fsinfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNUSED mesg_flags, - unsigned H5_ATTR_UNUSED *ioflags, size_t H5_ATTR_UNUSED p_size, const uint8_t *p) + unsigned H5_ATTR_UNUSED *ioflags, size_t p_size, const uint8_t *p) { - H5O_fsinfo_t * fsinfo = NULL; /* File space info message */ - H5F_mem_page_t ptype; /* Memory type for iteration */ - unsigned vers; /* message version */ + H5O_fsinfo_t * fsinfo = NULL; /* File space info message */ + H5F_mem_page_t ptype; /* Memory type for iteration */ + unsigned vers; /* message version */ + const uint8_t *p_end = p + p_size; void * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -136,8 +137,12 @@ H5O__fsinfo_decode(H5F_t *f, H5O_t H5_ATTR_UNUSED *open_oh, unsigned H5_ATTR_UNU fsinfo->threshold = threshold; if (HADDR_UNDEF == (fsinfo->eoa_pre_fsm_fsalloc = H5F_get_eoa(f, H5FD_MEM_DEFAULT))) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "unable to get file size") - for (type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; type++) + for (type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; type++) { + if (p + H5_SIZEOF_HADDR_T > p_end) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, + "ran off end of input buffer while decoding") H5F_addr_decode(f, &p, &(fsinfo->fs_addr[type - 1])); + } break; case H5F_FILE_SPACE_ALL: diff --git a/test/cve_2020_10810.h5 b/test/cve_2020_10810.h5 new file mode 100644 index 0000000..5cface3 Binary files /dev/null and b/test/cve_2020_10810.h5 differ diff --git a/test/ohdr.c b/test/ohdr.c index b7af77f..afcea87 100644 --- a/test/ohdr.c +++ b/test/ohdr.c @@ -457,6 +457,59 @@ error: } /* test_ohdr_swmr() */ /* + * Tests bad object header messages. + * + * Currently tests for CVE-2020-10810 fixes but can be expanded to handle + * other CVE badness. + */ + +/* This is a generated file that can be obtained from: + * + * https://nvd.nist.gov/vuln/detail/CVE-2020-10810 + * + * It was formerly named H5AC_unpin_entry_POC + */ +#define CVE_2020_10810_FILENAME "cve_2020_10810.h5" + +static herr_t +test_ohdr_badness(hid_t fapl) +{ + hid_t fid = H5I_INVALID_HID; + + /* CVE-2020-10810 involved a malformed fsinfo message + * This test ensures the fundamental problem is fixed. Running it under + * valgrind et al. will ensure that the memory leaks and invalid access + * are fixed. + */ + TESTING("Fix for CVE-2020-10810"); + + H5E_BEGIN_TRY + { + /* This should fail due to the malformed fsinfo message. It should + * fail gracefully and not segfault. + */ + fid = H5Fopen(CVE_2020_10810_FILENAME, H5F_ACC_RDWR, fapl); + } + H5E_END_TRY; + + if (fid >= 0) + FAIL_PUTS_ERROR("should not have been able to open malformed file"); + + PASSED(); + + return SUCCEED; + +error: + H5E_BEGIN_TRY + { + H5Fclose(fid); + } + H5E_END_TRY; + + return FAIL; +} + +/* * To test objects with unknown messages in a file with: * a) H5O_BOGUS_VALID_ID: * --the bogus_id is within the range of H5O_msg_class_g[] @@ -2047,6 +2100,9 @@ main(void) } /* high */ } /* low */ + /* Verify bad ohdr message fixes work */ + test_ohdr_badness(fapl); + /* Verify symbol table messages are cached */ if (h5_verify_cached_stabs(FILENAME, fapl) < 0) TEST_ERROR -- cgit v0.12 From c0f1db4b96d43cd2f7bcd0134e107061e866d6e3 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Thu, 12 Aug 2021 07:50:16 -0500 Subject: Update examples version used by CMake (#882) --- bin/release | 6 +++--- config/cmake/HDF5_Examples.cmake.in | 2 +- config/cmake/HDF5_Examples_options.cmake | 4 +++- config/cmake/scripts/HDF5options.cmake | 2 +- release_docs/README_HDF5_CMake | 2 +- 5 files changed, 9 insertions(+), 7 deletions(-) diff --git a/bin/release b/bin/release index 16e6bd2..0726b8c 100755 --- a/bin/release +++ b/bin/release @@ -235,7 +235,7 @@ tar2cmakezip() # step 3: add LIBAEC.tar.gz, ZLib.tar.gz and cmake files cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmziptmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmziptmpsubdir - cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.2-Source.zip $cmziptmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.3-Source.zip $cmziptmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-master.zip $cmziptmpsubdir cp $cmziptmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmziptmpsubdir cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmziptmpsubdir @@ -331,7 +331,7 @@ tar2cmaketgz() # step 3: add LIBAEC.tar.gz, ZLib.tar.gz and cmake files cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir - cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.2-Source.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.3-Source.tar.gz $cmgztmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-master.tar.gz $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir @@ -415,7 +415,7 @@ tar2hpccmaketgz() # step 3: add LIBAEC.tar.gz, ZLib.tar.gz and cmake files cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir - cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.2-Source.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.3-Source.tar.gz $cmgztmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/hdf5_plugins-master.tar.gz $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir diff --git a/config/cmake/HDF5_Examples.cmake.in b/config/cmake/HDF5_Examples.cmake.in index 5911005..d77e2df 100644 --- a/config/cmake/HDF5_Examples.cmake.in +++ b/config/cmake/HDF5_Examples.cmake.in @@ -77,7 +77,7 @@ set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DSITE:STRING=${CTEST_SITE} -DBUILDN #TAR_SOURCE - name of tarfile #if(NOT DEFINED TAR_SOURCE) -# set(CTEST_USE_TAR_SOURCE "HDF5Examples-1.14.2-Source") +# set(CTEST_USE_TAR_SOURCE "HDF5Examples-1.14.3-Source") #endif() ############################################################################################################### diff --git a/config/cmake/HDF5_Examples_options.cmake b/config/cmake/HDF5_Examples_options.cmake index 236a814..cdd49eb 100644 --- a/config/cmake/HDF5_Examples_options.cmake +++ b/config/cmake/HDF5_Examples_options.cmake @@ -52,12 +52,14 @@ ### enable parallel program builds #set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_ENABLE_PARALLEL:BOOL=ON") +############################################################################################# +### match the hdf5 library namespace +set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_NAMESPACE:STRING=hdf5::") ############################################################################################# ### enable threadsafe program builds #set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF_ENABLE_THREADSAFE:BOOL=ON") - ############################################################################################# ### enable test program builds, requires reference files in testfiles subdirectory #set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DBUILD_TESTING:BOOL=ON") diff --git a/config/cmake/scripts/HDF5options.cmake b/config/cmake/scripts/HDF5options.cmake index e3c4c1a..8a152d1 100644 --- a/config/cmake/scripts/HDF5options.cmake +++ b/config/cmake/scripts/HDF5options.cmake @@ -69,7 +69,7 @@ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRIN #set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_SZIP_ENCODING:BOOL=OFF") #### package examples #### -#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACK_EXAMPLES:BOOL=ON -DHDF5_EXAMPLES_COMPRESSED:STRING=HDF5Examples-1.14.2-Source.tar.gz -DHDF5_EXAMPLES_COMPRESSED_DIR:PATH=${CTEST_SCRIPT_DIRECTORY}") +#set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_PACK_EXAMPLES:BOOL=ON -DHDF5_EXAMPLES_COMPRESSED:STRING=HDF5Examples-1.14.3-Source.tar.gz -DHDF5_EXAMPLES_COMPRESSED_DIR:PATH=${CTEST_SCRIPT_DIRECTORY}") ############################################################################################# ### enable parallel builds diff --git a/release_docs/README_HDF5_CMake b/release_docs/README_HDF5_CMake index 1242695..c63eaca 100644 --- a/release_docs/README_HDF5_CMake +++ b/release_docs/README_HDF5_CMake @@ -10,7 +10,7 @@ This tar file contains LIBAEC.tar.gz source for building SZIP replacement ZLib.tar.gz source for building ZLIB hdf5_plugins.tar.gz source for building compression plugins - HDF5Examples-1.14.2-Source.tar.gz source for building examples + HDF5Examples-1.14.3-Source.tar.gz source for building examples For more information about building HDF5 with CMake, see USING_HDF5_CMake.txt in hdf5-1.13.0/release_docs, or -- cgit v0.12 From 6968906418f9ecfc383ad960c975e65a504743b7 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Mon, 16 Aug 2021 08:05:25 -0500 Subject: Fix clang/sanitizer for windows (#883) * Fix clang/sanitizer for windows * Restrict MSVC version for sanitize --- config/sanitizer/sanitizers.cmake | 10 +++++----- config/toolchain/clang.cmake | 9 +++++++-- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/config/sanitizer/sanitizers.cmake b/config/sanitizer/sanitizers.cmake index 0803279..22b9e3c 100644 --- a/config/sanitizer/sanitizers.cmake +++ b/config/sanitizer/sanitizers.cmake @@ -33,10 +33,10 @@ if(USE_SANITIZER) if(INTEL_CLANG OR "${CMAKE_C_COMPILER_ID}" MATCHES "Clang") set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - append("-fno-omit-frame-pointer" CMAKE_C_SANITIZER_FLAGS CMAKE_CXX_SANITIZER_FLAGS) - message(STATUS "Building with sanitize, base flags=${CMAKE_C_SANITIZER_FLAGS}") - if(UNIX) + append("-fno-omit-frame-pointer" CMAKE_C_SANITIZER_FLAGS CMAKE_CXX_SANITIZER_FLAGS) + message(STATUS "Building with sanitize, base flags=${CMAKE_C_SANITIZER_FLAGS}") + if(uppercase_CMAKE_BUILD_TYPE STREQUAL "DEBUG") append("-O1" CMAKE_C_SANITIZER_FLAGS CMAKE_CXX_SANITIZER_FLAGS) endif() @@ -80,10 +80,10 @@ if(USE_SANITIZER) message( FATAL_ERROR "Unsupported value of USE_SANITIZER: ${USE_SANITIZER}") endif() - elseif(MSVC) + elseif(MSVC AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 19.29) if(USE_SANITIZER MATCHES "([Aa]ddress)") message(STATUS "Building with Address sanitizer") - append("-fsanitize=address" CMAKE_C_SANITIZER_FLAGS CMAKE_CXX_SANITIZER_FLAGS) + append("/fsanitize=address" CMAKE_C_SANITIZER_FLAGS CMAKE_CXX_SANITIZER_FLAGS) else() message(FATAL_ERROR "This sanitizer not yet supported in the MSVC environment: ${USE_SANITIZER}") endif() diff --git a/config/toolchain/clang.cmake b/config/toolchain/clang.cmake index 5e3b13a..af176aa 100644 --- a/config/toolchain/clang.cmake +++ b/config/toolchain/clang.cmake @@ -3,8 +3,13 @@ set(CMAKE_COMPILER_VENDOR "clang") -set(CMAKE_C_COMPILER clang) -set(CMAKE_CXX_COMPILER clang++) +if(WIN32) + set(CMAKE_C_COMPILER clang-cl) + set(CMAKE_CXX_COMPILER clang-cl) +else() + set(CMAKE_C_COMPILER clang) + set(CMAKE_CXX_COMPILER clang++) +endif() set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # the following is used if cross-compiling -- cgit v0.12 From 451b472cdd63c8fef15c0866e0d3784e33a8b1c6 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Mon, 16 Aug 2021 08:06:59 -0500 Subject: Fixed the CMake processing for C++ warnings and flags (#888) --- MANIFEST | 3 +- config/cmake/HDFCXXCompilerFlags.cmake | 298 +++++++++++++++------------ config/cmake/HDFCompilerFlags.cmake | 303 ++++++++++++++++------------ config/intel-warnings/18 | 6 - config/intel-warnings/general-19 | 2 - config/intel-warnings/win-developer-general | 1 + config/intel-warnings/win-general | 1 + release_docs/RELEASE.txt | 9 + 8 files changed, 348 insertions(+), 275 deletions(-) delete mode 100644 config/intel-warnings/general-19 create mode 100644 config/intel-warnings/win-developer-general create mode 100644 config/intel-warnings/win-general diff --git a/MANIFEST b/MANIFEST index ed28027..a0f58ce 100644 --- a/MANIFEST +++ b/MANIFEST @@ -202,7 +202,8 @@ ./config/intel-warnings/18 ./config/intel-warnings/developer-general ./config/intel-warnings/general -./config/intel-warnings/general-19 +./config/intel-warnings/win-developer-general +./config/intel-warnings/win-general ./config/intel-warnings/ifort-general ./config/site-specific/BlankForm diff --git a/config/cmake/HDFCXXCompilerFlags.cmake b/config/cmake/HDFCXXCompilerFlags.cmake index bd14a0d..dc982c1 100644 --- a/config/cmake/HDFCXXCompilerFlags.cmake +++ b/config/cmake/HDFCXXCompilerFlags.cmake @@ -9,17 +9,45 @@ # If you do not have access to either file, you may request a copy from # help@hdfgroup.org. # +ENABLE_LANGUAGE (CXX) + set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED TRUE) + set(CMAKE_CXX_EXTENSIONS OFF) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_SANITIZER_FLAGS} ${CMAKE_CXX_FLAGS}") if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") - message (VERBOSE "Warnings Configuration: CXX default: ${CMAKE_CXX_FLAGS}") + message (VERBOSE "Warnings Configuration: CXX default: ${CMAKE_CXX_FLAGS}") endif () #----------------------------------------------------------------------------- # Compiler specific flags : Shouldn't there be compiler tests for these #----------------------------------------------------------------------------- +if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set(_INTEL_WINDOWS 1) +endif() + +if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" + AND "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") + set(_CLANG_MSVC_WINDOWS 1) +endif() + +# MSVC 14.28 enables C5105, but the Windows SDK 10.0.18362.0 triggers it. +if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.28) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd5105") +endif() + +if (CMAKE_CXX_COMPILER_ID STREQUAL SunPro AND + NOT DEFINED CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION) + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.13) + if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD EQUAL 98) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++03") + endif() + else() + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -library=stlport4") + endif() +endif() + if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED) set (CMAKE_CXX_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_CXX_FLAGS}") if (${HDF_CFG_NAME} MATCHES "Debug") @@ -71,163 +99,167 @@ endif () # HDF5 library compile options #----------------------------------------------------------------------------- -if (NOT MSVC AND NOT MINGW) - if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS") - list (APPEND HDF5_CMAKE_CXX_FLAGS "-erroff=%none -DBSD_COMP") - else () - # General flags - # - # Note that some of the flags listed here really should be developer - # flags (listed in a separate variable, below) but we put them here - # because they are not raised by the current code and we'd like to - # know if they do start showing up. - # - # NOTE: Don't add -Wpadded here since we can't/won't fix the (many) - # warnings that are emitted. If you need it, add it at configure time. - if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") +if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS") + list (APPEND HDF5_CMAKE_CXX_FLAGS "-erroff=%none -DBSD_COMP") +else () + # General flags + # + # Note that some of the flags listed here really should be developer + # flags (listed in a separate variable, below) but we put them here + # because they are not raised by the current code and we'd like to + # know if they do start showing up. + # + # NOTE: Don't add -Wpadded here since we can't/won't fix the (many) + # warnings that are emitted. If you need it, add it at configure time. + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + if (_INTEL_WINDOWS) + ADD_H5_FLAGS (HDF5_CMAKE_CXX_FLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/win-general") + else () ADD_H5_FLAGS (HDF5_CMAKE_CXX_FLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/general") + endif() + if (NOT _INTEL_WINDOWS) if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 15.0) ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/15") endif() if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 18.0) ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/18") endif() - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED - AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) - # add the general CXX flags for g++ compiler versions 4.8 and above. - ADD_H5_FLAGS (HDF5_CMAKE_CXX_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-general") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-error-general") - else () - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-noerror-general") - endif () + endif() + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED + AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) + # add the general CXX flags for g++ compiler versions 4.8 and above. + ADD_H5_FLAGS (HDF5_CMAKE_CXX_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-general") + if (HDF5_ENABLE_WARNINGS_AS_ERRORS) + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-error-general") + else () + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-noerror-general") endif () - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - ADD_H5_FLAGS (HDF5_CMAKE_CXX_FLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/general") - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "PGI") - list (APPEND HDF5_CMAKE_CXX_FLAGS "-Minform=inform") - endif () - if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") - message (VERBOSE "CMAKE_CXX_FLAGS_GENERAL=${HDF5_CMAKE_CXX_FLAGS}") endif () + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + ADD_H5_FLAGS (HDF5_CMAKE_CXX_FLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/general") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "PGI") + list (APPEND HDF5_CMAKE_CXX_FLAGS "-Minform=inform") endif () - - #----------------------------------------------------------------------------- - # Option to allow the user to enable developer warnings - # Developer warnings (suggestions from gcc, not code problems) - #----------------------------------------------------------------------------- - if (HDF5_ENABLE_DEV_WARNINGS) - message (STATUS "....HDF5 developer group warnings are enabled") - if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/developer-general") - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-general") - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/developer-general") - endif () - else () - if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-general") - elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/no-developer-general") - endif () + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (VERBOSE "CMAKE_CXX_FLAGS_GENERAL=${HDF5_CMAKE_CXX_FLAGS}") endif () +endif () +#----------------------------------------------------------------------------- +# Option to allow the user to enable developer warnings +# Developer warnings (suggestions from gcc, not code problems) +#----------------------------------------------------------------------------- +if (HDF5_ENABLE_DEV_WARNINGS) + message (STATUS "....HDF5 developer group warnings are enabled") + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/developer-general") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-general") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/developer-general") + endif () +else () if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") - # Technically, variable-length arrays are part of the C99 standard, but - # we should approach them a bit cautiously... Only needed for gcc 4.X - if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0 AND CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/4.8-4.last") - endif () + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-general") + elseif (CMAKE_CXX_COMPILER_ID STREQUAL "Clang") + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/no-developer-general") + endif () +endif () - # Append more extra warning flags that only gcc 4.8+ knows about - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-4.8") - if (HDF5_ENABLE_DEV_WARNINGS) - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-4.8") - else () - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-4.8") - endif () - endif () +if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") + # Technically, variable-length arrays are part of the C99 standard, but + # we should approach them a bit cautiously... Only needed for gcc 4.X + if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0 AND CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/4.8-4.last") + endif () - # Append more extra warning flags that only gcc 4.9+ knows about - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) + # Append more extra warning flags that only gcc 4.8+ knows about + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.8) + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-4.8") + if (HDF5_ENABLE_DEV_WARNINGS) # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-4.9") + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-4.8") + else () + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-4.8") endif () + endif () - # Append more extra warning flags that only gcc 5.1+ knows about - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) - # autotools always add the C flags with the CXX flags - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-5") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-error-5") - else () - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-noerror-5") - endif () - endif () + # Append more extra warning flags that only gcc 4.9+ knows about + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 4.9) + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-4.9") + endif () - # Append more extra warning flags that only gcc 6.x+ knows about - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0) - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/6") + # Append more extra warning flags that only gcc 5.1+ knows about + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) + # autotools always add the C flags with the CXX flags + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-5") + if (HDF5_ENABLE_WARNINGS_AS_ERRORS) + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-error-5") + else () + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-noerror-5") endif () + endif () - # Append more extra warning flags that only gcc 7.x+ knows about - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0) - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXxFLAGS2 "${HDF5_SOURCE_DIR}/config/gnu-warnings/7") - if (HDF5_ENABLE_DEV_WARNINGS) - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-7") - #else () - # ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-7") - endif () - endif () + # Append more extra warning flags that only gcc 6.x+ knows about + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 6.0) + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/6") + endif () - # Append more extra warning flags that only gcc 8.x+ knows about - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0) - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/8") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-8") - else () - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/noerror-8") - endif () - if (HDF5_ENABLE_DEV_WARNINGS) - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-8") - else () - # autotools always add the C flags with the CXX flags - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-8") - endif () + # Append more extra warning flags that only gcc 7.x+ knows about + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 7.0) + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXxFLAGS2 "${HDF5_SOURCE_DIR}/config/gnu-warnings/7") + if (HDF5_ENABLE_DEV_WARNINGS) + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-7") + #else () + # ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-7") endif () + endif () - # Append more extra warning flags that only gcc 9.x+ knows about - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) + # Append more extra warning flags that only gcc 8.x+ knows about + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 8.0) + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/8") + if (HDF5_ENABLE_WARNINGS_AS_ERRORS) + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-8") + else () + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/noerror-8") + endif () + if (HDF5_ENABLE_DEV_WARNINGS) # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-9") + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-8") + else () + # autotools always add the C flags with the CXX flags + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-8") endif () + endif () - # Append more extra warning flags that only gcc 9.3+ knows about - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.3) - # do not use C warnings, gnu-warnings 9.3, no cxx warniings - # ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/9.3") - endif () + # Append more extra warning flags that only gcc 9.x+ knows about + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.0) + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/cxx-9") + endif () - # Append more extra warning flags that only gcc 10.x+ knows about - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10.0) - if (HDF5_ENABLE_DEV_WARNINGS) - # Use the C warnings as CXX warnings are the same - ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-10") - #else () - # ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-10") - endif () + # Append more extra warning flags that only gcc 9.3+ knows about + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 9.3) + # do not use C warnings, gnu-warnings 9.3, no cxx warniings + # ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/9.3") + endif () + + # Append more extra warning flags that only gcc 10.x+ knows about + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 10.0) + if (HDF5_ENABLE_DEV_WARNINGS) + # Use the C warnings as CXX warnings are the same + ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-10") + #else () + # ADD_H5_FLAGS (H5_CXXFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-10") endif () endif () endif () @@ -270,7 +302,7 @@ endif () #----------------------------------------------------------------------------- if (HDF5_ENABLE_SYMBOLS MATCHES "YES") if(CMAKE_CXX_COMPILER_LOADED) - if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" AND NOT _INTEL_WINDOWS) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") @@ -278,8 +310,8 @@ if (HDF5_ENABLE_SYMBOLS MATCHES "YES") endif () elseif (HDF5_ENABLE_SYMBOLS MATCHES "NO") if(CMAKE_CXX_COMPILER_LOADED) - if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - set (CMAKE_CXX_FLAGS "${CMAKE_C_FLAGS} -Wl,-s") + if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" AND NOT _INTEL_WINDOWS) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-s") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -s") endif () diff --git a/config/cmake/HDFCompilerFlags.cmake b/config/cmake/HDFCompilerFlags.cmake index e38a92b..de5b563 100644 --- a/config/cmake/HDFCompilerFlags.cmake +++ b/config/cmake/HDFCompilerFlags.cmake @@ -14,14 +14,43 @@ set(CMAKE_C_STANDARD_REQUIRED TRUE) set (CMAKE_C_FLAGS "${CMAKE_C99_STANDARD_COMPILE_OPTION} ${CMAKE_C_FLAGS}") set (CMAKE_C_FLAGS "${CMAKE_C_SANITIZER_FLAGS} ${CMAKE_C_FLAGS}") -set (CMAKE_CXX_FLAGS "${CMAKE_CXX_SANITIZER_FLAGS} ${CMAKE_CXX_FLAGS}") - if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") - message (VERBOSE "Warnings Configuration: default: ${CMAKE_C_FLAGS} : ${CMAKE_CXX_FLAGS}") + message (VERBOSE "Warnings Configuration: C default: ${CMAKE_C_FLAGS}") endif () #----------------------------------------------------------------------------- # Compiler specific flags : Shouldn't there be compiler tests for these #----------------------------------------------------------------------------- +if(WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Intel") + set(_INTEL_WINDOWS 1) +endif() + +if(WIN32 AND CMAKE_C_COMPILER_ID STREQUAL "Clang" + AND "x${CMAKE_C_SIMULATE_ID}" STREQUAL "xMSVC") + set(_CLANG_MSVC_WINDOWS 1) +endif() + +# Disable deprecation warnings for standard C functions. +# really only needed for newer versions of VS, but should +# not hurt other versions, and this will work into the +# future +if(MSVC OR _INTEL_WINDOWS OR _CLANG_MSVC_WINDOWS) + add_definitions(-D_CRT_SECURE_NO_DEPRECATE -D_CRT_NONSTDC_NO_DEPRECATE) +else() +endif() + +if(MSVC) + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -stack:10000000") +endif() + +# MSVC 14.28 enables C5105, but the Windows SDK 10.0.18362.0 triggers it. +if(CMAKE_C_COMPILER_ID STREQUAL "MSVC" AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 19.28) + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -wd5105") +endif() + +if(_CLANG_MSVC_WINDOWS AND "x${CMAKE_C_COMPILER_FRONTEND_VARIANT}" STREQUAL "xGNU") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Xlinker -stack:20000000") +endif() + if (CMAKE_COMPILER_IS_GNUCC) set (CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS}") if (${HDF_CFG_NAME} MATCHES "Debug") @@ -81,33 +110,37 @@ endif () # HDF5 library compile options #----------------------------------------------------------------------------- -if (NOT MSVC AND NOT MINGW) - #----------------------------------------------------------------------------- - # Option to allow the user to interpret certain warnings as errors - # - # This should NOT be on by default as it can cause a lot of conflicts with - # new operating systems and compiler versions. Header files that are out of - # our control (MPI, HDFS, etc.) can also raise warnings. - #----------------------------------------------------------------------------- - option (HDF5_ENABLE_WARNINGS_AS_ERRORS "Interpret some warnings as errors" OFF) - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - message (STATUS "...some warnings will be interpreted as errors") - endif () +#----------------------------------------------------------------------------- +# Option to allow the user to interpret certain warnings as errors +# +# This should NOT be on by default as it can cause a lot of conflicts with +# new operating systems and compiler versions. Header files that are out of +# our control (MPI, HDFS, etc.) can also raise warnings. +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_WARNINGS_AS_ERRORS "Interpret some warnings as errors" OFF) +if (HDF5_ENABLE_WARNINGS_AS_ERRORS) + message (STATUS "...some warnings will be interpreted as errors") +endif () - if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS") - list (APPEND HDF5_CMAKE_C_FLAGS "-erroff=%none -DBSD_COMP") - else () - # General flags - # - # Note that some of the flags listed here really should be developer - # flags (listed in a separate variable, below) but we put them here - # because they are not raised by the current code and we'd like to - # know if they do start showing up. - # - # NOTE: Don't add -Wpadded here since we can't/won't fix the (many) - # warnings that are emitted. If you need it, add it at configure time. - if (CMAKE_C_COMPILER_ID STREQUAL "Intel") +if (${CMAKE_SYSTEM_NAME} MATCHES "SunOS") + list (APPEND HDF5_CMAKE_C_FLAGS "-erroff=%none -DBSD_COMP") +else () + # General flags + # + # Note that some of the flags listed here really should be developer + # flags (listed in a separate variable, below) but we put them here + # because they are not raised by the current code and we'd like to + # know if they do start showing up. + # + # NOTE: Don't add -Wpadded here since we can't/won't fix the (many) + # warnings that are emitted. If you need it, add it at configure time. + if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + if (_INTEL_WINDOWS) + ADD_H5_FLAGS (HDF5_CMAKE_C_FLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/win-general") + else () ADD_H5_FLAGS (HDF5_CMAKE_C_FLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/general") + endif() + if (NOT _INTEL_WINDOWS) if(NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 15.0) ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/15") endif() @@ -116,133 +149,137 @@ if (NOT MSVC AND NOT MINGW) if(NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 18.0) ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/18") endif() - elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU") - # Add general CFlags for GCC versions 4.8 and above - if (CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) - ADD_H5_FLAGS (HDF5_CMAKE_C_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/general") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-general") - else () - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/noerror-general") - endif () - endif () - # gcc automatically inlines based on the optimization level - # this is just a failsafe - list (APPEND H5_CFLAGS "-finline-functions") - elseif (CMAKE_C_COMPILER_ID MATCHES "[Cc]lang") - ADD_H5_FLAGS (HDF5_CMAKE_C_FLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/general") + endif() + elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU") + # Add general CFlags for GCC versions 4.8 and above + if (CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) + ADD_H5_FLAGS (HDF5_CMAKE_C_FLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/general") if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/error-general") + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-general") else () - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/noerror-general") + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/noerror-general") endif () - elseif (CMAKE_C_COMPILER_ID STREQUAL "PGI") - list (APPEND HDF5_CMAKE_C_FLAGS "-Minform=inform") endif () - if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") - message (VERBOSE "CMAKE_C_FLAGS_GENERAL=${HDF5_CMAKE_C_FLAGS}") + # gcc automatically inlines based on the optimization level + # this is just a failsafe + list (APPEND H5_CFLAGS "-finline-functions") + elseif (CMAKE_C_COMPILER_ID MATCHES "[Cc]lang") + ADD_H5_FLAGS (HDF5_CMAKE_C_FLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/general") + if (HDF5_ENABLE_WARNINGS_AS_ERRORS) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/error-general") + else () + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/noerror-general") endif () + elseif (CMAKE_C_COMPILER_ID STREQUAL "PGI") + list (APPEND HDF5_CMAKE_C_FLAGS "-Minform=inform") endif () + if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") + message (VERBOSE "CMAKE_C_FLAGS_GENERAL=${HDF5_CMAKE_C_FLAGS}") + endif () +endif () - #----------------------------------------------------------------------------- - # Option to allow the user to enable developer warnings - # Developer warnings (suggestions from gcc, not code problems) - #----------------------------------------------------------------------------- - option (HDF5_ENABLE_DEV_WARNINGS "Enable HDF5 developer group warnings" OFF) - if (HDF5_ENABLE_DEV_WARNINGS) - message (STATUS "....HDF5 developer group warnings are enabled") - if (CMAKE_C_COMPILER_ID STREQUAL "Intel") +#----------------------------------------------------------------------------- +# Option to allow the user to enable developer warnings +# Developer warnings (suggestions from gcc, not code problems) +#----------------------------------------------------------------------------- +option (HDF5_ENABLE_DEV_WARNINGS "Enable HDF5 developer group warnings" OFF) +if (HDF5_ENABLE_DEV_WARNINGS) + message (STATUS "....HDF5 developer group warnings are enabled") + if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + if (_INTEL_WINDOWS) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/win-developer-general") + else () ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/intel-warnings/developer-general") - elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-general") - elseif (CMAKE_C_COMPILER_ID MATCHES "[Cc]lang") - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/developer-general") - endif () - else () - if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-general") - elseif (CMAKE_C_COMPILER_ID MATCHES "[Cc]lang") - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/no-developer-general") endif () + elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-general") + elseif (CMAKE_C_COMPILER_ID MATCHES "[Cc]lang") + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/developer-general") endif () +else () + if (CMAKE_C_COMPILER_ID STREQUAL "GNU" AND CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-general") + elseif (CMAKE_C_COMPILER_ID MATCHES "[Cc]lang") + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/clang-warnings/no-developer-general") + endif () +endif () - if (CMAKE_C_COMPILER_ID STREQUAL "GNU") - # Technically, variable-length arrays are part of the C99 standard, but - # we should approach them a bit cautiously... Only needed for gcc 4.X - if (CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0 AND CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/4.8-4.last") - endif () +if (CMAKE_C_COMPILER_ID STREQUAL "GNU") + # Technically, variable-length arrays are part of the C99 standard, but + # we should approach them a bit cautiously... Only needed for gcc 4.X + if (CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0 AND CMAKE_C_COMPILER_VERSION VERSION_GREATER_EQUAL 4.8) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/4.8-4.last") + endif () - # Append more extra warning flags that only gcc 4.8+ knows about - if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 4.8) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/4.8") - if (HDF5_ENABLE_DEV_WARNINGS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-4.8") - else () - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-4.8") - endif () + # Append more extra warning flags that only gcc 4.8+ knows about + if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 4.8) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/4.8") + if (HDF5_ENABLE_DEV_WARNINGS) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-4.8") + else () + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-4.8") endif () + endif () - # Append more extra warning flags that only gcc 4.9+ knows about - if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 4.9) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/4.9") - endif () + # Append more extra warning flags that only gcc 4.9+ knows about + if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 4.9) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/4.9") + endif () - # Append more extra warning flags that only gcc 5.x+ knows about - if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/5") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-5") - else () - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-5") - endif () + # Append more extra warning flags that only gcc 5.x+ knows about + if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/5") + if (HDF5_ENABLE_WARNINGS_AS_ERRORS) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-5") + else () + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-5") endif () + endif () - # Append more extra warning flags that only gcc 6.x+ knows about - if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 6.0) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/6") - endif () + # Append more extra warning flags that only gcc 6.x+ knows about + if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 6.0) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/6") + endif () - # Append more extra warning flags that only gcc 7.x+ knows about - if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 7.0) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/7") - if (HDF5_ENABLE_DEV_WARNINGS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-7") - #else () - # ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-7") - endif () + # Append more extra warning flags that only gcc 7.x+ knows about + if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 7.0) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/7") + if (HDF5_ENABLE_DEV_WARNINGS) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-7") + #else () + # ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-7") endif () + endif () - # Append more extra warning flags that only gcc 8.x+ knows about - if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/8") - if (HDF5_ENABLE_WARNINGS_AS_ERRORS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-8") - endif () - if (HDF5_ENABLE_DEV_WARNINGS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-8") - else () - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-8") - endif () + # Append more extra warning flags that only gcc 8.x+ knows about + if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 8.0) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/8") + if (HDF5_ENABLE_WARNINGS_AS_ERRORS) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/error-8") endif () - - # Append more extra warning flags that only gcc 9.x+ knows about - if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/9") + if (HDF5_ENABLE_DEV_WARNINGS) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-8") + else () + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-8") endif () + endif () - # Append more extra warning flags that only gcc 9.3+ knows about - if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.3) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/9.3") - endif () + # Append more extra warning flags that only gcc 9.x+ knows about + if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.0) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/9") + endif () - # Append more extra warning flags that only gcc 10.x+ knows about - if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 10.0) - if (HDF5_ENABLE_DEV_WARNINGS) - ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-10") - #else () - # ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-10") - endif () + # Append more extra warning flags that only gcc 9.3+ knows about + if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 9.3) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/9.3") + endif () + + # Append more extra warning flags that only gcc 10.x+ knows about + if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 10.0) + if (HDF5_ENABLE_DEV_WARNINGS) + ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/developer-10") + #else () + # ADD_H5_FLAGS (H5_CFLAGS "${HDF5_SOURCE_DIR}/config/gnu-warnings/no-developer-10") endif () endif () endif () @@ -297,13 +334,13 @@ MARK_AS_ADVANCED (HDF5_ENABLE_ASSERTS) set (HDF5_ENABLE_SYMBOLS "OFF" CACHE STRING "Add debug symbols to the library independent of the build mode and optimization level (OFF NO YES)") set_property (CACHE HDF5_ENABLE_SYMBOLS PROPERTY STRINGS OFF NO YES) if (HDF5_ENABLE_SYMBOLS MATCHES "YES") - if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + if (CMAKE_C_COMPILER_ID STREQUAL "Intel" AND NOT _INTEL_WINDOWS) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g") elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -fno-omit-frame-pointer") endif () elseif (HDF5_ENABLE_SYMBOLS MATCHES "NO") - if (CMAKE_C_COMPILER_ID STREQUAL "Intel") + if (CMAKE_C_COMPILER_ID STREQUAL "Intel" AND NOT _INTEL_WINDOWS) set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Wl,-s") elseif (CMAKE_C_COMPILER_ID STREQUAL "GNU") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -s") diff --git a/config/intel-warnings/18 b/config/intel-warnings/18 index 565b47a..02bcdea 100644 --- a/config/intel-warnings/18 +++ b/config/intel-warnings/18 @@ -1,8 +1,2 @@ --Wextra-tokens --Wformat --Wformat-security -Wic-pointer --Wshadow -Wsign-compare --Wtrigraphs --Wwrite-strings diff --git a/config/intel-warnings/general-19 b/config/intel-warnings/general-19 deleted file mode 100644 index e35af30..0000000 --- a/config/intel-warnings/general-19 +++ /dev/null @@ -1,2 +0,0 @@ -# this is only available before oneapi versions --Wcheck diff --git a/config/intel-warnings/win-developer-general b/config/intel-warnings/win-developer-general new file mode 100644 index 0000000..6cd7ed7 --- /dev/null +++ b/config/intel-warnings/win-developer-general @@ -0,0 +1 @@ +/Wport diff --git a/config/intel-warnings/win-general b/config/intel-warnings/win-general new file mode 100644 index 0000000..ef54b2b --- /dev/null +++ b/config/intel-warnings/win-general @@ -0,0 +1 @@ +/Wall diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 396629c..b3264a7 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1117,6 +1117,15 @@ Bug Fixes since HDF5-1.12.0 release Configuration ------------- + - Fixed CMake C++ compiler flags + + A recent refactoring of the C++ configure files accidently removed the + file that executed the enable_language command for C++ needed by the + HDFCXXCompilerFlags.cmake file. Also updated the intel warnings files, + including adding support for windows platforms. + + (ADB - 2021/08/10) + - Better support for libaec (open-source Szip library) in CMake Implemented better support for libaec 1.0.5 (or later) library. This version -- cgit v0.12 From 04c504664b2cbb63b6a5cdf8c58a93dc65699f6d Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Wed, 18 Aug 2021 10:58:02 -0500 Subject: Fix formatting (#923) --- src/H5public.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5public.h b/src/H5public.h index dbccbf8..e7ed518 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -60,7 +60,7 @@ #define MPICH_SKIP_MPICXX 1 #endif #ifndef OMPI_SKIP_MPICXX -#define OMPI_SKIP_MPICXX 1 +#define OMPI_SKIP_MPICXX 1 #endif #include #ifndef MPI_FILE_NULL /* MPIO may be defined in mpi.h already */ -- cgit v0.12 From af7989e858d383322ced07873c2e19b433e74988 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Wed, 18 Aug 2021 11:02:29 -0500 Subject: Correct dataset close in java test (#919) * Correct dataset close * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- java/test/TestH5Arw.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/java/test/TestH5Arw.java b/java/test/TestH5Arw.java index 282b736..8ce2fee 100644 --- a/java/test/TestH5Arw.java +++ b/java/test/TestH5Arw.java @@ -100,7 +100,7 @@ public class TestH5Arw { if (H5aid >= 0) try {H5.H5Aclose(H5aid);} catch (Exception ex) {} if (H5did >= 0) - try {H5.H5Aclose(H5did);} catch (Exception ex) {} + try {H5.H5Dclose(H5did);} catch (Exception ex) {} if (H5fid > 0) try {H5.H5Fclose(H5fid);} catch (Exception ex) {} H5fid = HDF5Constants.H5I_INVALID_HID; -- cgit v0.12 From c019db60eecc173e1d3b118c9aeb7885865fa50f Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Wed, 18 Aug 2021 11:05:02 -0500 Subject: Remove non-published short arg names (#917) * Remove non-published short arg names * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- tools/src/h5perf/pio_perf.c | 116 ----------------------------------- tools/src/h5perf/sio_perf.c | 136 ------------------------------------------ tools/test/perform/zip_perf.c | 51 ---------------- 3 files changed, 303 deletions(-) diff --git a/tools/src/h5perf/pio_perf.c b/tools/src/h5perf/pio_perf.c index 96cba2d..41245d9 100644 --- a/tools/src/h5perf/pio_perf.c +++ b/tools/src/h5perf/pio_perf.c @@ -130,145 +130,29 @@ static const char *s_opts = "a:A:B:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; static const char *s_opts = "a:A:bB:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; #endif /* 1 */ static struct h5_long_options l_opts[] = {{"align", require_arg, 'a'}, - {"alig", require_arg, 'a'}, - {"ali", require_arg, 'a'}, - {"al", require_arg, 'a'}, {"api", require_arg, 'A'}, - {"ap", require_arg, 'A'}, #if 0 /* a sighting of the elusive binary option */ { "binary", no_arg, 'b' }, - { "binar", no_arg, 'b' }, - { "bina", no_arg, 'b' }, - { "bin", no_arg, 'b' }, - { "bi", no_arg, 'b' }, #endif /* 0 */ {"block-size", require_arg, 'B'}, - {"block-siz", require_arg, 'B'}, - {"block-si", require_arg, 'B'}, - {"block-s", require_arg, 'B'}, - {"block-", require_arg, 'B'}, - {"block", require_arg, 'B'}, - {"bloc", require_arg, 'B'}, - {"blo", require_arg, 'B'}, - {"bl", require_arg, 'B'}, {"chunk", no_arg, 'c'}, - {"chun", no_arg, 'c'}, - {"chu", no_arg, 'c'}, - {"ch", no_arg, 'c'}, {"collective", no_arg, 'C'}, - {"collectiv", no_arg, 'C'}, - {"collecti", no_arg, 'C'}, - {"collect", no_arg, 'C'}, - {"collec", no_arg, 'C'}, - {"colle", no_arg, 'C'}, - {"coll", no_arg, 'C'}, - {"col", no_arg, 'C'}, - {"co", no_arg, 'C'}, {"debug", require_arg, 'D'}, - {"debu", require_arg, 'D'}, - {"deb", require_arg, 'D'}, - {"de", require_arg, 'D'}, {"geometry", no_arg, 'g'}, - {"geometr", no_arg, 'g'}, - {"geomet", no_arg, 'g'}, - {"geome", no_arg, 'g'}, - {"geom", no_arg, 'g'}, - {"geo", no_arg, 'g'}, - {"ge", no_arg, 'g'}, {"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, {"interleaved", require_arg, 'I'}, - {"interleave", require_arg, 'I'}, - {"interleav", require_arg, 'I'}, - {"interlea", require_arg, 'I'}, - {"interle", require_arg, 'I'}, - {"interl", require_arg, 'I'}, - {"inter", require_arg, 'I'}, - {"inte", require_arg, 'I'}, - {"int", require_arg, 'I'}, - {"in", require_arg, 'I'}, {"max-num-processes", require_arg, 'P'}, - {"max-num-processe", require_arg, 'P'}, - {"max-num-process", require_arg, 'P'}, - {"max-num-proces", require_arg, 'P'}, - {"max-num-proce", require_arg, 'P'}, - {"max-num-proc", require_arg, 'P'}, - {"max-num-pro", require_arg, 'P'}, - {"max-num-pr", require_arg, 'P'}, - {"max-num-p", require_arg, 'P'}, {"min-num-processes", require_arg, 'p'}, - {"min-num-processe", require_arg, 'p'}, - {"min-num-process", require_arg, 'p'}, - {"min-num-proces", require_arg, 'p'}, - {"min-num-proce", require_arg, 'p'}, - {"min-num-proc", require_arg, 'p'}, - {"min-num-pro", require_arg, 'p'}, - {"min-num-pr", require_arg, 'p'}, - {"min-num-p", require_arg, 'p'}, {"max-xfer-size", require_arg, 'X'}, - {"max-xfer-siz", require_arg, 'X'}, - {"max-xfer-si", require_arg, 'X'}, - {"max-xfer-s", require_arg, 'X'}, - {"max-xfer", require_arg, 'X'}, - {"max-xfe", require_arg, 'X'}, - {"max-xf", require_arg, 'X'}, - {"max-x", require_arg, 'X'}, {"min-xfer-size", require_arg, 'x'}, - {"min-xfer-siz", require_arg, 'x'}, - {"min-xfer-si", require_arg, 'x'}, - {"min-xfer-s", require_arg, 'x'}, - {"min-xfer", require_arg, 'x'}, - {"min-xfe", require_arg, 'x'}, - {"min-xf", require_arg, 'x'}, - {"min-x", require_arg, 'x'}, {"num-bytes", require_arg, 'e'}, - {"num-byte", require_arg, 'e'}, - {"num-byt", require_arg, 'e'}, - {"num-by", require_arg, 'e'}, - {"num-b", require_arg, 'e'}, {"num-dsets", require_arg, 'd'}, - {"num-dset", require_arg, 'd'}, - {"num-dse", require_arg, 'd'}, - {"num-ds", require_arg, 'd'}, - {"num-d", require_arg, 'd'}, {"num-files", require_arg, 'F'}, - {"num-file", require_arg, 'F'}, - {"num-fil", require_arg, 'F'}, - {"num-fi", require_arg, 'F'}, - {"num-f", require_arg, 'F'}, {"num-iterations", require_arg, 'i'}, - {"num-iteration", require_arg, 'i'}, - {"num-iteratio", require_arg, 'i'}, - {"num-iterati", require_arg, 'i'}, - {"num-iterat", require_arg, 'i'}, - {"num-itera", require_arg, 'i'}, - {"num-iter", require_arg, 'i'}, - {"num-ite", require_arg, 'i'}, - {"num-it", require_arg, 'i'}, - {"num-i", require_arg, 'i'}, {"output", require_arg, 'o'}, - {"outpu", require_arg, 'o'}, - {"outp", require_arg, 'o'}, - {"out", require_arg, 'o'}, - {"ou", require_arg, 'o'}, {"threshold", require_arg, 'T'}, - {"threshol", require_arg, 'T'}, - {"thresho", require_arg, 'T'}, - {"thresh", require_arg, 'T'}, - {"thres", require_arg, 'T'}, - {"thre", require_arg, 'T'}, - {"thr", require_arg, 'T'}, - {"th", require_arg, 'T'}, {"write-only", require_arg, 'w'}, - {"write-onl", require_arg, 'w'}, - {"write-on", require_arg, 'w'}, - {"write-o", require_arg, 'w'}, - {"write", require_arg, 'w'}, - {"writ", require_arg, 'w'}, - {"wri", require_arg, 'w'}, - {"wr", require_arg, 'w'}, {NULL, 0, '\0'}}; struct options { diff --git a/tools/src/h5perf/sio_perf.c b/tools/src/h5perf/sio_perf.c index 51a7825..7bd58d8 100644 --- a/tools/src/h5perf/sio_perf.c +++ b/tools/src/h5perf/sio_perf.c @@ -99,168 +99,32 @@ static const char *progname = "h5perf_serial"; */ static const char * s_opts = "a:A:B:c:Cd:D:e:F:ghi:Imno:p:P:r:stT:v:wx:X:"; static struct h5_long_options l_opts[] = {{"align", require_arg, 'a'}, - {"alig", require_arg, 'a'}, - {"ali", require_arg, 'a'}, - {"al", require_arg, 'a'}, {"api", require_arg, 'A'}, - {"ap", require_arg, 'A'}, #if 0 /* a sighting of the elusive binary option */ { "binary", no_arg, 'b' }, - { "binar", no_arg, 'b' }, - { "bina", no_arg, 'b' }, - { "bin", no_arg, 'b' }, - { "bi", no_arg, 'b' }, #endif /* 0 */ {"block-size", require_arg, 'B'}, - {"block-siz", require_arg, 'B'}, - {"block-si", require_arg, 'B'}, - {"block-s", require_arg, 'B'}, - {"block-", require_arg, 'B'}, - {"block", require_arg, 'B'}, - {"bloc", require_arg, 'B'}, - {"blo", require_arg, 'B'}, - {"bl", require_arg, 'B'}, {"chunk", no_arg, 'c'}, - {"chun", no_arg, 'c'}, - {"chu", no_arg, 'c'}, - {"ch", no_arg, 'c'}, {"collective", no_arg, 'C'}, - {"collectiv", no_arg, 'C'}, - {"collecti", no_arg, 'C'}, - {"collect", no_arg, 'C'}, - {"collec", no_arg, 'C'}, - {"colle", no_arg, 'C'}, - {"coll", no_arg, 'C'}, - {"col", no_arg, 'C'}, - {"co", no_arg, 'C'}, {"debug", require_arg, 'D'}, - {"debu", require_arg, 'D'}, - {"deb", require_arg, 'D'}, - {"de", require_arg, 'D'}, {"file-driver", require_arg, 'v'}, - {"file-drive", require_arg, 'v'}, - {"file-driv", require_arg, 'v'}, - {"file-dri", require_arg, 'v'}, - {"file-dr", require_arg, 'v'}, - {"file-d", require_arg, 'v'}, - {"file-", require_arg, 'v'}, - {"file", require_arg, 'v'}, - {"fil", require_arg, 'v'}, - {"fi", require_arg, 'v'}, {"geometry", no_arg, 'g'}, - {"geometr", no_arg, 'g'}, - {"geomet", no_arg, 'g'}, - {"geome", no_arg, 'g'}, - {"geom", no_arg, 'g'}, - {"geo", no_arg, 'g'}, - {"ge", no_arg, 'g'}, {"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, {"interleaved", require_arg, 'I'}, - {"interleave", require_arg, 'I'}, - {"interleav", require_arg, 'I'}, - {"interlea", require_arg, 'I'}, - {"interle", require_arg, 'I'}, - {"interl", require_arg, 'I'}, - {"inter", require_arg, 'I'}, - {"inte", require_arg, 'I'}, - {"int", require_arg, 'I'}, - {"in", require_arg, 'I'}, {"max-num-processes", require_arg, 'P'}, - {"max-num-processe", require_arg, 'P'}, - {"max-num-process", require_arg, 'P'}, - {"max-num-proces", require_arg, 'P'}, - {"max-num-proce", require_arg, 'P'}, - {"max-num-proc", require_arg, 'P'}, - {"max-num-pro", require_arg, 'P'}, - {"max-num-pr", require_arg, 'P'}, - {"max-num-p", require_arg, 'P'}, {"min-num-processes", require_arg, 'p'}, - {"min-num-processe", require_arg, 'p'}, - {"min-num-process", require_arg, 'p'}, - {"min-num-proces", require_arg, 'p'}, - {"min-num-proce", require_arg, 'p'}, - {"min-num-proc", require_arg, 'p'}, - {"min-num-pro", require_arg, 'p'}, - {"min-num-pr", require_arg, 'p'}, - {"min-num-p", require_arg, 'p'}, {"max-xfer-size", require_arg, 'X'}, - {"max-xfer-siz", require_arg, 'X'}, - {"max-xfer-si", require_arg, 'X'}, - {"max-xfer-s", require_arg, 'X'}, - {"max-xfer", require_arg, 'X'}, - {"max-xfe", require_arg, 'X'}, - {"max-xf", require_arg, 'X'}, - {"max-x", require_arg, 'X'}, {"min-xfer-size", require_arg, 'x'}, - {"min-xfer-siz", require_arg, 'x'}, - {"min-xfer-si", require_arg, 'x'}, - {"min-xfer-s", require_arg, 'x'}, - {"min-xfer", require_arg, 'x'}, - {"min-xfe", require_arg, 'x'}, - {"min-xf", require_arg, 'x'}, - {"min-x", require_arg, 'x'}, {"num-bytes", require_arg, 'e'}, - {"num-byte", require_arg, 'e'}, - {"num-byt", require_arg, 'e'}, - {"num-by", require_arg, 'e'}, - {"num-b", require_arg, 'e'}, {"num-dsets", require_arg, 'd'}, - {"num-dset", require_arg, 'd'}, - {"num-dse", require_arg, 'd'}, - {"num-ds", require_arg, 'd'}, - {"num-d", require_arg, 'd'}, {"num-files", require_arg, 'F'}, - {"num-file", require_arg, 'F'}, - {"num-fil", require_arg, 'F'}, - {"num-fi", require_arg, 'F'}, - {"num-f", require_arg, 'F'}, {"num-iterations", require_arg, 'i'}, - {"num-iteration", require_arg, 'i'}, - {"num-iteratio", require_arg, 'i'}, - {"num-iterati", require_arg, 'i'}, - {"num-iterat", require_arg, 'i'}, - {"num-itera", require_arg, 'i'}, - {"num-iter", require_arg, 'i'}, - {"num-ite", require_arg, 'i'}, - {"num-it", require_arg, 'i'}, - {"num-i", require_arg, 'i'}, {"order", require_arg, 'r'}, - {"orde", require_arg, 'r'}, - {"ord", require_arg, 'r'}, - {"or", require_arg, 'r'}, {"output", require_arg, 'o'}, - {"outpu", require_arg, 'o'}, - {"outp", require_arg, 'o'}, - {"out", require_arg, 'o'}, - {"ou", require_arg, 'o'}, {"extendable", no_arg, 't'}, - {"extendabl", no_arg, 't'}, - {"extendab", no_arg, 't'}, - {"extenda", no_arg, 't'}, - {"extend", no_arg, 't'}, - {"exten", no_arg, 't'}, - {"exte", no_arg, 't'}, - {"ext", no_arg, 't'}, - {"ex", no_arg, 't'}, {"threshold", require_arg, 'T'}, - {"threshol", require_arg, 'T'}, - {"thresho", require_arg, 'T'}, - {"thresh", require_arg, 'T'}, - {"thres", require_arg, 'T'}, - {"thre", require_arg, 'T'}, - {"thr", require_arg, 'T'}, - {"th", require_arg, 'T'}, {"write-only", require_arg, 'w'}, - {"write-onl", require_arg, 'w'}, - {"write-on", require_arg, 'w'}, - {"write-o", require_arg, 'w'}, - {"write", require_arg, 'w'}, - {"writ", require_arg, 'w'}, - {"wri", require_arg, 'w'}, - {"wr", require_arg, 'w'}, {NULL, 0, '\0'}}; struct options { diff --git a/tools/test/perform/zip_perf.c b/tools/test/perform/zip_perf.c index 6231587..05ad083 100644 --- a/tools/test/perform/zip_perf.c +++ b/tools/test/perform/zip_perf.c @@ -67,62 +67,11 @@ static void compress_buffer(Bytef *dest, uLongf *destLen, const Bytef *source, u static const char * s_opts = "hB:b:c:p:rs:0123456789"; static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"compressability", require_arg, 'c'}, - {"compressabilit", require_arg, 'c'}, - {"compressabili", require_arg, 'c'}, - {"compressabil", require_arg, 'c'}, - {"compressabi", require_arg, 'c'}, - {"compressab", require_arg, 'c'}, - {"compressa", require_arg, 'c'}, - {"compress", require_arg, 'c'}, - {"compres", require_arg, 'c'}, - {"compre", require_arg, 'c'}, - {"compr", require_arg, 'c'}, - {"comp", require_arg, 'c'}, - {"com", require_arg, 'c'}, - {"co", require_arg, 'c'}, {"file-size", require_arg, 's'}, - {"file-siz", require_arg, 's'}, - {"file-si", require_arg, 's'}, - {"file-s", require_arg, 's'}, - {"file", require_arg, 's'}, - {"fil", require_arg, 's'}, - {"fi", require_arg, 's'}, {"max-buffer-size", require_arg, 'B'}, - {"max-buffer-siz", require_arg, 'B'}, - {"max-buffer-si", require_arg, 'B'}, - {"max-buffer-s", require_arg, 'B'}, - {"max-buffer", require_arg, 'B'}, - {"max-buffe", require_arg, 'B'}, - {"max-buff", require_arg, 'B'}, - {"max-buf", require_arg, 'B'}, - {"max-bu", require_arg, 'B'}, - {"max-b", require_arg, 'B'}, - {"max", require_arg, 'B'}, {"min-buffer-size", require_arg, 'b'}, - {"min-buffer-siz", require_arg, 'b'}, - {"min-buffer-si", require_arg, 'b'}, - {"min-buffer-s", require_arg, 'b'}, - {"min-buffer", require_arg, 'b'}, - {"min-buffe", require_arg, 'b'}, - {"min-buff", require_arg, 'b'}, - {"min-buf", require_arg, 'b'}, - {"min-bu", require_arg, 'b'}, - {"min-b", require_arg, 'b'}, - {"min", require_arg, 'b'}, {"prefix", require_arg, 'p'}, - {"prefi", require_arg, 'p'}, - {"pref", require_arg, 'p'}, - {"pre", require_arg, 'p'}, - {"pr", require_arg, 'p'}, {"random-test", no_arg, 'r'}, - {"random-tes", no_arg, 'r'}, - {"random-te", no_arg, 'r'}, - {"random-t", no_arg, 'r'}, - {"random", no_arg, 'r'}, - {"rando", no_arg, 'r'}, - {"rand", no_arg, 'r'}, - {"ran", no_arg, 'r'}, - {"ra", no_arg, 'r'}, {NULL, 0, '\0'}}; /* -- cgit v0.12 From c88c0f2008a0a431ee64ae7a36ed29f927ba3e13 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Thu, 19 Aug 2021 13:07:49 -0500 Subject: Correct number of args to Windows HDunsetenv macro (#939) * Correct number of args to Windows HDunsetenv macro * Correct format --- src/H5win32defs.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/H5win32defs.h b/src/H5win32defs.h index 030b65b..44cf50c 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -80,7 +80,7 @@ struct timezone { #define HDstrtok_r(X, Y, Z) strtok_s(X, Y, Z) #define HDtzset() _tzset() #define HDunlink(S) _unlink(S) -#define HDunsetenv(N, V, O) Wsetenv(N, "", 1) +#define HDunsetenv(N) Wsetenv(N, "", 1) #define HDwrite(F, M, Z) _write(F, M, Z) #ifndef H5_HAVE_MINGW -- cgit v0.12 From 2524b57a6f69e1a24212cba12d504748e13e9743 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Thu, 19 Aug 2021 15:56:37 -0500 Subject: Develop windows compiler fixes (#912) * Fix windows compiler issues Restore MS C++ Default exception handling behavior #885 Remove clang toolchain as it skips compiler checks * Add check and header gaurd for pwd.h * Revert change * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- config/cmake/H5pubconf.h.in | 15 +++++---- config/cmake/HDFCXXCompilerFlags.cmake | 54 ++++++++++++++++-------------- config/cmake_ext_mod/ConfigureChecks.cmake | 2 +- configure.ac | 2 +- src/H5private.h | 2 ++ 5 files changed, 42 insertions(+), 33 deletions(-) diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 5a94be3..04c0c9f 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -271,6 +271,9 @@ /* Define to 1 if you have the header file. */ #cmakedefine H5_HAVE_PTHREAD_H @H5_HAVE_PTHREAD_H@ +/* Define to 1 if you have the header file. */ +#cmakedefine H5_HAVE_PWD_H @H5_HAVE_PWD_H@ + /* Define to 1 if you have the header file. */ #cmakedefine H5_HAVE_QUADMATH_H @H5_HAVE_QUADMATH_H@ @@ -631,6 +634,12 @@ PTHREAD_SCOPE_SYSTEM) call. */ #cmakedefine H5_SYSTEM_SCOPE_THREADS @H5_SYSTEM_SCOPE_THREADS@ +/* Define using v1.6 public API symbols by default */ +#cmakedefine H5_USE_16_API_DEFAULT @H5_USE_16_API_DEFAULT@ + +/* Define using v1.8 public API symbols by default */ +#cmakedefine H5_USE_18_API_DEFAULT @H5_USE_18_API_DEFAULT@ + /* Define using v1.10 public API symbols by default */ #cmakedefine H5_USE_110_API_DEFAULT @H5_USE_110_API_DEFAULT@ @@ -640,12 +649,6 @@ /* Define using v1.14 public API symbols by default */ #cmakedefine H5_USE_114_API_DEFAULT @H5_USE_114_API_DEFAULT@ -/* Define using v1.6 public API symbols by default */ -#cmakedefine H5_USE_16_API_DEFAULT @H5_USE_16_API_DEFAULT@ - -/* Define using v1.8 public API symbols by default */ -#cmakedefine H5_USE_18_API_DEFAULT @H5_USE_18_API_DEFAULT@ - /* Define if the library will use file locking */ #cmakedefine H5_USE_FILE_LOCKING @H5_USE_FILE_LOCKING@ diff --git a/config/cmake/HDFCXXCompilerFlags.cmake b/config/cmake/HDFCXXCompilerFlags.cmake index dc982c1..a121d0a 100644 --- a/config/cmake/HDFCXXCompilerFlags.cmake +++ b/config/cmake/HDFCXXCompilerFlags.cmake @@ -11,10 +11,10 @@ # ENABLE_LANGUAGE (CXX) -set(CMAKE_CXX_STANDARD 11) -set(CMAKE_CXX_STANDARD_REQUIRED TRUE) +set (CMAKE_CXX_STANDARD 11) +set (CMAKE_CXX_STANDARD_REQUIRED TRUE) -set(CMAKE_CXX_EXTENSIONS OFF) +set (CMAKE_CXX_EXTENSIONS OFF) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_SANITIZER_FLAGS} ${CMAKE_CXX_FLAGS}") if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") @@ -23,30 +23,34 @@ endif () #----------------------------------------------------------------------------- # Compiler specific flags : Shouldn't there be compiler tests for these #----------------------------------------------------------------------------- -if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Intel") - set(_INTEL_WINDOWS 1) -endif() +if (WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Intel") + set (_INTEL_WINDOWS 1) +endif () -if(WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" +if (WIN32 AND CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND "x${CMAKE_CXX_SIMULATE_ID}" STREQUAL "xMSVC") - set(_CLANG_MSVC_WINDOWS 1) + set (_CLANG_MSVC_WINDOWS 1) endif() # MSVC 14.28 enables C5105, but the Windows SDK 10.0.18362.0 triggers it. -if(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.28) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd5105") -endif() +if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND CMAKE_CXX_COMPILER_LOADED) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /EHsc") + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 19.28) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -wd5105") + endif () +endif () -if (CMAKE_CXX_COMPILER_ID STREQUAL SunPro AND - NOT DEFINED CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION) - if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.13) - if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD EQUAL 98) - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++03") - endif() - else() - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -library=stlport4") - endif() -endif() +if (CMAKE_CXX_COMPILER_ID STREQUAL SunPro AND CMAKE_CXX_COMPILER_LOADED) + if (NOT DEFINED CMAKE_CXX${CMAKE_CXX_STANDARD}_STANDARD_COMPILE_OPTION) + if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.13) + if (NOT CMAKE_CXX_STANDARD OR CMAKE_CXX_STANDARD EQUAL 98) + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++03") + endif () + else () + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -library=stlport4") + endif () + endif () +endif () if (CMAKE_COMPILER_IS_GNUCXX AND CMAKE_CXX_COMPILER_LOADED) set (CMAKE_CXX_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_CXX_FLAGS}") @@ -301,7 +305,7 @@ endif () # This option will force/override the default setting for all configurations #----------------------------------------------------------------------------- if (HDF5_ENABLE_SYMBOLS MATCHES "YES") - if(CMAKE_CXX_COMPILER_LOADED) + if (CMAKE_CXX_COMPILER_LOADED) if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" AND NOT _INTEL_WINDOWS) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -g") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") @@ -309,7 +313,7 @@ if (HDF5_ENABLE_SYMBOLS MATCHES "YES") endif () endif () elseif (HDF5_ENABLE_SYMBOLS MATCHES "NO") - if(CMAKE_CXX_COMPILER_LOADED) + if (CMAKE_CXX_COMPILER_LOADED) if (CMAKE_CXX_COMPILER_ID STREQUAL "Intel" AND NOT _INTEL_WINDOWS) set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wl,-s") elseif (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") @@ -323,7 +327,7 @@ endif () # This option will force/override the default setting for all configurations #----------------------------------------------------------------------------- if (HDF5_ENABLE_PROFILING) - if(CMAKE_CXX_COMPILER_LOADED) + if (CMAKE_CXX_COMPILER_LOADED) list (APPEND HDF5_CMAKE_CXX_FLAGS "${PROFILE_CXXFLAGS}") endif () endif () @@ -333,7 +337,7 @@ endif () # This option will force/override the default setting for all configurations #----------------------------------------------------------------------------- if (HDF5_ENABLE_OPTIMIZATION) - if(CMAKE_CXX_COMPILER_LOADED) + if (CMAKE_CXX_COMPILER_LOADED) list (APPEND HDF5_CMAKE_CXX_FLAGS "${OPTIMIZE_CXXFLAGS}") endif () endif () diff --git a/config/cmake_ext_mod/ConfigureChecks.cmake b/config/cmake_ext_mod/ConfigureChecks.cmake index 27c7045..1dd1440 100644 --- a/config/cmake_ext_mod/ConfigureChecks.cmake +++ b/config/cmake_ext_mod/ConfigureChecks.cmake @@ -115,7 +115,7 @@ CHECK_INCLUDE_FILE_CONCAT ("sys/types.h" ${HDF_PREFIX}_HAVE_SYS_TYPES_H) CHECK_INCLUDE_FILE_CONCAT ("features.h" ${HDF_PREFIX}_HAVE_FEATURES_H) CHECK_INCLUDE_FILE_CONCAT ("dirent.h" ${HDF_PREFIX}_HAVE_DIRENT_H) CHECK_INCLUDE_FILE_CONCAT ("unistd.h" ${HDF_PREFIX}_HAVE_UNISTD_H) - +CHECK_INCLUDE_FILE_CONCAT ("pwd.h" ${HDF_PREFIX}_HAVE_PWD_H) CHECK_INCLUDE_FILE_CONCAT ("globus/common.h" ${HDF_PREFIX}_HAVE_GLOBUS_COMMON_H) CHECK_INCLUDE_FILE_CONCAT ("pdb.h" ${HDF_PREFIX}_HAVE_PDB_H) CHECK_INCLUDE_FILE_CONCAT ("pthread.h" ${HDF_PREFIX}_HAVE_PTHREAD_H) diff --git a/configure.ac b/configure.ac index cbb192d..d955a9f 100644 --- a/configure.ac +++ b/configure.ac @@ -1225,7 +1225,7 @@ AC_CHECK_LIB([dl], [dlopen]) ## ## Unix -AC_CHECK_HEADERS([dirent.h features.h unistd.h]) +AC_CHECK_HEADERS([dirent.h features.h unistd.h pwd.h]) AC_CHECK_HEADERS([sys/file.h sys/ioctl.h sys/resource.h]) AC_CHECK_HEADERS([sys/stat.h sys/time.h sys/types.h]) diff --git a/src/H5private.h b/src/H5private.h index 802dd90..7b18df0 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -44,7 +44,9 @@ #include #endif #ifdef H5_HAVE_UNISTD_H +#ifdef H5_HAVE_PWD_H #include +#endif #include #include #endif -- cgit v0.12 From 564380d176ae71d9f4ece9df51957e3f817be45d Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Thu, 19 Aug 2021 15:57:48 -0500 Subject: Fix issue #924 in develop branch: the ${HDF5_CXXLINKER} does not work in h5c++ (#925) * Committing clang-format changes * Fix issue #924, ${HDF5_CXXLINKER} does not work in h5c++. 2 instances of CLINKER corrected to CXXLINKER comment border aligned Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- c++/src/h5c++.in | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/c++/src/h5c++.in b/c++/src/h5c++.in index cf993b9..573d20d 100644 --- a/c++/src/h5c++.in +++ b/c++/src/h5c++.in @@ -35,10 +35,10 @@ HL="@HL@" ## (Advanced usage - know what you're doing - you're on your own here.) ## ## The four variables below can be used to insert paths and flags in ## ## CPPFLAGS, CXXFLAGS, LDFLAGS, or LIBS in the h5cc compile line: ## -## $CLINKER $H5BLD_CPPFLAGS $CPPFLAGS $H5BLD_CXXFLAGS $CXXFLAGS ## +## $CXXLINKER $H5BLD_CPPFLAGS $CPPFLAGS $H5BLD_CXXFLAGS $CXXFLAGS ## ## $LDFLAGS $LIBS $clibpath $link_objs $link_args $shared_link ## ## ## -## These settings can be overridden by setting HDF5_CXXFLAGS, ## +## These settings can be overridden by setting HDF5_CXXFLAGS, ## ## HDF5_CPPFLAGS, HDF5_LDFLAGS, or HDF5_LIBS in the environment. ## ## ## ############################################################################ @@ -93,7 +93,7 @@ H5BLD_LDFLAGS="@AM_LDFLAGS@ @LDFLAGS@" H5BLD_LIBS="@LIBS@" CXX="${HDF5_CXX:-$CXXBASE}" -CXXLINKER="${HDF5_CLINKER:-$CXXLINKERBASE}" +CXXLINKER="${HDF5_CXXLINKER:-$CXXLINKERBASE}" CXXFLAGS="${HDF5_CXXFLAGS:-$CXXFLAGSBASE}" CPPFLAGS="${HDF5_CPPFLAGS:-$CPPFLAGSBASE}" LDFLAGS="${HDF5_LDFLAGS:-$LDFLAGSBASE}" -- cgit v0.12 From 3600d33f10db1d520f360930c40bf99e282af101 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Thu, 19 Aug 2021 16:03:05 -0500 Subject: CMake should use the output dir programs for shell scripts (#928) --- test/ShellTests.cmake | 118 ++--------------------------------------------- test/test_usecases.sh.in | 22 +++++---- test/testswmr.sh.in | 40 +++++++++------- test/testvdsswmr.sh.in | 16 +++++-- 4 files changed, 52 insertions(+), 144 deletions(-) diff --git a/test/ShellTests.cmake b/test/ShellTests.cmake index 812121e..9614152 100644 --- a/test/ShellTests.cmake +++ b/test/ShellTests.cmake @@ -19,7 +19,8 @@ if (UNIX) find_program (SH_PROGRAM bash) if (SH_PROGRAM) - + set (srcdir ${HDF5_TEST_SOURCE_DIR}) + set (bindir ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}) ############################################################################## # configure scripts to test dir ############################################################################## @@ -33,13 +34,8 @@ if (UNIX) ############################################################################## # copy test programs to test dir ############################################################################## - add_custom_command ( - TARGET swmr_check_compat_vfd - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_check_compat_vfd" - ) - + #shell script creates dir + #file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/swmr_test") add_custom_command ( TARGET swmr_check_compat_vfd POST_BUILD @@ -47,118 +43,12 @@ if (UNIX) ARGS -E copy_if_different "${HDF5_SOURCE_DIR}/bin/output_filter.sh" "${HDF5_TEST_BINARY_DIR}/H5TEST/bin/output_filter.sh" ) - file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/flushrefresh_test") - add_custom_command ( - TARGET flushrefresh - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/flushrefresh" - ) - #shell script creates dir #file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/usecases_test") - add_custom_command ( - TARGET use_append_mchunks - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/use_append_mchunks" - ) - add_custom_command ( - TARGET use_disable_mdc_flushes - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/use_disable_mdc_flushes" - ) - add_custom_command ( - TARGET twriteorder - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/twriteorder" - ) - add_custom_command ( - TARGET use_append_chunk - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/use_append_chunk" - ) file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/swmr_test") - add_custom_command ( - TARGET swmr_generator - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_test/swmr_generator" - ) - add_custom_command ( - TARGET swmr_start_write - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_test/swmr_start_write" - ) - add_custom_command ( - TARGET swmr_reader - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_test/swmr_reader" - ) - add_custom_command ( - TARGET swmr_writer - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_test/swmr_writer" - ) - add_custom_command ( - TARGET swmr_remove_reader - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_test/swmr_remove_reader" - ) - add_custom_command ( - TARGET swmr_remove_writer - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_test/swmr_remove_writer" - ) - add_custom_command ( - TARGET swmr_addrem_writer - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_test/swmr_addrem_writer" - ) - add_custom_command ( - TARGET swmr_sparse_reader - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_test/swmr_sparse_reader" - ) - add_custom_command ( - TARGET swmr_sparse_writer - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/swmr_test/swmr_sparse_writer" - ) file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/H5TEST/vds_swmr_test") - add_custom_command ( - TARGET vds_swmr_gen - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/vds_swmr_gen" - ) - add_custom_command ( - TARGET vds_swmr_writer - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/vds_swmr_writer" - ) - add_custom_command ( - TARGET vds_swmr_reader - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different "$" "${HDF5_TEST_BINARY_DIR}/H5TEST/vds_swmr_reader" - ) - - ############################################################################## ############################################################################## diff --git a/test/test_usecases.sh.in b/test/test_usecases.sh.in index bb53697..eaa875e 100644 --- a/test/test_usecases.sh.in +++ b/test/test_usecases.sh.in @@ -22,10 +22,16 @@ # exit codes are okay (0). srcdir=@srcdir@ +bindir=@bindir@ + +# If the bindir directory is not set just use current (.). +if test -z "$bindir"; then + bindir=. +fi # Check to see if the VFD specified by the HDF5_DRIVER environment variable # supports SWMR. -./swmr_check_compat_vfd +$bindir/swmr_check_compat_vfd rc=$? if [[ $rc != 0 ]] ; then echo @@ -85,7 +91,7 @@ TOOLTEST() { # Run test. TESTING $program $@ ( - $RUNSERIAL ./$program "$@" + $RUNSERIAL $bindir/$program "$@" ) >$actual 2>$actual_err exit_code=$? @@ -178,14 +184,14 @@ fi # main body for p in $USECASES_PROGRAMS; do - TOOLTEST ./$p - TOOLTEST ./$p -z 256 + TOOLTEST $p + TOOLTEST $p -z 256 tmpfile=/tmp/datatfile.$$ - TOOLTEST ./$p -f $tmpfile; rm -f $tmpfile - TOOLTEST ./$p -l w - TOOLTEST ./$p -l r + TOOLTEST $p -f $tmpfile; rm -f $tmpfile + TOOLTEST $p -l w + TOOLTEST $p -l r # use case 1.9, testing with multi-planes chunks - TOOLTEST ./$p -z 256 -y 5 # 5 planes chunks + TOOLTEST $p -z 256 -y 5 # 5 planes chunks # cleanup temp datafile if test -z "$HDF5_NOCLEANUP"; then rm -f $p.h5 diff --git a/test/testswmr.sh.in b/test/testswmr.sh.in index 2df23c6..37e8f9c 100644 --- a/test/testswmr.sh.in +++ b/test/testswmr.sh.in @@ -17,6 +17,7 @@ # Albert Cheng, 2009/07/22 srcdir=@srcdir@ +bindir=@bindir@ ############################################################################### ## test parameters @@ -97,9 +98,14 @@ if test -z "$srcdir"; then srcdir=. fi +# If the bindir directory is not set just use current (.). +if test -z "$bindir"; then + bindir=. +fi + # Check to see if the VFD specified by the HDF5_DRIVER environment variable # supports SWMR. -./swmr_check_compat_vfd +$bindir/swmr_check_compat_vfd rc=$? if [ $rc -ne 0 ] ; then echo @@ -172,7 +178,7 @@ do echo "###############################################################################" # Launch the Generator without SWMR_WRITE echo launch the swmr_generator - ./swmr_generator $compress $index_type + $bindir/swmr_generator $compress $index_type if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -180,7 +186,7 @@ do # Launch the Generator with SWMR_WRITE echo launch the swmr_generator with SWMR_WRITE - ./swmr_generator -s $compress $index_type + $bindir/swmr_generator -s $compress $index_type if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -204,7 +210,7 @@ do # Launch the Writer echo launch the swmr_start_writer seed="" # Put -r command here - ./swmr_start_write $compress $index_type $Nrecords $seed 2>&1 |tee swmr_writer.out & + $bindir/swmr_start_write $compress $index_type $Nrecords $seed 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -220,7 +226,7 @@ do while [ $n -lt $Nreaders ]; do #seed="-r ${seeds[$n]}" seed="" - ./swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & + $bindir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -265,7 +271,7 @@ do # Launch the Generator echo launch the swmr_generator - ./swmr_generator -s $compress $index_type + $bindir/swmr_generator -s $compress $index_type if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -277,7 +283,7 @@ do # Launch the Writer echo launch the swmr_writer seed="" # Put -r command here - ./swmr_writer -o $Nrecords $seed 2>&1 |tee swmr_writer.out & + $bindir/swmr_writer -o $Nrecords $seed 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -292,7 +298,7 @@ do while [ $n -lt $Nreaders ]; do #seed="-r ${seeds[$n]}" seed="" - ./swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & + $bindir/swmr_reader $Nsecs_add $seed 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -340,7 +346,7 @@ do # Launch the Remove Writer echo launch the swmr_remove_writer seed="" # Put -r command here - ./swmr_remove_writer -o $Nrecs_rem $seed 2>&1 |tee swmr_writer.out & + $bindir/swmr_remove_writer -o $Nrecs_rem $seed 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -355,7 +361,7 @@ do while [ $n -lt $Nreaders ]; do #seed="-r ${seeds[$n]}" seed="" - ./swmr_remove_reader $Nsecs_rem $seed 2>&1 |tee swmr_reader.out.$n & + $bindir/swmr_remove_reader $Nsecs_rem $seed 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -400,7 +406,7 @@ do # Launch the Generator echo launch the swmr_generator - ./swmr_generator $compress $index_type + $bindir/swmr_generator $compress $index_type if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -409,7 +415,7 @@ do # Launch the Writer (not in parallel - just to rebuild the datasets) echo launch the swmr_writer seed="" # Put -r command here - ./swmr_writer $Nrecords $seed + $bindir/swmr_writer $Nrecords $seed if test $? -ne 0; then echo writer had error nerrors=`expr $nerrors + 1` @@ -421,7 +427,7 @@ do # Launch the Add/Remove Writer echo launch the swmr_addrem_writer seed="" # Put -r command here - ./swmr_addrem_writer $Nrecords $seed 2>&1 |tee swmr_writer.out & + $bindir/swmr_addrem_writer $Nrecords $seed 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -436,7 +442,7 @@ do while [ $n -lt $Nreaders ]; do #seed="-r ${seeds[$n]}" seed="" - ./swmr_remove_reader $Nsecs_addrem $seed 2>&1 |tee swmr_reader.out.$n & + $bindir/swmr_remove_reader $Nsecs_addrem $seed 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done @@ -484,7 +490,7 @@ do # created by the generator. echo launch the swmr_generator seed="" # Put -r command here - ./swmr_generator $compress $index_type $seed + $bindir/swmr_generator $compress $index_type $seed if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -494,7 +500,7 @@ do rm -f $WRITER_MESSAGE # Launch the Sparse writer echo launch the swmr_sparse_writer - nice -n 20 ./swmr_sparse_writer $Nrecs_spa 2>&1 |tee swmr_writer.out & + nice -n 20 $bindir/swmr_sparse_writer $Nrecs_spa 2>&1 |tee swmr_writer.out & pid_writer=$! $DPRINT pid_writer=$pid_writer @@ -507,7 +513,7 @@ do echo launch $Nrdrs_spa swmr_sparse_readers while [ $n -lt $Nrdrs_spa ]; do # The sparse reader spits out a LOT of data so it's set to 'quiet' - ./swmr_sparse_reader -q $Nrecs_spa 2>&1 |tee swmr_reader.out.$n & + $bindir/swmr_sparse_reader -q $Nrecs_spa 2>&1 |tee swmr_reader.out.$n & pid_readers="$pid_readers $!" n=`expr $n + 1` done diff --git a/test/testvdsswmr.sh.in b/test/testvdsswmr.sh.in index 9673aa3..5399903 100644 --- a/test/testvdsswmr.sh.in +++ b/test/testvdsswmr.sh.in @@ -17,6 +17,7 @@ # Dana Robinson, November 2015 srcdir=@srcdir@ +bindir=@bindir@ ############################################################################### ## test parameters @@ -30,7 +31,7 @@ nerrors=0 ## definitions for message file to coordinate test runs ############################################################################### WRITER_MESSAGE=SWMR_WRITER_MESSAGE # The message file created by writer that the open is complete - # This should be the same as the define in "./swmr_common.h" + # This should be the same as the define in "$bindir/swmr_common.h" MESSAGE_TIMEOUT=300 # Message timeout length in secs # This should be the same as the define in "./h5test.h" @@ -83,9 +84,14 @@ if test -z "$srcdir"; then srcdir=. fi +# If the bindir directory is not set just use current (.). +if test -z "$bindir"; then + bindir=. +fi + # Check to see if the VFD specified by the HDF5_DRIVER environment variable # supports SWMR. -./swmr_check_compat_vfd +$bindir/swmr_check_compat_vfd rc=$? if [ $rc -ne 0 ] ; then echo @@ -148,7 +154,7 @@ echo "########################################################################## # Launch the file generator echo launch the generator -./vds_swmr_gen +$bindir/vds_swmr_gen if test $? -ne 0; then echo generator had error nerrors=`expr $nerrors + 1` @@ -166,7 +172,7 @@ echo "launch the $Nwriters SWMR VDS writers (1 per source)" pid_writers="" n=0 while [ $n -lt $Nwriters ]; do - ./vds_swmr_writer $n & + $bindir/vds_swmr_writer $n & pid_writers="$pid_writers $!" n=`expr $n + 1` done @@ -181,7 +187,7 @@ echo launch $Nreaders SWMR readers pid_readers="" n=0 while [ $n -lt $Nreaders ]; do - ./vds_swmr_reader & + $bindir/vds_swmr_reader & pid_readers="$pid_readers $!" n=`expr $n + 1` done -- cgit v0.12 From de60f08b0df0abdcac1c6959b393a6862dadd4ce Mon Sep 17 00:00:00 2001 From: Jan-Willem Blokland Date: Thu, 19 Aug 2021 23:45:43 +0200 Subject: CMake: (fix) ph5example test (#935) Ensure that the used number of processes times an integer number is equal to SPACE1_DIM1 and SPACE_DIM2. These variables are equal to 24 and are defined in ph5example.c. If this is not the case the ph5example test will fail. --- examples/CMakeTests.cmake | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/examples/CMakeTests.cmake b/examples/CMakeTests.cmake index b422078..70142c8 100644 --- a/examples/CMakeTests.cmake +++ b/examples/CMakeTests.cmake @@ -98,11 +98,14 @@ endif () ### Windows pops up a modal permission dialog on this test if (H5_HAVE_PARALLEL AND HDF5_TEST_PARALLEL AND NOT WIN32) + # Ensure that 24 is a multiple of the number of processes. + # The number 24 corresponds to SPACE1_DIM1 and SPACE1_DIM2 defined in ph5example.c + math(EXPR NUMPROCS "24 / ((24 + ${MPIEXEC_MAX_NUMPROCS} - 1) / ${MPIEXEC_MAX_NUMPROCS})") if (HDF5_ENABLE_USING_MEMCHECKER) - add_test (NAME MPI_TEST_EXAMPLES-ph5example COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) + add_test (NAME MPI_TEST_EXAMPLES-ph5example COMMAND ${MPIEXEC_EXECUTABLE} ${MPIEXEC_NUMPROC_FLAG} ${NUMPROCS} ${MPIEXEC_PREFLAGS} $ ${MPIEXEC_POSTFLAGS}) else () add_test (NAME MPI_TEST_EXAMPLES-ph5example COMMAND "${CMAKE_COMMAND}" - -D "TEST_PROGRAM=${MPIEXEC_EXECUTABLE};${MPIEXEC_NUMPROC_FLAG};${MPIEXEC_MAX_NUMPROCS};${MPIEXEC_PREFLAGS};$;${MPIEXEC_POSTFLAGS}" + -D "TEST_PROGRAM=${MPIEXEC_EXECUTABLE};${MPIEXEC_NUMPROC_FLAG};${NUMPROCS};${MPIEXEC_PREFLAGS};$;${MPIEXEC_POSTFLAGS}" -D "TEST_ARGS:STRING=" -D "TEST_EXPECT=0" -D "TEST_OUTPUT=ph5example.out" -- cgit v0.12 From 09283ffbcb921efa89a7a1ca728f1d4b7064df8e Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Fri, 20 Aug 2021 16:32:15 -0400 Subject: Get class name null str (#899) * Fixed MSVC compile error in C++23 mode Error message from MSVC with C++23 enabled: error C2440: 'return': cannot convert from 'int' to 'std::basic_string' * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- c++/src/H5PropList.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/c++/src/H5PropList.cpp b/c++/src/H5PropList.cpp index 7dca716..7ee8395 100644 --- a/c++/src/H5PropList.cpp +++ b/c++/src/H5PropList.cpp @@ -543,7 +543,7 @@ PropList::getPropSize(const H5std_string &name) const // Function: PropList::getClassName ///\brief Return the name of a generic property list class. ///\return A string containing the class name, if success, otherwise, -/// a NULL string. +/// an empty string. // Programmer: Binh-Minh Ribler - April, 2004 //-------------------------------------------------------------------------- H5std_string @@ -557,8 +557,9 @@ PropList::getClassName() const return (class_name); } else - return 0; + return ""; } + //-------------------------------------------------------------------------- // Function: PropList::getNumProps ///\brief Returns the number of properties in this property list or class. -- cgit v0.12 From b38bc071e956bec607b180a7d3f8217197393f02 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Fri, 20 Aug 2021 15:49:53 -0500 Subject: Remove incorrect compiler check for windows clang sanitizer (#942) --- config/sanitizer/sanitizers.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/sanitizer/sanitizers.cmake b/config/sanitizer/sanitizers.cmake index 22b9e3c..58c4050 100644 --- a/config/sanitizer/sanitizers.cmake +++ b/config/sanitizer/sanitizers.cmake @@ -80,7 +80,7 @@ if(USE_SANITIZER) message( FATAL_ERROR "Unsupported value of USE_SANITIZER: ${USE_SANITIZER}") endif() - elseif(MSVC AND NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 19.29) + elseif(MSVC) if(USE_SANITIZER MATCHES "([Aa]ddress)") message(STATUS "Building with Address sanitizer") append("/fsanitize=address" CMAKE_C_SANITIZER_FLAGS CMAKE_CXX_SANITIZER_FLAGS) -- cgit v0.12 From b5f5c59f297be19096051282068cdcc14275eb34 Mon Sep 17 00:00:00 2001 From: Ben Boeckel Date: Mon, 23 Aug 2021 13:36:21 +0000 Subject: cmake: simplify try_run result comparisons (#950) Instead of looking at `MATCHES 0` which runs a regex and returns a false positive for any result such as `10`, check if they are equal to `0`. Also quote `0` to avoid looking it up as a variable (which was left behind prior to the fix to use `${RETURN_VAR}` in the `c_run` macro). --- config/cmake/ConfigureChecks.cmake | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index 4db3ad4..263cedf 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -149,7 +149,7 @@ if (NOT WINDOWS) OUTPUT_VARIABLE OUTPUT ) if (TEST_DIRECT_VFD_WORKS_COMPILE) - if (TEST_DIRECT_VFD_WORKS_RUN MATCHES 0) + if (TEST_DIRECT_VFD_WORKS_RUN EQUAL "0") HDF_FUNCTION_TEST (HAVE_DIRECT) set (CMAKE_REQUIRED_DEFINITIONS "${CMAKE_REQUIRED_DEFINITIONS} -D_GNU_SOURCE") add_definitions ("-D_GNU_SOURCE") @@ -259,8 +259,8 @@ macro (C_RUN FUNCTION_NAME SOURCE_CODE RETURN_VAR) message (VERBOSE "* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * ") endif () - if (${COMPILE_RESULT_VAR}) - if (${RUN_RESULT_VAR} MATCHES 0) + if (COMPILE_RESULT_VAR) + if (RUN_RESULT_VAR EQUAL "0") set (${RETURN_VAR} 1 CACHE INTERNAL "Have C function ${FUNCTION_NAME}") if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") message (VERBOSE "Testing C ${FUNCTION_NAME} - OK") @@ -318,7 +318,7 @@ message (STATUS "Testing maximum decimal precision for C - ${PROG_OUTPUT4}") list (GET PROG_OUTPUT4 0 H5_LDBL_DIG) list (GET PROG_OUTPUT4 1 H5_FLT128_DIG) -if (${HDF_PREFIX}_SIZEOF___FLOAT128 EQUAL 0 OR FLT128_DIG EQUAL 0) +if (${HDF_PREFIX}_SIZEOF___FLOAT128 EQUAL "0" OR FLT128_DIG EQUAL "0") set (${HDF_PREFIX}_HAVE_FLOAT128 0) set (${HDF_PREFIX}_SIZEOF___FLOAT128 0) set (_PAC_C_MAX_REAL_PRECISION ${H5_LDBL_DIG}) @@ -344,7 +344,7 @@ macro (H5ConversionTests TEST msg) OUTPUT_VARIABLE OUTPUT ) if (${TEST}_COMPILE) - if (${TEST}_RUN MATCHES 0) + if (${TEST}_RUN EQUAL "0") set (${TEST} 1 CACHE INTERNAL ${msg}) if (CMAKE_VERSION VERSION_GREATER_EQUAL "3.15.0") message (VERBOSE "${msg}... yes") -- cgit v0.12 From f6c49fe8911a39810b236f7babf09a70fcc76c7b Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Mon, 23 Aug 2021 17:14:53 -0400 Subject: More clang tidy (#908) * Pacify clang-analyzer-unix.cstring.NullArg * Apply some bugprone-suspicious-string-compare * Apply some readability-simplify-boolean-expr * Apply some readability-make-member-function-const * Apple some bugprone-macro-parentheses * Changed an f suffix to L for `long double` * Applied some readability-uppercase-literal-suffix automatically * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- c++/src/H5DataType.cpp | 5 +-- c++/src/H5IdComponent.cpp | 5 +-- c++/test/tattr.cpp | 18 +++++----- c++/test/ttypes.cpp | 2 +- hl/c++/src/H5PacketTable.cpp | 25 ++++++-------- hl/c++/src/H5PacketTable.h | 20 +++++------ hl/c++/test/ptableTest.cpp | 2 +- hl/examples/ex_table_01.c | 8 ++--- hl/examples/ex_table_02.c | 10 +++--- hl/examples/ex_table_03.c | 6 ++-- hl/examples/ex_table_04.c | 10 +++--- hl/examples/ex_table_05.c | 4 +-- hl/examples/ex_table_06.c | 2 +- hl/examples/ex_table_07.c | 10 +++--- hl/examples/ex_table_08.c | 10 +++--- hl/examples/ex_table_09.c | 10 +++--- hl/examples/ex_table_10.c | 8 ++--- hl/examples/ex_table_11.c | 10 +++--- hl/examples/ex_table_12.c | 10 +++--- hl/test/gen_test_ds.c | 2 +- hl/test/test_packet.c | 8 ++--- hl/test/test_table.c | 6 ++-- src/H5Dchunk.c | 4 +-- src/H5Fefc.c | 8 ++--- src/H5SL.c | 4 +-- src/H5SM.c | 2 +- src/H5T.c | 8 ++--- src/H5Zscaleoffset.c | 58 ++++++++++++++++---------------- test/big.c | 4 +-- test/cmpd_dset.c | 4 +-- test/dsets.c | 10 +++--- test/dt_arith.c | 8 ++--- test/dtransform.c | 70 +++++++++++++++++++-------------------- test/dtypes.c | 10 +++--- test/gheap.c | 2 +- test/pool.c | 2 +- test/set_extent.c | 2 +- test/tconfig.c | 5 +-- test/testhdf5.h | 2 +- test/tfile.c | 2 +- test/tmeta.c | 4 +-- test/tsohm.c | 2 +- test/tunicode.c | 2 +- test/vds.c | 12 +++---- tools/lib/h5diff_array.c | 8 ++--- tools/src/h5perf/sio_engine.c | 2 +- tools/src/h5perf/sio_perf.c | 2 +- tools/src/misc/h5clear.c | 2 +- tools/test/h5dump/h5dumpgentest.c | 4 +-- tools/test/h5repack/h5repacktst.c | 4 +-- tools/test/perform/chunk_cache.c | 6 ++-- tools/test/perform/zip_perf.c | 2 +- 52 files changed, 219 insertions(+), 227 deletions(-) diff --git a/c++/src/H5DataType.cpp b/c++/src/H5DataType.cpp index b00a526..af58a90 100644 --- a/c++/src/H5DataType.cpp +++ b/c++/src/H5DataType.cpp @@ -338,10 +338,7 @@ DataType::encode() bool DataType::hasBinaryDesc() const { - if (encoded_buf != NULL) - return true; - else - return false; + return encoded_buf != NULL; } //-------------------------------------------------------------------------- diff --git a/c++/src/H5IdComponent.cpp b/c++/src/H5IdComponent.cpp index d546d32..e1fc687 100644 --- a/c++/src/H5IdComponent.cpp +++ b/c++/src/H5IdComponent.cpp @@ -432,10 +432,7 @@ IdComponent::p_valid_id(const hid_t obj_id) return false; H5I_type_t id_type = H5Iget_type(obj_id); - if (id_type <= H5I_BADID || id_type >= H5I_NTYPES) - return false; - else - return true; + return (id_type > H5I_BADID && id_type < H5I_NTYPES); } // Notes about IdComponent::id diff --git a/c++/test/tattr.cpp b/c++/test/tattr.cpp index 35db334..26699d2 100644 --- a/c++/test/tattr.cpp +++ b/c++/test/tattr.cpp @@ -285,7 +285,7 @@ test_attr_getname() // Check for existence of attribute FATTR1_NAME bool attr_exists = fid1.attrExists(FATTR1_NAME); - if (attr_exists == false) + if (!attr_exists) throw InvalidActionException("H5File::attrExists", "Attribute should exist but does not"); // Open attribute @@ -361,7 +361,7 @@ test_attr_getname() // Check for existence of attribute attr_exists = dataset.attrExists(ATTR1_NAME); - if (attr_exists == false) + if (!attr_exists) throw InvalidActionException("H5File::attrExists", "Attribute should exist but does not"); // Open attribute @@ -410,7 +410,7 @@ test_attr_rename() // Check for existence of attribute bool attr_exists = fid1.attrExists(FATTR1_NAME); - if (attr_exists == false) + if (!attr_exists) throw InvalidActionException("H5File::attrExists", "Attribute should exist but does not"); // Change attribute name @@ -436,7 +436,7 @@ test_attr_rename() // Check for existence of attribute attr_exists = dataset.attrExists(ATTR1_NAME); - if (attr_exists == false) + if (!attr_exists) throw InvalidActionException("H5File::attrExists", "Attribute should exist but does not"); // Change attribute name @@ -464,7 +464,7 @@ test_attr_rename() // Check for existence of second attribute attr_exists = dataset.attrExists(ATTR2_NAME); - if (attr_exists == false) + if (!attr_exists) throw InvalidActionException("H5File::attrExists", "Attribute should exist but does not"); // Open the second attribute @@ -492,7 +492,7 @@ test_attr_rename() // Check for existence of attribute after renaming attr_exists = dataset.attrExists(ATTR1_NAME); - if (attr_exists == false) + if (!attr_exists) throw InvalidActionException("H5File::attrExists", "Attribute should exist but does not"); PASSED(); @@ -1681,13 +1681,13 @@ test_attr_exists() // Check for existence of attribute bool attr_exists = fid1.attrExists(ATTR1_FL_STR_NAME); - if (attr_exists == false) + if (!attr_exists) throw InvalidActionException("H5File::attrExists", "fid1, ATTR1_FL_STR_NAMEAttribute should exist but does not"); // Check for existence of attribute attr_exists = fid1.attrExists(FATTR1_NAME); - if (attr_exists == false) + if (!attr_exists) throw InvalidActionException("H5File::attrExists", "fid1,FATTR2_NAMEAttribute should exist but does not"); @@ -1696,7 +1696,7 @@ test_attr_exists() // Check for existence of attribute attr_exists = group.attrExists(ATTR2_NAME); - if (attr_exists == false) + if (!attr_exists) throw InvalidActionException("H5File::attrExists", "group, ATTR2_NAMEAttribute should exist but does not"); diff --git a/c++/test/ttypes.cpp b/c++/test/ttypes.cpp index 1cedef6..69c93d1 100644 --- a/c++/test/ttypes.cpp +++ b/c++/test/ttypes.cpp @@ -699,7 +699,7 @@ test_named() } // Check that it is committed. - if (itype.committed() == false) + if (!itype.committed()) cerr << "IntType::committed() returned false" << endl; // We should not be able to modify a type after it has been committed. diff --git a/hl/c++/src/H5PacketTable.cpp b/hl/c++/src/H5PacketTable.cpp index 2908626..086c702 100644 --- a/hl/c++/src/H5PacketTable.cpp +++ b/hl/c++/src/H5PacketTable.cpp @@ -62,12 +62,9 @@ PacketTable::~PacketTable() * any trouble making or opening the packet table. */ bool -PacketTable::IsValid() +PacketTable::IsValid() const { - if (H5PTis_valid(table_id) == 0) - return true; - else - return false; + return H5PTis_valid(table_id) == 0; } /* IsVariableLength @@ -75,7 +72,7 @@ PacketTable::IsValid() * and 0, otherwise. Returns -1 if the table is invalid (not open). */ int -PacketTable::IsVariableLength() +PacketTable::IsVariableLength() const { return H5PTis_varlen(table_id); } @@ -84,7 +81,7 @@ PacketTable::IsVariableLength() * Sets the index to point to the first packet in the packet table */ void -PacketTable::ResetIndex() +PacketTable::ResetIndex() const { H5PTcreate_index(table_id); } @@ -94,7 +91,7 @@ PacketTable::ResetIndex() * Returns 0 on success, negative on failure (if index is out of bounds) */ int -PacketTable::SetIndex(hsize_t index) +PacketTable::SetIndex(hsize_t index) const { return H5PTset_index(table_id, index); } @@ -104,7 +101,7 @@ PacketTable::SetIndex(hsize_t index) * Returns 0 on success, negative on failure (if index is out of bounds) */ hsize_t -PacketTable::GetIndex(int &error) +PacketTable::GetIndex(int &error) const { hsize_t index; @@ -121,7 +118,7 @@ PacketTable::GetIndex(int &error) * error is set to negative. */ hsize_t -PacketTable::GetPacketCount(int &error) +PacketTable::GetPacketCount(int &error) const { hsize_t npackets; @@ -133,7 +130,7 @@ PacketTable::GetPacketCount(int &error) * Returns the identifier of the packet table */ hid_t -PacketTable::GetTableId() +PacketTable::GetTableId() const { return table_id; } @@ -145,7 +142,7 @@ PacketTable::GetTableId() * the desired functionality cannot be performed via the packet table ID. */ hid_t -PacketTable::GetDatatype() +PacketTable::GetDatatype() const { return H5PTget_type(table_id); } @@ -157,7 +154,7 @@ PacketTable::GetDatatype() * the desired functionality cannot be performed via the packet table ID. */ hid_t -PacketTable::GetDataset() +PacketTable::GetDataset() const { return H5PTget_dataset(table_id); } @@ -169,7 +166,7 @@ PacketTable::GetDataset() * Returns 0 on success, negative on error. */ int -PacketTable::FreeBuff(size_t numStructs, hvl_t *buffer) +PacketTable::FreeBuff(size_t numStructs, hvl_t *buffer) const { return H5PTfree_vlen_buff(table_id, numStructs, buffer); } diff --git a/hl/c++/src/H5PacketTable.h b/hl/c++/src/H5PacketTable.h index eae66f1..c1f1eb5 100644 --- a/hl/c++/src/H5PacketTable.h +++ b/hl/c++/src/H5PacketTable.h @@ -54,39 +54,39 @@ class H5_HLCPPDLL PacketTable { * Use this after the constructor to ensure HDF did not have * any trouble making or opening the packet table. */ - bool IsValid(); + bool IsValid() const; /* IsVariableLength * Return 1 if this packet table uses variable-length datatype, * return 0 if it is Fixed Length. Returns -1 if the table is * invalid (not open). */ - int IsVariableLength(); + int IsVariableLength() const; /* ResetIndex * Sets the "current packet" index to point to the first packet in the * packet table */ - void ResetIndex(); + void ResetIndex() const; /* SetIndex * Sets the current packet to point to the packet specified by index. * Returns 0 on success, negative on failure (if index is out of bounds) */ - int SetIndex(hsize_t index); + int SetIndex(hsize_t index) const; /* GetIndex * Returns the position of the current packet. * On failure, returns 0 and error is set to negative. */ - hsize_t GetIndex(int &error); + hsize_t GetIndex(int &error) const; /* GetPacketCount * Returns the number of packets in the packet table. Error * is set to 0 on success. On failure, returns 0 and * error is set to negative. */ - hsize_t GetPacketCount(int &error); + hsize_t GetPacketCount(int &error) const; hsize_t GetPacketCount() @@ -98,7 +98,7 @@ class H5_HLCPPDLL PacketTable { /* GetTableId * Returns the identifier of the packet table. */ - hid_t GetTableId(); + hid_t GetTableId() const; /* GetDatatype * Returns the datatype identifier used by the packet table, on success, @@ -106,7 +106,7 @@ class H5_HLCPPDLL PacketTable { * Note: it is best to avoid using this identifier in applications, unless * the desired functionality cannot be performed via the packet table ID. */ - hid_t GetDatatype(); + hid_t GetDatatype() const; /* GetDataset * Returns the dataset identifier associated with the packet table, on @@ -114,7 +114,7 @@ class H5_HLCPPDLL PacketTable { * Note: it is best to avoid using this identifier in applications, unless * the desired functionality cannot be performed via the packet table ID. */ - hid_t GetDataset(); + hid_t GetDataset() const; /* FreeBuff * Frees the buffers created when variable-length packets are read. @@ -122,7 +122,7 @@ class H5_HLCPPDLL PacketTable { * location in memory. * Returns 0 on success, negative on error. */ - int FreeBuff(size_t numStructs, hvl_t *buffer); + int FreeBuff(size_t numStructs, hvl_t *buffer) const; protected: hid_t table_id; diff --git a/hl/c++/test/ptableTest.cpp b/hl/c++/test/ptableTest.cpp index 6deb24d..ab49303 100644 --- a/hl/c++/test/ptableTest.cpp +++ b/hl/c++/test/ptableTest.cpp @@ -620,7 +620,7 @@ TestHDFFV_9758() for (hsize_t i = 0; i < NUM_PACKETS; i++) { s1[i].a = static_cast(i); - s1[i].b = 1.0f * static_cast(i * i); + s1[i].b = 1.0F * static_cast(i * i); s1[i].c = 1.0 / static_cast(i + 1); HDsprintf(s1[i].d, "string%" PRIuHSIZE "", i); s1[i].e = static_cast(100 + i); diff --git a/hl/examples/ex_table_01.c b/hl/examples/ex_table_01.c index 8635acf..3294ef4 100644 --- a/hl/examples/ex_table_01.c +++ b/hl/examples/ex_table_01.c @@ -50,10 +50,10 @@ main(void) sizeof(dst_buf[0].pressure), sizeof(dst_buf[0].temperature)}; /* Define an array of Particles */ - Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, - {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, - {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, - {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; + Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, + {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, + {"four", 40, 40, 4.0F, 40.0}, {"five", 50, 50, 5.0F, 50.0}, + {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; /* Define field information */ const char *field_names[NFIELDS] = {"Name", "Latitude", "Longitude", "Pressure", "Temperature"}; diff --git a/hl/examples/ex_table_02.c b/hl/examples/ex_table_02.c index 9c476b3..fb2cad6 100644 --- a/hl/examples/ex_table_02.c +++ b/hl/examples/ex_table_02.c @@ -42,10 +42,10 @@ main(void) Particle dst_buf[NRECORDS + NRECORDS_ADD]; /* Define an array of Particles */ - Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, - {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, - {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, - {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; + Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, + {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, + {"four", 40, 40, 4.0F, 40.0}, {"five", 50, 50, 5.0F, 50.0}, + {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; /* Calculate the size and the offsets of our struct members in memory */ size_t dst_size = sizeof(Particle); @@ -66,7 +66,7 @@ main(void) int i; /* Append particles */ - Particle particle_in[NRECORDS_ADD] = {{"eight", 80, 80, 8.0f, 80.0}, {"nine", 90, 90, 9.0f, 90.0}}; + Particle particle_in[NRECORDS_ADD] = {{"eight", 80, 80, 8.0F, 80.0}, {"nine", 90, 90, 9.0F, 90.0}}; /* Initialize the field field_type */ string_type = H5Tcopy(H5T_C_S1); diff --git a/hl/examples/ex_table_03.c b/hl/examples/ex_table_03.c index 31cf970..3caa45e 100644 --- a/hl/examples/ex_table_03.c +++ b/hl/examples/ex_table_03.c @@ -46,14 +46,14 @@ main(void) size_t dst_offset[NFIELDS] = {HOFFSET(Particle, name), HOFFSET(Particle, lati), HOFFSET(Particle, longi), HOFFSET(Particle, pressure), HOFFSET(Particle, temperature)}; - Particle p = {"zero", 0, 0, 0.0f, 0.0}; + Particle p = {"zero", 0, 0, 0.0F, 0.0}; size_t dst_sizes[NFIELDS] = {sizeof(p.name), sizeof(p.lati), sizeof(p.longi), sizeof(p.pressure), sizeof(p.temperature)}; /* Define field information */ const char *field_names[NFIELDS] = {"Name", "Latitude", "Longitude", "Pressure", "Temperature"}; /* Fill value particle */ - Particle fill_data[1] = {{"no data", -1, -1, -99.0f, -99.0}}; + Particle fill_data[1] = {{"no data", -1, -1, -99.0F, -99.0}}; hid_t field_type[NFIELDS]; hid_t string_type; hid_t file_id; @@ -63,7 +63,7 @@ main(void) int i; /* Define 2 new particles to write */ - Particle particle_in[NRECORDS_WRITE] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}}; + Particle particle_in[NRECORDS_WRITE] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}}; /* Initialize the field field_type */ string_type = H5Tcopy(H5T_C_S1); diff --git a/hl/examples/ex_table_04.c b/hl/examples/ex_table_04.c index 863fe15..9b39ef7 100644 --- a/hl/examples/ex_table_04.c +++ b/hl/examples/ex_table_04.c @@ -65,23 +65,23 @@ main(void) hid_t string_type; hid_t file_id; hsize_t chunk_size = 10; - Particle fill_data[1] = {{"no data", -1, -1, -99.0f, -99.0}}; /* Fill value particle */ + Particle fill_data[1] = {{"no data", -1, -1, -99.0F, -99.0}}; /* Fill value particle */ hsize_t start; /* Record to start reading/writing */ hsize_t nrecords; /* Number of records to read/write */ int compress = 0; int i; Particle *p_data = NULL; /* Initially no data */ float pressure_in[NRECORDS_ADD] = /* Define new values for the field "Pressure" */ - {0.0f, 1.0f, 2.0f}; + {0.0F, 1.0F, 2.0F}; Position position_in[NRECORDS_ADD] = {/* Define new values for "Latitude,Longitude" */ {0, 0}, {10, 10}, {20, 20}}; NamePressure namepre_in[NRECORDS_ADD] = /* Define new values for "Name,Pressure" */ { - {"zero", 0.0f}, - {"one", 1.0f}, - {"two", 2.0f}, + {"zero", 0.0F}, + {"one", 1.0F}, + {"two", 2.0F}, }; size_t field_sizes_pos[2] = {sizeof(position_in[0].longi), sizeof(position_in[0].lati)}; size_t field_sizes_pre[1] = {sizeof(namepre_in[0].pressure)}; diff --git a/hl/examples/ex_table_05.c b/hl/examples/ex_table_05.c index 337bfb6..cac406b 100644 --- a/hl/examples/ex_table_05.c +++ b/hl/examples/ex_table_05.c @@ -64,7 +64,7 @@ main(void) hid_t string_type; hid_t file_id; hsize_t chunk_size = 10; - Particle fill_data[1] = {{"no data", -1, -1, -99.0f, -99.0}}; /* Fill value particle */ + Particle fill_data[1] = {{"no data", -1, -1, -99.0F, -99.0}}; /* Fill value particle */ int compress = 0; hsize_t nfields; hsize_t start; /* Record to start reading/writing */ @@ -72,7 +72,7 @@ main(void) int i; /* Define new values for the field "Pressure" */ - float pressure_in[NRECORDS_ADD] = {0.0f, 1.0f, 2.0f}; + float pressure_in[NRECORDS_ADD] = {0.0F, 1.0F, 2.0F}; int field_index_pre[1] = {3}; int field_index_pos[2] = {1, 2}; diff --git a/hl/examples/ex_table_06.c b/hl/examples/ex_table_06.c index f6b67c8..cc1bc01 100644 --- a/hl/examples/ex_table_06.c +++ b/hl/examples/ex_table_06.c @@ -49,7 +49,7 @@ main(void) hid_t string_type; hid_t file_id; hsize_t chunk_size = 10; - Particle fill_data[1] = {{"no data", -1, -1, -99.0f, -99.0}}; /* Fill value particle */ + Particle fill_data[1] = {{"no data", -1, -1, -99.0F, -99.0}}; /* Fill value particle */ int compress = 0; hsize_t nfields_out; hsize_t nrecords_out; diff --git a/hl/examples/ex_table_07.c b/hl/examples/ex_table_07.c index ab36613..e05a681 100644 --- a/hl/examples/ex_table_07.c +++ b/hl/examples/ex_table_07.c @@ -44,10 +44,10 @@ main(void) HOFFSET(Particle, pressure), HOFFSET(Particle, temperature)}; /* Define an array of Particles */ - Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, - {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, - {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, - {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; + Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, + {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, + {"four", 40, 40, 4.0F, 40.0}, {"five", 50, 50, 5.0F, 50.0}, + {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; const char *field_names[NFIELDS] = /* Define field information */ {"Name", "Latitude", "Longitude", "Pressure", "Temperature"}; @@ -56,7 +56,7 @@ main(void) hid_t file_id; hsize_t chunk_size = 10; int compress = 0; - Particle fill_data[1] = {{"no data", -1, -1, -99.0f, -99.0}}; + Particle fill_data[1] = {{"no data", -1, -1, -99.0F, -99.0}}; hsize_t start; /* Record to start reading */ hsize_t nrecords; /* Number of records to insert/delete */ hsize_t nfields_out; diff --git a/hl/examples/ex_table_08.c b/hl/examples/ex_table_08.c index 5d447dd..1063172 100644 --- a/hl/examples/ex_table_08.c +++ b/hl/examples/ex_table_08.c @@ -41,10 +41,10 @@ main(void) Particle dst_buf[NRECORDS + NRECORDS_INS]; /* Define an array of Particles */ - Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, - {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, - {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, - {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; + Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, + {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, + {"four", 40, 40, 4.0F, 40.0}, {"five", 50, 50, 5.0F, 50.0}, + {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; /* Calculate the size and the offsets of our struct members in memory */ size_t dst_size = sizeof(Particle); @@ -54,7 +54,7 @@ main(void) sizeof(p_data[0].pressure), sizeof(p_data[0].temperature)}; /* Define an array of Particles to insert */ - Particle p_data_insert[NRECORDS_INS] = {{"new", 30, 30, 3.0f, 30.0}, {"new", 40, 40, 4.0f, 40.0}}; + Particle p_data_insert[NRECORDS_INS] = {{"new", 30, 30, 3.0F, 30.0}, {"new", 40, 40, 4.0F, 40.0}}; /* Define field information */ const char *field_names[NFIELDS] = {"Name", "Latitude", "Longitude", "Pressure", "Temperature"}; diff --git a/hl/examples/ex_table_09.c b/hl/examples/ex_table_09.c index 381925f..171de2e 100644 --- a/hl/examples/ex_table_09.c +++ b/hl/examples/ex_table_09.c @@ -49,10 +49,10 @@ main(void) sizeof(dst_buf[0].pressure), sizeof(dst_buf[0].temperature)}; /* Define an array of Particles */ - Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, - {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, - {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, - {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; + Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, + {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, + {"four", 40, 40, 4.0F, 40.0}, {"five", 50, 50, 5.0F, 50.0}, + {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; /* Define field information */ const char *field_names[NFIELDS] = {"Name", "Latitude", "Longitude", "Pressure", "Temperature"}; @@ -61,7 +61,7 @@ main(void) hid_t file_id; hsize_t chunk_size = 10; int compress = 0; - Particle fill_data[1] = {{"no data", -1, -1, -99.0f, -99.0}}; /* Fill value particle */ + Particle fill_data[1] = {{"no data", -1, -1, -99.0F, -99.0}}; /* Fill value particle */ hsize_t start1; /* Record to start reading from 1st table */ hsize_t nrecords; /* Number of records to insert */ hsize_t start2; /* Record to start writing in 2nd table */ diff --git a/hl/examples/ex_table_10.c b/hl/examples/ex_table_10.c index 4ba5d64..c974e84 100644 --- a/hl/examples/ex_table_10.c +++ b/hl/examples/ex_table_10.c @@ -40,10 +40,10 @@ main(void) } Particle; /* Define an array of Particles */ - Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, - {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, - {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, - {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; + Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, + {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, + {"four", 40, 40, 4.0F, 40.0}, {"five", 50, 50, 5.0F, 50.0}, + {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; Particle dst_buf[2 * NRECORDS]; /* Calculate the size and the offsets of our struct members in memory */ diff --git a/hl/examples/ex_table_11.c b/hl/examples/ex_table_11.c index 9bf3927..6b38b95 100644 --- a/hl/examples/ex_table_11.c +++ b/hl/examples/ex_table_11.c @@ -38,10 +38,10 @@ main(void) } Particle1; /* Define an array of Particles */ - Particle1 p_data[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, - {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, - {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, - {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; + Particle1 p_data[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, + {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, + {"four", 40, 40, 4.0F, 40.0}, {"five", 50, 50, 5.0F, 50.0}, + {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; /* Calculate the size and the offsets of our struct members in memory */ size_t dst_size1 = sizeof(Particle1); @@ -56,7 +56,7 @@ main(void) hid_t file_id; hsize_t chunk_size = 10; int compress = 0; - Particle1 fill_data[1] = {{"no data", -1, -1, -99.0f, -99.0}}; + Particle1 fill_data[1] = {{"no data", -1, -1, -99.0F, -99.0}}; int fill_data_new[1] = {-100}; hsize_t position; hsize_t nfields_out; diff --git a/hl/examples/ex_table_12.c b/hl/examples/ex_table_12.c index 3e7c27a..a1a2c54 100644 --- a/hl/examples/ex_table_12.c +++ b/hl/examples/ex_table_12.c @@ -44,10 +44,10 @@ main(void) HOFFSET(Particle, pressure), HOFFSET(Particle, temperature)}; /* Define an array of Particles */ - Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, - {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, - {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, - {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; + Particle p_data[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, + {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, + {"four", 40, 40, 4.0F, 40.0}, {"five", 50, 50, 5.0F, 50.0}, + {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; /* Define field information */ const char *field_names[NFIELDS] = {"Name", "Latitude", "Longitude", "Pressure", "Temperature"}; @@ -56,7 +56,7 @@ main(void) hid_t file_id; hsize_t chunk_size = 10; int compress = 0; - Particle fill_data[1] = {{"no data", -1, -1, -99.0f, -99.0}}; + Particle fill_data[1] = {{"no data", -1, -1, -99.0F, -99.0}}; hsize_t nfields_out; hsize_t nrecords_out; diff --git a/hl/test/gen_test_ds.c b/hl/test/gen_test_ds.c index 368c083..a56e6cf 100644 --- a/hl/test/gen_test_ds.c +++ b/hl/test/gen_test_ds.c @@ -75,7 +75,7 @@ main(int argc, char **argv) int nerrors = 0; char filename[65]; - if (argc < 2) { + if (argc < 2 || !argv[0] || !argv[1]) { HDprintf("Usage: gen_test [le | be]\n"); return 1; } diff --git a/hl/test/test_packet.c b/hl/test/test_packet.c index 311f36a..5f30d4b 100644 --- a/hl/test/test_packet.c +++ b/hl/test/test_packet.c @@ -45,10 +45,10 @@ typedef struct particle_t { * a static array of particles for writing and checking reads *------------------------------------------------------------------------- */ -static particle_t testPart[NRECORDS] = {{"zero", 0, 0, 0.0f, 0.0}, {"one", 10, 10, 1.0f, 10.0}, - {"two", 20, 20, 2.0f, 20.0}, {"three", 30, 30, 3.0f, 30.0}, - {"four", 40, 40, 4.0f, 40.0}, {"five", 50, 50, 5.0f, 50.0}, - {"six", 60, 60, 6.0f, 60.0}, {"seven", 70, 70, 7.0f, 70.0}}; +static particle_t testPart[NRECORDS] = {{"zero", 0, 0, 0.0F, 0.0}, {"one", 10, 10, 1.0F, 10.0}, + {"two", 20, 20, 2.0F, 20.0}, {"three", 30, 30, 3.0F, 30.0}, + {"Four", 40, 40, 4.0F, 40.0}, {"Five", 50, 50, 5.0F, 50.0}, + {"six", 60, 60, 6.0F, 60.0}, {"seven", 70, 70, 7.0F, 70.0}}; /*------------------------------------------------------------------------- * function that compares one particle diff --git a/hl/test/test_table.c b/hl/test/test_table.c index 531f778..6199e27 100644 --- a/hl/test/test_table.c +++ b/hl/test/test_table.c @@ -245,7 +245,7 @@ test_table(hid_t fid, int do_write) {"six", 60, 6.0, 60.0, 60}, {"seven", 70, 7.0, 70.0, 70}}; /* buffers for the field "Pressure" and "New_field" */ - float pressure_in[NRECORDS] = {0.0f, 1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f}; + float pressure_in[NRECORDS] = {0.0F, 1.0F, 2.0F, 3.0F, 4.0F, 5.0F, 6.0F, 7.0F}; float pressure_out[NRECORDS]; int buf_new[NRECORDS] = {0, 1, 2, 3, 4, 5, 6, 7}; /* buffers for the fields "Latitude,Longitude" */ @@ -254,8 +254,8 @@ test_table(hid_t fid, int do_write) /* buffers for the fields "Name,Pressure" */ namepressure_t namepre_out[NRECORDS]; namepressure_t namepre_in[NRECORDS] = { - {"zero", 0.0f}, {"one", 1.0f}, {"two", 2.0f}, {"three", 3.0f}, - {"four", 4.0f}, {"five", 5.0f}, {"six", 6.0f}, {"seven", 7.0f}, + {"zero", 0.0F}, {"one", 1.0F}, {"two", 2.0F}, {"three", 3.0F}, + {"four", 4.0F}, {"five", 5.0F}, {"six", 6.0F}, {"seven", 7.0F}, }; /*------------------------------------------------------------------------- diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index ccae784..6600417 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -107,9 +107,9 @@ /*#define H5D_CHUNK_DEBUG */ /* Flags for the "edge_chunk_state" field below */ -#define H5D_RDCC_DISABLE_FILTERS 0x01u /* Disable filters on this chunk */ +#define H5D_RDCC_DISABLE_FILTERS 0x01U /* Disable filters on this chunk */ #define H5D_RDCC_NEWLY_DISABLED_FILTERS \ - 0x02u /* Filters have been disabled since \ + 0x02U /* Filters have been disabled since \ * the last flush */ /******************/ diff --git a/src/H5Fefc.c b/src/H5Fefc.c index 3f9a22f..2d4eff2 100644 --- a/src/H5Fefc.c +++ b/src/H5Fefc.c @@ -36,10 +36,10 @@ #include "H5Pprivate.h" /* Property lists */ /* Special values for the "tag" field below */ -#define H5F_EFC_TAG_DEFAULT -1 -#define H5F_EFC_TAG_LOCK -2 -#define H5F_EFC_TAG_CLOSE -3 -#define H5F_EFC_TAG_DONTCLOSE -4 +#define H5F_EFC_TAG_DEFAULT (-1) +#define H5F_EFC_TAG_LOCK (-2) +#define H5F_EFC_TAG_CLOSE (-3) +#define H5F_EFC_TAG_DONTCLOSE (-4) /* Structure for each entry in a file's external file cache */ typedef struct H5F_efc_ent_t { diff --git a/src/H5SL.c b/src/H5SL.c index 166019b..ba9721c 100644 --- a/src/H5SL.c +++ b/src/H5SL.c @@ -209,7 +209,7 @@ /* Check if we need to increase allocation of forward pointers */ \ if (LVL + 1 >= 1u << X->log_nalloc) { \ H5SL_node_t **_tmp; \ - HDassert(LVL + 1 == 1u << X->log_nalloc); \ + HDassert(LVL + 1 == 1U << X->log_nalloc); \ /* Double the amount of allocated space */ \ X->log_nalloc++; \ \ @@ -251,7 +251,7 @@ /* Check if we can reduce the allocation of forward pointers */ \ if (LVL <= 1u << (X->log_nalloc - 1)) { \ H5SL_node_t **_tmp; \ - HDassert(LVL == 1u << (X->log_nalloc - 1)); \ + HDassert(LVL == 1U << (X->log_nalloc - 1)); \ X->log_nalloc--; \ \ /* Allocate space for new forward pointers */ \ diff --git a/src/H5SM.c b/src/H5SM.c index 5f055bb..b09cb45 100644 --- a/src/H5SM.c +++ b/src/H5SM.c @@ -1079,7 +1079,7 @@ H5SM_try_share(H5F_t *f, H5O_t *open_oh, unsigned defer_flags, unsigned type_id, ssize_t index_num; htri_t tri_ret; #ifndef NDEBUG - unsigned deferred_type = -1u; + unsigned deferred_type = -1U; #endif htri_t ret_value = TRUE; diff --git a/src/H5T.c b/src/H5T.c index 1b0e8c9..19a3d39 100644 --- a/src/H5T.c +++ b/src/H5T.c @@ -290,7 +290,7 @@ #define H5T_INIT_TYPE_SET_SIZE(SIZE) \ { \ dt->shared->size = SIZE; \ - dt->shared->u.atomic.prec = 8 * SIZE; \ + dt->shared->u.atomic.prec = 8 * (SIZE); \ } #define H5T_INIT_TYPE_NOSET_SIZE(SIZE) \ @@ -327,7 +327,7 @@ H5_GLUE3(H5T_INIT_TYPE_, GUTS, _CORE) \ \ /* Register result */ \ - if ((GLOBAL = H5I_register(H5I_DATATYPE, dt, FALSE)) < 0) \ + if (((GLOBAL) = H5I_register(H5I_DATATYPE, dt, FALSE)) < 0) \ HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register datatype atom") \ } @@ -560,8 +560,8 @@ size_t H5T_NATIVE_UINT_FAST64_ALIGN_g = 0; /* Useful floating-point values for conversion routines */ /* (+/- Inf for all floating-point types) */ -float H5T_NATIVE_FLOAT_POS_INF_g = 0.0f; -float H5T_NATIVE_FLOAT_NEG_INF_g = 0.0f; +float H5T_NATIVE_FLOAT_POS_INF_g = 0.0F; +float H5T_NATIVE_FLOAT_NEG_INF_g = 0.0F; double H5T_NATIVE_DOUBLE_POS_INF_g = 0.0; double H5T_NATIVE_DOUBLE_NEG_INF_g = 0.0; diff --git a/src/H5Zscaleoffset.c b/src/H5Zscaleoffset.c index 2e1a474..a8a63bf 100644 --- a/src/H5Zscaleoffset.c +++ b/src/H5Zscaleoffset.c @@ -425,22 +425,22 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ #define H5Z_scaleoffset_check_3(i, type, pow_fun, round_fun, max, min, minbits, D_val) \ { \ if (sizeof(type) == sizeof(int)) { \ - if (round_fun(max * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)) > \ - pow_fun(2.0f, (type)(sizeof(int) * 8 - 1))) { \ + if (round_fun(max * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)) > \ + pow_fun(2.0F, (type)(sizeof(int) * 8 - 1))) { \ *minbits = sizeof(int) * 8; \ goto done; \ } \ } \ else if (sizeof(type) == sizeof(long)) { \ - if (round_fun(max * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)) > \ - pow_fun(2.0f, (type)(sizeof(long) * 8 - 1))) { \ + if (round_fun(max * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)) > \ + pow_fun(2.0F, (type)(sizeof(long) * 8 - 1))) { \ *minbits = sizeof(long) * 8; \ goto done; \ } \ } \ else if (sizeof(type) == sizeof(long long)) { \ - if (round_fun(max * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)) > \ - pow_fun(2.0f, (type)(sizeof(long long) * 8 - 1))) { \ + if (round_fun(max * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)) > \ + pow_fun(2.0F, (type)(sizeof(long long) * 8 - 1))) { \ *minbits = sizeof(long long) * 8; \ goto done; \ } \ @@ -530,27 +530,27 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ { \ if (sizeof(type) == sizeof(int)) \ for (i = 0; i < d_nelmts; i++) { \ - if (abs_fun(buf[i] - filval) < pow_fun(10.0f, (type)-D_val)) \ + if (abs_fun(buf[i] - filval) < pow_fun(10.0F, (type)-D_val)) \ *(int *)((void *)&buf[i]) = (int)(((unsigned int)1 << *minbits) - 1); \ else \ - *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - \ - min * pow_fun(10.0f, (type)D_val)); \ + *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - \ + min * pow_fun(10.0F, (type)D_val)); \ } \ else if (sizeof(type) == sizeof(long)) \ for (i = 0; i < d_nelmts; i++) { \ - if (abs_fun(buf[i] - filval) < pow_fun(10.0f, (type)-D_val)) \ + if (abs_fun(buf[i] - filval) < pow_fun(10.0F, (type)-D_val)) \ *(long *)((void *)&buf[i]) = (long)(((unsigned long)1 << *minbits) - 1); \ else \ - *(long *)((void *)&buf[i]) = lround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - \ - min * pow_fun(10.0f, (type)D_val)); \ + *(long *)((void *)&buf[i]) = lround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - \ + min * pow_fun(10.0F, (type)D_val)); \ } \ else if (sizeof(type) == sizeof(long long)) \ for (i = 0; i < d_nelmts; i++) { \ - if (abs_fun(buf[i] - filval) < pow_fun(10.0f, (type)-D_val)) \ + if (abs_fun(buf[i] - filval) < pow_fun(10.0F, (type)-D_val)) \ *(long long *)((void *)&buf[i]) = (long long)(((unsigned long long)1 << *minbits) - 1); \ else \ - *(long long *)((void *)&buf[i]) = llround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - \ - min * pow_fun(10.0f, (type)D_val)); \ + *(long long *)((void *)&buf[i]) = llround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - \ + min * pow_fun(10.0F, (type)D_val)); \ } \ else \ HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \ @@ -561,16 +561,16 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ { \ if (sizeof(type) == sizeof(int)) \ for (i = 0; i < d_nelmts; i++) \ - *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - \ - min * pow_fun(10.0f, (type)D_val)); \ + *(int *)((void *)&buf[i]) = (int)lround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - \ + min * pow_fun(10.0F, (type)D_val)); \ else if (sizeof(type) == sizeof(long)) \ for (i = 0; i < d_nelmts; i++) \ *(long *)((void *)&buf[i]) = \ - lround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)); \ + lround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)); \ else if (sizeof(type) == sizeof(long long)) \ for (i = 0; i < d_nelmts; i++) \ *(long long *)((void *)&buf[i]) = \ - llround_fun(buf[i] * pow_fun(10.0f, (type)D_val) - min * pow_fun(10.0f, (type)D_val)); \ + llround_fun(buf[i] * pow_fun(10.0F, (type)D_val) - min * pow_fun(10.0F, (type)D_val)); \ else \ HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \ } @@ -606,8 +606,8 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ H5Z_scaleoffset_get_filval_2(type, cd_values, filval) \ H5Z_scaleoffset_max_min_3(i, d_nelmts, buf, filval, max, min, D_val) \ H5Z_scaleoffset_check_3(i, type, pow_fun, round_fun, max, min, minbits, D_val) span = \ - (unsigned long long)(llround_fun(max * pow_fun(10.0f, (type)D_val) - \ - min * pow_fun(10.0f, (type)D_val)) + \ + (unsigned long long)(llround_fun(max * pow_fun(10.0F, (type)D_val) - \ + min * pow_fun(10.0F, (type)D_val)) + \ 1); \ *minbits = H5Z__scaleoffset_log2(span + 1); \ if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \ @@ -617,8 +617,8 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ else { /* fill value undefined */ \ H5Z_scaleoffset_max_min_2(i, d_nelmts, buf, max, min) \ H5Z_scaleoffset_check_3(i, type, pow_fun, round_fun, max, min, minbits, D_val) span = \ - (unsigned long long)(llround_fun(max * pow_fun(10.0f, (type)D_val) - \ - min * pow_fun(10.0f, (type)D_val)) + \ + (unsigned long long)(llround_fun(max * pow_fun(10.0F, (type)D_val) - \ + min * pow_fun(10.0F, (type)D_val)) + \ 1); \ *minbits = H5Z__scaleoffset_log2(span); \ if (*minbits != sizeof(type) * 8) /* change values if minbits != full precision */ \ @@ -685,19 +685,19 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ buf[i] = \ (type)((*(int *)((void *)&buf[i]) == (int)(((unsigned int)1 << minbits) - 1)) \ ? filval \ - : (type)(*(int *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \ + : (type)(*(int *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ else if (sizeof(type) == sizeof(long)) \ for (i = 0; i < d_nelmts; i++) \ buf[i] = \ (type)((*(long *)((void *)&buf[i]) == (long)(((unsigned long)1 << minbits) - 1)) \ ? filval \ - : (type)(*(long *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \ + : (type)(*(long *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ else if (sizeof(type) == sizeof(long long)) \ for (i = 0; i < d_nelmts; i++) \ buf[i] = (type)( \ (*(long long *)((void *)&buf[i]) == (long long)(((unsigned long long)1 << minbits) - 1)) \ ? filval \ - : (type)(*(long long *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \ + : (type)(*(long long *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ else \ HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \ } @@ -707,13 +707,13 @@ H5Z_class2_t H5Z_SCALEOFFSET[1] = {{ { \ if (sizeof(type) == sizeof(int)) \ for (i = 0; i < d_nelmts; i++) \ - buf[i] = ((type)(*(int *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \ + buf[i] = ((type)(*(int *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ else if (sizeof(type) == sizeof(long)) \ for (i = 0; i < d_nelmts; i++) \ - buf[i] = ((type)(*(long *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \ + buf[i] = ((type)(*(long *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ else if (sizeof(type) == sizeof(long long)) \ for (i = 0; i < d_nelmts; i++) \ - buf[i] = ((type)(*(long long *)((void *)&buf[i])) / pow_fun(10.0f, (type)D_val) + min); \ + buf[i] = ((type)(*(long long *)((void *)&buf[i])) / pow_fun(10.0F, (type)D_val) + min); \ else \ HGOTO_ERROR(H5E_PLINE, H5E_BADTYPE, FAIL, "cannot find matched integer dataype") \ } diff --git a/test/big.c b/test/big.c index 36fb27d..2bbae67 100644 --- a/test/big.c +++ b/test/big.c @@ -56,8 +56,8 @@ #define DNAME "big.data" #define WRT_N 50 -#define WRT_SIZE 4 * 1024 -#define FAMILY_SIZE 1024 * 1024 * 1024 +#define WRT_SIZE (4 * 1024) +#define FAMILY_SIZE (1024 * 1024 * 1024) #define GB (HDoff_t)0x40000000L diff --git a/test/cmpd_dset.c b/test/cmpd_dset.c index 4c3233d..8777096 100644 --- a/test/cmpd_dset.c +++ b/test/cmpd_dset.c @@ -110,8 +110,8 @@ typedef struct { long long r, s, t; } stype4; -#define NX 100u -#define NY 2000u +#define NX 100U +#define NY 2000U #define PACK_NMEMBS 100 /*------------------------------------------------------------------------- diff --git a/test/dsets.c b/test/dsets.c index 26322ad..922f370 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -1396,10 +1396,10 @@ test_conv_buffer(hid_t fid) cf->a[j][k][l] = 10 * (j + 1) + l + k; for (j = 0; j < DIM2; j++) - cf->b[j] = 100.0f * (float)(j + 1) + 0.01f * (float)j; + cf->b[j] = 100.0F * (float)(j + 1) + 0.01F * (float)j; for (j = 0; j < DIM3; j++) - cf->c[j] = 100.0f * (float)(j + 1) + 0.02f * (float)j; + cf->c[j] = 100.0F * (float)(j + 1) + 0.02F * (float)j; /* Create data space */ if ((space = H5Screate(H5S_SCALAR)) < 0) @@ -3324,8 +3324,8 @@ test_nbit_float(hid_t file) /* orig_data[] are initialized to be within the range that can be represented by * dataset datatype (no precision loss during datatype conversion) */ - float orig_data[2][5] = {{188384.0f, 19.103516f, -1.0831790e9f, -84.242188f, 5.2045898f}, - {-49140.0f, 2350.25f, -3.2110596e-1f, 6.4998865e-5f, -0.0f}}; + float orig_data[2][5] = {{188384.0F, 19.103516F, -1.0831790e9F, -84.242188F, 5.2045898F}, + {-49140.0F, 2350.25F, -3.2110596e-1F, 6.4998865e-5F, -0.0F}}; float new_data[2][5]; size_t precision, offset; size_t i, j; @@ -11642,7 +11642,7 @@ test_unfiltered_edge_chunks(hid_t fapl) TEST_ERROR /* Add "count" filter */ - if (H5Pset_filter(dcpl, H5Z_FILTER_COUNT, 0u, (size_t)0, NULL) < 0) + if (H5Pset_filter(dcpl, H5Z_FILTER_COUNT, 0U, (size_t)0, NULL) < 0) TEST_ERROR /* Disable filters on partial chunks */ diff --git a/test/dt_arith.c b/test/dt_arith.c index 7b79102..91e31d5 100644 --- a/test/dt_arith.c +++ b/test/dt_arith.c @@ -53,7 +53,7 @@ const char *FILENAME[] = {"dt_arith1", "dt_arith2", NULL}; * endian. If local variable `endian' is H5T_ORDER_BE then the result will * be I, otherwise the result will be Z-(I+1). */ -#define ENDIAN(Z, I, E) (H5T_ORDER_BE == E ? (I) : (Z) - ((I) + 1)) +#define ENDIAN(Z, I, E) (H5T_ORDER_BE == (E) ? (I) : (Z) - ((I) + 1)) typedef enum dtype_t { INT_SCHAR, @@ -3058,10 +3058,10 @@ test_conv_flt_1(const char *name, int run_test, hid_t src, hid_t dst) /* Check the software results against the hardware */ for (j = 0; j < nelmts; j++) { underflow = 0; - hw_f = 911.0f; - hw_d = 911.0f; + hw_f = 911.0F; + hw_d = 911.0F; #if H5_SIZEOF_LONG_DOUBLE != H5_SIZEOF_DOUBLE - hw_ld = 911.0f; + hw_ld = 911.0L; #endif /* The hardware conversion */ diff --git a/test/dtransform.c b/test/dtransform.c index 9445d83..8d3a8c0 100644 --- a/test/dtransform.c +++ b/test/dtransform.c @@ -32,30 +32,30 @@ hid_t dset_id_int_chunk = -1; hid_t dset_id_float_chunk = -1; const float windchillFfloat[ROWS][COLS] = { - {36.0f, 31.0f, 25.0f, 19.0f, 13.0f, 7.0f, 1.0f, -5.0f, -11.0f, -16.0f, -22.0f, -28.0f, -34.0f, -40.0f, - -46.0f, -52.0f, -57.0f, -63.0f}, - {34.0f, 27.0f, 21.0f, 15.0f, 9.0f, 3.0f, -4.0f, -10.0f, -16.0f, -22.0f, -28.0f, -35.0f, -41.0f, -47.0f, - -53.0f, -59.0f, -66.0f, -72.0f}, - {32.0f, 25.0f, 19.0f, 13.0f, 6.0f, 0.0f, -7.0f, -13.0f, -19.0f, -26.0f, -32.0f, -39.0f, -45.0f, -51.0f, - -58.0f, -64.0f, -71.0f, -77.0f}, - {30.0f, 24.0f, 17.0f, 11.0f, 4.0f, -2.0f, -9.0f, -15.0f, -22.0f, -29.0f, -35.0f, -42.0f, -48.0f, -55.0f, - -61.0f, -68.0f, -74.0f, -81.0f}, - {29.0f, 23.0f, 16.0f, 9.0f, 3.0f, -4.0f, -11.0f, -17.0f, -24.0f, -31.0f, -37.0f, -44.0f, -51.0f, -58.0f, - -64.0f, -71.0f, -78.0f, -84.0f}, - {28.0f, 22.0f, 15.0f, 8.0f, 1.0f, -5.0f, -12.0f, -19.0f, -26.0f, -33.0f, -39.0f, -46.0f, -53.0f, -60.0f, - -67.0f, -73.0f, -80.0f, -87.0f}, - {28.0f, 21.0f, 14.0f, 7.0f, 0.0f, -7.0f, -14.0f, -21.0f, -27.0f, -34.0f, -41.0f, -48.0f, -55.0f, -62.0f, - -69.0f, -76.0f, -82.0f, -89.0f}, - {27.0f, 20.0f, 13.0f, 6.0f, -1.0f, -8.0f, -15.0f, -22.0f, -29.0f, -36.0f, -43.0f, -50.0f, -57.0f, -64.0f, - -71.0f, -78.0f, -84.0f, -91.0f}, - {26.0f, 19.0f, 12.0f, 5.0f, -2.0f, -9.0f, -16.0f, -23.0f, -30.0f, -37.0f, -44.0f, -51.0f, -58.0f, -65.0f, - -72.0f, -79.0f, -86.0f, -93.0f}, - {26.0f, 19.0f, 12.0f, 4.0f, -3.0f, -10.0f, -17.0f, -24.0f, -31.0f, -38.0f, -45.0f, -52.0f, -60.0f, -67.0f, - -74.0f, -81.0f, -88.0f, -95.0f}, - {25.0f, 18.0f, 11.0f, 4.0f, -3.0f, -11.0f, -18.0f, -25.0f, -32.0f, -39.0f, -46.0f, -54.0f, -61.0f, -68.0f, - -75.0f, -82.0f, -89.0f, -97.0f}, - {25.0f, 17.0f, 10.0f, 3.0f, -4.0f, -11.0f, -19.0f, -26.0f, -33.0f, -40.0f, -48.0f, -55.0f, -62.0f, -69.0f, - -76.0f, -84.0f, -91.0f, -98.0f}}; + {36.0F, 31.0F, 25.0F, 19.0F, 13.0F, 7.0F, 1.0F, -5.0F, -11.0F, -16.0F, -22.0F, -28.0F, -34.0F, -40.0F, + -46.0F, -52.0F, -57.0F, -63.0F}, + {34.0F, 27.0F, 21.0F, 15.0F, 9.0F, 3.0F, -4.0F, -10.0F, -16.0F, -22.0F, -28.0F, -35.0F, -41.0F, -47.0F, + -53.0F, -59.0F, -66.0F, -72.0F}, + {32.0F, 25.0F, 19.0F, 13.0F, 6.0F, 0.0F, -7.0F, -13.0F, -19.0F, -26.0F, -32.0F, -39.0F, -45.0F, -51.0F, + -58.0F, -64.0F, -71.0F, -77.0F}, + {30.0F, 24.0F, 17.0F, 11.0F, 4.0F, -2.0F, -9.0F, -15.0F, -22.0F, -29.0F, -35.0F, -42.0F, -48.0F, -55.0F, + -61.0F, -68.0F, -74.0F, -81.0F}, + {29.0F, 23.0F, 16.0F, 9.0F, 3.0F, -4.0F, -11.0F, -17.0F, -24.0F, -31.0F, -37.0F, -44.0F, -51.0F, -58.0F, + -64.0F, -71.0F, -78.0F, -84.0F}, + {28.0F, 22.0F, 15.0F, 8.0F, 1.0F, -5.0F, -12.0F, -19.0F, -26.0F, -33.0F, -39.0F, -46.0F, -53.0F, -60.0F, + -67.0F, -73.0F, -80.0F, -87.0F}, + {28.0F, 21.0F, 14.0F, 7.0F, 0.0F, -7.0F, -14.0F, -21.0F, -27.0F, -34.0F, -41.0F, -48.0F, -55.0F, -62.0F, + -69.0F, -76.0F, -82.0F, -89.0F}, + {27.0F, 20.0F, 13.0F, 6.0F, -1.0F, -8.0F, -15.0F, -22.0F, -29.0F, -36.0F, -43.0F, -50.0F, -57.0F, -64.0F, + -71.0F, -78.0F, -84.0F, -91.0F}, + {26.0F, 19.0F, 12.0F, 5.0F, -2.0F, -9.0F, -16.0F, -23.0F, -30.0F, -37.0F, -44.0F, -51.0F, -58.0F, -65.0F, + -72.0F, -79.0F, -86.0F, -93.0F}, + {26.0F, 19.0F, 12.0F, 4.0F, -3.0F, -10.0F, -17.0F, -24.0F, -31.0F, -38.0F, -45.0F, -52.0F, -60.0F, -67.0F, + -74.0F, -81.0F, -88.0F, -95.0F}, + {25.0F, 18.0F, 11.0F, 4.0F, -3.0F, -11.0F, -18.0F, -25.0F, -32.0F, -39.0F, -46.0F, -54.0F, -61.0F, -68.0F, + -75.0F, -82.0F, -89.0F, -97.0F}, + {25.0F, 17.0F, 10.0F, 3.0F, -4.0F, -11.0F, -19.0F, -26.0F, -33.0F, -40.0F, -48.0F, -55.0F, -62.0F, -69.0F, + -76.0F, -84.0F, -91.0F, -98.0F}}; const int transformData[ROWS][COLS] = {{36, 31, 25, 19, 13, 7, 1, 5, 11, 16, 22, 28, 34, 40, 46, 52, 57, 63}, {34, 27, 21, 15, 9, 3, 4, 10, 16, 22, 28, 35, 41, 47, 53, 59, 66, 1}, @@ -77,9 +77,9 @@ const int transformData[ROWS][COLS] = {{36, 31, 25, 19, 13, 7, 1, 5, 11, 16, 22, for (i = 0; i < ROWS; i++) \ for (j = 0; j < COLS; j++) { \ if (!((((VAR1)[i][j] >= (TYPE)((VAR2)[i][j])) && \ - (((VAR1)[i][j] - TOL) < (TYPE)((VAR2)[i][j]))) || \ + (((VAR1)[i][j] - (TOL)) < (TYPE)((VAR2)[i][j]))) || \ (((VAR1)[i][j] <= (TYPE)((VAR2)[i][j])) && \ - (((VAR1)[i][j] + TOL) > (TYPE)((VAR2)[i][j]))))) { \ + (((VAR1)[i][j] + (TOL)) > (TYPE)((VAR2)[i][j]))))) { \ H5_FAILED(); \ HDfprintf(stderr, " ERROR: Conversion failed to match computed data\n"); \ goto error; \ @@ -94,8 +94,8 @@ const int transformData[ROWS][COLS] = {{36, 31, 25, 19, 13, 7, 1, 5, 11, 16, 22, \ for (i = 0; i < ROWS; i++) \ for (j = 0; j < COLS; j++) { \ - if (!(((VAR1)[i][j] <= ((TYPE)(VAR2)[i][j] + TOL)) && \ - ((VAR1)[i][j] >= ((TYPE)(VAR2)[i][j] - TOL)))) { \ + if (!(((VAR1)[i][j] <= ((TYPE)(VAR2)[i][j] + (TOL))) && \ + ((VAR1)[i][j] >= ((TYPE)(VAR2)[i][j] - (TOL))))) { \ H5_FAILED(); \ HDfprintf(stderr, " ERROR: Conversion failed to match computed data\n"); \ goto error; \ @@ -540,19 +540,19 @@ test_poly(const hid_t dxpl_id_polynomial) for (row = 0; row < ROWS; row++) for (col = 0; col < COLS; col++) { - windchillC = (int)((5.0f / 9.0f) * (windchillFfloat[row][col] - 32)); - polyflres[row][col] = ((2.0f + (float)windchillC) * (((float)windchillC - 8.0f) / 2.0f)); + windchillC = (int)((5.0F / 9.0F) * (windchillFfloat[row][col] - 32)); + polyflres[row][col] = ((2.0F + (float)windchillC) * (((float)windchillC - 8.0F) / 2.0F)); } TESTING("data transform, polynomial transform (int->float)") if (H5Dread(dset_id_int, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, dxpl_id_polynomial, polyflread) < 0) TEST_ERROR - COMPARE(float, polyflread, polyflres, 2.0f) + COMPARE(float, polyflread, polyflres, 2.0F) for (row = 0; row < ROWS; row++) for (col = 0; col < COLS; col++) { - windchillC = (int)((5.0f / 9.0f) * (windchillFfloat[row][col] - 32)); + windchillC = (int)((5.0F / 9.0F) * (windchillFfloat[row][col] - 32)); polyflres[row][col] = (float)((2 + windchillC) * ((windchillC - 8) / 2)); } @@ -752,7 +752,7 @@ test_copy(const hid_t dxpl_id_c_to_f_copy, const hid_t dxpl_id_polynomial_copy) for (row = 0; row < ROWS; row++) for (col = 0; col < COLS; col++) { - windchillC = (int)((5.0f / 9.0f) * (windchillFfloat[row][col] - 32)); + windchillC = (int)((5.0F / 9.0F) * (windchillFfloat[row][col] - 32)); polyflres[row][col] = (float)((2 + windchillC) * ((windchillC - 8) / 2)); } @@ -786,7 +786,7 @@ test_trivial(const hid_t dxpl_id_simple) TEST_ERROR for (row = 0; row < ROWS; row++) for (col = 0; col < COLS; col++) { - if ((windchillFfloatread[row][col] - 4.8f) > FLOAT_TOL) + if ((windchillFfloatread[row][col] - 4.8F) > FLOAT_TOL) FAIL_PUTS_ERROR(" ERROR: Conversion failed to match computed data\n"); } @@ -843,7 +843,7 @@ test_getset(const hid_t dxpl_id_c_to_f) for (row = 0; row < ROWS; row++) for (col = 0; col < COLS; col++) { - if ((windchillFfloatread[row][col] - 4.8f) > FLOAT_TOL) + if ((windchillFfloatread[row][col] - 4.8F) > FLOAT_TOL) FAIL_PUTS_ERROR(" ERROR: Conversion failed to match computed data\n") } diff --git a/test/dtypes.c b/test/dtypes.c index 2a18302..8b3101c 100644 --- a/test/dtypes.c +++ b/test/dtypes.c @@ -70,7 +70,7 @@ FAIL_STACK_ERROR \ if ((NMEMBS) != H5I_nmembers(H5I_DATATYPE)) { \ H5_FAILED(); \ - HDprintf(" #dtype ids expected: %lld; found: %lld\n", (long long)NMEMBS, \ + HDprintf(" #dtype ids expected: %lld; found: %lld\n", (long long)(NMEMBS), \ (long long)H5I_nmembers(H5I_DATATYPE)); \ goto error; \ } @@ -6683,13 +6683,13 @@ static int test_int_float_except(void) { #if H5_SIZEOF_INT == 4 && H5_SIZEOF_FLOAT == 4 - float buf[CONVERT_SIZE] = {(float)INT_MIN - 172.0f, (float)INT_MAX - 32.0f, (float)INT_MAX - 68.0f, - (float)4.5f}; + float buf[CONVERT_SIZE] = {(float)INT_MIN - 172.0F, (float)INT_MAX - 32.0F, (float)INT_MAX - 68.0F, + (float)4.5F}; int buf_int[CONVERT_SIZE] = {INT_MIN, INT_MAX, INT_MAX - 127, 4}; - float buf_float[CONVERT_SIZE] = {(float)INT_MIN, (float)INT_MAX + 1.0f, (float)INT_MAX - 127.0f, 4}; + float buf_float[CONVERT_SIZE] = {(float)INT_MIN, (float)INT_MAX + 1.0F, (float)INT_MAX - 127.0F, 4}; int * intp; /* Pointer to buffer, as integers */ int buf2[CONVERT_SIZE] = {INT_MIN, INT_MAX, INT_MAX - 72, 0}; - float buf2_float[CONVERT_SIZE] = {(float)INT_MIN, (float)INT_MAX, (float)INT_MAX - 127.0f, (float)0.0f}; + float buf2_float[CONVERT_SIZE] = {(float)INT_MIN, (float)INT_MAX, (float)INT_MAX - 127.0F, (float)0.0F}; int buf2_int[CONVERT_SIZE] = {INT_MIN, INT_MAX, INT_MAX - 127, 0}; float *floatp; /* Pointer to buffer #2, as floats */ hid_t dxpl; /* Dataset transfer property list */ diff --git a/test/gheap.c b/test/gheap.c index 95594ee..567fd65 100644 --- a/test/gheap.c +++ b/test/gheap.c @@ -503,7 +503,7 @@ test_ooo_indices(hid_t fapl) GHEAP_REPEATED_ERR(" Unable to insert object into global heap") /* Check that the index is as expected */ - if (obj[j].idx != ((1000 * i) + j - (1000 * ((~i & 1)))) % ((1u << 16) - 1) + 1) + if (obj[j].idx != ((1000 * i) + j - (1000 * ((~i & 1)))) % ((1U << 16) - 1) + 1) GHEAP_REPEATED_ERR(" Unexpected global heap index"); } diff --git a/test/pool.c b/test/pool.c index 9ce1846..c508025 100644 --- a/test/pool.c +++ b/test/pool.c @@ -34,7 +34,7 @@ #define MPOOL_LARGE_BLOCK (MPOOL_PAGE_SIZE * 3) #define MPOOL_NUM_SMALL_BLOCKS 64 #define MPOOL_SMALL_BLOCK 1 -#define MPOOL_NUM_RANDOM 10 * 1024 +#define MPOOL_NUM_RANDOM (10 * 1024) #define MPOOL_RANDOM_MAX_SIZE (MPOOL_PAGE_SIZE * 2) /*------------------------------------------------------------------------- diff --git a/test/set_extent.c b/test/set_extent.c index 5a07362..156dd19 100644 --- a/test/set_extent.c +++ b/test/set_extent.c @@ -39,7 +39,7 @@ const char *FILENAME[] = {"set_extent1", "set_extent2", "set_extent3", "set_exte #define CONFIG_EARLY_ALLOC 0x04u #define CONFIG_UNFILT_EDGE 0x08u #define CONFIG_ALL (CONFIG_COMPRESS + CONFIG_FILL + CONFIG_EARLY_ALLOC + CONFIG_UNFILT_EDGE) -#define FILL_VALUE -1 +#define FILL_VALUE (-1) #define DO_RANKS_PRINT_CONFIG(TEST) \ { \ HDprintf(" Config:\n"); \ diff --git a/test/tconfig.c b/test/tconfig.c index e3c6a2c..101de9a 100644 --- a/test/tconfig.c +++ b/test/tconfig.c @@ -37,8 +37,9 @@ /* verify if the sizeof(type) matches size defined in macro. */ /* Needs this extra step so that we can print the macro name. */ #define vrfy_macrosize(type, macro, macroname) \ - if (sizeof(type) != macro) \ - TestErrPrintf("Error: sizeof(%s) is %zu but %s is %d\n", #type, sizeof(type), macroname, (int)macro); + if (sizeof(type) != (macro)) \ + TestErrPrintf("Error: sizeof(%s) is %zu but %s is %d\n", #type, sizeof(type), macroname, \ + (int)(macro)); /* local routine prototypes */ void test_config_ctypes(void); diff --git a/test/testhdf5.h b/test/testhdf5.h index 5fb01a8..ba5fa71 100644 --- a/test/testhdf5.h +++ b/test/testhdf5.h @@ -136,7 +136,7 @@ "%s \n", \ (where), (int)__LINE__, __FILE__, x); \ } \ - if (HDstrcmp(x, val)) { \ + if (HDstrcmp(x, val) != 0) { \ TestErrPrintf("*** UNEXPECTED VALUE from %s should be %s, but is %s at line %4d " \ "in %s\n", \ where, val, x, (int)__LINE__, __FILE__); \ diff --git a/test/tfile.c b/test/tfile.c index 62f3c73..0014d08 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -125,7 +125,7 @@ #define TEST_THRESHOLD10 10 /* Free space section threshold */ #define FSP_SIZE_DEF 4096 /* File space page size default */ #define FSP_SIZE512 512 /* File space page size */ -#define FSP_SIZE1G 1024 * 1024 * 1024 /* File space page size */ +#define FSP_SIZE1G (1024 * 1024 * 1024) /* File space page size */ /* Declaration for test_libver_macros2() */ #define FILE6 "tfile6.h5" /* Test file */ diff --git a/test/tmeta.c b/test/tmeta.c index 2609703..e70db16 100644 --- a/test/tmeta.c +++ b/test/tmeta.c @@ -22,9 +22,9 @@ #include "testhdf5.h" #include "H5Fprivate.h" -#define TEST_INT16_VALUE -7641 +#define TEST_INT16_VALUE (-7641) #define TEST_UINT16_VALUE 45002 -#define TEST_INT32_VALUE -981236 +#define TEST_INT32_VALUE (-981236) #define TEST_UINT32_VALUE 3476589 uint8_t compar_buffer[] = { diff --git a/test/tsohm.c b/test/tsohm.c index fcbb06a..b4ece0b 100644 --- a/test/tsohm.c +++ b/test/tsohm.c @@ -622,7 +622,7 @@ size1_helper(hid_t file, const char *filename, hid_t fapl_id, hbool_t test_file_ HDprintf("Can't read data\n"); \ goto error; \ } \ - if ((rdata.i1 != wdata.i1) || (rdata.i2 != wdata.i2) || HDstrcmp(rdata.str, wdata.str)) { \ + if ((rdata.i1 != wdata.i1) || (rdata.i2 != wdata.i2) || HDstrcmp(rdata.str, wdata.str) != 0) { \ H5_FAILED(); \ AT(); \ HDprintf("incorrect read data\n"); \ diff --git a/test/tunicode.c b/test/tunicode.c index 52341bb..1b696ac 100644 --- a/test/tunicode.c +++ b/test/tunicode.c @@ -33,7 +33,7 @@ #define RANK 1 #define COMP_INT_VAL 7 -#define COMP_FLOAT_VAL -42.0F +#define COMP_FLOAT_VAL (-42.0F) #define COMP_DOUBLE_VAL 42.0F /* Test function prototypes */ diff --git a/test/vds.c b/test/vds.c index 88ac4eb..ac9bb5d 100644 --- a/test/vds.c +++ b/test/vds.c @@ -78,12 +78,12 @@ char vds_test_str_g[128] = ""; #endif /* VDS_TEST_VERBOSE */ /* I/O test config flags */ -#define TEST_IO_CLOSE_SRC 0x01u -#define TEST_IO_DIFFERENT_FILE 0x02u -#define TEST_IO_REOPEN_VIRT 0x04u -#define TEST_IO_FCLOSE_SEMI 0x08u -#define TEST_IO_FCLOSE_STRONG 0x10u -#define TEST_IO_NTESTS 0x20u +#define TEST_IO_CLOSE_SRC 0x01U +#define TEST_IO_DIFFERENT_FILE 0x02U +#define TEST_IO_REOPEN_VIRT 0x04U +#define TEST_IO_FCLOSE_SEMI 0x08U +#define TEST_IO_FCLOSE_STRONG 0x10U +#define TEST_IO_NTESTS 0x20U #define LIST_DOUBLE_SIZE (H5D_VIRTUAL_DEF_LIST_SIZE + 1) diff --git a/tools/lib/h5diff_array.c b/tools/lib/h5diff_array.c index 43ded12..2d6c66c 100644 --- a/tools/lib/h5diff_array.c +++ b/tools/lib/h5diff_array.c @@ -104,9 +104,9 @@ static hbool_t not_comparable; per = -1; \ not_comparable = FALSE; \ both_zero = FALSE; \ - if (H5_DBL_ABS_EQUAL(0, (double)A) && H5_DBL_ABS_EQUAL(0, (double)B)) \ + if (H5_DBL_ABS_EQUAL(0, (double)(A)) && H5_DBL_ABS_EQUAL(0, (double)(B))) \ both_zero = TRUE; \ - if (!H5_DBL_ABS_EQUAL(0, (double)A)) \ + if (!H5_DBL_ABS_EQUAL(0, (double)(A))) \ per = (double)ABS((double)((B) - (A)) / (double)(A)); \ else \ not_comparable = TRUE; \ @@ -117,9 +117,9 @@ static hbool_t not_comparable; per = -1; \ not_comparable = FALSE; \ both_zero = FALSE; \ - if (H5_DBL_ABS_EQUAL(0, (double)A) && H5_DBL_ABS_EQUAL(0, (double)B)) \ + if (H5_DBL_ABS_EQUAL(0, (double)(A)) && H5_DBL_ABS_EQUAL(0, (double)(B))) \ both_zero = TRUE; \ - if (!H5_DBL_ABS_EQUAL(0, (double)A)) \ + if (!H5_DBL_ABS_EQUAL(0, (double)(A))) \ per = ABS((double)((TYPE)((B) - (A))) / (double)(A)); \ else \ not_comparable = TRUE; \ diff --git a/tools/src/h5perf/sio_engine.c b/tools/src/h5perf/sio_engine.c index 1af2318..e69a7cd 100644 --- a/tools/src/h5perf/sio_engine.c +++ b/tools/src/h5perf/sio_engine.c @@ -50,7 +50,7 @@ /* verify: if val is false (0), print mesg. */ #define VRFY(val, mesg) \ do { \ - if (!val) { \ + if (!(val)) { \ ERRMSG(mesg); \ GOTOERROR(FAIL); \ } \ diff --git a/tools/src/h5perf/sio_perf.c b/tools/src/h5perf/sio_perf.c index 7bd58d8..1b200b0 100644 --- a/tools/src/h5perf/sio_perf.c +++ b/tools/src/h5perf/sio_perf.c @@ -65,7 +65,7 @@ #define SIO_HDF5 0x4 /* report 0.0 in case t is zero too */ -#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL(t, 0.0) ? 0.0 : ((((double)bytes) / (double)ONE_MB) / (t))) +#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL(t, 0.0) ? 0.0 : ((((double)(bytes)) / (double)ONE_MB) / (t))) #ifndef TRUE #define TRUE 1 diff --git a/tools/src/misc/h5clear.c b/tools/src/misc/h5clear.c index 008dae3..face2f0 100644 --- a/tools/src/misc/h5clear.c +++ b/tools/src/misc/h5clear.c @@ -32,7 +32,7 @@ #define H5F_ACS_SKIP_EOF_CHECK_NAME "skip_eof_check" /* Default increment is 1 megabytes for the --increment option */ -#define DEFAULT_INCREMENT 1024 * 1024 +#define DEFAULT_INCREMENT (1024 * 1024) static char * fname_g = NULL; static hbool_t clear_status_flags = FALSE; diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index 29922e8..e74054c 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -199,8 +199,8 @@ const H5L_class_t UD_link_class[1] = {{ #define DIM1 20 #define DIM2 10 -#define CDIM1 DIM1 / 2 -#define CDIM2 DIM2 / 2 +#define CDIM1 (DIM1 / 2) +#define CDIM2 (DIM2 / 2) #define RANK 2 /* Dataspace of 0 dimension size */ diff --git a/tools/test/h5repack/h5repacktst.c b/tools/test/h5repack/h5repacktst.c index acad74e..b62fa53 100644 --- a/tools/test/h5repack/h5repacktst.c +++ b/tools/test/h5repack/h5repacktst.c @@ -111,8 +111,8 @@ const char *H5REPACK_FILENAMES[] = {"h5repack_big_out", NULL}; #define DIM1 40 #define DIM2 20 -#define CDIM1 DIM1 / 2 -#define CDIM2 DIM2 / 2 +#define CDIM1 (DIM1 / 2) +#define CDIM2 (DIM2 / 2) #define RANK 2 /* Size of userblock (for userblock test) */ diff --git a/tools/test/perform/chunk_cache.c b/tools/test/perform/chunk_cache.c index 25fa34c..b515961 100644 --- a/tools/test/perform/chunk_cache.c +++ b/tools/test/perform/chunk_cache.c @@ -26,9 +26,9 @@ #define RANK 2 #define DSET1_NAME "partial_chunks" -#define DSET1_DIM1 9 * 1000 +#define DSET1_DIM1 (9 * 1000) #define DSET1_DIM2 9 -#define CHUNK1_DIM1 2 * 1000 +#define CHUNK1_DIM1 (2 * 1000) #define CHUNK1_DIM2 2 #define DSET2_NAME "hash_value" @@ -38,7 +38,7 @@ #define CHUNK2_DIM2 100 #define RDCC_NSLOTS 5 -#define RDCC_NBYTES 1024 * 1024 * 10 +#define RDCC_NBYTES (1024 * 1024 * 10) #define RDCC_W0 0.75F #define FILTER_COUNTER 306 diff --git a/tools/test/perform/zip_perf.c b/tools/test/perform/zip_perf.c index 05ad083..d8cd3a1 100644 --- a/tools/test/perform/zip_perf.c +++ b/tools/test/perform/zip_perf.c @@ -35,7 +35,7 @@ #define MICROSECOND 1000000.0 /* report 0.0 in case t is zero too */ -#define MB_PER_SEC(bytes, t) ((fabs(t) < 0.0000000001) ? 0.0 : ((((double)bytes) / (double)ONE_MB) / (t))) +#define MB_PER_SEC(bytes, t) ((fabs(t) < 0.0000000001) ? 0.0 : ((((double)(bytes)) / (double)ONE_MB) / (t))) #ifndef TRUE #define TRUE 1 -- cgit v0.12 From e9765e6c09de2e278e2821faef656b6d61854196 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Mon, 23 Aug 2021 16:45:29 -0700 Subject: Move pwd header check to alphabetical order (#954) --- configure.ac | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/configure.ac b/configure.ac index d955a9f..258f96c 100644 --- a/configure.ac +++ b/configure.ac @@ -1225,7 +1225,7 @@ AC_CHECK_LIB([dl], [dlopen]) ## ## Unix -AC_CHECK_HEADERS([dirent.h features.h unistd.h pwd.h]) +AC_CHECK_HEADERS([dirent.h features.h pwd.h unistd.h]) AC_CHECK_HEADERS([sys/file.h sys/ioctl.h sys/resource.h]) AC_CHECK_HEADERS([sys/stat.h sys/time.h sys/types.h]) -- cgit v0.12 From 131402a92de9bbd5df60d5859c37c4820c60d6b9 Mon Sep 17 00:00:00 2001 From: Sean McBride Date: Wed, 25 Aug 2021 00:05:23 -0400 Subject: More various warnings (#958) * Committing clang-format changes * Fixed various -Wdouble-promotion warnings * Fixed -Wshadow warning for `optopt` conflict On macOS at least, there is a global various named `optopt` in unistd.h. * Committing clang-format changes Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- src/H5ACprivate.h | 16 ++++++++-------- src/H5Cquery.c | 2 +- src/H5Pdxpl.c | 2 +- src/H5system.c | 24 ++++++++++++------------ test/cache_common.c | 2 +- test/enc_dec_plist.c | 4 ++-- test/fillval.c | 6 +++--- test/hyperslab.c | 6 +++--- test/set_extent.c | 4 ++-- test/tattr.c | 7 +++---- test/timer.c | 18 +++++++++--------- test/tmisc.c | 2 +- test/tvltypes.c | 8 ++++---- 13 files changed, 50 insertions(+), 51 deletions(-) diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index b14194b..e9b8a47 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -275,24 +275,24 @@ typedef struct H5AC_proxy_entry_t { /* size_t min_size = */ ( 1 * 1024 * 1024), \ /* long int epoch_length = */ 50000, \ /* enum H5C_cache_incr_mode incr_mode = */ H5C_incr__threshold, \ - /* double lower_hr_threshold = */ 0.9f, \ - /* double increment = */ 2.0f, \ + /* double lower_hr_threshold = */ 0.9, \ + /* double increment = */ 2.0, \ /* hbool_t apply_max_increment = */ TRUE, \ /* size_t max_increment = */ (4 * 1024 * 1024), \ /* enum H5C_cache_flash_incr_mode */ \ /* flash_incr_mode = */ H5C_flash_incr__add_space, \ - /* double flash_multiple = */ 1.4f, \ - /* double flash_threshold = */ 0.25f, \ + /* double flash_multiple = */ 1.4, \ + /* double flash_threshold = */ 0.25, \ /* enum H5C_cache_decr_mode decr_mode = */ H5C_decr__age_out_with_threshold,\ - /* double upper_hr_threshold = */ 0.999f, \ - /* double decrement = */ 0.9f, \ + /* double upper_hr_threshold = */ 0.999, \ + /* double decrement = */ 0.9, \ /* hbool_t apply_max_decrement = */ TRUE, \ /* size_t max_decrement = */ (1 * 1024 * 1024), \ /* int epochs_before_eviction = */ 3, \ /* hbool_t apply_empty_reserve = */ TRUE, \ - /* double empty_reserve = */ 0.1f, \ + /* double empty_reserve = */ 0.1, \ /* size_t dirty_bytes_threshold = */ (256 * 1024), \ - /* int metadata_write_strategy = */ \ + /* int metadata_write_strategy = */ \ H5AC__DEFAULT_METADATA_WRITE_STRATEGY \ } #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5Cquery.c b/src/H5Cquery.c index 6cf8ffc..710ce2a 100644 --- a/src/H5Cquery.c +++ b/src/H5Cquery.c @@ -201,7 +201,7 @@ H5C_get_cache_hit_rate(const H5C_t *cache_ptr, double *hit_rate_ptr) if (cache_ptr->cache_accesses > 0) *hit_rate_ptr = ((double)(cache_ptr->cache_hits)) / ((double)(cache_ptr->cache_accesses)); else - *hit_rate_ptr = 0.0f; + *hit_rate_ptr = 0.0; done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Pdxpl.c b/src/H5Pdxpl.c index 5f83a1d..a0c38f8 100644 --- a/src/H5Pdxpl.c +++ b/src/H5Pdxpl.c @@ -68,7 +68,7 @@ #define H5D_XFER_BTREE_SPLIT_RATIO_SIZE sizeof(double[3]) #define H5D_XFER_BTREE_SPLIT_RATIO_DEF \ { \ - 0.1f, 0.5f, 0.9f \ + 0.1, 0.5, 0.9 \ } #define H5D_XFER_BTREE_SPLIT_RATIO_ENC H5P__dxfr_btree_split_ratio_enc #define H5D_XFER_BTREE_SPLIT_RATIO_DEC H5P__dxfr_btree_split_ratio_dec diff --git a/src/H5system.c b/src/H5system.c index eba8341..fe379ad 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -958,8 +958,8 @@ const char *H5_optarg; /* Flag argument (or value) */ int H5_get_option(int argc, const char **argv, const char *opts, const struct h5_long_options *l_opts) { - static int sp = 1; /* character index in current token */ - int optopt = '?'; /* option character passed back to user */ + static int sp = 1; /* character index in current token */ + int optchar = '?'; /* option character passed back to user */ if (sp == 1) { /* check for more flag-like tokens */ @@ -990,7 +990,7 @@ H5_get_option(int argc, const char **argv, const char *opts, const struct h5_lon for (i = 0; l_opts && l_opts[i].name; i++) { if (HDstrcmp(arg, l_opts[i].name) == 0) { /* we've found a matching long command line flag */ - optopt = l_opts[i].shortval; + optchar = l_opts[i].shortval; if (l_opts[i].has_arg != no_arg) { if (H5_optarg == NULL) { @@ -1003,7 +1003,7 @@ H5_get_option(int argc, const char **argv, const char *opts, const struct h5_lon if (H5_opterr) HDfprintf(stderr, "%s: option required for \"--%s\" flag\n", argv[0], arg); - optopt = '?'; + optchar = '?'; } } } @@ -1012,7 +1012,7 @@ H5_get_option(int argc, const char **argv, const char *opts, const struct h5_lon if (H5_opterr) HDfprintf(stderr, "%s: no option required for \"%s\" flag\n", argv[0], arg); - optopt = '?'; + optchar = '?'; } } break; @@ -1024,7 +1024,7 @@ H5_get_option(int argc, const char **argv, const char *opts, const struct h5_lon if (H5_opterr) HDfprintf(stderr, "%s: unknown option \"%s\"\n", argv[0], arg); - optopt = '?'; + optchar = '?'; } H5_optind++; @@ -1036,11 +1036,11 @@ H5_get_option(int argc, const char **argv, const char *opts, const struct h5_lon register char *cp; /* pointer into current token */ /* short command line option */ - optopt = argv[H5_optind][sp]; + optchar = argv[H5_optind][sp]; - if (optopt == ':' || (cp = HDstrchr(opts, optopt)) == 0) { + if (optchar == ':' || (cp = HDstrchr(opts, optchar)) == 0) { if (H5_opterr) - HDfprintf(stderr, "%s: unknown option \"%c\"\n", argv[0], optopt); + HDfprintf(stderr, "%s: unknown option \"%c\"\n", argv[0], optchar); /* if no chars left in this token, move to next token */ if (argv[H5_optind][++sp] == '\0') { @@ -1058,9 +1058,9 @@ H5_get_option(int argc, const char **argv, const char *opts, const struct h5_lon } else if (++H5_optind >= argc) { if (H5_opterr) - HDfprintf(stderr, "%s: value expected for option \"%c\"\n", argv[0], optopt); + HDfprintf(stderr, "%s: value expected for option \"%c\"\n", argv[0], optchar); - optopt = '?'; + optchar = '?'; } else { /* flag value is next token */ @@ -1098,5 +1098,5 @@ H5_get_option(int argc, const char **argv, const char *opts, const struct h5_lon } /* return the current flag character found */ - return optopt; + return optchar; } diff --git a/test/cache_common.c b/test/cache_common.c index 07c70f0..2d88e05 100644 --- a/test/cache_common.c +++ b/test/cache_common.c @@ -4994,7 +4994,7 @@ check_and_validate_cache_hit_rate(hid_t file_id, double *hit_rate_ptr, hbool_t d } else { - expected_hit_rate = 0.0F; + expected_hit_rate = 0.0; } result = H5Fget_mdc_hit_rate(file_id, &hit_rate); diff --git a/test/enc_dec_plist.c b/test/enc_dec_plist.c index 9b4879e..60b229a 100644 --- a/test/enc_dec_plist.c +++ b/test/enc_dec_plist.c @@ -350,7 +350,7 @@ main(void) TESTING("DXPL Encoding/Decoding"); - if ((H5Pset_btree_ratios(dxpl, 0.2f, 0.6f, 0.2f)) < 0) + if ((H5Pset_btree_ratios(dxpl, 0.2, 0.6, 0.2)) < 0) FAIL_STACK_ERROR if ((H5Pset_hyper_vector_size(dxpl, 5)) < 0) FAIL_STACK_ERROR @@ -544,7 +544,7 @@ main(void) FAIL_STACK_ERROR if ((H5Pset_alignment(fapl, 2, 1024)) < 0) FAIL_STACK_ERROR - if ((H5Pset_cache(fapl, 1024, 128, 10485760, 0.3f)) < 0) + if ((H5Pset_cache(fapl, 1024, 128, 10485760, 0.3)) < 0) FAIL_STACK_ERROR if ((H5Pset_elink_file_cache_size(fapl, 10485760)) < 0) FAIL_STACK_ERROR diff --git a/test/fillval.c b/test/fillval.c index 4215c89..8de6ef1 100644 --- a/test/fillval.c +++ b/test/fillval.c @@ -1050,7 +1050,7 @@ test_rdwr_cases(hid_t file, hid_t dcpl, const char *dname, void *_fillval, H5D_f for (u = 0; u < nelmts; u++) { buf_c[u].a = 1111.11F; buf_c[u].x = 2222; - buf_c[u].y = 3333.3333F; + buf_c[u].y = 3333.3333; buf_c[u].z = 'd'; } if (H5Dwrite(dset2, ctype_id, mspace, fspace, H5P_DEFAULT, buf_c) < 0) @@ -1304,7 +1304,7 @@ test_rdwr(hid_t fapl, const char *base_name, H5D_layout_t layout) if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; HDmemset(&fill_ctype, 0, sizeof(fill_ctype)); - fill_ctype.y = 4444.4444F; + fill_ctype.y = 4444.4444; if (H5Pset_fill_value(dcpl, ctype_id, &fill_ctype) < 0) goto error; nerrors += test_rdwr_cases(file, dcpl, "dset11", &fill_ctype, H5D_FILL_TIME_ALLOC, layout, @@ -1370,7 +1370,7 @@ test_rdwr(hid_t fapl, const char *base_name, H5D_layout_t layout) if (H5Pset_fill_time(dcpl, H5D_FILL_TIME_ALLOC) < 0) goto error; HDmemset(&fill_ctype, 0, sizeof(fill_ctype)); - fill_ctype.y = 4444.4444F; + fill_ctype.y = 4444.4444; if (H5Pset_fill_value(dcpl, ctype_id, &fill_ctype) < 0) goto error; nerrors += test_rdwr_cases(file, dcpl, "dset12", &fill_ctype, H5D_FILL_TIME_ALLOC, layout, H5T_COMPOUND, diff --git a/test/hyperslab.c b/test/hyperslab.c index 1f57e3b..1c55259 100644 --- a/test/hyperslab.c +++ b/test/hyperslab.c @@ -585,10 +585,10 @@ test_multifill(size_t nx) for (i = 0; i < nx; i++) { src[i].left = 1111111; - src[i].mid = 12345.6789F; + src[i].mid = 12345.6789; src[i].right = 2222222; dst[i].left = 3333333; - dst[i].mid = 98765.4321F; + dst[i].mid = 98765.4321; dst[i].right = 4444444; } /* end for */ @@ -597,7 +597,7 @@ test_multifill(size_t nx) * over and over again. */ fill.left = 55555555; - fill.mid = 3.1415927F; + fill.mid = 3.1415927; fill.right = 66666666; src_stride = 0; diff --git a/test/set_extent.c b/test/set_extent.c index 156dd19..3078c2d 100644 --- a/test/set_extent.c +++ b/test/set_extent.c @@ -134,11 +134,11 @@ main(void) TEST_ERROR /* Set chunk cache so only part of the chunks can be cached on fapl */ - if (H5Pset_cache(fapl, 0, (size_t)8, 256 * sizeof(int), 0.75F) < 0) + if (H5Pset_cache(fapl, 0, (size_t)8, 256 * sizeof(int), 0.75) < 0) TEST_ERROR /* Disable chunk caching on fapl2 */ - if (H5Pset_cache(fapl2, 0, (size_t)0, (size_t)0, 0.0F) < 0) + if (H5Pset_cache(fapl2, 0, (size_t)0, (size_t)0, 0.0) < 0) TEST_ERROR /* Set the "use the latest version of the format" bounds for creating objects in the file */ diff --git a/test/tattr.c b/test/tattr.c index b602222..756e139 100644 --- a/test/tattr.c +++ b/test/tattr.c @@ -95,8 +95,7 @@ int attr_data2[ATTR2_DIM1][ATTR2_DIM2] = {{7614, -416}, {197814, -3}}; /* Test d #define ATTR3_DIM2 2 #define ATTR3_DIM3 2 double attr_data3[ATTR3_DIM1][ATTR3_DIM2][ATTR3_DIM3] = { - {{2.3F, -26.1F}, {0.123F, -10.0F}}, - {{973.23F, -0.91827F}, {2.0F, 23.0F}}}; /* Test data for 3rd attribute */ + {{2.3, -26.1}, {0.123, -10.0}}, {{973.23, -0.91827}, {2.0, 23.0}}}; /* Test data for 3rd attribute */ #define ATTR4_NAME "Attr4" #define ATTR4_RANK 2 @@ -113,8 +112,8 @@ struct attr4_struct { double d; char c; } attr_data4[ATTR4_DIM1][ATTR4_DIM2] = { - {{3, -26.1F, 'd'}, {-100000, 0.123F, '3'}}, - {{-23, 981724.2F, 'Q'}, {0, 2.0F, '\n'}}}; /* Test data for 4th attribute */ + {{3, -26.1, 'd'}, {-100000, 0.123, '3'}}, + {{-23, 981724.2, 'Q'}, {0, 2.0, '\n'}}}; /* Test data for 4th attribute */ #define ATTR5_NAME "Attr5" #define ATTR5_RANK 0 diff --git a/test/timer.c b/test/timer.c index bb474ca..11b7427 100644 --- a/test/timer.c +++ b/test/timer.c @@ -43,55 +43,55 @@ test_time_formatting(void) TESTING("Time string formats"); /* < 0, N/A */ - s = H5_timer_get_time_string(-1.0F); + s = H5_timer_get_time_string(-1.0); if (NULL == s || HDstrcmp(s, "N/A") != 0) TEST_ERROR; HDfree(s); /* 0 0 */ - s = H5_timer_get_time_string(0.0F); + s = H5_timer_get_time_string(0.0); if (NULL == s || HDstrcmp(s, "0.0 s") != 0) TEST_ERROR; HDfree(s); /* < 1 us nanoseconds */ - s = H5_timer_get_time_string(123.0E-9F); + s = H5_timer_get_time_string(123.0E-9); if (NULL == s || HDstrcmp(s, "123 ns") != 0) TEST_ERROR; HDfree(s); /* < 1 ms microseconds */ - s = H5_timer_get_time_string(23.456E-6F); + s = H5_timer_get_time_string(23.456E-6); if (NULL == s || HDstrcmp(s, "23.5 us") != 0) TEST_ERROR; HDfree(s); /* < 1 s milliseconds */ - s = H5_timer_get_time_string(4.56789E-3F); + s = H5_timer_get_time_string(4.56789E-3); if (NULL == s || HDstrcmp(s, "4.6 ms") != 0) TEST_ERROR; HDfree(s); /* < 1 min seconds */ - s = H5_timer_get_time_string(3.14F); + s = H5_timer_get_time_string(3.14); if (NULL == s || HDstrcmp(s, "3.14 s") != 0) TEST_ERROR; HDfree(s); /* < 1 hr mins, secs */ - s = H5_timer_get_time_string(2521.0F); + s = H5_timer_get_time_string(2521.0); if (NULL == s || HDstrcmp(s, "42 m 1 s") != 0) TEST_ERROR; HDfree(s); /* < 1 d hrs, mins, secs */ - s = H5_timer_get_time_string(9756.0F); + s = H5_timer_get_time_string(9756.0); if (NULL == s || HDstrcmp(s, "2 h 42 m 36 s") != 0) TEST_ERROR; HDfree(s); /* > 1 d days, hrs, mins, secs */ - s = H5_timer_get_time_string(280802.0F); + s = H5_timer_get_time_string(280802.0); if (NULL == s || HDstrcmp(s, "3 d 6 h 0 m 2 s") != 0) TEST_ERROR; HDfree(s); diff --git a/test/tmisc.c b/test/tmisc.c index 302da0d..b267330 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -5276,7 +5276,7 @@ test_misc28(void) * bytes). */ fapl = H5Pcreate(H5P_FILE_ACCESS); CHECK(fapl, FAIL, "H5Pcreate"); - ret = H5Pset_cache(fapl, MISC28_NSLOTS, MISC28_NSLOTS, MISC28_SIZE, 0.75F); + ret = H5Pset_cache(fapl, MISC28_NSLOTS, MISC28_NSLOTS, MISC28_SIZE, 0.75); CHECK(ret, FAIL, "H5Pset_cache"); /* Create the dcpl and set the chunk size */ diff --git a/test/tvltypes.c b/test/tvltypes.c index 50b2d7a..03a8ad3 100644 --- a/test/tvltypes.c +++ b/test/tvltypes.c @@ -2526,10 +2526,10 @@ test_vltypes_fill_value(void) hsize_t small_dims[] = {SPACE4_DIM_SMALL}; hsize_t large_dims[] = {SPACE4_DIM_LARGE}; size_t dset_elmts = 0; /* Number of elements in a particular dataset */ - const dtype1_struct fill1 = {1, 2, "foobar", "", NULL, "\0", "dead", - 3, 4.0F, 100.0F, 1.0F, "liquid", "meter"}; - const dtype1_struct wdata = {3, 4, "", NULL, "\0", "foo", "two", 6, 8.0F, 200.0F, 2.0F, "solid", "yard"}; - dtype1_struct * rbuf = NULL; /* Buffer for reading data */ + const dtype1_struct fill1 = {1, 2, "foobar", "", NULL, "\0", "dead", + 3, 4.0, 100.0, 1.0, "liquid", "meter"}; + const dtype1_struct wdata = {3, 4, "", NULL, "\0", "foo", "two", 6, 8.0, 200.0, 2.0, "solid", "yard"}; + dtype1_struct * rbuf = NULL; /* Buffer for reading data */ size_t mem_used = 0; /* Memory used during allocation */ H5D_layout_t layout; /* Dataset storage layout */ char dset_name1[64], dset_name2[64]; /* Dataset names */ -- cgit v0.12 From 794acf489fa093a3a119723c7cea74cb2880624b Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Mon, 30 Aug 2021 09:09:06 -0500 Subject: Fixed failure on BE with xlf (#968) * fixed missed closing of a dataset * fixed missed closing of a dataset * fixed typo in error return * Committing clang-format changes * minor edits * code format * Committing clang-format changes * code format * minor edit * switched from using MPI_count, to actual bytes written for H5FD_mpio_debug rw debugging * Committing clang-format changes * changed size_i in printf to reflect the I/O. * Committing clang-format changes * Fixed seg fault with xlf on BE with -qintsize=8 Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- fortran/test/tH5Z.F90 | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fortran/test/tH5Z.F90 b/fortran/test/tH5Z.F90 index 0eaa941..8c39fea 100644 --- a/fortran/test/tH5Z.F90 +++ b/fortran/test/tH5Z.F90 @@ -192,7 +192,8 @@ CONTAINS INTEGER(HSIZE_T), DIMENSION(2) :: chunk_dims = (/NN, MM/) INTEGER :: rank = 2 ! Dataset rank - INTEGER, DIMENSION(N,M) :: dset_data, data_out ! Data buffers + INTEGER, DIMENSION(N,M) :: dset_data ! Data buffers + INTEGER, DIMENSION(:,:), ALLOCATABLE :: data_out ! Data buffers INTEGER :: error ! Error flag INTEGER :: num_errors = 0 ! Number of data errors @@ -363,8 +364,9 @@ CONTAINS ! ! Read the dataset. ! + ALLOCATE(data_out(1:N,1:M)) CALL h5dread_f (dset_id, H5T_NATIVE_INTEGER, data_out, data_dims, error) - CALL check("h5dread_f", error, total_error) + CALL check("h5dread_f", error, total_error) ! !Compare the data. @@ -385,6 +387,7 @@ CONTAINS 100 IF (num_errors .GT. 0) THEN total_error=total_error + 1 END IF + DEALLOCATE(data_out) ! ! End access to the dataset and release resources used by it. -- cgit v0.12 From 01fe2549a36d7635b2cbe6f6a57cbcce29c1335b Mon Sep 17 00:00:00 2001 From: Scot Breitenfeld Date: Mon, 30 Aug 2021 09:56:32 -0500 Subject: Rework of PR #826 (#972) * H5Fget_name_f fixed to handle correctly trailing whitespaces and newly allocated buffers. Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com> --- fortran/src/H5Fff.F90 | 4 +- fortran/test/fortranlib_test.F90 | 4 ++ fortran/test/tH5F.F90 | 122 ++++++++++++++++++++++++++++++++++++++- release_docs/RELEASE.txt | 6 ++ 4 files changed, 133 insertions(+), 3 deletions(-) diff --git a/fortran/src/H5Fff.F90 b/fortran/src/H5Fff.F90 index f772558..a273431 100644 --- a/fortran/src/H5Fff.F90 +++ b/fortran/src/H5Fff.F90 @@ -824,7 +824,7 @@ CONTAINS SUBROUTINE h5fget_name_f(obj_id, buf, size, hdferr) IMPLICIT NONE INTEGER(HID_T), INTENT(IN) :: obj_id ! Object identifier - CHARACTER(LEN=*), INTENT(INOUT) :: buf + CHARACTER(LEN=*), INTENT(OUT) :: buf ! Buffer to hold file name INTEGER(SIZE_T), INTENT(OUT) :: size ! Size of the file name INTEGER, INTENT(OUT) :: hdferr ! Error code: 0 on success, @@ -844,7 +844,7 @@ CONTAINS CHARACTER(KIND=C_CHAR), DIMENSION(*), INTENT(OUT) :: buf END FUNCTION h5fget_name_c END INTERFACE - buflen = LEN_TRIM(buf) + buflen = LEN(buf) hdferr = h5fget_name_c(obj_id, size, buf, buflen) END SUBROUTINE h5fget_name_f !****s* H5F/h5fget_filesize_f diff --git a/fortran/test/fortranlib_test.F90 b/fortran/test/fortranlib_test.F90 index eff4657..1640a8f 100644 --- a/fortran/test/fortranlib_test.F90 +++ b/fortran/test/fortranlib_test.F90 @@ -74,6 +74,10 @@ PROGRAM fortranlibtest CALL write_test_status(ret_total_error, ' Reopen test', total_error) ret_total_error = 0 + CALL get_name_test(cleanup, ret_total_error) + CALL write_test_status(ret_total_error, ' Get name test', total_error) + + ret_total_error = 0 CALL file_close(cleanup, ret_total_error) CALL write_test_status(ret_total_error, ' File open/close test', total_error) diff --git a/fortran/test/tH5F.F90 b/fortran/test/tH5F.F90 index f4a3232..06dc6de 100644 --- a/fortran/test/tH5F.F90 +++ b/fortran/test/tH5F.F90 @@ -21,7 +21,8 @@ ! * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * ! ! CONTAINS SUBROUTINES -! mountingtest, reopentest, plisttest, file_close, file_space +! mountingtest, reopentest, get_name_test, plisttest, +! file_close, file_space ! !***** ! @@ -580,6 +581,125 @@ CONTAINS END SUBROUTINE reopentest +! The following subroutine checks that h5fget_name_f produces +! correct output for a given obj_id and filename. +! + SUBROUTINE check_get_name(obj_id, fix_filename, total_error) + USE HDF5 ! This module contains all necessary modules + USE TH5_MISC + IMPLICIT NONE + INTEGER(HID_T) :: obj_id ! Object identifier + CHARACTER(LEN=80), INTENT(IN) :: fix_filename ! Expected filename + INTEGER, INTENT(INOUT) :: total_error ! Error count + + CHARACTER(LEN=80):: file_name ! Filename buffer + INTEGER:: error ! HDF5 error code + INTEGER(SIZE_T):: name_size ! Filename length + ! + !Get file name from the dataset identifier + ! + + ! Use an uninitialized buffer + CALL h5fget_name_f(obj_id, file_name, name_size, error) + CALL check("h5fget_name_f",error,total_error) + IF(name_size .NE. LEN_TRIM(fix_filename))THEN + WRITE(*,*) " file name size obtained from the object id is incorrect" + total_error = total_error + 1 + ENDIF + IF(file_name(1:name_size) .NE. TRIM(fix_filename)) THEN + WRITE(*,*) " file name obtained from the object id is incorrect" + total_error = total_error + 1 + END IF + + ! Use a buffer initialized with spaces + file_name(:) = " " + CALL h5fget_name_f(obj_id, file_name, name_size, error) + CALL check("h5fget_name_f",error,total_error) + IF(name_size .NE. LEN_TRIM(fix_filename))THEN + WRITE(*,*) " file name size obtained from the object id is incorrect" + total_error = total_error + 1 + ENDIF + IF(file_name(1:name_size) .NE. TRIM(fix_filename)) THEN + WRITE(*,*) " file name obtained from the object id is incorrect" + total_error = total_error + 1 + END IF + + ! Use a buffer initialized with non-whitespace characters + file_name(:) = "a" + CALL h5fget_name_f(obj_id, file_name, name_size, error) + CALL check("h5fget_name_f",error,total_error) + IF(name_size .NE. LEN_TRIM(fix_filename))THEN + WRITE(*,*) " file name size obtained from the object id is incorrect" + total_error = total_error + 1 + ENDIF + IF(file_name(1:name_size) .NE. TRIM(fix_filename)) THEN + WRITE(*,*) " file name obtained from the object id is incorrect" + total_error = total_error + 1 + END IF + + END SUBROUTINE check_get_name + +! The following subroutine tests h5fget_name_f. +! It creates the file which has name "filename.h5" and +! tests that h5fget_name_f also returns the name "filename.h5" +! + + SUBROUTINE get_name_test(cleanup, total_error) + USE HDF5 ! This module contains all necessary modules + USE TH5_MISC + IMPLICIT NONE + LOGICAL, INTENT(IN) :: cleanup + INTEGER, INTENT(INOUT) :: total_error + + CHARACTER(LEN=*), PARAMETER :: filename = "filename" + CHARACTER(LEN=80) :: fix_filename + + INTEGER(HID_T) :: file_id ! File identifier + INTEGER(HID_T) :: g_id ! Group identifier + + ! + ! Flag to check operation success + ! + INTEGER :: error + + ! + ! Create file "filename.h5" using default properties. + ! + CALL h5_fixname_f(filename, fix_filename, H5P_DEFAULT_F, error) + IF (error .NE. 0) THEN + WRITE(*,*) "Cannot modify filename" + STOP + ENDIF + CALL h5fcreate_f(fix_filename, H5F_ACC_TRUNC_F, file_id, error) + CALL check("h5fcreate_f",error,total_error) + + ! + ! Create group. + ! + CALL h5gopen_f(file_id,"/",g_id, error) + CALL check("h5gopen_f",error,total_error) + + CALL check_get_name(file_id, fix_filename, total_error) + CALL check_get_name(g_id, fix_filename, total_error) + + ! Close the group. + ! + CALL h5gclose_f(g_id, error) + CALL check("h5gclose_f",error,total_error) + + ! + ! Close the file identifiers. + ! + CALL h5fclose_f(file_id, error) + CALL check("h5fclose_f",error,total_error) + + IF(cleanup) CALL h5_cleanup_f(filename, H5P_DEFAULT_F, error) + CALL check("h5_cleanup_f", error, total_error) + RETURN + + END SUBROUTINE get_name_test + + ! ! The following example demonstrates how to get creation property list, ! and access property list. diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index b3264a7..86074d0 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -745,6 +745,12 @@ New Features Fortran Library: ---------------- + + - H5Fget_name_f fixed to handle correctly trailing whitespaces and + newly allocated buffers. + + (MSB - 2021/08/30, github-826,972) + - Add wrappers for H5Pset/get_file_locking() API calls h5pget_file_locking_f() -- cgit v0.12