From f8ecdf846c7e054d874ea92baf5c9b6c2b47e836 Mon Sep 17 00:00:00 2001 From: Allen Byrne <50328838+byrnHDF@users.noreply.github.com> Date: Mon, 23 Aug 2021 13:50:17 -0500 Subject: 1.12 Tools merges from develop (#949) * Tools merges from develop Brings the tools getopt(3) replacement into the main library (#803) Perform option arg variables (#834) Removes unused commented-out code from h5jamgentest.c (#851) Reorg tools perform to provide h5perf for installation (#884) * Fix missing files * Remove empty files * Remove empty file * Resync defines * Fix endif typo * Add Makfile to list * Fix typo * Remove H5_GCC_DIAG_ON line to match removal of H5_GCC_DIAG_OFF line in sio_engine.c/ Co-authored-by: Dana Robinson <43805+derobins@users.noreply.github.com> Co-authored-by: Larry Knox --- MANIFEST | 17 +- config/cmake/H5pubconf.h.in | 11 +- config/cmake_ext_mod/ConfigureChecks.cmake | 16 +- configure.ac | 1 + hl/tools/h5watch/h5watch.c | 60 +- hl/tools/h5watch/h5watchgentest.c | 4 +- release_docs/RELEASE.txt | 34 +- src/H5private.h | 872 ++++---- src/H5system.c | 221 +- src/H5win32defs.h | 78 +- test/h5test.c | 10 +- tools/lib/h5tools_utils.c | 169 +- tools/lib/h5tools_utils.h | 52 - tools/lib/io_timer.h | 2 +- tools/src/CMakeLists.txt | 5 +- tools/src/Makefile.am | 2 +- tools/src/h5copy/h5copy.c | 48 +- tools/src/h5diff/h5diff_common.c | 108 +- tools/src/h5dump/h5dump.c | 262 +-- tools/src/h5dump/h5dump_xml.c | 39 +- tools/src/h5format_convert/h5format_convert.c | 43 +- tools/src/h5import/h5import.c | 162 +- tools/src/h5jam/h5jam.c | 20 +- tools/src/h5jam/h5unjam.c | 20 +- tools/src/h5perf/CMakeLists.txt | 103 + tools/src/h5perf/Makefile.am | 63 + tools/src/h5perf/perf.c | 802 ++++++++ tools/src/h5perf/pio_engine.c | 2747 +++++++++++++++++++++++++ tools/src/h5perf/pio_perf.c | 1718 ++++++++++++++++ tools/src/h5perf/pio_perf.h | 109 + tools/src/h5perf/sio_engine.c | 1330 ++++++++++++ tools/src/h5perf/sio_perf.c | 1301 ++++++++++++ tools/src/h5perf/sio_perf.h | 104 + tools/src/h5repack/h5repack_main.c | 162 +- tools/src/h5stat/h5stat.c | 141 +- tools/src/misc/h5clear.c | 51 +- tools/src/misc/h5mkgrp.c | 28 +- tools/test/h5jam/getub.c | 18 +- tools/test/h5jam/h5jamgentest.c | 447 ++-- tools/test/h5jam/tellub.c | 12 +- tools/test/h5stat/CMakeTests.cmake | 2 +- tools/test/h5stat/h5stat_gentest.c | 145 +- tools/test/h5stat/testh5stat.sh.in | 2 +- tools/test/misc/CMakeTestsClear.cmake | 4 +- tools/test/misc/testh5clear.sh.in | 4 +- tools/test/perform/CMakeLists.txt | 81 +- tools/test/perform/Makefile.am | 23 +- tools/test/perform/perf.c | 476 ----- tools/test/perform/pio_engine.c | 2747 ------------------------- tools/test/perform/pio_perf.c | 1699 --------------- tools/test/perform/pio_perf.h | 100 - tools/test/perform/pio_standalone.c | 192 +- tools/test/perform/pio_standalone.h | 34 +- tools/test/perform/sio_engine.c | 1338 ------------ tools/test/perform/sio_perf.c | 1437 ------------- tools/test/perform/sio_perf.h | 104 - tools/test/perform/sio_standalone.c | 57 +- tools/test/perform/sio_standalone.h | 34 +- tools/test/perform/zip_perf.c | 81 +- 59 files changed, 10013 insertions(+), 9939 deletions(-) create mode 100644 tools/src/h5perf/CMakeLists.txt create mode 100644 tools/src/h5perf/Makefile.am create mode 100644 tools/src/h5perf/perf.c create mode 100644 tools/src/h5perf/pio_engine.c create mode 100644 tools/src/h5perf/pio_perf.c create mode 100644 tools/src/h5perf/pio_perf.h create mode 100644 tools/src/h5perf/sio_engine.c create mode 100644 tools/src/h5perf/sio_perf.c create mode 100644 tools/src/h5perf/sio_perf.h delete mode 100644 tools/test/perform/perf.c delete mode 100644 tools/test/perform/pio_engine.c delete mode 100644 tools/test/perform/pio_perf.c delete mode 100644 tools/test/perform/pio_perf.h delete mode 100644 tools/test/perform/sio_engine.c delete mode 100644 tools/test/perform/sio_perf.c delete mode 100644 tools/test/perform/sio_perf.h diff --git a/MANIFEST b/MANIFEST index 465ddeb..7014c4c 100644 --- a/MANIFEST +++ b/MANIFEST @@ -2930,6 +2930,15 @@ ./tools/testfiles/h5mkgrp_single_p.ls ./tools/testfiles/h5mkgrp_single_l.ls +./tools/src/h5perf/Makefile.am +./tools/src/h5perf/perf.c +./tools/src/h5perf/pio_engine.c +./tools/src/h5perf/pio_perf.c +./tools/src/h5perf/pio_perf.h +./tools/src/h5perf/sio_engine.c +./tools/src/h5perf/sio_perf.c +./tools/src/h5perf/sio_perf.h + ./tools/test/perform/Makefile.am ./tools/test/perform/build_h5perf_alone.sh ./tools/test/perform/build_h5perf_serial_alone.sh @@ -2939,16 +2948,9 @@ ./tools/test/perform/gen_report.pl ./tools/test/perform/iopipe.c ./tools/test/perform/overhead.c -./tools/test/perform/perf.c ./tools/test/perform/perf_meta.c -./tools/test/perform/pio_engine.c -./tools/test/perform/pio_perf.c -./tools/test/perform/pio_perf.h ./tools/test/perform/pio_standalone.c ./tools/test/perform/pio_standalone.h -./tools/test/perform/sio_engine.c -./tools/test/perform/sio_perf.c -./tools/test/perform/sio_perf.h ./tools/test/perform/sio_standalone.c ./tools/test/perform/sio_standalone.h ./tools/test/perform/zip_perf.c @@ -3694,6 +3696,7 @@ ./tools/test/h5stat/CMakeLists.txt ./tools/test/h5stat/CMakeTests.cmake ./tools/src/misc/CMakeLists.txt +./tools/src/h5perf/CMakeLists.txt ./tools/test/misc/CMakeLists.txt ./tools/test/misc/CMakeTestsClear.cmake ./tools/test/misc/CMakeTestsMkgrp.cmake diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index 266f7b7..a1f9b2c 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -35,6 +35,10 @@ /* Define if dev_t is a scalar */ #cmakedefine H5_DEV_T_IS_SCALAR @H5_DEV_T_IS_SCALAR@ +/* Define if your system is IBM ppc64le and cannot convert some long double + values correctly. */ +#cmakedefine H5_DISABLE_SOME_LDOUBLE_CONV @H5_DISABLE_SOME_LDOUBLE_CONV@ + /* Define to dummy `main' function (if any) required to link to the Fortran libraries. */ #cmakedefine H5_FC_DUMMY_MAIN @H5_FC_DUMMY_MAIN@ @@ -111,7 +115,7 @@ #cmakedefine H5_HAVE_CODESTACK @H5_HAVE_CODESTACK@ /* Define to 1 if you have the header file. */ -#cmakedefine H5_HAVE_CURL_H @H5_HAVE_CURL_H@ +#cmakedefine H5_HAVE_CURL_CURL_H @H5_HAVE_CURL_H@ /* Define if Darwin or Mac OS X */ #cmakedefine H5_HAVE_DARWIN @H5_HAVE_DARWIN@ @@ -361,7 +365,7 @@ /* Define to 1 if you have the `stat64' function. */ #cmakedefine H5_HAVE_STAT64 @H5_HAVE_STAT64@ -/* Define if `struct stat' has the `st_blocks' field */ +/* Define if struct stat has the st_blocks field */ #cmakedefine H5_HAVE_STAT_ST_BLOCKS @H5_HAVE_STAT_ST_BLOCKS@ /* Define to 1 if you have the header file. */ @@ -511,9 +515,6 @@ with special algorithm. */ #cmakedefine H5_LONG_TO_LDOUBLE_SPECIAL @H5_LONG_TO_LDOUBLE_SPECIAL@ -/* Define if your system is power6 and cannot convert some long double values. */ -#cmakedefine H5_DISABLE_SOME_LDOUBLE_CONV @H5_DISABLE_SOME_LDOUBLE_CONV@ - /* Define to the sub-directory where libtool stores uninstalled libraries. */ #cmakedefine H5_LT_OBJDIR @H5_LT_OBJDIR@ diff --git a/config/cmake_ext_mod/ConfigureChecks.cmake b/config/cmake_ext_mod/ConfigureChecks.cmake index 699dfa4..45b62ea 100644 --- a/config/cmake_ext_mod/ConfigureChecks.cmake +++ b/config/cmake_ext_mod/ConfigureChecks.cmake @@ -395,15 +395,13 @@ HDF_CHECK_TYPE_SIZE (uint_least64_t ${HDF_PREFIX}_SIZEOF_UINT_LEAST64_T) HDF_CHECK_TYPE_SIZE (int_fast64_t ${HDF_PREFIX}_SIZEOF_INT_FAST64_T) HDF_CHECK_TYPE_SIZE (uint_fast64_t ${HDF_PREFIX}_SIZEOF_UINT_FAST64_T) -if (NOT APPLE) - HDF_CHECK_TYPE_SIZE (size_t ${HDF_PREFIX}_SIZEOF_SIZE_T) - HDF_CHECK_TYPE_SIZE (ssize_t ${HDF_PREFIX}_SIZEOF_SSIZE_T) - if (NOT ${HDF_PREFIX}_SIZEOF_SSIZE_T) - set (${HDF_PREFIX}_SIZEOF_SSIZE_T 0) - endif () - if (MINGW OR NOT WINDOWS) - HDF_CHECK_TYPE_SIZE (ptrdiff_t ${HDF_PREFIX}_SIZEOF_PTRDIFF_T) - endif () +HDF_CHECK_TYPE_SIZE (size_t ${HDF_PREFIX}_SIZEOF_SIZE_T) +HDF_CHECK_TYPE_SIZE (ssize_t ${HDF_PREFIX}_SIZEOF_SSIZE_T) +if (NOT ${HDF_PREFIX}_SIZEOF_SSIZE_T) + set (${HDF_PREFIX}_SIZEOF_SSIZE_T 0) +endif () +if (MINGW OR NOT WINDOWS) + HDF_CHECK_TYPE_SIZE (ptrdiff_t ${HDF_PREFIX}_SIZEOF_PTRDIFF_T) endif () HDF_CHECK_TYPE_SIZE (off_t ${HDF_PREFIX}_SIZEOF_OFF_T) diff --git a/configure.ac b/configure.ac index 8929ba4..49b7c89 100644 --- a/configure.ac +++ b/configure.ac @@ -4162,6 +4162,7 @@ AC_CONFIG_FILES([src/libhdf5.settings tools/test/misc/vds/Makefile tools/test/h5stat/Makefile tools/test/h5stat/testh5stat.sh + tools/src/h5perf/Makefile tools/test/perform/Makefile examples/Makefile examples/run-c-ex.sh diff --git a/hl/tools/h5watch/h5watch.c b/hl/tools/h5watch/h5watch.c index a5e82ca..afbc31d 100644 --- a/hl/tools/h5watch/h5watch.c +++ b/hl/tools/h5watch/h5watch.c @@ -58,27 +58,27 @@ static void parse_command_line(int argc, const char *argv[]); * The long-named ones can be partially spelled. When * adding more, make sure that they don't clash with each other. */ -static const char * s_opts = "?"; -static struct long_options l_opts[] = {{"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, - {"dim", no_arg, 'd'}, {"di", no_arg, 'd'}, - {"label", no_arg, 'l'}, {"labe", no_arg, 'l'}, - {"lab", no_arg, 'l'}, {"la", no_arg, 'l'}, - {"simple", no_arg, 'S'}, {"simpl", no_arg, 'S'}, - {"simp", no_arg, 'S'}, {"sim", no_arg, 'S'}, - {"si", no_arg, 'S'}, {"hexdump", no_arg, 'x'}, - {"hexdum", no_arg, 'x'}, {"hexdu", no_arg, 'x'}, - {"hexd", no_arg, 'x'}, {"hex", no_arg, 'x'}, - {"width", require_arg, 'w'}, {"widt", require_arg, 'w'}, - {"wid", require_arg, 'w'}, {"wi", require_arg, 'w'}, - {"polling", require_arg, 'p'}, {"pollin", require_arg, 'p'}, - {"polli", require_arg, 'p'}, {"poll", require_arg, 'p'}, - {"pol", require_arg, 'p'}, {"po", require_arg, 'p'}, - {"fields", require_arg, 'f'}, {"field", require_arg, 'f'}, - {"fiel", require_arg, 'f'}, {"fie", require_arg, 'f'}, - {"fi", require_arg, 'f'}, {"version", no_arg, 'V'}, - {"versio", no_arg, 'V'}, {"versi", no_arg, 'V'}, - {"vers", no_arg, 'V'}, {"ver", no_arg, 'V'}, - {"ve", no_arg, 'V'}, {NULL, 0, '\0'}}; +static const char * s_opts = "?"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, + {"dim", no_arg, 'd'}, {"di", no_arg, 'd'}, + {"label", no_arg, 'l'}, {"labe", no_arg, 'l'}, + {"lab", no_arg, 'l'}, {"la", no_arg, 'l'}, + {"simple", no_arg, 'S'}, {"simpl", no_arg, 'S'}, + {"simp", no_arg, 'S'}, {"sim", no_arg, 'S'}, + {"si", no_arg, 'S'}, {"hexdump", no_arg, 'x'}, + {"hexdum", no_arg, 'x'}, {"hexdu", no_arg, 'x'}, + {"hexd", no_arg, 'x'}, {"hex", no_arg, 'x'}, + {"width", require_arg, 'w'}, {"widt", require_arg, 'w'}, + {"wid", require_arg, 'w'}, {"wi", require_arg, 'w'}, + {"polling", require_arg, 'p'}, {"pollin", require_arg, 'p'}, + {"polli", require_arg, 'p'}, {"poll", require_arg, 'p'}, + {"pol", require_arg, 'p'}, {"po", require_arg, 'p'}, + {"fields", require_arg, 'f'}, {"field", require_arg, 'f'}, + {"fiel", require_arg, 'f'}, {"fie", require_arg, 'f'}, + {"fi", require_arg, 'f'}, {"version", no_arg, 'V'}, + {"versio", no_arg, 'V'}, {"versi", no_arg, 'V'}, + {"vers", no_arg, 'V'}, {"ver", no_arg, 'V'}, + {"ve", no_arg, 'V'}, {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: doprint() @@ -677,7 +677,7 @@ parse_command_line(int argc, const char *argv[]) } /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case '?': case 'h': /* --help */ @@ -691,7 +691,7 @@ parse_command_line(int argc, const char *argv[]) break; case 'w': /* --width=N */ - g_display_width = (int)HDstrtol(opt_arg, NULL, 0); + g_display_width = (int)HDstrtol(H5_optarg, NULL, 0); if (g_display_width < 0) { usage(h5tools_getprogname()); leave(EXIT_FAILURE); @@ -711,8 +711,8 @@ parse_command_line(int argc, const char *argv[]) break; case 'p': /* --polling=N */ - /* g_polling_interval = HDstrtod(opt_arg, NULL); */ - if ((tmp = (int)HDstrtol(opt_arg, NULL, 10)) <= 0) { + /* g_polling_interval = HDstrtod(H5_optarg, NULL); */ + if ((tmp = (int)HDstrtol(H5_optarg, NULL, 10)) <= 0) { usage(h5tools_getprogname()); leave(EXIT_FAILURE); } @@ -721,7 +721,7 @@ parse_command_line(int argc, const char *argv[]) case 'f': /* --fields= */ if (g_list_of_fields == NULL) { - if ((g_list_of_fields = HDstrdup(opt_arg)) == NULL) { + if ((g_list_of_fields = HDstrdup(H5_optarg)) == NULL) { error_msg("memory allocation failed (file %s:line %d)\n", __FILE__, __LINE__); leave(EXIT_FAILURE); } @@ -729,7 +729,7 @@ parse_command_line(int argc, const char *argv[]) else { char *str; - if ((str = HDstrdup(opt_arg)) == NULL) { + if ((str = HDstrdup(H5_optarg)) == NULL) { error_msg("memory allocation failed (file %s:line %d)\n", __FILE__, __LINE__); leave(EXIT_FAILURE); } @@ -751,7 +751,7 @@ parse_command_line(int argc, const char *argv[]) } /* check for object to be processed */ - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing dataset name\n"); usage(h5tools_getprogname()); leave(EXIT_FAILURE); @@ -821,7 +821,7 @@ main(int argc, const char *argv[]) /* parse command line options */ parse_command_line(argc, argv); - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing dataset name\n"); usage(h5tools_getprogname()); leave(EXIT_FAILURE); @@ -845,7 +845,7 @@ main(int argc, const char *argv[]) * then there must have been something wrong with the file (perhaps it * doesn't exist). */ - if ((fname = HDstrdup(argv[opt_ind])) == NULL) { + if ((fname = HDstrdup(argv[H5_optind])) == NULL) { error_msg("memory allocation failed (file %s:line %d)\n", __FILE__, __LINE__); h5tools_setstatus(EXIT_FAILURE); goto done; diff --git a/hl/tools/h5watch/h5watchgentest.c b/hl/tools/h5watch/h5watchgentest.c index c83e485..44cb0cb 100644 --- a/hl/tools/h5watch/h5watchgentest.c +++ b/hl/tools/h5watch/h5watchgentest.c @@ -224,7 +224,7 @@ main(void) one_cbuf[i].field2.b.a = 20; one_cbuf[i].field2.b.b = 40; one_cbuf[i].field2.b.c = 80; - one_cbuf[i].field3 = 3.0F; + one_cbuf[i].field3 = 3.0; one_cbuf[i].field4.a = 4; one_cbuf[i].field4.b = 8; } /* end for */ @@ -313,7 +313,7 @@ main(void) two_cbuf[i].field2.b.a = 20; two_cbuf[i].field2.b.b = 40; two_cbuf[i].field2.b.c = 80; - two_cbuf[i].field3 = 3.0F; + two_cbuf[i].field3 = 3.0; two_cbuf[i].field4.a = 4; two_cbuf[i].field4.b = 8; } /* end for */ diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index ecac75f..df429d7 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -108,7 +108,7 @@ New Features The Doxygen documentation has been updated and passing values larger than UINT32_MAX for size_hint will now produce a normal HDF5 error. - + (DER - 2021/04/29, HDFFV-11241) @@ -136,7 +136,7 @@ New Features The string is now handled properly and the segfault no longer occurs. (DER - 2021/04/27, HDFFV-11239) - + Parallel Library: ----------------- @@ -160,7 +160,26 @@ New Features Tools: ------ - - + - Refactored the perform tools and removed depends on test library. + + Moved the perf and h5perf tools from tools/test/perform to + tools/src/h5perf so that they can be installed. This required + that the test library dependency be removed by copying the + needed functions from h5test.c. + The standalone scripts and other perform tools remain in the + tools/test/perform folder. + + (ADB - 2021/08/10) + + - Removed partial long exceptions + + Some of the tools accepted shortened versions of the long options + (ex: --datas instead of --dataset). These were implemented inconsistently, + are difficult to maintian, and occasionally block useful long option + names. These partial long options have been removed from all the tools. + + (DER - 2021/08/03) + Support for new platforms, languages and compilers @@ -177,7 +196,7 @@ Bug Fixes since HDF5-1.12.1 release Java Library ------------ - - + - Configuration @@ -209,12 +228,12 @@ Bug Fixes since HDF5-1.12.1 release Fortran API ----------- - - + - High-Level Library ------------------ - - + - Fortran High-Level APIs @@ -224,7 +243,7 @@ Bug Fixes since HDF5-1.12.1 release Documentation ------------- - - + - F90 APIs @@ -459,7 +478,6 @@ Known Problems CMake vs. Autotools installations ================================= - While both build systems produce similar results, there are differences. Each system produces the same set of folders on linux (only CMake works on standard Windows); bin, include, lib and share. Autotools places the diff --git a/src/H5private.h b/src/H5private.h index ec33898..c9b4ab3 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -58,10 +58,10 @@ #include #endif -/* - * If _POSIX_VERSION is defined in unistd.h then this system is Posix.1 - * compliant. Otherwise all bets are off. - */ +/* POSIX headers */ +#ifdef H5_HAVE_SYS_TIME_H +#include +#endif #ifdef H5_HAVE_UNISTD_H #ifdef H5_HAVE_PWD_H #include @@ -84,9 +84,9 @@ #endif /* - * The `struct stat' data type for stat() and fstat(). This is a Posix file - * but often apears on non-Posix systems also. The `struct stat' is required - * for hdf5 to compile, although only a few fields are actually used. + * The `struct stat' data type for stat() and fstat(). This is a POSIX file + * but often apears on non-POSIX systems also. The `struct stat' is required + * for HDF5 to compile, although only a few fields are actually used. */ #ifdef H5_HAVE_SYS_STAT_H #include @@ -176,10 +176,14 @@ #define H5_DEFAULT_VOL H5VL_NATIVE #ifdef H5_HAVE_WIN32_API + /* The following two defines must be before any windows headers are included */ #define WIN32_LEAN_AND_MEAN /* Exclude rarely-used stuff from Windows headers */ #define NOGDI /* Exclude Graphic Display Interface macros */ +#include + +#include /* For _getcwd() */ #ifdef H5_HAVE_WINSOCK2_H #include #endif @@ -188,9 +192,6 @@ #include /* For _beginthread() */ #endif -#include -#include /* For _getcwd() */ - #endif /*H5_HAVE_WIN32_API*/ #ifndef F_OK @@ -636,228 +637,245 @@ typedef struct { haddr_t addr; /* The unique address of the object's header in that file */ } H5_obj_t; -/* - * Redefine all the POSIX functions. We should never see a POSIX - * function (or any other non-HDF5 function) in the source! +#define H5_SIZEOF_H5_STAT_SIZE_T H5_SIZEOF_OFF_T + +/* Put all Windows-specific definitions in H5win32defs.h so we + * can (mostly) assume a POSIX platform. Not all of the POSIX calls + * will have a Windows equivalent so some #ifdef protection is still + * necessary (e.g., fork()). + */ +#include "H5win32defs.h" + +/* Platform-independent definitions for struct stat and off_t */ +#ifndef H5_HAVE_WIN32_API +/* These definitions differ in Windows and are defined in + * H5win32defs for that platform. */ +typedef struct stat h5_stat_t; +typedef off_t h5_stat_size_t; +#define HDoff_t off_t +#endif -/* Put all platform-specific definitions in the following file */ -/* so that the following definitions are platform free. */ -#include "H5win32defs.h" /* For Windows-specific definitions */ +/* Redefine all the POSIX and C functions. We should never see an + * undecorated POSIX or C function (or any other non-HDF5 function) + * in the source. + */ #ifndef HDabort #define HDabort() abort() -#endif /* HDabort */ +#endif #ifndef HDabs #define HDabs(X) abs(X) -#endif /* HDabs */ +#endif #ifndef HDaccept -#define HDaccept(A, B, C) accept((A), (B), (C)) /* mirror VFD */ -#endif /* HDaccept */ +#define HDaccept(A, B, C) accept((A), (B), (C)) +#endif #ifndef HDaccess #define HDaccess(F, M) access(F, M) -#endif /* HDaccess */ +#endif #ifndef HDacos #define HDacos(X) acos(X) -#endif /* HDacos */ +#endif #ifndef HDalarm #ifdef H5_HAVE_ALARM #define HDalarm(N) alarm(N) -#else /* H5_HAVE_ALARM */ +#else #define HDalarm(N) (0) -#endif /* H5_HAVE_ALARM */ -#endif /* HDalarm */ +#endif +#endif #ifndef HDasctime #define HDasctime(T) asctime(T) -#endif /* HDasctime */ +#endif #ifndef HDasin #define HDasin(X) asin(X) -#endif /* HDasin */ +#endif #ifndef HDasprintf #define HDasprintf asprintf /*varargs*/ -#endif /* HDasprintf */ +#endif #ifndef HDassert #define HDassert(X) assert(X) -#endif /* HDassert */ +#endif #ifndef HDatan #define HDatan(X) atan(X) -#endif /* HDatan */ +#endif #ifndef HDatan2 #define HDatan2(X, Y) atan2(X, Y) -#endif /* HDatan2 */ +#endif #ifndef HDatexit #define HDatexit(F) atexit(F) -#endif /* HDatexit */ +#endif #ifndef HDatof #define HDatof(S) atof(S) -#endif /* HDatof */ +#endif #ifndef HDatoi #define HDatoi(S) atoi(S) -#endif /* HDatoi */ +#endif #ifndef HDatol #define HDatol(S) atol(S) -#endif /* HDatol */ +#endif #ifndef HDatoll #define HDatoll(S) atoll(S) -#endif /* HDatol */ +#endif #ifndef HDbind -#define HDbind(A, B, C) bind((A), (B), (C)) /* mirror VFD */ -#endif /* HDbind */ +#define HDbind(A, B, C) bind((A), (B), (C)) +#endif #ifndef HDbsearch #define HDbsearch(K, B, N, Z, F) bsearch(K, B, N, Z, F) -#endif /* HDbsearch */ +#endif #ifndef HDcalloc #define HDcalloc(N, Z) calloc(N, Z) -#endif /* HDcalloc */ +#endif #ifndef HDceil #define HDceil(X) ceil(X) -#endif /* HDceil */ +#endif #ifndef HDcfgetispeed #define HDcfgetispeed(T) cfgetispeed(T) -#endif /* HDcfgetispeed */ +#endif #ifndef HDcfgetospeed #define HDcfgetospeed(T) cfgetospeed(T) -#endif /* HDcfgetospeed */ +#endif #ifndef HDcfsetispeed #define HDcfsetispeed(T, S) cfsetispeed(T, S) -#endif /* HDcfsetispeed */ +#endif #ifndef HDcfsetospeed #define HDcfsetospeed(T, S) cfsetospeed(T, S) -#endif /* HDcfsetospeed */ +#endif #ifndef HDchdir #define HDchdir(S) chdir(S) -#endif /* HDchdir */ +#endif #ifndef HDchmod #define HDchmod(S, M) chmod(S, M) -#endif /* HDchmod */ +#endif #ifndef HDchown #define HDchown(S, O, G) chown(S, O, G) -#endif /* HDchown */ +#endif #ifndef HDclearerr #define HDclearerr(F) clearerr(F) -#endif /* HDclearerr */ +#endif #ifndef HDclock #define HDclock() clock() -#endif /* HDclock */ +#endif #ifndef HDclock_gettime #define HDclock_gettime(CID, TS) clock_gettime(CID, TS) -#endif /* HDclock_gettime */ +#endif #ifndef HDclose #define HDclose(F) close(F) -#endif /* HDclose */ +#endif #ifndef HDclosedir #define HDclosedir(D) closedir(D) -#endif /* HDclosedir */ +#endif #ifndef HDconnect -#define HDconnect(A, B, C) connect((A), (B), (C)) /* mirror VFD */ -#endif /* HDconnect */ +#define HDconnect(A, B, C) connect((A), (B), (C)) +#endif #ifndef HDcos #define HDcos(X) cos(X) -#endif /* HDcos */ +#endif #ifndef HDcosh #define HDcosh(X) cosh(X) -#endif /* HDcosh */ +#endif #ifndef HDcreat #define HDcreat(S, M) creat(S, M) -#endif /* HDcreat */ +#endif #ifndef HDctermid #define HDctermid(S) ctermid(S) -#endif /* HDctermid */ +#endif #ifndef HDctime #define HDctime(T) ctime(T) -#endif /* HDctime */ +#endif #ifndef HDcuserid #define HDcuserid(S) cuserid(S) -#endif /* HDcuserid */ +#endif #ifndef HDdifftime #ifdef H5_HAVE_DIFFTIME #define HDdifftime(X, Y) difftime(X, Y) -#else /* H5_HAVE_DIFFTIME */ +#else #define HDdifftime(X, Y) ((double)(X) - (double)(Y)) -#endif /* H5_HAVE_DIFFTIME */ -#endif /* HDdifftime */ +#endif +#endif #ifndef HDdiv #define HDdiv(X, Y) div(X, Y) -#endif /* HDdiv */ +#endif #ifndef HDdup #define HDdup(F) dup(F) -#endif /* HDdup */ +#endif #ifndef HDdup2 #define HDdup2(F, I) dup2(F, I) -#endif /* HDdup2 */ +#endif /* execl() variable arguments */ /* execle() variable arguments */ /* execlp() variable arguments */ #ifndef HDexecv #define HDexecv(S, AV) execv(S, AV) -#endif /* HDexecv */ +#endif #ifndef HDexecve #define HDexecve(S, AV, E) execve(S, AV, E) -#endif /* HDexecve */ +#endif #ifndef HDexecvp #define HDexecvp(S, AV) execvp(S, AV) -#endif /* HDexecvp */ +#endif #ifndef HDexit #define HDexit(N) exit(N) -#endif /* HDexit */ +#endif #ifndef HD_exit #define HD_exit(N) _exit(N) -#endif /* HD_exit */ +#endif #ifndef HDexp #define HDexp(X) exp(X) -#endif /* HDexp */ +#endif #ifndef HDexp2 #define HDexp2(X) exp2(X) -#endif /* HDexp2 */ +#endif #ifndef HDfabs #define HDfabs(X) fabs(X) -#endif /* HDfabs */ +#endif /* use ABS() because fabsf() fabsl() are not common yet. */ #ifndef HDfabsf #define HDfabsf(X) ABS(X) -#endif /* HDfabsf */ +#endif #ifndef HDfabsl #define HDfabsl(X) ABS(X) -#endif /* HDfabsl */ +#endif #ifndef HDfclose #define HDfclose(F) fclose(F) -#endif /* HDfclose */ +#endif #ifdef H5_HAVE_FCNTL #ifndef HDfcntl #define HDfcntl(F, C, ...) fcntl(F, C, __VA_ARGS__) -#endif /* HDfcntl */ -#endif /* H5_HAVE_FCNTL */ +#endif +#endif #ifndef HDfdopen #define HDfdopen(N, S) fdopen(N, S) -#endif /* HDfdopen */ +#endif #ifndef HDfeof #define HDfeof(F) feof(F) -#endif /* HDfeof */ +#endif #ifndef HDferror #define HDferror(F) ferror(F) -#endif /* HDferror */ +#endif #ifndef HDfflush #define HDfflush(F) fflush(F) -#endif /* HDfflush */ +#endif #ifndef HDfgetc #define HDfgetc(F) fgetc(F) -#endif /* HDfgetc */ +#endif #ifndef HDfgetpos #define HDfgetpos(F, P) fgetpos(F, P) -#endif /* HDfgetpos */ +#endif #ifndef HDfgets #define HDfgets(S, N, F) fgets(S, N, F) -#endif /* HDfgets */ +#endif #ifndef HDfileno #define HDfileno(F) fileno(F) -#endif /* HDfileno */ +#endif + /* Since flock is so prevalent, always build these functions * when possible to avoid them becoming dead code. */ #ifdef H5_HAVE_FCNTL H5_DLL int Pflock(int fd, int operation); -#endif /* H5_HAVE_FCNTL */ +#endif H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation); + #ifndef HDflock /* NOTE: flock(2) is not present on all POSIX systems. * If it is not present, we try a flock() equivalent based on @@ -871,760 +889,764 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation); #define HDflock(F, L) Pflock(F, L) #else #define HDflock(F, L) Nflock(F, L) -#endif /* H5_HAVE_FLOCK */ +#endif + #endif /* HDflock */ + #ifndef HDfloor #define HDfloor(X) floor(X) -#endif /* HDfloor */ +#endif #ifndef HDfmod #define HDfmod(X, Y) fmod(X, Y) -#endif /* HDfmod */ +#endif #ifndef HDfopen #define HDfopen(S, M) fopen(S, M) -#endif /* HDfopen */ +#endif #ifndef HDfork #define HDfork() fork() -#endif /* HDfork */ +#endif #ifndef HDfprintf #define HDfprintf fprintf #endif #ifndef HDfpathconf #define HDfpathconf(F, N) fpathconf(F, N) -#endif /* HDfpathconf */ +#endif #ifndef HDfputc #define HDfputc(C, F) fputc(C, F) -#endif /* HDfputc */ +#endif #ifndef HDfputs #define HDfputs(S, F) fputs(S, F) -#endif /* HDfputs */ +#endif #ifndef HDfread #define HDfread(M, Z, N, F) fread(M, Z, N, F) -#endif /* HDfread */ +#endif #ifndef HDfree #define HDfree(M) free(M) -#endif /* HDfree */ +#endif #ifndef HDfreopen #define HDfreopen(S, M, F) freopen(S, M, F) -#endif /* HDfreopen */ +#endif #ifndef HDfrexp #define HDfrexp(X, N) frexp(X, N) -#endif /* HDfrexp */ +#endif /* Check for Cray-specific 'frexpf()' and 'frexpl()' routines */ #ifndef HDfrexpf #ifdef H5_HAVE_FREXPF #define HDfrexpf(X, N) frexpf(X, N) -#else /* H5_HAVE_FREXPF */ +#else #define HDfrexpf(X, N) frexp(X, N) -#endif /* H5_HAVE_FREXPF */ -#endif /* HDfrexpf */ +#endif +#endif #ifndef HDfrexpl #ifdef H5_HAVE_FREXPL #define HDfrexpl(X, N) frexpl(X, N) -#else /* H5_HAVE_FREXPL */ +#else #define HDfrexpl(X, N) frexp(X, N) -#endif /* H5_HAVE_FREXPL */ -#endif /* HDfrexpl */ -/* fscanf() variable arguments */ +#endif +#endif +#ifndef HDfscanf +#define HDfscanf fscanf +#endif #ifndef HDfseek #define HDfseek(F, O, W) fseeko(F, O, W) -#endif /* HDfseek */ +#endif #ifndef HDfsetpos #define HDfsetpos(F, P) fsetpos(F, P) -#endif /* HDfsetpos */ +#endif #ifndef HDfstat #define HDfstat(F, B) fstat(F, B) -#endif /* HDfstat */ -#ifndef HDlstat -#define HDlstat(S, B) lstat(S, B) -#endif /* HDlstat */ -#ifndef HDstat -#define HDstat(S, B) stat(S, B) -#endif /* HDstat */ - -#ifndef H5_HAVE_WIN32_API -/* These definitions differ in Windows and are defined in - * H5win32defs for that platform. - */ -typedef struct stat h5_stat_t; -typedef off_t h5_stat_size_t; -#define HDoff_t off_t -#endif /* H5_HAVE_WIN32_API */ - -#define H5_SIZEOF_H5_STAT_SIZE_T H5_SIZEOF_OFF_T - +#endif #ifndef HDftell #define HDftell(F) ftell(F) -#endif /* HDftell */ +#endif #ifndef HDftruncate #define HDftruncate(F, L) ftruncate(F, L) -#endif /* HDftruncate */ +#endif #ifndef HDfwrite #define HDfwrite(M, Z, N, F) fwrite(M, Z, N, F) -#endif /* HDfwrite */ +#endif #ifndef HDgetc #define HDgetc(F) getc(F) -#endif /* HDgetc */ +#endif #ifndef HDgetchar #define HDgetchar() getchar() -#endif /* HDgetchar */ +#endif #ifndef HDgetcwd #define HDgetcwd(S, Z) getcwd(S, Z) -#endif /* HDgetcwd */ +#endif #ifndef HDgetdcwd #define HDgetdcwd(D, S, Z) getcwd(S, Z) -#endif /* HDgetdcwd */ +#endif + +/* Windows only - set to zero on other systems */ #ifndef HDgetdrive #define HDgetdrive() 0 -#endif /* HDgetdrive */ +#endif + #ifndef HDgetegid #define HDgetegid() getegid() -#endif /* HDgetegid() */ +#endif #ifndef HDgetenv #define HDgetenv(S) getenv(S) -#endif /* HDgetenv */ +#endif #ifndef HDgeteuid #define HDgeteuid() geteuid() -#endif /* HDgeteuid */ +#endif #ifndef HDgetgid #define HDgetgid() getgid() -#endif /* HDgetgid */ +#endif #ifndef HDgetgrgid #define HDgetgrgid(G) getgrgid(G) -#endif /* HDgetgrgid */ +#endif #ifndef HDgetgrnam #define HDgetgrnam(S) getgrnam(S) -#endif /* HDgetgrnam */ +#endif #ifndef HDgetgroups #define HDgetgroups(Z, G) getgroups(Z, G) -#endif /* HDgetgroups */ +#endif #ifndef HDgethostbyaddr -#define HDgethostbyaddr(A, B, C) gethostbyaddr((A), (B), (C)) /* mirror VFD */ -#endif /* HDgethostbyaddr */ +#define HDgethostbyaddr(A, B, C) gethostbyaddr((A), (B), (C)) +#endif #ifndef HDgethostname #define HDgethostname(N, L) gethostname(N, L) -#endif /* HDgethostname */ +#endif #ifndef HDgetlogin #define HDgetlogin() getlogin() -#endif /* HDgetlogin */ +#endif #ifndef HDgetpgrp #define HDgetpgrp() getpgrp() -#endif /* HDgetpgrp */ +#endif #ifndef HDgetpid #define HDgetpid() getpid() -#endif /* HDgetpid */ +#endif #ifndef HDgetppid #define HDgetppid() getppid() -#endif /* HDgetppid */ +#endif #ifndef HDgetpwnam #define HDgetpwnam(S) getpwnam(S) -#endif /* HDgetpwnam */ +#endif #ifndef HDgetpwuid #define HDgetpwuid(U) getpwuid(U) -#endif /* HDgetpwuid */ +#endif #ifndef HDgetrusage #define HDgetrusage(X, S) getrusage(X, S) -#endif /* HDgetrusage */ +#endif /* Don't define HDgets - gets() was deprecated in C99 and removed in C11 */ #ifdef HDgets #undef HDgets -#endif /* HDgets */ +#endif #ifndef HDgettimeofday #define HDgettimeofday(S, P) gettimeofday(S, P) -#endif /* HDgettimeofday */ +#endif #ifndef HDgetuid #define HDgetuid() getuid() -#endif /* HDgetuid */ +#endif #ifndef HDgmtime #define HDgmtime(T) gmtime(T) -#endif /* HDgmtime */ +#endif #ifndef HDhtonl -#define HDhtonl(X) htonl((X)) /* mirror VFD */ -#endif /* HDhtonl */ +#define HDhtonl(X) htonl((X)) +#endif #ifndef HDhtons -#define HDhtons(X) htons((X)) /* mirror VFD */ -#endif /* HDhtons */ +#define HDhtons(X) htons((X)) +#endif #ifndef HDinet_addr -#define HDinet_addr(C) inet_addr((C)) /* mirror VFD */ -#endif /* HDinet_addr */ +#define HDinet_addr(C) inet_addr((C)) +#endif #ifndef HDinet_ntoa -#define HDinet_ntoa(C) inet_ntoa((C)) /* mirror VFD */ -#endif /* HDinet_ntoa */ +#define HDinet_ntoa(C) inet_ntoa((C)) +#endif #ifndef HDisalnum -#define HDisalnum(C) isalnum((int)(C)) /*cast for solaris warning*/ -#endif /* HDisalnum */ +#define HDisalnum(C) isalnum((int)(C)) /* Cast for Solaris warning */ +#endif #ifndef HDisalpha -#define HDisalpha(C) isalpha((int)(C)) /*cast for solaris warning*/ -#endif /* HDisalpha */ +#define HDisalpha(C) isalpha((int)(C)) /* Cast for Solaris warning */ +#endif #ifndef HDisatty #define HDisatty(F) isatty(F) -#endif /* HDisatty */ +#endif #ifndef HDiscntrl -#define HDiscntrl(C) iscntrl((int)(C)) /*cast for solaris warning*/ -#endif /* HDiscntrl */ +#define HDiscntrl(C) iscntrl((int)(C)) /* Cast for solaris warning */ +#endif #ifndef HDisdigit -#define HDisdigit(C) isdigit((int)(C)) /*cast for solaris warning*/ -#endif /* HDisdigit */ +#define HDisdigit(C) isdigit((int)(C)) /* Cast for Solaris warning */ +#endif #ifndef HDisgraph -#define HDisgraph(C) isgraph((int)(C)) /*cast for solaris warning*/ -#endif /* HDisgraph */ +#define HDisgraph(C) isgraph((int)(C)) /* Cast for Solaris warning*/ +#endif #ifndef HDislower -#define HDislower(C) islower((int)(C)) /*cast for solaris warning*/ -#endif /* HDislower */ +#define HDislower(C) islower((int)(C)) /* Cast for Solaris warning */ +#endif #ifndef HDisnan #define HDisnan(X) isnan(X) -#endif /* HDisnan */ +#endif #ifndef HDisprint -#define HDisprint(C) isprint((int)(C)) /*cast for solaris warning*/ -#endif /* HDisprint */ +#define HDisprint(C) isprint((int)(C)) /* Cast for Solaris warning */ +#endif #ifndef HDispunct -#define HDispunct(C) ispunct((int)(C)) /*cast for solaris warning*/ -#endif /* HDispunct */ +#define HDispunct(C) ispunct((int)(C)) /* Cast for Solaris warning */ +#endif #ifndef HDisspace -#define HDisspace(C) isspace((int)(C)) /*cast for solaris warning*/ -#endif /* HDisspace */ +#define HDisspace(C) isspace((int)(C)) /* Cast for Solaris warning */ +#endif #ifndef HDisupper -#define HDisupper(C) isupper((int)(C)) /*cast for solaris warning*/ -#endif /* HDisupper */ +#define HDisupper(C) isupper((int)(C)) /* Cast for Solaris warning */ +#endif #ifndef HDisxdigit -#define HDisxdigit(C) isxdigit((int)(C)) /*cast for solaris warning*/ -#endif /* HDisxdigit */ +#define HDisxdigit(C) isxdigit((int)(C)) /* Cast for Solaris warning */ +#endif #ifndef HDkill #define HDkill(P, S) kill(P, S) -#endif /* HDkill */ +#endif #ifndef HDlabs #define HDlabs(X) labs(X) -#endif /* HDlabs */ +#endif #ifndef HDldexp #define HDldexp(X, N) ldexp(X, N) -#endif /* HDldexp */ +#endif #ifndef HDldiv #define HDldiv(X, Y) ldiv(X, Y) -#endif /* HDldiv */ +#endif #ifndef HDlink #define HDlink(OLD, NEW) link(OLD, NEW) -#endif /* HDlink */ +#endif #ifndef HDlisten -#define HDlisten(A, B) listen((A), (B)) /* mirror VFD */ -#endif /* HDlisten */ +#define HDlisten(A, B) listen((A), (B)) +#endif #ifndef HDllround #define HDllround(V) llround(V) -#endif /* HDround */ +#endif #ifndef HDllroundf #define HDllroundf(V) llroundf(V) -#endif /* HDllroundf */ +#endif #ifndef HDllroundl #define HDllroundl(V) llroundl(V) -#endif /* HDllroundl */ +#endif #ifndef HDlocaleconv #define HDlocaleconv() localeconv() -#endif /* HDlocaleconv */ +#endif #ifndef HDlocaltime #define HDlocaltime(T) localtime(T) -#endif /* HDlocaltime */ +#endif #ifndef HDlog #define HDlog(X) log(X) -#endif /* HDlog */ +#endif #ifndef HDlog10 #define HDlog10(X) log10(X) -#endif /* HDlog10 */ +#endif #ifndef HDlongjmp #define HDlongjmp(J, N) longjmp(J, N) -#endif /* HDlongjmp */ +#endif #ifndef HDlround #define HDlround(V) lround(V) -#endif /* HDround */ +#endif #ifndef HDlroundf #define HDlroundf(V) lroundf(V) -#endif /* HDlroundf */ +#endif #ifndef HDlroundl #define HDlroundl(V) lroundl(V) -#endif /* HDroundl */ +#endif #ifndef HDlseek #define HDlseek(F, O, W) lseek(F, O, W) -#endif /* HDlseek */ +#endif +#ifndef HDlstat +#define HDlstat(S, B) lstat(S, B) +#endif #ifndef HDmalloc #define HDmalloc(Z) malloc(Z) -#endif /* HDmalloc */ +#endif #ifndef HDposix_memalign #define HDposix_memalign(P, A, Z) posix_memalign(P, A, Z) -#endif /* HDposix_memalign */ +#endif #ifndef HDmblen #define HDmblen(S, N) mblen(S, N) -#endif /* HDmblen */ +#endif #ifndef HDmbstowcs #define HDmbstowcs(P, S, Z) mbstowcs(P, S, Z) -#endif /* HDmbstowcs */ +#endif #ifndef HDmbtowc #define HDmbtowc(P, S, Z) mbtowc(P, S, Z) -#endif /* HDmbtowc */ +#endif #ifndef HDmemchr #define HDmemchr(S, C, Z) memchr(S, C, Z) -#endif /* HDmemchr */ +#endif #ifndef HDmemcmp #define HDmemcmp(X, Y, Z) memcmp(X, Y, Z) -#endif /* HDmemcmp */ +#endif #ifndef HDmemcpy #define HDmemcpy(X, Y, Z) memcpy(X, Y, Z) -#endif /* HDmemcpy */ +#endif #ifndef HDmemmove #define HDmemmove(X, Y, Z) memmove((char *)(X), (const char *)(Y), Z) -#endif /* HDmemmove */ +#endif #ifndef HDmemset #define HDmemset(X, C, Z) memset(X, C, Z) -#endif /* HDmemset */ +#endif #ifndef HDmkdir #define HDmkdir(S, M) mkdir(S, M) -#endif /* HDmkdir */ +#endif #ifndef HDmkfifo #define HDmkfifo(S, M) mkfifo(S, M) -#endif /* HDmkfifo */ +#endif #ifndef HDmktime #define HDmktime(T) mktime(T) -#endif /* HDmktime */ +#endif #ifndef HDmodf #define HDmodf(X, Y) modf(X, Y) -#endif /* HDmodf */ +#endif #ifndef HDnanosleep #define HDnanosleep(N, O) nanosleep(N, O) -#endif /* HDnanosleep */ +#endif #ifndef HDntohl -#define HDntohl(A) ntohl((A)) /* mirror VFD */ -#endif /* HDntohl */ +#define HDntohl(A) ntohl((A)) +#endif #ifndef HDntohs -#define HDntohs(A) ntohs((A)) /* mirror VFD */ -#endif /* HDntohs */ +#define HDntohs(A) ntohs((A)) +#endif #ifndef HDopen #define HDopen(F, ...) open(F, __VA_ARGS__) -#endif /* HDopen */ +#endif #ifndef HDopendir #define HDopendir(S) opendir(S) -#endif /* HDopendir */ +#endif #ifndef HDpathconf #define HDpathconf(S, N) pathconf(S, N) -#endif /* HDpathconf */ +#endif #ifndef HDpause #define HDpause() pause() -#endif /* HDpause */ +#endif #ifndef HDperror #define HDperror(S) perror(S) -#endif /* HDperror */ +#endif #ifndef HDpipe #define HDpipe(F) pipe(F) -#endif /* HDpipe */ +#endif #ifndef HDpow #define HDpow(X, Y) pow(X, Y) -#endif /* HDpow */ +#endif #ifndef HDpowf #define HDpowf(X, Y) powf(X, Y) -#endif /* HDpowf */ +#endif #ifndef HDpread #define HDpread(F, B, C, O) pread(F, B, C, O) -#endif /* HDpread */ +#endif #ifndef HDprintf -#define HDprintf printf -#endif /* HDprintf */ +#define HDprintf printf /*varargs*/ +#endif #ifndef HDputc #define HDputc(C, F) putc(C, F) -#endif /* HDputc*/ +#endif #ifndef HDputchar #define HDputchar(C) putchar(C) -#endif /* HDputchar */ +#endif #ifndef HDputs #define HDputs(S) puts(S) -#endif /* HDputs */ +#endif #ifndef HDpwrite #define HDpwrite(F, B, C, O) pwrite(F, B, C, O) -#endif /* HDpwrite */ +#endif #ifndef HDqsort #define HDqsort(M, N, Z, F) qsort(M, N, Z, F) -#endif /* HDqsort*/ +#endif #ifndef HDraise #define HDraise(N) raise(N) -#endif /* HDraise */ +#endif +/* clang-format off */ #ifdef H5_HAVE_RAND_R -#ifndef HDrandom -#define HDrandom() HDrand() -#endif /* HDrandom */ -H5_DLL int HDrand(void); -#ifndef HDsrandom -#define HDsrandom(S) HDsrand(S) -#endif /* HDsrandom */ -H5_DLL void HDsrand(unsigned int seed); +# ifndef HDrandom +# define HDrandom() HDrand() +# endif + H5_DLL int HDrand(void); +# ifndef HDsrandom +# define HDsrandom(S) HDsrand(S) +# endif + H5_DLL void HDsrand(unsigned int seed); #elif defined(H5_HAVE_RANDOM) -#ifndef HDrand -#define HDrand() random() -#endif /* HDrand */ -#ifndef HDrandom -#define HDrandom() random() -#endif /* HDrandom */ -#ifndef HDsrand -#define HDsrand(S) srandom(S) -#endif /* HDsrand */ -#ifndef HDsrandom -#define HDsrandom(S) srandom(S) -#endif /* HDsrandom */ -#else /* H5_HAVE_RANDOM */ -#ifndef HDrand -#define HDrand() rand() -#endif /* HDrand */ -#ifndef HDrandom -#define HDrandom() rand() -#endif /* HDrandom */ -#ifndef HDsrand -#define HDsrand(S) srand(S) -#endif /* HDsrand */ -#ifndef HDsrandom -#define HDsrandom(S) srand(S) -#endif /* HDsrandom */ -#endif /* H5_HAVE_RANDOM */ +# ifndef HDrand +# define HDrand() random() +# endif +# ifndef HDrandom +# define HDrandom() random() +# endif +# ifndef HDsrand +# define HDsrand(S) srandom(S) +# endif +# ifndef HDsrandom +# define HDsrandom(S) srandom(S) +# endif +#else +# ifndef HDrand +# define HDrand() rand() +# endif +# ifndef HDrandom +# define HDrandom() rand() +# endif +# ifndef HDsrand +# define HDsrand(S) srand(S) +# endif +# ifndef HDsrandom +# define HDsrandom(S) srand(S) +# endif +#endif +/* clang-format on */ #ifndef HDread #define HDread(F, M, Z) read(F, M, Z) -#endif /* HDread */ +#endif #ifndef HDreaddir #define HDreaddir(D) readdir(D) -#endif /* HDreaddir */ +#endif #ifndef HDrealloc #define HDrealloc(M, Z) realloc(M, Z) -#endif /* HDrealloc */ +#endif #ifndef HDrealpath #define HDrealpath(F1, F2) realpath(F1, F2) -#endif /* HDrealloc */ +#endif #ifndef HDremove #define HDremove(S) remove(S) -#endif /* HDremove */ +#endif #ifndef HDrename #define HDrename(OLD, NEW) rename(OLD, NEW) -#endif /* HDrename */ +#endif #ifndef HDrewind #define HDrewind(F) rewind(F) -#endif /* HDrewind */ +#endif #ifndef HDrewinddir #define HDrewinddir(D) rewinddir(D) -#endif /* HDrewinddir */ +#endif #ifndef HDround #define HDround(V) round(V) -#endif /* HDround */ +#endif #ifndef HDroundf #define HDroundf(V) roundf(V) -#endif /* HDroundf */ +#endif #ifndef HDroundl #define HDroundl(V) roundl(V) -#endif /* HDroundl */ +#endif #ifndef HDrmdir #define HDrmdir(S) rmdir(S) -#endif /* HDrmdir */ -/* scanf() variable arguments */ +#endif +#ifndef HDscanf +#define HDscanf scanf /*varargs*/ +#endif #ifndef HDselect #define HDselect(N, RD, WR, ER, T) select(N, RD, WR, ER, T) -#endif /* HDsetbuf */ +#endif #ifndef HDsetbuf #define HDsetbuf(F, S) setbuf(F, S) -#endif /* HDsetbuf */ +#endif #ifndef HDsetenv #define HDsetenv(N, V, O) setenv(N, V, O) -#endif /* HDsetenv */ +#endif #ifndef HDsetgid #define HDsetgid(G) setgid(G) -#endif /* HDsetgid */ +#endif #ifndef HDsetjmp #define HDsetjmp(J) setjmp(J) -#endif /* HDsetjmp */ +#endif #ifndef HDsetlocale #define HDsetlocale(N, S) setlocale(N, S) -#endif /* HDsetlocale */ +#endif #ifndef HDsetpgid #define HDsetpgid(P, PG) setpgid(P, PG) -#endif /* HDsetpgid */ +#endif #ifndef HDsetsid #define HDsetsid() setsid() -#endif /* HDsetsid */ +#endif #ifndef HDsetsockopt -#define HDsetsockopt(A, B, C, D, E) setsockopt((A), (B), (C), (D), (E)) /* mirror VFD */ -#endif /* HDsetsockopt */ +#define HDsetsockopt(A, B, C, D, E) setsockopt((A), (B), (C), (D), (E)) +#endif #ifndef HDsetuid #define HDsetuid(U) setuid(U) -#endif /* HDsetuid */ +#endif #ifndef HDsetvbuf #define HDsetvbuf(F, S, M, Z) setvbuf(F, S, M, Z) -#endif /* HDsetvbuf */ +#endif #ifndef HDshutdown -#define HDshutdown(A, B) shutdown((A), (B)) /* mirror VFD */ -#endif /* HDshutdown */ +#define HDshutdown(A, B) shutdown((A), (B)) +#endif #ifndef HDsigaction #define HDsigaction(S, A, O) sigaction((S), (A), (O)) -#endif /* HDsigaction */ +#endif #ifndef HDsigaddset #define HDsigaddset(S, N) sigaddset(S, N) -#endif /* HDsigaddset */ +#endif #ifndef HDsigdelset #define HDsigdelset(S, N) sigdelset(S, N) -#endif /* HDsigdelset */ +#endif #ifndef HDsigemptyset #define HDsigemptyset(S) sigemptyset(S) -#endif /* HDsigemptyset */ +#endif #ifndef HDsigfillset #define HDsigfillset(S) sigfillset(S) -#endif /* HDsigfillset */ +#endif #ifndef HDsigismember #define HDsigismember(S, N) sigismember(S, N) -#endif /* HDsigismember */ +#endif #ifndef HDsiglongjmp #define HDsiglongjmp(J, N) siglongjmp(J, N) -#endif /* HDsiglongjmp */ +#endif #ifndef HDsignal #define HDsignal(N, F) signal(N, F) -#endif /* HDsignal */ +#endif #ifndef HDsigpending #define HDsigpending(S) sigpending(S) -#endif /* HDsigpending */ +#endif #ifndef HDsigprocmask #define HDsigprocmask(H, S, O) sigprocmask(H, S, O) -#endif /* HDsigprocmask */ +#endif #ifndef HDsigsetjmp #define HDsigsetjmp(J, N) sigsetjmp(J, N) -#endif /* HDsigsetjmp */ +#endif #ifndef HDsigsuspend #define HDsigsuspend(S) sigsuspend(S) -#endif /* HDsigsuspend */ +#endif #ifndef HDsin #define HDsin(X) sin(X) -#endif /* HDsin */ +#endif #ifndef HDsinh #define HDsinh(X) sinh(X) -#endif /* HDsinh */ +#endif #ifndef HDsleep #define HDsleep(N) sleep(N) -#endif /* HDsleep */ +#endif #ifndef HDsnprintf #define HDsnprintf snprintf /*varargs*/ -#endif /* HDsnprintf */ +#endif #ifndef HDsocket -#define HDsocket(A, B, C) socket((A), (B), (C)) /* mirror VFD */ -#endif /* HDsocket */ +#define HDsocket(A, B, C) socket((A), (B), (C)) +#endif #ifndef HDsprintf #define HDsprintf sprintf /*varargs*/ -#endif /* HDsprintf */ +#endif #ifndef HDsqrt #define HDsqrt(X) sqrt(X) -#endif /* HDsqrt */ +#endif #ifndef HDsscanf -#define HDsscanf(S, FMT, ...) sscanf(S, FMT, __VA_ARGS__) -#endif /* HDsscanf */ +#define HDsscanf sscanf /*varargs*/ +#endif +#ifndef HDstat +#define HDstat(S, B) stat(S, B) +#endif #ifndef HDstrcat #define HDstrcat(X, Y) strcat(X, Y) -#endif /* HDstrcat */ +#endif #ifndef HDstrchr #define HDstrchr(S, C) strchr(S, C) -#endif /* HDstrchr */ +#endif #ifndef HDstrcmp #define HDstrcmp(X, Y) strcmp(X, Y) -#endif /* HDstrcmp */ +#endif #ifndef HDstrcasecmp #define HDstrcasecmp(X, Y) strcasecmp(X, Y) -#endif /* HDstrcasecmp */ +#endif #ifndef HDstrcoll #define HDstrcoll(X, Y) strcoll(X, Y) -#endif /* HDstrcoll */ +#endif #ifndef HDstrcpy #define HDstrcpy(X, Y) strcpy(X, Y) -#endif /* HDstrcpy */ +#endif #ifndef HDstrcspn #define HDstrcspn(X, Y) strcspn(X, Y) -#endif /* HDstrcspn */ +#endif #ifndef HDstrdup #define HDstrdup(S) strdup(S) -#endif /* HDstrdup */ +#endif #ifndef HDstrerror #define HDstrerror(N) strerror(N) -#endif /* HDstrerror */ +#endif #ifndef HDstrftime #define HDstrftime(S, Z, F, T) strftime(S, Z, F, T) -#endif /* HDstrftime */ +#endif #ifndef HDstrlen #define HDstrlen(S) strlen(S) -#endif /* HDstrlen */ +#endif #ifndef HDstrncat #define HDstrncat(X, Y, Z) strncat(X, Y, Z) -#endif /* HDstrncat */ +#endif #ifndef HDstrncmp #define HDstrncmp(X, Y, Z) strncmp(X, Y, Z) -#endif /* HDstrncmp */ +#endif #ifndef HDstrncpy #define HDstrncpy(X, Y, Z) strncpy(X, Y, Z) -#endif /* HDstrncpy */ +#endif #ifndef HDstrpbrk #define HDstrpbrk(X, Y) strpbrk(X, Y) -#endif /* HDstrpbrk */ +#endif #ifndef HDstrrchr #define HDstrrchr(S, C) strrchr(S, C) -#endif /* HDstrrchr */ +#endif #ifndef HDstrspn #define HDstrspn(X, Y) strspn(X, Y) -#endif /* HDstrspn */ +#endif #ifndef HDstrstr #define HDstrstr(X, Y) strstr(X, Y) -#endif /* HDstrstr */ +#endif #ifndef HDstrtod #define HDstrtod(S, R) strtod(S, R) -#endif /* HDstrtod */ +#endif #ifndef HDstrtok #define HDstrtok(X, Y) strtok(X, Y) -#endif /* HDstrtok */ +#endif #ifndef HDstrtok_r #define HDstrtok_r(X, Y, Z) strtok_r(X, Y, Z) -#endif /* HDstrtok */ +#endif #ifndef HDstrtol #define HDstrtol(S, R, N) strtol(S, R, N) -#endif /* HDstrtol */ +#endif #ifndef HDstrtoll #ifdef H5_HAVE_STRTOLL #define HDstrtoll(S, R, N) strtoll(S, R, N) #else H5_DLL int64_t HDstrtoll(const char *s, const char **rest, int base); -#endif /* H5_HAVE_STRTOLL */ -#endif /* HDstrtoll */ +#endif +#endif #ifndef HDstrtoul #define HDstrtoul(S, R, N) strtoul(S, R, N) -#endif /* HDstrtoul */ +#endif #ifndef HDstrtoull #define HDstrtoull(S, R, N) strtoull(S, R, N) -#endif /* HDstrtoul */ +#endif #ifndef HDstrtoumax #define HDstrtoumax(S, R, N) strtoumax(S, R, N) -#endif /* HDstrtoumax */ +#endif #ifndef HDstrxfrm #define HDstrxfrm(X, Y, Z) strxfrm(X, Y, Z) -#endif /* HDstrxfrm */ +#endif #ifdef H5_HAVE_SYMLINK #ifndef HDsymlink #define HDsymlink(F1, F2) symlink(F1, F2) -#endif /* HDsymlink */ -#endif /* H5_HAVE_SYMLINK */ +#endif +#endif #ifndef HDsysconf #define HDsysconf(N) sysconf(N) -#endif /* HDsysconf */ +#endif #ifndef HDsystem #define HDsystem(S) system(S) -#endif /* HDsystem */ +#endif #ifndef HDtan #define HDtan(X) tan(X) -#endif /* HDtan */ +#endif #ifndef HDtanh #define HDtanh(X) tanh(X) -#endif /* HDtanh */ +#endif #ifndef HDtcdrain #define HDtcdrain(F) tcdrain(F) -#endif /* HDtcdrain */ +#endif #ifndef HDtcflow #define HDtcflow(F, A) tcflow(F, A) -#endif /* HDtcflow */ +#endif #ifndef HDtcflush #define HDtcflush(F, N) tcflush(F, N) -#endif /* HDtcflush */ +#endif #ifndef HDtcgetattr #define HDtcgetattr(F, T) tcgetattr(F, T) -#endif /* HDtcgetattr */ +#endif #ifndef HDtcgetpgrp #define HDtcgetpgrp(F) tcgetpgrp(F) -#endif /* HDtcgetpgrp */ +#endif #ifndef HDtcsendbreak #define HDtcsendbreak(F, N) tcsendbreak(F, N) -#endif /* HDtcsendbreak */ +#endif #ifndef HDtcsetattr #define HDtcsetattr(F, O, T) tcsetattr(F, O, T) -#endif /* HDtcsetattr */ +#endif #ifndef HDtcsetpgrp #define HDtcsetpgrp(F, N) tcsetpgrp(F, N) -#endif /* HDtcsetpgrp */ +#endif #ifndef HDtime #define HDtime(T) time(T) -#endif /* HDtime */ +#endif #ifndef HDtimes #define HDtimes(T) times(T) -#endif /* HDtimes*/ +#endif #ifndef HDtmpfile #define HDtmpfile() tmpfile() -#endif /* HDtmpfile */ +#endif #ifndef HDtmpnam #define HDtmpnam(S) tmpnam(S) -#endif /* HDtmpnam */ +#endif #ifndef HDtolower #define HDtolower(C) tolower(C) -#endif /* HDtolower */ +#endif #ifndef HDtoupper #define HDtoupper(C) toupper(C) -#endif /* HDtoupper */ +#endif #ifndef HDttyname #define HDttyname(F) ttyname(F) -#endif /* HDttyname */ +#endif #ifndef HDtzset #define HDtzset() tzset() -#endif /* HDtzset */ +#endif #ifndef HDumask #define HDumask(N) umask(N) -#endif /* HDumask */ +#endif #ifndef HDuname #define HDuname(S) uname(S) -#endif /* HDuname */ +#endif #ifndef HDungetc #define HDungetc(C, F) ungetc(C, F) -#endif /* HDungetc */ +#endif #ifndef HDunlink #define HDunlink(S) unlink(S) -#endif /* HDunlink */ +#endif +#ifndef HDunsetenv +#define HDunsetenv(S) unsetenv(S) +#endif #ifndef HDutime #define HDutime(S, T) utime(S, T) -#endif /* HDutime */ +#endif + #ifndef HDvasprintf #ifdef H5_HAVE_VASPRINTF #define HDvasprintf(RET, FMT, A) vasprintf(RET, FMT, A) #else H5_DLL int HDvasprintf(char **bufp, const char *fmt, va_list _ap); -#endif /* H5_HAVE_VASPRINTF */ -#endif /* HDvasprintf */ +#endif +#endif + #ifndef HDva_arg #define HDva_arg(A, T) va_arg(A, T) -#endif /* HDva_arg */ +#endif #ifndef HDva_copy #define HDva_copy(D, S) va_copy(D, S) -#endif /* HDva_copy */ +#endif #ifndef HDva_end #define HDva_end(A) va_end(A) -#endif /* HDva_end */ +#endif #ifndef HDva_start #define HDva_start(A, P) va_start(A, P) -#endif /* HDva_start */ +#endif #ifndef HDvfprintf #define HDvfprintf(F, FMT, A) vfprintf(F, FMT, A) -#endif /* HDvfprintf */ +#endif #ifndef HDvprintf #define HDvprintf(FMT, A) vprintf(FMT, A) -#endif /* HDvprintf */ +#endif #ifndef HDvsprintf #define HDvsprintf(S, FMT, A) vsprintf(S, FMT, A) -#endif /* HDvsprintf */ +#endif #ifndef HDvsnprintf #define HDvsnprintf(S, N, FMT, A) vsnprintf(S, N, FMT, A) -#endif /* HDvsnprintf */ +#endif #ifndef HDwait #define HDwait(W) wait(W) -#endif /* HDwait */ +#endif #ifndef HDwaitpid #define HDwaitpid(P, W, O) waitpid(P, W, O) -#endif /* HDwaitpid */ +#endif #ifndef HDwcstombs #define HDwcstombs(S, P, Z) wcstombs(S, P, Z) -#endif /* HDwcstombs */ +#endif #ifndef HDwctomb #define HDwctomb(S, C) wctomb(S, C) -#endif /* HDwctomb */ +#endif #ifndef HDwrite #define HDwrite(F, M, Z) write(F, M, Z) -#endif /* HDwrite */ +#endif /* Macro for "stringizing" an integer in the C preprocessor (use H5_TOSTRING) */ /* (use H5_TOSTRING, H5_STRINGIZE is just part of the implementation) */ @@ -2009,7 +2031,8 @@ typedef struct H5_api_struct { /* Macros for thread cancellation-safe mechanism */ #define H5_API_UNSET_CANCEL H5TS_cancel_count_inc(); -#define H5_API_SET_CANCEL H5TS_cancel_count_dec(); + +#define H5_API_SET_CANCEL H5TS_cancel_count_dec(); extern H5_api_t H5_g; @@ -2569,6 +2592,57 @@ H5_DLL double H5_get_time(void); H5_DLL herr_t H5_build_extpath(const char *name, char **extpath /*out*/); H5_DLL herr_t H5_combine_path(const char *path1, const char *path2, char **full_name /*out*/); +/* getopt(3) equivalent that papers over the lack of long options on BSD + * and lack of Windows support. + */ +H5_DLLVAR int H5_opterr; /* get_option prints errors if this is on */ +H5_DLLVAR int H5_optind; /* token pointer */ +H5_DLLVAR const char *H5_optarg; /* flag argument (or value) */ + +enum h5_arg_level { + no_arg = 0, /* doesn't take an argument */ + require_arg, /* requires an argument */ + optional_arg /* argument is optional */ +}; + +/* + * get_option determines which options are specified on the command line and + * returns a pointer to any arguments possibly associated with the option in + * the ``H5_optarg'' variable. get_option returns the shortname equivalent of + * the option. The long options are specified in the following way: + * + * struct h5_long_options foo[] = { + * { "filename", require_arg, 'f' }, + * { "append", no_arg, 'a' }, + * { "width", require_arg, 'w' }, + * { NULL, 0, 0 } + * }; + * + * Long named options can have arguments specified as either: + * + * ``--param=arg'' or ``--param arg'' + * + * Short named options can have arguments specified as either: + * + * ``-w80'' or ``-w 80'' + * + * and can have more than one short named option specified at one time: + * + * -aw80 + * + * in which case those options which expect an argument need to come at the + * end. + */ +struct h5_long_options { + const char * name; /* Name of the long option */ + enum h5_arg_level has_arg; /* Whether we should look for an arg */ + char shortval; /* The shortname equivalent of long arg + * this gets returned from get_option + */ +}; + +H5_DLL int H5_get_option(int argc, const char **argv, const char *opt, const struct h5_long_options *l_opt); + #ifdef H5_HAVE_PARALLEL /* Generic MPI functions */ H5_DLL hsize_t H5_mpi_set_bigio_count(hsize_t new_count); diff --git a/src/H5system.c b/src/H5system.c index 144e0bc..5cbc480 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -460,6 +460,9 @@ Wgettimeofday(struct timeval *tv, struct timezone *tz) * Interestingly, getenv *is* available in the Windows * POSIX layer, just not setenv. * + * Note: Passing an empty string ("") for the value will remove + * the variable from the environment (like unsetenv(3)) + * * Return: Success: 0 * Failure: non-zero error code * @@ -471,14 +474,14 @@ Wgettimeofday(struct timeval *tv, struct timezone *tz) int Wsetenv(const char *name, const char *value, int overwrite) { - size_t bufsize; - errno_t err; - /* If we're not overwriting, check if the environment variable exists. * If it does (i.e.: the required buffer size to store the variable's * value is non-zero), then return an error code. */ if (!overwrite) { + size_t bufsize; + errno_t err; + err = getenv_s(&bufsize, NULL, 0, name); if (err || bufsize) return (int)err; @@ -487,7 +490,7 @@ Wsetenv(const char *name, const char *value, int overwrite) return (int)_putenv_s(name, value); } /* end Wsetenv() */ -#ifdef H5_HAVE_WINSOCK2_H +#ifdef H5_HAVE_WIN32_API #pragma comment(lib, "advapi32.lib") #endif @@ -572,12 +575,12 @@ char * Wgetlogin(void) { -#ifdef H5_HAVE_WINSOCK2_H +#ifdef H5_HAVE_WIN32_API DWORD bufferCount = WloginBuffer_count; if (GetUserName(Wlogin_buffer, &bufferCount) != 0) return (Wlogin_buffer); else -#endif /* H5_HAVE_WINSOCK2_H */ +#endif return NULL; } @@ -964,10 +967,7 @@ done: * Note that commodity hardware is probably going to have a * resolution of milliseconds, not nanoseconds. * - * Return: SUCCEED/FAIL - * - * Programmer: Quincey Koziol - * October 01, 2016 + * Return: void *-------------------------------------------------------------------------- */ void @@ -976,21 +976,40 @@ H5_nanosleep(uint64_t nanosec) FUNC_ENTER_NOAPI_NOINIT_NOERR #ifdef H5_HAVE_WIN32_API + DWORD dwMilliseconds = (DWORD)HDceil(nanosec / 1.0e6); + DWORD ignore; - /* On Windows, Sleep() is in milliseconds. Passing 0 to Sleep() - * causes the thread to relinquish the rest of its time slice. + /* Windows can't sleep at a ns resolution. Best we can do is ~1 ms. We + * don't care about the return value since the second parameter + * (bAlertable) is FALSE, so it will always be zero. */ - Sleep(nanosec / (1000 * 1000)); + ignore = SleepEx(dwMilliseconds, FALSE); #else - { - struct timespec sleeptime; /* Struct to hold time to sleep */ - /* Set up time to sleep */ - sleeptime.tv_sec = 0; - sleeptime.tv_nsec = (long)nanosec; + const uint64_t nanosec_per_sec = 1000 * 1000 * 1000; + struct timespec sleeptime; /* Struct to hold time to sleep */ + + /* Set up time to sleep + * + * Assuming ILP32 or LP64 or wider architecture, (long)operand + * satisfies 0 <= operand < nanosec_per_sec < LONG_MAX. + * + * It's harder to be sure that we don't overflow time_t. + */ + sleeptime.tv_sec = (time_t)(nanosec / nanosec_per_sec); + sleeptime.tv_nsec = (long)(nanosec % nanosec_per_sec); - HDnanosleep(&sleeptime, NULL); + /* Sleep for up to `sleeptime` and, in the event of an interruption, + * save the unslept time back to `sleeptime`. + */ + while (HDnanosleep(&sleeptime, &sleeptime) == -1) { + /* If we were just interrupted, sleep for the remaining time. + * Otherwise, the error was essentially impossible, so just stop + * sleeping. + */ + if (errno != EINTR) + break; } #endif @@ -1042,3 +1061,167 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5_expand_windows_env_vars() */ #endif /* H5_HAVE_WIN32_API */ + +/* Global variables */ +int H5_opterr = 1; /* Get_option prints errors if this is on */ +int H5_optind = 1; /* Token pointer */ +const char *H5_optarg; /* Flag argument (or value) */ + +/*------------------------------------------------------------------------- + * Function: H5_get_option + * + * Purpose: Determine the command-line options a user specified. We can + * accept both short and long type command-lines. + * + * Return: Success: The short valued "name" of the command line + * parameter or EOF if there are no more + * parameters to process. + * + * Failure: A question mark. + *------------------------------------------------------------------------- + */ +int +H5_get_option(int argc, const char **argv, const char *opts, const struct h5_long_options *l_opts) +{ + static int sp = 1; /* character index in current token */ + int optopt = '?'; /* option character passed back to user */ + + if (sp == 1) { + /* check for more flag-like tokens */ + if (H5_optind >= argc || argv[H5_optind][0] != '-' || argv[H5_optind][1] == '\0') { + return EOF; + } + else if (HDstrcmp(argv[H5_optind], "--") == 0) { + H5_optind++; + return EOF; + } + } + + if (sp == 1 && argv[H5_optind][0] == '-' && argv[H5_optind][1] == '-') { + /* long command line option */ + int i; + const char ch = '='; + char * arg = HDstrdup(&argv[H5_optind][2]); + size_t arg_len = 0; + + H5_optarg = strchr(&argv[H5_optind][2], ch); + arg_len = HDstrlen(&argv[H5_optind][2]); + if (H5_optarg) { + arg_len -= HDstrlen(H5_optarg); + H5_optarg++; /* skip the equal sign */ + } + arg[arg_len] = 0; + + for (i = 0; l_opts && l_opts[i].name; i++) { + if (HDstrcmp(arg, l_opts[i].name) == 0) { + /* we've found a matching long command line flag */ + optopt = l_opts[i].shortval; + + if (l_opts[i].has_arg != no_arg) { + if (H5_optarg == NULL) { + if (l_opts[i].has_arg != optional_arg) { + if (H5_optind < (argc - 1)) + if (argv[H5_optind + 1][0] != '-') + H5_optarg = argv[++H5_optind]; + } + else if (l_opts[i].has_arg == require_arg) { + if (H5_opterr) + HDfprintf(stderr, "%s: option required for \"--%s\" flag\n", argv[0], arg); + + optopt = '?'; + } + } + } + else { + if (H5_optarg) { + if (H5_opterr) + HDfprintf(stderr, "%s: no option required for \"%s\" flag\n", argv[0], arg); + + optopt = '?'; + } + } + break; + } + } + + if (l_opts[i].name == NULL) { + /* exhausted all of the l_opts we have and still didn't match */ + if (H5_opterr) + HDfprintf(stderr, "%s: unknown option \"%s\"\n", argv[0], arg); + + optopt = '?'; + } + + H5_optind++; + sp = 1; + + HDfree(arg); + } + else { + register char *cp; /* pointer into current token */ + + /* short command line option */ + optopt = argv[H5_optind][sp]; + + if (optopt == ':' || (cp = HDstrchr(opts, optopt)) == 0) { + if (H5_opterr) + HDfprintf(stderr, "%s: unknown option \"%c\"\n", argv[0], optopt); + + /* if no chars left in this token, move to next token */ + if (argv[H5_optind][++sp] == '\0') { + H5_optind++; + sp = 1; + } + return '?'; + } + + if (*++cp == ':') { + /* if a value is expected, get it */ + if (argv[H5_optind][sp + 1] != '\0') { + /* flag value is rest of current token */ + H5_optarg = &argv[H5_optind++][sp + 1]; + } + else if (++H5_optind >= argc) { + if (H5_opterr) + HDfprintf(stderr, "%s: value expected for option \"%c\"\n", argv[0], optopt); + + optopt = '?'; + } + else { + /* flag value is next token */ + H5_optarg = argv[H5_optind++]; + } + + sp = 1; + } + /* wildcard argument */ + else if (*cp == '*') { + /* check the next argument */ + H5_optind++; + /* we do have an extra argument, check if not last */ + if ((H5_optind + 1) < argc) { + if (argv[H5_optind][0] != '-') { + H5_optarg = argv[H5_optind++]; + } + else { + H5_optarg = NULL; + } + } + else { + H5_optarg = NULL; + } + } + else { + /* set up to look at next char in token, next time */ + if (argv[H5_optind][++sp] == '\0') { + /* no more in current token, so setup next token */ + H5_optind++; + sp = 1; + } + H5_optarg = NULL; + } + } + + /* return the current flag character found */ + return optopt; +} diff --git a/src/H5win32defs.h b/src/H5win32defs.h index 88911b0..985ad3c 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -29,24 +29,36 @@ typedef struct _stati64 h5_stat_t; typedef __int64 h5_stat_size_t; -#define HDaccess(F, M) _access(F, M) -#define HDchdir(S) _chdir(S) -#define HDclose(F) _close(F) -#define HDcreat(S, M) Wopen_utf8(S, O_CREAT | O_TRUNC | O_RDWR, M) -#define HDdup(F) _dup(F) -#define HDfdopen(N, S) _fdopen(N, S) -#define HDfileno(F) _fileno(F) -#define HDfstat(F, B) _fstati64(F, B) -#define HDisatty(F) _isatty(F) +#ifdef H5_HAVE_VISUAL_STUDIO -#define HDgetcwd(S, Z) _getcwd(S, Z) -#define HDgetdcwd(D, S, Z) _getdcwd(D, S, Z) -#define HDgetdrive() _getdrive() -#define HDlseek(F, O, W) _lseeki64(F, O, W) -#define HDlstat(S, B) _lstati64(S, B) -#define HDmkdir(S, M) _mkdir(S) -#define HDnanosleep(N, O) Wnanosleep(N, O) -#define HDoff_t __int64 +struct timezone { + int tz_minuteswest; + int tz_dsttime; +}; + +#endif /* H5_HAVE_VISUAL_STUDIO */ + +#define HDaccess(F, M) _access(F, M) +#define HDchdir(S) _chdir(S) +#define HDclose(F) _close(F) +#define HDcreat(S, M) Wopen_utf8(S, O_CREAT | O_TRUNC | O_RDWR, M) +#define HDdup(F) _dup(F) +#define HDfdopen(N, S) _fdopen(N, S) +#define HDfileno(F) _fileno(F) +#define HDflock(F, L) Wflock(F, L) +#define HDfstat(F, B) _fstati64(F, B) +#define HDgetcwd(S, Z) _getcwd(S, Z) +#define HDgetdcwd(D, S, Z) _getdcwd(D, S, Z) +#define HDgetdrive() _getdrive() +#define HDgetlogin() Wgetlogin() +#define HDgettimeofday(V, Z) Wgettimeofday(V, Z) +#define HDisatty(F) _isatty(F) +#define HDlseek(F, O, W) _lseeki64(F, O, W) +#define HDlstat(S, B) _lstati64(S, B) +#define HDmemset(X, C, Z) memset((void *)(X), C, Z) /* Cast avoids MSVC warning */ +#define HDmkdir(S, M) _mkdir(S) +#define HDnanosleep(N, O) Wnanosleep(N, O) +#define HDoff_t __int64 /* Note that the variadic HDopen macro is using a VC++ extension * where the comma is dropped if nothing is passed to the ellipsis. @@ -56,9 +68,11 @@ typedef __int64 h5_stat_size_t; #else #define HDopen(S, F, ...) Wopen_utf8(S, F, ##__VA_ARGS__) #endif + #define HDread(F, M, Z) _read(F, M, Z) #define HDremove(S) Wremove_utf8(S) #define HDrmdir(S) _rmdir(S) +#define HDsetenv(N, V, O) Wsetenv(N, V, O) #define HDsetvbuf(F, S, M, Z) setvbuf(F, S, M, (Z > 1 ? Z : 2)) #define HDsleep(S) Sleep(S * 1000) #define HDstat(S, B) _stati64(S, B) @@ -67,21 +81,13 @@ typedef __int64 h5_stat_size_t; #define HDstrtok_r(X, Y, Z) strtok_s(X, Y, Z) #define HDtzset() _tzset() #define HDunlink(S) _unlink(S) +#define HDunsetenv(N) Wsetenv(N, "", 1) #define HDwrite(F, M, Z) _write(F, M, Z) -#ifdef H5_HAVE_VISUAL_STUDIO - -/* - * The (void*) cast just avoids a compiler warning in MSVC - */ -#define HDmemset(X, C, Z) memset((void *)(X), C, Z) - -struct timezone { - int tz_minuteswest; - int tz_dsttime; -}; - -#endif /* H5_HAVE_VISUAL_STUDIO */ +#ifndef H5_HAVE_MINGW +#define HDftruncate(F, L) _chsize_s(F, L) +#define HDfseek(F, O, W) _fseeki64(F, O, W) +#endif /* H5_HAVE_MINGW */ #ifdef __cplusplus extern "C" { @@ -99,16 +105,4 @@ H5_DLL int H5_get_win32_times(H5_timevals_t *tvs); } #endif /* __cplusplus */ -#define HDgettimeofday(V, Z) Wgettimeofday(V, Z) -#define HDsetenv(N, V, O) Wsetenv(N, V, O) -#define HDflock(F, L) Wflock(F, L) -#define HDgetlogin() Wgetlogin() - -/* Non-POSIX functions */ - -#ifndef H5_HAVE_MINGW -#define HDftruncate(F, L) _chsize_s(F, L) -#define HDfseek(F, O, W) _fseeki64(F, O, W) -#endif /* H5_HAVE_MINGW */ - #endif /* H5_HAVE_WIN32_API */ diff --git a/test/h5test.c b/test/h5test.c index b37409a..bb2f5cc 100644 --- a/test/h5test.c +++ b/test/h5test.c @@ -1237,9 +1237,10 @@ h5_set_info_object(void) valp++; /* copy key/value pair into temporary buffer */ - len = strcspn(valp, ";"); - next = &valp[len]; - key_val = (char *)HDcalloc(1, len + 1); + len = HDstrcspn(valp, ";"); + next = &valp[len]; + if (NULL == (key_val = (char *)HDcalloc(1, len + 1))) + return -1; /* increment the next pointer past the terminating semicolon */ if (*next == ';') @@ -1635,6 +1636,9 @@ h5_make_local_copy(const char *origfilename, const char *local_copy_name) void * buf = NULL; /* Buffer for copying data */ const char *filename = H5_get_srcdir_filename(origfilename); /* Get the test file name to copy */ + if (!filename) + goto error; + /* Allocate copy buffer */ if (NULL == (buf = HDcalloc((size_t)1, (size_t)READ_BUF_SIZE))) goto error; diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index 6aec6b8..f2407bf 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -25,12 +25,8 @@ #include "H5FDros3.h" #endif -/* global variables */ -unsigned h5tools_nCols = 80; -/* ``get_option'' variables */ -int opt_err = 1; /*get_option prints errors if this is on */ -int opt_ind = 1; /*token pointer */ -const char * opt_arg; /*flag argument (or value) */ +/* Global variables */ +unsigned h5tools_nCols = 80; static int h5tools_d_status = 0; static const char *h5tools_progname = "h5tools"; @@ -162,167 +158,6 @@ help_ref_msg(FILE *output) HDfprintf(output, "see the <%s> entry in the 'HDF5 Reference Manual'.\n", h5tools_getprogname()); } -/*------------------------------------------------------------------------- - * Function: get_option - * - * Purpose: Determine the command-line options a user specified. We can - * accept both short and long type command-lines. - * - * Return: Success: The short valued "name" of the command line - * parameter or EOF if there are no more - * parameters to process. - * - * Failure: A question mark. - *------------------------------------------------------------------------- - */ -int -get_option(int argc, const char **argv, const char *opts, const struct long_options *l_opts) -{ - static int sp = 1; /* character index in current token */ - int opt_opt = '?'; /* option character passed back to user */ - - if (sp == 1) { - /* check for more flag-like tokens */ - if (opt_ind >= argc || argv[opt_ind][0] != '-' || argv[opt_ind][1] == '\0') { - return EOF; - } - else if (HDstrcmp(argv[opt_ind], "--") == 0) { - opt_ind++; - return EOF; - } - } - - if (sp == 1 && argv[opt_ind][0] == '-' && argv[opt_ind][1] == '-') { - /* long command line option */ - int i; - const char ch = '='; - char * arg = HDstrdup(&argv[opt_ind][2]); - size_t arg_len = 0; - - opt_arg = strchr(&argv[opt_ind][2], ch); - arg_len = HDstrlen(&argv[opt_ind][2]); - if (opt_arg) { - arg_len -= HDstrlen(opt_arg); - opt_arg++; /* skip the equal sign */ - } - arg[arg_len] = 0; - - for (i = 0; l_opts && l_opts[i].name; i++) { - if (HDstrcmp(arg, l_opts[i].name) == 0) { - /* we've found a matching long command line flag */ - opt_opt = l_opts[i].shortval; - - if (l_opts[i].has_arg != no_arg) { - if (opt_arg == NULL) { - if (l_opts[i].has_arg != optional_arg) { - if (opt_ind < (argc - 1)) - if (argv[opt_ind + 1][0] != '-') - opt_arg = argv[++opt_ind]; - } - else if (l_opts[i].has_arg == require_arg) { - if (opt_err) - HDfprintf(rawerrorstream, "%s: option required for \"--%s\" flag\n", argv[0], - arg); - - opt_opt = '?'; - } - } - } - else { - if (opt_arg) { - if (opt_err) - HDfprintf(rawerrorstream, "%s: no option required for \"%s\" flag\n", argv[0], - arg); - - opt_opt = '?'; - } - } - break; - } - } - - if (l_opts[i].name == NULL) { - /* exhausted all of the l_opts we have and still didn't match */ - if (opt_err) - HDfprintf(rawerrorstream, "%s: unknown option \"%s\"\n", argv[0], arg); - - opt_opt = '?'; - } - - opt_ind++; - sp = 1; - - HDfree(arg); - } - else { - register char *cp; /* pointer into current token */ - - /* short command line option */ - opt_opt = argv[opt_ind][sp]; - - if (opt_opt == ':' || (cp = HDstrchr(opts, opt_opt)) == 0) { - if (opt_err) - HDfprintf(rawerrorstream, "%s: unknown option \"%c\"\n", argv[0], opt_opt); - - /* if no chars left in this token, move to next token */ - if (argv[opt_ind][++sp] == '\0') { - opt_ind++; - sp = 1; - } - return '?'; - } - - if (*++cp == ':') { - /* if a value is expected, get it */ - if (argv[opt_ind][sp + 1] != '\0') { - /* flag value is rest of current token */ - opt_arg = &argv[opt_ind++][sp + 1]; - } - else if (++opt_ind >= argc) { - if (opt_err) - HDfprintf(rawerrorstream, "%s: value expected for option \"%c\"\n", argv[0], opt_opt); - - opt_opt = '?'; - } - else { - /* flag value is next token */ - opt_arg = argv[opt_ind++]; - } - - sp = 1; - } - /* wildcard argument */ - else if (*cp == '*') { - /* check the next argument */ - opt_ind++; - /* we do have an extra argument, check if not last */ - if ((opt_ind + 1) < argc) { - if (argv[opt_ind][0] != '-') { - opt_arg = argv[opt_ind++]; - } - else { - opt_arg = NULL; - } - } - else { - opt_arg = NULL; - } - } - else { - /* set up to look at next char in token, next time */ - if (argv[opt_ind][++sp] == '\0') { - /* no more in current token, so setup next token */ - opt_ind++; - sp = 1; - } - opt_arg = NULL; - } - } - - /* return the current flag character found */ - return opt_opt; -} - /***************************************************************************** * * Function: parse_tuple() diff --git a/tools/lib/h5tools_utils.h b/tools/lib/h5tools_utils.h index 307ce20..095ad6c 100644 --- a/tools/lib/h5tools_utils.h +++ b/tools/lib/h5tools_utils.h @@ -40,58 +40,6 @@ H5TOOLS_DLLVAR FILE *overflow_file; H5TOOLS_DLLVAR hsize_t H5TOOLS_MALLOCSIZE; /* size of hyperslab buffer when a dataset is bigger than H5TOOLS_MALLOCSIZE */ H5TOOLS_DLLVAR hsize_t H5TOOLS_BUFSIZE; -/* - * begin get_option section - */ -H5TOOLS_DLLVAR int opt_err; /* getoption prints errors if this is on */ -H5TOOLS_DLLVAR int opt_ind; /* token pointer */ -H5TOOLS_DLLVAR const char *opt_arg; /* flag argument (or value) */ - -enum { - no_arg = 0, /* doesn't take an argument */ - require_arg, /* requires an argument */ - optional_arg /* argument is optional */ -}; - -/* - * get_option determines which options are specified on the command line and - * returns a pointer to any arguments possibly associated with the option in - * the ``opt_arg'' variable. get_option returns the shortname equivalent of - * the option. The long options are specified in the following way: - * - * struct long_options foo[] = { - * { "filename", require_arg, 'f' }, - * { "append", no_arg, 'a' }, - * { "width", require_arg, 'w' }, - * { NULL, 0, 0 } - * }; - * - * Long named options can have arguments specified as either: - * - * ``--param=arg'' or ``--param arg'' - * - * Short named options can have arguments specified as either: - * - * ``-w80'' or ``-w 80'' - * - * and can have more than one short named option specified at one time: - * - * -aw80 - * - * in which case those options which expect an argument need to come at the - * end. - */ -typedef struct long_options { - const char *name; /* name of the long option */ - int has_arg; /* whether we should look for an arg */ - char shortval; /* the shortname equivalent of long arg - * this gets returned from get_option */ -} long_options; - -H5TOOLS_DLL int get_option(int argc, const char **argv, const char *opt, const struct long_options *l_opt); -/* - * end get_option section - */ /*struct taken from the dumper. needed in table struct*/ typedef struct obj_t { diff --git a/tools/lib/io_timer.h b/tools/lib/io_timer.h index 0269b93..02951e1b 100644 --- a/tools/lib/io_timer.h +++ b/tools/lib/io_timer.h @@ -27,7 +27,7 @@ #ifdef H5_HAVE_WINSOCK2_H #include -#endif /* H5_HAVE_WINSOCK2_H */ +#endif /* The different types of timers we can have */ typedef enum timer_type_ { diff --git a/tools/src/CMakeLists.txt b/tools/src/CMakeLists.txt index 8c3e361..e291f61 100644 --- a/tools/src/CMakeLists.txt +++ b/tools/src/CMakeLists.txt @@ -13,7 +13,7 @@ add_subdirectory (misc) #-- Add the h5import and test executables add_subdirectory (h5import) -#-- h5Repack executables +#-- h5repack executables add_subdirectory (h5repack) #-- Add the h5dump and test executables @@ -30,3 +30,6 @@ add_subdirectory (h5dump) #-- Add the h5format_convert and test executables add_subdirectory (h5format_convert) + +#-- h5perf executables +add_subdirectory (h5perf) diff --git a/tools/src/Makefile.am b/tools/src/Makefile.am index 397bd31..5af7d06 100644 --- a/tools/src/Makefile.am +++ b/tools/src/Makefile.am @@ -23,6 +23,6 @@ CONFIG=ordered # All subdirectories SUBDIRS=h5diff h5ls h5dump misc h5import h5repack h5jam h5copy \ - h5format_convert h5stat + h5format_convert h5stat h5perf include $(top_srcdir)/config/conclude.am diff --git a/tools/src/h5copy/h5copy.c b/tools/src/h5copy/h5copy.c index f618914..4e9a25e 100644 --- a/tools/src/h5copy/h5copy.c +++ b/tools/src/h5copy/h5copy.c @@ -19,23 +19,23 @@ #define PROGRAMNAME "h5copy" /* command-line options: short and long-named parameters */ -static const char * s_opts = "d:f:hi:o:ps:vVE"; -static struct long_options l_opts[] = {{"destination", require_arg, 'd'}, - {"flag", require_arg, 'f'}, - {"help", no_arg, 'h'}, - {"input", require_arg, 'i'}, - {"output", require_arg, 'o'}, - {"parents", no_arg, 'p'}, - {"source", require_arg, 's'}, - {"verbose", no_arg, 'v'}, - {"version", no_arg, 'V'}, - {"enable-error-stack", no_arg, 'E'}, - {NULL, 0, '\0'}}; -char * fname_src = NULL; -char * fname_dst = NULL; -char * oname_src = NULL; -char * oname_dst = NULL; -char * str_flag = NULL; +static const char * s_opts = "d:f:hi:o:ps:vVE"; +static struct h5_long_options l_opts[] = {{"destination", require_arg, 'd'}, + {"flag", require_arg, 'f'}, + {"help", no_arg, 'h'}, + {"input", require_arg, 'i'}, + {"output", require_arg, 'o'}, + {"parents", no_arg, 'p'}, + {"source", require_arg, 's'}, + {"verbose", no_arg, 'v'}, + {"version", no_arg, 'V'}, + {"enable-error-stack", no_arg, 'E'}, + {NULL, 0, '\0'}}; +char * fname_src = NULL; +char * fname_dst = NULL; +char * oname_src = NULL; +char * oname_dst = NULL; +char * str_flag = NULL; /*------------------------------------------------------------------------- * Function: leave @@ -242,19 +242,19 @@ main(int argc, const char *argv[]) } /* end if */ /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'd': - oname_dst = HDstrdup(opt_arg); + oname_dst = HDstrdup(H5_optarg); break; case 'f': /* validate flag */ - if (parse_flag(opt_arg, &flag) < 0) { + if (parse_flag(H5_optarg, &flag) < 0) { usage(); leave(EXIT_FAILURE); } - str_flag = HDstrdup(opt_arg); + str_flag = HDstrdup(H5_optarg); break; case 'h': @@ -263,11 +263,11 @@ main(int argc, const char *argv[]) break; case 'i': - fname_src = HDstrdup(opt_arg); + fname_src = HDstrdup(H5_optarg); break; case 'o': - fname_dst = HDstrdup(opt_arg); + fname_dst = HDstrdup(H5_optarg); break; case 'p': @@ -275,7 +275,7 @@ main(int argc, const char *argv[]) break; case 's': - oname_src = HDstrdup(opt_arg); + oname_src = HDstrdup(H5_optarg); break; case 'V': diff --git a/tools/src/h5diff/h5diff_common.c b/tools/src/h5diff/h5diff_common.c index 1b81824..b669087 100644 --- a/tools/src/h5diff/h5diff_common.c +++ b/tools/src/h5diff/h5diff_common.c @@ -25,30 +25,30 @@ static int check_d_input(const char *); * Command-line options: The user can specify short or long-named * parameters. */ -static const char * s_opts = "hVrv*qn:d:p:NcelxE:A:S"; -static struct long_options l_opts[] = {{"help", no_arg, 'h'}, - {"version", no_arg, 'V'}, - {"report", no_arg, 'r'}, - {"verbose", optional_arg, 'v'}, - {"quiet", no_arg, 'q'}, - {"count", require_arg, 'n'}, - {"delta", require_arg, 'd'}, - {"relative", require_arg, 'p'}, - {"nan", no_arg, 'N'}, - {"compare", no_arg, 'c'}, - {"use-system-epsilon", no_arg, 'e'}, - {"follow-symlinks", no_arg, 'l'}, - {"no-dangling-links", no_arg, 'x'}, - {"exclude-path", require_arg, 'E'}, - {"exclude-attribute", require_arg, 'A'}, - {"enable-error-stack", no_arg, 'S'}, - {"vol-value-1", require_arg, '1'}, - {"vol-name-1", require_arg, '2'}, - {"vol-info-1", require_arg, '3'}, - {"vol-value-2", require_arg, '4'}, - {"vol-name-2", require_arg, '5'}, - {"vol-info-2", require_arg, '6'}, - {NULL, 0, '\0'}}; +static const char * s_opts = "hVrv*qn:d:p:NcelxE:A:S"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, + {"version", no_arg, 'V'}, + {"report", no_arg, 'r'}, + {"verbose", optional_arg, 'v'}, + {"quiet", no_arg, 'q'}, + {"count", require_arg, 'n'}, + {"delta", require_arg, 'd'}, + {"relative", require_arg, 'p'}, + {"nan", no_arg, 'N'}, + {"compare", no_arg, 'c'}, + {"use-system-epsilon", no_arg, 'e'}, + {"follow-symlinks", no_arg, 'l'}, + {"no-dangling-links", no_arg, 'x'}, + {"exclude-path", require_arg, 'E'}, + {"exclude-attribute", require_arg, 'A'}, + {"enable-error-stack", no_arg, 'S'}, + {"vol-value-1", require_arg, '1'}, + {"vol-name-1", require_arg, '2'}, + {"vol-info-1", require_arg, '3'}, + {"vol-value-2", require_arg, '4'}, + {"vol-name-2", require_arg, '5'}, + {"vol-info-2", require_arg, '6'}, + {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: check_options @@ -232,7 +232,7 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char exclude_attr_head = NULL; /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { default: usage(); @@ -256,20 +256,20 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char * special check for short opt */ if (!strcmp(argv[i], "-v")) { - if (opt_arg != NULL) - opt_ind--; + if (H5_optarg != NULL) + H5_optind--; opts->mode_verbose_level = 0; break; } else if (!strncmp(argv[i], "-v", (size_t)2)) { - if (opt_arg != NULL) - opt_ind--; + if (H5_optarg != NULL) + H5_optind--; opts->mode_verbose_level = atoi(&argv[i][2]); break; } else { - if (opt_arg != NULL) - opts->mode_verbose_level = HDatoi(opt_arg); + if (H5_optarg != NULL) + opts->mode_verbose_level = HDatoi(H5_optarg); else opts->mode_verbose_level = 0; } @@ -308,7 +308,7 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char } /* init */ - exclude_node->obj_path = opt_arg; + exclude_node->obj_path = H5_optarg; exclude_node->obj_type = H5TRAV_TYPE_UNKNOWN; exclude_prev = exclude_head; @@ -336,7 +336,7 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char } /* init */ - exclude_attr_node->obj_path = opt_arg; + exclude_attr_node->obj_path = H5_optarg; exclude_attr_node->obj_type = H5TRAV_TYPE_UNKNOWN; exclude_attr_prev = exclude_attr_head; @@ -356,23 +356,23 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char case 'd': opts->delta_bool = 1; - if (check_d_input(opt_arg) == -1) { - HDprintf("<-d %s> is not a valid option\n", opt_arg); + if (check_d_input(H5_optarg) == -1) { + HDprintf("<-d %s> is not a valid option\n", H5_optarg); usage(); h5diff_exit(EXIT_FAILURE); } - opts->delta = HDatof(opt_arg); + opts->delta = HDatof(H5_optarg); /* do not check against default, the DBL_EPSILON is being replaced by user */ break; case 'p': opts->percent_bool = 1; - if (check_p_input(opt_arg) == -1) { - HDprintf("<-p %s> is not a valid option\n", opt_arg); + if (check_p_input(H5_optarg) == -1) { + HDprintf("<-p %s> is not a valid option\n", H5_optarg); usage(); h5diff_exit(EXIT_FAILURE); } - opts->percent = HDatof(opt_arg); + opts->percent = HDatof(H5_optarg); /* -p 0 is the same as default */ if (H5_DBL_ABS_EQUAL(opts->percent, 0.0)) @@ -381,12 +381,12 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char case 'n': opts->count_bool = 1; - if (check_n_input(opt_arg) == -1) { - HDprintf("<-n %s> is not a valid option\n", opt_arg); + if (check_n_input(H5_optarg) == -1) { + HDprintf("<-n %s> is not a valid option\n", H5_optarg); usage(); h5diff_exit(EXIT_FAILURE); } - opts->count = HDstrtoull(opt_arg, NULL, 0); + opts->count = HDstrtoull(H5_optarg, NULL, 0); break; case 'N': @@ -403,34 +403,34 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char case '1': opts->vol_info[0].type = VOL_BY_VALUE; - opts->vol_info[0].u.value = (H5VL_class_value_t)HDatoi(opt_arg); + opts->vol_info[0].u.value = (H5VL_class_value_t)HDatoi(H5_optarg); opts->custom_vol[0] = TRUE; break; case '2': opts->vol_info[0].type = VOL_BY_NAME; - opts->vol_info[0].u.name = opt_arg; + opts->vol_info[0].u.name = H5_optarg; opts->custom_vol[0] = TRUE; break; case '3': - opts->vol_info[0].info_string = opt_arg; + opts->vol_info[0].info_string = H5_optarg; break; case '4': opts->vol_info[1].type = VOL_BY_VALUE; - opts->vol_info[1].u.value = (H5VL_class_value_t)HDatoi(opt_arg); + opts->vol_info[1].u.value = (H5VL_class_value_t)HDatoi(H5_optarg); opts->custom_vol[1] = TRUE; break; case '5': opts->vol_info[1].type = VOL_BY_NAME; - opts->vol_info[1].u.name = opt_arg; + opts->vol_info[1].u.name = H5_optarg; opts->custom_vol[1] = TRUE; break; case '6': - opts->vol_info[1].info_string = opt_arg; + opts->vol_info[1].info_string = H5_optarg; break; } } @@ -447,15 +447,15 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char opts->exclude_attr = exclude_attr_head; /* check for file names to be processed */ - if (argc <= opt_ind || argv[opt_ind + 1] == NULL) { + if (argc <= H5_optind || argv[H5_optind + 1] == NULL) { error_msg("missing file names\n"); usage(); h5diff_exit(EXIT_FAILURE); } - *fname1 = argv[opt_ind]; - *fname2 = argv[opt_ind + 1]; - *objname1 = argv[opt_ind + 2]; + *fname1 = argv[H5_optind]; + *fname2 = argv[H5_optind + 1]; + *objname1 = argv[H5_optind + 2]; H5TOOLS_DEBUG("file1 = %s", *fname1); H5TOOLS_DEBUG("file2 = %s", *fname2); @@ -466,8 +466,8 @@ parse_command_line(int argc, const char *argv[], const char **fname1, const char } H5TOOLS_DEBUG("objname1 = %s", *objname1); - if (argv[opt_ind + 3] != NULL) { - *objname2 = argv[opt_ind + 3]; + if (argv[H5_optind + 3] != NULL) { + *objname2 = argv[H5_optind + 3]; } else { *objname2 = *objname1; diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index c9c8185..cab340d 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -80,132 +80,54 @@ struct handler_t { */ /* The following initialization makes use of C language concatenating */ /* "xxx" "yyy" into "xxxyyy". */ -static const char * s_opts = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HM:N:O*RS:VX:"; -static struct long_options l_opts[] = {{"attribute", require_arg, 'a'}, - {"attribut", require_arg, 'a'}, - {"attribu", require_arg, 'a'}, - {"attrib", require_arg, 'a'}, - {"attri", require_arg, 'a'}, - {"attr", require_arg, 'a'}, - {"att", require_arg, 'a'}, - {"at", require_arg, 'a'}, - {"binary", optional_arg, 'b'}, - {"count", require_arg, 'c'}, - {"coun", require_arg, 'c'}, - {"cou", require_arg, 'c'}, - {"co", require_arg, 'c'}, - {"dataset", require_arg, 'd'}, - {"datase", require_arg, 'd'}, - {"datas", require_arg, 'd'}, - {"escape", no_arg, 'e'}, - {"filedriver", require_arg, 'f'}, - {"filedrive", require_arg, 'f'}, - {"filedriv", require_arg, 'f'}, - {"filedri", require_arg, 'f'}, - {"filedr", require_arg, 'f'}, - {"filed", require_arg, 'f'}, - {"file", require_arg, 'f'}, - {"fil", require_arg, 'f'}, - {"fi", require_arg, 'f'}, - {"group", require_arg, 'g'}, - {"grou", require_arg, 'g'}, - {"gro", require_arg, 'g'}, - {"gr", require_arg, 'g'}, - {"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"object-ids", no_arg, 'i'}, - {"object-id", no_arg, 'i'}, - {"object-i", no_arg, 'i'}, - {"object", no_arg, 'i'}, - {"objec", no_arg, 'i'}, - {"obje", no_arg, 'i'}, - {"obj", no_arg, 'i'}, - {"ob", no_arg, 'i'}, - {"block", require_arg, 'k'}, - {"bloc", require_arg, 'k'}, - {"blo", require_arg, 'k'}, - {"bl", require_arg, 'k'}, - {"soft-link", require_arg, 'l'}, - {"soft-lin", require_arg, 'l'}, - {"soft-li", require_arg, 'l'}, - {"soft-l", require_arg, 'l'}, - {"soft", require_arg, 'l'}, - {"sof", require_arg, 'l'}, - {"format", require_arg, 'm'}, - {"contents", optional_arg, 'n'}, - {"output", optional_arg, 'o'}, - {"outpu", optional_arg, 'o'}, - {"outp", optional_arg, 'o'}, - {"out", optional_arg, 'o'}, - {"ou", optional_arg, 'o'}, - {"properties", no_arg, 'p'}, - {"sort_by", require_arg, 'q'}, - {"string", no_arg, 'r'}, - {"strin", no_arg, 'r'}, - {"start", require_arg, 's'}, - {"star", require_arg, 's'}, - {"sta", require_arg, 's'}, - {"datatype", require_arg, 't'}, - {"datatyp", require_arg, 't'}, - {"dataty", require_arg, 't'}, - {"datat", require_arg, 't'}, - {"use-dtd", no_arg, 'u'}, - {"use-dt", no_arg, 'u'}, - {"use-d", no_arg, 'u'}, - {"use-", no_arg, 'u'}, - {"use", no_arg, 'u'}, - {"us", no_arg, 'u'}, - {"u", no_arg, 'u'}, - {"vds-view-first-missing", no_arg, 'v'}, - {"width", require_arg, 'w'}, - {"widt", require_arg, 'w'}, - {"wid", require_arg, 'w'}, - {"wi", require_arg, 'w'}, - {"xml", no_arg, 'x'}, - {"xm", no_arg, 'x'}, - {"noindex", no_arg, 'y'}, - {"sort_order", require_arg, 'z'}, - {"onlyattr", optional_arg, 'A'}, - {"superblock", no_arg, 'B'}, - {"boot-block", no_arg, 'B'}, - {"boot-bloc", no_arg, 'B'}, - {"boot-blo", no_arg, 'B'}, - {"boot-bl", no_arg, 'B'}, - {"boot-b", no_arg, 'B'}, - {"boot", no_arg, 'B'}, - {"boo", no_arg, 'B'}, - {"bo", no_arg, 'B'}, - {"no-compact-subset", no_arg, 'C'}, - {"xml-dtd", require_arg, 'D'}, - {"xml-dt", require_arg, 'D'}, - {"xml-d", require_arg, 'D'}, - {"enable-error-stack", optional_arg, 'E'}, - {"form", require_arg, 'F'}, - {"vds-gap-size", require_arg, 'G'}, - {"header", no_arg, 'H'}, - {"heade", no_arg, 'H'}, - {"head", no_arg, 'H'}, - {"hea", no_arg, 'H'}, - {"packed-bits", require_arg, 'M'}, - {"any_path", require_arg, 'N'}, - {"ddl", optional_arg, 'O'}, - {"region", no_arg, 'R'}, - {"stride", require_arg, 'S'}, - {"strid", require_arg, 'S'}, - {"version", no_arg, 'V'}, - {"versio", no_arg, 'V'}, - {"versi", no_arg, 'V'}, - {"vers", no_arg, 'V'}, - {"ver", no_arg, 'V'}, - {"ve", no_arg, 'V'}, - {"xml-ns", require_arg, 'X'}, - {"xml-n", require_arg, 'X'}, - {"s3-cred", require_arg, '$'}, - {"hdfs-attrs", require_arg, '#'}, - {"vol-value", require_arg, '1'}, - {"vol-name", require_arg, '2'}, - {"vol-info", require_arg, '3'}, - {NULL, 0, '\0'}}; +static const char * s_opts = "a:b*c:d:ef:g:hik:l:m:n*o*pq:rs:t:uvw:xyz:A*BCD:E*F:G:HM:N:O*RS:VX:"; +static struct h5_long_options l_opts[] = {{"attribute", require_arg, 'a'}, + {"binary", optional_arg, 'b'}, + {"count", require_arg, 'c'}, + {"dataset", require_arg, 'd'}, + {"escape", no_arg, 'e'}, + {"filedriver", require_arg, 'f'}, + {"group", require_arg, 'g'}, + {"help", no_arg, 'h'}, + {"object-ids", no_arg, 'i'}, + {"block", require_arg, 'k'}, + {"soft-link", require_arg, 'l'}, + {"format", require_arg, 'm'}, + {"contents", optional_arg, 'n'}, + {"output", optional_arg, 'o'}, + {"properties", no_arg, 'p'}, + {"sort_by", require_arg, 'q'}, + {"string", no_arg, 'r'}, + {"start", require_arg, 's'}, + {"datatype", require_arg, 't'}, + {"use-dtd", no_arg, 'u'}, + {"vds-view-first-missing", no_arg, 'v'}, + {"width", require_arg, 'w'}, + {"xml", no_arg, 'x'}, + {"noindex", no_arg, 'y'}, + {"sort_order", require_arg, 'z'}, + {"onlyattr", optional_arg, 'A'}, + {"superblock", no_arg, 'B'}, + {"boot-block", no_arg, 'B'}, + {"no-compact-subset", no_arg, 'C'}, + {"xml-dtd", require_arg, 'D'}, + {"enable-error-stack", optional_arg, 'E'}, + {"form", require_arg, 'F'}, + {"vds-gap-size", require_arg, 'G'}, + {"header", no_arg, 'H'}, + {"packed-bits", require_arg, 'M'}, + {"any_path", require_arg, 'N'}, + {"ddl", optional_arg, 'O'}, + {"region", no_arg, 'R'}, + {"stride", require_arg, 'S'}, + {"version", no_arg, 'V'}, + {"xml-ns", require_arg, 'X'}, + {"s3-cred", require_arg, '$'}, + {"hdfs-attrs", require_arg, '#'}, + {"vol-value", require_arg, '1'}, + {"vol-name", require_arg, '2'}, + {"vol-info", require_arg, '3'}, + {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: leave @@ -917,7 +839,7 @@ parse_command_line(int argc, const char *argv[]) } /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { parse_start: switch ((char)opt) { case 'R': @@ -931,8 +853,8 @@ parse_start: case 'n': dump_opts.display_fi = TRUE; last_was_dset = FALSE; - if (opt_arg != NULL) - h5trav_set_verbose(HDatoi(opt_arg)); + if (H5_optarg != NULL) + h5trav_set_verbose(HDatoi(H5_optarg)); break; case 'p': dump_opts.display_dcpl = TRUE; @@ -949,8 +871,8 @@ parse_start: last_was_dset = FALSE; break; case 'A': - if (opt_arg != NULL) { - if (0 == HDatoi(opt_arg)) + if (H5_optarg != NULL) { + if (0 == HDatoi(H5_optarg)) dump_opts.include_attrs = FALSE; } else { @@ -974,7 +896,7 @@ parse_start: goto done; break; case 'w': { - int sh5tools_nCols = HDatoi(opt_arg); + int sh5tools_nCols = HDatoi(H5_optarg); if (sh5tools_nCols <= 0) h5tools_nCols = 65535; @@ -988,7 +910,7 @@ parse_start: for (i = 0; i < argc; i++) if (!hand[i].func) { hand[i].func = handle_paths; - hand[i].obj = HDstrdup(opt_arg); + hand[i].obj = HDstrdup(H5_optarg); break; } @@ -1000,7 +922,7 @@ parse_start: for (i = 0; i < argc; i++) if (!hand[i].func) { hand[i].func = handle_attributes; - hand[i].obj = HDstrdup(opt_arg); + hand[i].obj = HDstrdup(H5_optarg); break; } @@ -1012,7 +934,7 @@ parse_start: for (i = 0; i < argc; i++) if (!hand[i].func) { hand[i].func = handle_datasets; - hand[i].obj = HDstrdup(opt_arg); + hand[i].obj = HDstrdup(H5_optarg); hand[i].subset_info = parse_subset_params(hand[i].obj); last_dset = &hand[i]; break; @@ -1021,7 +943,7 @@ parse_start: last_was_dset = TRUE; break; case 'f': - driver_name_g = opt_arg; + driver_name_g = H5_optarg; break; case 'g': dump_opts.display_all = 0; @@ -1029,7 +951,7 @@ parse_start: for (i = 0; i < argc; i++) if (!hand[i].func) { hand[i].func = handle_groups; - hand[i].obj = HDstrdup(opt_arg); + hand[i].obj = HDstrdup(H5_optarg); break; } @@ -1041,7 +963,7 @@ parse_start: for (i = 0; i < argc; i++) if (!hand[i].func) { hand[i].func = handle_links; - hand[i].obj = HDstrdup(opt_arg); + hand[i].obj = HDstrdup(H5_optarg); break; } @@ -1053,7 +975,7 @@ parse_start: for (i = 0; i < argc; i++) if (!hand[i].func) { hand[i].func = handle_datatypes; - hand[i].obj = HDstrdup(opt_arg); + hand[i].obj = HDstrdup(H5_optarg); break; } @@ -1061,7 +983,7 @@ parse_start: break; case 'O': - if (h5tools_set_output_file(opt_arg, 0) < 0) { + if (h5tools_set_output_file(H5_optarg, 0) < 0) { usage(h5tools_getprogname()); goto error; } @@ -1069,20 +991,20 @@ parse_start: case 'o': if (bin_output) { - if (h5tools_set_data_output_file(opt_arg, 1) < 0) { + if (h5tools_set_data_output_file(H5_optarg, 1) < 0) { usage(h5tools_getprogname()); goto error; } } else { if (dump_opts.display_attr_data && !dump_opts.display_data) { - if (h5tools_set_attr_output_file(opt_arg, 0) < 0) { + if (h5tools_set_attr_output_file(H5_optarg, 0) < 0) { usage(h5tools_getprogname()); goto error; } } if (dump_opts.display_data || dump_opts.display_all) { - if (h5tools_set_data_output_file(opt_arg, 0) < 0) { + if (h5tools_set_data_output_file(H5_optarg, 0) < 0) { usage(h5tools_getprogname()); goto error; } @@ -1091,12 +1013,12 @@ parse_start: dump_opts.usingdasho = TRUE; last_was_dset = FALSE; - outfname_g = opt_arg; + outfname_g = H5_optarg; break; case 'b': - if (opt_arg != NULL) { - if ((bin_form = set_binary_form(opt_arg)) < 0) { + if (H5_optarg != NULL) { + if ((bin_form = set_binary_form(H5_optarg)) < 0) { /* failed to set binary form */ usage(h5tools_getprogname()); goto error; @@ -1115,7 +1037,7 @@ parse_start: break; case 'q': - if ((sort_by = set_sort_by(opt_arg)) < 0) { + if ((sort_by = set_sort_by(H5_optarg)) < 0) { /* failed to set "sort by" form */ usage(h5tools_getprogname()); goto error; @@ -1123,7 +1045,7 @@ parse_start: break; case 'z': - if ((sort_order = set_sort_order(opt_arg)) < 0) { + if ((sort_order = set_sort_order(H5_optarg)) < 0) { /* failed to set "sort order" form */ usage(h5tools_getprogname()); goto error; @@ -1135,7 +1057,7 @@ parse_start: error_msg("option \"-%c\" can only be used after --dataset option\n", opt); goto error; } - if (parse_mask_list(opt_arg) != SUCCEED) { + if (parse_mask_list(H5_optarg) != SUCCEED) { usage(h5tools_getprogname()); goto error; } @@ -1145,7 +1067,7 @@ parse_start: dump_opts.display_vds_first = TRUE; break; case 'G': - dump_opts.vds_gap_size = HDatoi(opt_arg); + dump_opts.vds_gap_size = HDatoi(H5_optarg); if (dump_opts.vds_gap_size < 0) { usage(h5tools_getprogname()); goto error; @@ -1172,13 +1094,13 @@ parse_start: case 'D': /* specify alternative XML DTD or schema */ /* To Do: check format of this value? */ - xml_dtd_uri_g = opt_arg; + xml_dtd_uri_g = H5_optarg; h5tools_nCols = 0; break; case 'm': /* specify alternative floating point printing format */ - fp_format = opt_arg; + fp_format = H5_optarg; h5tools_nCols = 0; break; @@ -1189,10 +1111,10 @@ parse_start: usage(h5tools_getprogname()); goto error; } - if (HDstrcmp(opt_arg, ":") == 0) + if (HDstrcmp(H5_optarg, ":") == 0) xmlnsprefix = ""; else - xmlnsprefix = opt_arg; + xmlnsprefix = H5_optarg; h5tools_nCols = 0; break; /** end XML parameters **/ @@ -1238,33 +1160,33 @@ parse_start: HDfree(s->start.data); s->start.data = NULL; } - parse_hsize_list(opt_arg, &s->start); + parse_hsize_list(H5_optarg, &s->start); break; case 'S': if (s->stride.data) { HDfree(s->stride.data); s->stride.data = NULL; } - parse_hsize_list(opt_arg, &s->stride); + parse_hsize_list(H5_optarg, &s->stride); break; case 'c': if (s->count.data) { HDfree(s->count.data); s->count.data = NULL; } - parse_hsize_list(opt_arg, &s->count); + parse_hsize_list(H5_optarg, &s->count); break; case 'k': if (s->block.data) { HDfree(s->block.data); s->block.data = NULL; } - parse_hsize_list(opt_arg, &s->block); + parse_hsize_list(H5_optarg, &s->block); break; default: goto end_collect; } - } while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF); + } while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF); end_collect: last_was_dset = FALSE; @@ -1277,8 +1199,8 @@ end_collect: /** end subsetting parameters **/ case 'E': - if (opt_arg != NULL) - enable_error_stack = HDatoi(opt_arg); + if (H5_optarg != NULL) + enable_error_stack = HDatoi(H5_optarg); else enable_error_stack = 1; break; @@ -1294,7 +1216,7 @@ end_collect: case '$': #ifdef H5_HAVE_ROS3_VFD - if (h5tools_parse_ros3_fapl_tuple(opt_arg, ',', &ros3_fa_g) < 0) { + if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa_g) < 0) { error_msg("failed to parse S3 VFD credential info\n"); usage(h5tools_getprogname()); free_handler(hand, argc); @@ -1311,7 +1233,7 @@ end_collect: case '#': #ifdef H5_HAVE_LIBHDFS - if (h5tools_parse_hdfs_fapl_tuple(opt_arg, ',', &hdfs_fa_g) < 0) { + if (h5tools_parse_hdfs_fapl_tuple(H5_optarg, ',', &hdfs_fa_g) < 0) { error_msg("failed to parse HDFS VFD configuration info\n"); usage(h5tools_getprogname()); free_handler(hand, argc); @@ -1328,18 +1250,18 @@ end_collect: case '1': vol_info_g.type = VOL_BY_VALUE; - vol_info_g.u.value = (H5VL_class_value_t)HDatoi(opt_arg); + vol_info_g.u.value = (H5VL_class_value_t)HDatoi(H5_optarg); use_custom_vol_g = TRUE; break; case '2': vol_info_g.type = VOL_BY_NAME; - vol_info_g.u.name = opt_arg; + vol_info_g.u.name = H5_optarg; use_custom_vol_g = TRUE; break; case '3': - vol_info_g.info_string = opt_arg; + vol_info_g.info_string = H5_optarg; break; case '?': @@ -1351,7 +1273,7 @@ end_collect: parse_end: /* check for file name to be processed */ - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing file name\n"); usage(h5tools_getprogname()); goto error; @@ -1443,7 +1365,7 @@ main(int argc, const char *argv[]) } } - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing file name\n"); usage(h5tools_getprogname()); h5tools_setstatus(EXIT_FAILURE); @@ -1496,8 +1418,8 @@ main(int argc, const char *argv[]) } } - while (opt_ind < argc) { - fname = HDstrdup(argv[opt_ind++]); + while (H5_optind < argc) { + fname = HDstrdup(argv[H5_optind++]); fid = h5tools_fopen(fname, H5F_ACC_RDONLY, fapl_id, (fapl_id == H5P_DEFAULT) ? FALSE : TRUE, NULL, 0); diff --git a/tools/src/h5dump/h5dump_xml.c b/tools/src/h5dump/h5dump_xml.c index 5507dcc..0e881df 100644 --- a/tools/src/h5dump/h5dump_xml.c +++ b/tools/src/h5dump/h5dump_xml.c @@ -2365,14 +2365,21 @@ xml_dump_named_datatype(hid_t type, const char *name) h5tools_context_t ctx; /* print context */ h5tool_format_t * outputformat = &xml_dataformat; h5tool_format_t string_dataformat; - char * tmp; - char * dtxid; - char * parentxid; - char * t_tmp; - char * t_prefix; - char * t_name; + char * tmp = NULL; + char * dtxid = NULL; + char * parentxid = NULL; + char * t_tmp = NULL; + char * t_prefix = NULL; + char * t_name = NULL; tmp = (char *)HDmalloc(HDstrlen(prefix) + HDstrlen(name) + 2); + if (tmp == NULL) { + indentation(dump_indent); + error_msg("internal error (file %s:line %d)\n", __FILE__, __LINE__); + h5tools_setstatus(EXIT_FAILURE); + goto done; + } + HDstrcpy(tmp, prefix); HDstrcat(tmp, "/"); HDstrcat(tmp, name); @@ -2627,6 +2634,13 @@ xml_dump_group(hid_t gid, const char *name) } else { tmp = (char *)HDmalloc(HDstrlen(prefix) + HDstrlen(name) + 2); + if (tmp == NULL) { + indentation(dump_indent); + error_msg("internal error (file %s:line %d)\n", __FILE__, __LINE__); + h5tools_setstatus(EXIT_FAILURE); + return; + } + HDstrcpy(tmp, prefix); par = HDstrdup(tmp); cp = HDstrrchr(par, '/'); @@ -3146,8 +3160,11 @@ xml_print_strs(hid_t did, int source) } bp = (char *)buf; - if (!is_vlstr) + if (!is_vlstr) { onestring = (char *)HDcalloc(tsiz, sizeof(char)); + if (onestring == NULL) + goto error; + } /* setup */ HDmemset(&buffer, 0, sizeof(h5tools_str_t)); @@ -3768,6 +3785,14 @@ xml_dump_dataset(hid_t did, const char *name, struct subset_t H5_ATTR_UNUSED *ss char *pstr = (char *)HDmalloc((size_t)100); tmp = (char *)HDmalloc(HDstrlen(prefix) + HDstrlen(name) + 2); + if (tmp == NULL) { + error_msg("buffer allocation failed\n"); + h5tools_setstatus(EXIT_FAILURE); + HDfree(rstr); + HDfree(pstr); + return; + } + HDstrcpy(tmp, prefix); HDstrcat(tmp, "/"); HDstrcat(tmp, name); diff --git a/tools/src/h5format_convert/h5format_convert.c b/tools/src/h5format_convert/h5format_convert.c index 321f274..ddf129c 100644 --- a/tools/src/h5format_convert/h5format_convert.c +++ b/tools/src/h5format_convert/h5format_convert.c @@ -38,28 +38,11 @@ static int verbose_g = 0; * Command-line options: The user can specify short or long-named * parameters. */ -static const char * s_opts = "hVvd:n"; -static struct long_options l_opts[] = {{"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, - {"version", no_arg, 'V'}, - {"version", no_arg, 'V'}, - {"versio", no_arg, 'V'}, - {"versi", no_arg, 'V'}, - {"vers", no_arg, 'V'}, - {"verbose", no_arg, 'v'}, - {"verbos", no_arg, 'v'}, - {"verbo", no_arg, 'v'}, - {"verb", no_arg, 'v'}, - {"dname", require_arg, 'd'}, - {"dnam", require_arg, 'd'}, - {"dna", require_arg, 'd'}, - {"dn", require_arg, 'd'}, - {"noop", no_arg, 'n'}, - {"noo", no_arg, 'n'}, - {"no", no_arg, 'n'}, - {"enable-error-stack", no_arg, 'E'}, - {NULL, 0, '\0'}}; +static const char * s_opts = "hVvd:n"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"version", no_arg, 'V'}, + {"verbose", no_arg, 'v'}, {"dname", require_arg, 'd'}, + {"noop", no_arg, 'n'}, {"enable-error-stack", no_arg, 'E'}, + {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage @@ -123,7 +106,7 @@ parse_command_line(int argc, const char **argv) } /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'h': usage(h5tools_getprogname()); @@ -140,11 +123,11 @@ parse_command_line(int argc, const char **argv) break; case 'd': /* -d dname */ - if (opt_arg != NULL && *opt_arg) - dname_g = HDstrdup(opt_arg); + if (H5_optarg != NULL && *H5_optarg) + dname_g = HDstrdup(H5_optarg); if (dname_g == NULL) { h5tools_setstatus(EXIT_FAILURE); - error_msg("No dataset name\n", opt_arg); + error_msg("No dataset name\n", H5_optarg); usage(h5tools_getprogname()); goto error; } @@ -167,19 +150,19 @@ parse_command_line(int argc, const char **argv) } /* switch */ } /* while */ - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing file name\n"); usage(h5tools_getprogname()); h5tools_setstatus(EXIT_FAILURE); goto error; } - fname_g = HDstrdup(argv[opt_ind]); + fname_g = HDstrdup(argv[H5_optind]); - return (0); + return 0; error: - return (-1); + return -1; ; } /* parse_command_line() */ diff --git a/tools/src/h5import/h5import.c b/tools/src/h5import/h5import.c index 2a65591..04e1c35 100644 --- a/tools/src/h5import/h5import.c +++ b/tools/src/h5import/h5import.c @@ -489,7 +489,7 @@ readIntegerData(FILE *strm, struct Input *in) case 0: /* TEXTIN */ in08 = (H5DT_INT8 *)in->data; for (i = 0; i < len; i++, in08++) { - if (fscanf(strm, "%hd", &temp16) != 1) { + if (HDfscanf(strm, "%hd", &temp16) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -521,7 +521,7 @@ readIntegerData(FILE *strm, struct Input *in) switch (in->inputClass) { case 0: /* TEXTIN */ for (i = 0; i < len; i++, in16++) { - if (fscanf(strm, "%hd", in16) != 1) { + if (HDfscanf(strm, "%hd", in16) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -558,7 +558,7 @@ readIntegerData(FILE *strm, struct Input *in) switch (in->inputClass) { case 0: /* TEXTIN */ for (i = 0; i < len; i++, in32++) { - if (fscanf(strm, "%d", in32) != 1) { + if (HDfscanf(strm, "%d", in32) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -595,7 +595,7 @@ readIntegerData(FILE *strm, struct Input *in) switch (in->inputClass) { case 0: /* TEXTIN */ for (i = 0; i < len; i++, in64++) { - if (fscanf(strm, "%s", buffer) < 1) { + if (HDfscanf(strm, "%s", buffer) < 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -664,7 +664,7 @@ readUIntegerData(FILE *strm, struct Input *in) case 6: /* TEXTUIN */ in08 = (H5DT_UINT8 *)in->data; for (i = 0; i < len; i++, in08++) { - if (fscanf(strm, "%hu", &temp16) != 1) { + if (HDfscanf(strm, "%hu", &temp16) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -693,7 +693,7 @@ readUIntegerData(FILE *strm, struct Input *in) switch (in->inputClass) { case 6: /* TEXTUIN */ for (i = 0; i < len; i++, in16++) { - if (fscanf(strm, "%hu", in16) != 1) { + if (HDfscanf(strm, "%hu", in16) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -729,7 +729,7 @@ readUIntegerData(FILE *strm, struct Input *in) switch (in->inputClass) { case 6: /* TEXTUIN */ for (i = 0; i < len; i++, in32++) { - if (fscanf(strm, "%u", in32) != 1) { + if (HDfscanf(strm, "%u", in32) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -766,7 +766,7 @@ readUIntegerData(FILE *strm, struct Input *in) switch (in->inputClass) { case 6: /* TEXTUIN */ for (i = 0; i < len; i++, in64++) { - if (fscanf(strm, "%s", buffer) < 1) { + if (HDfscanf(strm, "%s", buffer) < 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -832,7 +832,7 @@ readFloatData(FILE *strm, struct Input *in) switch (in->inputClass) { case 1: /* TEXTFP */ for (i = 0; i < len; i++, fp32++) { - if (fscanf(strm, "%f", fp32) != 1) { + if (HDfscanf(strm, "%f", fp32) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -845,7 +845,7 @@ readFloatData(FILE *strm, struct Input *in) case 2: /*TEXTFPE */ for (i = 0; i < len; i++, fp32++) { - if (fscanf(strm, "%f", fp32) != 1) { + if (HDfscanf(strm, "%f", fp32) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -884,7 +884,7 @@ readFloatData(FILE *strm, struct Input *in) switch (in->inputClass) { case 1: /* TEXTFP */ for (i = 0; i < len; i++, fp64++) { - if (fscanf(strm, "%lf", fp64) != 1) { + if (HDfscanf(strm, "%lf", fp64) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -897,7 +897,7 @@ readFloatData(FILE *strm, struct Input *in) case 2: /*TEXTFPE */ for (i = 0; i < len; i++, fp64++) { - if (fscanf(strm, "%lf", fp64) != 1) { + if (HDfscanf(strm, "%lf", fp64) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -1469,16 +1469,16 @@ processConfigurationFile(char *infile, struct Input *in) goto error; } - scanret = fscanf(strm, "%254s", key); + scanret = HDfscanf(strm, "%254s", key); if ((scanret == 1) && !HDstrcmp("HDF5", key)) { #ifdef H5DEBUGIMPORT int pndx; HDprintf("\nh5dump file\n"); #endif in->h5dumpInput = 1; - scanret = fscanf(strm, "%254s", temp); /* filename */ - scanret = fscanf(strm, "%254s", temp); /* start bracket */ - scanret = fscanf(strm, "%254s", key); /* DATASET */ + scanret = HDfscanf(strm, "%254s", temp); /* filename */ + scanret = HDfscanf(strm, "%254s", temp); /* start bracket */ + scanret = HDfscanf(strm, "%254s", key); /* DATASET */ while (scanret == 1) { if (!HDstrcmp("DATASET", key)) { /* PATH */ #ifdef H5DEBUGIMPORT @@ -1488,7 +1488,7 @@ processConfigurationFile(char *infile, struct Input *in) (void)HDfprintf(stderr, err3a, infile); goto error; } - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1500,7 +1500,7 @@ processConfigurationFile(char *infile, struct Input *in) goto error; } in->configOptionVector[PATH] = 1; - scanret = fscanf(strm, "%254s", temp); /* start bracket */ + scanret = HDfscanf(strm, "%254s", temp); /* start bracket */ #ifdef H5DEBUGIMPORT HDprintf("h5dump DATASET %s found\n", temp); #endif @@ -1514,7 +1514,7 @@ processConfigurationFile(char *infile, struct Input *in) goto error; } - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1550,20 +1550,20 @@ processConfigurationFile(char *infile, struct Input *in) #ifdef H5DEBUGIMPORT HDprintf("h5dump DATATYPE STRING found\n"); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* start bracket */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start bracket */ (void)HDfprintf(stderr, "%s", err18); goto error; } #ifdef H5DEBUGIMPORT HDprintf("h5dump DATATYPE STRING %s found\n", temp); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* string properties */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* string properties */ (void)HDfprintf(stderr, "%s", err18); goto error; } while (get_next_prop) { if (!HDstrcmp("STRSIZE", temp)) { /* STRSIZE */ - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err19); goto error; } @@ -1582,8 +1582,8 @@ processConfigurationFile(char *infile, struct Input *in) #endif } } - else if (!HDstrcmp("STRPAD", temp)) { /* STRPAD */ - if (fscanf(strm, "%254s", temp) != 1) { /* STRPAD type */ + else if (!HDstrcmp("STRPAD", temp)) { /* STRPAD */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* STRPAD type */ (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1591,8 +1591,8 @@ processConfigurationFile(char *infile, struct Input *in) HDprintf("h5dump DATATYPE STRING STRPAD %s found\n", temp); #endif } - else if (!HDstrcmp("CSET", key)) { /* CSET */ - if (fscanf(strm, "%254s", temp) != 1) { /* CSET type */ + else if (!HDstrcmp("CSET", key)) { /* CSET */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* CSET type */ (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1600,8 +1600,8 @@ processConfigurationFile(char *infile, struct Input *in) HDprintf("h5dump DATATYPE STRING CSET %s found\n", temp); #endif } - else if (!HDstrcmp("CTYPE", temp)) { /* CTYPE */ - if (fscanf(strm, "%254s", temp) != 1) { /* CTYPE type */ + else if (!HDstrcmp("CTYPE", temp)) { /* CTYPE */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* CTYPE type */ (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1609,7 +1609,7 @@ processConfigurationFile(char *infile, struct Input *in) HDprintf("h5dump DATATYPE STRING CTYPE %s found\n", temp); #endif } /* if(!HDstrcmp("CSET", key)) */ - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1628,7 +1628,7 @@ processConfigurationFile(char *infile, struct Input *in) #ifdef H5DEBUGIMPORT HDprintf("h5dump DATASPACE key\n"); #endif - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1644,14 +1644,14 @@ processConfigurationFile(char *infile, struct Input *in) #ifdef H5DEBUGIMPORT HDprintf("h5dump DATASPACE SIMPLE found\n"); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* start bracket */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start bracket */ (void)HDfprintf(stderr, err6b, infile); goto error; } #ifdef H5DEBUGIMPORT HDprintf("h5dump DATASPACE SIMPLE %s found\n", temp); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* start paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start paren */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -1662,7 +1662,7 @@ processConfigurationFile(char *infile, struct Input *in) int get_next_dim = 1; int i = 0; - if (fscanf(strm, "%254s", temp) != 1) { /* Dimension with optional comma */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* Dimension with optional comma */ (void)HDfprintf(stderr, err16c, infile); goto error; } @@ -1672,7 +1672,7 @@ processConfigurationFile(char *infile, struct Input *in) while (get_next_dim) { char *more = temp; temp_dims[icount] = HDstrtoull(more, &more, 10); - if (fscanf(strm, "%254s", temp) != 1) { /* Dimension or end paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* Dimension or end paren */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -1716,7 +1716,7 @@ processConfigurationFile(char *infile, struct Input *in) (void)HDfprintf(stderr, err5b, infile); goto error; } - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1728,7 +1728,7 @@ processConfigurationFile(char *infile, struct Input *in) (hsize_t *)HDmalloc((size_t)in->rank * sizeof(hsize_t))) == NULL) { goto error; } - if (fscanf(strm, "%254s", temp) != 1) { /* start paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start paren */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -1742,7 +1742,7 @@ processConfigurationFile(char *infile, struct Input *in) #ifdef H5DEBUGIMPORT HDprintf("h5dump DATASPACE SIMPLE process max dim values\n"); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* max dim with optional comma */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* max dim with optional comma */ (void)HDfprintf(stderr, err16c, infile); goto error; } @@ -1762,7 +1762,7 @@ processConfigurationFile(char *infile, struct Input *in) char *more = temp; in->maxsizeOfDimension[i] = HDstrtoull(more, &more, 10); } - if (fscanf(strm, "%254s", temp) != 1) { /* max dim or end paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* max dim or end paren */ (void)HDfprintf(stderr, err16c, infile); goto error; } @@ -1793,7 +1793,7 @@ processConfigurationFile(char *infile, struct Input *in) (void)HDfprintf(stderr, err16c, infile); goto error; } - scanret = fscanf(strm, "%254s", temp); /* end bracket */ + scanret = HDfscanf(strm, "%254s", temp); /* end bracket */ #ifdef H5DEBUGIMPORT HDprintf("h5dump DATASPACE SIMPLE %s found\n", temp); #endif @@ -1808,14 +1808,14 @@ processConfigurationFile(char *infile, struct Input *in) #ifdef H5DEBUGIMPORT HDprintf("h5dump STORAGE_LAYOUT key\n"); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* start bracket */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start bracket */ (void)HDfprintf(stderr, err6b, infile); goto error; } #ifdef H5DEBUGIMPORT HDprintf("h5dump STORAGE_LAYOUT %s found\n", temp); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* CHUNKED */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* CHUNKED */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -1827,7 +1827,7 @@ processConfigurationFile(char *infile, struct Input *in) (void)HDfprintf(stderr, "Unable to allocate dynamic memory.\n"); goto error; } - if (fscanf(strm, "%254s", temp) != 1) { /* start paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start paren */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -1838,7 +1838,7 @@ processConfigurationFile(char *infile, struct Input *in) int get_next_dim = 1; int icount = 0; - if (fscanf(strm, "%254s", temp) != 1) { /* Dimension with optional comma */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* Dimension with optional comma */ (void)HDfprintf(stderr, err16c, infile); goto error; } @@ -1848,7 +1848,7 @@ processConfigurationFile(char *infile, struct Input *in) while (get_next_dim) { char *more = temp; in->sizeOfChunk[icount] = HDstrtoull(more, &more, 10); - if (fscanf(strm, "%254s", temp) != 1) { /* Dimension or end paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* Dimension or end paren */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -1880,7 +1880,7 @@ processConfigurationFile(char *infile, struct Input *in) (void)HDfprintf(stderr, err5b, infile); goto error; } - if (fscanf(strm, "%254s", temp) != 1) { /* SIZE */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* SIZE */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -1888,7 +1888,7 @@ processConfigurationFile(char *infile, struct Input *in) HDprintf("h5dump STORAGE_LAYOUT CHUNKED %s found\n", temp); #endif if (!HDstrcmp("SIZE", temp)) { /* SIZE */ - if (fscanf(strm, "%d", (&ival)) != 1) { + if (HDfscanf(strm, "%d", (&ival)) != 1) { (void)HDfprintf(stderr, "%s", err19); goto error; } @@ -1897,7 +1897,7 @@ processConfigurationFile(char *infile, struct Input *in) #endif } while (HDstrcmp("}", temp) != 0) { - if (fscanf(strm, "%254s", temp) != 1) { /* end bracket */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* end bracket */ (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1912,14 +1912,14 @@ processConfigurationFile(char *infile, struct Input *in) #ifdef H5DEBUGIMPORT HDprintf("h5dump FILTERS key\n"); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* start bracket */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start bracket */ (void)HDfprintf(stderr, err6b, infile); goto error; } #ifdef H5DEBUGIMPORT HDprintf("h5dump FILTERS %s found\n", temp); #endif - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -1930,28 +1930,28 @@ processConfigurationFile(char *infile, struct Input *in) #ifdef H5DEBUGIMPORT HDprintf("h5dump FILTERS COMPRESSION found\n"); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* DEFLATE */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* DEFLATE */ (void)HDfprintf(stderr, "%s", err18); goto error; } #ifdef H5DEBUGIMPORT HDprintf("h5dump FILTERS COMPRESSION %s found\n", temp); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* bgin bracket */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* bgin bracket */ (void)HDfprintf(stderr, "%s", err18); goto error; } #ifdef H5DEBUGIMPORT HDprintf("h5dump FILTERS COMPRESSION %s found\n", temp); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* LEVEL */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* LEVEL */ (void)HDfprintf(stderr, "%s", err18); goto error; } #ifdef H5DEBUGIMPORT HDprintf("h5dump FILTERS COMPRESSION %s found\n", temp); #endif - if (fscanf(strm, "%d", (&ival)) != 1) { + if (HDfscanf(strm, "%d", (&ival)) != 1) { (void)HDfprintf(stderr, "%s", err19); goto error; } @@ -1959,7 +1959,7 @@ processConfigurationFile(char *infile, struct Input *in) HDprintf("h5dump FILTERS COMPRESSION LEVEL %d found\n", ival); #endif in->compressionParam = ival; - if (fscanf(strm, "%254s", temp) != 1) { /* end bracket */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* end bracket */ (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1981,7 +1981,7 @@ processConfigurationFile(char *infile, struct Input *in) #endif in->configOptionVector[COMPRESS] = 0; } - if (fscanf(strm, "%254s", temp) != 1) { /* end bracket */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* end bracket */ (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -1995,14 +1995,14 @@ processConfigurationFile(char *infile, struct Input *in) #ifdef H5DEBUGIMPORT HDprintf("h5dump SUBSET key\n"); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* start bracket */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start bracket */ (void)HDfprintf(stderr, err20, infile); goto error; } #ifdef H5DEBUGIMPORT HDprintf("h5dump SUBSET %s found\n", temp); #endif - if (fscanf(strm, "%254s", temp) != 1) { /* SUBSET keyword */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* SUBSET keyword */ (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -2012,7 +2012,7 @@ processConfigurationFile(char *infile, struct Input *in) while (get_next_prop) { if (!HDstrcmp("COUNT", temp)) { /* COUNT */ int icount = 0; - if (fscanf(strm, "%254s", temp) != 1) { /* start paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start paren */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -2023,7 +2023,7 @@ processConfigurationFile(char *infile, struct Input *in) int get_next_dim = 1; int i = 0; - if (fscanf(strm, "%254s", temp) != 1) { /* Dimension with optional comma */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* Dimension with optional comma */ (void)HDfprintf(stderr, err16c, infile); goto error; } @@ -2033,7 +2033,7 @@ processConfigurationFile(char *infile, struct Input *in) while (get_next_dim) { char *more = temp; temp_dims[icount] = HDstrtoull(more, &more, 10); - if (fscanf(strm, "%254s", temp) != 1) { /* Dimension or end paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* Dimension or end paren */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -2068,7 +2068,7 @@ processConfigurationFile(char *infile, struct Input *in) } /* if(!HDstrcmp("COUNT", temp)) COUNT */ if (!HDstrcmp("BLOCK", temp)) { /* BLOCK */ int icount = 0; - if (fscanf(strm, "%254s", temp) != 1) { /* start paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* start paren */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -2079,7 +2079,7 @@ processConfigurationFile(char *infile, struct Input *in) int get_next_dim = 1; int i = 0; - if (fscanf(strm, "%254s", temp) != 1) { /* Dimension with optional comma */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* Dimension with optional comma */ (void)HDfprintf(stderr, err16c, infile); goto error; } @@ -2089,7 +2089,7 @@ processConfigurationFile(char *infile, struct Input *in) while (get_next_dim) { char *more = temp; temp_dims[icount] = HDstrtoull(more, &more, 10); - if (fscanf(strm, "%254s", temp) != 1) { /* Dimension or end paren */ + if (HDfscanf(strm, "%254s", temp) != 1) { /* Dimension or end paren */ (void)HDfprintf(stderr, err6b, infile); goto error; } @@ -2122,7 +2122,7 @@ processConfigurationFile(char *infile, struct Input *in) in->configOptionVector[DIM] = 1; } /* if(!HDstrcmp("(", key)) start paren */ } /* if(!HDstrcmp("BLOCK", temp)) BLOCK */ - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -2141,7 +2141,7 @@ processConfigurationFile(char *infile, struct Input *in) scanret = 0; break; } - scanret = fscanf(strm, "%254s", key); + scanret = HDfscanf(strm, "%254s", key); } #ifdef H5DEBUGIMPORT HDprintf("h5dump path"); @@ -2182,7 +2182,7 @@ processConfigurationFile(char *infile, struct Input *in) (void)HDfprintf(stderr, err3a, infile); goto error; } - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -2199,7 +2199,7 @@ processConfigurationFile(char *infile, struct Input *in) goto error; } - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err18); goto error; } @@ -2226,7 +2226,7 @@ processConfigurationFile(char *infile, struct Input *in) (void)HDfprintf(stderr, err5a, infile); goto error; } - if (fscanf(strm, "%254d", (&ival)) != 1) { + if (HDfscanf(strm, "%254d", (&ival)) != 1) { (void)HDfprintf(stderr, "%s", err19); goto error; } @@ -2423,7 +2423,7 @@ processConfigurationFile(char *infile, struct Input *in) default: break; } - scanret = fscanf(strm, "%254s", key); + scanret = HDfscanf(strm, "%254s", key); } /* @@ -2602,7 +2602,7 @@ getOutputClass(struct Input *in, FILE *strm) const char *err1 = "Unable to get 'string' value.\n"; const char *err2 = "Invalid value for output class.\n"; - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -2637,7 +2637,7 @@ getOutputSize(struct Input *in, FILE *strm) const char *err1 = "Unable to get integer value.\n"; const char *err2 = "Invalid value for output size.\n"; - if (fscanf(strm, "%d", (&ival)) != 1) { + if (HDfscanf(strm, "%d", (&ival)) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -3502,7 +3502,7 @@ getInputByteOrder(struct Input *in, FILE *strm) const char *err1 = "Unable to get 'string' value.\n"; const char *err2 = "Invalid value for input byte-order.\n"; - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -3524,7 +3524,7 @@ getRank(struct Input *in, FILE *strm) const char *err1 = "Unable to get integer value.\n"; const char *err2 = "Invalid value for rank.\n"; - if (fscanf(strm, "%d", (&ival)) != 1) { + if (HDfscanf(strm, "%d", (&ival)) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -3553,7 +3553,7 @@ getDimensionSizes(struct Input *in, FILE *strm) return (-1); } - while (fscanf(strm, "%llu", (&ullval)) == 1) + while (HDfscanf(strm, "%llu", (&ullval)) == 1) in->sizeOfDimension[i++] = ullval; if (in->rank != i) { @@ -3579,7 +3579,7 @@ getChunkedDimensionSizes(struct Input *in, FILE *strm) return (-1); } - while (fscanf(strm, "%llu", (&ullval)) == 1) + while (HDfscanf(strm, "%llu", (&ullval)) == 1) in->sizeOfChunk[i++] = ullval; if (in->rank != i) { @@ -3612,7 +3612,7 @@ getMaximumDimensionSizes(struct Input *in, FILE *strm) return (-1); } - while (fscanf(strm, "%lld", (&llval)) == 1) { + while (HDfscanf(strm, "%lld", (&llval)) == 1) { if (llval == -1) in->maxsizeOfDimension[i++] = H5S_UNLIMITED; else @@ -3642,7 +3642,7 @@ getOutputArchitecture(struct Input *in, FILE *strm) const char *err1 = "Unable to get 'string' value.\n"; const char *err2 = "Invalid value for output architecture.\n"; - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -3675,7 +3675,7 @@ getOutputByteOrder(struct Input *in, FILE *strm) const char *err1 = "Unable to get 'string' value.\n"; const char *err2 = "Invalid value for output byte-order.\n"; - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -3708,7 +3708,7 @@ getCompressionType(struct Input *in, FILE *strm) const char *err1 = "Unable to get 'string' value.\n"; const char *err2 = "Invalid value for compression.\n"; - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -3750,7 +3750,7 @@ getCompressionParameter(struct Input *in, FILE *strm) switch (in->compressionType) { case 0: /* GZIP */ - if (fscanf(strm, "%d", (&ival)) != 1) { + if (HDfscanf(strm, "%d", (&ival)) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } @@ -3774,7 +3774,7 @@ getExternalFilename(struct Input *in, FILE *strm) char temp[255]; const char *err1 = "Unable to get 'string' value.\n"; - if (fscanf(strm, "%254s", temp) != 1) { + if (HDfscanf(strm, "%254s", temp) != 1) { (void)HDfprintf(stderr, "%s", err1); return (-1); } diff --git a/tools/src/h5jam/h5jam.c b/tools/src/h5jam/h5jam.c index 5bcbf45..7f3385c 100644 --- a/tools/src/h5jam/h5jam.c +++ b/tools/src/h5jam/h5jam.c @@ -34,14 +34,10 @@ char *ub_file = NULL; * parameters. The long-named ones can be partially spelled. When * adding more, make sure that they don't clash with each other. */ -static const char * s_opts = "hi:u:o:c:V"; /* add more later ? */ -static struct long_options l_opts[] = { - {"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, {"i", require_arg, 'i'}, /* input file */ - {"u", require_arg, 'u'}, /* user block file */ - {"o", require_arg, 'o'}, /* output file */ - {"clobber", no_arg, 'c'}, /* clobber existing UB */ - {"clobbe", no_arg, 'c'}, {"clobb", no_arg, 'c'}, {"clob", no_arg, 'c'}, - {"clo", no_arg, 'c'}, {"cl", no_arg, 'c'}, {NULL, 0, '\0'}}; +static const char * s_opts = "hi:u:o:c:V"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"i", require_arg, 'i'}, + {"u", require_arg, 'u'}, {"o", require_arg, 'o'}, + {"clobber", no_arg, 'c'}, {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage @@ -113,16 +109,16 @@ parse_command_line(int argc, const char *argv[]) int opt = FALSE; /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'o': - output_file = HDstrdup(opt_arg); + output_file = HDstrdup(H5_optarg); break; case 'i': - input_file = HDstrdup(opt_arg); + input_file = HDstrdup(H5_optarg); break; case 'u': - ub_file = HDstrdup(opt_arg); + ub_file = HDstrdup(H5_optarg); break; case 'c': do_clobber = TRUE; diff --git a/tools/src/h5jam/h5unjam.c b/tools/src/h5jam/h5unjam.c index cde7ec7..fa23b06 100644 --- a/tools/src/h5jam/h5unjam.c +++ b/tools/src/h5jam/h5unjam.c @@ -35,14 +35,10 @@ char *ub_file = NULL; * parameters. The long-named ones can be partially spelled. When * adding more, make sure that they don't clash with each other. */ -static const char * s_opts = "hu:i:o:d:V"; -static struct long_options l_opts[] = { - {"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, {"i", require_arg, 'i'}, /* input file */ - {"u", require_arg, 'u'}, /* user block file */ - {"o", require_arg, 'o'}, /* output file */ - {"delete", no_arg, 'd'}, /* delete ub */ - {"delet", no_arg, 'd'}, {"dele", no_arg, 'd'}, {"del", no_arg, 'd'}, - {"de", no_arg, 'd'}, {NULL, 0, '\0'}}; +static const char * s_opts = "hu:i:o:d:V"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"i", require_arg, 'i'}, + {"u", require_arg, 'u'}, {"o", require_arg, 'o'}, + {"delete", no_arg, 'd'}, {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage @@ -101,23 +97,23 @@ parse_command_line(int argc, const char *argv[]) int opt = FALSE; /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'o': - output_file = HDstrdup(opt_arg); + output_file = HDstrdup(H5_optarg); if (output_file) h5tools_set_data_output_file(output_file, 1); break; case 'i': - input_file = HDstrdup(opt_arg); + input_file = HDstrdup(H5_optarg); if (input_file) h5tools_set_input_file(input_file, 1); break; ; case 'u': - ub_file = HDstrdup(opt_arg); + ub_file = HDstrdup(H5_optarg); if (ub_file) h5tools_set_output_file(ub_file, 1); else diff --git a/tools/src/h5perf/CMakeLists.txt b/tools/src/h5perf/CMakeLists.txt new file mode 100644 index 0000000..36b0b2f --- /dev/null +++ b/tools/src/h5perf/CMakeLists.txt @@ -0,0 +1,103 @@ +cmake_minimum_required (VERSION 3.12) +project (HDF5_TOOLS_SRC_H5PERF C) + +# -------------------------------------------------------------------- +# Add the executables +# -------------------------------------------------------------------- +#-- Adding test for h5perf_serial +set (h5perf_serial_SOURCES + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/sio_perf.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/sio_engine.c +) +add_executable (h5perf_serial ${h5perf_serial_SOURCES}) +target_include_directories (h5perf_serial PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") +if (NOT ONLY_SHARED_LIBS) + TARGET_C_PROPERTIES (h5perf_serial STATIC) + target_link_libraries (h5perf_serial PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) +else () + TARGET_C_PROPERTIES (h5perf_serial SHARED) + target_link_libraries (h5perf_serial PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) +endif () +set_target_properties (h5perf_serial PROPERTIES FOLDER perform) +set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5perf_serial") + +set (H5_DEP_EXECUTABLES h5perf_serial) + +#----------------------------------------------------------------------------- +# Add Target to clang-format +#----------------------------------------------------------------------------- +if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TOOLS_SRC_H5PERF_h5perf_serial_FORMAT h5perf_serial) +endif () + +if (H5_HAVE_PARALLEL) + if (UNIX) + #-- Adding test for perf - only on unix systems + set (perf_SOURCES + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/perf.c + ) + add_executable (perf ${perf_SOURCES}) + target_include_directories (perf PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + if (NOT ONLY_SHARED_LIBS) + TARGET_C_PROPERTIES (perf STATIC) + target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} "$<$:${MPI_C_LIBRARIES}>") + else () + TARGET_C_PROPERTIES (perf SHARED) + target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} "$<$:${MPI_C_LIBRARIES}>") + endif () + set_target_properties (perf PROPERTIES FOLDER perform) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};perf") + + set (H5_DEP_EXECUTABLES perf) + + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TOOLS_SRC_H5PERF_perf_FORMAT perf) + endif () + endif () + + #-- Adding test for h5perf + set (h5perf_SOURCES + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/pio_perf.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/pio_engine.c + ) + add_executable (h5perf ${h5perf_SOURCES}) + target_include_directories (h5perf PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") + if (NOT ONLY_SHARED_LIBS) + TARGET_C_PROPERTIES (h5perf STATIC) + target_link_libraries (h5perf PRIVATE ${LINK_LIBS} ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET} "$<$:${MPI_C_LIBRARIES}>") + else () + TARGET_C_PROPERTIES (h5perf SHARED) + target_link_libraries (h5perf PRIVATE ${LINK_LIBS} ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET} "$<$:${MPI_C_LIBRARIES}>") + endif () + set_target_properties (h5perf PROPERTIES FOLDER perform) + set_global_variable (HDF5_UTILS_TO_EXPORT "${HDF5_UTILS_TO_EXPORT};h5perf") + + set (H5_DEP_EXECUTABLES h5perf) + + #----------------------------------------------------------------------------- + # Add Target to clang-format + #----------------------------------------------------------------------------- + if (HDF5_ENABLE_FORMATTERS) + clang_format (HDF5_TOOLS_SRC_H5PERF_h5perf_FORMAT h5perf) + endif () +endif () + +#----------------------------------------------------------------------------- +# Rules for Installation of tools using make Install target +#----------------------------------------------------------------------------- +if (HDF5_EXPORTED_TARGETS) + foreach (exec ${H5_DEP_EXECUTABLES}) + INSTALL_PROGRAM_PDB (${exec} ${HDF5_INSTALL_BIN_DIR} toolsapplications) + endforeach () + + install ( + TARGETS + ${H5_DEP_EXECUTABLES} + EXPORT + ${HDF5_EXPORTED_TARGETS} + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications + ) +endif () diff --git a/tools/src/h5perf/Makefile.am b/tools/src/h5perf/Makefile.am new file mode 100644 index 0000000..e8a9fdd --- /dev/null +++ b/tools/src/h5perf/Makefile.am @@ -0,0 +1,63 @@ +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://www.hdfgroup.org/licenses. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +## +# +# HDF5 Library Performance Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib + +# bin_PROGRAMS will be installed. +if BUILD_PARALLEL_CONDITIONAL + bin_PROGRAMS=h5perf_serial perf h5perf +else + bin_PROGRAMS=h5perf_serial +endif + +# Add h5perf and h5perf_serial specific linker flags here +h5perf_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) +h5perf_serial_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) + +# Some programs are not built or run by default, but can be built by hand or by +# specifying --enable-build-all at configure time. +# Also, some of these programs should only be built in parallel. +# Currently there is no such program. +if BUILD_PARALLEL_CONDITIONAL + PARA_BUILD_ALL= +endif +if BUILD_ALL_CONDITIONAL + BUILD_ALL_PROGS=$(PARA_BUILD_ALL) +endif + +# Define programs that will be run in 'make check' +# List them in the order they should be run. +# Parallel test programs. +if BUILD_PARALLEL_CONDITIONAL + TEST_PROG_PARA=h5perf perf +endif + +h5perf_SOURCES=pio_perf.c pio_engine.c +h5perf_serial_SOURCES=sio_perf.c sio_engine.c + +# All of the programs depend on the main hdf5 library, and some of them +# depend on test or tools library. +LDADD=$(LIBHDF5) +h5perf_LDADD=$(LIBH5TOOLS) $(LIBHDF5) +h5perf_serial_LDADD=$(LIBH5TOOLS) $(LIBHDF5) +perf_LDADD=$(LIBHDF5) + +include $(top_srcdir)/config/conclude.am diff --git a/tools/src/h5perf/perf.c b/tools/src/h5perf/perf.c new file mode 100644 index 0000000..2fbcee5 --- /dev/null +++ b/tools/src/h5perf/perf.c @@ -0,0 +1,802 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Author: Albert Cheng of NCSA, May 1, 2001. + * This is derived from code given to me by Robert Ross. + * + * NOTE: This code assumes that all command line arguments make it out to all + * the processes that make up the parallel job, which isn't always the case. + * So if it doesn't work on some platform, that might be why. + */ + +#include "hdf5.h" +#include "H5private.h" + +#ifdef H5_HAVE_PARALLEL + +#ifdef H5_STDC_HEADERS +#include +#include +#include +#include +#include +#include + +#ifdef H5_HAVE_SYS_STAT_H +#include +#endif + +#if defined(H5_TIME_WITH_SYS_TIME) +#include +#include +#elif defined(H5_HAVE_SYS_TIME_H) +#include +#else +#include +#endif + +#ifdef H5_HAVE_SYS_TYPES_H +#include +#endif + +#ifdef H5_HAVE_UNISTD_H +#include +#endif + +#include +#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ +#include +#endif + +/* Macro definitions */ +/* Verify: + * if val is false (0), print mesg and if fatal is true (non-zero), die. + */ +#define H5FATAL 1 +#define VRFY(val, mesg, fatal) \ + do { \ + if (!val) { \ + printf("Proc %d: ", mynod); \ + printf("*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ + if (fatal) { \ + fflush(stdout); \ + goto die_jar_jar_die; \ + } \ + } \ + } while (0) +#define RANK 1 +#define MAX_PATH 1024 + +hsize_t dims[RANK]; /* dataset dim sizes */ +hsize_t block[RANK], stride[RANK], count[RANK]; +hsize_t start[RANK]; +hid_t fid; /* HDF5 file ID */ +hid_t acc_tpl; /* File access templates */ +hid_t sid; /* Dataspace ID */ +hid_t file_dataspace; /* File dataspace ID */ +hid_t mem_dataspace; /* memory dataspace ID */ +hid_t dataset; /* Dataset ID */ +hsize_t opt_alignment = 1; +hsize_t opt_threshold = 1; +int opt_split_vfd = 0; +char * meta_ext, *raw_ext; /* holds the meta and raw file extension if */ + /* opt_split_vfd is set */ + +/* DEFAULT VALUES FOR OPTIONS */ +int64_t opt_block = 1048576 * 16; +int opt_iter = 1; +int opt_stripe = -1; +int opt_correct = 0; +int amode = O_RDWR | O_CREAT; +char opt_file[256] = "perftest.out"; +char opt_pvfstab[256] = "notset"; +int opt_pvfstab_set = 0; + +const char *FILENAME[] = {opt_file, NULL}; + +/* function prototypes */ +static int parse_args(int argc, char **argv); + +#ifndef H5_HAVE_UNISTD_H +/* globals needed for getopt */ +extern char *optarg; +#endif + +#ifndef HDF5_PARAPREFIX +#define HDF5_PARAPREFIX "" +#endif +char * paraprefix = NULL; /* for command line option para-prefix */ +MPI_Info h5_io_info_g = MPI_INFO_NULL; /* MPI INFO object for IO */ + +static char *h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fullname, + size_t size, hbool_t nest_printf, hbool_t subst_for_superblock); + +int +main(int argc, char **argv) +{ + char * buf, *tmp, *buf2 = NULL, *tmp2 = NULL, *check; + int i, j, mynod = 0, nprocs = 1, my_correct = 1, correct, myerrno; + double stim, etim; + double write_tim = 0; + double read_tim = 0; + double read_bw, write_bw; + double max_read_tim, max_write_tim; + double min_read_tim, min_write_tim; + double ave_read_tim, ave_write_tim; + int64_t iter_jump = 0; + char filename[MAX_PATH]; + herr_t ret; /* Generic return value */ + + /* startup MPI and determine the rank of this process */ + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + MPI_Comm_rank(MPI_COMM_WORLD, &mynod); + + /* parse the command line arguments */ + parse_args(argc, argv); + + if (mynod == 0) + printf("# Using hdf5-io calls.\n"); + +#ifdef H5_HAVE_UNISTD_H + /* Kind of a weird hack- if the location of the pvfstab file was + * specified on the command line, then spit out this location into + * the appropriate environment variable. + */ + if (opt_pvfstab_set) { + if ((setenv("PVFSTAB_FILE", opt_pvfstab, 1)) < 0) { + perror("setenv"); + goto die_jar_jar_die; + } + } +#endif + + /* this is how much of the file data is covered on each iteration of + * the test. used to help determine the seek offset on each + * iteration */ + iter_jump = nprocs * opt_block; + + /* setup a buffer of data to write */ + if (!(tmp = (char *)malloc((size_t)opt_block + 256))) { + perror("malloc"); + goto die_jar_jar_die; + } + buf = tmp + 128 - (((long)tmp) % 128); /* align buffer */ + + if (opt_correct) { + /* do the same buffer setup for verifiable data */ + if (!(tmp2 = (char *)malloc((size_t)opt_block + 256))) { + perror("malloc2"); + goto die_jar_jar_die; + } + buf2 = tmp + 128 - (((long)tmp) % 128); + } + + /* setup file access template with parallel IO access. */ + if (opt_split_vfd) { + hid_t mpio_pl; + + mpio_pl = H5Pcreate(H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "", H5FATAL); + ret = H5Pset_fapl_mpio(mpio_pl, MPI_COMM_WORLD, MPI_INFO_NULL); + VRFY((ret >= 0), "", H5FATAL); + + /* set optional allocation alignment */ + if (opt_alignment * opt_threshold != 1) { + ret = H5Pset_alignment(acc_tpl, opt_threshold, opt_alignment); + VRFY((ret >= 0), "H5Pset_alignment succeeded", !H5FATAL); + } + + /* setup file access template */ + acc_tpl = H5Pcreate(H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "", H5FATAL); + ret = H5Pset_fapl_split(acc_tpl, meta_ext, mpio_pl, raw_ext, mpio_pl); + VRFY((ret >= 0), "H5Pset_fapl_split succeeded", H5FATAL); + ret = H5Pclose(mpio_pl); + VRFY((ret >= 0), "H5Pclose mpio_pl succeeded", H5FATAL); + } + else { + /* setup file access template */ + acc_tpl = H5Pcreate(H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "", H5FATAL); + ret = H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL); + VRFY((ret >= 0), "", H5FATAL); + + /* set optional allocation alignment */ + if (opt_alignment * opt_threshold != 1) { + ret = H5Pset_alignment(acc_tpl, opt_threshold, opt_alignment); + VRFY((ret >= 0), "H5Pset_alignment succeeded", !H5FATAL); + } + } + + h5_fixname_real(FILENAME[0], acc_tpl, NULL, filename, sizeof filename, FALSE, FALSE); + + /* create the parallel file */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); + VRFY((fid >= 0), "H5Fcreate succeeded", H5FATAL); + + /* define a contiquous dataset of opt_iter*nprocs*opt_block chars */ + dims[0] = (hsize_t)opt_iter * (hsize_t)nprocs * (hsize_t)opt_block; + sid = H5Screate_simple(RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded", H5FATAL); + dataset = H5Dcreate2(fid, "Dataset1", H5T_NATIVE_CHAR, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded", H5FATAL); + + /* create the memory dataspace and the file dataspace */ + dims[0] = (hsize_t)opt_block; + mem_dataspace = H5Screate_simple(RANK, dims, NULL); + VRFY((mem_dataspace >= 0), "", H5FATAL); + file_dataspace = H5Dget_space(dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded", H5FATAL); + + /* now each process writes a block of opt_block chars in round robbin + * fashion until the whole dataset is covered. + */ + for (j = 0; j < opt_iter; j++) { + /* setup a file dataspace selection */ + start[0] = (hsize_t)((j * iter_jump) + (mynod * opt_block)); + stride[0] = block[0] = (hsize_t)opt_block; + count[0] = 1; + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded", H5FATAL); + + if (opt_correct) /* fill in buffer for iteration */ { + for (i = mynod + j, check = buf; i < opt_block; i++, check++) + *check = (char)i; + } + + /* discover the starting time of the operation */ + MPI_Barrier(MPI_COMM_WORLD); + stim = MPI_Wtime(); + + /* write data */ + ret = H5Dwrite(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded", !H5FATAL); + + /* discover the ending time of the operation */ + etim = MPI_Wtime(); + + write_tim += (etim - stim); + + /* we are done with this "write" iteration */ + } + + /* close dataset and file */ + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose succeeded", H5FATAL); + ret = H5Fclose(fid); + VRFY((ret >= 0), "H5Fclose succeeded", H5FATAL); + + /* wait for everyone to synchronize at this point */ + MPI_Barrier(MPI_COMM_WORLD); + + /* reopen the file for reading */ + fid = H5Fopen(filename, H5F_ACC_RDONLY, acc_tpl); + VRFY((fid >= 0), "", H5FATAL); + + /* open the dataset */ + dataset = H5Dopen2(fid, "Dataset1", H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen succeeded", H5FATAL); + + /* we can re-use the same mem_dataspace and file_dataspace + * the H5Dwrite used since the dimension size is the same. + */ + + /* we are going to repeat the read the same pattern the write used */ + for (j = 0; j < opt_iter; j++) { + /* setup a file dataspace selection */ + start[0] = (hsize_t)((j * iter_jump) + (mynod * opt_block)); + stride[0] = block[0] = (hsize_t)opt_block; + count[0] = 1; + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded", H5FATAL); + /* seek to the appropriate spot give the current iteration and + * rank within the MPI processes */ + + /* discover the start time */ + MPI_Barrier(MPI_COMM_WORLD); + stim = MPI_Wtime(); + + /* read in the file data */ + if (!opt_correct) { + ret = H5Dread(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf); + } + else { + ret = H5Dread(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf2); + } + myerrno = errno; + + /* discover the end time */ + etim = MPI_Wtime(); + read_tim += (etim - stim); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded", !H5FATAL); + + if (ret < 0) + HDfprintf(stderr, "node %d, read error, loc = %" PRId64 ": %s\n", mynod, mynod * opt_block, + strerror(myerrno)); + + /* if the user wanted to check correctness, compare the write + * buffer to the read buffer */ + if (opt_correct && memcmp(buf, buf2, (size_t)opt_block)) { + HDfprintf(stderr, "node %d, correctness test failed\n", mynod); + my_correct = 0; + MPI_Allreduce(&my_correct, &correct, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); + } + + /* we are done with this read iteration */ + } + + /* close dataset and file */ + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose succeeded", H5FATAL); + ret = H5Fclose(fid); + VRFY((ret >= 0), "H5Fclose succeeded", H5FATAL); + ret = H5Pclose(acc_tpl); + VRFY((ret >= 0), "H5Pclose succeeded", H5FATAL); + + /* compute the read and write times */ + MPI_Allreduce(&read_tim, &max_read_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + MPI_Allreduce(&read_tim, &min_read_tim, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(&read_tim, &ave_read_tim, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + /* calculate the average from the sum */ + ave_read_tim = ave_read_tim / nprocs; + + MPI_Allreduce(&write_tim, &max_write_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); + MPI_Allreduce(&write_tim, &min_write_tim, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); + MPI_Allreduce(&write_tim, &ave_write_tim, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); + + /* calculate the average from the sum */ + ave_write_tim = ave_write_tim / nprocs; + + /* print out the results on one node */ + if (mynod == 0) { + read_bw = (double)((int64_t)(opt_block * nprocs * opt_iter)) / (max_read_tim * 1000000.0); + write_bw = (double)((int64_t)(opt_block * nprocs * opt_iter)) / (max_write_tim * 1000000.0); + + printf("nr_procs = %d, nr_iter = %d, blk_sz = %ld\n", nprocs, opt_iter, (long)opt_block); + + printf("# total_size = %ld\n", (long)(opt_block * nprocs * opt_iter)); + + printf("# Write: min_time = %f, max_time = %f, mean_time = %f\n", min_write_tim, max_write_tim, + ave_write_tim); + printf("# Read: min_time = %f, max_time = %f, mean_time = %f\n", min_read_tim, max_read_tim, + ave_read_tim); + + printf("Write bandwidth = %f Mbytes/sec\n", write_bw); + printf("Read bandwidth = %f Mbytes/sec\n", read_bw); + + if (opt_correct) { + printf("Correctness test %s.\n", correct ? "passed" : "failed"); + } + } + +die_jar_jar_die: + +#ifdef H5_HAVE_UNISTD + /* Clear the environment variable if it was set earlier */ + if (opt_pvfstab_set) { + unsetenv("PVFSTAB_FILE"); + } +#endif + + free(tmp); + if (opt_correct) + free(tmp2); + + MPI_Finalize(); + + return (0); +} + +static int +parse_args(int argc, char **argv) +{ + int c; + + while ((c = getopt(argc, argv, "s:b:i:f:p:a:2:c")) != EOF) { + switch (c) { + case 's': /* stripe */ + opt_stripe = atoi(optarg); + break; + case 'b': /* block size */ + opt_block = atoi(optarg); + break; + case 'i': /* iterations */ + opt_iter = atoi(optarg); + break; + case 'f': /* filename */ + strncpy(opt_file, optarg, 255); + FILENAME[0] = opt_file; + break; + case 'p': /* pvfstab file */ + strncpy(opt_pvfstab, optarg, 255); + opt_pvfstab_set = 1; + break; + case 'a': /* aligned allocation. + * syntax: -a/ + * e.g., -a4096/512 allocate at 4096 bytes + * boundary if request size >= 512. + */ + { + char *p; + + opt_alignment = (hsize_t)HDatoi(optarg); + if (NULL != (p = (char *)HDstrchr(optarg, '/'))) + opt_threshold = (hsize_t)HDatoi(p + 1); + } + HDfprintf(stdout, "alignment/threshold=%" PRIuHSIZE "/%" PRIuHSIZE "\n", opt_alignment, + opt_threshold); + break; + case '2': /* use 2-files, i.e., split file driver */ + opt_split_vfd = 1; + /* get meta and raw file extension. */ + /* syntax is , */ + meta_ext = raw_ext = optarg; + while (*raw_ext != '\0') { + if (*raw_ext == ',') { + *raw_ext = '\0'; + raw_ext++; + break; + } + raw_ext++; + } + printf("split-file-vfd used: %s,%s\n", meta_ext, raw_ext); + break; + case 'c': /* correctness */ + opt_correct = 1; + break; + case '?': /* unknown */ + default: + break; + } + } + + return (0); +} +/*------------------------------------------------------------------------- + * Function: getenv_all + * + * Purpose: Used to get the environment that the root MPI task has. + * name specifies which environment variable to look for + * val is the string to which the value of that environment + * variable will be copied. + * + * NOTE: The pointer returned by this function is only + * valid until the next call to getenv_all and the data + * stored there must be copied somewhere else before any + * further calls to getenv_all take place. + * + * Return: pointer to a string containing the value of the environment variable + * NULL if the varialbe doesn't exist in task 'root's environment. + * + * Programmer: Leon Arber + * 4/4/05 + * + * Modifications: + * Use original getenv if MPI is not initialized. This happens + * one uses the PHDF5 library to build a serial nature code. + * Albert 2006/04/07 + * + *------------------------------------------------------------------------- + */ +char * +getenv_all(MPI_Comm comm, int root, const char *name) +{ + int mpi_size, mpi_rank, mpi_initialized, mpi_finalized; + int len; + static char *env = NULL; + + HDassert(name); + + MPI_Initialized(&mpi_initialized); + MPI_Finalized(&mpi_finalized); + + if (mpi_initialized && !mpi_finalized) { + MPI_Comm_rank(comm, &mpi_rank); + MPI_Comm_size(comm, &mpi_size); + HDassert(root < mpi_size); + + /* The root task does the getenv call + * and sends the result to the other tasks */ + if (mpi_rank == root) { + env = HDgetenv(name); + if (env) { + len = (int)HDstrlen(env); + MPI_Bcast(&len, 1, MPI_INT, root, comm); + MPI_Bcast(env, len, MPI_CHAR, root, comm); + } + else { + /* len -1 indicates that the variable was not in the environment */ + len = -1; + MPI_Bcast(&len, 1, MPI_INT, root, comm); + } + } + else { + MPI_Bcast(&len, 1, MPI_INT, root, comm); + if (len >= 0) { + if (env == NULL) + env = (char *)HDmalloc((size_t)len + 1); + else if (HDstrlen(env) < (size_t)len) + env = (char *)HDrealloc(env, (size_t)len + 1); + + MPI_Bcast(env, len, MPI_CHAR, root, comm); + env[len] = '\0'; + } + else { + if (env) + HDfree(env); + env = NULL; + } + } +#ifndef NDEBUG + MPI_Barrier(comm); +#endif + } + else { + /* use original getenv */ + if (env) + HDfree(env); + env = HDgetenv(name); + } /* end if */ + + return env; +} + +/*------------------------------------------------------------------------- + * Function: h5_fixname_real + * + * Purpose: Create a file name from a file base name like `test' and + * return it through the FULLNAME (at most SIZE characters + * counting the null terminator). The full name is created by + * prepending the contents of HDF5_PREFIX (separated from the + * base name by a slash) and appending a file extension based on + * the driver supplied, resulting in something like + * `ufs:/u/matzke/test.h5'. + * + * Return: Success: The FULLNAME pointer. + * + * Failure: NULL if BASENAME or FULLNAME is the null + * pointer or if FULLNAME isn't large enough for + * the result. + * + * Programmer: Robb Matzke + * Thursday, November 19, 1998 + * + *------------------------------------------------------------------------- + */ +static char * +h5_fixname_real(const char *base_name, hid_t fapl, const char *_suffix, char *fullname, size_t size, + hbool_t nest_printf, hbool_t subst_for_superblock) +{ + const char *prefix = NULL; + const char *env = NULL; /* HDF5_DRIVER environment variable */ + char * ptr, last = '\0'; + const char *suffix = _suffix; + size_t i, j; + hid_t driver = -1; + int isppdriver = 0; /* if the driver is MPI parallel */ + + if (!base_name || !fullname || size < 1) + return NULL; + + HDmemset(fullname, 0, size); + + /* figure out the suffix */ + if (H5P_DEFAULT != fapl) { + if ((driver = H5Pget_driver(fapl)) < 0) + return NULL; + + if (suffix) { + if (H5FD_FAMILY == driver) { + if (subst_for_superblock) + suffix = "00000.h5"; + else + suffix = nest_printf ? "%%05d.h5" : "%05d.h5"; + } + else if (H5FD_MULTI == driver) { + + /* Get the environment variable, if it exists, in case + * we are using the split driver since both of those + * use the multi VFD under the hood. + */ + env = HDgetenv("HDF5_DRIVER"); +#ifdef HDF5_DRIVER + /* Use the environment variable, then the compile-time constant */ + if (!env) + env = HDF5_DRIVER; +#endif + if (env && !HDstrcmp(env, "split")) { + /* split VFD */ + if (subst_for_superblock) + suffix = "-m.h5"; + else + suffix = NULL; + } + else { + /* multi VFD */ + if (subst_for_superblock) + suffix = "-s.h5"; + else + suffix = NULL; + } + } + } + } + + /* Must first check fapl is not H5P_DEFAULT (-1) because H5FD_XXX + * could be of value -1 if it is not defined. + */ + isppdriver = H5P_DEFAULT != fapl && (H5FD_MPIO == driver); + + /* Check what prefix to use for test files. Process HDF5_PARAPREFIX and + * HDF5_PREFIX. + * Use different ones depending on parallel or serial driver used. + * (The #ifdef is needed to prevent compile failure in case MPI is not + * configured.) + */ + if (isppdriver) { + /* + * For parallel: + * First use command line option, then the environment + * variable, then try the constant + */ + static int explained = 0; + + prefix = (paraprefix ? paraprefix : getenv_all(MPI_COMM_WORLD, 0, "HDF5_PARAPREFIX")); + + if (!prefix && !explained) { + /* print hint by process 0 once. */ + int mpi_rank; + + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + if (mpi_rank == 0) + HDprintf("*** Hint ***\n" + "You can use environment variable HDF5_PARAPREFIX to " + "run parallel test files in a\n" + "different directory or to add file type prefix. e.g.,\n" + " HDF5_PARAPREFIX=pfs:/PFS/user/me\n" + " export HDF5_PARAPREFIX\n" + "*** End of Hint ***\n"); + + explained = TRUE; +#ifdef HDF5_PARAPREFIX + prefix = HDF5_PARAPREFIX; +#endif /* HDF5_PARAPREFIX */ + } + } + else { + /* + * For serial: + * First use the environment variable, then try the constant + */ + prefix = HDgetenv("HDF5_PREFIX"); + +#ifdef HDF5_PREFIX + if (!prefix) + prefix = HDF5_PREFIX; +#endif /* HDF5_PREFIX */ + } + + /* Prepend the prefix value to the base name */ + if (prefix && *prefix) { + if (isppdriver) { + /* This is a parallel system */ + char *subdir; + + if (!HDstrcmp(prefix, HDF5_PARAPREFIX)) { + /* + * If the prefix specifies the HDF5_PARAPREFIX directory, then + * default to using the "/tmp/$USER" or "/tmp/$LOGIN" + * directory instead. + */ + char *user, *login; + + user = HDgetenv("USER"); + login = HDgetenv("LOGIN"); + subdir = (user ? user : login); + + if (subdir) { + for (i = 0; i < size && prefix[i]; i++) + fullname[i] = prefix[i]; + + fullname[i++] = '/'; + + for (j = 0; i < size && subdir[j]; ++i, ++j) + fullname[i] = subdir[j]; + } + } + + if (!fullname[0]) { + /* We didn't append the prefix yet */ + HDstrncpy(fullname, prefix, size); + fullname[size - 1] = '\0'; + } + + if (HDstrlen(fullname) + HDstrlen(base_name) + 1 < size) { + /* + * Append the base_name with a slash first. Multiple + * slashes are handled below. + */ + h5_stat_t buf; + + if (HDstat(fullname, &buf) < 0) + /* The directory doesn't exist just yet */ + if (HDmkdir(fullname, (mode_t)0755) < 0 && errno != EEXIST) + /* + * We couldn't make the "/tmp/${USER,LOGIN}" + * subdirectory. Default to PREFIX's original + * prefix value. + */ + HDstrcpy(fullname, prefix); + + HDstrcat(fullname, "/"); + HDstrcat(fullname, base_name); + } + else { + /* Buffer is too small */ + return NULL; + } + } + else { + if (HDsnprintf(fullname, size, "%s/%s", prefix, base_name) == (int)size) + /* Buffer is too small */ + return NULL; + } + } + else if (HDstrlen(base_name) >= size) { + /* Buffer is too small */ + return NULL; + } + else { + HDstrcpy(fullname, base_name); + } + + /* Append a suffix */ + if (suffix) { + if (HDstrlen(fullname) + HDstrlen(suffix) >= size) + return NULL; + + HDstrcat(fullname, suffix); + } + + /* Remove any double slashes in the filename */ + for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) { + if (*ptr != '/' || last != '/') + fullname[j++] = *ptr; + + last = *ptr; + } + + return fullname; +} + +/* + * Local variables: + * c-indent-level: 3 + * c-basic-offset: 3 + * tab-width: 3 + * End: + */ + +#else /* H5_HAVE_PARALLEL */ +/* dummy program since H5_HAVE_PARALLEL is not configured in */ +int +main(int H5_ATTR_UNUSED argc, char H5_ATTR_UNUSED **argv) +{ + printf("No parallel performance because parallel is not configured in\n"); + return (0); +} +#endif /* H5_HAVE_PARALLEL */ diff --git a/tools/src/h5perf/pio_engine.c b/tools/src/h5perf/pio_engine.c new file mode 100644 index 0000000..65194b0 --- /dev/null +++ b/tools/src/h5perf/pio_engine.c @@ -0,0 +1,2747 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Author: Albert Cheng of NCSA, Oct 24, 2001. + */ + +#include "hdf5.h" + +#ifdef H5_STDC_HEADERS +#include +#include +#include +#include +#endif + +#ifdef H5_HAVE_UNISTD_H +#include +#include +#endif + +#ifdef H5_HAVE_SYS_STAT_H +#include +#endif + +#ifdef H5_HAVE_PARALLEL + +#include + +#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ +#include +#endif /* !MPI_FILE_NULL */ + +#include "pio_perf.h" + +/* Macro definitions */ + +#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 6 +#define H5DCREATE(fd, name, type, space, dcpl) H5Dcreate(fd, name, type, space, dcpl) +#define H5DOPEN(fd, name) H5Dopen(fd, name) +#else +#define H5DCREATE(fd, name, type, space, dcpl) \ + H5Dcreate2(fd, name, type, space, H5P_DEFAULT, dcpl, H5P_DEFAULT) +#define H5DOPEN(fd, name) H5Dopen2(fd, name, H5P_DEFAULT) +#endif + +/* sizes of various items. these sizes won't change during program execution */ +/* The following three must have the same type */ +#define ELMT_H5_TYPE H5T_NATIVE_UCHAR + +#define GOTOERROR(errcode) \ + { \ + ret_code = errcode; \ + goto done; \ + } +#define ERRMSG(mesg) \ + { \ + HDfprintf(stderr, "Proc %d: ", pio_mpi_rank_g); \ + HDfprintf(stderr, "*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ + } + +/* verify: if val is false (0), print mesg. */ +#define VRFY(val, mesg) \ + do { \ + if (!val) { \ + ERRMSG(mesg); \ + GOTOERROR(FAIL); \ + } \ + } while (0) + +/* POSIX I/O macros */ +#ifdef H5_HAVE_WIN32_API +/* Can't link against the library, so this test will use the older, non-Unicode + * _open() call on Windows. + */ +#define HDopen(S, F, ...) _open(S, F | _O_BINARY, __VA_ARGS__) +#endif /* H5_HAVE_WIN32_API */ +#define POSIXCREATE(fn) HDopen(fn, O_CREAT | O_TRUNC | O_RDWR, 0600) +#define POSIXOPEN(fn, F) HDopen(fn, F, 0600) +#define POSIXCLOSE(F) HDclose(F) +#define POSIXSEEK(F, L) HDlseek(F, L, SEEK_SET) +#define POSIXWRITE(F, B, S) HDwrite(F, B, S) +#define POSIXREAD(F, B, S) HDread(F, B, S) + +enum { PIO_CREATE = 1, PIO_WRITE = 2, PIO_READ = 4 }; + +/* Global variables */ +static int clean_file_g = -1; /*whether to cleanup temporary test */ +/*files. -1 is not defined; */ +/*0 is no cleanup; 1 is do cleanup */ + +/* + * In a parallel machine, the filesystem suitable for compiling is + * unlikely a parallel file system that is suitable for parallel I/O. + * There is no standard pathname for the parallel file system. /tmp + * is about the best guess. + */ +#ifndef HDF5_PARAPREFIX +#define HDF5_PARAPREFIX "" +#endif /* !HDF5_PARAPREFIX */ + +#ifndef MIN +#define MIN(a, b) ((a) < (b) ? (a) : (b)) +#endif /* !MIN */ + +/* the different types of file descriptors we can expect */ +typedef union _file_descr { + int posixfd; /* POSIX file handle*/ + MPI_File mpifd; /* MPI file */ + hid_t h5fd; /* HDF5 file */ +} file_descr; + +/* local functions */ +static char * pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size); +static herr_t do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nelmts, + size_t buf_size, void *buffer); +static herr_t do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nelmts, + size_t buf_size, void *buffer /*out*/); +static herr_t do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags); +static herr_t do_fclose(iotype iot, file_descr *fd); +static void do_cleanupfile(iotype iot, char *fname); +static off_t sqrto(off_t); + +/* + * Function: do_pio + * Purpose: PIO Engine where Parallel IO are executed. + * Return: results + * Programmer: Albert Cheng, Bill Wendling 2001/12/12 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +results +do_pio(parameters param) +{ + /* return codes */ + herr_t ret_code = 0; /*return code */ + results res; + + file_descr fd; + iotype iot; + + char fname[FILENAME_MAX]; + long nf; + long ndsets; + off_t nbytes; /*number of bytes per dataset */ + off_t snbytes; /*general dataset size */ + /*for 1D, it is the actual dataset size */ + /*for 2D, it is the size of a side of the dataset square */ + char * buffer = NULL; /*data buffer pointer */ + size_t buf_size; /*general buffer size in bytes */ + /*for 1D, it is the actual buffer size */ + /*for 2D, it is the length of the buffer rectangle */ + size_t blk_size; /*data block size in bytes */ + size_t bsize; /*actual buffer size */ + + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + + /* Sanity check parameters */ + + /* IO type */ + iot = param.io_type; + + switch (iot) { + case MPIO: + fd.mpifd = MPI_FILE_NULL; + res.timers = io_time_new(MPI_CLOCK); + break; + case POSIXIO: + fd.posixfd = -1; + res.timers = io_time_new(MPI_CLOCK); + break; + case PHDF5: + fd.h5fd = -1; + res.timers = io_time_new(MPI_CLOCK); + break; + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", iot); + GOTOERROR(FAIL); + } + + ndsets = param.num_dsets; /* number of datasets per file */ + nbytes = param.num_bytes; /* number of bytes per dataset */ + buf_size = param.buf_size; + blk_size = param.blk_size; + + if (!param.dim2d) { + snbytes = nbytes; /* General dataset size */ + bsize = buf_size; /* Actual buffer size */ + } + else { + snbytes = sqrto(nbytes); /* General dataset size */ + bsize = buf_size * blk_size; /* Actual buffer size */ + } + + if (param.num_files < 0) { + HDfprintf(stderr, "number of files must be >= 0 (%ld)\n", param.num_files); + GOTOERROR(FAIL); + } + + if (ndsets < 0) { + HDfprintf(stderr, "number of datasets per file must be >= 0 (%ld)\n", ndsets); + GOTOERROR(FAIL); + } + + if (param.num_procs <= 0) { + HDfprintf(stderr, "maximum number of process to use must be > 0 (%d)\n", param.num_procs); + GOTOERROR(FAIL); + } + + /* Validate transfer buffer size & block size*/ + if (blk_size <= 0) { + HDfprintf(stderr, "Transfer block size (%zu) must be > 0\n", blk_size); + GOTOERROR(FAIL); + } + if (buf_size <= 0) { + HDfprintf(stderr, "Transfer buffer size (%zu) must be > 0\n", buf_size); + GOTOERROR(FAIL); + } + if ((buf_size % blk_size) != 0) { + HDfprintf(stderr, + "Transfer buffer size (%zu) must be a multiple of the " + "interleaved I/O block size (%zu)\n", + buf_size, blk_size); + GOTOERROR(FAIL); + } + if ((snbytes % pio_mpi_nprocs_g) != 0) { + HDfprintf(stderr, + "Dataset size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " + "number of processes (%d)\n", + (long long)snbytes, pio_mpi_nprocs_g); + GOTOERROR(FAIL); + } + + if (!param.dim2d) { + if (((size_t)(snbytes / pio_mpi_nprocs_g) % buf_size) != 0) { + HDfprintf(stderr, + "Dataset size/process (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " + "trasfer buffer size (%zu)\n", + (long long)(snbytes / pio_mpi_nprocs_g), buf_size); + GOTOERROR(FAIL); + } + } + else { + if (((size_t)snbytes % buf_size) != 0) { + HDfprintf(stderr, + "Dataset side size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " + "trasfer buffer size (%zu)\n", + (long long)snbytes, buf_size); + GOTOERROR(FAIL); + } + } + + /* Allocate transfer buffer */ + if ((buffer = malloc(bsize)) == NULL) { + HDfprintf(stderr, "malloc for transfer buffer size (%zu) failed\n", bsize); + GOTOERROR(FAIL); + } + + if (pio_debug_level >= 4) { + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + + /* output all of the times for all iterations */ + if (myrank == 0) + HDfprintf(output, "Timer details:\n"); + } + + for (nf = 1; nf <= param.num_files; nf++) { + /* + * Write performance measurement + */ + /* Open file for write */ + char base_name[256]; + + HDsprintf(base_name, "#pio_tmp_%lu", nf); + pio_create_filename(iot, base_name, fname, sizeof(fname)); + if (pio_debug_level > 0) + HDfprintf(output, "rank %d: data filename=%s\n", pio_mpi_rank_g, fname); + + /* Need barrier to make sure everyone starts at the same time */ + MPI_Barrier(pio_comm_g); + + io_time_set(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTART); + hrc = do_fopen(¶m, fname, &fd, PIO_CREATE | PIO_WRITE); + + VRFY((hrc == SUCCESS), "do_fopen failed"); + + io_time_set(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTART); + hrc = do_write(&res, &fd, ¶m, ndsets, nbytes, buf_size, buffer); + io_time_set(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTOP); + + VRFY((hrc == SUCCESS), "do_write failed"); + + /* Close file for write */ + hrc = do_fclose(iot, &fd); + + io_time_set(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_fclose failed"); + + if (!param.h5_write_only) { + /* + * Read performance measurement + */ + /* Need barrier to make sure everyone is done writing and has + * closed the file. Also to make sure everyone starts reading + * at the same time. + */ + MPI_Barrier(pio_comm_g); + + /* Open file for read */ + io_time_set(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTART); + hrc = do_fopen(¶m, fname, &fd, PIO_READ); + + VRFY((hrc == SUCCESS), "do_fopen failed"); + + io_time_set(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTART); + hrc = do_read(&res, &fd, ¶m, ndsets, nbytes, buf_size, buffer); + io_time_set(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_read failed"); + + /* Close file for read */ + hrc = do_fclose(iot, &fd); + + io_time_set(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_fclose failed"); + } + + /* Need barrier to make sure everyone is done with the file */ + /* before it may be removed by do_cleanupfile */ + MPI_Barrier(pio_comm_g); + do_cleanupfile(iot, fname); + } + +done: + /* clean up */ + /* release HDF5 objects */ + + /* close any opened files */ + /* no remove(fname) because that should have happened normally. */ + switch (iot) { + case POSIXIO: + if (fd.posixfd != -1) + hrc = do_fclose(iot, &fd); + break; + case MPIO: + if (fd.mpifd != MPI_FILE_NULL) + hrc = do_fclose(iot, &fd); + break; + case PHDF5: + if (fd.h5fd != -1) + hrc = do_fclose(iot, &fd); + break; + default: + break; + } + + /* release generic resources */ + if (buffer) + HDfree(buffer); + res.ret_code = ret_code; + return res; +} + +/* + * Function: pio_create_filename + * Purpose: Create a new filename to write to. Determine the correct + * suffix to append to the filename by the type of I/O we're + * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if + * USER or LOGIN are specified in the environment. + * Return: Pointer to filename or NULL + * Programmer: Bill Wendling, 21. November 2001 + * Modifications: + */ +static char * +pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size) +{ + const char *prefix, *suffix = ""; + char * ptr, last = '\0'; + size_t i, j; + + if (!base_name || !fullname || size < 1) + return NULL; + + HDmemset(fullname, 0, size); + + switch (iot) { + case POSIXIO: + suffix = ".posix"; + break; + case MPIO: + suffix = ".mpio"; + break; + case PHDF5: + suffix = ".h5"; + break; + default: + break; + } + + /* First use the environment variable and then try the constant */ + prefix = HDgetenv("HDF5_PARAPREFIX"); + +#ifdef HDF5_PARAPREFIX + if (!prefix) + prefix = HDF5_PARAPREFIX; +#endif /* HDF5_PARAPREFIX */ + + /* Prepend the prefix value to the base name */ + if (prefix && *prefix) { + /* If the prefix specifies the HDF5_PARAPREFIX directory, then + * default to using the "/tmp/$USER" or "/tmp/$LOGIN" + * directory instead. */ + register char *user, *login, *subdir; + + user = HDgetenv("USER"); + login = HDgetenv("LOGIN"); + subdir = (user ? user : login); + + if (subdir) { + for (i = 0; i < size - 1 && prefix[i]; i++) + fullname[i] = prefix[i]; + + fullname[i++] = '/'; + + for (j = 0; i < size && subdir[j]; i++, j++) + fullname[i] = subdir[j]; + } + else { + /* We didn't append the prefix yet */ + HDstrncpy(fullname, prefix, size); + fullname[size - 1] = '\0'; + } + + if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) { + /* Append the base_name with a slash first. Multiple slashes are + * handled below. */ + h5_stat_t buf; + + if (HDstat(fullname, &buf) < 0) + /* The directory doesn't exist just yet */ + if (HDmkdir(fullname, (mode_t)0755) < 0 && errno != EEXIST) { + /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. + * Default to PREFIX's original prefix value. */ + HDstrcpy(fullname, prefix); + } + + HDstrcat(fullname, "/"); + HDstrcat(fullname, base_name); + } + else { + /* Buffer is too small */ + return NULL; + } + } + else if (HDstrlen(base_name) >= size) { + /* Buffer is too small */ + return NULL; + } + else { + HDstrcpy(fullname, base_name); + } + + /* Append a suffix */ + if (suffix) { + if (HDstrlen(fullname) + HDstrlen(suffix) >= size) + return NULL; + + HDstrcat(fullname, suffix); + } + + /* Remove any double slashes in the filename */ + for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) { + if (*ptr != '/' || last != '/') + fullname[j++] = *ptr; + + last = *ptr; + } + + return fullname; +} + +/* + * Function: do_write + * Purpose: Write the required amount of data to the file. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static herr_t +do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbytes, size_t buf_size, + void *buffer) +{ + int ret_code = SUCCESS; + int rc; /*routine return code */ + long ndset; + size_t blk_size; /* The block size to subdivide the xfer buffer into */ + off_t nbytes_xfer; /* Total number of bytes transferred so far */ + size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */ + size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */ + char dname[64]; + off_t dset_offset = 0; /*dataset offset in a file */ + off_t bytes_begin[2]; /*first elmt this process transfer */ + off_t bytes_count; /*number of elmts this process transfer */ + off_t snbytes = 0; /*size of a side of the dataset square */ + unsigned char *buf_p; /* Current buffer pointer */ + + /* POSIX variables */ + off_t file_offset; /* File offset of the next transfer */ + off_t file_offset_advance; /* File offset advance after each I/O operation */ + off_t posix_file_offset; /* Base file offset of the next transfer */ + + /* MPI variables */ + MPI_Offset mpi_file_offset; /* Base file offset of the next transfer*/ + MPI_Offset mpi_offset; /* Offset in MPI file */ + MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */ + MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */ + MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */ + MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */ + MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */ + MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */ + MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */ + MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */ + MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */ + MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */ + MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */ + MPI_Status mpi_status; + int mrc; /* MPI return code */ + + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + hsize_t h5dims[2]; /*dataset dim sizes */ + hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ + hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ + hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ + hsize_t h5block[2]; /*dataspace selection */ + hsize_t h5stride[2]; + hsize_t h5count[2]; + hsize_t h5start[2]; + hssize_t h5offset[2]; /* Selection offset within dataspace */ + hid_t h5dcpl = H5I_INVALID_HID; /* Dataset creation property list */ + hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ + + /* Get the parameters from the parameter block */ + blk_size = parms->blk_size; + + /* There are two kinds of transfer patterns, contiguous and interleaved. + * Let 0,1,2,...,n be data accessed by process 0,1,2,...,n + * where n is rank of the last process. + * In contiguous pattern, data are accessed as + * 000...111...222...nnn... + * In interleaved pattern, data are accessed as + * 012...n012...n... + * These are all in the scope of one dataset. + */ + + /* 1D dataspace */ + if (!parms->dim2d) { + /* Contiguous Pattern: */ + if (!parms->interleaved) { + bytes_begin[0] = (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); + } /* end if */ + /* Interleaved Pattern: */ + else { + bytes_begin[0] = (off_t)(blk_size * (size_t)pio_mpi_rank_g); + } /* end else */ + + /* Prepare buffer for verifying data */ + if (parms->verify) + memset(buffer, pio_mpi_rank_g + 1, buf_size); + } /* end if */ + /* 2D dataspace */ + else { + /* nbytes is always the number of bytes per dataset (1D or 2D). If the + dataspace is 2D, snbytes is the size of a side of the dataset square. + */ + snbytes = sqrto(nbytes); + + /* Contiguous Pattern: */ + if (!parms->interleaved) { + bytes_begin[0] = (off_t)((double)snbytes * pio_mpi_rank_g / pio_mpi_nprocs_g); + bytes_begin[1] = 0; + } /* end if */ + /* Interleaved Pattern: */ + else { + bytes_begin[0] = 0; + + if (!parms->h5_use_chunks || parms->io_type == PHDF5) + bytes_begin[1] = (off_t)(blk_size * (size_t)pio_mpi_rank_g); + else + bytes_begin[1] = (off_t)(blk_size * blk_size * (size_t)pio_mpi_rank_g); + } /* end else */ + + /* Prepare buffer for verifying data */ + if (parms->verify) + HDmemset(buffer, pio_mpi_rank_g + 1, buf_size * blk_size); + } /* end else */ + + /* Calculate the total number of bytes (bytes_count) to be + * transferred by this process. It may be different for different + * transfer pattern due to rounding to integral values. + */ + /* + * Calculate the beginning bytes of this process and the next. + * bytes_count is the difference between these two beginnings. + * This way, it eliminates any rounding errors. + * (This is tricky, don't mess with the formula, rounding errors + * can easily get introduced) */ + bytes_count = (off_t)(((double)nbytes * (pio_mpi_rank_g + 1)) / pio_mpi_nprocs_g) - + (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); + + /* debug */ + if (pio_debug_level >= 4) { + HDprint_rank(output); + if (!parms->dim2d) { + HDfprintf(output, + "Debug(do_write): " + "buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH + "d\n", + buf_size, (long long)bytes_begin[0], (long long)bytes_count); + } + else { + HDfprintf(output, + "Debug(do_write): " + "linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH + "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n", + buf_size * blk_size, (long long)bytes_begin[0], (long long)bytes_begin[1], + (long long)bytes_count); + } + } + + /* I/O Access specific setup */ + switch (parms->io_type) { + case POSIXIO: + /* No extra setup */ + break; + + case MPIO: /* MPI-I/O setup */ + /* 1D dataspace */ + if (!parms->dim2d) { + /* Build block's derived type */ + mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Build file's derived type */ + mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)pio_mpi_nprocs_g, mpi_blk_type, + &mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit file type */ + mrc = MPI_Type_commit(&mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Commit buffer type */ + mrc = MPI_Type_commit(&mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + } /* end if */ + /* 2D dataspace */ + else { + /* Build partial buffer derived type for contiguous access */ + + mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE, &mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit partial buffer derived type */ + mrc = MPI_Type_commit(&mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build contiguous file's derived type */ + mrc = MPI_Type_vector((int)blk_size, (int)1, (int)((size_t)snbytes / buf_size), + mpi_partial_buffer_cont, &mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit contiguous file type */ + mrc = MPI_Type_commit(&mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build partial buffer derived type for interleaved access */ + mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit partial buffer derived type */ + mrc = MPI_Type_commit(&mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build interleaved file's derived type */ + mrc = MPI_Type_vector((int)buf_size, (int)1, (int)((size_t)snbytes / blk_size), + mpi_partial_buffer_inter, &mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit interleaved file type */ + mrc = MPI_Type_commit(&mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build full buffer derived type */ + mrc = MPI_Type_contiguous((int)(blk_size * buf_size), MPI_BYTE, &mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit full buffer derived type */ + mrc = MPI_Type_commit(&mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build full chunk derived type */ + mrc = MPI_Type_contiguous((int)(blk_size * blk_size), MPI_BYTE, &mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit full chunk derived type */ + mrc = MPI_Type_commit(&mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build chunk interleaved file's derived type */ + mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)((size_t)snbytes / blk_size), + mpi_full_chunk, &mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit chunk interleaved file type */ + mrc = MPI_Type_commit(&mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + } /* end else */ + break; + + case PHDF5: /* HDF5 setup */ + /* 1D dataspace */ + if (!parms->dim2d) { + if (nbytes > 0) { + /* define a contiguous dataset of nbytes native bytes */ + h5dims[0] = (hsize_t)nbytes; + h5dset_space_id = H5Screate_simple(1, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + /* Set up the file dset space id to select the pattern to access */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5stride[0] = h5block[0] = blk_size; + h5count[0] = buf_size / blk_size; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5stride[0] = blk_size * (size_t)pio_mpi_nprocs_g; + h5block[0] = blk_size; + h5count[0] = buf_size / blk_size; + } /* end else */ + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, + h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + } /* end if */ + else { + h5dset_space_id = H5Screate(H5S_SCALAR); + VRFY((h5dset_space_id >= 0), "H5Screate"); + } /* end else */ + + /* Create the memory dataspace that corresponds to the xfer buffer */ + if (buf_size > 0) { + h5dims[0] = buf_size; + h5mem_space_id = H5Screate_simple(1, h5dims, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + } /* end if */ + else { + h5mem_space_id = H5Screate(H5S_SCALAR); + VRFY((h5mem_space_id >= 0), "H5Screate"); + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + if (nbytes > 0) { + /* define a contiguous dataset of nbytes native bytes */ + h5dims[0] = (hsize_t)snbytes; + h5dims[1] = (hsize_t)snbytes; + h5dset_space_id = H5Screate_simple(2, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + /* Set up the file dset space id to select the pattern to access */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5start[1] = (hsize_t)bytes_begin[1]; + h5stride[0] = 1; + h5stride[1] = h5block[0] = h5block[1] = blk_size; + h5count[0] = 1; + h5count[1] = buf_size / blk_size; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5start[1] = (hsize_t)bytes_begin[1]; + h5stride[0] = blk_size; + h5stride[1] = blk_size * (size_t)pio_mpi_nprocs_g; + h5block[0] = h5block[1] = blk_size; + h5count[0] = buf_size / blk_size; + h5count[1] = 1; + } /* end else */ + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, + h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + } /* end if */ + else { + h5dset_space_id = H5Screate(H5S_SCALAR); + VRFY((h5dset_space_id >= 0), "H5Screate"); + } /* end else */ + + /* Create the memory dataspace that corresponds to the xfer buffer */ + if (buf_size > 0) { + if (!parms->interleaved) { + h5dims[0] = blk_size; + h5dims[1] = buf_size; + } + else { + h5dims[0] = buf_size; + h5dims[1] = blk_size; + } + h5mem_space_id = H5Screate_simple(2, h5dims, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + } /* end if */ + else { + h5mem_space_id = H5Screate(H5S_SCALAR); + VRFY((h5mem_space_id >= 0), "H5Screate"); + } /* end else */ + } /* end else */ + + /* Create the dataset transfer property list */ + h5dxpl = H5Pcreate(H5P_DATASET_XFER); + if (h5dxpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + /* Change to collective I/O, if asked */ + if (parms->collective) { + hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + break; + + default: + break; + } /* end switch */ + + for (ndset = 1; ndset <= ndsets; ++ndset) { + + /* Calculate dataset offset within a file */ + + /* create dataset */ + switch (parms->io_type) { + case POSIXIO: + case MPIO: + /* both posix and mpi io just need dataset offset in file*/ + dset_offset = (ndset - 1) * nbytes; + break; + + case PHDF5: + h5dcpl = H5Pcreate(H5P_DATASET_CREATE); + if (h5dcpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + /* 1D dataspace */ + if (!parms->dim2d) { + /* Make the dataset chunked if asked */ + if (parms->h5_use_chunks) { + /* Set the chunk size to be the same as the buffer size */ + h5dims[0] = blk_size; + hrc = H5Pset_chunk(h5dcpl, 1, h5dims); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + } /* end if */ + else { + /* 2D dataspace */ + if (parms->h5_use_chunks) { + /* Set the chunk size to be the same as the block size */ + h5dims[0] = blk_size; + h5dims[1] = blk_size; + hrc = H5Pset_chunk(h5dcpl, 2, h5dims); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + } /* end else */ + + HDsprintf(dname, "Dataset_%ld", ndset); + h5ds_id = H5DCREATE(fd->h5fd, dname, ELMT_H5_TYPE, h5dset_space_id, h5dcpl); + + if (h5ds_id < 0) { + HDfprintf(stderr, "HDF5 Dataset Create failed\n"); + GOTOERROR(FAIL); + } + + hrc = H5Pclose(h5dcpl); + /* verifying the close of the dcpl */ + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Close failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + break; + } + + /* The task is to transfer bytes_count bytes, starting at + * bytes_begin position, using transfer buffer of buf_size bytes. + * If interleaved, select buf_size at a time, in round robin + * fashion, according to number of process. Otherwise, select + * all bytes_count in contiguous. + */ + nbytes_xfer = 0; + + /* 1D dataspace */ + if (!parms->dim2d) { + /* Set base file offset for all I/O patterns and POSIX access */ + posix_file_offset = dset_offset + bytes_begin[0]; + + /* Set base file offset for all I/O patterns and MPI access */ + mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]); + } /* end if */ + else { + /* Set base file offset for all I/O patterns and POSIX access */ + posix_file_offset = dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]; + + /* Set base file offset for all I/O patterns and MPI access */ + mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]); + } /* end else */ + + /* Start "raw data" write timer */ + io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART); + + while (nbytes_xfer < bytes_count) { + /* Write */ + /* Calculate offset of write within a dataset/file */ + switch (parms->io_type) { + case POSIXIO: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Contiguous pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + (off_t)nbytes_xfer; + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are written */ + rc = ((ssize_t)buf_size == POSIXWRITE(fd->posixfd, buffer, buf_size)); + VRFY((rc != 0), "POSIXWRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size; + + /* Loop over the buffers to write */ + while (nbytes_toxfer > 0) { + /* Skip offset over blocks of other processes */ + file_offset = posix_file_offset + (off_t)(nbytes_xfer * pio_mpi_nprocs_g); + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are written */ + rc = ((ssize_t)blk_size == POSIXWRITE(fd->posixfd, buf_p, blk_size)); + VRFY((rc != 0), "POSIXWRITE"); + + /* Advance location in buffer */ + buf_p += blk_size; + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)blk_size; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= blk_size; + } /* end while */ + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* Contiguous storage */ + if (!parms->h5_use_chunks) { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + + (off_t)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * + (blk_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / blk_size) % (size_t)snbytes)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = buf_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = (off_t)snbytes; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute file offset */ + file_offset = + posix_file_offset + + (off_t)(((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / + (size_t)snbytes) * + (buf_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % + (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = (off_t)snbytes; + } /* end else */ + } /* end if */ + /* Chunked storage */ + else { + /*Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + (off_t)nbytes_xfer; + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * buf_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = 0; + } /* end if */ + /*Interleaved access pattern */ + else { + /* Compute file offset */ + /* Before simplification */ + /* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size) + *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size + *snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size)) + *pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */ + + file_offset = posix_file_offset + + (off_t)((((size_t)nbytes_xfer / (buf_size / blk_size) * + (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * blk_size)) * + (buf_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / (buf_size / blk_size)) * + (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * blk_size)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * blk_size; + + /* Global offset advance after each I/O operation */ + /* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */ + file_offset_advance = (off_t)snbytes * (off_t)blk_size; + } /* end else */ + } /* end else */ + + /* Common code for file access */ + + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size * blk_size; + + /* Loop over portions of the buffer to write */ + while (nbytes_toxfer > 0) { + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are written */ + rc = ((ssize_t)nbytes_xfer_advance == + POSIXWRITE(fd->posixfd, buf_p, nbytes_xfer_advance)); + VRFY((rc != 0), "POSIXWRITE"); + + /* Advance location in buffer */ + buf_p += nbytes_xfer_advance; + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)nbytes_xfer_advance; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= nbytes_xfer_advance; + + /* Partially advance file offset */ + file_offset += file_offset_advance; + } /* end while */ + + } /* end else */ + + break; + + case MPIO: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Independent file access */ + if (!parms->collective) { + /* Contiguous pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Perform independent write */ + mrc = + MPI_File_write_at(fd->mpifd, mpi_offset, buffer, + (int)(buf_size / blk_size), mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size; + + /* Loop over the buffers to write */ + while (nbytes_toxfer > 0) { + /* Skip offset over blocks of other processes */ + mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); + + /* Perform independent write */ + mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p, (int)1, + mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance location in buffer */ + buf_p += blk_size; + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)blk_size; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= blk_size; + } /* end while */ + } /* end else */ + } /* end if */ + /* Collective file access */ + else { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Perform independent write */ + mrc = MPI_File_write_at_all(fd->mpifd, mpi_offset, buffer, + (int)(buf_size / blk_size), mpi_blk_type, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); + + /* Set the file view */ + mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, + (char *)"native", h5_io_info_g); + VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); + + /* Perform write */ + mrc = MPI_File_write_at_all(fd->mpifd, 0, buffer, (int)(buf_size / blk_size), + mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)buf_size; + } /* end else */ + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* Contiguous storage */ + if (!parms->h5_use_chunks) { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = + mpi_file_offset + + (MPI_Offset)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * + (blk_size * (size_t)snbytes)) + + (MPI_Offset)(((size_t)nbytes_xfer / blk_size) % (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = buf_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = snbytes; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_cont_type; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute offset in file */ + mpi_offset = + mpi_file_offset + + (MPI_Offset)( + ((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / + (size_t)snbytes) * + (buf_size * (size_t)snbytes)) + + (MPI_Offset)( + (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % + (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = snbytes; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_inter_type; + } /* end else */ + } /* end if */ + /* Chunked storage */ + else { + /*Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * buf_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = 0; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_full_buffer; + } /* end if */ + /*Interleaved access pattern */ + else { + /* Compute offset in file */ + /* Before simplification */ + /* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size) + *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))* + (buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+ + ((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes + /blk_size*(blk_size*blk_size)); */ + mpi_offset = mpi_file_offset + + (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size) * + (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * blk_size)) * + (buf_size * (size_t)snbytes)) + + (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size)) * + (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * blk_size)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * blk_size; + + /* Global offset advance after each I/O operation */ + /* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */ + mpi_offset_advance = (MPI_Offset)((size_t)snbytes * blk_size); + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_chunk_inter_type; + } /* end else */ + } /* end else */ + + /* Common code for independent file access */ + if (!parms->collective) { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size * blk_size; + + /* Loop over portions of the buffer to write */ + while (nbytes_toxfer > 0) { + /* Perform independent write */ + mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p, + (int)nbytes_xfer_advance, MPI_BYTE, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance location in buffer */ + buf_p += nbytes_xfer_advance; + + /* Advance global offset in dataset */ + nbytes_xfer += (ssize_t)nbytes_xfer_advance; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= nbytes_xfer_advance; + + /* Partially advance global offset in dataset */ + mpi_offset += mpi_offset_advance; + } /* end while */ + } /* end if */ + + /* Common code for collective file access */ + else { + /* Set the file view */ + mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, + (char *)"native", h5_io_info_g); + VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); + + /* Perform write */ + MPI_File_write_at_all(fd->mpifd, 0, buffer, (int)(buf_size * blk_size), MPI_BYTE, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size * (off_t)blk_size; + } /* end else */ + + } /* end else */ + + break; + + case PHDF5: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Set up the file dset space id to move the selection to process */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5offset[0] = nbytes_xfer; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5offset[0] = (nbytes_xfer * pio_mpi_nprocs_g); + } /* end else */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Write the buffer out */ + hrc = + H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dwrite"); + + /* Increment number of bytes transferred */ + nbytes_xfer += (ssize_t)buf_size; + } /* end if */ + /* 2D dataspace */ + else { + /* Set up the file dset space id to move the selection to process */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5offset[0] = + (hssize_t)(((size_t)nbytes_xfer / ((size_t)snbytes * blk_size)) * blk_size); + h5offset[1] = + (hssize_t)(((size_t)nbytes_xfer % ((size_t)snbytes * blk_size)) / blk_size); + + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5offset[0] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * buf_size)) * + buf_size); + h5offset[1] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * buf_size)) / + buf_size); + + } /* end else */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Write the buffer out */ + hrc = + H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dwrite"); + + /* Increment number of bytes transferred */ + nbytes_xfer += (off_t)buf_size * (off_t)blk_size; + + } /* end else */ + + break; + + default: + break; + } /* switch (parms->io_type) */ + } /* end while */ + + /* Stop "raw data" write timer */ + io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP); + + /* Calculate write time */ + + /* Close dataset. Only HDF5 needs to do an explicit close. */ + if (parms->io_type == PHDF5) { + hrc = H5Dclose(h5ds_id); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Close failed\n"); + GOTOERROR(FAIL); + } + + h5ds_id = H5I_INVALID_HID; + } /* end if */ + } /* end for */ + +done: + /* release MPI-I/O objects */ + if (parms->io_type == MPIO) { + /* 1D dataspace */ + if (!parms->dim2d) { + /* Free file type */ + mrc = MPI_Type_free(&mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free buffer type */ + mrc = MPI_Type_free(&mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + } /* end if */ + /* 2D dataspace */ + else { + /* Free partial buffer type for contiguous access */ + mrc = MPI_Type_free(&mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free contiguous file type */ + mrc = MPI_Type_free(&mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free partial buffer type for interleaved access */ + mrc = MPI_Type_free(&mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free interleaved file type */ + mrc = MPI_Type_free(&mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free full buffer type */ + mrc = MPI_Type_free(&mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free full chunk type */ + mrc = MPI_Type_free(&mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free chunk interleaved file type */ + mrc = MPI_Type_free(&mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + } /* end else */ + } /* end if */ + + /* release HDF5 objects */ + if (h5dset_space_id != -1) { + hrc = H5Sclose(h5dset_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); + ret_code = FAIL; + } + else { + h5dset_space_id = H5I_INVALID_HID; + } + } + + if (h5mem_space_id != -1) { + hrc = H5Sclose(h5mem_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); + ret_code = FAIL; + } + else { + h5mem_space_id = H5I_INVALID_HID; + } + } + + if (h5dxpl != -1) { + hrc = H5Pclose(h5dxpl); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); + ret_code = FAIL; + } + else { + h5dxpl = H5I_INVALID_HID; + } + } + + return ret_code; +} + +static off_t +sqrto(off_t x) +{ + double root_x = sqrt((double)x); + return (off_t)root_x; +} + +/* + * Function: do_read + * Purpose: read the required amount of data from the file. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng 2001/12/13 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static herr_t +do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbytes, size_t buf_size, + void *buffer /*out*/) +{ + int ret_code = SUCCESS; + int rc; /*routine return code */ + long ndset; + size_t blk_size; /* The block size to subdivide the xfer buffer into */ + size_t bsize; /* Size of the actual buffer */ + off_t nbytes_xfer; /* Total number of bytes transferred so far */ + size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */ + size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */ + char dname[64]; + off_t dset_offset = 0; /*dataset offset in a file */ + off_t bytes_begin[2]; /*first elmt this process transfer */ + off_t bytes_count; /*number of elmts this process transfer */ + off_t snbytes = 0; /*size of a side of the dataset square */ + unsigned char *buf_p; /* Current buffer pointer */ + + /* POSIX variables */ + off_t file_offset; /* File offset of the next transfer */ + off_t file_offset_advance; /* File offset advance after each I/O operation */ + off_t posix_file_offset; /* Base file offset of the next transfer */ + + /* MPI variables */ + MPI_Offset mpi_file_offset; /* Base file offset of the next transfer*/ + MPI_Offset mpi_offset; /* Offset in MPI file */ + MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */ + MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */ + MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */ + MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */ + MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */ + MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */ + MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */ + MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */ + MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */ + MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */ + MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */ + MPI_Status mpi_status; + int mrc; /* MPI return code */ + + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + hsize_t h5dims[2]; /*dataset dim sizes */ + hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ + hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ + hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ + hsize_t h5block[2]; /*dataspace selection */ + hsize_t h5stride[2]; + hsize_t h5count[2]; + hsize_t h5start[2]; + hssize_t h5offset[2]; /* Selection offset within dataspace */ + hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ + + /* Get the parameters from the parameter block */ + blk_size = parms->blk_size; + + /* There are two kinds of transfer patterns, contiguous and interleaved. + * Let 0,1,2,...,n be data accessed by process 0,1,2,...,n + * where n is rank of the last process. + * In contiguous pattern, data are accessed as + * 000...111...222...nnn... + * In interleaved pattern, data are accessed as + * 012...n012...n... + * These are all in the scope of one dataset. + */ + + /* 1D dataspace */ + if (!parms->dim2d) { + bsize = buf_size; + /* Contiguous Pattern: */ + if (!parms->interleaved) { + bytes_begin[0] = (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); + } /* end if */ + /* Interleaved Pattern: */ + else { + bytes_begin[0] = (off_t)blk_size * (off_t)pio_mpi_rank_g; + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* nbytes is always the number of bytes per dataset (1D or 2D). If the + dataspace is 2D, snbytes is the size of a side of the 'dataset square'. + */ + snbytes = sqrto(nbytes); + + bsize = buf_size * blk_size; + + /* Contiguous Pattern: */ + if (!parms->interleaved) { + bytes_begin[0] = (off_t)((double)snbytes * pio_mpi_rank_g / pio_mpi_nprocs_g); + bytes_begin[1] = 0; + } /* end if */ + /* Interleaved Pattern: */ + else { + bytes_begin[0] = 0; + + if (!parms->h5_use_chunks || parms->io_type == PHDF5) + bytes_begin[1] = (off_t)blk_size * (off_t)pio_mpi_rank_g; + else + bytes_begin[1] = (off_t)blk_size * (off_t)blk_size * (off_t)pio_mpi_rank_g; + } /* end else */ + } /* end else */ + + /* Calculate the total number of bytes (bytes_count) to be + * transferred by this process. It may be different for different + * transfer pattern due to rounding to integral values. + */ + /* + * Calculate the beginning bytes of this process and the next. + * bytes_count is the difference between these two beginnings. + * This way, it eliminates any rounding errors. + * (This is tricky, don't mess with the formula, rounding errors + * can easily get introduced) */ + bytes_count = (off_t)(((double)nbytes * (pio_mpi_rank_g + 1)) / pio_mpi_nprocs_g) - + (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); + + /* debug */ + if (pio_debug_level >= 4) { + HDprint_rank(output); + if (!parms->dim2d) { + HDfprintf(output, + "Debug(do_write): " + "buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH + "d\n", + buf_size, (long long)bytes_begin[0], (long long)bytes_count); + } + else { + HDfprintf(output, + "Debug(do_write): " + "linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH + "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n", + buf_size * blk_size, (long long)bytes_begin[0], (long long)bytes_begin[1], + (long long)bytes_count); + } + } + + /* I/O Access specific setup */ + switch (parms->io_type) { + case POSIXIO: + /* No extra setup */ + break; + + case MPIO: /* MPI-I/O setup */ + /* 1D dataspace */ + if (!parms->dim2d) { + /* Build block's derived type */ + mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Build file's derived type */ + mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)pio_mpi_nprocs_g, mpi_blk_type, + &mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit file type */ + mrc = MPI_Type_commit(&mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Commit buffer type */ + mrc = MPI_Type_commit(&mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + } /* end if */ + /* 2D dataspace */ + else { + /* Build partial buffer derived type for contiguous access */ + mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE, &mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit partial buffer derived type */ + mrc = MPI_Type_commit(&mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build contiguous file's derived type */ + mrc = MPI_Type_vector((int)blk_size, (int)1, (int)((size_t)snbytes / buf_size), + mpi_partial_buffer_cont, &mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit contiguous file type */ + mrc = MPI_Type_commit(&mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build partial buffer derived type for interleaved access */ + mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit partial buffer derived type */ + mrc = MPI_Type_commit(&mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build interleaved file's derived type */ + mrc = MPI_Type_vector((int)buf_size, (int)1, (int)((size_t)snbytes / blk_size), + mpi_partial_buffer_inter, &mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit interleaved file type */ + mrc = MPI_Type_commit(&mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build full buffer derived type */ + mrc = MPI_Type_contiguous((int)(blk_size * buf_size), MPI_BYTE, &mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit full buffer derived type */ + mrc = MPI_Type_commit(&mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build full chunk derived type */ + mrc = MPI_Type_contiguous((int)(blk_size * blk_size), MPI_BYTE, &mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit full chunk derived type */ + mrc = MPI_Type_commit(&mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + + /* Build chunk interleaved file's derived type */ + mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)((size_t)snbytes / blk_size), + mpi_full_chunk, &mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); + + /* Commit chunk interleaved file type */ + mrc = MPI_Type_commit(&mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); + } /* end else */ + break; + + case PHDF5: /* HDF5 setup */ + /* 1D dataspace */ + if (!parms->dim2d) { + if (nbytes > 0) { + /* define a contiguous dataset of nbytes native bytes */ + h5dims[0] = (hsize_t)nbytes; + h5dset_space_id = H5Screate_simple(1, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + /* Set up the file dset space id to select the pattern to access */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5stride[0] = h5block[0] = blk_size; + h5count[0] = buf_size / blk_size; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5stride[0] = blk_size * (size_t)pio_mpi_nprocs_g; + h5block[0] = blk_size; + h5count[0] = buf_size / blk_size; + } /* end else */ + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, + h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + } /* end if */ + else { + h5dset_space_id = H5Screate(H5S_SCALAR); + VRFY((h5dset_space_id >= 0), "H5Screate"); + } /* end else */ + + /* Create the memory dataspace that corresponds to the xfer buffer */ + if (buf_size > 0) { + h5dims[0] = buf_size; + h5mem_space_id = H5Screate_simple(1, h5dims, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + } /* end if */ + else { + h5mem_space_id = H5Screate(H5S_SCALAR); + VRFY((h5mem_space_id >= 0), "H5Screate"); + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + if (nbytes > 0) { + /* define a contiguous dataset of nbytes native bytes */ + h5dims[0] = (hsize_t)snbytes; + h5dims[1] = (hsize_t)snbytes; + h5dset_space_id = H5Screate_simple(2, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + /* Set up the file dset space id to select the pattern to access */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5start[1] = (hsize_t)bytes_begin[1]; + h5stride[0] = 1; + h5stride[1] = h5block[0] = h5block[1] = blk_size; + h5count[0] = 1; + h5count[1] = buf_size / blk_size; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5start[0] = (hsize_t)bytes_begin[0]; + h5start[1] = (hsize_t)bytes_begin[1]; + h5stride[0] = blk_size; + h5stride[1] = blk_size * (size_t)pio_mpi_nprocs_g; + h5block[0] = h5block[1] = blk_size; + h5count[0] = buf_size / blk_size; + h5count[1] = 1; + } /* end else */ + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, + h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + } /* end if */ + else { + h5dset_space_id = H5Screate(H5S_SCALAR); + VRFY((h5dset_space_id >= 0), "H5Screate"); + } /* end else */ + + /* Create the memory dataspace that corresponds to the xfer buffer */ + if (buf_size > 0) { + if (!parms->interleaved) { + h5dims[0] = blk_size; + h5dims[1] = buf_size; + } + else { + h5dims[0] = buf_size; + h5dims[1] = blk_size; + } + h5mem_space_id = H5Screate_simple(2, h5dims, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + } /* end if */ + else { + h5mem_space_id = H5Screate(H5S_SCALAR); + VRFY((h5mem_space_id >= 0), "H5Screate"); + } /* end else */ + } /* end else */ + + /* Create the dataset transfer property list */ + h5dxpl = H5Pcreate(H5P_DATASET_XFER); + if (h5dxpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + /* Change to collective I/O, if asked */ + if (parms->collective) { + hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + break; + + default: + break; + } /* end switch */ + + for (ndset = 1; ndset <= ndsets; ++ndset) { + + /* Calculate dataset offset within a file */ + + /* create dataset */ + switch (parms->io_type) { + case POSIXIO: + case MPIO: + /* both posix and mpi io just need dataset offset in file*/ + dset_offset = (ndset - 1) * nbytes; + break; + + case PHDF5: + HDsprintf(dname, "Dataset_%ld", ndset); + h5ds_id = H5DOPEN(fd->h5fd, dname); + if (h5ds_id < 0) { + HDfprintf(stderr, "HDF5 Dataset open failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + break; + } + + /* The task is to transfer bytes_count bytes, starting at + * bytes_begin position, using transfer buffer of buf_size bytes. + * If interleaved, select buf_size at a time, in round robin + * fashion, according to number of process. Otherwise, select + * all bytes_count in contiguous. + */ + nbytes_xfer = 0; + + /* 1D dataspace */ + if (!parms->dim2d) { + /* Set base file offset for all I/O patterns and POSIX access */ + posix_file_offset = dset_offset + bytes_begin[0]; + + /* Set base file offset for all I/O patterns and MPI access */ + mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]); + } /* end if */ + else { + /* Set base file offset for all I/O patterns and POSIX access */ + posix_file_offset = dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]; + + /* Set base file offset for all I/O patterns and MPI access */ + mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]); + } /* end else */ + + /* Start "raw data" read timer */ + io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTART); + + while (nbytes_xfer < bytes_count) { + /* Read */ + /* Calculate offset of read within a dataset/file */ + switch (parms->io_type) { + case POSIXIO: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Contiguous pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + (off_t)nbytes_xfer; + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are read */ + rc = ((ssize_t)buf_size == POSIXREAD(fd->posixfd, buffer, buf_size)); + VRFY((rc != 0), "POSIXREAD"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size; + + /* Loop over the buffers to read */ + while (nbytes_toxfer > 0) { + /* Skip offset over blocks of other processes */ + file_offset = posix_file_offset + (off_t)(nbytes_xfer * pio_mpi_nprocs_g); + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are read */ + rc = ((ssize_t)blk_size == POSIXREAD(fd->posixfd, buf_p, blk_size)); + VRFY((rc != 0), "POSIXREAD"); + + /* Advance location in buffer */ + buf_p += blk_size; + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)blk_size; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= blk_size; + } /* end while */ + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* Contiguous storage */ + if (!parms->h5_use_chunks) { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + + (off_t)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * + (blk_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / blk_size) % (size_t)snbytes)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = buf_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = (off_t)snbytes; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute file offset */ + file_offset = + posix_file_offset + + (off_t)(((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / + (size_t)snbytes) * + (buf_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % + (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = (off_t)snbytes; + } /* end else */ + } /* end if */ + /* Chunked storage */ + else { + /*Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute file offset */ + file_offset = posix_file_offset + (off_t)nbytes_xfer; + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * buf_size; + + /* Global offset advance after each I/O operation */ + file_offset_advance = 0; + } /* end if */ + /*Interleaved access pattern */ + else { + /* Compute file offset */ + /* Before simplification */ + /* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size) + *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size + *snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size)) + *pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */ + + file_offset = posix_file_offset + + (off_t)((((size_t)nbytes_xfer / (buf_size / blk_size) * + (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * blk_size)) * + (buf_size * (size_t)snbytes) + + (((size_t)nbytes_xfer / (buf_size / blk_size)) * + (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * blk_size)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * blk_size; + + /* Global offset advance after each I/O operation */ + /* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */ + file_offset_advance = (off_t)((size_t)snbytes * blk_size); + } /* end else */ + } /* end else */ + + /* Common code for file access */ + + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size * blk_size; + + /* Loop over portions of the buffer to read */ + while (nbytes_toxfer > 0) { + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + + /* check if all bytes are read */ + rc = ((ssize_t)nbytes_xfer_advance == + POSIXREAD(fd->posixfd, buf_p, nbytes_xfer_advance)); + VRFY((rc != 0), "POSIXREAD"); + + /* Advance location in buffer */ + buf_p += nbytes_xfer_advance; + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)nbytes_xfer_advance; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= nbytes_xfer_advance; + + /* Partially advance file offset */ + file_offset += file_offset_advance; + } /* end while */ + + } /* end else */ + break; + + case MPIO: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Independent file access */ + if (!parms->collective) { + /* Contiguous pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Perform independent read */ + mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buffer, + (int)(buf_size / blk_size), mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size; + + /* Loop over the buffers to read */ + while (nbytes_toxfer > 0) { + /* Skip offset over blocks of other processes */ + mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); + + /* Perform independent read */ + mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p, (int)1, mpi_blk_type, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance location in buffer */ + buf_p += blk_size; + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)blk_size; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= blk_size; + } /* end while */ + } /* end else */ + } /* end if */ + /* Collective file access */ + else { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Perform collective read */ + mrc = MPI_File_read_at_all(fd->mpifd, mpi_offset, buffer, + (int)(buf_size / blk_size), mpi_blk_type, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); + + /* Set the file view */ + mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, + (char *)"native", h5_io_info_g); + VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); + + /* Perform collective read */ + mrc = MPI_File_read_at_all(fd->mpifd, 0, buffer, (int)(buf_size / blk_size), + mpi_blk_type, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size; + } /* end else */ + } /* end else */ + } /* end if */ + /* 2D dataspace */ + else { + /* Contiguous storage */ + if (!parms->h5_use_chunks) { + /* Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = + mpi_file_offset + + (MPI_Offset)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * + (blk_size * (size_t)snbytes)) + + (MPI_Offset)(((size_t)nbytes_xfer / blk_size) % (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = buf_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = snbytes; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_cont_type; + } /* end if */ + /* Interleaved access pattern */ + else { + /* Compute offset in file */ + mpi_offset = + mpi_file_offset + + (MPI_Offset)( + ((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / + (size_t)snbytes) * + (buf_size * (size_t)snbytes)) + + (MPI_Offset)( + (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % + (size_t)snbytes); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = snbytes; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_inter_type; + } /* end else */ + } /* end if */ + /* Chunked storage */ + else { + /*Contiguous access pattern */ + if (!parms->interleaved) { + /* Compute offset in file */ + mpi_offset = mpi_file_offset + nbytes_xfer; + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * buf_size; + + /* Global offset advance after each I/O operation */ + mpi_offset_advance = 0; + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_full_buffer; + } /* end if */ + /*Interleaved access pattern */ + else { + /* Compute offset in file */ + /* Before simplification */ + /* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size) + *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))* + (buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+ + ((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes + /blk_size*(blk_size*blk_size)); */ + mpi_offset = mpi_file_offset + + (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size) * + (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * blk_size)) * + (buf_size * (size_t)snbytes)) + + (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size)) * + (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * blk_size)); + + /* Number of bytes to be transferred per I/O operation */ + nbytes_xfer_advance = blk_size * blk_size; + + /* Global offset advance after each I/O operation */ + /* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */ + mpi_offset_advance = (MPI_Offset)((size_t)snbytes * blk_size); + + /* MPI type to be used for collective access */ + mpi_collective_type = mpi_chunk_inter_type; + } /* end else */ + } /* end else */ + + /* Common code for independent file access */ + if (!parms->collective) { + /* Set the base of user's buffer */ + buf_p = (unsigned char *)buffer; + + /* Set the number of bytes to transfer this time */ + nbytes_toxfer = buf_size * blk_size; + + /* Loop over portions of the buffer to read */ + while (nbytes_toxfer > 0) { + /* Perform independent read */ + mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p, (int)nbytes_xfer_advance, + MPI_BYTE, &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance location in buffer */ + buf_p += nbytes_xfer_advance; + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)nbytes_xfer_advance; + + /* Decrement number of bytes left this time */ + nbytes_toxfer -= nbytes_xfer_advance; + + /* Partially advance global offset in dataset */ + mpi_offset += mpi_offset_advance; + } /* end while */ + } /* end if */ + + /* Common code for collective file access */ + else { + /* Set the file view */ + mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, + (char *)"native", h5_io_info_g); + VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); + + /* Perform read */ + MPI_File_read_at_all(fd->mpifd, 0, buffer, (int)(buf_size * blk_size), MPI_BYTE, + &mpi_status); + VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); + + /* Advance global offset in dataset */ + nbytes_xfer += (off_t)buf_size * (off_t)blk_size; + } /* end else */ + + } /* end else */ + break; + + case PHDF5: + /* 1D dataspace */ + if (!parms->dim2d) { + /* Set up the file dset space id to move the selection to process */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5offset[0] = nbytes_xfer; + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5offset[0] = (nbytes_xfer * pio_mpi_nprocs_g); + } /* end else */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Read the buffer in */ + hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dread"); + + /* Increment number of bytes transferred */ + nbytes_xfer += (off_t)buf_size; + } /* end if */ + /* 2D dataspace */ + else { + /* Set up the file dset space id to move the selection to process */ + if (!parms->interleaved) { + /* Contiguous pattern */ + h5offset[0] = + (hssize_t)(((size_t)nbytes_xfer / ((size_t)snbytes * blk_size)) * blk_size); + h5offset[1] = + (hssize_t)(((size_t)nbytes_xfer % ((size_t)snbytes * blk_size)) / blk_size); + } /* end if */ + else { + /* Interleaved access pattern */ + /* Skip offset over blocks of other processes */ + h5offset[0] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) / + ((size_t)snbytes * buf_size)) * + buf_size); + h5offset[1] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) % + ((size_t)snbytes * buf_size)) / + buf_size); + + } /* end else */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Write the buffer out */ + hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dread"); + + /* Increment number of bytes transferred */ + nbytes_xfer += (off_t)buf_size * (off_t)blk_size; + + } /* end else */ + break; + + default: + break; + } /* switch (parms->io_type) */ + + /* Verify raw data, if asked */ + if (parms->verify) { + /* Verify data read */ + unsigned char *ucharptr = (unsigned char *)buffer; + size_t i; + int nerror = 0; + + for (i = 0; i < bsize; ++i) { + if (*ucharptr++ != pio_mpi_rank_g + 1) { + if (++nerror < 20) { + /* report at most 20 errors */ + HDprint_rank(output); + HDfprintf(output, + "read data error, expected (%d), " + "got (%d)\n", + pio_mpi_rank_g + 1, (int)*(ucharptr - 1)); + } /* end if */ + } /* end if */ + } /* end for */ + if (nerror >= 20) { + HDprint_rank(output); + HDfprintf(output, "..."); + HDfprintf(output, "total read data errors=%d\n", nerror); + } /* end if */ + } /* if (parms->verify) */ + + } /* end while */ + + /* Stop "raw data" read timer */ + io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTOP); + + /* Calculate read time */ + + /* Close dataset. Only HDF5 needs to do an explicit close. */ + if (parms->io_type == PHDF5) { + hrc = H5Dclose(h5ds_id); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Close failed\n"); + GOTOERROR(FAIL); + } + + h5ds_id = H5I_INVALID_HID; + } /* end if */ + } /* end for */ + +done: + /* release MPI-I/O objects */ + if (parms->io_type == MPIO) { + /* 1D dataspace */ + if (!parms->dim2d) { + /* Free file type */ + mrc = MPI_Type_free(&mpi_file_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free buffer type */ + mrc = MPI_Type_free(&mpi_blk_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + } /* end if */ + /* 2D dataspace */ + else { + /* Free partial buffer type for contiguous access */ + mrc = MPI_Type_free(&mpi_partial_buffer_cont); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free contiguous file type */ + mrc = MPI_Type_free(&mpi_cont_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free partial buffer type for interleaved access */ + mrc = MPI_Type_free(&mpi_partial_buffer_inter); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free interleaved file type */ + mrc = MPI_Type_free(&mpi_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free full buffer type */ + mrc = MPI_Type_free(&mpi_full_buffer); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free full chunk type */ + mrc = MPI_Type_free(&mpi_full_chunk); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + + /* Free chunk interleaved file type */ + mrc = MPI_Type_free(&mpi_chunk_inter_type); + VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); + } /* end else */ + } /* end if */ + + /* release HDF5 objects */ + if (h5dset_space_id != -1) { + hrc = H5Sclose(h5dset_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); + ret_code = FAIL; + } + else { + h5dset_space_id = H5I_INVALID_HID; + } + } + + if (h5mem_space_id != -1) { + hrc = H5Sclose(h5mem_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); + ret_code = FAIL; + } + else { + h5mem_space_id = H5I_INVALID_HID; + } + } + + if (h5dxpl != -1) { + hrc = H5Pclose(h5dxpl); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); + ret_code = FAIL; + } + else { + h5dxpl = H5I_INVALID_HID; + } + } + + return ret_code; +} + +/* + * Function: do_fopen + * Purpose: Open the specified file. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: + */ +static herr_t +do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) +{ + int ret_code = SUCCESS, mrc; + hid_t acc_tpl = H5I_INVALID_HID; /* file access templates */ + + switch (param->io_type) { + case POSIXIO: + if (flags & (PIO_CREATE | PIO_WRITE)) + fd->posixfd = POSIXCREATE(fname); + else + fd->posixfd = POSIXOPEN(fname, O_RDONLY); + + if (fd->posixfd < 0) { + HDfprintf(stderr, "POSIX File Open failed(%s)\n", fname); + GOTOERROR(FAIL); + } + + /* The perils of POSIX I/O in a parallel environment. The problem is: + * + * - Process n opens a file with truncation and then starts + * writing to the file. + * - Process m also opens the file with truncation, but after + * process n has already started to write to the file. Thus, + * all of the stuff process n wrote is now lost. + */ + MPI_Barrier(pio_comm_g); + + break; + + case MPIO: + if (flags & (PIO_CREATE | PIO_WRITE)) { + MPI_File_delete(fname, h5_io_info_g); + mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_CREATE | MPI_MODE_RDWR, h5_io_info_g, + &fd->mpifd); + + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI File Open failed(%s)\n", fname); + GOTOERROR(FAIL); + } + + /*since MPI_File_open with MPI_MODE_CREATE does not truncate */ + /*filesize , set size to 0 explicitedly. */ + mrc = MPI_File_set_size(fd->mpifd, (MPI_Offset)0); + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI_File_set_size failed\n"); + GOTOERROR(FAIL); + } + } + else { + mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_RDONLY, h5_io_info_g, &fd->mpifd); + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI File Open failed(%s)\n", fname); + GOTOERROR(FAIL); + } + } + + break; + + case PHDF5: + if ((acc_tpl = H5Pcreate(H5P_FILE_ACCESS)) < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + /* Set the file driver to the MPI-IO driver */ + if (H5Pset_fapl_mpio(acc_tpl, pio_comm_g, h5_io_info_g) < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } + + /* Set the alignment of objects in HDF5 file */ + if (H5Pset_alignment(acc_tpl, param->h5_thresh, param->h5_align) < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } + + /* create the parallel file */ + if (flags & (PIO_CREATE | PIO_WRITE)) + fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); + else + fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, acc_tpl); + if (fd->h5fd < 0) { + HDfprintf(stderr, "HDF5 File Create failed(%s)\n", fname); + GOTOERROR(FAIL); + } + + /* verifying the close of the acc_tpl */ + if (H5Pclose(acc_tpl) < 0) { + HDfprintf(stderr, "HDF5 Property List Close failed\n"); + GOTOERROR(FAIL); + } + + break; + + default: + break; + } + +done: + return ret_code; +} + +/* + * Function: do_fclose + * Purpose: Close the specified file descriptor. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: + */ +static herr_t +do_fclose(iotype iot, file_descr *fd /*out*/) +{ + herr_t ret_code = SUCCESS, hrc; + int mrc = 0, rc = 0; + + switch (iot) { + case POSIXIO: + rc = POSIXCLOSE(fd->posixfd); + + if (rc != 0) { + HDfprintf(stderr, "POSIX File Close failed\n"); + GOTOERROR(FAIL); + } + + fd->posixfd = -1; + break; + + case MPIO: + mrc = MPI_File_close(&fd->mpifd); + + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI File close failed\n"); + GOTOERROR(FAIL); + } + + fd->mpifd = MPI_FILE_NULL; + break; + + case PHDF5: + hrc = H5Fclose(fd->h5fd); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 File Close failed\n"); + GOTOERROR(FAIL); + } + + fd->h5fd = -1; + break; + + default: + break; + } + +done: + return ret_code; +} + +/* + * Function: do_fclose + * Purpose: Cleanup temporary file unless HDF5_NOCLEANUP is set. + * Only Proc 0 of the PIO communicator will do the cleanup. + * Other processes just return. + * Return: void + * Programmer: Albert Cheng 2001/12/12 + * Modifications: + */ +static void +do_cleanupfile(iotype iot, char *fname) +{ + if (pio_mpi_rank_g != 0) + return; + + if (clean_file_g == -1) + clean_file_g = (getenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; + + if (clean_file_g) { + switch (iot) { + case POSIXIO: + HDremove(fname); + break; + case MPIO: + case PHDF5: + MPI_File_delete(fname, h5_io_info_g); + break; + default: + break; + } + } +} + +#ifdef TIME_MPI +/* instrument the MPI_File_wrirte_xxx and read_xxx calls to measure + * pure time spent in MPI_File code. + */ +int +MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, + MPI_Status *status) +{ + int err; + io_time_set(timer_g, HDF5_MPI_READ, TSTART); + err = PMPI_File_read_at(fh, offset, buf, count, datatype, status); + io_time_set(timer_g, HDF5_MPI_READ, TSTOP); + return err; +} + +int +MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, + MPI_Status *status) +{ + int err; + io_time_set(timer_g, HDF5_MPI_READ, TSTART); + err = PMPI_File_read_at_all(fh, offset, buf, count, datatype, status); + io_time_set(timer_g, HDF5_MPI_READ, TSTOP); + return err; +} + +int +MPI_File_write_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, + MPI_Status *status) +{ + int err; + io_time_set(timer_g, HDF5_MPI_WRITE, TSTART); + err = PMPI_File_write_at(fh, offset, buf, count, datatype, status); + io_time_set(timer_g, HDF5_MPI_WRITE, TSTOP); + return err; +} + +int +MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, + MPI_Status *status) +{ + int err; + io_time_set(timer_g, HDF5_MPI_WRITE, TSTART); + err = PMPI_File_write_at_all(fh, offset, buf, count, datatype, status); + io_time_set(timer_g, HDF5_MPI_WRITE, TSTOP); + return err; +} + +#endif /* TIME_MPI */ +#endif /* H5_HAVE_PARALLEL */ diff --git a/tools/src/h5perf/pio_perf.c b/tools/src/h5perf/pio_perf.c new file mode 100644 index 0000000..41245d9 --- /dev/null +++ b/tools/src/h5perf/pio_perf.c @@ -0,0 +1,1718 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Parallel HDF5 Performance Testing Code + * -------------------------------------- + * + * Portable code to test performance on the different platforms we support. + * This is what the report should look like: + * + * nprocs = Max#Procs + * IO API = POSIXIO + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * IO API = MPIO + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * IO API = PHDF5 + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * nprocs = Max#Procs / 2 + * + * . . . + * + */ + +/* system header files */ +#include +#include +#include + +#include "hdf5.h" + +#ifdef H5_HAVE_PARALLEL + +/* library header files */ +#include + +/* our header files */ +#include "pio_perf.h" + +/* useful macros */ +#define TAB_SPACE 4 + +#define ONE_KB 1024 +#define ONE_MB (ONE_KB * ONE_KB) +#define ONE_GB (ONE_MB * ONE_KB) + +#define PIO_POSIX 0x1 +#define PIO_MPI 0x2 +#define PIO_HDF5 0x4 + +#ifdef STANDALONE +#define DBL_EPSILON 2.2204460492503131e-16 +#define H5_DBL_ABS_EQUAL(X, Y) (fabs((X) - (Y)) < DBL_EPSILON) +#endif + +/* report 0.0 in case t is zero too */ +#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL((t), 0.0) ? 0.0 : ((((double)bytes) / ONE_MB) / (t))) + +#ifndef TRUE +#define TRUE 1 +#endif /* TRUE */ +#ifndef FALSE +#define FALSE (!TRUE) +#endif /* FALSE */ + +/* global variables */ +FILE * output; /* output file */ +int comm_world_rank_g; /* my rank in MPI_COMM_RANK */ +int comm_world_nprocs_g; /* num. of processes of MPI_COMM_WORLD */ +MPI_Comm pio_comm_g; /* Communicator to run the PIO */ +int pio_mpi_rank_g; /* MPI rank of pio_comm_g */ +int pio_mpi_nprocs_g; /* Number of processes of pio_comm_g */ +int pio_debug_level = 0; /* The debug level: + * 0 - Off + * 1 - Minimal + * 2 - Some more + * 3 - Maximal + * 4 - Maximal & then some + */ + +/* local variables */ +static const char *progname = "h5perf"; + +#ifndef HDF5_PARAPREFIX +#define HDF5_PARAPREFIX "" +#endif +char * paraprefix = NULL; /* for command line option para-prefix */ +MPI_Info h5_io_info_g = MPI_INFO_NULL; /* MPI INFO object for IO */ + +/* + * Command-line options: The user can specify short or long-named + * parameters. The long-named ones can be partially spelled. When + * adding more, make sure that they don't clash with each other. + */ +#if 1 +static const char *s_opts = "a:A:B:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; +#else +static const char *s_opts = "a:A:bB:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; +#endif /* 1 */ +static struct h5_long_options l_opts[] = {{"align", require_arg, 'a'}, + {"api", require_arg, 'A'}, +#if 0 + /* a sighting of the elusive binary option */ + { "binary", no_arg, 'b' }, +#endif /* 0 */ + {"block-size", require_arg, 'B'}, + {"chunk", no_arg, 'c'}, + {"collective", no_arg, 'C'}, + {"debug", require_arg, 'D'}, + {"geometry", no_arg, 'g'}, + {"help", no_arg, 'h'}, + {"interleaved", require_arg, 'I'}, + {"max-num-processes", require_arg, 'P'}, + {"min-num-processes", require_arg, 'p'}, + {"max-xfer-size", require_arg, 'X'}, + {"min-xfer-size", require_arg, 'x'}, + {"num-bytes", require_arg, 'e'}, + {"num-dsets", require_arg, 'd'}, + {"num-files", require_arg, 'F'}, + {"num-iterations", require_arg, 'i'}, + {"output", require_arg, 'o'}, + {"threshold", require_arg, 'T'}, + {"write-only", require_arg, 'w'}, + {NULL, 0, '\0'}}; + +struct options { + long io_types; /* bitmask of which I/O types to test */ + const char *output_file; /* file to print report to */ + long num_dsets; /* number of datasets */ + long num_files; /* number of files */ + off_t num_bpp; /* number of bytes per proc per dset */ + int num_iters; /* number of iterations */ + int max_num_procs; /* maximum number of processes to use */ + int min_num_procs; /* minimum number of processes to use */ + size_t max_xfer_size; /* maximum transfer buffer size */ + size_t min_xfer_size; /* minimum transfer buffer size */ + size_t blk_size; /* Block size */ + unsigned interleaved; /* Interleaved vs. contiguous blocks */ + unsigned collective; /* Collective vs. independent I/O */ + unsigned dim2d; /* 1D vs. 2D geometry */ + int print_times; /* print times as well as throughputs */ + int print_raw; /* print raw data throughput info */ + off_t h5_alignment; /* alignment in HDF5 file */ + off_t h5_threshold; /* threshold for alignment in HDF5 file */ + int h5_use_chunks; /* Make HDF5 dataset chunked */ + int h5_write_only; /* Perform the write tests only */ + int verify; /* Verify data correctness */ +}; + +typedef struct _minmax { + double min; + double max; + double sum; + int num; +} minmax; + +/* local functions */ +static off_t parse_size_directive(const char *size); +static struct options *parse_command_line(int argc, char *argv[]); +static void run_test_loop(struct options *options); +static int run_test(iotype iot, parameters parms, struct options *opts); +static void output_all_info(minmax *mm, int count, int indent_level); +static void get_minmax(minmax *mm, double val); +static minmax accumulate_minmax_stuff(minmax *mm, int count); +static int create_comm_world(int num_procs, int *doing_pio); +static int destroy_comm_world(void); +static void output_results(const struct options *options, const char *name, minmax *table, int table_size, + off_t data_size); +static void output_times(const struct options *options, const char *name, minmax *table, int table_size); +static void output_report(const char *fmt, ...); +static void print_indent(register int indent); +static void usage(const char *prog); +static void report_parameters(struct options *opts); +static off_t squareo(off_t); + +/* + * Function: main + * Purpose: Start things up. Initialize MPI and then call the test looping + * function. + * Return: EXIT_SUCCESS or EXIT_FAILURE + * Programmer: Bill Wendling, 30. October 2001 + * Modifications: + */ +int +main(int argc, char *argv[]) +{ + int ret; + int exit_value = EXIT_SUCCESS; + struct options *opts = NULL; + +#ifndef STANDALONE + /* Initialize h5tools lib */ + h5tools_init(); +#endif + + output = stdout; + + /* initialize MPI and get the maximum num of processors we started with */ + MPI_Init(&argc, &argv); + ret = MPI_Comm_size(MPI_COMM_WORLD, &comm_world_nprocs_g); + + if (ret != MPI_SUCCESS) { + HDfprintf(stderr, "%s: MPI_Comm_size call failed\n", progname); + + if (ret == MPI_ERR_COMM) + HDfprintf(stderr, "invalid MPI communicator\n"); + else + HDfprintf(stderr, "invalid argument\n"); + + exit_value = EXIT_FAILURE; + goto finish; + } + + ret = MPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank_g); + + if (ret != MPI_SUCCESS) { + HDfprintf(stderr, "%s: MPI_Comm_rank call failed\n", progname); + + if (ret == MPI_ERR_COMM) + HDfprintf(stderr, "invalid MPI communicator\n"); + else + HDfprintf(stderr, "invalid argument\n"); + + exit_value = EXIT_FAILURE; + goto finish; + } + + pio_comm_g = MPI_COMM_WORLD; + + h5_set_info_object(); + opts = parse_command_line(argc, argv); + + if (!opts) { + exit_value = EXIT_FAILURE; + goto finish; + } + + if (opts->output_file) { + if ((output = HDfopen(opts->output_file, "w")) == NULL) { + HDfprintf(stderr, "%s: cannot open output file\n", progname); + perror(opts->output_file); + goto finish; + } + } + + if ((pio_debug_level == 0 && comm_world_rank_g == 0) || pio_debug_level > 0) + report_parameters(opts); + + run_test_loop(opts); + +finish: + MPI_Finalize(); + free(opts); + return exit_value; +} + +off_t +squareo(off_t x) +{ + return x * x; +} + +/* + * Function: run_test_loop + * Purpose: Run the I/O tests. Write the results to OUTPUT. + * + * - The slowest changing part of the test is the number of + * processors to use. For each loop iteration, we divide that + * number by 2 and rerun the test. + * + * - The second slowest is what type of IO API to perform. We have + * three choices: POSIXIO, MPI-IO, and PHDF5. + * + * - Then we change the size of the buffer. This information is + * inferred from the number of datasets to create and the number + * of integers to put into each dataset. The backend code figures + * this out. + * + * Return: Nothing + * Programmer: Bill Wendling, 30. October 2001 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static void +run_test_loop(struct options *opts) +{ + parameters parms; + int num_procs; + int doing_pio; /* if this process is doing PIO */ + + parms.num_files = opts->num_files; + parms.num_dsets = opts->num_dsets; + parms.num_iters = opts->num_iters; + parms.blk_size = opts->blk_size; + parms.interleaved = opts->interleaved; + parms.collective = opts->collective; + parms.dim2d = opts->dim2d; + parms.h5_align = (hsize_t)opts->h5_alignment; + parms.h5_thresh = (hsize_t)opts->h5_threshold; + parms.h5_use_chunks = opts->h5_use_chunks; + parms.h5_write_only = opts->h5_write_only; + parms.verify = opts->verify; + + /* start with max_num_procs and decrement it by half for each loop. */ + /* if performance needs restart, fewer processes may be needed. */ + for (num_procs = opts->max_num_procs; num_procs >= opts->min_num_procs; num_procs >>= 1) { + register size_t buf_size; + + parms.num_procs = num_procs; + + if (create_comm_world(parms.num_procs, &doing_pio) != SUCCESS) { + /* do something harsh */ + } + + /* only processes doing PIO will run the tests */ + if (doing_pio) { + output_report("Number of processors = %ld\n", parms.num_procs); + + /* multiply the xfer buffer size by 2 for each loop iteration */ + for (buf_size = opts->min_xfer_size; buf_size <= opts->max_xfer_size; buf_size <<= 1) { + parms.buf_size = buf_size; + + if (parms.dim2d) { + parms.num_bytes = squareo(opts->num_bpp * parms.num_procs); + if (parms.interleaved) + output_report("Transfer Buffer Size: %ldx%ld bytes, File size: %.2f MB\n", buf_size, + opts->blk_size, + ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); + else + output_report("Transfer Buffer Size: %ldx%ld bytes, File size: %.2f MB\n", + opts->blk_size, buf_size, + ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); + + print_indent(1); + output_report(" # of files: %ld, # of datasets: %ld, dataset size: %.2fx%.2f KB\n", + parms.num_files, parms.num_dsets, + (double)(opts->num_bpp * parms.num_procs) / ONE_KB, + (double)(opts->num_bpp * parms.num_procs) / ONE_KB); + } + else { + parms.num_bytes = (off_t)opts->num_bpp * parms.num_procs; + output_report("Transfer Buffer Size: %ld bytes, File size: %.2f MB\n", buf_size, + ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); + + print_indent(1); + output_report(" # of files: %ld, # of datasets: %ld, dataset size: %.2f MB\n", + parms.num_files, parms.num_dsets, + (double)(opts->num_bpp * parms.num_procs) / ONE_MB); + } + + if (opts->io_types & PIO_POSIX) + run_test(POSIXIO, parms, opts); + + if (opts->io_types & PIO_MPI) + run_test(MPIO, parms, opts); + + if (opts->io_types & PIO_HDF5) + run_test(PHDF5, parms, opts); + + /* Run the tests once if buf_size==0, but then break out */ + if (buf_size == 0) + break; + } + + if (destroy_comm_world() != SUCCESS) { + /* do something harsh */ + } + } + } +} + +/* + * Function: run_test + * Purpose: Inner loop call to actually run the I/O test. + * Return: Nothing + * Programmer: Bill Wendling, 18. December 2001 + * Modifications: + */ +static int +run_test(iotype iot, parameters parms, struct options *opts) +{ + results res; + register int i, ret_value = SUCCESS; + int comm_size; + off_t raw_size; + minmax * write_mpi_mm_table = NULL; + minmax * write_mm_table = NULL; + minmax * write_gross_mm_table = NULL; + minmax * write_raw_mm_table = NULL; + minmax * read_mpi_mm_table = NULL; + minmax * read_mm_table = NULL; + minmax * read_gross_mm_table = NULL; + minmax * read_raw_mm_table = NULL; + minmax * read_open_mm_table = NULL; + minmax * read_close_mm_table = NULL; + minmax * write_open_mm_table = NULL; + minmax * write_close_mm_table = NULL; + minmax write_mpi_mm = {0.0, 0.0, 0.0, 0}; + minmax write_mm = {0.0, 0.0, 0.0, 0}; + minmax write_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax write_raw_mm = {0.0, 0.0, 0.0, 0}; + minmax read_mpi_mm = {0.0, 0.0, 0.0, 0}; + minmax read_mm = {0.0, 0.0, 0.0, 0}; + minmax read_gross_mm = {0.0, 0.0, 0.0, 0}; + minmax read_raw_mm = {0.0, 0.0, 0.0, 0}; + minmax read_open_mm = {0.0, 0.0, 0.0, 0}; + minmax read_close_mm = {0.0, 0.0, 0.0, 0}; + minmax write_open_mm = {0.0, 0.0, 0.0, 0}; + minmax write_close_mm = {0.0, 0.0, 0.0, 0}; + + raw_size = parms.num_files * (off_t)parms.num_dsets * (off_t)parms.num_bytes; + parms.io_type = iot; + print_indent(2); + output_report("IO API = "); + + switch (iot) { + case POSIXIO: + output_report("POSIX\n"); + break; + case MPIO: + output_report("MPIO\n"); + break; + case PHDF5: + output_report("PHDF5 (w/MPI-IO driver)\n"); + break; + default: + break; + } + + MPI_Comm_size(pio_comm_g, &comm_size); + + /* allocate space for tables minmax and that it is sufficient */ + /* to initialize all elements to zeros by calloc. */ + write_mpi_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_gross_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_raw_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_open_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + write_close_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + if (!parms.h5_write_only) { + read_mpi_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_gross_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_raw_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_open_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + read_close_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); + } + + /* Do IO iteration times, collecting statistics each time */ + for (i = 0; i < parms.num_iters; ++i) { + double t; + + MPI_Barrier(pio_comm_g); + res = do_pio(parms); + + /* gather all of the "mpi write" times */ + t = io_time_get(res.timers, HDF5_MPI_WRITE); + get_minmax(&write_mpi_mm, t); + + write_mpi_mm_table[i] = write_mpi_mm; + + /* gather all of the "write" times */ + t = io_time_get(res.timers, HDF5_FINE_WRITE_FIXED_DIMS); + get_minmax(&write_mm, t); + + write_mm_table[i] = write_mm; + + /* gather all of the "write" times from open to close */ + t = io_time_get(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS); + get_minmax(&write_gross_mm, t); + + write_gross_mm_table[i] = write_gross_mm; + + /* gather all of the raw "write" times */ + t = io_time_get(res.timers, HDF5_RAW_WRITE_FIXED_DIMS); + get_minmax(&write_raw_mm, t); + + write_raw_mm_table[i] = write_raw_mm; + + /* gather all of the file open times (time from open to first write) */ + t = io_time_get(res.timers, HDF5_FILE_WRITE_OPEN); + get_minmax(&write_open_mm, t); + + write_open_mm_table[i] = write_open_mm; + + /* gather all of the file close times (time from last write to close) */ + t = io_time_get(res.timers, HDF5_FILE_WRITE_CLOSE); + get_minmax(&write_close_mm, t); + + write_close_mm_table[i] = write_close_mm; + + if (!parms.h5_write_only) { + /* gather all of the "mpi read" times */ + t = io_time_get(res.timers, HDF5_MPI_READ); + get_minmax(&read_mpi_mm, t); + + read_mpi_mm_table[i] = read_mpi_mm; + + /* gather all of the "read" times */ + t = io_time_get(res.timers, HDF5_FINE_READ_FIXED_DIMS); + get_minmax(&read_mm, t); + + read_mm_table[i] = read_mm; + + /* gather all of the "read" times from open to close */ + t = io_time_get(res.timers, HDF5_GROSS_READ_FIXED_DIMS); + get_minmax(&read_gross_mm, t); + + read_gross_mm_table[i] = read_gross_mm; + + /* gather all of the raw "read" times */ + t = io_time_get(res.timers, HDF5_RAW_READ_FIXED_DIMS); + get_minmax(&read_raw_mm, t); + + read_raw_mm_table[i] = read_raw_mm; + + /* gather all of the file open times (time from open to first read) */ + t = io_time_get(res.timers, HDF5_FILE_READ_OPEN); + get_minmax(&read_open_mm, t); + + read_open_mm_table[i] = read_open_mm; + + /* gather all of the file close times (time from last read to close) */ + t = io_time_get(res.timers, HDF5_FILE_READ_CLOSE); + get_minmax(&read_close_mm, t); + + read_close_mm_table[i] = read_close_mm; + } + + io_time_destroy(res.timers); + } + + /* + * Show various statistics + */ + /* Write statistics */ + /* Print the raw data throughput if desired */ + if (opts->print_raw) { + /* accumulate and output the max, min, and average "raw write" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Raw Data Write details:\n"); + output_all_info(write_raw_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); + } /* end if */ + + /* show mpi write statics */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("MPI Write details:\n"); + output_all_info(write_mpi_mm_table, parms.num_iters, 4); + } + + /* We don't currently output the MPI write results */ + + /* accumulate and output the max, min, and average "write" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write details:\n"); + output_all_info(write_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Write", write_mm_table, parms.num_iters, raw_size); + + /* accumulate and output the max, min, and average "gross write" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write Open-Close details:\n"); + output_all_info(write_gross_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Write Open-Close", write_gross_mm_table, parms.num_iters, raw_size); + + if (opts->print_times) { + output_times(opts, "Write File Open", write_open_mm_table, parms.num_iters); + output_times(opts, "Write File Close", write_close_mm_table, parms.num_iters); + } + + /* Print out time from open to first write */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write file open details:\n"); + output_all_info(write_open_mm_table, parms.num_iters, 4); + } + + /* Print out time from last write to close */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write file close details:\n"); + output_all_info(write_close_mm_table, parms.num_iters, 4); + } + + if (!parms.h5_write_only) { + /* Read statistics */ + /* Print the raw data throughput if desired */ + if (opts->print_raw) { + /* accumulate and output the max, min, and average "raw read" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Raw Data Read details:\n"); + output_all_info(read_raw_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); + } /* end if */ + + /* show mpi read statics */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("MPI Read details:\n"); + output_all_info(read_mpi_mm_table, parms.num_iters, 4); + } + + /* We don't currently output the MPI read results */ + + /* accumulate and output the max, min, and average "read" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read details:\n"); + output_all_info(read_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Read", read_mm_table, parms.num_iters, raw_size); + + /* accumulate and output the max, min, and average "gross read" times */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read Open-Close details:\n"); + output_all_info(read_gross_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Read Open-Close", read_gross_mm_table, parms.num_iters, raw_size); + + if (opts->print_times) { + output_times(opts, "Read File Open", read_open_mm_table, parms.num_iters); + output_times(opts, "Read File Close", read_close_mm_table, parms.num_iters); + } + + /* Print out time from open to first read */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read file open details:\n"); + output_all_info(read_open_mm_table, parms.num_iters, 4); + } + + /* Print out time from last read to close */ + if (pio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read file close details:\n"); + output_all_info(read_close_mm_table, parms.num_iters, 4); + } + } + + /* clean up our mess */ + free(write_mpi_mm_table); + free(write_mm_table); + free(write_gross_mm_table); + free(write_raw_mm_table); + free(write_open_mm_table); + free(write_close_mm_table); + + if (!parms.h5_write_only) { + free(read_mpi_mm_table); + free(read_mm_table); + free(read_gross_mm_table); + free(read_raw_mm_table); + free(read_open_mm_table); + free(read_close_mm_table); + } + + return ret_value; +} + +/* + * Function: output_all_info + * Purpose: + * Return: Nothing + * Programmer: Bill Wendling, 29. January 2002 + * Modifications: + */ +static void +output_all_info(minmax *mm, int count, int indent_level) +{ + int i; + + for (i = 0; i < count; ++i) { + print_indent(indent_level); + output_report("Iteration %d:\n", i + 1); + print_indent(indent_level + 1); + output_report("Minimum Time: %.2fs\n", mm[i].min); + print_indent(indent_level + 1); + output_report("Maximum Time: %.2fs\n", mm[i].max); + } +} + +/* + * Function: h5_set_info_object + * Purpose: Process environment variables setting to set up MPI Info + * object. + * Return: 0 if all is fine; otherwise non-zero. + * Programmer: Albert Cheng, 2002/05/21. + * Modifications: + * Bill Wendling, 2002/05/31 + * Modified so that the HDF5_MPI_INFO environment variable can + * be a semicolon separated list of "key=value" pairings. Most + * of the code is to remove any whitespaces which might be + * surrounding the "key=value" pairs. + */ +int +h5_set_info_object(void) +{ + char *envp; /* environment pointer */ + int ret_value = 0; + + /* handle any MPI INFO hints via $HDF5_MPI_INFO */ + if ((envp = HDgetenv("HDF5_MPI_INFO")) != NULL) { + char *next, *valp; + + valp = envp = next = HDstrdup(envp); + + if (!valp) + return 0; + + /* create an INFO object if not created yet */ + if (h5_io_info_g == MPI_INFO_NULL) + MPI_Info_create(&h5_io_info_g); + + do { + size_t len; + char * key_val, *endp, *namep; + + if (*valp == ';') + valp++; + + /* copy key/value pair into temporary buffer */ + len = strcspn(valp, ";"); + next = &valp[len]; + key_val = (char *)HDcalloc(1, len + 1); + + /* increment the next pointer past the terminating semicolon */ + if (*next == ';') + ++next; + + namep = HDstrncpy(key_val, valp, len); + + /* pass up any beginning whitespaces */ + while (*namep && (*namep == ' ' || *namep == '\t')) + namep++; + + if (!*namep) + continue; /* was all white space, so move to next k/v pair */ + + /* eat up any ending white spaces */ + endp = &namep[HDstrlen(namep) - 1]; + + while (endp && (*endp == ' ' || *endp == '\t')) + *endp-- = '\0'; + + /* find the '=' */ + valp = HDstrchr(namep, '='); + + if (valp != NULL) { /* it's a valid key/value pairing */ + char *tmp_val = valp + 1; + + /* change '=' to \0, move valp down one */ + *valp-- = '\0'; + + /* eat up ending whitespace on the "key" part */ + while (*valp == ' ' || *valp == '\t') + *valp-- = '\0'; + + valp = tmp_val; + + /* eat up beginning whitespace on the "value" part */ + while (*valp == ' ' || *valp == '\t') + *valp++ = '\0'; + + /* actually set the darned thing */ + if (MPI_SUCCESS != MPI_Info_set(h5_io_info_g, namep, valp)) { + HDprintf("MPI_Info_set failed\n"); + ret_value = -1; + } + } + + valp = next; + HDfree(key_val); + } while (next && *next); + + HDfree(envp); + } + + return ret_value; +} + +/* + * Function: h5_dump_info_object + * Purpose: Display content of an MPI Info object + * Return: void + * Programmer: Albert Cheng 2002/05/21 + * Modifications: + */ +void +h5_dump_info_object(MPI_Info info) +{ + char key[MPI_MAX_INFO_KEY + 1]; + char value[MPI_MAX_INFO_VAL + 1]; + int flag; + int i, nkeys; + + HDprintf("Dumping MPI Info Object (up to %d bytes per item):\n", MPI_MAX_INFO_VAL); + if (info == MPI_INFO_NULL) { + HDprintf("object is MPI_INFO_NULL\n"); + } + else { + MPI_Info_get_nkeys(info, &nkeys); + HDprintf("object has %d items\n", nkeys); + for (i = 0; i < nkeys; i++) { + MPI_Info_get_nthkey(info, i, key); + MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); + HDprintf("%s=%s\n", key, value); + } + } +} + +/* + * Function: get_minmax + * Purpose: Gather all the min, max and total of val. + * Return: Nothing + * Programmer: Bill Wendling, 21. December 2001 + * Modifications: + * Use MPI_Allreduce to do it. -akc, 2002/01/11 + */ +static void +get_minmax(minmax *mm, double val) +{ + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + MPI_Comm_size(pio_comm_g, &mm->num); + + MPI_Allreduce(&val, &mm->max, 1, MPI_DOUBLE, MPI_MAX, pio_comm_g); + MPI_Allreduce(&val, &mm->min, 1, MPI_DOUBLE, MPI_MIN, pio_comm_g); + MPI_Allreduce(&val, &mm->sum, 1, MPI_DOUBLE, MPI_SUM, pio_comm_g); +} + +/* + * Function: accumulate_minmax_stuff + * Purpose: Accumulate the minimum, maximum, and average of the times + * across all processes. + * Return: TOTAL_MM - the total of all of these. + * Programmer: Bill Wendling, 21. December 2001 + * Modifications: + * Changed to use seconds instead of MB/s - QAK, 5/9/02 + */ +static minmax +accumulate_minmax_stuff(minmax *mm, int count) +{ + int i; + minmax total_mm; + + total_mm.sum = 0.0f; + total_mm.max = -DBL_MAX; + total_mm.min = DBL_MAX; + total_mm.num = count; + + for (i = 0; i < count; ++i) { + double m = mm[i].max; + + total_mm.sum += m; + + if (m < total_mm.min) + total_mm.min = m; + + if (m > total_mm.max) + total_mm.max = m; + } + + return total_mm; +} + +/* + * Function: create_comm_world + * Purpose: Create an MPI Comm world and store it in pio_comm_g, which + * is a global variable. + * Return: SUCCESS on success. + * FAIL otherwise. + * Programmer: Bill Wendling, 19. December 2001 + * Modifications: + */ +static int +create_comm_world(int num_procs, int *doing_pio) +{ + /* MPI variables */ + int mrc; /* return values */ + int color; /* for communicator creation */ + int myrank, nprocs; + + pio_comm_g = MPI_COMM_NULL; + + /* + * Create a sub communicator for this PIO run. Easier to use the first N + * processes. + */ + MPI_Comm_size(MPI_COMM_WORLD, &nprocs); + + if (num_procs > nprocs) { + HDfprintf(stderr, "number of process(%d) must be <= number of processes in MPI_COMM_WORLD(%d)\n", + num_procs, nprocs); + goto error_done; + } + + MPI_Comm_rank(MPI_COMM_WORLD, &myrank); + color = (myrank < num_procs); + mrc = MPI_Comm_split(MPI_COMM_WORLD, color, myrank, &pio_comm_g); + + if (mrc != MPI_SUCCESS) { + HDfprintf(stderr, "MPI_Comm_split failed\n"); + goto error_done; + } + + if (!color) { + /* not involved in this run */ + mrc = destroy_comm_world(); + goto done; + } + + /* determine the MPI rank in the PIO communicator */ + MPI_Comm_size(pio_comm_g, &pio_mpi_nprocs_g); + MPI_Comm_rank(pio_comm_g, &pio_mpi_rank_g); + +done: + *doing_pio = color; + return SUCCESS; + +error_done: + destroy_comm_world(); + return FAIL; +} + +/* + * Function: destroy_comm_world + * Purpose: Destroy the created MPI Comm world which is stored in the + * pio_comm_g global variable. + * Return: SUCCESS on success. + * FAIL otherwise. + * Programmer: Bill Wendling, 19. December 2001 + * Modifications: + */ +static int +destroy_comm_world(void) +{ + int mrc = SUCCESS; /* return code */ + + /* release MPI resources */ + if (pio_comm_g != MPI_COMM_NULL) + mrc = (MPI_Comm_free(&pio_comm_g) == MPI_SUCCESS ? SUCCESS : FAIL); + + return mrc; +} + +/* + * Function: output_results + * Purpose: Print information about the time & bandwidth for a given + * minmax & # of iterations. + * Return: Nothing + * Programmer: Quincey Koziol, 9. May 2002 + * Modifications: + */ +static void +output_results(const struct options *opts, const char *name, minmax *table, int table_size, off_t data_size) +{ + minmax total_mm; + + total_mm = accumulate_minmax_stuff(table, table_size); + + print_indent(3); + output_report("%s (%d iteration(s)):\n", name, table_size); + + /* Note: The maximum throughput uses the minimum amount of time & vice versa */ + + print_indent(4); + output_report("Maximum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.min)); + if (opts->print_times) + output_report(" (%7.3f s)\n", total_mm.min); + else + output_report("\n"); + + print_indent(4); + output_report("Average Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.sum / total_mm.num)); + if (opts->print_times) + output_report(" (%7.3f s)\n", (total_mm.sum / total_mm.num)); + else + output_report("\n"); + + print_indent(4); + output_report("Minimum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.max)); + if (opts->print_times) + output_report(" (%7.3f s)\n", total_mm.max); + else + output_report("\n"); +} + +static void +output_times(const struct options *opts, const char *name, minmax *table, int table_size) +{ + minmax total_mm; + + total_mm = accumulate_minmax_stuff(table, table_size); + + print_indent(3); + output_report("%s (%d iteration(s)):\n", name, table_size); + + /* Note: The maximum throughput uses the minimum amount of time & vice versa */ + + print_indent(4); + output_report("Minimum Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, (total_mm.min)); + + print_indent(4); + output_report("Average Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, + (total_mm.sum / total_mm.num)); + + print_indent(4); + output_report("Maximum Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, (total_mm.max)); +} + +/* + * Function: output_report + * Purpose: Print a line of the report. Only do so if I'm the 0 process. + * Return: Nothing + * Programmer: Bill Wendling, 19. December 2001 + * Modifications: + */ +static void +output_report(const char *fmt, ...) +{ + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + + if (myrank == 0) { + va_list ap; + + HDva_start(ap, fmt); + HDvfprintf(output, fmt, ap); + HDva_end(ap); + } +} + +/* + * Function: print_indent + * Purpose: Print spaces to indent a new line of text for pretty printing + * things. + * Return: Nothing + * Programmer: Bill Wendling, 29. October 2001 + * Modifications: + */ +static void +print_indent(register int indent) +{ + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + + if (myrank == 0) { + indent *= TAB_SPACE; + + for (; indent > 0; --indent) + HDfputc(' ', output); + } +} + +static void +recover_size_and_print(long long val, const char *end) +{ + if (val >= ONE_KB && (val % ONE_KB) == 0) { + if (val >= ONE_MB && (val % ONE_MB) == 0) { + if (val >= ONE_GB && (val % ONE_GB) == 0) + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "GB%s", + val / ONE_GB, end); + else + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "MB%s", + val / ONE_MB, end); + } + else { + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "KB%s", + val / ONE_KB, end); + } + } + else { + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "%s", + val, end); + } +} + +static void +print_io_api(long io_types) +{ + if (io_types & PIO_POSIX) + HDfprintf(output, "posix "); + if (io_types & PIO_MPI) + HDfprintf(output, "mpiio "); + if (io_types & PIO_HDF5) + HDfprintf(output, "phdf5 "); + HDfprintf(output, "\n"); +} + +static void +report_parameters(struct options *opts) +{ + int rank = comm_world_rank_g; + + print_version("HDF5 Library"); /* print library version */ + HDfprintf(output, "rank %d: ==== Parameters ====\n", rank); + + HDfprintf(output, "rank %d: IO API=", rank); + print_io_api(opts->io_types); + + HDfprintf(output, "rank %d: Number of files=%ld\n", rank, opts->num_files); + HDfprintf(output, "rank %d: Number of datasets=%ld\n", rank, opts->num_dsets); + HDfprintf(output, "rank %d: Number of iterations=%d\n", rank, opts->num_iters); + HDfprintf(output, "rank %d: Number of processes=%d:%d\n", rank, opts->min_num_procs, opts->max_num_procs); + + if (opts->dim2d) { + HDfprintf(output, "rank %d: Number of bytes per process per dataset=", rank); + recover_size_and_print((long long)(opts->num_bpp * opts->num_bpp * opts->min_num_procs), ":"); + recover_size_and_print((long long)(opts->num_bpp * opts->num_bpp * opts->max_num_procs), "\n"); + + HDfprintf(output, "rank %d: Size of dataset(s)=", rank); + recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), "x"); + recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), ":"); + recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "x"); + recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "\n"); + + HDfprintf(output, "rank %d: File size=", rank); + recover_size_and_print((long long)(squareo(opts->num_bpp * opts->min_num_procs) * opts->num_dsets), + ":"); + recover_size_and_print((long long)(squareo(opts->num_bpp * opts->max_num_procs) * opts->num_dsets), + "\n"); + + HDfprintf(output, "rank %d: Transfer buffer size=", rank); + if (opts->interleaved) { + recover_size_and_print((long long)opts->min_xfer_size, "x"); + recover_size_and_print((long long)opts->blk_size, ":"); + recover_size_and_print((long long)opts->max_xfer_size, "x"); + recover_size_and_print((long long)opts->blk_size, "\n"); + } + else { + recover_size_and_print((long long)opts->blk_size, "x"); + recover_size_and_print((long long)opts->min_xfer_size, ":"); + recover_size_and_print((long long)opts->blk_size, "x"); + recover_size_and_print((long long)opts->max_xfer_size, "\n"); + } + HDfprintf(output, "rank %d: Block size=", rank); + recover_size_and_print((long long)opts->blk_size, "x"); + recover_size_and_print((long long)opts->blk_size, "\n"); + } + else { + HDfprintf(output, "rank %d: Number of bytes per process per dataset=", rank); + recover_size_and_print((long long)opts->num_bpp, "\n"); + + HDfprintf(output, "rank %d: Size of dataset(s)=", rank); + recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), ":"); + recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "\n"); + + HDfprintf(output, "rank %d: File size=", rank); + recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs * opts->num_dsets), ":"); + recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs * opts->num_dsets), "\n"); + + HDfprintf(output, "rank %d: Transfer buffer size=", rank); + recover_size_and_print((long long)opts->min_xfer_size, ":"); + recover_size_and_print((long long)opts->max_xfer_size, "\n"); + HDfprintf(output, "rank %d: Block size=", rank); + recover_size_and_print((long long)opts->blk_size, "\n"); + } + + HDfprintf(output, "rank %d: Block Pattern in Dataset=", rank); + if (opts->interleaved) + HDfprintf(output, "Interleaved\n"); + else + HDfprintf(output, "Contiguous\n"); + + HDfprintf(output, "rank %d: I/O Method for MPI and HDF5=", rank); + if (opts->collective) + HDfprintf(output, "Collective\n"); + else + HDfprintf(output, "Independent\n"); + + HDfprintf(output, "rank %d: Geometry=", rank); + if (opts->dim2d) + HDfprintf(output, "2D\n"); + else + HDfprintf(output, "1D\n"); + + HDfprintf(output, "rank %d: VFL used for HDF5 I/O=%s\n", rank, "MPI-IO driver"); + + HDfprintf(output, "rank %d: Data storage method in HDF5=", rank); + if (opts->h5_use_chunks) + HDfprintf(output, "Chunked\n"); + else + HDfprintf(output, "Contiguous\n"); + + { + char *prefix = HDgetenv("HDF5_PARAPREFIX"); + + HDfprintf(output, "rank %d: Env HDF5_PARAPREFIX=%s\n", rank, (prefix ? prefix : "not set")); + } + + HDfprintf(output, "rank %d: ", rank); + h5_dump_info_object(h5_io_info_g); + + HDfprintf(output, "rank %d: ==== End of Parameters ====\n", rank); + HDfprintf(output, "\n"); +} + +/* + * Function: parse_command_line + * Purpose: Parse the command line options and return a STRUCT OPTIONS + * structure which will need to be freed by the calling function. + * Return: Pointer to an OPTIONS structure + * Programmer: Bill Wendling, 31. October 2001 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static struct options * +parse_command_line(int argc, char *argv[]) +{ + register int opt; + struct options *cl_opts; + + cl_opts = (struct options *)malloc(sizeof(struct options)); + + cl_opts->output_file = NULL; + cl_opts->io_types = 0; /* will set default after parsing options */ + cl_opts->num_dsets = 1; + cl_opts->num_files = 1; + cl_opts->num_bpp = 0; + cl_opts->num_iters = 1; + cl_opts->max_num_procs = comm_world_nprocs_g; + cl_opts->min_num_procs = 1; + cl_opts->max_xfer_size = 0; + cl_opts->min_xfer_size = 0; + cl_opts->blk_size = 0; + cl_opts->interleaved = 0; /* Default to contiguous blocks in dataset */ + cl_opts->collective = 0; /* Default to independent I/O access */ + cl_opts->dim2d = 0; /* Default to 1D */ + cl_opts->print_times = FALSE; /* Printing times is off by default */ + cl_opts->print_raw = FALSE; /* Printing raw data throughput is off by default */ + cl_opts->h5_alignment = 1; /* No alignment for HDF5 objects by default */ + cl_opts->h5_threshold = 1; /* No threshold for aligning HDF5 objects by default */ + cl_opts->h5_use_chunks = FALSE; /* Don't chunk the HDF5 dataset by default */ + cl_opts->h5_write_only = FALSE; /* Do both read and write by default */ + cl_opts->verify = FALSE; /* No Verify data correctness by default */ + + while ((opt = H5_get_option(argc, (const char **)argv, s_opts, l_opts)) != EOF) { + switch ((char)opt) { + case 'a': + cl_opts->h5_alignment = parse_size_directive(H5_optarg); + break; + case 'A': { + const char *end = H5_optarg; + + while (end && *end != '\0') { + char buf[10]; + int i; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (isalnum(*end) && i < 10) + buf[i++] = *end; + + if (!HDstrcasecmp(buf, "phdf5")) { + cl_opts->io_types |= PIO_HDF5; + } + else if (!HDstrcasecmp(buf, "mpiio")) { + cl_opts->io_types |= PIO_MPI; + } + else if (!HDstrcasecmp(buf, "posix")) { + cl_opts->io_types |= PIO_POSIX; + } + else { + HDfprintf(stderr, "pio_perf: invalid --api option %s\n", buf); + HDexit(EXIT_FAILURE); + } + + if (*end == '\0') + break; + + end++; + } + } + + break; +#if 0 + case 'b': + /* the future "binary" option */ + break; +#endif /* 0 */ + case 'B': + cl_opts->blk_size = (size_t)parse_size_directive(H5_optarg); + break; + case 'c': + /* Turn on chunked HDF5 dataset creation */ + cl_opts->h5_use_chunks = TRUE; + break; + case 'C': + cl_opts->collective = 1; + break; + case 'd': + cl_opts->num_dsets = atoi(H5_optarg); + break; + case 'D': { + const char *end = H5_optarg; + + while (end && *end != '\0') { + char buf[10]; + int i; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + if (HDstrlen(buf) > 1 || HDisdigit(buf[0])) { + size_t j; + + for (j = 0; j < 10 && buf[j] != '\0'; ++j) + if (!isdigit(buf[j])) { + HDfprintf(stderr, "pio_perf: invalid --debug option %s\n", buf); + HDexit(EXIT_FAILURE); + } + + pio_debug_level = atoi(buf); + + if (pio_debug_level > 4) + pio_debug_level = 4; + else if (pio_debug_level < 0) + pio_debug_level = 0; + } + else { + switch (*buf) { + case 'r': + /* Turn on raw data throughput info */ + cl_opts->print_raw = TRUE; + break; + case 't': + /* Turn on time printing */ + cl_opts->print_times = TRUE; + break; + case 'v': + /* Turn on verify data correctness*/ + cl_opts->verify = TRUE; + break; + default: + HDfprintf(stderr, "pio_perf: invalid --debug option %s\n", buf); + HDexit(EXIT_FAILURE); + } + } + + if (*end == '\0') + break; + + end++; + } + } + + break; + case 'e': + cl_opts->num_bpp = parse_size_directive(H5_optarg); + break; + case 'F': + cl_opts->num_files = HDatoi(H5_optarg); + break; + case 'g': + cl_opts->dim2d = 1; + break; + case 'i': + cl_opts->num_iters = HDatoi(H5_optarg); + break; + case 'I': + cl_opts->interleaved = 1; + break; + case 'o': + cl_opts->output_file = H5_optarg; + break; + case 'p': + cl_opts->min_num_procs = HDatoi(H5_optarg); + break; + case 'P': + cl_opts->max_num_procs = HDatoi(H5_optarg); + break; + case 'T': + cl_opts->h5_threshold = parse_size_directive(H5_optarg); + break; + case 'w': + cl_opts->h5_write_only = TRUE; + break; + case 'x': + cl_opts->min_xfer_size = (size_t)parse_size_directive(H5_optarg); + break; + case 'X': + cl_opts->max_xfer_size = (size_t)parse_size_directive(H5_optarg); + break; + case 'h': + case '?': + default: + usage(progname); + HDfree(cl_opts); + return NULL; + } + } + + if (cl_opts->num_bpp == 0) { + if (cl_opts->dim2d == 0) + cl_opts->num_bpp = 256 * ONE_KB; + else + cl_opts->num_bpp = 8 * ONE_KB; + } + + if (cl_opts->max_xfer_size == 0) + cl_opts->max_xfer_size = (size_t)cl_opts->num_bpp; + + if (cl_opts->min_xfer_size == 0) + cl_opts->min_xfer_size = (size_t)(cl_opts->num_bpp) / 2; + + if (cl_opts->blk_size == 0) + cl_opts->blk_size = (size_t)(cl_opts->num_bpp) / 2; + + /* set default if none specified yet */ + if (!cl_opts->io_types) + cl_opts->io_types = PIO_HDF5 | PIO_MPI | PIO_POSIX; /* run all API */ + + /* verify parameters sanity. Adjust if needed. */ + /* cap xfer_size with bytes per process */ + if (!cl_opts->dim2d) { + if (cl_opts->min_xfer_size > (size_t)cl_opts->num_bpp) + cl_opts->min_xfer_size = (size_t)cl_opts->num_bpp; + if (cl_opts->max_xfer_size > (size_t)cl_opts->num_bpp) + cl_opts->max_xfer_size = (size_t)cl_opts->num_bpp; + } + if (cl_opts->min_xfer_size > cl_opts->max_xfer_size) + cl_opts->min_xfer_size = cl_opts->max_xfer_size; + if (cl_opts->blk_size > (size_t)cl_opts->num_bpp) + cl_opts->blk_size = (size_t)cl_opts->num_bpp; + /* check range of number of processes */ + if (cl_opts->min_num_procs <= 0) + cl_opts->min_num_procs = 1; + if (cl_opts->max_num_procs <= 0) + cl_opts->max_num_procs = 1; + if (cl_opts->min_num_procs > cl_opts->max_num_procs) + cl_opts->min_num_procs = cl_opts->max_num_procs; + /* check iteration */ + if (cl_opts->num_iters <= 0) + cl_opts->num_iters = 1; + + return cl_opts; +} + +/* + * Function: parse_size_directive + * Purpose: Parse the size directive passed on the commandline. The size + * directive is an integer followed by a size indicator: + * + * K, k - Kilobyte + * M, m - Megabyte + * G, g - Gigabyte + * + * Return: The size as a off_t because this is related to file size. + * If an unknown size indicator is used, then the program will + * exit with EXIT_FAILURE as the return value. + * Programmer: Bill Wendling, 18. December 2001 + * Modifications: + */ +static off_t +parse_size_directive(const char *size) +{ + off_t s; + char *endptr; + + s = HDstrtol(size, &endptr, 10); + + if (endptr && *endptr) { + while (*endptr != '\0' && (*endptr == ' ' || *endptr == '\t')) + ++endptr; + + switch (*endptr) { + case 'K': + case 'k': + s *= ONE_KB; + break; + case 'M': + case 'm': + s *= ONE_MB; + break; + case 'G': + case 'g': + s *= ONE_GB; + break; + default: + HDfprintf(stderr, "Illegal size specifier '%c'\n", *endptr); + HDexit(EXIT_FAILURE); + } + } + + return s; +} + +/* + * Function: usage + * Purpose: Print a usage message and then exit. + * Return: Nothing + * Programmer: Bill Wendling, 31. October 2001 + * Modifications: + * Added 2D testing (Christian Chilan, 10. August 2005) + */ +static void +usage(const char *prog) +{ + int myrank; + + MPI_Comm_rank(pio_comm_g, &myrank); + + if (myrank == 0) { + print_version(prog); + HDprintf("usage: %s [OPTIONS]\n", prog); + HDprintf(" OPTIONS\n"); + HDprintf(" -h, --help Print a usage message and exit\n"); + HDprintf(" -a S, --align=S Alignment of objects in HDF5 file [default: 1]\n"); + HDprintf(" -A AL, --api=AL Which APIs to test [default: all of them]\n"); +#if 0 + HDprintf(" -b, --binary The elusive binary option\n"); +#endif /* 0 */ + HDprintf(" -B S, --block-size=S Block size within transfer buffer\n"); + HDprintf(" (see below for description)\n"); + HDprintf(" [default: half the number of bytes per process\n"); + HDprintf(" per dataset]\n"); + HDprintf(" -c, --chunk Create HDF5 datasets using chunked storage\n"); + HDprintf(" [default: contiguous storage]\n"); + HDprintf(" -C, --collective Use collective I/O for MPI and HDF5 APIs\n"); + HDprintf(" [default: independent I/O)\n"); + HDprintf(" -d N, --num-dsets=N Number of datasets per file [default: 1]\n"); + HDprintf(" -D DL, --debug=DL Indicate the debugging level\n"); + HDprintf(" [default: no debugging]\n"); + HDprintf(" -e S, --num-bytes=S Number of bytes per process per dataset\n"); + HDprintf(" (see below for description)\n"); + HDprintf(" [default: 256K for 1D, 8K for 2D]\n"); + HDprintf(" -F N, --num-files=N Number of files [default: 1]\n"); + HDprintf(" -g, --geometry Use 2D geometry [default: 1D geometry]\n"); + HDprintf(" -i N, --num-iterations=N Number of iterations to perform [default: 1]\n"); + HDprintf(" -I, --interleaved Interleaved access pattern\n"); + HDprintf(" (see below for example)\n"); + HDprintf(" [default: Contiguous access pattern]\n"); + HDprintf(" -o F, --output=F Output raw data into file F [default: none]\n"); + HDprintf(" -p N, --min-num-processes=N Minimum number of processes to use [default: 1]\n"); + HDprintf(" -P N, --max-num-processes=N Maximum number of processes to use\n"); + HDprintf(" [default: all MPI_COMM_WORLD processes ]\n"); + HDprintf(" -T S, --threshold=S Threshold for alignment of objects in HDF5 file\n"); + HDprintf(" [default: 1]\n"); + HDprintf(" -w, --write-only Perform write tests not the read tests\n"); + HDprintf(" -x S, --min-xfer-size=S Minimum transfer buffer size\n"); + HDprintf(" (see below for description)\n"); + HDprintf(" [default: half the number of bytes per process\n"); + HDprintf(" per dataset]\n"); + HDprintf(" -X S, --max-xfer-size=S Maximum transfer buffer size\n"); + HDprintf(" [default: the number of bytes per process per\n"); + HDprintf(" dataset]\n"); + HDprintf("\n"); + HDprintf(" F - is a filename.\n"); + HDprintf(" N - is an integer >=0.\n"); + HDprintf(" S - is a size specifier, an integer >=0 followed by a size indicator:\n"); + HDprintf(" K - Kilobyte (%d)\n", ONE_KB); + HDprintf(" M - Megabyte (%d)\n", ONE_MB); + HDprintf(" G - Gigabyte (%d)\n", ONE_GB); + HDprintf("\n"); + HDprintf(" Example: '37M' is 37 megabytes or %d bytes\n", 37 * ONE_MB); + HDprintf("\n"); + HDprintf(" AL - is an API list. Valid values are:\n"); + HDprintf(" phdf5 - Parallel HDF5\n"); + HDprintf(" mpiio - MPI-I/O\n"); + HDprintf(" posix - POSIX\n"); + HDprintf("\n"); + HDprintf(" Example: --api=mpiio,phdf5\n"); + HDprintf("\n"); + HDprintf(" Dataset size:\n"); + HDprintf(" Depending on the selected geometry, each test dataset is either a linear\n"); + HDprintf(" array of size bytes-per-process * num-processes, or a square array of size\n"); + HDprintf(" (bytes-per-process * num-processes) x (bytes-per-process * num-processes).\n"); + HDprintf("\n"); + HDprintf(" Block size vs. Transfer buffer size:\n"); + HDprintf(" buffer-size controls the size of the memory buffer, which is broken into\n"); + HDprintf(" blocks and written to the file. Depending on the selected geometry, each\n"); + HDprintf(" block can be a linear array of size block-size or a square array of size\n"); + HDprintf(" block-size x block-size. The arrangement in which blocks are written is\n"); + HDprintf(" determined by the access pattern.\n"); + HDprintf("\n"); + HDprintf(" In 1D geometry, the transfer buffer is a linear array of size buffer-size.\n"); + HDprintf(" In 2D geometry, it is a rectangular array of size block-size x buffer-size\n"); + HDprintf(" or buffer-size x block-size if interleaved pattern is selected.\n"); + HDprintf("\n"); + HDprintf(" Interleaved and Contiguous patterns in 1D geometry:\n"); + HDprintf(" When contiguous access pattern is chosen, the dataset is evenly divided\n"); + HDprintf(" into num-processes regions and each process writes data to its own region.\n"); + HDprintf(" When interleaved blocks are written to a dataset, space for the first\n"); + HDprintf(" block of the first process is allocated in the dataset, then space is\n"); + HDprintf(" allocated for the first block of the second process, etc. until space is\n"); + HDprintf(" allocated for the first block of each process, then space is allocated for\n"); + HDprintf(" the second block of the first process, the second block of the second\n"); + HDprintf(" process, etc.\n"); + HDprintf("\n"); + HDprintf(" For example, with a 3 process run, 512KB bytes-per-process, 256KB transfer\n"); + HDprintf(" buffer size, and 64KB block size, each process must issue 2 transfer\n"); + HDprintf(" requests to complete access to the dataset.\n"); + HDprintf(" Contiguous blocks of the first transfer request are written like so:\n"); + HDprintf(" 1111----2222----3333----\n"); + HDprintf(" Interleaved blocks of the first transfer request are written like so:\n"); + HDprintf(" 123123123123------------\n"); + HDprintf(" The actual number of I/O operations involved in a transfer request\n"); + HDprintf(" depends on the access pattern and communication mode.\n"); + HDprintf(" When using independent I/O with interleaved pattern, each process\n"); + HDprintf(" performs 4 small non-contiguous I/O operations per transfer request.\n"); + HDprintf(" If collective I/O is turned on, the combined content of the buffers of\n"); + HDprintf(" the 3 processes will be written using one collective I/O operation\n"); + HDprintf(" per transfer request.\n"); + HDprintf("\n"); + HDprintf(" For information about access patterns in 2D geometry, please refer to the\n"); + HDprintf(" HDF5 Reference Manual.\n"); + HDprintf("\n"); + HDprintf(" DL - is a list of debugging flags. Valid values are:\n"); + HDprintf(" 1 - Minimal\n"); + HDprintf(" 2 - Not quite everything\n"); + HDprintf(" 3 - Everything\n"); + HDprintf(" 4 - The kitchen sink\n"); + HDprintf(" r - Raw data I/O throughput information\n"); + HDprintf(" t - Times as well as throughputs\n"); + HDprintf(" v - Verify data correctness\n"); + HDprintf("\n"); + HDprintf(" Example: --debug=2,r,t\n"); + HDprintf("\n"); + HDprintf(" Environment variables:\n"); + HDprintf(" HDF5_NOCLEANUP Do not remove data files if set [default remove]\n"); + HDprintf(" HDF5_MPI_INFO MPI INFO object key=value separated by ;\n"); + HDprintf(" HDF5_PARAPREFIX Paralllel data files prefix\n"); + fflush(stdout); + } /* end if */ +} /* end usage() */ + +#else /* H5_HAVE_PARALLEL */ + +/* + * Function: main + * Purpose: Dummy main() function for if HDF5 was configured without + * parallel stuff. + * Return: EXIT_SUCCESS + * Programmer: Bill Wendling, 14. November 2001 + */ +int +main(void) +{ + HDprintf("No parallel IO performance because parallel is not configured\n"); + return EXIT_SUCCESS; +} /* end main */ + +#endif /* !H5_HAVE_PARALLEL */ diff --git a/tools/src/h5perf/pio_perf.h b/tools/src/h5perf/pio_perf.h new file mode 100644 index 0000000..8924c20 --- /dev/null +++ b/tools/src/h5perf/pio_perf.h @@ -0,0 +1,109 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef PIO_PERF_H +#define PIO_PERF_H + +#ifndef STANDALONE +#include "io_timer.h" +#include "H5private.h" +#include "h5tools.h" +#include "h5tools_utils.h" +#else +#include "io_timer.h" +#include "pio_standalone.h" +#endif + +#ifdef H5_HAVE_PARALLEL +extern MPI_Info h5_io_info_g; /* MPI INFO object for IO */ +#endif + +#ifdef H5_HAVE_PARALLEL +int h5_set_info_object(void); +void h5_dump_info_object(MPI_Info info); +#endif + +/* setup the dataset no fill option if this is v1.5 or more */ +#if H5_VERS_MAJOR > 1 || H5_VERS_MINOR > 4 +#define H5_HAVE_NOFILL 1 +#endif + +typedef enum iotype_ { + POSIXIO, + MPIO, + PHDF5 + /*NUM_TYPES*/ +} iotype; + +typedef struct parameters_ { + iotype io_type; /* The type of IO test to perform */ + int num_procs; /* Maximum number of processes to use */ + long num_files; /* Number of files to create */ + long num_dsets; /* Number of datasets to create */ + off_t num_bytes; /* Number of bytes in each dset */ + int num_iters; /* Number of times to loop doing the IO */ + size_t buf_size; /* Buffer size */ + size_t blk_size; /* Block size */ + unsigned interleaved; /* Interleaved vs. contiguous blocks */ + unsigned collective; /* Collective vs. independent I/O */ + unsigned dim2d; /* 1D vs. 2D */ + hsize_t h5_align; /* HDF5 object alignment */ + hsize_t h5_thresh; /* HDF5 object alignment threshold */ + int h5_use_chunks; /* Make HDF5 dataset chunked */ + int h5_write_only; /* Perform the write tests only */ + int verify; /* Verify data correctness */ +} parameters; + +typedef struct results_ { + herr_t ret_code; + io_time_t *timers; +} results; + +#ifndef SUCCESS +#define SUCCESS 0 +#endif /* !SUCCESS */ + +#ifndef FAIL +#define FAIL -1 +#endif /* !FAIL */ + +extern FILE * output; /* output file */ +extern io_time_t *timer_g; /* timer: global for stub functions */ +extern int comm_world_rank_g; /* my rank in MPI_COMM_RANK */ +extern int comm_world_nprocs_g; /* num. of processes of MPI_COMM_WORLD */ +extern MPI_Comm pio_comm_g; /* Communicator to run the PIO */ +extern int pio_mpi_rank_g; /* MPI rank of pio_comm_g */ +extern int pio_mpi_nprocs_g; /* number of processes of pio_comm_g */ +extern int pio_debug_level; /* The debug level: + * 0 - Off + * 1 - Minimal + * 2 - Some more + * 3 - Maximal + * 4 - Even More Debugging (timer stuff) + */ + +#define HDprint_rank(f) /* print rank in MPI_COMM_WORLD */ HDfprintf(f, "%d: ", comm_world_rank_g); +#define HDprint_size(f) /* print size of MPI_COMM_WORLD */ HDfprintf(f, "%d", comm_world_nprocs_g); +#define HDprint_rank_size(f) /* print rank/size of MPI_COMM_WORLD */ \ + HDfprintf(f, "%d/%d: ", comm_world_rank_g, comm_world_nprocs_g); + +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +extern results do_pio(parameters param); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* PIO_PERF_H */ diff --git a/tools/src/h5perf/sio_engine.c b/tools/src/h5perf/sio_engine.c new file mode 100644 index 0000000..dc2c68e --- /dev/null +++ b/tools/src/h5perf/sio_engine.c @@ -0,0 +1,1330 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Author: Christian Chilan, April 2008 + */ + +#include "hdf5.h" + +#ifdef H5_STDC_HEADERS +#include +#include +#include +#include +#endif + +#ifdef H5_HAVE_UNISTD_H +#include +#include +#endif + +#ifdef H5_HAVE_SYS_STAT_H +#include +#endif + +#include "sio_perf.h" + +/* Macro definitions */ + +/* sizes of various items. these sizes won't change during program execution */ +#define ELMT_H5_TYPE H5T_NATIVE_UCHAR + +#define GOTOERROR(errcode) \ + { \ + ret_code = errcode; \ + goto done; \ + } +#define ERRMSG(mesg) \ + { \ + HDfprintf(stderr, "*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ + } + +/* verify: if val is false (0), print mesg. */ +#define VRFY(val, mesg) \ + do { \ + if (!val) { \ + ERRMSG(mesg); \ + GOTOERROR(FAIL); \ + } \ + } while (0) + +/* POSIX I/O macros */ +#ifdef H5_HAVE_WIN32_API +/* Can't link against the library, so this test will use the older, non-Unicode + * _open() call on Windows. + */ +#define HDopen(S, F, ...) _open(S, F | _O_BINARY, __VA_ARGS__) +#endif /* H5_HAVE_WIN32_API */ +#define POSIXCREATE(fn) HDopen(fn, O_CREAT | O_TRUNC | O_RDWR, 0600) +#define POSIXOPEN(fn, F) HDopen(fn, F, 0600) +#define POSIXCLOSE(F) HDclose(F) +#define POSIXSEEK(F, L) HDlseek(F, L, SEEK_SET) +#define POSIXWRITE(F, B, S) HDwrite(F, B, S) +#define POSIXREAD(F, B, S) HDread(F, B, S) + +enum { SIO_CREATE = 1, SIO_WRITE = 2, SIO_READ = 4 }; + +/* Global variables */ +static int clean_file_g = -1; /*whether to cleanup temporary test */ +/*files. -1 is not defined; */ +/*0 is no cleanup; 1 is do cleanup */ + +/* the different types of file descriptors we can expect */ +typedef union { + int posixfd; /* POSIX file handle*/ + hid_t h5fd; /* HDF5 file */ +} file_descr; + +/* local functions */ +static char * sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size, + parameters *param); +static herr_t do_write(results *res, file_descr *fd, parameters *parms, void *buffer); +static herr_t do_read(results *res, file_descr *fd, parameters *parms, void *buffer); +static herr_t dset_write(int local_dim, file_descr *fd, parameters *parms, void *buffer); +static herr_t posix_buffer_write(int local_dim, file_descr *fd, parameters *parms, void *buffer); +static herr_t dset_read(int localrank, file_descr *fd, parameters *parms, void *buffer, const char *buffer2); +static herr_t posix_buffer_read(int local_dim, file_descr *fd, parameters *parms, void *buffer); +static herr_t do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags); +hid_t set_vfd(parameters *param); +static herr_t do_fclose(iotype iot, file_descr *fd); +static void do_cleanupfile(iotype iot, char *fname); + +/* global variables */ +static HDoff_t offset[MAX_DIMS]; /* dataset size in bytes */ +static size_t buf_offset[MAX_DIMS]; /* dataset size in bytes */ +static int order[MAX_DIMS]; /* dimension access order */ +static size_t linear_buf_size; /* linear buffer size */ +static int cont_dim; /* lowest dimension for contiguous POSIX + access */ +static size_t cont_size; /* size of contiguous POSIX access */ +static hid_t fapl; /* file access list */ +static unsigned char *buf_p; /* buffer pointer */ +static const char * multi_letters = "msbrglo"; /* string for multi driver */ + +/* HDF5 global variables */ +static hsize_t h5count[MAX_DIMS]; /*selection count */ +static hssize_t h5offset[MAX_DIMS]; /* Selection offset within dataspace */ +static hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ +static hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ +static hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ +static hid_t h5dcpl = H5I_INVALID_HID; /* Dataset creation property list */ +static hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ + +/* + * Function: do_sio + * Purpose: SIO Engine where IO are executed. + * Return: results + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ +void +do_sio(parameters param, results *res) +{ + char * buffer = NULL; /*data buffer pointer */ + size_t buf_size[MAX_DIMS]; /* general buffer size in bytes */ + file_descr fd; /* file handles */ + iotype iot; /* API type */ + char base_name[256]; /* test file base name */ + /* return codes */ + herr_t ret_code = 0; /*return code */ + + char fname[FILENAME_MAX]; /* test file name */ + int i; + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + + /* Sanity check parameters */ + + /* IO type */ + iot = param.io_type; + + switch (iot) { + case POSIXIO: + fd.posixfd = -1; + res->timers = io_time_new(SYS_CLOCK); + break; + case HDF5: + fd.h5fd = -1; + res->timers = io_time_new(SYS_CLOCK); + break; + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + GOTOERROR(FAIL); + } + + linear_buf_size = 1; + + for (i = 0; i < param.rank; i++) { + buf_size[i] = param.buf_size[i]; + order[i] = param.order[i]; + linear_buf_size *= buf_size[i]; + buf_offset[i] = 0; + offset[i] = 0; + + /* Validate transfer buffer size */ + if (param.buf_size[i] <= 0) { + HDfprintf(stderr, "Transfer buffer size[%d] (%zu) must be > 0\n", i, buf_size[i]); + GOTOERROR(FAIL); + } + + if ((param.dset_size[i] % param.buf_size[i]) != 0) { + HDfprintf(stderr, + "Dataset size[%d] (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " + "trasfer buffer size[%d] (%zu)\n", + param.rank, (long long)param.dset_size[i], param.rank, param.buf_size[i]); + GOTOERROR(FAIL); + } + } + + /* Allocate transfer buffer */ + if ((buffer = (char *)malloc(linear_buf_size)) == NULL) { + HDfprintf(stderr, "malloc for transfer buffer size (%zu) failed\n", linear_buf_size); + GOTOERROR(FAIL); + } + + if (sio_debug_level >= 4) + + /* output all of the times for all iterations */ + HDfprintf(output, "Timer details:\n"); + + /* + * Write performance measurement + */ + /* Open file for write */ + + HDstrcpy(base_name, "#sio_tmp"); + sio_create_filename(iot, base_name, fname, sizeof(fname), ¶m); + + if (sio_debug_level > 0) + HDfprintf(output, "data filename=%s\n", fname); + + io_time_set(res->timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTART); + hrc = do_fopen(¶m, fname, &fd, SIO_CREATE | SIO_WRITE); + VRFY((hrc == SUCCESS), "do_fopen failed"); + + io_time_set(res->timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTART); + hrc = do_write(res, &fd, ¶m, buffer); + io_time_set(res->timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_write failed"); + + /* Close file for write */ + hrc = do_fclose(iot, &fd); + io_time_set(res->timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_fclose failed"); + + if (!param.h5_write_only) { + /* + * Read performance measurement + */ + + /* Open file for read */ + io_time_set(res->timers, HDF5_GROSS_READ_FIXED_DIMS, TSTART); + hrc = do_fopen(¶m, fname, &fd, SIO_READ); + VRFY((hrc == SUCCESS), "do_fopen failed"); + + io_time_set(res->timers, HDF5_FINE_READ_FIXED_DIMS, TSTART); + hrc = do_read(res, &fd, ¶m, buffer); + io_time_set(res->timers, HDF5_FINE_READ_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_read failed"); + + /* Close file for read */ + hrc = do_fclose(iot, &fd); + + io_time_set(res->timers, HDF5_GROSS_READ_FIXED_DIMS, TSTOP); + VRFY((hrc == SUCCESS), "do_fclose failed"); + } + + do_cleanupfile(iot, fname); + +done: + /* clean up */ + /* release HDF5 objects */ + + /* close any opened files */ + /* no remove(fname) because that should have happened normally. */ + switch (iot) { + case POSIXIO: + if (fd.posixfd != -1) + hrc = do_fclose(iot, &fd); + break; + case HDF5: + if (fd.h5fd != -1) + hrc = do_fclose(iot, &fd); + break; + default: + /* unknown request */ + HDassert(0 && "Unknown IO type"); + break; + } + + /* release generic resources */ + if (buffer) + free(buffer); + + res->ret_code = ret_code; +} + +/* + * Function: sio_create_filename + * Purpose: Create a new filename to write to. Determine the correct + * suffix to append to the filename by the type of I/O we're + * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if + * USER or LOGIN are specified in the environment. + * Return: Pointer to filename or NULL + * Programmer: Bill Wendling, 21. November 2001 + * Modifications: Support for file drivers. Christian Chilan, April, 2008 + */ +static char * +sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size, parameters *param) +{ + const char *prefix, *suffix = ""; + char * ptr, last = '\0'; + size_t i, j; + vfdtype vfd; + vfd = param->vfd; + + if (!base_name || !fullname || size < 1) + return NULL; + + memset(fullname, 0, size); + + switch (iot) { + case POSIXIO: + suffix = ".posix"; + break; + case HDF5: + suffix = ".h5"; + if (vfd == family) + suffix = "%05d.h5"; + else if (vfd == multi) + suffix = NULL; + break; + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + HDassert(0 && "Unknown IO type"); + break; + } + + /* First use the environment variable and then try the constant */ + prefix = HDgetenv("HDF5_PREFIX"); + +#ifdef HDF5_PREFIX + if (!prefix) + prefix = HDF5_PREFIX; +#endif /* HDF5_PREFIX */ + + /* Prepend the prefix value to the base name */ + if (prefix && *prefix) { + /* If the prefix specifies the HDF5_PREFIX directory, then + * default to using the "/tmp/$USER" or "/tmp/$LOGIN" + * directory instead. */ + register char *user, *login, *subdir; + + user = HDgetenv("USER"); + login = HDgetenv("LOGIN"); + subdir = (user ? user : login); + + if (subdir) { + for (i = 0; i < size - 1 && prefix[i]; i++) + fullname[i] = prefix[i]; + + fullname[i++] = '/'; + + for (j = 0; i < size && subdir[j]; i++, j++) + fullname[i] = subdir[j]; + } + else { + /* We didn't append the prefix yet */ + HDstrncpy(fullname, prefix, size); + fullname[size - 1] = '\0'; + } + + if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) { + /* Append the base_name with a slash first. Multiple slashes are + * handled below. */ + h5_stat_t buf; + + if (HDstat(fullname, &buf) < 0) + /* The directory doesn't exist just yet */ + if (HDmkdir(fullname, 0755) < 0 && errno != EEXIST) { + /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. + * Default to PREFIX's original prefix value. */ + HDstrcpy(fullname, prefix); + } + + HDstrcat(fullname, "/"); + HDstrcat(fullname, base_name); + } + else { + /* Buffer is too small */ + return NULL; + } + } + else if (strlen(base_name) >= size) { + /* Buffer is too small */ + return NULL; + } + else { + HDstrcpy(fullname, base_name); + } + + /* Append a suffix */ + if (suffix) { + if (HDstrlen(fullname) + HDstrlen(suffix) >= size) + return NULL; + + HDstrcat(fullname, suffix); + } + + /* Remove any double slashes in the filename */ + for (ptr = fullname, i = j = 0; ptr && (i < size); i++, ptr++) { + if (*ptr != '/' || last != '/') + fullname[j++] = *ptr; + + last = *ptr; + } + + return fullname; +} + +/* + * Function: do_write + * Purpose: Write the required amount of data to the file. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ +static herr_t +do_write(results *res, file_descr *fd, parameters *parms, void *buffer) +{ + int ret_code = SUCCESS; + char dname[64]; + int i; + size_t u; + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + hsize_t h5dims[MAX_DIMS]; /*dataset dim sizes */ + hsize_t h5chunk[MAX_DIMS]; /*dataset dim sizes */ + hsize_t h5block[MAX_DIMS]; /*dataspace selection */ + hsize_t h5stride[MAX_DIMS]; /*selection stride */ + hsize_t h5start[MAX_DIMS]; /*selection start */ + hsize_t h5maxdims[MAX_DIMS]; + int rank; /*rank of dataset */ + + /* Prepare buffer for verifying data */ + /* if (parms->verify) + memset(buffer,1,linear_buf_size); */ + + buf_p = (unsigned char *)buffer; + + for (u = 0; u < linear_buf_size; u++) + buf_p[u] = u % 128; + + rank = parms->rank; + + for (i = 0; i < rank; i++) + h5offset[i] = offset[i] = 0; + + /* I/O Access specific setup */ + switch (parms->io_type) { + case POSIXIO: + + /* determine lowest dimension for contiguous POSIX access */ + cont_dim = rank; + + for (i = rank - 1; i >= 0; i--) { + if (parms->buf_size[i] == parms->dset_size[i]) + cont_dim = i; + else + break; + } + + /* determine size of the contiguous POSIX access */ + cont_size = (!cont_dim) ? 1 : parms->buf_size[cont_dim - 1]; + for (i = cont_dim; i < rank; i++) + cont_size *= parms->buf_size[i]; + + break; + + case HDF5: /* HDF5 setup */ + + for (i = 0; i < rank; i++) { + h5dims[i] = parms->dset_size[i]; + h5start[i] = 0; + h5stride[i] = 1; + h5block[i] = 1; + h5count[i] = parms->buf_size[i]; + h5chunk[i] = parms->chk_size[i]; + h5maxdims[i] = H5S_UNLIMITED; + } + + if (parms->h5_use_chunks && parms->h5_extendable) { + h5dset_space_id = H5Screate_simple(rank, h5count, h5maxdims); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + } + else { + h5dset_space_id = H5Screate_simple(rank, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + } + + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + + /* Create the memory dataspace that corresponds to the xfer buffer */ + h5mem_space_id = H5Screate_simple(rank, h5count, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + + /* Create the dataset transfer property list */ + h5dxpl = H5Pcreate(H5P_DATASET_XFER); + if (h5dxpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + break; + + default: + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + GOTOERROR(FAIL); + break; + } /* end switch */ + + /* create dataset */ + switch (parms->io_type) { + case POSIXIO: + break; + + case HDF5: + h5dcpl = H5Pcreate(H5P_DATASET_CREATE); + + if (h5dcpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + if (parms->h5_use_chunks) { + /* Set the chunk size to be the same as the buffer size */ + hrc = H5Pset_chunk(h5dcpl, rank, h5chunk); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Set failed\n"); + GOTOERROR(FAIL); + } /* end if */ + } /* end if */ + + HDsprintf(dname, "Dataset_%ld", (unsigned long)parms->num_bytes); + h5ds_id = + H5Dcreate2(fd->h5fd, dname, ELMT_H5_TYPE, h5dset_space_id, H5P_DEFAULT, h5dcpl, H5P_DEFAULT); + + if (h5ds_id < 0) { + HDfprintf(stderr, "HDF5 Dataset Create failed\n"); + GOTOERROR(FAIL); + } + + hrc = H5Pclose(h5dcpl); + /* verifying the close of the dcpl */ + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Property List Close failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + GOTOERROR(FAIL); + break; + } + + /* Start "raw data" write timer */ + io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART); + + /* Perform write */ + hrc = dset_write(rank - 1, fd, parms, buffer); + + if (hrc < 0) { + HDfprintf(stderr, "Error in dataset write\n"); + GOTOERROR(FAIL); + } + + /* Stop "raw data" write timer */ + io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP); + + /* Calculate write time */ + + /* Close dataset. Only HDF5 needs to do an explicit close. */ + if (parms->io_type == HDF5) { + hrc = H5Dclose(h5ds_id); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Close failed\n"); + GOTOERROR(FAIL); + } + + h5ds_id = H5I_INVALID_HID; + } /* end if */ + +done: + + /* release HDF5 objects */ + if (h5dset_space_id != -1) { + hrc = H5Sclose(h5dset_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); + ret_code = FAIL; + } + else { + h5dset_space_id = H5I_INVALID_HID; + } + } + + if (h5mem_space_id != -1) { + hrc = H5Sclose(h5mem_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); + ret_code = FAIL; + } + else { + h5mem_space_id = H5I_INVALID_HID; + } + } + + if (h5dxpl != -1) { + hrc = H5Pclose(h5dxpl); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); + ret_code = FAIL; + } + else { + h5dxpl = H5I_INVALID_HID; + } + } + + return ret_code; +} + +/* + * Function: dset_write + * Purpose: Write buffer into the dataset. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ +static herr_t +dset_write(int local_dim, file_descr *fd, parameters *parms, void *buffer) +{ + int cur_dim = order[local_dim] - 1; + int ret_code = SUCCESS; + int k; + hsize_t dims[MAX_DIMS], maxdims[MAX_DIMS]; + hsize_t i; + int j; + herr_t hrc; + + /* iterates according to the dimensions in order array */ + for (i = 0; i < parms->dset_size[cur_dim]; i += parms->buf_size[cur_dim]) { + + h5offset[cur_dim] = (hssize_t)i; + offset[cur_dim] = (HDoff_t)i; + + if (local_dim > 0) { + + dset_write(local_dim - 1, fd, parms, buffer); + } + else { + + switch (parms->io_type) { + + case POSIXIO: + /* initialize POSIX offset in the buffer */ + for (j = 0; j < parms->rank; j++) + buf_offset[j] = 0; + buf_p = (unsigned char *)buffer; + /* write POSIX buffer */ + posix_buffer_write(0, fd, parms, buffer); + break; + + case HDF5: + /* if dimensions are extendable, extend them as needed during access */ + if (parms->h5_use_chunks && parms->h5_extendable) { + + hrc = H5Sget_simple_extent_dims(h5dset_space_id, dims, maxdims); + VRFY((hrc >= 0), "H5Sget_simple_extent_dims"); + + for (k = 0; k < parms->rank; k++) { + + HDassert(h5offset[k] >= 0); + if (dims[k] <= (hsize_t)h5offset[k]) { + dims[k] = dims[k] + h5count[k]; + hrc = H5Sset_extent_simple(h5dset_space_id, parms->rank, dims, maxdims); + VRFY((hrc >= 0), "H5Sset_extent_simple"); + hrc = H5Dset_extent(h5ds_id, dims); + VRFY((hrc >= 0), "H5Dextend"); + } + } + } + /* applies offset */ + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + + /* Write the buffer out */ + hrc = H5Sget_simple_extent_dims(h5dset_space_id, dims, maxdims); + hrc = H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dwrite"); + + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + HDassert(0 && "Unknown IO type"); + break; + } /* switch (parms->io_type) */ + } + } +done: + return ret_code; +} + +/* + * Function: posix_buffer_write + * Purpose: Write buffer into the POSIX file considering contiguity. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ + +static herr_t +posix_buffer_write(int local_dim, file_descr *fd, parameters *parms, void *buffer) +{ + int ret_code = SUCCESS; + + /* if dimension is not contiguous, call recursively */ + if (local_dim < parms->rank - 1 && local_dim != cont_dim) { + size_t u; + + for (u = 0; u < parms->buf_size[local_dim]; u++) { + buf_offset[local_dim] = u; + posix_buffer_write(local_dim + 1, fd, parms, buffer); + + /* if next dimension is cont_dim, it will fill out the buffer + traversing the entire dimension local_dim without the need + of performing iteration */ + if (local_dim + 1 == cont_dim) + break; + } + /* otherwise, perform contiguous POSIX access */ + } + else { + HDoff_t d_offset; + HDoff_t linear_dset_offset = 0; + int i, j, rc; + + buf_offset[local_dim] = 0; + + /* determine offset in the buffer */ + for (i = 0; i < parms->rank; i++) { + d_offset = 1; + + for (j = i + 1; j < parms->rank; j++) + d_offset *= (HDoff_t)parms->dset_size[j]; + + linear_dset_offset += (offset[i] + (HDoff_t)buf_offset[i]) * d_offset; + } + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, linear_dset_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + /* check if all bytes are written */ + rc = ((ssize_t)cont_size == POSIXWRITE(fd->posixfd, buf_p, cont_size)); + VRFY((rc != 0), "POSIXWRITE"); + + /* Advance location in buffer */ + buf_p += cont_size; + } + +done: + return ret_code; +} + +/* + * Function: do_read + * Purpose: Read the required amount of data to the file. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ +static herr_t +do_read(results *res, file_descr *fd, parameters *parms, void *buffer) +{ + char * buffer2 = NULL; /* Buffer for data verification */ + int ret_code = SUCCESS; + char dname[64]; + int i; + size_t u; + /* HDF5 variables */ + herr_t hrc; /*HDF5 return code */ + hsize_t h5dims[MAX_DIMS]; /*dataset dim sizes */ + hsize_t h5block[MAX_DIMS]; /*dataspace selection */ + hsize_t h5stride[MAX_DIMS]; /*selection stride */ + hsize_t h5start[MAX_DIMS]; /*selection start */ + int rank; + + /* Allocate data verification buffer */ + if (NULL == (buffer2 = (char *)malloc(linear_buf_size))) { + HDfprintf(stderr, "malloc for data verification buffer size (%zu) failed\n", linear_buf_size); + GOTOERROR(FAIL); + } /* end if */ + + /* Prepare buffer for verifying data */ + for (u = 0; u < linear_buf_size; u++) + buffer2[u] = (char)(u % 128); + + rank = parms->rank; + for (i = 0; i < rank; i++) + h5offset[i] = offset[i] = 0; + + /* I/O Access specific setup */ + switch (parms->io_type) { + case POSIXIO: + cont_dim = rank; + + for (i = rank - 1; i >= 0; i--) { + if (parms->buf_size[i] == parms->dset_size[i]) + cont_dim = i; + else + break; + } + cont_size = (!cont_dim) ? 1 : parms->buf_size[cont_dim - 1]; + for (i = cont_dim; i < rank; i++) + cont_size *= parms->buf_size[i]; + + break; + + case HDF5: /* HDF5 setup */ + for (i = 0; i < rank; i++) { + h5dims[i] = parms->dset_size[i]; + h5start[i] = 0; + h5stride[i] = 1; + h5block[i] = 1; + h5count[i] = parms->buf_size[i]; + } + + h5dset_space_id = H5Screate_simple(rank, h5dims, NULL); + VRFY((h5dset_space_id >= 0), "H5Screate_simple"); + + hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, h5block); + VRFY((hrc >= 0), "H5Sselect_hyperslab"); + + /* Create the memory dataspace that corresponds to the xfer buffer */ + h5mem_space_id = H5Screate_simple(rank, h5count, NULL); + VRFY((h5mem_space_id >= 0), "H5Screate_simple"); + + /* Create the dataset transfer property list */ + h5dxpl = H5Pcreate(H5P_DATASET_XFER); + if (h5dxpl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + GOTOERROR(FAIL); + break; + } /* end switch */ + + /* create dataset */ + switch (parms->io_type) { + case POSIXIO: + break; + + case HDF5: + HDsprintf(dname, "Dataset_%ld", (long)parms->num_bytes); + h5ds_id = H5Dopen2(fd->h5fd, dname, H5P_DEFAULT); + if (h5ds_id < 0) { + HDfprintf(stderr, "HDF5 Dataset open failed\n"); + GOTOERROR(FAIL); + } + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + GOTOERROR(FAIL); + break; + } /* end switch */ + + /* Start "raw data" read timer */ + io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTART); + hrc = dset_read(rank - 1, fd, parms, buffer, buffer2); + + if (hrc < 0) { + HDfprintf(stderr, "Error in dataset read\n"); + GOTOERROR(FAIL); + } + + /* Stop "raw data" read timer */ + io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTOP); + + /* Calculate read time */ + + /* Close dataset. Only HDF5 needs to do an explicit close. */ + if (parms->io_type == HDF5) { + hrc = H5Dclose(h5ds_id); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Close failed\n"); + GOTOERROR(FAIL); + } + + h5ds_id = H5I_INVALID_HID; + } /* end if */ + +done: + + /* release HDF5 objects */ + if (h5dset_space_id != -1) { + hrc = H5Sclose(h5dset_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); + ret_code = FAIL; + } + else { + h5dset_space_id = H5I_INVALID_HID; + } + } + + if (h5mem_space_id != -1) { + hrc = H5Sclose(h5mem_space_id); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); + ret_code = FAIL; + } + else { + h5mem_space_id = H5I_INVALID_HID; + } + } + + if (h5dxpl != -1) { + hrc = H5Pclose(h5dxpl); + if (hrc < 0) { + HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); + ret_code = FAIL; + } + else { + h5dxpl = H5I_INVALID_HID; + } + } + + /* release generic resources */ + if (buffer2) + free(buffer2); + + return ret_code; +} + +/* + * Function: dset_read + * Purpose: Read buffer into the dataset. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ + +static herr_t +dset_read(int local_dim, file_descr *fd, parameters *parms, void *buffer, const char *buffer2) +{ + int cur_dim = order[local_dim] - 1; + hsize_t i; + int j; + herr_t hrc; + int ret_code = SUCCESS; + + /* iterate on the current dimension */ + for (i = 0; i < parms->dset_size[cur_dim]; i += parms->buf_size[cur_dim]) { + + h5offset[cur_dim] = (hssize_t)i; + offset[cur_dim] = (HDoff_t)i; + + /* if traverse in order array is incomplete, recurse */ + if (local_dim > 0) { + + ret_code = dset_read(local_dim - 1, fd, parms, buffer, buffer2); + + /* otherwise, write buffer into dataset */ + } + else { + + switch (parms->io_type) { + + case POSIXIO: + for (j = 0; j < parms->rank; j++) { + buf_offset[j] = 0; + } + buf_p = (unsigned char *)buffer; + posix_buffer_read(0, fd, parms, buffer); + break; + + case HDF5: + hrc = H5Soffset_simple(h5dset_space_id, h5offset); + VRFY((hrc >= 0), "H5Soffset_simple"); + /* Read the buffer out */ + hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); + VRFY((hrc >= 0), "H5Dread"); + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); + HDassert(0 && "Unknown IO type"); + break; + } /* switch (parms->io_type) */ + } + } +done: + return ret_code; +} + +/* + * Function: posix_buffer_read + * Purpose: Read buffer into the POSIX file considering contiguity. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ + +static herr_t +posix_buffer_read(int local_dim, file_descr *fd, parameters *parms, void *buffer) +{ + int ret_code = SUCCESS; + + /* if local dimension is not contiguous, recurse */ + if (local_dim < parms->rank - 1 && local_dim != cont_dim) { + size_t u; + + for (u = 0; u < parms->buf_size[local_dim]; u++) { + buf_offset[local_dim] = u; + ret_code = posix_buffer_read(local_dim + 1, fd, parms, buffer); + if (local_dim + 1 == cont_dim) + break; + } + /* otherwise, perform contiguous POSIX access */ + } + else { + HDoff_t d_offset; + HDoff_t linear_dset_offset = 0; + int i, j, rc; + + buf_offset[local_dim] = 0; + /* determine offset in buffer */ + for (i = 0; i < parms->rank; i++) { + d_offset = 1; + + for (j = i + 1; j < parms->rank; j++) + d_offset *= (HDoff_t)parms->dset_size[j]; + + linear_dset_offset += (offset[i] + (HDoff_t)buf_offset[i]) * d_offset; + } + + /* only care if seek returns error */ + rc = POSIXSEEK(fd->posixfd, linear_dset_offset) < 0 ? -1 : 0; + VRFY((rc == 0), "POSIXSEEK"); + /* check if all bytes are read */ + rc = ((ssize_t)cont_size == POSIXREAD(fd->posixfd, buf_p, cont_size)); + VRFY((rc != 0), "POSIXREAD"); + + /* Advance location in buffer */ + buf_p += cont_size; + } +done: + return ret_code; +} + +/* + * Function: do_fopen + * Purpose: Open the specified file. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: Support for file drivers, Christian Chilan, April, 2008 + */ +static herr_t +do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) +{ + int ret_code = SUCCESS; + hid_t fcpl; + + switch (param->io_type) { + case POSIXIO: + if (flags & (SIO_CREATE | SIO_WRITE)) + fd->posixfd = POSIXCREATE(fname); + else + fd->posixfd = POSIXOPEN(fname, O_RDONLY); + + if (fd->posixfd < 0) { + HDfprintf(stderr, "POSIX File Open failed(%s)\n", fname); + GOTOERROR(FAIL); + } + + break; + + case HDF5: + + fapl = set_vfd(param); + + if (fapl < 0) { + HDfprintf(stderr, "HDF5 Property List Create failed\n"); + GOTOERROR(FAIL); + } + + fcpl = H5Pcreate(H5P_FILE_CREATE); + if (param->page_size) { + H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1); + H5Pset_file_space_page_size(fcpl, param->page_size); + if (param->page_buffer_size) + H5Pset_page_buffer_size(fapl, param->page_buffer_size, 0, 0); + } + + /* create the parallel file */ + if (flags & (SIO_CREATE | SIO_WRITE)) { + fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, fcpl, fapl); + } + else { + fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, fapl); + } + + if (fd->h5fd < 0) { + HDfprintf(stderr, "HDF5 File Create failed(%s)\n", fname); + GOTOERROR(FAIL); + } + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)param->io_type); + GOTOERROR(FAIL); + break; + } + +done: + return ret_code; +} + +/* + * Function: set_vfd + * Purpose: Sets file driver. + * Return: SUCCESS or FAIL + * Programmer: Christian Chilan, April, 2008 + * Modifications: + */ + +hid_t +set_vfd(parameters *param) +{ + hid_t my_fapl = H5I_INVALID_HID; + vfdtype vfd; + + vfd = param->vfd; + + if ((my_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) + return -1; + + if (vfd == sec2) { + /* Unix read() and write() system calls */ + if (H5Pset_fapl_sec2(my_fapl) < 0) + return -1; + } + else if (vfd == stdio) { + /* Standard C fread() and fwrite() system calls */ + if (H5Pset_fapl_stdio(my_fapl) < 0) + return -1; + } + else if (vfd == core) { + /* In-core temporary file with 1MB increment */ + if (H5Pset_fapl_core(my_fapl, (size_t)1024 * 1024, TRUE) < 0) + return -1; + } + else if (vfd == split) { + /* Split meta data and raw data each using default driver */ + if (H5Pset_fapl_split(my_fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0) + return -1; + } + else if (vfd == multi) { + /* Multi-file driver, general case of the split driver */ + H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; + hid_t memb_fapl[H5FD_MEM_NTYPES]; + const char *memb_name[H5FD_MEM_NTYPES]; + char sv[H5FD_MEM_NTYPES][1024]; + haddr_t memb_addr[H5FD_MEM_NTYPES]; + H5FD_mem_t mt; + + HDmemset(memb_map, 0, sizeof memb_map); + HDmemset(memb_fapl, 0, sizeof memb_fapl); + HDmemset(memb_name, 0, sizeof memb_name); + HDmemset(memb_addr, 0, sizeof memb_addr); + + HDassert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { + memb_fapl[mt] = H5P_DEFAULT; + HDsprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]); + memb_name[mt] = sv[mt]; + memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10); + } + + if (H5Pset_fapl_multi(my_fapl, memb_map, memb_fapl, memb_name, memb_addr, FALSE) < 0) { + return -1; + } + } + else if (vfd == family) { + hsize_t fam_size = 1 * 1024 * 1024; /*100 MB*/ + + /* Family of files, each 1MB and using the default driver */ + /* if ((val=HDstrtok(NULL, " \t\n\r"))) + fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024); */ + if (H5Pset_fapl_family(my_fapl, fam_size, H5P_DEFAULT) < 0) + return -1; + } + else if (vfd == direct) { +#ifdef H5_HAVE_DIRECT + /* Linux direct read() and write() system calls. Set memory boundary, file block size, + * and copy buffer size to the default values. */ + if (H5Pset_fapl_direct(my_fapl, 1024, 4096, 8 * 4096) < 0) + return -1; +#endif + } + else { + /* Unknown driver */ + return -1; + } + + return my_fapl; +} + +/* + * Function: do_fclose + * Purpose: Close the specified file descriptor. + * Return: SUCCESS or FAIL + * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 + * Modifications: + */ +static herr_t +do_fclose(iotype iot, file_descr *fd /*out*/) +{ + herr_t ret_code = SUCCESS, hrc; + int rc = 0; + + switch (iot) { + case POSIXIO: + rc = POSIXCLOSE(fd->posixfd); + + if (rc != 0) { + HDfprintf(stderr, "POSIX File Close failed\n"); + GOTOERROR(FAIL); + } + + fd->posixfd = -1; + break; + + case HDF5: + hrc = H5Fclose(fd->h5fd); + + if (hrc < 0) { + HDfprintf(stderr, "HDF5 File Close failed\n"); + GOTOERROR(FAIL); + } + + fd->h5fd = -1; + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + GOTOERROR(FAIL); + break; + } + +done: + return ret_code; +} + +/* + * Function: do_cleanupfile + * Purpose: Cleanup temporary file unless HDF5_NOCLEANUP is set. + * Return: void + * Programmer: Albert Cheng 2001/12/12 + * Modifications: Support for file drivers. Christian Chilan, April, 2008 + */ +static void +do_cleanupfile(iotype iot, char *filename) +{ + char temp[2048]; + int j; + hid_t driver; + + if (clean_file_g == -1) + clean_file_g = (HDgetenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; + + if (clean_file_g) { + + switch (iot) { + case POSIXIO: + HDremove(filename); + break; + + case HDF5: + driver = H5Pget_driver(fapl); + + if (driver == H5FD_FAMILY) { + for (j = 0; /*void*/; j++) { + HDsnprintf(temp, sizeof temp, filename, j); + + if (HDaccess(temp, F_OK) < 0) + break; + + HDremove(temp); + } + } + else if (driver == H5FD_CORE) { + hbool_t backing; /* Whether the core file has backing store */ + + H5Pget_fapl_core(fapl, NULL, &backing); + + /* If the file was stored to disk with bacing store, remove it */ + if (backing) + HDremove(filename); + } + else if (driver == H5FD_MULTI) { + H5FD_mem_t mt; + assert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); + + for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { + HDsnprintf(temp, sizeof temp, "%s-%c.h5", filename, multi_letters[mt]); + HDremove(temp); /*don't care if it fails*/ + } + } + else { + HDremove(filename); + } + H5Pclose(fapl); + break; + + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + HDassert(0 && "Unknown IO type"); + break; + } + } +} diff --git a/tools/src/h5perf/sio_perf.c b/tools/src/h5perf/sio_perf.c new file mode 100644 index 0000000..7bd58d8 --- /dev/null +++ b/tools/src/h5perf/sio_perf.c @@ -0,0 +1,1301 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Serial HDF5 Performance Testing Code + * -------------------------------------- + * + * Portable code to test performance on the different platforms we support. + * This is what the report should look like: + * + * nprocs = Max#Procs + * IO API = POSIXIO + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * + * IO API = HDF5 + * # Files = 1, # of dsets = 1000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * # Files = 1, # of dsets = 3000, Elements per dset = 37000 + * Write Results = x MB/s + * Read Results = x MB/s + * + * . . . + * + * + * . . . + * + */ + +/* system header files */ +#include +#include +#include + +#include "hdf5.h" + +/* our header files */ +#include "sio_perf.h" + +/* useful macros */ +#define TAB_SPACE 4 + +#define ONE_KB 1024 +#define ONE_MB (ONE_KB * ONE_KB) +#define ONE_GB (ONE_MB * ONE_KB) + +#define SIO_POSIX 0x1 +#define SIO_HDF5 0x4 + +/* report 0.0 in case t is zero too */ +#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL(t, 0.0) ? 0.0 : ((((double)bytes) / (double)ONE_MB) / (t))) + +#ifndef TRUE +#define TRUE 1 +#endif /* TRUE */ +#ifndef FALSE +#define FALSE (!TRUE) +#endif /* FALSE */ + +/* global variables */ +FILE *output; /* output file */ +int sio_debug_level = 0; /* The debug level: + * 0 - Off + * 1 - Minimal + * 2 - Some more + * 3 - Maximal + * 4 - Maximal & then some + */ + +/* local variables */ +static const char *progname = "h5perf_serial"; + +/* + * Command-line options: The user can specify short or long-named + * parameters. The long-named ones can be partially spelled. When + * adding more, make sure that they don't clash with each other. + */ + +/* + * It seems that only the options that accept additional information + * such as dataset size (-e) require the colon next to it. + */ +static const char * s_opts = "a:A:B:c:Cd:D:e:F:ghi:Imno:p:P:r:stT:v:wx:X:"; +static struct h5_long_options l_opts[] = {{"align", require_arg, 'a'}, + {"api", require_arg, 'A'}, +#if 0 + /* a sighting of the elusive binary option */ + { "binary", no_arg, 'b' }, +#endif /* 0 */ + {"block-size", require_arg, 'B'}, + {"chunk", no_arg, 'c'}, + {"collective", no_arg, 'C'}, + {"debug", require_arg, 'D'}, + {"file-driver", require_arg, 'v'}, + {"geometry", no_arg, 'g'}, + {"help", no_arg, 'h'}, + {"interleaved", require_arg, 'I'}, + {"max-num-processes", require_arg, 'P'}, + {"min-num-processes", require_arg, 'p'}, + {"max-xfer-size", require_arg, 'X'}, + {"min-xfer-size", require_arg, 'x'}, + {"num-bytes", require_arg, 'e'}, + {"num-dsets", require_arg, 'd'}, + {"num-files", require_arg, 'F'}, + {"num-iterations", require_arg, 'i'}, + {"order", require_arg, 'r'}, + {"output", require_arg, 'o'}, + {"extendable", no_arg, 't'}, + {"threshold", require_arg, 'T'}, + {"write-only", require_arg, 'w'}, + {NULL, 0, '\0'}}; + +struct options { + long io_types; /* bitmask of which I/O types to test */ + const char *output_file; /* file to print report to */ + long num_dsets; /* number of datasets */ + long num_files; /* number of files */ + off_t num_bpp; /* number of bytes per proc per dset */ + int num_iters; /* number of iterations */ + hsize_t dset_size[MAX_DIMS]; /* Dataset size */ + size_t buf_size[MAX_DIMS]; /* Buffer size */ + size_t chk_size[MAX_DIMS]; /* Chunk size */ + int order[MAX_DIMS]; /* Dimension access order */ + int dset_rank; /* Rank */ + int buf_rank; /* Rank */ + int order_rank; /* Rank */ + int chk_rank; /* Rank */ + int print_times; /* print times as well as throughputs */ + int print_raw; /* print raw data throughput info */ + hsize_t h5_alignment; /* alignment in HDF5 file */ + hsize_t h5_threshold; /* threshold for alignment in HDF5 file */ + int h5_use_chunks; /* Make HDF5 dataset chunked */ + int h5_write_only; /* Perform the write tests only */ + int h5_extendable; /* Perform the write tests only */ + int verify; /* Verify data correctness */ + vfdtype vfd; /* File driver */ + size_t page_buffer_size; + size_t page_size; +}; + +typedef struct { + double min; + double max; + double sum; + int num; +} minmax; + +/* local functions */ +static hsize_t parse_size_directive(const char *size); +static struct options *parse_command_line(int argc, const char *argv[]); +static void run_test_loop(struct options *options); +static int run_test(iotype iot, parameters parms, struct options *opts); +static void output_all_info(minmax *mm, int count, int indent_level); +static void get_minmax(minmax *mm, double val); +static void accumulate_minmax_stuff(const minmax *mm, int count, minmax *total_mm); +static void output_results(const struct options *options, const char *name, minmax *table, int table_size, + off_t data_size); +static void output_report(const char *fmt, ...); +static void print_indent(register int indent); +static void usage(const char *prog); +static void report_parameters(struct options *opts); + +/* + * Function: main + * Purpose: Start things up. + * Return: EXIT_SUCCESS or EXIT_FAILURE + * Programmer: Bill Wendling, 30. October 2001 + * Modifications: + */ +int +main(int argc, const char *argv[]) +{ + int exit_value = EXIT_SUCCESS; + struct options *opts = NULL; + +#ifndef STANDALONE + /* Initialize h5tools lib */ + h5tools_init(); +#endif + + output = stdout; + + opts = parse_command_line(argc, argv); + + if (!opts) { + exit_value = EXIT_FAILURE; + goto finish; + } + + if (opts->output_file) { + if ((output = HDfopen(opts->output_file, "w")) == NULL) { + HDfprintf(stderr, "%s: cannot open output file\n", progname); + HDperror(opts->output_file); + goto finish; + } + } + + report_parameters(opts); + + run_test_loop(opts); + +finish: + HDfree(opts); + return exit_value; +} + +/* + * Function: run_test_loop + * Purpose: Run the I/O tests. Write the results to OUTPUT. + * + * - The slowest changing part of the test is the number of + * processors to use. For each loop iteration, we divide that + * number by 2 and rerun the test. + * + * - The second slowest is what type of IO API to perform. We have + * three choices: POSIXIO, and HDF5. + * + * - Then we change the size of the buffer. This information is + * inferred from the number of datasets to create and the number + * of integers to put into each dataset. The backend code figures + * this out. + * + * Return: Nothing + * Programmer: Bill Wendling, 30. October 2001 + * Modifications: + * Added multidimensional testing (Christian Chilan, April, 2008) + */ +static void +run_test_loop(struct options *opts) +{ + parameters parms; + int i; + size_t buf_bytes; + + /* load options into parameter structure */ + parms.num_files = opts->num_files; + parms.num_dsets = opts->num_dsets; + parms.num_iters = opts->num_iters; + parms.rank = opts->dset_rank; + parms.h5_align = opts->h5_alignment; + parms.h5_thresh = opts->h5_threshold; + parms.h5_use_chunks = opts->h5_use_chunks; + parms.h5_extendable = opts->h5_extendable; + parms.h5_write_only = opts->h5_write_only; + parms.verify = opts->verify; + parms.vfd = opts->vfd; + parms.page_buffer_size = opts->page_buffer_size; + parms.page_size = opts->page_size; + + /* load multidimensional options */ + parms.num_bytes = 1; + buf_bytes = 1; + for (i = 0; i < parms.rank; i++) { + parms.buf_size[i] = opts->buf_size[i]; + parms.dset_size[i] = opts->dset_size[i]; + parms.chk_size[i] = opts->chk_size[i]; + parms.order[i] = opts->order[i]; + parms.num_bytes *= opts->dset_size[i]; + buf_bytes *= opts->buf_size[i]; + } + + /* print size information */ + output_report("Transfer Buffer Size (bytes): %d\n", buf_bytes); + output_report("File Size(MB): %.2f\n", ((double)parms.num_bytes) / ONE_MB); + + print_indent(0); + if (opts->io_types & SIO_POSIX) + run_test(POSIXIO, parms, opts); + + print_indent(0); + if (opts->io_types & SIO_HDF5) + run_test(HDF5, parms, opts); +} + +/* + * Function: run_test + * Purpose: Inner loop call to actually run the I/O test. + * Return: Nothing + * Programmer: Bill Wendling, 18. December 2001 + * Modifications: + */ +static int +run_test(iotype iot, parameters parms, struct options *opts) +{ + results res; + register int i, ret_value = SUCCESS; + off_t raw_size; + minmax * write_sys_mm_table = NULL; + minmax * write_mm_table = NULL; + minmax * write_gross_mm_table = NULL; + minmax * write_raw_mm_table = NULL; + minmax * read_sys_mm_table = NULL; + minmax * read_mm_table = NULL; + minmax * read_gross_mm_table = NULL; + minmax * read_raw_mm_table = NULL; + minmax write_sys_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax write_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax write_gross_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax write_raw_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax read_sys_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax read_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax read_gross_mm = {0.0F, 0.0F, 0.0F, 0}; + minmax read_raw_mm = {0.0F, 0.0F, 0.0F, 0}; + + raw_size = (off_t)parms.num_bytes; + parms.io_type = iot; + print_indent(2); + output_report("IO API = "); + + switch (iot) { + case POSIXIO: + output_report("POSIX\n"); + break; + case HDF5: + output_report("HDF5\n"); + break; + default: + /* unknown request */ + HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); + HDassert(0 && "Unknown IO tpe"); + break; + } + + /* allocate space for tables minmax and that it is sufficient */ + /* to initialize all elements to zeros by calloc. */ + write_sys_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + write_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + write_gross_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + write_raw_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + + if (!parms.h5_write_only) { + read_sys_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + read_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + read_gross_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + read_raw_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); + } + + /* Do IO iteration times, collecting statistics each time */ + for (i = 0; i < parms.num_iters; ++i) { + double t; + + do_sio(parms, &res); + + /* gather all of the "sys write" times */ + t = io_time_get(res.timers, HDF5_MPI_WRITE); + get_minmax(&write_sys_mm, t); + + write_sys_mm_table[i] = write_sys_mm; + + /* gather all of the "write" times */ + t = io_time_get(res.timers, HDF5_FINE_WRITE_FIXED_DIMS); + get_minmax(&write_mm, t); + + write_mm_table[i] = write_mm; + + /* gather all of the "write" times from open to close */ + t = io_time_get(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS); + get_minmax(&write_gross_mm, t); + + write_gross_mm_table[i] = write_gross_mm; + + /* gather all of the raw "write" times */ + t = io_time_get(res.timers, HDF5_RAW_WRITE_FIXED_DIMS); + get_minmax(&write_raw_mm, t); + + write_raw_mm_table[i] = write_raw_mm; + + if (!parms.h5_write_only) { + /* gather all of the "mpi read" times */ + t = io_time_get(res.timers, HDF5_MPI_READ); + get_minmax(&read_sys_mm, t); + + read_sys_mm_table[i] = read_sys_mm; + + /* gather all of the "read" times */ + t = io_time_get(res.timers, HDF5_FINE_READ_FIXED_DIMS); + get_minmax(&read_mm, t); + + read_mm_table[i] = read_mm; + + /* gather all of the "read" times from open to close */ + t = io_time_get(res.timers, HDF5_GROSS_READ_FIXED_DIMS); + get_minmax(&read_gross_mm, t); + + read_gross_mm_table[i] = read_gross_mm; + + /* gather all of the raw "read" times */ + t = io_time_get(res.timers, HDF5_RAW_READ_FIXED_DIMS); + get_minmax(&read_raw_mm, t); + + read_raw_mm_table[i] = read_gross_mm; + } + io_time_destroy(res.timers); + } + + /* + * Show various statistics + */ + /* Write statistics */ + /* Print the raw data throughput if desired */ + if (opts->print_raw) { + /* accumulate and output the max, min, and average "raw write" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Raw Data Write details:\n"); + output_all_info(write_raw_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); + } /* end if */ + + /* show sys write statics */ +#if 0 + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("MPI Write details:\n"); + output_all_info(write_sys_mm_table, parms.num_iters, 4); + } +#endif + /* We don't currently output the MPI write results */ + + /* accumulate and output the max, min, and average "write" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write details:\n"); + output_all_info(write_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Write", write_mm_table, parms.num_iters, raw_size); + + /* accumulate and output the max, min, and average "gross write" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Write Open-Close details:\n"); + output_all_info(write_gross_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Write Open-Close", write_gross_mm_table, parms.num_iters, raw_size); + + if (!parms.h5_write_only) { + /* Read statistics */ + /* Print the raw data throughput if desired */ + if (opts->print_raw) { + /* accumulate and output the max, min, and average "raw read" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Raw Data Read details:\n"); + output_all_info(read_raw_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); + } /* end if */ + + /* show mpi read statics */ +#if 0 + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("MPI Read details:\n"); + output_all_info(read_sys_mm_table, parms.num_iters, 4); + } +#endif + /* We don't currently output the MPI read results */ + + /* accumulate and output the max, min, and average "read" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read details:\n"); + output_all_info(read_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Read", read_mm_table, parms.num_iters, raw_size); + + /* accumulate and output the max, min, and average "gross read" times */ + if (sio_debug_level >= 3) { + /* output all of the times for all iterations */ + print_indent(3); + output_report("Read Open-Close details:\n"); + output_all_info(read_gross_mm_table, parms.num_iters, 4); + } + + output_results(opts, "Read Open-Close", read_gross_mm_table, parms.num_iters, raw_size); + } + + /* clean up our mess */ + HDfree(write_sys_mm_table); + HDfree(write_mm_table); + HDfree(write_gross_mm_table); + HDfree(write_raw_mm_table); + + if (!parms.h5_write_only) { + HDfree(read_sys_mm_table); + HDfree(read_mm_table); + HDfree(read_gross_mm_table); + HDfree(read_raw_mm_table); + } + + return ret_value; +} + +/* + * Function: output_all_info + * Purpose: + * Return: Nothing + * Programmer: Bill Wendling, 29. January 2002 + * Modifications: + */ +static void +output_all_info(minmax *mm, int count, int indent_level) +{ + int i; + + for (i = 0; i < count; ++i) { + print_indent(indent_level); + output_report("Iteration %d:\n", i + 1); + print_indent(indent_level + 1); + output_report("Minimum Time: %.2fs\n", mm[i].min); + print_indent(indent_level + 1); + output_report("Maximum Time: %.2fs\n", mm[i].max); + } +} + +/* + * Function: get_minmax + * Purpose: Gather all the min, max and total of val. + * Return: Nothing + * Programmer: Bill Wendling, 21. December 2001 + * Modifications: + * Use MPI_Allreduce to do it. -akc, 2002/01/11 + */ + +static void +get_minmax(minmax *mm, double val) +{ + mm->max = val; + mm->min = val; + mm->sum = val; +} + +/* + * Function: accumulate_minmax_stuff + * Purpose: Accumulate the minimum, maximum, and average of the times + * across all processes. + * Return: TOTAL_MM - the total of all of these. + * Programmer: Bill Wendling, 21. December 2001 + * Modifications: + * Changed to use seconds instead of MB/s - QAK, 5/9/02 + */ +static void +accumulate_minmax_stuff(const minmax *mm, int count, minmax *total_mm) +{ + int i; + + total_mm->sum = 0.0F; + total_mm->max = -DBL_MAX; + total_mm->min = DBL_MAX; + total_mm->num = count; + + for (i = 0; i < count; ++i) { + double m = mm[i].max; + + total_mm->sum += m; + + if (m < total_mm->min) + total_mm->min = m; + + if (m > total_mm->max) + total_mm->max = m; + } +} + +/* + * Function: output_results + * Purpose: Print information about the time & bandwidth for a given + * minmax & # of iterations. + * Return: Nothing + * Programmer: Quincey Koziol, 9. May 2002 + * Modifications: + */ +static void +output_results(const struct options *opts, const char *name, minmax *table, int table_size, off_t data_size) +{ + minmax total_mm; + + accumulate_minmax_stuff(table, table_size, &total_mm); + + print_indent(3); + output_report("%s (%d iteration(s)):\n", name, table_size); + + /* Note: The maximum throughput uses the minimum amount of time & vice versa */ + + print_indent(4); + output_report("Maximum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.min)); + if (opts->print_times) + output_report(" (%7.3f s)\n", total_mm.min); + else + output_report("\n"); + + print_indent(4); + output_report("Average Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.sum / total_mm.num)); + if (opts->print_times) + output_report(" (%7.3f s)\n", (total_mm.sum / total_mm.num)); + else + output_report("\n"); + + print_indent(4); + output_report("Minimum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.max)); + if (opts->print_times) + output_report(" (%7.3f s)\n", total_mm.max); + else + output_report("\n"); +} + +/* + * Function: output_report + * Purpose: Print a line of the report. Only do so if I'm the 0 process. + * Return: Nothing + * Programmer: Bill Wendling, 19. December 2001 + * Modifications: + */ +static void +output_report(const char *fmt, ...) +{ + va_list ap; + + HDva_start(ap, fmt); + HDvfprintf(output, fmt, ap); + HDva_end(ap); +} + +/* + * Function: print_indent + * Purpose: Print spaces to indent a new line of text for pretty printing + * things. + * Return: Nothing + * Programmer: Bill Wendling, 29. October 2001 + * Modifications: + */ +static void +print_indent(register int indent) +{ + indent *= TAB_SPACE; + + for (; indent > 0; --indent) + HDfputc(' ', output); +} + +static void +recover_size_and_print(long long val, const char *end) +{ + if (val >= ONE_KB && (val % ONE_KB) == 0) { + if (val >= ONE_MB && (val % ONE_MB) == 0) { + if (val >= ONE_GB && (val % ONE_GB) == 0) + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "GB%s", + val / ONE_GB, end); + else + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "MB%s", + val / ONE_MB, end); + } + else { + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "KB%s", + val / ONE_KB, end); + } + } + else { + HDfprintf(output, + "%" H5_PRINTF_LL_WIDTH "d" + "%s", + val, end); + } +} + +static void +print_io_api(long io_types) +{ + if (io_types & SIO_POSIX) + HDfprintf(output, "posix "); + if (io_types & SIO_HDF5) + HDfprintf(output, "hdf5 "); + HDfprintf(output, "\n"); +} + +static void +report_parameters(struct options *opts) +{ + int i, rank; + rank = opts->dset_rank; + + print_version("HDF5 Library"); /* print library version */ + HDfprintf(output, "==== Parameters ====\n"); + + HDfprintf(output, "IO API="); + print_io_api(opts->io_types); + + HDfprintf(output, "Number of iterations=%d\n", opts->num_iters); + + HDfprintf(output, "Dataset size="); + + for (i = 0; i < rank; i++) + recover_size_and_print((long long)opts->dset_size[i], " "); + HDfprintf(output, "\n"); + + HDfprintf(output, "Transfer buffer size="); + for (i = 0; i < rank; i++) + recover_size_and_print((long long)opts->buf_size[i], " "); + HDfprintf(output, "\n"); + + if (opts->page_size) { + HDfprintf(output, "Page Aggregation Enabled. Page size = %zu\n", opts->page_size); + if (opts->page_buffer_size) + HDfprintf(output, "Page Buffering Enabled. Page Buffer size = %zu\n", opts->page_buffer_size); + else + HDfprintf(output, "Page Buffering Disabled\n"); + } + else + HDfprintf(output, "Page Aggregation Disabled\n"); + + HDfprintf(output, "Dimension access order="); + for (i = 0; i < rank; i++) + recover_size_and_print((long long)opts->order[i], " "); + HDfprintf(output, "\n"); + + if (opts->io_types & SIO_HDF5) { + + HDfprintf(output, "HDF5 data storage method="); + + if (opts->h5_use_chunks) { + + HDfprintf(output, "Chunked\n"); + HDfprintf(output, "HDF5 chunk size="); + for (i = 0; i < rank; i++) + recover_size_and_print((long long)opts->chk_size[i], " "); + HDfprintf(output, "\n"); + + HDfprintf(output, "HDF5 dataset dimensions="); + if (opts->h5_extendable) { + HDfprintf(output, "Extendable\n"); + } + else { + HDfprintf(output, "Fixed\n"); + } + } + else { + HDfprintf(output, "Contiguous\n"); + } + + HDfprintf(output, "HDF5 file driver="); + if (opts->vfd == sec2) { + HDfprintf(output, "sec2\n"); + } + else if (opts->vfd == stdio) { + HDfprintf(output, "stdio\n"); + } + else if (opts->vfd == core) { + HDfprintf(output, "core\n"); + } + else if (opts->vfd == split) { + HDfprintf(output, "split\n"); + } + else if (opts->vfd == multi) { + HDfprintf(output, "multi\n"); + } + else if (opts->vfd == family) { + HDfprintf(output, "family\n"); + } + else if (opts->vfd == direct) { + HDfprintf(output, "direct\n"); + } + } + + { + char *prefix = HDgetenv("HDF5_PREFIX"); + + HDfprintf(output, "Env HDF5_PREFIX=%s\n", (prefix ? prefix : "not set")); + } + + HDfprintf(output, "==== End of Parameters ====\n"); + HDfprintf(output, "\n"); +} + +/* + * Function: parse_command_line + * Purpose: Parse the command line options and return a STRUCT OPTIONS + * structure which will need to be freed by the calling function. + * Return: Pointer to an OPTIONS structure + * Programmer: Bill Wendling, 31. October 2001 + * Modifications: + * Added multidimensional testing (Christian Chilan, April, 2008) + */ +static struct options * +parse_command_line(int argc, const char *argv[]) +{ + int opt; + struct options *cl_opts; + int i, default_rank, actual_rank, ranks[4]; + + cl_opts = (struct options *)HDmalloc(sizeof(struct options)); + + cl_opts->page_buffer_size = 0; + cl_opts->page_size = 0; + + cl_opts->output_file = NULL; + cl_opts->io_types = 0; /* will set default after parsing options */ + cl_opts->num_iters = 1; + + default_rank = 2; + + cl_opts->dset_rank = 0; + cl_opts->buf_rank = 0; + cl_opts->chk_rank = 0; + cl_opts->order_rank = 0; + + for (i = 0; i < MAX_DIMS; i++) { + cl_opts->buf_size[i] = (size_t)((i + 1) * 10); + cl_opts->dset_size[i] = (hsize_t)((i + 1) * 100); + cl_opts->chk_size[i] = (size_t)((i + 1) * 10); + cl_opts->order[i] = i + 1; + } + + cl_opts->vfd = sec2; + + cl_opts->print_times = FALSE; /* Printing times is off by default */ + cl_opts->print_raw = FALSE; /* Printing raw data throughput is off by default */ + cl_opts->h5_alignment = 1; /* No alignment for HDF5 objects by default */ + cl_opts->h5_threshold = 1; /* No threshold for aligning HDF5 objects by default */ + cl_opts->h5_use_chunks = FALSE; /* Don't chunk the HDF5 dataset by default */ + cl_opts->h5_write_only = FALSE; /* Do both read and write by default */ + cl_opts->h5_extendable = FALSE; /* Use extendable dataset */ + cl_opts->verify = FALSE; /* No Verify data correctness by default */ + + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { + switch ((char)opt) { + case 'a': + cl_opts->h5_alignment = parse_size_directive(H5_optarg); + break; + case 'G': + cl_opts->page_size = parse_size_directive(H5_optarg); + break; + case 'b': + cl_opts->page_buffer_size = parse_size_directive(H5_optarg); + break; + case 'A': { + const char *end = H5_optarg; + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + if (!HDstrcasecmp(buf, "hdf5")) { + cl_opts->io_types |= SIO_HDF5; + } + else if (!HDstrcasecmp(buf, "posix")) { + cl_opts->io_types |= SIO_POSIX; + } + else { + HDfprintf(stderr, "sio_perf: invalid --api option %s\n", buf); + HDexit(EXIT_FAILURE); + } + + if (*end == '\0') + break; + + end++; + } + } + + break; +#if 0 + case 'b': + /* the future "binary" option */ + break; +#endif /* 0 */ + case 'c': + /* Turn on chunked HDF5 dataset creation */ + cl_opts->h5_use_chunks = 1; + { + const char *end = H5_optarg; + int j = 0; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + cl_opts->chk_size[j] = parse_size_directive(buf); + + j++; + + if (*end == '\0') + break; + + end++; + } + cl_opts->chk_rank = j; + } + + break; + + case 'D': { + const char *end = H5_optarg; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + if (HDstrlen(buf) > 1 || HDisdigit(buf[0])) { + size_t j; + + for (j = 0; j < 10 && buf[j] != '\0'; ++j) + if (!HDisdigit(buf[j])) { + HDfprintf(stderr, "sio_perf: invalid --debug option %s\n", buf); + HDexit(EXIT_FAILURE); + } + + sio_debug_level = atoi(buf); + + if (sio_debug_level > 4) + sio_debug_level = 4; + else if (sio_debug_level < 0) + sio_debug_level = 0; + } + else { + switch (*buf) { + case 'r': + /* Turn on raw data throughput info */ + cl_opts->print_raw = TRUE; + break; + case 't': + /* Turn on time printing */ + cl_opts->print_times = TRUE; + break; + case 'v': + /* Turn on verify data correctness*/ + cl_opts->verify = TRUE; + break; + default: + HDfprintf(stderr, "sio_perf: invalid --debug option %s\n", buf); + HDexit(EXIT_FAILURE); + } + } + + if (*end == '\0') + break; + + end++; + } + } + + break; + case 'e': { + const char *end = H5_optarg; + int j = 0; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + cl_opts->dset_size[j] = parse_size_directive(buf); + + j++; + + if (*end == '\0') + break; + + end++; + } + cl_opts->dset_rank = j; + } + + break; + + case 'i': + cl_opts->num_iters = HDatoi(H5_optarg); + break; + case 'o': + cl_opts->output_file = H5_optarg; + break; + case 'T': + cl_opts->h5_threshold = parse_size_directive(H5_optarg); + break; + case 'v': + if (!HDstrcasecmp(H5_optarg, "sec2")) { + cl_opts->vfd = sec2; + } + else if (!HDstrcasecmp(H5_optarg, "stdio")) { + cl_opts->vfd = stdio; + } + else if (!HDstrcasecmp(H5_optarg, "core")) { + cl_opts->vfd = core; + } + else if (!HDstrcasecmp(H5_optarg, "split")) { + cl_opts->vfd = split; + } + else if (!HDstrcasecmp(H5_optarg, "multi")) { + cl_opts->vfd = multi; + } + else if (!HDstrcasecmp(H5_optarg, "family")) { + cl_opts->vfd = family; + } + else if (!HDstrcasecmp(H5_optarg, "direct")) { + cl_opts->vfd = direct; + } + else { + HDfprintf(stderr, "sio_perf: invalid --api option %s\n", H5_optarg); + HDexit(EXIT_FAILURE); + } + break; + case 'w': + cl_opts->h5_write_only = TRUE; + break; + case 't': + cl_opts->h5_extendable = TRUE; + break; + case 'x': { + const char *end = H5_optarg; + int j = 0; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + cl_opts->buf_size[j] = parse_size_directive(buf); + + j++; + + if (*end == '\0') + break; + + end++; + } + cl_opts->buf_rank = j; + } + + break; + + case 'r': { + const char *end = H5_optarg; + int j = 0; + + while (end && *end != '\0') { + char buf[10]; + + HDmemset(buf, '\0', sizeof(buf)); + + for (i = 0; *end != '\0' && *end != ','; ++end) + if (HDisalnum(*end) && i < 10) + buf[i++] = *end; + + cl_opts->order[j] = (int)parse_size_directive(buf); + + j++; + + if (*end == '\0') + break; + + end++; + } + + cl_opts->order_rank = j; + } + + break; + + case 'h': + case '?': + default: + usage(progname); + HDfree(cl_opts); + return NULL; + } + } + + /* perform rank consistency analysis */ + actual_rank = 0; + + ranks[0] = cl_opts->dset_rank; + ranks[1] = cl_opts->buf_rank; + ranks[2] = cl_opts->order_rank; + ranks[3] = cl_opts->chk_rank; + + for (i = 0; i < 4; i++) { + if (ranks[i] > 0) { + if (!actual_rank) { + actual_rank = ranks[i]; + } + else { + if (actual_rank != ranks[i]) + exit(EXIT_FAILURE); + } + } + } + + if (!actual_rank) + actual_rank = default_rank; + + cl_opts->dset_rank = actual_rank; + cl_opts->buf_rank = actual_rank; + cl_opts->order_rank = actual_rank; + cl_opts->chk_rank = actual_rank; + + for (i = 0; i < actual_rank; i++) { + if (cl_opts->order[i] > actual_rank) { + exit(EXIT_FAILURE); + } + } + + /* set default if none specified yet */ + if (!cl_opts->io_types) + cl_opts->io_types = SIO_HDF5 | SIO_POSIX; /* run all API */ + + /* verify parameters sanity. Adjust if needed. */ + /* cap xfer_size with bytes per process */ + if (cl_opts->num_iters <= 0) + cl_opts->num_iters = 1; + + return cl_opts; +} + +/* + * Function: parse_size_directive + * Purpose: Parse the size directive passed on the commandline. The size + * directive is an integer followed by a size indicator: + * + * K, k - Kilobyte + * M, m - Megabyte + * G, g - Gigabyte + * + * Return: The size as a off_t because this is related to file size. + * If an unknown size indicator is used, then the program will + * exit with EXIT_FAILURE as the return value. + * Programmer: Bill Wendling, 18. December 2001 + * Modifications: + */ + +static hsize_t +parse_size_directive(const char *size) +{ + hsize_t s; + char * endptr; + + s = HDstrtoull(size, &endptr, 10); + + if (endptr && *endptr) { + while (*endptr != '\0' && (*endptr == ' ' || *endptr == '\t')) + ++endptr; + + switch (*endptr) { + case 'K': + case 'k': + s *= ONE_KB; + break; + + case 'M': + case 'm': + s *= ONE_MB; + break; + + case 'G': + case 'g': + s *= ONE_GB; + break; + + default: + HDfprintf(stderr, "Illegal size specifier '%c'\n", *endptr); + HDexit(EXIT_FAILURE); + } + } + + return s; +} + +/* + * Function: usage + * Purpose: Print a usage message and then exit. + * Return: Nothing + * Programmer: Bill Wendling, 31. October 2001 + * Modifications: + */ +static void +usage(const char *prog) +{ + print_version(prog); + HDprintf("usage: %s [OPTIONS]\n", prog); + HDprintf(" OPTIONS\n"); + HDprintf(" -h Print an usage message and exit\n"); + HDprintf(" -A AL Which APIs to test\n"); + HDprintf(" [default: all of them]\n"); + HDprintf(" -c SL Selects chunked storage and defines chunks dimensions\n"); + HDprintf(" and sizes\n"); + HDprintf(" [default: Off]\n"); + HDprintf(" -e SL Dimensions and sizes of dataset\n"); + HDprintf(" [default: 100,200]\n"); + HDprintf(" -i N Number of iterations to perform\n"); + HDprintf(" [default: 1]\n"); + HDprintf(" -r NL Dimension access order (see below for description)\n"); + HDprintf(" [default: 1,2]\n"); + HDprintf(" -t Selects extendable dimensions for HDF5 dataset\n"); + HDprintf(" [default: Off]\n"); + HDprintf(" -v VFD Selects file driver for HDF5 access\n"); + HDprintf(" [default: sec2]\n"); + HDprintf(" -w Perform write tests, not the read tests\n"); + HDprintf(" [default: Off]\n"); + HDprintf(" -x SL Dimensions and sizes of the transfer buffer\n"); + HDprintf(" [default: 10,20]\n"); + HDprintf("\n"); + HDprintf(" N - is an integer > 0.\n"); + HDprintf("\n"); + HDprintf(" S - is a size specifier, an integer > 0 followed by a size indicator:\n"); + HDprintf(" K - Kilobyte (%d)\n", ONE_KB); + HDprintf(" M - Megabyte (%d)\n", ONE_MB); + HDprintf(" G - Gigabyte (%d)\n", ONE_GB); + HDprintf("\n"); + HDprintf(" Example: '37M' is 37 megabytes or %d bytes\n", 37 * ONE_MB); + HDprintf("\n"); + HDprintf(" AL - is an API list. Valid values are:\n"); + HDprintf(" hdf5 - HDF5\n"); + HDprintf(" posix - POSIX\n"); + HDprintf("\n"); + HDprintf(" Example: -A posix,hdf5\n"); + HDprintf("\n"); + HDprintf(" NL - is list of integers (N) separated by commas.\n"); + HDprintf("\n"); + HDprintf(" Example: 1,2,3\n"); + HDprintf("\n"); + HDprintf(" SL - is list of size specifiers (S) separated by commas.\n"); + HDprintf("\n"); + HDprintf(" Example: 2K,2K,3K\n"); + HDprintf("\n"); + HDprintf(" The example defines an object (dataset, tranfer buffer) with three\n"); + HDprintf(" dimensions. Be aware that as the number of dimensions increases, the\n"); + HDprintf(" the total size of the object increases exponentially.\n"); + HDprintf("\n"); + HDprintf(" VFD - is an HDF5 file driver specifier. Valid values are:\n"); + HDprintf(" sec2, stdio, core, split, multi, family, direct\n"); + HDprintf("\n"); + HDprintf(" Dimension access order:\n"); + HDprintf(" Data access starts at the cardinal origin of the dataset using the\n"); + HDprintf(" transfer buffer. The next access occurs on a dataset region next to\n"); + HDprintf(" the previous one. For a multidimensional dataset, there are several\n"); + HDprintf(" directions as to where to proceed. This can be specified in the dimension\n"); + HDprintf(" access order. For example, -r 1,2 states that the tool should traverse\n"); + HDprintf(" dimension 1 first, and then dimension 2.\n"); + HDprintf("\n"); + HDprintf(" Environment variables:\n"); + HDprintf(" HDF5_NOCLEANUP Do not remove data files if set [default remove]\n"); + HDprintf(" HDF5_PREFIX Data file prefix\n"); + HDprintf("\n"); + HDfflush(stdout); +} /* end usage() */ diff --git a/tools/src/h5perf/sio_perf.h b/tools/src/h5perf/sio_perf.h new file mode 100644 index 0000000..d998377 --- /dev/null +++ b/tools/src/h5perf/sio_perf.h @@ -0,0 +1,104 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://www.hdfgroup.org/licenses. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +#ifndef SIO_PERF_H +#define SIO_PERF_H + +#ifndef STANDALONE +#include "io_timer.h" +#include "H5private.h" +#include "h5tools.h" +#include "h5tools_utils.h" +#else +#include "io_timer.h" +#include "sio_standalone.h" +#endif + +/* setup the dataset no fill option if this is v1.5 or more */ +#if H5_VERS_MAJOR > 1 || H5_VERS_MINOR > 4 +#define H5_HAVE_NOFILL 1 +#endif + +#define MAX_DIMS 32 + +typedef enum iotype_ { + POSIXIO, + HDF5 + /*NUM_TYPES*/ +} iotype; + +typedef enum vfdtype_ { + sec2, + stdio, + core, + split, + multi, + family, + direct + /*NUM_TYPES*/ +} vfdtype; + +typedef struct parameters_ { + iotype io_type; /* The type of IO test to perform */ + vfdtype vfd; + long num_files; /* Number of files to create */ + long num_dsets; /* Number of datasets to create */ + hsize_t num_bytes; /* Number of bytes in each dset */ + int num_iters; /* Number of times to loop doing the IO */ + int rank; /* Rank of dataset */ + hsize_t dset_size[MAX_DIMS]; /* Dataset size */ + size_t buf_size[MAX_DIMS]; /* Buffer size */ + size_t chk_size[MAX_DIMS]; /* Chunk size */ + int order[MAX_DIMS]; /* Buffer size */ + hsize_t h5_align; /* HDF5 object alignment */ + hsize_t h5_thresh; /* HDF5 object alignment threshold */ + int h5_use_chunks; /* Make HDF5 dataset chunked */ + int h5_extendable; /* Make HDF5 dataset chunked */ + int h5_write_only; /* Perform the write tests only */ + int verify; /* Verify data correctness */ + size_t page_size; + size_t page_buffer_size; +} parameters; + +typedef struct results_ { + herr_t ret_code; + io_time_t *timers; +} results; + +#ifndef SUCCESS +#define SUCCESS 0 +#endif /* !SUCCESS */ + +#ifndef FAIL +#define FAIL -1 +#endif /* !FAIL */ + +extern FILE * output; /* output file */ +extern io_time_t *timer_g; /* timer: global for stub functions */ +extern int sio_debug_level; /* The debug level: + * 0 - Off + * 1 - Minimal + * 2 - Some more + * 3 - Maximal + * 4 - Even More Debugging (timer stuff) + */ +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ + +extern void do_sio(parameters param, results *res); + +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#endif /* SIO_PERF_H */ diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index 1f1b821..3526268 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -31,44 +31,44 @@ const char *outfile = NULL; * Command-line options: The user can specify short or long-named * parameters. */ -static const char * s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:vz:EG:LM:P:S:T:VXW1:2:3:4:5:6:"; -static struct long_options l_opts[] = {{"alignment", require_arg, 'a'}, - {"block", require_arg, 'b'}, - {"compact", require_arg, 'c'}, - {"indexed", require_arg, 'd'}, - {"file", require_arg, 'e'}, - {"filter", require_arg, 'f'}, - {"help", no_arg, 'h'}, - {"infile", require_arg, 'i'}, /* for backward compability */ - {"low", require_arg, 'j'}, - {"high", require_arg, 'k'}, - {"layout", require_arg, 'l'}, - {"minimum", require_arg, 'm'}, - {"native", no_arg, 'n'}, - {"outfile", require_arg, 'o'}, /* for backward compability */ - {"sort_by", require_arg, 'q'}, - {"ssize", require_arg, 's'}, - {"threshold", require_arg, 't'}, - {"ublock", require_arg, 'u'}, - {"verbose", no_arg, 'v'}, - {"sort_order", require_arg, 'z'}, - {"enable-error-stack", no_arg, 'E'}, - {"fs_pagesize", require_arg, 'G'}, - {"latest", no_arg, 'L'}, - {"metadata_block_size", require_arg, 'M'}, - {"fs_persist", require_arg, 'P'}, - {"fs_strategy", require_arg, 'S'}, - {"fs_threshold", require_arg, 'T'}, - {"version", no_arg, 'V'}, - {"merge", no_arg, 'X'}, - {"prune", no_arg, 'W'}, - {"src-vol-value", require_arg, '1'}, - {"src-vol-name", require_arg, '2'}, - {"src-vol-info", require_arg, '3'}, - {"dst-vol-value", require_arg, '4'}, - {"dst-vol-name", require_arg, '5'}, - {"dst-vol-info", require_arg, '6'}, - {NULL, 0, '\0'}}; +static const char * s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:vz:EG:LM:P:S:T:VXW1:2:3:4:5:6:"; +static struct h5_long_options l_opts[] = {{"alignment", require_arg, 'a'}, + {"block", require_arg, 'b'}, + {"compact", require_arg, 'c'}, + {"indexed", require_arg, 'd'}, + {"file", require_arg, 'e'}, + {"filter", require_arg, 'f'}, + {"help", no_arg, 'h'}, + {"infile", require_arg, 'i'}, /* for backward compability */ + {"low", require_arg, 'j'}, + {"high", require_arg, 'k'}, + {"layout", require_arg, 'l'}, + {"minimum", require_arg, 'm'}, + {"native", no_arg, 'n'}, + {"outfile", require_arg, 'o'}, /* for backward compability */ + {"sort_by", require_arg, 'q'}, + {"ssize", require_arg, 's'}, + {"threshold", require_arg, 't'}, + {"ublock", require_arg, 'u'}, + {"verbose", no_arg, 'v'}, + {"sort_order", require_arg, 'z'}, + {"enable-error-stack", no_arg, 'E'}, + {"fs_pagesize", require_arg, 'G'}, + {"latest", no_arg, 'L'}, + {"metadata_block_size", require_arg, 'M'}, + {"fs_persist", require_arg, 'P'}, + {"fs_strategy", require_arg, 'S'}, + {"fs_threshold", require_arg, 'T'}, + {"version", no_arg, 'V'}, + {"merge", no_arg, 'X'}, + {"prune", no_arg, 'W'}, + {"src-vol-value", require_arg, '1'}, + {"src-vol-name", require_arg, '2'}, + {"src-vol-info", require_arg, '3'}, + {"dst-vol-value", require_arg, '4'}, + {"dst-vol-name", require_arg, '5'}, + {"dst-vol-info", require_arg, '6'}, + {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage @@ -363,7 +363,7 @@ read_info(const char *filename, pack_opt_t *options) /* cycle until end of file reached */ while (1) { - if (EOF == fscanf(fp, "%9s", stype)) + if (EOF == HDfscanf(fp, "%9s", stype)) break; /* Info indicator must be for layout or filter */ @@ -378,7 +378,7 @@ read_info(const char *filename, pack_opt_t *options) i = 0; c = '0'; while (c != ' ') { - if (fscanf(fp, "%c", &c) < 0 && HDferror(fp)) { + if (HDfscanf(fp, "%c", &c) < 0 && HDferror(fp)) { error_msg("fscanf error\n"); h5tools_setstatus(EXIT_FAILURE); ret_value = EXIT_FAILURE; @@ -390,7 +390,7 @@ read_info(const char *filename, pack_opt_t *options) c = '0'; /* go until end */ while (c != ' ') { - if (fscanf(fp, "%c", &c) < 0 && HDferror(fp)) { + if (HDfscanf(fp, "%c", &c) < 0 && HDferror(fp)) { error_msg("fscanf error\n"); h5tools_setstatus(EXIT_FAILURE); ret_value = EXIT_FAILURE; @@ -497,18 +497,18 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) HDmemset(&out_vol_info, 0, sizeof(h5tools_vol_info_t)); /* parse command line options */ - while (EOF != (opt = get_option(argc, argv, s_opts, l_opts))) { + while (EOF != (opt = H5_get_option(argc, argv, s_opts, l_opts))) { switch ((char)opt) { /* -i for backward compatibility */ case 'i': - infile = opt_arg; + infile = H5_optarg; has_i++; break; /* -o for backward compatibility */ case 'o': - outfile = opt_arg; + outfile = H5_optarg; has_o++; break; @@ -530,7 +530,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) case 'f': /* parse the -f filter option */ - if (h5repack_addfilter(opt_arg, options) < 0) { + if (h5repack_addfilter(H5_optarg, options) < 0) { error_msg("in parsing filter\n"); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; @@ -540,7 +540,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) case 'l': /* parse the -l layout option */ - if (h5repack_addlayout(opt_arg, options) < 0) { + if (h5repack_addlayout(H5_optarg, options) < 0) { error_msg("in parsing layout\n"); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; @@ -549,9 +549,9 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) break; case 'm': - options->min_comp = HDstrtoull(opt_arg, NULL, 0); + options->min_comp = HDstrtoull(H5_optarg, NULL, 0); if ((int)options->min_comp <= 0) { - error_msg("invalid minimum compress size <%s>\n", opt_arg); + error_msg("invalid minimum compress size <%s>\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -559,8 +559,8 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) break; case 'e': - if ((ret_value = read_info(opt_arg, options)) < 0) { - error_msg("failed to read from repack options file <%s>\n", opt_arg); + if ((ret_value = read_info(H5_optarg, options)) < 0) { + error_msg("failed to read from repack options file <%s>\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -576,7 +576,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) break; case 'j': - bound = HDatoi(opt_arg); + bound = HDatoi(H5_optarg); if (bound < H5F_LIBVER_EARLIEST || bound > H5F_LIBVER_LATEST) { error_msg("in parsing low bound\n"); h5tools_setstatus(EXIT_FAILURE); @@ -587,7 +587,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) break; case 'k': - bound = HDatoi(opt_arg); + bound = HDatoi(H5_optarg); if (bound < H5F_LIBVER_EARLIEST || bound > H5F_LIBVER_LATEST) { error_msg("in parsing high bound\n"); h5tools_setstatus(EXIT_FAILURE); @@ -606,13 +606,13 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) break; case 'c': - options->grp_compact = HDatoi(opt_arg); + options->grp_compact = HDatoi(H5_optarg); if (options->grp_compact > 0) options->latest = TRUE; /* must use latest format */ break; case 'd': - options->grp_indexed = HDatoi(opt_arg); + options->grp_indexed = HDatoi(H5_optarg); if (options->grp_indexed > 0) options->latest = TRUE; /* must use latest format */ break; @@ -620,10 +620,10 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) case 's': { int idx = 0; int ssize = 0; - char *msgPtr = HDstrchr(opt_arg, ':'); + char *msgPtr = HDstrchr(H5_optarg, ':'); options->latest = TRUE; /* must use latest format */ if (msgPtr == NULL) { - ssize = HDatoi(opt_arg); + ssize = HDatoi(H5_optarg); for (idx = 0; idx < 5; idx++) options->msg_size[idx] = ssize; } @@ -632,7 +632,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) HDstrcpy(msgType, msgPtr + 1); msgPtr[0] = '\0'; - ssize = HDatoi(opt_arg); + ssize = HDatoi(H5_optarg); if (!HDstrncmp(msgType, "dspace", 6)) options->msg_size[0] = ssize; else if (!HDstrncmp(msgType, "dtype", 5)) @@ -647,25 +647,25 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) } break; case 'u': - options->ublock_filename = opt_arg; + options->ublock_filename = H5_optarg; break; case 'b': - options->ublock_size = (hsize_t)HDatol(opt_arg); + options->ublock_size = (hsize_t)HDatol(H5_optarg); break; case 'M': - options->meta_block_size = (hsize_t)HDatol(opt_arg); + options->meta_block_size = (hsize_t)HDatol(H5_optarg); break; case 't': - options->threshold = (hsize_t)HDatol(opt_arg); + options->threshold = (hsize_t)HDatol(H5_optarg); break; case 'a': - options->alignment = HDstrtoull(opt_arg, NULL, 0); + options->alignment = HDstrtoull(H5_optarg, NULL, 0); if (options->alignment < 1) { - error_msg("invalid alignment size\n", opt_arg); + error_msg("invalid alignment size\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -675,7 +675,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) case 'S': { char strategy[MAX_NC_NAME]; - HDstrcpy(strategy, opt_arg); + HDstrcpy(strategy, H5_optarg); if (!HDstrcmp(strategy, "FSM_AGGR")) options->fs_strategy = H5F_FSPACE_STRATEGY_FSM_AGGR; else if (!HDstrcmp(strategy, "PAGE")) @@ -685,7 +685,7 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) else if (!HDstrcmp(strategy, "NONE")) options->fs_strategy = H5F_FSPACE_STRATEGY_NONE; else { - error_msg("invalid file space management strategy\n", opt_arg); + error_msg("invalid file space management strategy\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -696,29 +696,29 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) } break; case 'P': - options->fs_persist = HDatoi(opt_arg); + options->fs_persist = HDatoi(H5_optarg); if (options->fs_persist == 0) /* To distinguish the "specified" zero value */ options->fs_persist = -1; break; case 'T': - options->fs_threshold = HDatol(opt_arg); + options->fs_threshold = HDatol(H5_optarg); if (options->fs_threshold == 0) /* To distinguish the "specified" zero value */ options->fs_threshold = -1; break; case 'G': - options->fs_pagesize = HDstrtoll(opt_arg, NULL, 0); + options->fs_pagesize = HDstrtoll(H5_optarg, NULL, 0); if (options->fs_pagesize == 0) /* To distinguish the "specified" zero value */ options->fs_pagesize = -1; break; case 'q': - if (H5_INDEX_UNKNOWN == (sort_by = set_sort_by(opt_arg))) { - error_msg("failed to set sort by form <%s>\n", opt_arg); + if (H5_INDEX_UNKNOWN == (sort_by = set_sort_by(H5_optarg))) { + error_msg("failed to set sort by form <%s>\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -726,8 +726,8 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) break; case 'z': - if (H5_ITER_UNKNOWN == (sort_order = set_sort_order(opt_arg))) { - error_msg("failed to set sort order form <%s>\n", opt_arg); + if (H5_ITER_UNKNOWN == (sort_order = set_sort_order(H5_optarg))) { + error_msg("failed to set sort order form <%s>\n", H5_optarg); h5tools_setstatus(EXIT_FAILURE); ret_value = -1; goto done; @@ -740,34 +740,34 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) case '1': in_vol_info.type = VOL_BY_VALUE; - in_vol_info.u.value = (H5VL_class_value_t)HDatoi(opt_arg); + in_vol_info.u.value = (H5VL_class_value_t)HDatoi(H5_optarg); custom_in_fapl = TRUE; break; case '2': in_vol_info.type = VOL_BY_NAME; - in_vol_info.u.name = opt_arg; + in_vol_info.u.name = H5_optarg; custom_in_fapl = TRUE; break; case '3': - in_vol_info.info_string = opt_arg; + in_vol_info.info_string = H5_optarg; break; case '4': out_vol_info.type = VOL_BY_VALUE; - out_vol_info.u.value = (H5VL_class_value_t)HDatoi(opt_arg); + out_vol_info.u.value = (H5VL_class_value_t)HDatoi(H5_optarg); custom_out_fapl = TRUE; break; case '5': out_vol_info.type = VOL_BY_NAME; - out_vol_info.u.name = opt_arg; + out_vol_info.u.name = H5_optarg; custom_out_fapl = TRUE; break; case '6': - out_vol_info.info_string = opt_arg; + out_vol_info.info_string = H5_optarg; break; default: @@ -777,9 +777,9 @@ parse_command_line(int argc, const char **argv, pack_opt_t *options) /* If neither -i nor -o given, get in and out files positionally */ if (0 == (has_i + has_o)) { - if (argv[opt_ind] != NULL && argv[opt_ind + 1] != NULL) { - infile = argv[opt_ind]; - outfile = argv[opt_ind + 1]; + if (argv[H5_optind] != NULL && argv[H5_optind + 1] != NULL) { + infile = argv[H5_optind]; + outfile = argv[H5_optind + 1]; if (!HDstrcmp(infile, outfile)) { error_msg("file names cannot be the same\n"); diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index 853f3ce..d97a628 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -171,106 +171,23 @@ struct handler_t { static const char *s_opts = "Aa:Ddm:EFfhGgl:sSTO:Vw:H:"; /* e.g. "filemetadata" has to precede "file"; "groupmetadata" has to precede "group" etc. */ -static struct long_options l_opts[] = {{"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, - {"filemetadata", no_arg, 'F'}, - {"filemetadat", no_arg, 'F'}, - {"filemetada", no_arg, 'F'}, - {"filemetad", no_arg, 'F'}, - {"filemeta", no_arg, 'F'}, - {"filemet", no_arg, 'F'}, - {"fileme", no_arg, 'F'}, - {"filem", no_arg, 'F'}, - {"file", no_arg, 'f'}, - {"fil", no_arg, 'f'}, - {"fi", no_arg, 'f'}, - {"groupmetadata", no_arg, 'G'}, - {"groupmetadat", no_arg, 'G'}, - {"groupmetada", no_arg, 'G'}, - {"groupmetad", no_arg, 'G'}, - {"groupmeta", no_arg, 'G'}, - {"groupmet", no_arg, 'G'}, - {"groupme", no_arg, 'G'}, - {"groupm", no_arg, 'G'}, - {"group", no_arg, 'g'}, - {"grou", no_arg, 'g'}, - {"gro", no_arg, 'g'}, - {"gr", no_arg, 'g'}, - {"links", require_arg, 'l'}, - {"link", require_arg, 'l'}, - {"lin", require_arg, 'l'}, - {"li", require_arg, 'l'}, - {"dsetmetadata", no_arg, 'D'}, - {"dsetmetadat", no_arg, 'D'}, - {"dsetmetada", no_arg, 'D'}, - {"dsetmetad", no_arg, 'D'}, - {"dsetmeta", no_arg, 'D'}, - {"dsetmet", no_arg, 'D'}, - {"dsetme", no_arg, 'D'}, - {"dsetm", no_arg, 'D'}, - {"dset", no_arg, 'd'}, - {"dse", no_arg, 'd'}, - {"ds", no_arg, 'd'}, - {"dims", require_arg, 'm'}, - {"dim", require_arg, 'm'}, - {"di", require_arg, 'm'}, - {"dtypemetadata", no_arg, 'T'}, - {"dtypemetadat", no_arg, 'T'}, - {"dtypemetada", no_arg, 'T'}, - {"dtypemetad", no_arg, 'T'}, - {"dtypemeta", no_arg, 'T'}, - {"dtypemet", no_arg, 'T'}, - {"dtypeme", no_arg, 'T'}, - {"dtypem", no_arg, 'T'}, - {"dtype", no_arg, 'T'}, - {"dtyp", no_arg, 'T'}, - {"dty", no_arg, 'T'}, - {"dt", no_arg, 'T'}, - {"object", require_arg, 'O'}, - {"objec", require_arg, 'O'}, - {"obje", require_arg, 'O'}, - {"obj", require_arg, 'O'}, - {"ob", require_arg, 'O'}, - {"version", no_arg, 'V'}, - {"versio", no_arg, 'V'}, - {"versi", no_arg, 'V'}, - {"vers", no_arg, 'V'}, - {"ver", no_arg, 'V'}, - {"ve", no_arg, 'V'}, - {"attribute", no_arg, 'A'}, - {"attribut", no_arg, 'A'}, - {"attribu", no_arg, 'A'}, - {"attrib", no_arg, 'A'}, - {"attri", no_arg, 'A'}, - {"attr", no_arg, 'A'}, - {"att", no_arg, 'A'}, - {"at", no_arg, 'A'}, - {"enable-error-stack", no_arg, 'E'}, - {"numattrs", require_arg, 'a'}, - {"numattr", require_arg, 'a'}, - {"numatt", require_arg, 'a'}, - {"numat", require_arg, 'a'}, - {"numa", require_arg, 'a'}, - {"num", require_arg, 'a'}, - {"nu", require_arg, 'a'}, - {"freespace", no_arg, 's'}, - {"freespac", no_arg, 's'}, - {"freespa", no_arg, 's'}, - {"freesp", no_arg, 's'}, - {"frees", no_arg, 's'}, - {"free", no_arg, 's'}, - {"fre", no_arg, 's'}, - {"fr", no_arg, 's'}, - {"summary", no_arg, 'S'}, - {"summar", no_arg, 'S'}, - {"summa", no_arg, 'S'}, - {"summ", no_arg, 'S'}, - {"sum", no_arg, 'S'}, - {"su", no_arg, 'S'}, - {"s3-cred", require_arg, 'w'}, - {"hdfs-attrs", require_arg, 'H'}, - {NULL, 0, '\0'}}; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, + {"filemetadata", no_arg, 'F'}, + {"groupmetadata", no_arg, 'G'}, + {"links", require_arg, 'l'}, + {"dsetmetadata", no_arg, 'D'}, + {"dims", require_arg, 'm'}, + {"dtypemetadata", no_arg, 'T'}, + {"object", require_arg, 'O'}, + {"version", no_arg, 'V'}, + {"attribute", no_arg, 'A'}, + {"enable-error-stack", no_arg, 'E'}, + {"numattrs", require_arg, 'a'}, + {"freespace", no_arg, 's'}, + {"summary", no_arg, 'S'}, + {"s3-cred", require_arg, 'w'}, + {"hdfs-attrs", require_arg, 'H'}, + {NULL, 0, '\0'}}; static void leave(int ret) @@ -920,7 +837,7 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) struct handler_t *hand = NULL; /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'h': usage(h5tools_getprogname()); @@ -959,8 +876,8 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) break; case 'l': - if (opt_arg) { - sgroups_threshold = HDatoi(opt_arg); + if (H5_optarg) { + sgroups_threshold = HDatoi(H5_optarg); if (sgroups_threshold < 1) { error_msg("Invalid threshold for small groups\n"); goto error; @@ -982,8 +899,8 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) break; case 'm': - if (opt_arg) { - sdsets_threshold = HDatoi(opt_arg); + if (H5_optarg) { + sdsets_threshold = HDatoi(H5_optarg); if (sdsets_threshold < 1) { error_msg("Invalid threshold for small datasets\n"); goto error; @@ -1005,8 +922,8 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) break; case 'a': - if (opt_arg) { - sattrs_threshold = HDatoi(opt_arg); + if (H5_optarg) { + sattrs_threshold = HDatoi(H5_optarg); if (sattrs_threshold < 1) { error_msg("Invalid threshold for small # of attributes\n"); goto error; @@ -1046,7 +963,7 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) /* Store object names */ for (u = 0; u < hand->obj_count; u++) - if (NULL == (hand->obj[u] = HDstrdup(opt_arg))) { + if (NULL == (hand->obj[u] = HDstrdup(H5_optarg))) { error_msg("unable to allocate memory for object name\n"); goto error; } /* end if */ @@ -1054,7 +971,7 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) case 'w': #ifdef H5_HAVE_ROS3_VFD - if (h5tools_parse_ros3_fapl_tuple(opt_arg, ',', &ros3_fa) < 0) { + if (h5tools_parse_ros3_fapl_tuple(H5_optarg, ',', &ros3_fa) < 0) { error_msg("failed to parse S3 VFD credential info\n"); goto error; } @@ -1068,7 +985,7 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) case 'H': #ifdef H5_HAVE_LIBHDFS - if (h5tools_parse_hdfs_fapl_tuple(opt_arg, ',', &hdfs_fa) < 0) { + if (h5tools_parse_hdfs_fapl_tuple(H5_optarg, ',', &hdfs_fa) < 0) { error_msg("failed to parse HDFS VFD configuration info\n"); goto error; } @@ -1087,7 +1004,7 @@ parse_command_line(int argc, const char *argv[], struct handler_t **hand_ret) } /* end while */ /* check for file name to be processed */ - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing file name\n"); usage(h5tools_getprogname()); goto error; @@ -1811,7 +1728,7 @@ main(int argc, const char *argv[]) } } - fname = argv[opt_ind]; + fname = argv[H5_optind]; /* Check for filename given */ if (fname) { diff --git a/tools/src/misc/h5clear.c b/tools/src/misc/h5clear.c index ac7c0ca..008dae3 100644 --- a/tools/src/misc/h5clear.c +++ b/tools/src/misc/h5clear.c @@ -44,40 +44,11 @@ static hsize_t increment = DEFAULT_INCREMENT; /* * Command-line options: only publicize long options */ -static const char * s_opts = "hVsmzi*"; -static struct long_options l_opts[] = {{"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, - {"version", no_arg, 'V'}, - {"version", no_arg, 'V'}, - {"versio", no_arg, 'V'}, - {"versi", no_arg, 'V'}, - {"vers", no_arg, 'V'}, - {"status", no_arg, 's'}, - {"statu", no_arg, 's'}, - {"stat", no_arg, 's'}, - {"sta", no_arg, 's'}, - {"st", no_arg, 's'}, - {"image", no_arg, 'm'}, - {"imag", no_arg, 'm'}, - {"ima", no_arg, 'm'}, - {"im", no_arg, 'm'}, - {"filesize", no_arg, 'z'}, - {"filesiz", no_arg, 'z'}, - {"filesi", no_arg, 'z'}, - {"files", no_arg, 'z'}, - {"file", no_arg, 'z'}, - {"fil", no_arg, 'z'}, - {"fi", no_arg, 'z'}, - {"increment", optional_arg, 'i'}, - {"incremen", optional_arg, 'i'}, - {"increme", optional_arg, 'i'}, - {"increm", optional_arg, 'i'}, - {"incre", optional_arg, 'i'}, - {"incr", optional_arg, 'i'}, - {"inc", optional_arg, 'i'}, - {"in", optional_arg, 'i'}, - {NULL, 0, '\0'}}; +static const char * s_opts = "hVsmzi*"; +static struct h5_long_options l_opts[] = { + {"help", no_arg, 'h'}, {"version", no_arg, 'V'}, {"status", no_arg, 's'}, + {"image", no_arg, 'm'}, {"filesize", no_arg, 'z'}, {"increment", optional_arg, 'i'}, + {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage @@ -150,7 +121,7 @@ parse_command_line(int argc, const char **argv) } /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'h': usage(h5tools_getprogname()); @@ -176,12 +147,12 @@ parse_command_line(int argc, const char **argv) case 'i': increment_eoa_eof = TRUE; - if (opt_arg != NULL) { - if (HDatoi(opt_arg) < 0) { + if (H5_optarg != NULL) { + if (HDatoi(H5_optarg) < 0) { usage(h5tools_getprogname()); goto done; } - increment = (hsize_t)HDatoi(opt_arg); + increment = (hsize_t)HDatoi(H5_optarg); } break; @@ -193,14 +164,14 @@ parse_command_line(int argc, const char **argv) } /* end while */ /* check for file name to be processed */ - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing file name\n"); usage(h5tools_getprogname()); h5tools_setstatus(EXIT_FAILURE); goto error; } /* end if */ - fname_g = HDstrdup(argv[opt_ind]); + fname_g = HDstrdup(argv[H5_optind]); done: return (0); diff --git a/tools/src/misc/h5mkgrp.c b/tools/src/misc/h5mkgrp.c index 516191c..a85ee4d 100644 --- a/tools/src/misc/h5mkgrp.c +++ b/tools/src/misc/h5mkgrp.c @@ -22,8 +22,8 @@ int d_status = EXIT_SUCCESS; /* command-line options: short and long-named parameters */ -static const char * s_opts = "hlpvV"; -static struct long_options l_opts[] = { +static const char * s_opts = "hlpvV"; +static struct h5_long_options l_opts[] = { {"help", no_arg, 'h'}, {"latest", no_arg, 'l'}, {"parents", no_arg, 'p'}, {"verbose", no_arg, 'v'}, {"version", no_arg, 'V'}, {"vol-value", require_arg, '1'}, {"vol-name", require_arg, '2'}, {"vol-info", require_arg, '3'}, {NULL, 0, '\0'}}; @@ -140,7 +140,7 @@ parse_command_line(int argc, const char *argv[], mkgrp_opt_t *options) HDmemset(&vol_info, 0, sizeof(h5tools_vol_info_t)); /* Parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { /* Display 'help' */ case 'h': @@ -171,18 +171,18 @@ parse_command_line(int argc, const char *argv[], mkgrp_opt_t *options) case '1': vol_info.type = VOL_BY_VALUE; - vol_info.u.value = (H5VL_class_value_t)HDatoi(opt_arg); + vol_info.u.value = (H5VL_class_value_t)HDatoi(H5_optarg); custom_fapl = TRUE; break; case '2': vol_info.type = VOL_BY_NAME; - vol_info.u.name = opt_arg; + vol_info.u.name = H5_optarg; custom_fapl = TRUE; break; case '3': - vol_info.info_string = opt_arg; + vol_info.info_string = H5_optarg; break; /* Bad command line argument */ @@ -193,33 +193,33 @@ parse_command_line(int argc, const char *argv[], mkgrp_opt_t *options) } /* end while */ /* Check for file name to be processed */ - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing file name\n"); usage(h5tools_getprogname()); leave(EXIT_FAILURE); } /* Retrieve file name */ - options->fname = HDstrdup(argv[opt_ind]); - opt_ind++; + options->fname = HDstrdup(argv[H5_optind]); + H5_optind++; /* Check for group(s) to be created */ - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing group name(s)\n"); usage(h5tools_getprogname()); leave(EXIT_FAILURE); } /* Allocate space for the group name pointers */ - options->ngroups = (size_t)(argc - opt_ind); + options->ngroups = (size_t)(argc - H5_optind); options->groups = (char **)HDmalloc(options->ngroups * sizeof(char *)); /* Retrieve the group names */ curr_group = 0; - while (opt_ind < argc) { - options->groups[curr_group] = HDstrdup(argv[opt_ind]); + while (H5_optind < argc) { + options->groups[curr_group] = HDstrdup(argv[H5_optind]); curr_group++; - opt_ind++; + H5_optind++; } /* Setup a custom fapl for file accesses */ diff --git a/tools/test/h5jam/getub.c b/tools/test/h5jam/getub.c index a9b4437..fd21d6c 100644 --- a/tools/test/h5jam/getub.c +++ b/tools/test/h5jam/getub.c @@ -21,9 +21,9 @@ void parse_command_line(int argc, const char *argv[]); #define PROGRAM_NAME "getub" char *nbytes = NULL; -static const char * s_opts = "c:"; /* add more later ? */ -static struct long_options l_opts[] = {{"c", require_arg, 'c'}, /* input file */ - {NULL, 0, '\0'}}; +static const char * s_opts = "c:"; /* add more later ? */ +static struct h5_long_options l_opts[] = {{"c", require_arg, 'c'}, /* input file */ + {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage @@ -57,10 +57,10 @@ parse_command_line(int argc, const char *argv[]) int opt; /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'c': - nbytes = HDstrdup(opt_arg); + nbytes = HDstrdup(H5_optarg); break; case '?': default: @@ -69,7 +69,7 @@ parse_command_line(int argc, const char *argv[]) } /* end switch */ } /* end while */ - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing file name\n"); usage(h5tools_getprogname()); HDexit(EXIT_FAILURE); @@ -100,16 +100,16 @@ main(int argc, const char *argv[]) goto error; } /* end if */ - if (argc <= (opt_ind)) { + if (argc <= (H5_optind)) { error_msg("missing file name\n"); usage(h5tools_getprogname()); goto error; } /* end if */ - filename = HDstrdup(argv[opt_ind]); + filename = HDstrdup(argv[H5_optind]); size = 0; - if (EOF == (res = sscanf(nbytes, "%u", &size))) { + if (EOF == (res = HDsscanf(nbytes, "%u", &size))) { /* fail */ error_msg("missing file name\n"); usage(h5tools_getprogname()); diff --git a/tools/test/h5jam/h5jamgentest.c b/tools/test/h5jam/h5jamgentest.c index 1da6b63..5632cc0 100644 --- a/tools/test/h5jam/h5jamgentest.c +++ b/tools/test/h5jam/h5jamgentest.c @@ -31,24 +31,7 @@ #define UBTXT3 "u511.txt" #define UBTXT4 "u512.txt" #define UBTXT5 "u513.txt" -/* not used yet -#define UBTXT6 "u1023.txt" -#define UBTXT7 "u1024.txt" -#define UBTXT8 "u1025.txt" -#define UBTXT9 "u2047.txt" -#define UBTXT10 "u2048.txt" -#define UBTXT11 "u2049.txt" -#define UBBIN1 "u0.dat" -#define UBBIN2 "u10.dat" -#define UBBIN3 "u511.dat" -#define UBBIN4 "u512.dat" -#define UBBIN5 "u513.dat" -*/ -/* not used yet -#define FILE1 "tnull.h5" -#define FILE2 "tnullwithub.h5" -*/ /* tall is same as dumper test */ #define FILE7 "tall.h5" #define FILE8 "twithub.h5" @@ -66,24 +49,13 @@ char pattern[11] = "abcdefghij"; #define BUF_SIZE 1024 -/* Element selection information */ - -typedef enum { RED, GREEN, BLUE, WHITE, BLACK } enumtype; - -/* Compound datatype */ -typedef struct s1_t { - unsigned int a; - unsigned int b; - float c; -} s1_t; - /* A UD link traversal function. Shouldn't actually be called. */ static hid_t UD_traverse(const char H5_ATTR_UNUSED *link_name, hid_t H5_ATTR_UNUSED cur_group, const void H5_ATTR_UNUSED *udata, size_t H5_ATTR_UNUSED udata_size, hid_t H5_ATTR_UNUSED lapl_id, hid_t H5_ATTR_UNUSED dxpl_id) { - return -1; + return H5I_INVALID_HID; } #define MY_LINKCLASS 187 @@ -115,261 +87,340 @@ g1.2.1 : slink g2 : dset2.1 dset2.2 udlink */ - -static void +static herr_t gent_ub(const char *filename, size_t ub_size, size_t ub_fill) { - hid_t fid, group, attr, dataset, space; - hid_t create_plist; + hid_t fid = H5I_INVALID_HID; + hid_t group = H5I_INVALID_HID; + hid_t attr = H5I_INVALID_HID; + hid_t dataset = H5I_INVALID_HID; + hid_t space = H5I_INVALID_HID; + hid_t create_plist = H5I_INVALID_HID; hsize_t dims[2]; int data[2][2], dset1[10][10], dset2[20]; char buf[BUF_SIZE]; int i, j; size_t u; float dset2_1[10], dset2_2[3][5]; - int fd; - char * bp; + int fd = -1; if (ub_size > 0) { - create_plist = H5Pcreate(H5P_FILE_CREATE); - H5Pset_userblock(create_plist, (hsize_t)ub_size); - fid = H5Fcreate(filename, H5F_ACC_TRUNC, create_plist, H5P_DEFAULT); + if ((create_plist = H5Pcreate(H5P_FILE_CREATE)) < 0) + goto error; + if (H5Pset_userblock(create_plist, (hsize_t)ub_size) < 0) + goto error; + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, create_plist, H5P_DEFAULT)) < 0) + goto error; } else { - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + if ((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; } - /* create groups */ - group = H5Gcreate2(fid, "/g1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + /* Create groups */ + if ((group = H5Gcreate2(fid, "/g1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - group = H5Gcreate2(fid, "/g2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + if ((group = H5Gcreate2(fid, "/g2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - group = H5Gcreate2(fid, "/g1/g1.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + if ((group = H5Gcreate2(fid, "/g1/g1.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - group = H5Gcreate2(fid, "/g1/g1.2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + if ((group = H5Gcreate2(fid, "/g1/g1.2", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - group = H5Gcreate2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); + if ((group = H5Gcreate2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; - /* root attributes */ - group = H5Gopen2(fid, "/", H5P_DEFAULT); + /* Root attributes */ + if ((group = H5Gopen2(fid, "/", H5P_DEFAULT)) < 0) + goto error; dims[0] = 10; - space = H5Screate_simple(1, dims, NULL); - attr = H5Acreate2(group, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT); - HDsprintf(buf, "abcdefghi"); - H5Awrite(attr, H5T_NATIVE_SCHAR, buf); - H5Sclose(space); - H5Aclose(attr); - - dims[0] = 2; - dims[1] = 2; - space = H5Screate_simple(2, dims, NULL); - attr = H5Acreate2(group, "attr2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((attr = H5Acreate2(group, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (HDsprintf(buf, "abcdefghi") < 0) + goto error; + if (H5Awrite(attr, H5T_NATIVE_SCHAR, buf) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Aclose(attr) < 0) + goto error; + + dims[0] = 2; + dims[1] = 2; + if ((space = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + if ((attr = H5Acreate2(group, "attr2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; data[0][0] = 0; data[0][1] = 1; data[1][0] = 2; data[1][1] = 3; - H5Awrite(attr, H5T_NATIVE_INT, data); - H5Sclose(space); - H5Aclose(attr); + if (H5Awrite(attr, H5T_NATIVE_INT, data) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Aclose(attr) < 0) + goto error; - H5Gclose(group); + if (H5Gclose(group) < 0) + goto error; - group = H5Gopen2(fid, "/g1/g1.1", H5P_DEFAULT); + if ((group = H5Gopen2(fid, "/g1/g1.1", H5P_DEFAULT)) < 0) + goto error; - /* dset1.1.1 */ + /* Dataset 1.1.1 */ dims[0] = 10; dims[1] = 10; - space = H5Screate_simple(2, dims, NULL); - dataset = H5Dcreate2(group, "dset1.1.1", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + if ((dataset = + H5Dcreate2(group, "dset1.1.1", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < 10; i++) for (j = 0; j < 10; j++) dset1[i][j] = j * i; - H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset1); - H5Sclose(space); + if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset1) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; - /* attributes of dset1.1.1 */ + /* Attributes of dset1.1.1 */ dims[0] = 27; - space = H5Screate_simple(1, dims, NULL); - attr = H5Acreate2(dataset, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT); - HDsprintf(buf, "1st attribute of dset1.1.1"); - H5Awrite(attr, H5T_NATIVE_SCHAR, buf); - H5Sclose(space); - H5Aclose(attr); + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((attr = H5Acreate2(dataset, "attr1", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (HDsprintf(buf, "1st attribute of dset1.1.1") < 0) + goto error; + if (H5Awrite(attr, H5T_NATIVE_SCHAR, buf) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Aclose(attr) < 0) + goto error; dims[0] = 27; - space = H5Screate_simple(1, dims, NULL); - attr = H5Acreate2(dataset, "attr2", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT); - HDsprintf(buf, "2nd attribute of dset1.1.1"); - H5Awrite(attr, H5T_NATIVE_SCHAR, buf); - H5Sclose(space); - H5Aclose(attr); - - H5Dclose(dataset); - - /* dset1.1.2 */ + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((attr = H5Acreate2(dataset, "attr2", H5T_STD_I8BE, space, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; + if (HDsprintf(buf, "2nd attribute of dset1.1.1") < 0) + goto error; + if (H5Awrite(attr, H5T_NATIVE_SCHAR, buf) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Aclose(attr) < 0) + goto error; + + if (H5Dclose(dataset) < 0) + goto error; + + /* Dataset 1.1.2 */ dims[0] = 20; - space = H5Screate_simple(1, dims, NULL); - dataset = H5Dcreate2(group, "dset1.1.2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((dataset = + H5Dcreate2(group, "dset1.1.2", H5T_STD_I32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < 20; i++) dset2[i] = i; - H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2); - H5Sclose(space); - H5Dclose(dataset); - - H5Gclose(group); - - /* external link */ - H5Lcreate_external("somefile", "somepath", fid, "/g1/g1.2/extlink", H5P_DEFAULT, H5P_DEFAULT); - - /* soft link */ - group = H5Gopen2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT); - H5Lcreate_soft("somevalue", group, "slink", H5P_DEFAULT, H5P_DEFAULT); - H5Gclose(group); - - group = H5Gopen2(fid, "/g2", H5P_DEFAULT); - - /* dset2.1 */ + if (H5Dwrite(dataset, H5T_NATIVE_INT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Dclose(dataset) < 0) + goto error; + + if (H5Gclose(group) < 0) + goto error; + + /* External link */ + if (H5Lcreate_external("somefile", "somepath", fid, "/g1/g1.2/extlink", H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + + /* Soft link */ + if ((group = H5Gopen2(fid, "/g1/g1.2/g1.2.1", H5P_DEFAULT)) < 0) + goto error; + if (H5Lcreate_soft("somevalue", group, "slink", H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + if (H5Gclose(group) < 0) + goto error; + + if ((group = H5Gopen2(fid, "/g2", H5P_DEFAULT)) < 0) + goto error; + + /* Dataset 2.1 */ dims[0] = 10; - space = H5Screate_simple(1, dims, NULL); - dataset = H5Dcreate2(group, "dset2.1", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(1, dims, NULL)) < 0) + goto error; + if ((dataset = + H5Dcreate2(group, "dset2.1", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < 10; i++) dset2_1[i] = (float)((float)i * 0.1F + 1.0F); - H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_1); - H5Sclose(space); - H5Dclose(dataset); - - /* dset2.2 */ + if (H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_1) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Dclose(dataset) < 0) + goto error; + + /* Dataset 2.2 */ dims[0] = 3; dims[1] = 5; - space = H5Screate_simple(2, dims, NULL); - dataset = H5Dcreate2(group, "dset2.2", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + if ((space = H5Screate_simple(2, dims, NULL)) < 0) + goto error; + if ((dataset = + H5Dcreate2(group, "dset2.2", H5T_IEEE_F32BE, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) < 0) + goto error; for (i = 0; i < 3; i++) for (j = 0; j < 5; j++) dset2_2[i][j] = (float)(((float)i + 1.0F) * (float)j * 0.1F); - H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_2); - H5Sclose(space); - H5Dclose(dataset); - - H5Gclose(group); - - /* user-defined link */ - H5Lregister(UD_link_class); - H5Lcreate_ud(fid, "/g2/udlink", (H5L_type_t)MY_LINKCLASS, NULL, (size_t)0, H5P_DEFAULT, H5P_DEFAULT); - - H5Fclose(fid); + if (H5Dwrite(dataset, H5T_NATIVE_FLOAT, H5S_ALL, H5S_ALL, H5P_DEFAULT, dset2_2) < 0) + goto error; + if (H5Sclose(space) < 0) + goto error; + if (H5Dclose(dataset) < 0) + goto error; + + if (H5Gclose(group) < 0) + goto error; + + /* User-defined link */ + if (H5Lregister(UD_link_class) < 0) + goto error; + if (H5Lcreate_ud(fid, "/g2/udlink", (H5L_type_t)MY_LINKCLASS, NULL, (size_t)0, H5P_DEFAULT, H5P_DEFAULT) < + 0) + goto error; + + /* MUST close the file ID before the user block code or you risk tripping + * over file locking issues. + */ + if (H5Fclose(fid) < 0) + goto error; /* If a user block is being used, write to it here */ if (ub_size > 0) { - HDassert(ub_size <= BUF_SIZE); + char *bp; - fd = HDopen(filename, O_RDWR); - HDassert(fd >= 0); + if (ub_size > BUF_SIZE) + goto error; - /* fill buf with pattern */ + if ((fd = HDopen(filename, O_RDWR)) < 0) + goto error; + + /* Fill buf with pattern */ HDmemset(buf, '\0', ub_size); bp = buf; for (u = 0; u < ub_fill; u++) *bp++ = pattern[u % 10]; - (void)HDwrite(fd, buf, ub_size); + if (HDwrite(fd, buf, ub_size) < 0) + goto error; + + if (HDclose(fd) < 0) + goto error; + } + return SUCCEED; + +error: + if (fd >= 0) HDclose(fd); + + H5E_BEGIN_TRY + { + H5Fclose(fid); + H5Gclose(group); + H5Aclose(attr); + H5Dclose(dataset); + H5Sclose(space); + H5Pclose(create_plist); } + H5E_END_TRY; + + return FAIL; } -static void +/* Creates a simple (i.e., not HDF5) text file and fills it with a pattern */ +static herr_t create_textfile(const char *name, size_t size) { - char * buf; - int fd; + char * buf = NULL; + int fd = -1; size_t i; - char * bp; + char * bp = NULL; - fd = HDcreat(name, 0777); - HDassert(fd >= 0); - buf = (char *)HDcalloc(size, (size_t)1); - HDassert(buf); + if ((fd = HDcreat(name, 0777)) < 0) + goto error; + if (NULL == (buf = (char *)HDcalloc(size, 1))) + goto error; - /* fill buf with pattern */ + /* Fill buf with pattern */ bp = buf; for (i = 0; i < size; i++) *bp++ = pattern[i % 10]; - (void)HDwrite(fd, buf, size); + if (HDwrite(fd, buf, size) < 0) + goto error; HDfree(buf); - HDclose(fd); -} - -#ifdef notdef -/* not used yet */ -void -create_binfile(char *name, off_t size) -{ - char *buf; - int fd; - int i; - char *bp; - fd = creat(name, 0777); - HDassert(fd >= 0); + return SUCCEED; - buf = HDcalloc(size, 1); - HDassert(buf); - - /* fill buf with pattern */ - bp = buf; - for (i = 0; i < size; i++) - *bp++ = (char)i & 0xff; - - (void)HDwrite(fd, buf, size); +error: + HDfree(buf); + if (fd >= 0) + HDclose(fd); - HDclose(fd); + return FAIL; } -#endif /*------------------------------------------------------------------------- * Function: main * *------------------------------------------------------------------------- */ - int main(void) { - - /* - create_textfile(UBTXT1, (size_t)0); - */ - create_textfile(UBTXT2, (size_t)10); - create_textfile(UBTXT3, (size_t)511); - create_textfile(UBTXT4, (size_t)512); - create_textfile(UBTXT5, (size_t)513); - /* - create_textfile(UBTXT6, (size_t)1023); - create_textfile(UBTXT7, (size_t)1024); - create_textfile(UBTXT8, (size_t)1025); - create_textfile(UBTXT9, (size_t)2047); - create_textfile(UBTXT10, (size_t)2048); - create_textfile(UBTXT11, (size_t)2049); - - create_binfile(UBBIN1, (off_t)0); - create_binfile(UBBIN2, (off_t)10); - create_binfile(UBBIN3, (off_t)511); - create_binfile(UBBIN4, (off_t)512); - create_binfile(UBBIN5, (off_t)513); - - */ - gent_ub(FILE7, (size_t)0, (size_t)0); - gent_ub(FILE8, (size_t)512, HDstrlen(pattern)); - gent_ub(FILE9, (size_t)1024, (size_t)513); - - return 0; + if (create_textfile(UBTXT2, 10) < 0) + goto error; + if (create_textfile(UBTXT3, 511) < 0) + goto error; + if (create_textfile(UBTXT4, 512) < 0) + goto error; + if (create_textfile(UBTXT5, 513) < 0) + goto error; + + if (gent_ub(FILE7, 0, 0) < 0) + goto error; + if (gent_ub(FILE8, 512, HDstrlen(pattern)) < 0) + goto error; + if (gent_ub(FILE9, 1024, 513) < 0) + goto error; + + return EXIT_SUCCESS; + +error: + HDfprintf(stderr, "h5jam test generator FAILED\n"); + return EXIT_FAILURE; } diff --git a/tools/test/h5jam/tellub.c b/tools/test/h5jam/tellub.c index 45d4af4..e6769ec 100644 --- a/tools/test/h5jam/tellub.c +++ b/tools/test/h5jam/tellub.c @@ -24,8 +24,8 @@ * parameters. The long-named ones can be partially spelled. When * adding more, make sure that they don't clash with each other. */ -static const char * s_opts = "h"; -static struct long_options l_opts[] = {{"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, {NULL, 0, '\0'}}; +static const char * s_opts = "h"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, {NULL, 0, '\0'}}; /*------------------------------------------------------------------------- * Function: usage @@ -61,7 +61,7 @@ parse_command_line(int argc, const char *argv[]) int opt; /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { case 'h': usage(h5tools_getprogname()); @@ -75,7 +75,7 @@ parse_command_line(int argc, const char *argv[]) } /* check for file name to be processed */ - if (argc <= opt_ind) { + if (argc <= H5_optind) { error_msg("missing file name\n"); usage(h5tools_getprogname()); h5tools_setstatus(EXIT_FAILURE); @@ -118,14 +118,14 @@ main(int argc, const char *argv[]) /* enable error reporting if command line option */ h5tools_error_report(); - if (argc <= (opt_ind)) { + if (argc <= (H5_optind)) { error_msg("missing file name\n"); usage(h5tools_getprogname()); h5tools_setstatus(EXIT_FAILURE); goto done; } - ifname = HDstrdup(argv[opt_ind]); + ifname = HDstrdup(argv[H5_optind]); testval = H5Fis_accessible(ifname, H5P_DEFAULT); diff --git a/tools/test/h5stat/CMakeTests.cmake b/tools/test/h5stat/CMakeTests.cmake index 79de6b9..d4238b5 100644 --- a/tools/test/h5stat/CMakeTests.cmake +++ b/tools/test/h5stat/CMakeTests.cmake @@ -207,7 +207,7 @@ # -d --di=15 ADD_H5_ERR_TEST (h5stat_err1_dims 1 -d --dims=-1 h5stat_threshold.h5) ADD_H5_TEST (h5stat_dims1 0 -gd -m 5 h5stat_threshold.h5) - ADD_H5_TEST (h5stat_dims2 0 -d --di=15 h5stat_threshold.h5) + ADD_H5_TEST (h5stat_dims2 0 -d --dims=15 h5stat_threshold.h5) # # Tests for -a option on h5stat_threshold.h5 # -a -2 (incorrect threshold value) diff --git a/tools/test/h5stat/h5stat_gentest.c b/tools/test/h5stat/h5stat_gentest.c index a9813e7..c775d6a 100644 --- a/tools/test/h5stat/h5stat_gentest.c +++ b/tools/test/h5stat/h5stat_gentest.c @@ -51,7 +51,7 @@ * Generate HDF5 file with latest format with * NUM_GRPS groups and NUM_ATTRS attributes for the dataset */ -static void +static herr_t gen_newgrat_file(const char *fname) { hid_t fcpl = H5I_INVALID_HID; /* File creation property */ @@ -117,6 +117,21 @@ gen_newgrat_file(const char *fname) } /* end for */ /* Close dataset, dataspace, datatype, file */ + if (H5Pclose(fapl) < 0) + goto error; + if (H5Pclose(fcpl) < 0) + goto error; + if (H5Dclose(did) < 0) + goto error; + if (H5Tclose(tid) < 0) + goto error; + if (H5Sclose(sid) < 0) + goto error; + if (H5Fclose(fid) < 0) + goto error; + + return SUCCEED; + error: H5E_BEGIN_TRY { @@ -130,6 +145,8 @@ error: H5Fclose(fid); } H5E_END_TRY; + + return FAIL; } /* gen_newgrat_file() */ /* @@ -139,7 +156,7 @@ error: * datasets. -a N (--numattrs=N): Set the threshold for the # of attributes when printing information for * small # of attributes. */ -static void +static herr_t gen_threshold_file(const char *fname) { hid_t fid = H5I_INVALID_HID; /* File ID */ @@ -302,6 +319,23 @@ gen_threshold_file(const char *fname) goto error; } + if (H5Gclose(gid) < 0) + goto error; + if (H5Sclose(sid0) < 0) + goto error; + if (H5Sclose(sid1) < 0) + goto error; + if (H5Sclose(sid2) < 0) + goto error; + if (H5Sclose(sid3) < 0) + goto error; + if (H5Sclose(sid4) < 0) + goto error; + if (H5Fclose(fid) < 0) + goto error; + + return SUCCEED; + error: H5E_BEGIN_TRY { @@ -317,6 +351,8 @@ error: } H5E_END_TRY; + return FAIL; + } /* gen_threshold_file() */ /* @@ -327,18 +363,21 @@ error: * one dataset: fixed dimension, chunked layout, w/ filters * */ -static void +static herr_t gen_idx_file(const char *fname) { - hid_t fapl = H5I_INVALID_HID; /* file access property id */ - hid_t fid = H5I_INVALID_HID; /* file id */ - hid_t sid = H5I_INVALID_HID; /* space id */ - hid_t dcpl = H5I_INVALID_HID; /* dataset creation property id */ - hid_t did = -1, did2 = H5I_INVALID_HID; /* dataset id */ - hsize_t dims[1] = {10}; /* dataset dimension */ - hsize_t c_dims[1] = {2}; /* chunk dimension */ - int i; /* local index variable */ - int buf[10]; /* data buffer */ + hid_t fapl = H5I_INVALID_HID; /* file access property id */ + hid_t fid = H5I_INVALID_HID; /* file id */ + hid_t sid = H5I_INVALID_HID; /* space id */ + hid_t dcpl = H5I_INVALID_HID; /* dataset creation property id */ + hid_t did = H5I_INVALID_HID; /* dataset id */ +#if defined(H5_HAVE_FILTER_DEFLATE) + hid_t did2 = H5I_INVALID_HID; /* dataset id (compressed) */ +#endif + hsize_t dims[1] = {10}; /* dataset dimension */ + hsize_t c_dims[1] = {2}; /* chunk dimension */ + int i; /* local index variable */ + int buf[10]; /* data buffer */ /* Get a copy of the file access property */ if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) @@ -390,6 +429,19 @@ gen_idx_file(const char *fname) #endif /* closing: dataspace, dataset, file */ + if (H5Pclose(fapl) < 0) + goto error; + if (H5Pclose(dcpl) < 0) + goto error; + if (H5Sclose(sid) < 0) + goto error; + if (H5Dclose(did) < 0) + goto error; + if (H5Fclose(fid) < 0) + goto error; + + return SUCCEED; + error: H5E_BEGIN_TRY { @@ -404,6 +456,8 @@ error: } H5E_END_TRY; + return FAIL; + } /* gen_idx_file() */ /* @@ -419,20 +473,21 @@ error: * H5O_refcount_decode in the jira issue. * */ -static void +static herr_t gen_err_refcount(const char *fname) { - hid_t fid = H5I_INVALID_HID; /* File identifier */ - hid_t sid = H5I_INVALID_HID; /* Dataspace message */ - hid_t did = H5I_INVALID_HID; /* Dataset identifier */ - hid_t gid = H5I_INVALID_HID; /* Group identifier */ - hid_t aid1 = -1, aid2 = H5I_INVALID_HID; /* Attribute identifier */ - hid_t tid = H5I_INVALID_HID; /* Datatype identifier */ - int i, n; /* Local index variables */ - int buf[10]; /* Data buffer */ - hsize_t dims[1]; /* Dimension size */ - int fd = -1; /* File descriptor */ - unsigned short val = 22; /* The refcount message ID */ + hid_t fid = H5I_INVALID_HID; /* File identifier */ + hid_t sid = H5I_INVALID_HID; /* Dataspace message */ + hid_t did = H5I_INVALID_HID; /* Dataset identifier */ + hid_t gid = H5I_INVALID_HID; /* Group identifier */ + hid_t aid1 = H5I_INVALID_HID; /* Attribute identifier */ + hid_t aid2 = H5I_INVALID_HID; /* Attribute identifier */ + hid_t tid = H5I_INVALID_HID; /* Datatype identifier */ + int i, n; /* Local index variables */ + int buf[10]; /* Data buffer */ + hsize_t dims[1]; /* Dimension size */ + int fd = -1; /* File descriptor */ + unsigned short val = 22; /* The refcount message ID */ /* Initialize data buffer */ n = 0; @@ -485,6 +540,10 @@ gen_err_refcount(const char *fname) goto error; if (H5Tclose(tid) < 0) goto error; + + /* Be sure to close this before opening the file again via open(), below, + * or you'll possibly trip over the file locking. + */ if (H5Fclose(fid) < 0) goto error; @@ -495,11 +554,16 @@ gen_err_refcount(const char *fname) with the committed datatype */ /* 24: the offset in the object header containing the version of the attribute message */ - if ((fd = HDopen(fname, O_RDWR, 0633)) >= 0) { - HDlseek(fd, 4520 + 24, SEEK_SET); - (void)HDwrite(fd, &val, 2); - HDclose(fd); - } + if ((fd = HDopen(fname, O_RDWR, 0633)) < 0) + goto error; + if (HDlseek(fd, 4520 + 24, SEEK_SET) < 0) + goto error; + if (HDwrite(fd, &val, 2) < 0) + goto error; + if (HDclose(fd) < 0) + goto error; + + return SUCCEED; error: H5E_BEGIN_TRY @@ -513,6 +577,11 @@ error: H5Fclose(fid); } H5E_END_TRY; + + if (fd >= 0) + HDclose(fd); + + return FAIL; } /* gen_err_refcount() */ /* @@ -542,14 +611,22 @@ error: int main(void) { - gen_newgrat_file(NEWGRAT_FILE); - gen_threshold_file(THRESHOLD_FILE); + if (gen_newgrat_file(NEWGRAT_FILE) < 0) + goto error; + if (gen_threshold_file(THRESHOLD_FILE) < 0) + goto error; /* Generate an HDF file to test for datasets with Fixed Array indexing */ - gen_idx_file(IDX_FILE); + if (gen_idx_file(IDX_FILE) < 0) + goto error; /* Generate a file with a refcount message ID */ - gen_err_refcount(ERR_REFCOUNT_FILE); + if (gen_err_refcount(ERR_REFCOUNT_FILE) < 0) + goto error; + + return EXIT_SUCCESS; - return 0; +error: + HDfprintf(stderr, "h5stat test generator FAILED\n"); + return EXIT_FAILURE; } diff --git a/tools/test/h5stat/testh5stat.sh.in b/tools/test/h5stat/testh5stat.sh.in index 470c381..647f06a 100644 --- a/tools/test/h5stat/testh5stat.sh.in +++ b/tools/test/h5stat/testh5stat.sh.in @@ -304,7 +304,7 @@ TOOLTEST h5stat_links5.ddl -g -l 40000 h5stat_newgrat.h5 # -d --di=15 TOOLTEST h5stat_err1_dims.ddl -d --dims=-1 h5stat_threshold.h5 TOOLTEST h5stat_dims1.ddl -gd -m 5 h5stat_threshold.h5 -TOOLTEST h5stat_dims2.ddl -d --di=15 h5stat_threshold.h5 +TOOLTEST h5stat_dims2.ddl -d --dims=15 h5stat_threshold.h5 # # Tests for -a option on h5stat_threshold.h5 # -a -2 (incorrect threshold value) diff --git a/tools/test/misc/CMakeTestsClear.cmake b/tools/test/misc/CMakeTestsClear.cmake index ec24a61..198a363 100644 --- a/tools/test/misc/CMakeTestsClear.cmake +++ b/tools/test/misc/CMakeTestsClear.cmake @@ -397,7 +397,7 @@ # # The following are tests to verify the expected exit code from h5clear: # "h5clear -m h5clear_mdc_image.h5" (valid option, existing file, succeed exit code) -# "h5clear --vers" (valid option, version #, succeed exit code) +# "h5clear --version" (valid option, version #, succeed exit code) # "h5clear -k" (invalid 1 option, no file, fail exit code) # "h5clear -k junk.h5" (invalid 1 option, nonexisting file, fail exit code) # "h5clear -l h5clear_sec2_v2.h5" (invalid 1 option, existing file, fail exit code) @@ -408,7 +408,7 @@ # "h5clear -m -l h5clear_sec2_v0.h5" (valid/invalid 2 options, existing file, fail exit code) # "h5clear -l -m h5clear_sec2_v0.h5" (invalid/valid 2 options, existing file, fail exit code) ADD_H5_RETTEST (h5clr_mdc_image "false" "-m" h5clear_mdc_image.h5) - ADD_H5_RETTEST (h5clr_vers "false" "--vers") + ADD_H5_RETTEST (h5clr_vers "false" "--version") ADD_H5_RETTEST (h5clr_k "true" "-k") ADD_H5_RETTEST (h5clr_k_junk "true" "-k" junk.h5) ADD_H5_RETTEST (h5clr_l_sec2 "true" "-l" h5clear_sec2_v2.h5) diff --git a/tools/test/misc/testh5clear.sh.in b/tools/test/misc/testh5clear.sh.in index c05e43d..a01fa4a 100644 --- a/tools/test/misc/testh5clear.sh.in +++ b/tools/test/misc/testh5clear.sh.in @@ -315,7 +315,7 @@ TOOLTEST_ERR orig_h5clear_sec2_v0.h5 -s -m "" h5clear_no_mdc_image.err # # The following are tests to verify the expected exit code from h5clear: # "h5clear -m h5clear_mdc_image.h5" (valid option, existing file, succeed exit code) -# "h5clear --vers" (valid option, version #, succeed exit code) +# "h5clear --version" (valid option, version #, succeed exit code) # "h5clear -k" (invalid 1 option, no file, fail exit code) # "h5clear -k junk.h5" (invalid 1 option, nonexisting file, fail exit code) # "h5clear -l h5clear_sec2_v2.h5" (invalid 1 option, existing file, fail exit code) @@ -326,7 +326,7 @@ TOOLTEST_ERR orig_h5clear_sec2_v0.h5 -s -m "" h5clear_no_mdc_image.err # "h5clear -m -l h5clear_sec2_v0.h5" (valid/invalid 2 options, existing file, fail exit code) # "h5clear -l -m h5clear_sec2_v0.h5" (invalid/valid 2 options, existing file, fail exit code) TOOLTEST h5clear_mdc_image.h5 -m "" $SUCCEED -TOOLTEST "" --vers "" $SUCCEED +TOOLTEST "" --version "" $SUCCEED TOOLTEST "" -k "" $FAIL TOOLTEST junk.h5 -k "" $FAIL TOOLTEST h5clear_sec2_v2.h5 -l "" $FAIL diff --git a/tools/test/perform/CMakeLists.txt b/tools/test/perform/CMakeLists.txt index 7399723..7bf79c5 100644 --- a/tools/test/perform/CMakeLists.txt +++ b/tools/test/perform/CMakeLists.txt @@ -4,39 +4,17 @@ project (HDF5_TOOLS_TEST_PERFORM C) # -------------------------------------------------------------------- # Add the executables # -------------------------------------------------------------------- -#-- Adding test for h5perf_serial -set (h5perf_serial_SOURCES - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_perf.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_engine.c -) -add_executable (h5perf_serial ${h5perf_serial_SOURCES}) -target_include_directories (h5perf_serial PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") -if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (h5perf_serial STATIC) - target_link_libraries (h5perf_serial PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET}) -else () - TARGET_C_PROPERTIES (h5perf_serial SHARED) - target_link_libraries (h5perf_serial PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) -endif () -set_target_properties (h5perf_serial PROPERTIES FOLDER perform) - -#----------------------------------------------------------------------------- -# Add Target to clang-format -#----------------------------------------------------------------------------- -if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_TOOLS_TEST_PERFORM_h5perf_serial_FORMAT h5perf_serial) -endif () if (HDF5_BUILD_PERFORM_STANDALONE) #-- Adding test for h5perf_serial_alone - io_timer.c includes set (h5perf_serial_alone_SOURCES ${HDF5_TOOLS_DIR}/lib/io_timer.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_perf.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_engine.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/sio_perf.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/sio_engine.c ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/sio_standalone.c ) add_executable (h5perf_serial_alone ${h5perf_serial_alone_SOURCES}) - target_include_directories (h5perf_serial_alone PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TOOLS_DIR}/lib;$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5perf_serial_alone PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TOOLS_DIR}/lib;${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR};${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (h5perf_serial_alone STATIC) target_link_libraries (h5perf_serial_alone PRIVATE ${HDF5_LIB_TARGET} "$<$:${MPI_C_LIBRARIES}>") @@ -188,63 +166,16 @@ if (HDF5_ENABLE_FORMATTERS) endif () if (H5_HAVE_PARALLEL AND HDF5_TEST_PARALLEL) - if (UNIX) - #-- Adding test for perf - only on unix systems - set (perf_SOURCES - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/perf.c - ) - add_executable (perf ${perf_SOURCES}) - target_include_directories (perf PRIVATE "${HDF5_TEST_SRC_DIR};${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") - if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (perf STATIC) - target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_TEST_LIB_TARGET} ${HDF5_LIB_TARGET}) - else () - TARGET_C_PROPERTIES (perf SHARED) - target_link_libraries (perf PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_TEST_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) - endif () - set_target_properties (perf PROPERTIES FOLDER perform) - - #----------------------------------------------------------------------------- - # Add Target to clang-format - #----------------------------------------------------------------------------- - if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_TOOLS_TEST_PERFORM_perf_FORMAT perf) - endif () - endif () - - #-- Adding test for h5perf - set (h5perf_SOURCES - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_perf.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_engine.c - ) - add_executable (h5perf ${h5perf_SOURCES}) - target_include_directories (h5perf PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") - if (NOT BUILD_SHARED_LIBS) - TARGET_C_PROPERTIES (h5perf STATIC) - target_link_libraries (h5perf PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_TEST_LIB_TARGET} ${HDF5_LIB_TARGET}) - else () - TARGET_C_PROPERTIES (h5perf SHARED) - target_link_libraries (h5perf PRIVATE ${HDF5_TOOLS_LIBSH_TARGET} ${HDF5_TEST_LIBSH_TARGET} ${HDF5_LIBSH_TARGET}) - endif () - set_target_properties (h5perf PROPERTIES FOLDER perform) - - #----------------------------------------------------------------------------- - # Add Target to clang-format - #----------------------------------------------------------------------------- - if (HDF5_ENABLE_FORMATTERS) - clang_format (HDF5_TOOLS_TEST_PERFORM_h5perf_FORMAT h5perf) - endif () - if (HDF5_BUILD_PERFORM_STANDALONE) #-- Adding test for h5perf set (h5perf_alone_SOURCES ${HDF5_TOOLS_DIR}/lib/io_timer.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_perf.c - ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_engine.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/pio_perf.c + ${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR}/pio_engine.c ${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR}/pio_standalone.c ) add_executable (h5perf_alone ${h5perf_alone_SOURCES}) - target_include_directories (h5perf_alone PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TOOLS_DIR}/lib;$<$:${MPI_C_INCLUDE_DIRS}>") + target_include_directories (h5perf_alone PRIVATE "${HDF5_SRC_DIR};${HDF5_SRC_BINARY_DIR};${HDF5_TOOLS_DIR}/lib;${HDF5_TOOLS_SRC_H5PERF_SOURCE_DIR};${HDF5_TOOLS_TEST_PERFORM_SOURCE_DIR};$<$:${MPI_C_INCLUDE_DIRS}>") if (NOT BUILD_SHARED_LIBS) TARGET_C_PROPERTIES (h5perf_alone STATIC) target_link_libraries (h5perf_alone PRIVATE ${HDF5_LIB_TARGET} ${LINK_LIBS} "$<$:${MPI_C_LIBRARIES}>") diff --git a/tools/test/perform/Makefile.am b/tools/test/perform/Makefile.am index 244ef3a..10b13fe 100644 --- a/tools/test/perform/Makefile.am +++ b/tools/test/perform/Makefile.am @@ -21,17 +21,6 @@ include $(top_srcdir)/config/commence.am AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -# bin_PROGRAMS will be installed. -if BUILD_PARALLEL_CONDITIONAL - bin_PROGRAMS=h5perf_serial h5perf -else - bin_PROGRAMS=h5perf_serial -endif - -# Add h5perf and h5perf_serial specific linker flags here -h5perf_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) -h5perf_serial_LDFLAGS = $(LT_STATIC_EXEC) $(AM_LDFLAGS) - # Some programs are not built or run by default, but can be built by hand or by # specifying --enable-build-all at configure time. # Also, some of these programs should only be built in parallel. @@ -47,18 +36,15 @@ endif # List them in the order they should be run. # Parallel test programs. if BUILD_PARALLEL_CONDITIONAL - TEST_PROG_PARA=h5perf perf + TEST_PROG_PARA= endif # Serial test programs. -TEST_PROG = iopipe chunk chunk_cache overhead zip_perf perf_meta h5perf_serial $(BUILD_ALL_PROGS) +TEST_PROG = iopipe chunk chunk_cache overhead zip_perf perf_meta $(BUILD_ALL_PROGS) # check_PROGRAMS will be built but not installed. Do not any executable # that is in bin_PROGRAMS already. Otherwise, it will be removed twice in # "make clean" and some systems, e.g., AIX, do not like it. -check_PROGRAMS= iopipe chunk chunk_cache overhead zip_perf perf_meta $(BUILD_ALL_PROGS) perf - -h5perf_SOURCES=pio_perf.c pio_engine.c -h5perf_serial_SOURCES=sio_perf.c sio_engine.c +check_PROGRAMS= $(TEST_PROG) $(BUILD_ALL_PROGS) # These are the files that `make clean' (and derivatives) will remove from # this directory. @@ -67,9 +53,6 @@ CLEANFILES=*.h5 *.raw *.dat x-gnuplot perftest.out # All of the programs depend on the main hdf5 library, and some of them # depend on test or tools library. LDADD=$(LIBHDF5) -h5perf_LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) -h5perf_serial_LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) -perf_LDADD=$(LIBH5TEST) $(LIBHDF5) iopipe_LDADD=$(LIBH5TEST) $(LIBHDF5) zip_perf_LDADD=$(LIBH5TOOLS) $(LIBH5TEST) $(LIBHDF5) perf_meta_LDADD=$(LIBH5TEST) $(LIBHDF5) diff --git a/tools/test/perform/perf.c b/tools/test/perform/perf.c deleted file mode 100644 index bf4b2a9..0000000 --- a/tools/test/perform/perf.c +++ /dev/null @@ -1,476 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * Copyright by the Board of Trustees of the University of Illinois. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Author: Albert Cheng of NCSA, May 1, 2001. - * This is derived from code given to me by Robert Ross. - * - * NOTE: This code assumes that all command line arguments make it out to all - * the processes that make up the parallel job, which isn't always the case. - * So if it doesn't work on some platform, that might be why. - */ - -#include "hdf5.h" -#include "H5private.h" -#include "h5test.h" - -#ifdef H5_HAVE_PARALLEL - -#ifdef H5_STDC_HEADERS -#include -#include -#include -#include -#include -#endif - -#ifdef H5_HAVE_UNISTD_H -#include -#include -#endif - -#ifdef H5_HAVE_SYS_STAT_H -#include -#endif - -#if defined(H5_TIME_WITH_SYS_TIME) -#include -#include -#elif defined(H5_HAVE_SYS_TIME_H) -#include -#else -#include -#endif - -#include -#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ -#include -#endif - -/* Macro definitions */ -/* Verify: - * if val is false (0), print mesg and if fatal is true (non-zero), die. - */ -#define H5FATAL 1 -#define VRFY(val, mesg, fatal) \ - do { \ - if (!val) { \ - printf("Proc %d: ", mynod); \ - printf("*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ - if (fatal) { \ - fflush(stdout); \ - goto die_jar_jar_die; \ - } \ - } \ - } while (0) -#define RANK 1 -#define MAX_PATH 1024 - -hsize_t dims[RANK]; /* dataset dim sizes */ -hsize_t block[RANK], stride[RANK], count[RANK]; -hsize_t start[RANK]; -hid_t fid; /* HDF5 file ID */ -hid_t acc_tpl; /* File access templates */ -hid_t sid; /* Dataspace ID */ -hid_t file_dataspace; /* File dataspace ID */ -hid_t mem_dataspace; /* memory dataspace ID */ -hid_t dataset; /* Dataset ID */ -hsize_t opt_alignment = 1; -hsize_t opt_threshold = 1; -int opt_split_vfd = 0; -char * meta_ext, *raw_ext; /* holds the meta and raw file extension if */ - /* opt_split_vfd is set */ - -/* DEFAULT VALUES FOR OPTIONS */ -int64_t opt_block = 1048576 * 16; -int opt_iter = 1; -int opt_stripe = -1; -int opt_correct = 0; -int amode = O_RDWR | O_CREAT; -char opt_file[256] = "perftest.out"; -char opt_pvfstab[256] = "notset"; -int opt_pvfstab_set = 0; - -const char *FILENAME[] = {opt_file, NULL}; - -/* function prototypes */ -static int parse_args(int argc, char **argv); - -#ifndef H5_HAVE_UNISTD_H -/* globals needed for getopt */ -extern char *optarg; -#endif - -int -main(int argc, char **argv) -{ - char * buf, *tmp, *buf2 = NULL, *tmp2 = NULL, *check; - int i, j, mynod = 0, nprocs = 1, my_correct = 1, correct, myerrno; - double stim, etim; - double write_tim = 0; - double read_tim = 0; - double read_bw, write_bw; - double max_read_tim, max_write_tim; - double min_read_tim, min_write_tim; - double ave_read_tim, ave_write_tim; - int64_t iter_jump = 0; - char filename[MAX_PATH]; - herr_t ret; /* Generic return value */ - - /* startup MPI and determine the rank of this process */ - MPI_Init(&argc, &argv); - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - MPI_Comm_rank(MPI_COMM_WORLD, &mynod); - - /* parse the command line arguments */ - parse_args(argc, argv); - - if (mynod == 0) - printf("# Using hdf5-io calls.\n"); - - /* kindof a weird hack- if the location of the pvfstab file was - * specified on the command line, then spit out this location into - * the appropriate environment variable: */ - -#if H5_HAVE_SETENV - /* no setenv or unsetenv */ - if (opt_pvfstab_set) { - if ((setenv("PVFSTAB_FILE", opt_pvfstab, 1)) < 0) { - perror("setenv"); - goto die_jar_jar_die; - } - } -#endif - - /* this is how much of the file data is covered on each iteration of - * the test. used to help determine the seek offset on each - * iteration */ - iter_jump = nprocs * opt_block; - - /* setup a buffer of data to write */ - if (!(tmp = (char *)malloc((size_t)opt_block + 256))) { - perror("malloc"); - goto die_jar_jar_die; - } - buf = tmp + 128 - (((long)tmp) % 128); /* align buffer */ - - if (opt_correct) { - /* do the same buffer setup for verifiable data */ - if (!(tmp2 = (char *)malloc((size_t)opt_block + 256))) { - perror("malloc2"); - goto die_jar_jar_die; - } - buf2 = tmp + 128 - (((long)tmp) % 128); - } - - /* setup file access template with parallel IO access. */ - if (opt_split_vfd) { - hid_t mpio_pl; - - mpio_pl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((acc_tpl >= 0), "", H5FATAL); - ret = H5Pset_fapl_mpio(mpio_pl, MPI_COMM_WORLD, MPI_INFO_NULL); - VRFY((ret >= 0), "", H5FATAL); - - /* set optional allocation alignment */ - if (opt_alignment * opt_threshold != 1) { - ret = H5Pset_alignment(acc_tpl, opt_threshold, opt_alignment); - VRFY((ret >= 0), "H5Pset_alignment succeeded", !H5FATAL); - } - - /* setup file access template */ - acc_tpl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((acc_tpl >= 0), "", H5FATAL); - ret = H5Pset_fapl_split(acc_tpl, meta_ext, mpio_pl, raw_ext, mpio_pl); - VRFY((ret >= 0), "H5Pset_fapl_split succeeded", H5FATAL); - ret = H5Pclose(mpio_pl); - VRFY((ret >= 0), "H5Pclose mpio_pl succeeded", H5FATAL); - } - else { - /* setup file access template */ - acc_tpl = H5Pcreate(H5P_FILE_ACCESS); - VRFY((acc_tpl >= 0), "", H5FATAL); - ret = H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL); - VRFY((ret >= 0), "", H5FATAL); - - /* set optional allocation alignment */ - if (opt_alignment * opt_threshold != 1) { - ret = H5Pset_alignment(acc_tpl, opt_threshold, opt_alignment); - VRFY((ret >= 0), "H5Pset_alignment succeeded", !H5FATAL); - } - } - - h5_fixname_no_suffix(FILENAME[0], acc_tpl, filename, sizeof filename); - - /* create the parallel file */ - fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); - VRFY((fid >= 0), "H5Fcreate succeeded", H5FATAL); - - /* define a contiquous dataset of opt_iter*nprocs*opt_block chars */ - dims[0] = (hsize_t)opt_iter * (hsize_t)nprocs * (hsize_t)opt_block; - sid = H5Screate_simple(RANK, dims, NULL); - VRFY((sid >= 0), "H5Screate_simple succeeded", H5FATAL); - dataset = H5Dcreate2(fid, "Dataset1", H5T_NATIVE_CHAR, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - VRFY((dataset >= 0), "H5Dcreate2 succeeded", H5FATAL); - - /* create the memory dataspace and the file dataspace */ - dims[0] = (hsize_t)opt_block; - mem_dataspace = H5Screate_simple(RANK, dims, NULL); - VRFY((mem_dataspace >= 0), "", H5FATAL); - file_dataspace = H5Dget_space(dataset); - VRFY((file_dataspace >= 0), "H5Dget_space succeeded", H5FATAL); - - /* now each process writes a block of opt_block chars in round robbin - * fashion until the whole dataset is covered. - */ - for (j = 0; j < opt_iter; j++) { - /* setup a file dataspace selection */ - start[0] = (hsize_t)((j * iter_jump) + (mynod * opt_block)); - stride[0] = block[0] = (hsize_t)opt_block; - count[0] = 1; - ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); - VRFY((ret >= 0), "H5Sset_hyperslab succeeded", H5FATAL); - - if (opt_correct) /* fill in buffer for iteration */ { - for (i = mynod + j, check = buf; i < opt_block; i++, check++) - *check = (char)i; - } - - /* discover the starting time of the operation */ - MPI_Barrier(MPI_COMM_WORLD); - stim = MPI_Wtime(); - - /* write data */ - ret = H5Dwrite(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf); - VRFY((ret >= 0), "H5Dwrite dataset1 succeeded", !H5FATAL); - - /* discover the ending time of the operation */ - etim = MPI_Wtime(); - - write_tim += (etim - stim); - - /* we are done with this "write" iteration */ - } - - /* close dataset and file */ - ret = H5Dclose(dataset); - VRFY((ret >= 0), "H5Dclose succeeded", H5FATAL); - ret = H5Fclose(fid); - VRFY((ret >= 0), "H5Fclose succeeded", H5FATAL); - - /* wait for everyone to synchronize at this point */ - MPI_Barrier(MPI_COMM_WORLD); - - /* reopen the file for reading */ - fid = H5Fopen(filename, H5F_ACC_RDONLY, acc_tpl); - VRFY((fid >= 0), "", H5FATAL); - - /* open the dataset */ - dataset = H5Dopen2(fid, "Dataset1", H5P_DEFAULT); - VRFY((dataset >= 0), "H5Dopen succeeded", H5FATAL); - - /* we can re-use the same mem_dataspace and file_dataspace - * the H5Dwrite used since the dimension size is the same. - */ - - /* we are going to repeat the read the same pattern the write used */ - for (j = 0; j < opt_iter; j++) { - /* setup a file dataspace selection */ - start[0] = (hsize_t)((j * iter_jump) + (mynod * opt_block)); - stride[0] = block[0] = (hsize_t)opt_block; - count[0] = 1; - ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); - VRFY((ret >= 0), "H5Sset_hyperslab succeeded", H5FATAL); - /* seek to the appropriate spot give the current iteration and - * rank within the MPI processes */ - - /* discover the start time */ - MPI_Barrier(MPI_COMM_WORLD); - stim = MPI_Wtime(); - - /* read in the file data */ - if (!opt_correct) { - ret = H5Dread(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf); - } - else { - ret = H5Dread(dataset, H5T_NATIVE_CHAR, mem_dataspace, file_dataspace, H5P_DEFAULT, buf2); - } - myerrno = errno; - - /* discover the end time */ - etim = MPI_Wtime(); - read_tim += (etim - stim); - VRFY((ret >= 0), "H5Dwrite dataset1 succeeded", !H5FATAL); - - if (ret < 0) - HDfprintf(stderr, "node %d, read error, loc = %" PRId64 ": %s\n", mynod, mynod * opt_block, - strerror(myerrno)); - - /* if the user wanted to check correctness, compare the write - * buffer to the read buffer */ - if (opt_correct && memcmp(buf, buf2, (size_t)opt_block)) { - HDfprintf(stderr, "node %d, correctness test failed\n", mynod); - my_correct = 0; - MPI_Allreduce(&my_correct, &correct, 1, MPI_INT, MPI_MIN, MPI_COMM_WORLD); - } - - /* we are done with this read iteration */ - } - - /* close dataset and file */ - ret = H5Dclose(dataset); - VRFY((ret >= 0), "H5Dclose succeeded", H5FATAL); - ret = H5Fclose(fid); - VRFY((ret >= 0), "H5Fclose succeeded", H5FATAL); - ret = H5Pclose(acc_tpl); - VRFY((ret >= 0), "H5Pclose succeeded", H5FATAL); - - /* compute the read and write times */ - MPI_Allreduce(&read_tim, &max_read_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - MPI_Allreduce(&read_tim, &min_read_tim, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); - MPI_Allreduce(&read_tim, &ave_read_tim, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - - /* calculate the average from the sum */ - ave_read_tim = ave_read_tim / nprocs; - - MPI_Allreduce(&write_tim, &max_write_tim, 1, MPI_DOUBLE, MPI_MAX, MPI_COMM_WORLD); - MPI_Allreduce(&write_tim, &min_write_tim, 1, MPI_DOUBLE, MPI_MIN, MPI_COMM_WORLD); - MPI_Allreduce(&write_tim, &ave_write_tim, 1, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - - /* calculate the average from the sum */ - ave_write_tim = ave_write_tim / nprocs; - - /* print out the results on one node */ - if (mynod == 0) { - read_bw = (double)((int64_t)(opt_block * nprocs * opt_iter)) / (max_read_tim * 1000000.0); - write_bw = (double)((int64_t)(opt_block * nprocs * opt_iter)) / (max_write_tim * 1000000.0); - - printf("nr_procs = %d, nr_iter = %d, blk_sz = %ld\n", nprocs, opt_iter, (long)opt_block); - - printf("# total_size = %ld\n", (long)(opt_block * nprocs * opt_iter)); - - printf("# Write: min_time = %f, max_time = %f, mean_time = %f\n", min_write_tim, max_write_tim, - ave_write_tim); - printf("# Read: min_time = %f, max_time = %f, mean_time = %f\n", min_read_tim, max_read_tim, - ave_read_tim); - - printf("Write bandwidth = %f Mbytes/sec\n", write_bw); - printf("Read bandwidth = %f Mbytes/sec\n", read_bw); - - if (opt_correct) { - printf("Correctness test %s.\n", correct ? "passed" : "failed"); - } - } - -die_jar_jar_die: - -#if H5_HAVE_SETENV - /* no setenv or unsetenv */ - /* clear the environment variable if it was set earlier */ - if (opt_pvfstab_set) { - unsetenv("PVFSTAB_FILE"); - } -#endif - - free(tmp); - if (opt_correct) - free(tmp2); - - MPI_Finalize(); - - return (0); -} - -static int -parse_args(int argc, char **argv) -{ - int c; - - while ((c = getopt(argc, argv, "s:b:i:f:p:a:2:c")) != EOF) { - switch (c) { - case 's': /* stripe */ - opt_stripe = atoi(optarg); - break; - case 'b': /* block size */ - opt_block = atoi(optarg); - break; - case 'i': /* iterations */ - opt_iter = atoi(optarg); - break; - case 'f': /* filename */ - strncpy(opt_file, optarg, 255); - FILENAME[0] = opt_file; - break; - case 'p': /* pvfstab file */ - strncpy(opt_pvfstab, optarg, 255); - opt_pvfstab_set = 1; - break; - case 'a': /* aligned allocation. - * syntax: -a/ - * e.g., -a4096/512 allocate at 4096 bytes - * boundary if request size >= 512. - */ - { - char *p; - - opt_alignment = (hsize_t)HDatoi(optarg); - if (NULL != (p = (char *)HDstrchr(optarg, '/'))) - opt_threshold = (hsize_t)HDatoi(p + 1); - } - HDfprintf(stdout, "alignment/threshold=%" PRIuHSIZE "/%" PRIuHSIZE "\n", opt_alignment, - opt_threshold); - break; - case '2': /* use 2-files, i.e., split file driver */ - opt_split_vfd = 1; - /* get meta and raw file extension. */ - /* syntax is , */ - meta_ext = raw_ext = optarg; - while (*raw_ext != '\0') { - if (*raw_ext == ',') { - *raw_ext = '\0'; - raw_ext++; - break; - } - raw_ext++; - } - printf("split-file-vfd used: %s,%s\n", meta_ext, raw_ext); - break; - case 'c': /* correctness */ - opt_correct = 1; - break; - case '?': /* unknown */ - default: - break; - } - } - - return (0); -} - -/* - * Local variables: - * c-indent-level: 3 - * c-basic-offset: 3 - * tab-width: 3 - * End: - */ - -#else /* H5_HAVE_PARALLEL */ -/* dummy program since H5_HAVE_PARALLEL is not configured in */ -int -main(int H5_ATTR_UNUSED argc, char H5_ATTR_UNUSED **argv) -{ - printf("No parallel performance because parallel is not configured in\n"); - return (0); -} -#endif /* H5_HAVE_PARALLEL */ diff --git a/tools/test/perform/pio_engine.c b/tools/test/perform/pio_engine.c deleted file mode 100644 index 65194b0..0000000 --- a/tools/test/perform/pio_engine.c +++ /dev/null @@ -1,2747 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Author: Albert Cheng of NCSA, Oct 24, 2001. - */ - -#include "hdf5.h" - -#ifdef H5_STDC_HEADERS -#include -#include -#include -#include -#endif - -#ifdef H5_HAVE_UNISTD_H -#include -#include -#endif - -#ifdef H5_HAVE_SYS_STAT_H -#include -#endif - -#ifdef H5_HAVE_PARALLEL - -#include - -#ifndef MPI_FILE_NULL /*MPIO may be defined in mpi.h already */ -#include -#endif /* !MPI_FILE_NULL */ - -#include "pio_perf.h" - -/* Macro definitions */ - -#if H5_VERS_MAJOR == 1 && H5_VERS_MINOR == 6 -#define H5DCREATE(fd, name, type, space, dcpl) H5Dcreate(fd, name, type, space, dcpl) -#define H5DOPEN(fd, name) H5Dopen(fd, name) -#else -#define H5DCREATE(fd, name, type, space, dcpl) \ - H5Dcreate2(fd, name, type, space, H5P_DEFAULT, dcpl, H5P_DEFAULT) -#define H5DOPEN(fd, name) H5Dopen2(fd, name, H5P_DEFAULT) -#endif - -/* sizes of various items. these sizes won't change during program execution */ -/* The following three must have the same type */ -#define ELMT_H5_TYPE H5T_NATIVE_UCHAR - -#define GOTOERROR(errcode) \ - { \ - ret_code = errcode; \ - goto done; \ - } -#define ERRMSG(mesg) \ - { \ - HDfprintf(stderr, "Proc %d: ", pio_mpi_rank_g); \ - HDfprintf(stderr, "*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ - } - -/* verify: if val is false (0), print mesg. */ -#define VRFY(val, mesg) \ - do { \ - if (!val) { \ - ERRMSG(mesg); \ - GOTOERROR(FAIL); \ - } \ - } while (0) - -/* POSIX I/O macros */ -#ifdef H5_HAVE_WIN32_API -/* Can't link against the library, so this test will use the older, non-Unicode - * _open() call on Windows. - */ -#define HDopen(S, F, ...) _open(S, F | _O_BINARY, __VA_ARGS__) -#endif /* H5_HAVE_WIN32_API */ -#define POSIXCREATE(fn) HDopen(fn, O_CREAT | O_TRUNC | O_RDWR, 0600) -#define POSIXOPEN(fn, F) HDopen(fn, F, 0600) -#define POSIXCLOSE(F) HDclose(F) -#define POSIXSEEK(F, L) HDlseek(F, L, SEEK_SET) -#define POSIXWRITE(F, B, S) HDwrite(F, B, S) -#define POSIXREAD(F, B, S) HDread(F, B, S) - -enum { PIO_CREATE = 1, PIO_WRITE = 2, PIO_READ = 4 }; - -/* Global variables */ -static int clean_file_g = -1; /*whether to cleanup temporary test */ -/*files. -1 is not defined; */ -/*0 is no cleanup; 1 is do cleanup */ - -/* - * In a parallel machine, the filesystem suitable for compiling is - * unlikely a parallel file system that is suitable for parallel I/O. - * There is no standard pathname for the parallel file system. /tmp - * is about the best guess. - */ -#ifndef HDF5_PARAPREFIX -#define HDF5_PARAPREFIX "" -#endif /* !HDF5_PARAPREFIX */ - -#ifndef MIN -#define MIN(a, b) ((a) < (b) ? (a) : (b)) -#endif /* !MIN */ - -/* the different types of file descriptors we can expect */ -typedef union _file_descr { - int posixfd; /* POSIX file handle*/ - MPI_File mpifd; /* MPI file */ - hid_t h5fd; /* HDF5 file */ -} file_descr; - -/* local functions */ -static char * pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size); -static herr_t do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nelmts, - size_t buf_size, void *buffer); -static herr_t do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nelmts, - size_t buf_size, void *buffer /*out*/); -static herr_t do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags); -static herr_t do_fclose(iotype iot, file_descr *fd); -static void do_cleanupfile(iotype iot, char *fname); -static off_t sqrto(off_t); - -/* - * Function: do_pio - * Purpose: PIO Engine where Parallel IO are executed. - * Return: results - * Programmer: Albert Cheng, Bill Wendling 2001/12/12 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -results -do_pio(parameters param) -{ - /* return codes */ - herr_t ret_code = 0; /*return code */ - results res; - - file_descr fd; - iotype iot; - - char fname[FILENAME_MAX]; - long nf; - long ndsets; - off_t nbytes; /*number of bytes per dataset */ - off_t snbytes; /*general dataset size */ - /*for 1D, it is the actual dataset size */ - /*for 2D, it is the size of a side of the dataset square */ - char * buffer = NULL; /*data buffer pointer */ - size_t buf_size; /*general buffer size in bytes */ - /*for 1D, it is the actual buffer size */ - /*for 2D, it is the length of the buffer rectangle */ - size_t blk_size; /*data block size in bytes */ - size_t bsize; /*actual buffer size */ - - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - - /* Sanity check parameters */ - - /* IO type */ - iot = param.io_type; - - switch (iot) { - case MPIO: - fd.mpifd = MPI_FILE_NULL; - res.timers = io_time_new(MPI_CLOCK); - break; - case POSIXIO: - fd.posixfd = -1; - res.timers = io_time_new(MPI_CLOCK); - break; - case PHDF5: - fd.h5fd = -1; - res.timers = io_time_new(MPI_CLOCK); - break; - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", iot); - GOTOERROR(FAIL); - } - - ndsets = param.num_dsets; /* number of datasets per file */ - nbytes = param.num_bytes; /* number of bytes per dataset */ - buf_size = param.buf_size; - blk_size = param.blk_size; - - if (!param.dim2d) { - snbytes = nbytes; /* General dataset size */ - bsize = buf_size; /* Actual buffer size */ - } - else { - snbytes = sqrto(nbytes); /* General dataset size */ - bsize = buf_size * blk_size; /* Actual buffer size */ - } - - if (param.num_files < 0) { - HDfprintf(stderr, "number of files must be >= 0 (%ld)\n", param.num_files); - GOTOERROR(FAIL); - } - - if (ndsets < 0) { - HDfprintf(stderr, "number of datasets per file must be >= 0 (%ld)\n", ndsets); - GOTOERROR(FAIL); - } - - if (param.num_procs <= 0) { - HDfprintf(stderr, "maximum number of process to use must be > 0 (%d)\n", param.num_procs); - GOTOERROR(FAIL); - } - - /* Validate transfer buffer size & block size*/ - if (blk_size <= 0) { - HDfprintf(stderr, "Transfer block size (%zu) must be > 0\n", blk_size); - GOTOERROR(FAIL); - } - if (buf_size <= 0) { - HDfprintf(stderr, "Transfer buffer size (%zu) must be > 0\n", buf_size); - GOTOERROR(FAIL); - } - if ((buf_size % blk_size) != 0) { - HDfprintf(stderr, - "Transfer buffer size (%zu) must be a multiple of the " - "interleaved I/O block size (%zu)\n", - buf_size, blk_size); - GOTOERROR(FAIL); - } - if ((snbytes % pio_mpi_nprocs_g) != 0) { - HDfprintf(stderr, - "Dataset size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "number of processes (%d)\n", - (long long)snbytes, pio_mpi_nprocs_g); - GOTOERROR(FAIL); - } - - if (!param.dim2d) { - if (((size_t)(snbytes / pio_mpi_nprocs_g) % buf_size) != 0) { - HDfprintf(stderr, - "Dataset size/process (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "trasfer buffer size (%zu)\n", - (long long)(snbytes / pio_mpi_nprocs_g), buf_size); - GOTOERROR(FAIL); - } - } - else { - if (((size_t)snbytes % buf_size) != 0) { - HDfprintf(stderr, - "Dataset side size (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "trasfer buffer size (%zu)\n", - (long long)snbytes, buf_size); - GOTOERROR(FAIL); - } - } - - /* Allocate transfer buffer */ - if ((buffer = malloc(bsize)) == NULL) { - HDfprintf(stderr, "malloc for transfer buffer size (%zu) failed\n", bsize); - GOTOERROR(FAIL); - } - - if (pio_debug_level >= 4) { - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - - /* output all of the times for all iterations */ - if (myrank == 0) - HDfprintf(output, "Timer details:\n"); - } - - for (nf = 1; nf <= param.num_files; nf++) { - /* - * Write performance measurement - */ - /* Open file for write */ - char base_name[256]; - - HDsprintf(base_name, "#pio_tmp_%lu", nf); - pio_create_filename(iot, base_name, fname, sizeof(fname)); - if (pio_debug_level > 0) - HDfprintf(output, "rank %d: data filename=%s\n", pio_mpi_rank_g, fname); - - /* Need barrier to make sure everyone starts at the same time */ - MPI_Barrier(pio_comm_g); - - io_time_set(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTART); - hrc = do_fopen(¶m, fname, &fd, PIO_CREATE | PIO_WRITE); - - VRFY((hrc == SUCCESS), "do_fopen failed"); - - io_time_set(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTART); - hrc = do_write(&res, &fd, ¶m, ndsets, nbytes, buf_size, buffer); - io_time_set(res.timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTOP); - - VRFY((hrc == SUCCESS), "do_write failed"); - - /* Close file for write */ - hrc = do_fclose(iot, &fd); - - io_time_set(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_fclose failed"); - - if (!param.h5_write_only) { - /* - * Read performance measurement - */ - /* Need barrier to make sure everyone is done writing and has - * closed the file. Also to make sure everyone starts reading - * at the same time. - */ - MPI_Barrier(pio_comm_g); - - /* Open file for read */ - io_time_set(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTART); - hrc = do_fopen(¶m, fname, &fd, PIO_READ); - - VRFY((hrc == SUCCESS), "do_fopen failed"); - - io_time_set(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTART); - hrc = do_read(&res, &fd, ¶m, ndsets, nbytes, buf_size, buffer); - io_time_set(res.timers, HDF5_FINE_READ_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_read failed"); - - /* Close file for read */ - hrc = do_fclose(iot, &fd); - - io_time_set(res.timers, HDF5_GROSS_READ_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_fclose failed"); - } - - /* Need barrier to make sure everyone is done with the file */ - /* before it may be removed by do_cleanupfile */ - MPI_Barrier(pio_comm_g); - do_cleanupfile(iot, fname); - } - -done: - /* clean up */ - /* release HDF5 objects */ - - /* close any opened files */ - /* no remove(fname) because that should have happened normally. */ - switch (iot) { - case POSIXIO: - if (fd.posixfd != -1) - hrc = do_fclose(iot, &fd); - break; - case MPIO: - if (fd.mpifd != MPI_FILE_NULL) - hrc = do_fclose(iot, &fd); - break; - case PHDF5: - if (fd.h5fd != -1) - hrc = do_fclose(iot, &fd); - break; - default: - break; - } - - /* release generic resources */ - if (buffer) - HDfree(buffer); - res.ret_code = ret_code; - return res; -} - -/* - * Function: pio_create_filename - * Purpose: Create a new filename to write to. Determine the correct - * suffix to append to the filename by the type of I/O we're - * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if - * USER or LOGIN are specified in the environment. - * Return: Pointer to filename or NULL - * Programmer: Bill Wendling, 21. November 2001 - * Modifications: - */ -static char * -pio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size) -{ - const char *prefix, *suffix = ""; - char * ptr, last = '\0'; - size_t i, j; - - if (!base_name || !fullname || size < 1) - return NULL; - - HDmemset(fullname, 0, size); - - switch (iot) { - case POSIXIO: - suffix = ".posix"; - break; - case MPIO: - suffix = ".mpio"; - break; - case PHDF5: - suffix = ".h5"; - break; - default: - break; - } - - /* First use the environment variable and then try the constant */ - prefix = HDgetenv("HDF5_PARAPREFIX"); - -#ifdef HDF5_PARAPREFIX - if (!prefix) - prefix = HDF5_PARAPREFIX; -#endif /* HDF5_PARAPREFIX */ - - /* Prepend the prefix value to the base name */ - if (prefix && *prefix) { - /* If the prefix specifies the HDF5_PARAPREFIX directory, then - * default to using the "/tmp/$USER" or "/tmp/$LOGIN" - * directory instead. */ - register char *user, *login, *subdir; - - user = HDgetenv("USER"); - login = HDgetenv("LOGIN"); - subdir = (user ? user : login); - - if (subdir) { - for (i = 0; i < size - 1 && prefix[i]; i++) - fullname[i] = prefix[i]; - - fullname[i++] = '/'; - - for (j = 0; i < size && subdir[j]; i++, j++) - fullname[i] = subdir[j]; - } - else { - /* We didn't append the prefix yet */ - HDstrncpy(fullname, prefix, size); - fullname[size - 1] = '\0'; - } - - if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) { - /* Append the base_name with a slash first. Multiple slashes are - * handled below. */ - h5_stat_t buf; - - if (HDstat(fullname, &buf) < 0) - /* The directory doesn't exist just yet */ - if (HDmkdir(fullname, (mode_t)0755) < 0 && errno != EEXIST) { - /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. - * Default to PREFIX's original prefix value. */ - HDstrcpy(fullname, prefix); - } - - HDstrcat(fullname, "/"); - HDstrcat(fullname, base_name); - } - else { - /* Buffer is too small */ - return NULL; - } - } - else if (HDstrlen(base_name) >= size) { - /* Buffer is too small */ - return NULL; - } - else { - HDstrcpy(fullname, base_name); - } - - /* Append a suffix */ - if (suffix) { - if (HDstrlen(fullname) + HDstrlen(suffix) >= size) - return NULL; - - HDstrcat(fullname, suffix); - } - - /* Remove any double slashes in the filename */ - for (ptr = fullname, i = j = 0; ptr && i < size; i++, ptr++) { - if (*ptr != '/' || last != '/') - fullname[j++] = *ptr; - - last = *ptr; - } - - return fullname; -} - -/* - * Function: do_write - * Purpose: Write the required amount of data to the file. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static herr_t -do_write(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbytes, size_t buf_size, - void *buffer) -{ - int ret_code = SUCCESS; - int rc; /*routine return code */ - long ndset; - size_t blk_size; /* The block size to subdivide the xfer buffer into */ - off_t nbytes_xfer; /* Total number of bytes transferred so far */ - size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */ - size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */ - char dname[64]; - off_t dset_offset = 0; /*dataset offset in a file */ - off_t bytes_begin[2]; /*first elmt this process transfer */ - off_t bytes_count; /*number of elmts this process transfer */ - off_t snbytes = 0; /*size of a side of the dataset square */ - unsigned char *buf_p; /* Current buffer pointer */ - - /* POSIX variables */ - off_t file_offset; /* File offset of the next transfer */ - off_t file_offset_advance; /* File offset advance after each I/O operation */ - off_t posix_file_offset; /* Base file offset of the next transfer */ - - /* MPI variables */ - MPI_Offset mpi_file_offset; /* Base file offset of the next transfer*/ - MPI_Offset mpi_offset; /* Offset in MPI file */ - MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */ - MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */ - MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */ - MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */ - MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */ - MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */ - MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */ - MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */ - MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */ - MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */ - MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */ - MPI_Status mpi_status; - int mrc; /* MPI return code */ - - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - hsize_t h5dims[2]; /*dataset dim sizes */ - hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ - hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ - hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ - hsize_t h5block[2]; /*dataspace selection */ - hsize_t h5stride[2]; - hsize_t h5count[2]; - hsize_t h5start[2]; - hssize_t h5offset[2]; /* Selection offset within dataspace */ - hid_t h5dcpl = H5I_INVALID_HID; /* Dataset creation property list */ - hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ - - /* Get the parameters from the parameter block */ - blk_size = parms->blk_size; - - /* There are two kinds of transfer patterns, contiguous and interleaved. - * Let 0,1,2,...,n be data accessed by process 0,1,2,...,n - * where n is rank of the last process. - * In contiguous pattern, data are accessed as - * 000...111...222...nnn... - * In interleaved pattern, data are accessed as - * 012...n012...n... - * These are all in the scope of one dataset. - */ - - /* 1D dataspace */ - if (!parms->dim2d) { - /* Contiguous Pattern: */ - if (!parms->interleaved) { - bytes_begin[0] = (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); - } /* end if */ - /* Interleaved Pattern: */ - else { - bytes_begin[0] = (off_t)(blk_size * (size_t)pio_mpi_rank_g); - } /* end else */ - - /* Prepare buffer for verifying data */ - if (parms->verify) - memset(buffer, pio_mpi_rank_g + 1, buf_size); - } /* end if */ - /* 2D dataspace */ - else { - /* nbytes is always the number of bytes per dataset (1D or 2D). If the - dataspace is 2D, snbytes is the size of a side of the dataset square. - */ - snbytes = sqrto(nbytes); - - /* Contiguous Pattern: */ - if (!parms->interleaved) { - bytes_begin[0] = (off_t)((double)snbytes * pio_mpi_rank_g / pio_mpi_nprocs_g); - bytes_begin[1] = 0; - } /* end if */ - /* Interleaved Pattern: */ - else { - bytes_begin[0] = 0; - - if (!parms->h5_use_chunks || parms->io_type == PHDF5) - bytes_begin[1] = (off_t)(blk_size * (size_t)pio_mpi_rank_g); - else - bytes_begin[1] = (off_t)(blk_size * blk_size * (size_t)pio_mpi_rank_g); - } /* end else */ - - /* Prepare buffer for verifying data */ - if (parms->verify) - HDmemset(buffer, pio_mpi_rank_g + 1, buf_size * blk_size); - } /* end else */ - - /* Calculate the total number of bytes (bytes_count) to be - * transferred by this process. It may be different for different - * transfer pattern due to rounding to integral values. - */ - /* - * Calculate the beginning bytes of this process and the next. - * bytes_count is the difference between these two beginnings. - * This way, it eliminates any rounding errors. - * (This is tricky, don't mess with the formula, rounding errors - * can easily get introduced) */ - bytes_count = (off_t)(((double)nbytes * (pio_mpi_rank_g + 1)) / pio_mpi_nprocs_g) - - (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); - - /* debug */ - if (pio_debug_level >= 4) { - HDprint_rank(output); - if (!parms->dim2d) { - HDfprintf(output, - "Debug(do_write): " - "buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH - "d\n", - buf_size, (long long)bytes_begin[0], (long long)bytes_count); - } - else { - HDfprintf(output, - "Debug(do_write): " - "linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH - "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n", - buf_size * blk_size, (long long)bytes_begin[0], (long long)bytes_begin[1], - (long long)bytes_count); - } - } - - /* I/O Access specific setup */ - switch (parms->io_type) { - case POSIXIO: - /* No extra setup */ - break; - - case MPIO: /* MPI-I/O setup */ - /* 1D dataspace */ - if (!parms->dim2d) { - /* Build block's derived type */ - mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Build file's derived type */ - mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)pio_mpi_nprocs_g, mpi_blk_type, - &mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit file type */ - mrc = MPI_Type_commit(&mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Commit buffer type */ - mrc = MPI_Type_commit(&mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - } /* end if */ - /* 2D dataspace */ - else { - /* Build partial buffer derived type for contiguous access */ - - mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE, &mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit partial buffer derived type */ - mrc = MPI_Type_commit(&mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build contiguous file's derived type */ - mrc = MPI_Type_vector((int)blk_size, (int)1, (int)((size_t)snbytes / buf_size), - mpi_partial_buffer_cont, &mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit contiguous file type */ - mrc = MPI_Type_commit(&mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build partial buffer derived type for interleaved access */ - mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit partial buffer derived type */ - mrc = MPI_Type_commit(&mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build interleaved file's derived type */ - mrc = MPI_Type_vector((int)buf_size, (int)1, (int)((size_t)snbytes / blk_size), - mpi_partial_buffer_inter, &mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit interleaved file type */ - mrc = MPI_Type_commit(&mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build full buffer derived type */ - mrc = MPI_Type_contiguous((int)(blk_size * buf_size), MPI_BYTE, &mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit full buffer derived type */ - mrc = MPI_Type_commit(&mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build full chunk derived type */ - mrc = MPI_Type_contiguous((int)(blk_size * blk_size), MPI_BYTE, &mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit full chunk derived type */ - mrc = MPI_Type_commit(&mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build chunk interleaved file's derived type */ - mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)((size_t)snbytes / blk_size), - mpi_full_chunk, &mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit chunk interleaved file type */ - mrc = MPI_Type_commit(&mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - } /* end else */ - break; - - case PHDF5: /* HDF5 setup */ - /* 1D dataspace */ - if (!parms->dim2d) { - if (nbytes > 0) { - /* define a contiguous dataset of nbytes native bytes */ - h5dims[0] = (hsize_t)nbytes; - h5dset_space_id = H5Screate_simple(1, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - /* Set up the file dset space id to select the pattern to access */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5stride[0] = h5block[0] = blk_size; - h5count[0] = buf_size / blk_size; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5stride[0] = blk_size * (size_t)pio_mpi_nprocs_g; - h5block[0] = blk_size; - h5count[0] = buf_size / blk_size; - } /* end else */ - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, - h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - } /* end if */ - else { - h5dset_space_id = H5Screate(H5S_SCALAR); - VRFY((h5dset_space_id >= 0), "H5Screate"); - } /* end else */ - - /* Create the memory dataspace that corresponds to the xfer buffer */ - if (buf_size > 0) { - h5dims[0] = buf_size; - h5mem_space_id = H5Screate_simple(1, h5dims, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - } /* end if */ - else { - h5mem_space_id = H5Screate(H5S_SCALAR); - VRFY((h5mem_space_id >= 0), "H5Screate"); - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - if (nbytes > 0) { - /* define a contiguous dataset of nbytes native bytes */ - h5dims[0] = (hsize_t)snbytes; - h5dims[1] = (hsize_t)snbytes; - h5dset_space_id = H5Screate_simple(2, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - /* Set up the file dset space id to select the pattern to access */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5start[1] = (hsize_t)bytes_begin[1]; - h5stride[0] = 1; - h5stride[1] = h5block[0] = h5block[1] = blk_size; - h5count[0] = 1; - h5count[1] = buf_size / blk_size; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5start[1] = (hsize_t)bytes_begin[1]; - h5stride[0] = blk_size; - h5stride[1] = blk_size * (size_t)pio_mpi_nprocs_g; - h5block[0] = h5block[1] = blk_size; - h5count[0] = buf_size / blk_size; - h5count[1] = 1; - } /* end else */ - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, - h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - } /* end if */ - else { - h5dset_space_id = H5Screate(H5S_SCALAR); - VRFY((h5dset_space_id >= 0), "H5Screate"); - } /* end else */ - - /* Create the memory dataspace that corresponds to the xfer buffer */ - if (buf_size > 0) { - if (!parms->interleaved) { - h5dims[0] = blk_size; - h5dims[1] = buf_size; - } - else { - h5dims[0] = buf_size; - h5dims[1] = blk_size; - } - h5mem_space_id = H5Screate_simple(2, h5dims, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - } /* end if */ - else { - h5mem_space_id = H5Screate(H5S_SCALAR); - VRFY((h5mem_space_id >= 0), "H5Screate"); - } /* end else */ - } /* end else */ - - /* Create the dataset transfer property list */ - h5dxpl = H5Pcreate(H5P_DATASET_XFER); - if (h5dxpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - /* Change to collective I/O, if asked */ - if (parms->collective) { - hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - break; - - default: - break; - } /* end switch */ - - for (ndset = 1; ndset <= ndsets; ++ndset) { - - /* Calculate dataset offset within a file */ - - /* create dataset */ - switch (parms->io_type) { - case POSIXIO: - case MPIO: - /* both posix and mpi io just need dataset offset in file*/ - dset_offset = (ndset - 1) * nbytes; - break; - - case PHDF5: - h5dcpl = H5Pcreate(H5P_DATASET_CREATE); - if (h5dcpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - /* 1D dataspace */ - if (!parms->dim2d) { - /* Make the dataset chunked if asked */ - if (parms->h5_use_chunks) { - /* Set the chunk size to be the same as the buffer size */ - h5dims[0] = blk_size; - hrc = H5Pset_chunk(h5dcpl, 1, h5dims); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - } /* end if */ - else { - /* 2D dataspace */ - if (parms->h5_use_chunks) { - /* Set the chunk size to be the same as the block size */ - h5dims[0] = blk_size; - h5dims[1] = blk_size; - hrc = H5Pset_chunk(h5dcpl, 2, h5dims); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - } /* end else */ - - HDsprintf(dname, "Dataset_%ld", ndset); - h5ds_id = H5DCREATE(fd->h5fd, dname, ELMT_H5_TYPE, h5dset_space_id, h5dcpl); - - if (h5ds_id < 0) { - HDfprintf(stderr, "HDF5 Dataset Create failed\n"); - GOTOERROR(FAIL); - } - - hrc = H5Pclose(h5dcpl); - /* verifying the close of the dcpl */ - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Close failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - break; - } - - /* The task is to transfer bytes_count bytes, starting at - * bytes_begin position, using transfer buffer of buf_size bytes. - * If interleaved, select buf_size at a time, in round robin - * fashion, according to number of process. Otherwise, select - * all bytes_count in contiguous. - */ - nbytes_xfer = 0; - - /* 1D dataspace */ - if (!parms->dim2d) { - /* Set base file offset for all I/O patterns and POSIX access */ - posix_file_offset = dset_offset + bytes_begin[0]; - - /* Set base file offset for all I/O patterns and MPI access */ - mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]); - } /* end if */ - else { - /* Set base file offset for all I/O patterns and POSIX access */ - posix_file_offset = dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]; - - /* Set base file offset for all I/O patterns and MPI access */ - mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]); - } /* end else */ - - /* Start "raw data" write timer */ - io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART); - - while (nbytes_xfer < bytes_count) { - /* Write */ - /* Calculate offset of write within a dataset/file */ - switch (parms->io_type) { - case POSIXIO: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Contiguous pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + (off_t)nbytes_xfer; - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are written */ - rc = ((ssize_t)buf_size == POSIXWRITE(fd->posixfd, buffer, buf_size)); - VRFY((rc != 0), "POSIXWRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size; - - /* Loop over the buffers to write */ - while (nbytes_toxfer > 0) { - /* Skip offset over blocks of other processes */ - file_offset = posix_file_offset + (off_t)(nbytes_xfer * pio_mpi_nprocs_g); - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are written */ - rc = ((ssize_t)blk_size == POSIXWRITE(fd->posixfd, buf_p, blk_size)); - VRFY((rc != 0), "POSIXWRITE"); - - /* Advance location in buffer */ - buf_p += blk_size; - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)blk_size; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= blk_size; - } /* end while */ - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* Contiguous storage */ - if (!parms->h5_use_chunks) { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + - (off_t)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * - (blk_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / blk_size) % (size_t)snbytes)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = buf_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = (off_t)snbytes; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute file offset */ - file_offset = - posix_file_offset + - (off_t)(((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / - (size_t)snbytes) * - (buf_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % - (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = (off_t)snbytes; - } /* end else */ - } /* end if */ - /* Chunked storage */ - else { - /*Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + (off_t)nbytes_xfer; - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * buf_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = 0; - } /* end if */ - /*Interleaved access pattern */ - else { - /* Compute file offset */ - /* Before simplification */ - /* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size) - *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size - *snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size)) - *pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */ - - file_offset = posix_file_offset + - (off_t)((((size_t)nbytes_xfer / (buf_size / blk_size) * - (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * blk_size)) * - (buf_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / (buf_size / blk_size)) * - (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * blk_size)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * blk_size; - - /* Global offset advance after each I/O operation */ - /* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */ - file_offset_advance = (off_t)snbytes * (off_t)blk_size; - } /* end else */ - } /* end else */ - - /* Common code for file access */ - - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size * blk_size; - - /* Loop over portions of the buffer to write */ - while (nbytes_toxfer > 0) { - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are written */ - rc = ((ssize_t)nbytes_xfer_advance == - POSIXWRITE(fd->posixfd, buf_p, nbytes_xfer_advance)); - VRFY((rc != 0), "POSIXWRITE"); - - /* Advance location in buffer */ - buf_p += nbytes_xfer_advance; - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)nbytes_xfer_advance; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= nbytes_xfer_advance; - - /* Partially advance file offset */ - file_offset += file_offset_advance; - } /* end while */ - - } /* end else */ - - break; - - case MPIO: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Independent file access */ - if (!parms->collective) { - /* Contiguous pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Perform independent write */ - mrc = - MPI_File_write_at(fd->mpifd, mpi_offset, buffer, - (int)(buf_size / blk_size), mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size; - - /* Loop over the buffers to write */ - while (nbytes_toxfer > 0) { - /* Skip offset over blocks of other processes */ - mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); - - /* Perform independent write */ - mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p, (int)1, - mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance location in buffer */ - buf_p += blk_size; - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)blk_size; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= blk_size; - } /* end while */ - } /* end else */ - } /* end if */ - /* Collective file access */ - else { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Perform independent write */ - mrc = MPI_File_write_at_all(fd->mpifd, mpi_offset, buffer, - (int)(buf_size / blk_size), mpi_blk_type, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); - - /* Set the file view */ - mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, - (char *)"native", h5_io_info_g); - VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); - - /* Perform write */ - mrc = MPI_File_write_at_all(fd->mpifd, 0, buffer, (int)(buf_size / blk_size), - mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)buf_size; - } /* end else */ - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* Contiguous storage */ - if (!parms->h5_use_chunks) { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = - mpi_file_offset + - (MPI_Offset)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * - (blk_size * (size_t)snbytes)) + - (MPI_Offset)(((size_t)nbytes_xfer / blk_size) % (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = buf_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = snbytes; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_cont_type; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute offset in file */ - mpi_offset = - mpi_file_offset + - (MPI_Offset)( - ((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / - (size_t)snbytes) * - (buf_size * (size_t)snbytes)) + - (MPI_Offset)( - (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % - (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = snbytes; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_inter_type; - } /* end else */ - } /* end if */ - /* Chunked storage */ - else { - /*Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * buf_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = 0; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_full_buffer; - } /* end if */ - /*Interleaved access pattern */ - else { - /* Compute offset in file */ - /* Before simplification */ - /* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size) - *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))* - (buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+ - ((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes - /blk_size*(blk_size*blk_size)); */ - mpi_offset = mpi_file_offset + - (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size) * - (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * blk_size)) * - (buf_size * (size_t)snbytes)) + - (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size)) * - (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * blk_size)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * blk_size; - - /* Global offset advance after each I/O operation */ - /* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */ - mpi_offset_advance = (MPI_Offset)((size_t)snbytes * blk_size); - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_chunk_inter_type; - } /* end else */ - } /* end else */ - - /* Common code for independent file access */ - if (!parms->collective) { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size * blk_size; - - /* Loop over portions of the buffer to write */ - while (nbytes_toxfer > 0) { - /* Perform independent write */ - mrc = MPI_File_write_at(fd->mpifd, mpi_offset, buf_p, - (int)nbytes_xfer_advance, MPI_BYTE, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance location in buffer */ - buf_p += nbytes_xfer_advance; - - /* Advance global offset in dataset */ - nbytes_xfer += (ssize_t)nbytes_xfer_advance; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= nbytes_xfer_advance; - - /* Partially advance global offset in dataset */ - mpi_offset += mpi_offset_advance; - } /* end while */ - } /* end if */ - - /* Common code for collective file access */ - else { - /* Set the file view */ - mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, - (char *)"native", h5_io_info_g); - VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); - - /* Perform write */ - MPI_File_write_at_all(fd->mpifd, 0, buffer, (int)(buf_size * blk_size), MPI_BYTE, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_WRITE"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size * (off_t)blk_size; - } /* end else */ - - } /* end else */ - - break; - - case PHDF5: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Set up the file dset space id to move the selection to process */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5offset[0] = nbytes_xfer; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5offset[0] = (nbytes_xfer * pio_mpi_nprocs_g); - } /* end else */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Write the buffer out */ - hrc = - H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dwrite"); - - /* Increment number of bytes transferred */ - nbytes_xfer += (ssize_t)buf_size; - } /* end if */ - /* 2D dataspace */ - else { - /* Set up the file dset space id to move the selection to process */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5offset[0] = - (hssize_t)(((size_t)nbytes_xfer / ((size_t)snbytes * blk_size)) * blk_size); - h5offset[1] = - (hssize_t)(((size_t)nbytes_xfer % ((size_t)snbytes * blk_size)) / blk_size); - - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5offset[0] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * buf_size)) * - buf_size); - h5offset[1] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * buf_size)) / - buf_size); - - } /* end else */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Write the buffer out */ - hrc = - H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dwrite"); - - /* Increment number of bytes transferred */ - nbytes_xfer += (off_t)buf_size * (off_t)blk_size; - - } /* end else */ - - break; - - default: - break; - } /* switch (parms->io_type) */ - } /* end while */ - - /* Stop "raw data" write timer */ - io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP); - - /* Calculate write time */ - - /* Close dataset. Only HDF5 needs to do an explicit close. */ - if (parms->io_type == PHDF5) { - hrc = H5Dclose(h5ds_id); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Close failed\n"); - GOTOERROR(FAIL); - } - - h5ds_id = H5I_INVALID_HID; - } /* end if */ - } /* end for */ - -done: - /* release MPI-I/O objects */ - if (parms->io_type == MPIO) { - /* 1D dataspace */ - if (!parms->dim2d) { - /* Free file type */ - mrc = MPI_Type_free(&mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free buffer type */ - mrc = MPI_Type_free(&mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - } /* end if */ - /* 2D dataspace */ - else { - /* Free partial buffer type for contiguous access */ - mrc = MPI_Type_free(&mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free contiguous file type */ - mrc = MPI_Type_free(&mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free partial buffer type for interleaved access */ - mrc = MPI_Type_free(&mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free interleaved file type */ - mrc = MPI_Type_free(&mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free full buffer type */ - mrc = MPI_Type_free(&mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free full chunk type */ - mrc = MPI_Type_free(&mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free chunk interleaved file type */ - mrc = MPI_Type_free(&mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - } /* end else */ - } /* end if */ - - /* release HDF5 objects */ - if (h5dset_space_id != -1) { - hrc = H5Sclose(h5dset_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); - ret_code = FAIL; - } - else { - h5dset_space_id = H5I_INVALID_HID; - } - } - - if (h5mem_space_id != -1) { - hrc = H5Sclose(h5mem_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); - ret_code = FAIL; - } - else { - h5mem_space_id = H5I_INVALID_HID; - } - } - - if (h5dxpl != -1) { - hrc = H5Pclose(h5dxpl); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); - ret_code = FAIL; - } - else { - h5dxpl = H5I_INVALID_HID; - } - } - - return ret_code; -} - -static off_t -sqrto(off_t x) -{ - double root_x = sqrt((double)x); - return (off_t)root_x; -} - -/* - * Function: do_read - * Purpose: read the required amount of data from the file. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng 2001/12/13 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static herr_t -do_read(results *res, file_descr *fd, parameters *parms, long ndsets, off_t nbytes, size_t buf_size, - void *buffer /*out*/) -{ - int ret_code = SUCCESS; - int rc; /*routine return code */ - long ndset; - size_t blk_size; /* The block size to subdivide the xfer buffer into */ - size_t bsize; /* Size of the actual buffer */ - off_t nbytes_xfer; /* Total number of bytes transferred so far */ - size_t nbytes_xfer_advance; /* Number of bytes transferred in a single I/O operation */ - size_t nbytes_toxfer; /* Number of bytes to transfer a particular time */ - char dname[64]; - off_t dset_offset = 0; /*dataset offset in a file */ - off_t bytes_begin[2]; /*first elmt this process transfer */ - off_t bytes_count; /*number of elmts this process transfer */ - off_t snbytes = 0; /*size of a side of the dataset square */ - unsigned char *buf_p; /* Current buffer pointer */ - - /* POSIX variables */ - off_t file_offset; /* File offset of the next transfer */ - off_t file_offset_advance; /* File offset advance after each I/O operation */ - off_t posix_file_offset; /* Base file offset of the next transfer */ - - /* MPI variables */ - MPI_Offset mpi_file_offset; /* Base file offset of the next transfer*/ - MPI_Offset mpi_offset; /* Offset in MPI file */ - MPI_Offset mpi_offset_advance; /* Offset advance after each I/O operation */ - MPI_Datatype mpi_file_type; /* MPI derived type for 1D file */ - MPI_Datatype mpi_blk_type; /* MPI derived type for 1D buffer */ - MPI_Datatype mpi_cont_type; /* MPI derived type for 2D contiguous file */ - MPI_Datatype mpi_partial_buffer_cont; /* MPI derived type for partial 2D contiguous buffer */ - MPI_Datatype mpi_inter_type; /* MPI derived type for 2D interleaved file */ - MPI_Datatype mpi_partial_buffer_inter; /* MPI derived type for partial 2D interleaved buffer */ - MPI_Datatype mpi_full_buffer; /* MPI derived type for 2D full buffer */ - MPI_Datatype mpi_full_chunk; /* MPI derived type for 2D full chunk */ - MPI_Datatype mpi_chunk_inter_type; /* MPI derived type for 2D chunk interleaved file */ - MPI_Datatype mpi_collective_type; /* Generic MPI derived type for 2D collective access */ - MPI_Status mpi_status; - int mrc; /* MPI return code */ - - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - hsize_t h5dims[2]; /*dataset dim sizes */ - hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ - hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ - hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ - hsize_t h5block[2]; /*dataspace selection */ - hsize_t h5stride[2]; - hsize_t h5count[2]; - hsize_t h5start[2]; - hssize_t h5offset[2]; /* Selection offset within dataspace */ - hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ - - /* Get the parameters from the parameter block */ - blk_size = parms->blk_size; - - /* There are two kinds of transfer patterns, contiguous and interleaved. - * Let 0,1,2,...,n be data accessed by process 0,1,2,...,n - * where n is rank of the last process. - * In contiguous pattern, data are accessed as - * 000...111...222...nnn... - * In interleaved pattern, data are accessed as - * 012...n012...n... - * These are all in the scope of one dataset. - */ - - /* 1D dataspace */ - if (!parms->dim2d) { - bsize = buf_size; - /* Contiguous Pattern: */ - if (!parms->interleaved) { - bytes_begin[0] = (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); - } /* end if */ - /* Interleaved Pattern: */ - else { - bytes_begin[0] = (off_t)blk_size * (off_t)pio_mpi_rank_g; - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* nbytes is always the number of bytes per dataset (1D or 2D). If the - dataspace is 2D, snbytes is the size of a side of the 'dataset square'. - */ - snbytes = sqrto(nbytes); - - bsize = buf_size * blk_size; - - /* Contiguous Pattern: */ - if (!parms->interleaved) { - bytes_begin[0] = (off_t)((double)snbytes * pio_mpi_rank_g / pio_mpi_nprocs_g); - bytes_begin[1] = 0; - } /* end if */ - /* Interleaved Pattern: */ - else { - bytes_begin[0] = 0; - - if (!parms->h5_use_chunks || parms->io_type == PHDF5) - bytes_begin[1] = (off_t)blk_size * (off_t)pio_mpi_rank_g; - else - bytes_begin[1] = (off_t)blk_size * (off_t)blk_size * (off_t)pio_mpi_rank_g; - } /* end else */ - } /* end else */ - - /* Calculate the total number of bytes (bytes_count) to be - * transferred by this process. It may be different for different - * transfer pattern due to rounding to integral values. - */ - /* - * Calculate the beginning bytes of this process and the next. - * bytes_count is the difference between these two beginnings. - * This way, it eliminates any rounding errors. - * (This is tricky, don't mess with the formula, rounding errors - * can easily get introduced) */ - bytes_count = (off_t)(((double)nbytes * (pio_mpi_rank_g + 1)) / pio_mpi_nprocs_g) - - (off_t)(((double)nbytes * pio_mpi_rank_g) / pio_mpi_nprocs_g); - - /* debug */ - if (pio_debug_level >= 4) { - HDprint_rank(output); - if (!parms->dim2d) { - HDfprintf(output, - "Debug(do_write): " - "buf_size=%zu, bytes_begin=%" H5_PRINTF_LL_WIDTH "d, bytes_count=%" H5_PRINTF_LL_WIDTH - "d\n", - buf_size, (long long)bytes_begin[0], (long long)bytes_count); - } - else { - HDfprintf(output, - "Debug(do_write): " - "linear buf_size=%zu, bytes_begin=(%" H5_PRINTF_LL_WIDTH "d,%" H5_PRINTF_LL_WIDTH - "d), bytes_count=%" H5_PRINTF_LL_WIDTH "d\n", - buf_size * blk_size, (long long)bytes_begin[0], (long long)bytes_begin[1], - (long long)bytes_count); - } - } - - /* I/O Access specific setup */ - switch (parms->io_type) { - case POSIXIO: - /* No extra setup */ - break; - - case MPIO: /* MPI-I/O setup */ - /* 1D dataspace */ - if (!parms->dim2d) { - /* Build block's derived type */ - mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Build file's derived type */ - mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)pio_mpi_nprocs_g, mpi_blk_type, - &mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit file type */ - mrc = MPI_Type_commit(&mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Commit buffer type */ - mrc = MPI_Type_commit(&mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - } /* end if */ - /* 2D dataspace */ - else { - /* Build partial buffer derived type for contiguous access */ - mrc = MPI_Type_contiguous((int)buf_size, MPI_BYTE, &mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit partial buffer derived type */ - mrc = MPI_Type_commit(&mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build contiguous file's derived type */ - mrc = MPI_Type_vector((int)blk_size, (int)1, (int)((size_t)snbytes / buf_size), - mpi_partial_buffer_cont, &mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit contiguous file type */ - mrc = MPI_Type_commit(&mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build partial buffer derived type for interleaved access */ - mrc = MPI_Type_contiguous((int)blk_size, MPI_BYTE, &mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit partial buffer derived type */ - mrc = MPI_Type_commit(&mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build interleaved file's derived type */ - mrc = MPI_Type_vector((int)buf_size, (int)1, (int)((size_t)snbytes / blk_size), - mpi_partial_buffer_inter, &mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit interleaved file type */ - mrc = MPI_Type_commit(&mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build full buffer derived type */ - mrc = MPI_Type_contiguous((int)(blk_size * buf_size), MPI_BYTE, &mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit full buffer derived type */ - mrc = MPI_Type_commit(&mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build full chunk derived type */ - mrc = MPI_Type_contiguous((int)(blk_size * blk_size), MPI_BYTE, &mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit full chunk derived type */ - mrc = MPI_Type_commit(&mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - - /* Build chunk interleaved file's derived type */ - mrc = MPI_Type_vector((int)(buf_size / blk_size), (int)1, (int)((size_t)snbytes / blk_size), - mpi_full_chunk, &mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_CREATE"); - - /* Commit chunk interleaved file type */ - mrc = MPI_Type_commit(&mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_COMMIT"); - } /* end else */ - break; - - case PHDF5: /* HDF5 setup */ - /* 1D dataspace */ - if (!parms->dim2d) { - if (nbytes > 0) { - /* define a contiguous dataset of nbytes native bytes */ - h5dims[0] = (hsize_t)nbytes; - h5dset_space_id = H5Screate_simple(1, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - /* Set up the file dset space id to select the pattern to access */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5stride[0] = h5block[0] = blk_size; - h5count[0] = buf_size / blk_size; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5stride[0] = blk_size * (size_t)pio_mpi_nprocs_g; - h5block[0] = blk_size; - h5count[0] = buf_size / blk_size; - } /* end else */ - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, - h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - } /* end if */ - else { - h5dset_space_id = H5Screate(H5S_SCALAR); - VRFY((h5dset_space_id >= 0), "H5Screate"); - } /* end else */ - - /* Create the memory dataspace that corresponds to the xfer buffer */ - if (buf_size > 0) { - h5dims[0] = buf_size; - h5mem_space_id = H5Screate_simple(1, h5dims, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - } /* end if */ - else { - h5mem_space_id = H5Screate(H5S_SCALAR); - VRFY((h5mem_space_id >= 0), "H5Screate"); - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - if (nbytes > 0) { - /* define a contiguous dataset of nbytes native bytes */ - h5dims[0] = (hsize_t)snbytes; - h5dims[1] = (hsize_t)snbytes; - h5dset_space_id = H5Screate_simple(2, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - /* Set up the file dset space id to select the pattern to access */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5start[1] = (hsize_t)bytes_begin[1]; - h5stride[0] = 1; - h5stride[1] = h5block[0] = h5block[1] = blk_size; - h5count[0] = 1; - h5count[1] = buf_size / blk_size; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5start[0] = (hsize_t)bytes_begin[0]; - h5start[1] = (hsize_t)bytes_begin[1]; - h5stride[0] = blk_size; - h5stride[1] = blk_size * (size_t)pio_mpi_nprocs_g; - h5block[0] = h5block[1] = blk_size; - h5count[0] = buf_size / blk_size; - h5count[1] = 1; - } /* end else */ - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, - h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - } /* end if */ - else { - h5dset_space_id = H5Screate(H5S_SCALAR); - VRFY((h5dset_space_id >= 0), "H5Screate"); - } /* end else */ - - /* Create the memory dataspace that corresponds to the xfer buffer */ - if (buf_size > 0) { - if (!parms->interleaved) { - h5dims[0] = blk_size; - h5dims[1] = buf_size; - } - else { - h5dims[0] = buf_size; - h5dims[1] = blk_size; - } - h5mem_space_id = H5Screate_simple(2, h5dims, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - } /* end if */ - else { - h5mem_space_id = H5Screate(H5S_SCALAR); - VRFY((h5mem_space_id >= 0), "H5Screate"); - } /* end else */ - } /* end else */ - - /* Create the dataset transfer property list */ - h5dxpl = H5Pcreate(H5P_DATASET_XFER); - if (h5dxpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - /* Change to collective I/O, if asked */ - if (parms->collective) { - hrc = H5Pset_dxpl_mpio(h5dxpl, H5FD_MPIO_COLLECTIVE); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - break; - - default: - break; - } /* end switch */ - - for (ndset = 1; ndset <= ndsets; ++ndset) { - - /* Calculate dataset offset within a file */ - - /* create dataset */ - switch (parms->io_type) { - case POSIXIO: - case MPIO: - /* both posix and mpi io just need dataset offset in file*/ - dset_offset = (ndset - 1) * nbytes; - break; - - case PHDF5: - HDsprintf(dname, "Dataset_%ld", ndset); - h5ds_id = H5DOPEN(fd->h5fd, dname); - if (h5ds_id < 0) { - HDfprintf(stderr, "HDF5 Dataset open failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - break; - } - - /* The task is to transfer bytes_count bytes, starting at - * bytes_begin position, using transfer buffer of buf_size bytes. - * If interleaved, select buf_size at a time, in round robin - * fashion, according to number of process. Otherwise, select - * all bytes_count in contiguous. - */ - nbytes_xfer = 0; - - /* 1D dataspace */ - if (!parms->dim2d) { - /* Set base file offset for all I/O patterns and POSIX access */ - posix_file_offset = dset_offset + bytes_begin[0]; - - /* Set base file offset for all I/O patterns and MPI access */ - mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0]); - } /* end if */ - else { - /* Set base file offset for all I/O patterns and POSIX access */ - posix_file_offset = dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]; - - /* Set base file offset for all I/O patterns and MPI access */ - mpi_file_offset = (MPI_Offset)(dset_offset + bytes_begin[0] * snbytes + bytes_begin[1]); - } /* end else */ - - /* Start "raw data" read timer */ - io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTART); - - while (nbytes_xfer < bytes_count) { - /* Read */ - /* Calculate offset of read within a dataset/file */ - switch (parms->io_type) { - case POSIXIO: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Contiguous pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + (off_t)nbytes_xfer; - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are read */ - rc = ((ssize_t)buf_size == POSIXREAD(fd->posixfd, buffer, buf_size)); - VRFY((rc != 0), "POSIXREAD"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size; - - /* Loop over the buffers to read */ - while (nbytes_toxfer > 0) { - /* Skip offset over blocks of other processes */ - file_offset = posix_file_offset + (off_t)(nbytes_xfer * pio_mpi_nprocs_g); - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are read */ - rc = ((ssize_t)blk_size == POSIXREAD(fd->posixfd, buf_p, blk_size)); - VRFY((rc != 0), "POSIXREAD"); - - /* Advance location in buffer */ - buf_p += blk_size; - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)blk_size; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= blk_size; - } /* end while */ - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* Contiguous storage */ - if (!parms->h5_use_chunks) { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + - (off_t)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * - (blk_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / blk_size) % (size_t)snbytes)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = buf_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = (off_t)snbytes; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute file offset */ - file_offset = - posix_file_offset + - (off_t)(((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / - (size_t)snbytes) * - (buf_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % - (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = (off_t)snbytes; - } /* end else */ - } /* end if */ - /* Chunked storage */ - else { - /*Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute file offset */ - file_offset = posix_file_offset + (off_t)nbytes_xfer; - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * buf_size; - - /* Global offset advance after each I/O operation */ - file_offset_advance = 0; - } /* end if */ - /*Interleaved access pattern */ - else { - /* Compute file offset */ - /* Before simplification */ - /* file_offset=posix_file_offset+(off_t)((nbytes_xfer/(buf_size/blk_size) - *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))*(buf_size/blk_size - *snbytes/blk_size*(blk_size*blk_size))+((nbytes_xfer/(buf_size/blk_size)) - *pio_mpi_nprocs_g)%(snbytes/blk_size*(blk_size*blk_size))); */ - - file_offset = posix_file_offset + - (off_t)((((size_t)nbytes_xfer / (buf_size / blk_size) * - (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * blk_size)) * - (buf_size * (size_t)snbytes) + - (((size_t)nbytes_xfer / (buf_size / blk_size)) * - (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * blk_size)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * blk_size; - - /* Global offset advance after each I/O operation */ - /* file_offset_advance = (off_t)(snbytes/blk_size*(blk_size*blk_size)); */ - file_offset_advance = (off_t)((size_t)snbytes * blk_size); - } /* end else */ - } /* end else */ - - /* Common code for file access */ - - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size * blk_size; - - /* Loop over portions of the buffer to read */ - while (nbytes_toxfer > 0) { - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, file_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - - /* check if all bytes are read */ - rc = ((ssize_t)nbytes_xfer_advance == - POSIXREAD(fd->posixfd, buf_p, nbytes_xfer_advance)); - VRFY((rc != 0), "POSIXREAD"); - - /* Advance location in buffer */ - buf_p += nbytes_xfer_advance; - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)nbytes_xfer_advance; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= nbytes_xfer_advance; - - /* Partially advance file offset */ - file_offset += file_offset_advance; - } /* end while */ - - } /* end else */ - break; - - case MPIO: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Independent file access */ - if (!parms->collective) { - /* Contiguous pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Perform independent read */ - mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buffer, - (int)(buf_size / blk_size), mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size; - - /* Loop over the buffers to read */ - while (nbytes_toxfer > 0) { - /* Skip offset over blocks of other processes */ - mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); - - /* Perform independent read */ - mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p, (int)1, mpi_blk_type, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance location in buffer */ - buf_p += blk_size; - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)blk_size; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= blk_size; - } /* end while */ - } /* end else */ - } /* end if */ - /* Collective file access */ - else { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Perform collective read */ - mrc = MPI_File_read_at_all(fd->mpifd, mpi_offset, buffer, - (int)(buf_size / blk_size), mpi_blk_type, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + (nbytes_xfer * pio_mpi_nprocs_g); - - /* Set the file view */ - mrc = MPI_File_set_view(fd->mpifd, mpi_offset, mpi_blk_type, mpi_file_type, - (char *)"native", h5_io_info_g); - VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); - - /* Perform collective read */ - mrc = MPI_File_read_at_all(fd->mpifd, 0, buffer, (int)(buf_size / blk_size), - mpi_blk_type, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size; - } /* end else */ - } /* end else */ - } /* end if */ - /* 2D dataspace */ - else { - /* Contiguous storage */ - if (!parms->h5_use_chunks) { - /* Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = - mpi_file_offset + - (MPI_Offset)((((size_t)nbytes_xfer / blk_size) / (size_t)snbytes) * - (blk_size * (size_t)snbytes)) + - (MPI_Offset)(((size_t)nbytes_xfer / blk_size) % (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = buf_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = snbytes; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_cont_type; - } /* end if */ - /* Interleaved access pattern */ - else { - /* Compute offset in file */ - mpi_offset = - mpi_file_offset + - (MPI_Offset)( - ((((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) / - (size_t)snbytes) * - (buf_size * (size_t)snbytes)) + - (MPI_Offset)( - (((size_t)nbytes_xfer / buf_size) * (size_t)pio_mpi_nprocs_g) % - (size_t)snbytes); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = snbytes; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_inter_type; - } /* end else */ - } /* end if */ - /* Chunked storage */ - else { - /*Contiguous access pattern */ - if (!parms->interleaved) { - /* Compute offset in file */ - mpi_offset = mpi_file_offset + nbytes_xfer; - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * buf_size; - - /* Global offset advance after each I/O operation */ - mpi_offset_advance = 0; - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_full_buffer; - } /* end if */ - /*Interleaved access pattern */ - else { - /* Compute offset in file */ - /* Before simplification */ - /* mpi_offset=mpi_file_offset+(nbytes_xfer/(buf_size/blk_size) - *pio_mpi_nprocs_g)/(snbytes/blk_size*(blk_size*blk_size))* - (buf_size/blk_size*snbytes/blk_size*(blk_size*blk_size))+ - ((nbytes_xfer/(buf_size/blk_size))*pio_mpi_nprocs_g)%(snbytes - /blk_size*(blk_size*blk_size)); */ - mpi_offset = mpi_file_offset + - (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size) * - (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * blk_size)) * - (buf_size * (size_t)snbytes)) + - (MPI_Offset)((((size_t)nbytes_xfer / (buf_size / blk_size)) * - (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * blk_size)); - - /* Number of bytes to be transferred per I/O operation */ - nbytes_xfer_advance = blk_size * blk_size; - - /* Global offset advance after each I/O operation */ - /* mpi_offset_advance = (MPI_Offset)(snbytes/blk_size*(blk_size*blk_size)); */ - mpi_offset_advance = (MPI_Offset)((size_t)snbytes * blk_size); - - /* MPI type to be used for collective access */ - mpi_collective_type = mpi_chunk_inter_type; - } /* end else */ - } /* end else */ - - /* Common code for independent file access */ - if (!parms->collective) { - /* Set the base of user's buffer */ - buf_p = (unsigned char *)buffer; - - /* Set the number of bytes to transfer this time */ - nbytes_toxfer = buf_size * blk_size; - - /* Loop over portions of the buffer to read */ - while (nbytes_toxfer > 0) { - /* Perform independent read */ - mrc = MPI_File_read_at(fd->mpifd, mpi_offset, buf_p, (int)nbytes_xfer_advance, - MPI_BYTE, &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance location in buffer */ - buf_p += nbytes_xfer_advance; - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)nbytes_xfer_advance; - - /* Decrement number of bytes left this time */ - nbytes_toxfer -= nbytes_xfer_advance; - - /* Partially advance global offset in dataset */ - mpi_offset += mpi_offset_advance; - } /* end while */ - } /* end if */ - - /* Common code for collective file access */ - else { - /* Set the file view */ - mrc = MPI_File_set_view(fd->mpifd, mpi_offset, MPI_BYTE, mpi_collective_type, - (char *)"native", h5_io_info_g); - VRFY((mrc == MPI_SUCCESS), "MPIO_VIEW"); - - /* Perform read */ - MPI_File_read_at_all(fd->mpifd, 0, buffer, (int)(buf_size * blk_size), MPI_BYTE, - &mpi_status); - VRFY((mrc == MPI_SUCCESS), "MPIO_READ"); - - /* Advance global offset in dataset */ - nbytes_xfer += (off_t)buf_size * (off_t)blk_size; - } /* end else */ - - } /* end else */ - break; - - case PHDF5: - /* 1D dataspace */ - if (!parms->dim2d) { - /* Set up the file dset space id to move the selection to process */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5offset[0] = nbytes_xfer; - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5offset[0] = (nbytes_xfer * pio_mpi_nprocs_g); - } /* end else */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Read the buffer in */ - hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dread"); - - /* Increment number of bytes transferred */ - nbytes_xfer += (off_t)buf_size; - } /* end if */ - /* 2D dataspace */ - else { - /* Set up the file dset space id to move the selection to process */ - if (!parms->interleaved) { - /* Contiguous pattern */ - h5offset[0] = - (hssize_t)(((size_t)nbytes_xfer / ((size_t)snbytes * blk_size)) * blk_size); - h5offset[1] = - (hssize_t)(((size_t)nbytes_xfer % ((size_t)snbytes * blk_size)) / blk_size); - } /* end if */ - else { - /* Interleaved access pattern */ - /* Skip offset over blocks of other processes */ - h5offset[0] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) / - ((size_t)snbytes * buf_size)) * - buf_size); - h5offset[1] = (hssize_t)((((size_t)nbytes_xfer * (size_t)pio_mpi_nprocs_g) % - ((size_t)snbytes * buf_size)) / - buf_size); - - } /* end else */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Write the buffer out */ - hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dread"); - - /* Increment number of bytes transferred */ - nbytes_xfer += (off_t)buf_size * (off_t)blk_size; - - } /* end else */ - break; - - default: - break; - } /* switch (parms->io_type) */ - - /* Verify raw data, if asked */ - if (parms->verify) { - /* Verify data read */ - unsigned char *ucharptr = (unsigned char *)buffer; - size_t i; - int nerror = 0; - - for (i = 0; i < bsize; ++i) { - if (*ucharptr++ != pio_mpi_rank_g + 1) { - if (++nerror < 20) { - /* report at most 20 errors */ - HDprint_rank(output); - HDfprintf(output, - "read data error, expected (%d), " - "got (%d)\n", - pio_mpi_rank_g + 1, (int)*(ucharptr - 1)); - } /* end if */ - } /* end if */ - } /* end for */ - if (nerror >= 20) { - HDprint_rank(output); - HDfprintf(output, "..."); - HDfprintf(output, "total read data errors=%d\n", nerror); - } /* end if */ - } /* if (parms->verify) */ - - } /* end while */ - - /* Stop "raw data" read timer */ - io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTOP); - - /* Calculate read time */ - - /* Close dataset. Only HDF5 needs to do an explicit close. */ - if (parms->io_type == PHDF5) { - hrc = H5Dclose(h5ds_id); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Close failed\n"); - GOTOERROR(FAIL); - } - - h5ds_id = H5I_INVALID_HID; - } /* end if */ - } /* end for */ - -done: - /* release MPI-I/O objects */ - if (parms->io_type == MPIO) { - /* 1D dataspace */ - if (!parms->dim2d) { - /* Free file type */ - mrc = MPI_Type_free(&mpi_file_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free buffer type */ - mrc = MPI_Type_free(&mpi_blk_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - } /* end if */ - /* 2D dataspace */ - else { - /* Free partial buffer type for contiguous access */ - mrc = MPI_Type_free(&mpi_partial_buffer_cont); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free contiguous file type */ - mrc = MPI_Type_free(&mpi_cont_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free partial buffer type for interleaved access */ - mrc = MPI_Type_free(&mpi_partial_buffer_inter); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free interleaved file type */ - mrc = MPI_Type_free(&mpi_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free full buffer type */ - mrc = MPI_Type_free(&mpi_full_buffer); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free full chunk type */ - mrc = MPI_Type_free(&mpi_full_chunk); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - - /* Free chunk interleaved file type */ - mrc = MPI_Type_free(&mpi_chunk_inter_type); - VRFY((mrc == MPI_SUCCESS), "MPIO_TYPE_FREE"); - } /* end else */ - } /* end if */ - - /* release HDF5 objects */ - if (h5dset_space_id != -1) { - hrc = H5Sclose(h5dset_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); - ret_code = FAIL; - } - else { - h5dset_space_id = H5I_INVALID_HID; - } - } - - if (h5mem_space_id != -1) { - hrc = H5Sclose(h5mem_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); - ret_code = FAIL; - } - else { - h5mem_space_id = H5I_INVALID_HID; - } - } - - if (h5dxpl != -1) { - hrc = H5Pclose(h5dxpl); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); - ret_code = FAIL; - } - else { - h5dxpl = H5I_INVALID_HID; - } - } - - return ret_code; -} - -/* - * Function: do_fopen - * Purpose: Open the specified file. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: - */ -static herr_t -do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) -{ - int ret_code = SUCCESS, mrc; - hid_t acc_tpl = H5I_INVALID_HID; /* file access templates */ - - switch (param->io_type) { - case POSIXIO: - if (flags & (PIO_CREATE | PIO_WRITE)) - fd->posixfd = POSIXCREATE(fname); - else - fd->posixfd = POSIXOPEN(fname, O_RDONLY); - - if (fd->posixfd < 0) { - HDfprintf(stderr, "POSIX File Open failed(%s)\n", fname); - GOTOERROR(FAIL); - } - - /* The perils of POSIX I/O in a parallel environment. The problem is: - * - * - Process n opens a file with truncation and then starts - * writing to the file. - * - Process m also opens the file with truncation, but after - * process n has already started to write to the file. Thus, - * all of the stuff process n wrote is now lost. - */ - MPI_Barrier(pio_comm_g); - - break; - - case MPIO: - if (flags & (PIO_CREATE | PIO_WRITE)) { - MPI_File_delete(fname, h5_io_info_g); - mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_CREATE | MPI_MODE_RDWR, h5_io_info_g, - &fd->mpifd); - - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI File Open failed(%s)\n", fname); - GOTOERROR(FAIL); - } - - /*since MPI_File_open with MPI_MODE_CREATE does not truncate */ - /*filesize , set size to 0 explicitedly. */ - mrc = MPI_File_set_size(fd->mpifd, (MPI_Offset)0); - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI_File_set_size failed\n"); - GOTOERROR(FAIL); - } - } - else { - mrc = MPI_File_open(pio_comm_g, fname, MPI_MODE_RDONLY, h5_io_info_g, &fd->mpifd); - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI File Open failed(%s)\n", fname); - GOTOERROR(FAIL); - } - } - - break; - - case PHDF5: - if ((acc_tpl = H5Pcreate(H5P_FILE_ACCESS)) < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - /* Set the file driver to the MPI-IO driver */ - if (H5Pset_fapl_mpio(acc_tpl, pio_comm_g, h5_io_info_g) < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } - - /* Set the alignment of objects in HDF5 file */ - if (H5Pset_alignment(acc_tpl, param->h5_thresh, param->h5_align) < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } - - /* create the parallel file */ - if (flags & (PIO_CREATE | PIO_WRITE)) - fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); - else - fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, acc_tpl); - if (fd->h5fd < 0) { - HDfprintf(stderr, "HDF5 File Create failed(%s)\n", fname); - GOTOERROR(FAIL); - } - - /* verifying the close of the acc_tpl */ - if (H5Pclose(acc_tpl) < 0) { - HDfprintf(stderr, "HDF5 Property List Close failed\n"); - GOTOERROR(FAIL); - } - - break; - - default: - break; - } - -done: - return ret_code; -} - -/* - * Function: do_fclose - * Purpose: Close the specified file descriptor. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: - */ -static herr_t -do_fclose(iotype iot, file_descr *fd /*out*/) -{ - herr_t ret_code = SUCCESS, hrc; - int mrc = 0, rc = 0; - - switch (iot) { - case POSIXIO: - rc = POSIXCLOSE(fd->posixfd); - - if (rc != 0) { - HDfprintf(stderr, "POSIX File Close failed\n"); - GOTOERROR(FAIL); - } - - fd->posixfd = -1; - break; - - case MPIO: - mrc = MPI_File_close(&fd->mpifd); - - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI File close failed\n"); - GOTOERROR(FAIL); - } - - fd->mpifd = MPI_FILE_NULL; - break; - - case PHDF5: - hrc = H5Fclose(fd->h5fd); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 File Close failed\n"); - GOTOERROR(FAIL); - } - - fd->h5fd = -1; - break; - - default: - break; - } - -done: - return ret_code; -} - -/* - * Function: do_fclose - * Purpose: Cleanup temporary file unless HDF5_NOCLEANUP is set. - * Only Proc 0 of the PIO communicator will do the cleanup. - * Other processes just return. - * Return: void - * Programmer: Albert Cheng 2001/12/12 - * Modifications: - */ -static void -do_cleanupfile(iotype iot, char *fname) -{ - if (pio_mpi_rank_g != 0) - return; - - if (clean_file_g == -1) - clean_file_g = (getenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; - - if (clean_file_g) { - switch (iot) { - case POSIXIO: - HDremove(fname); - break; - case MPIO: - case PHDF5: - MPI_File_delete(fname, h5_io_info_g); - break; - default: - break; - } - } -} - -#ifdef TIME_MPI -/* instrument the MPI_File_wrirte_xxx and read_xxx calls to measure - * pure time spent in MPI_File code. - */ -int -MPI_File_read_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status) -{ - int err; - io_time_set(timer_g, HDF5_MPI_READ, TSTART); - err = PMPI_File_read_at(fh, offset, buf, count, datatype, status); - io_time_set(timer_g, HDF5_MPI_READ, TSTOP); - return err; -} - -int -MPI_File_read_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status) -{ - int err; - io_time_set(timer_g, HDF5_MPI_READ, TSTART); - err = PMPI_File_read_at_all(fh, offset, buf, count, datatype, status); - io_time_set(timer_g, HDF5_MPI_READ, TSTOP); - return err; -} - -int -MPI_File_write_at(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status) -{ - int err; - io_time_set(timer_g, HDF5_MPI_WRITE, TSTART); - err = PMPI_File_write_at(fh, offset, buf, count, datatype, status); - io_time_set(timer_g, HDF5_MPI_WRITE, TSTOP); - return err; -} - -int -MPI_File_write_at_all(MPI_File fh, MPI_Offset offset, void *buf, int count, MPI_Datatype datatype, - MPI_Status *status) -{ - int err; - io_time_set(timer_g, HDF5_MPI_WRITE, TSTART); - err = PMPI_File_write_at_all(fh, offset, buf, count, datatype, status); - io_time_set(timer_g, HDF5_MPI_WRITE, TSTOP); - return err; -} - -#endif /* TIME_MPI */ -#endif /* H5_HAVE_PARALLEL */ diff --git a/tools/test/perform/pio_perf.c b/tools/test/perform/pio_perf.c deleted file mode 100644 index 6fb5043..0000000 --- a/tools/test/perform/pio_perf.c +++ /dev/null @@ -1,1699 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Parallel HDF5 Performance Testing Code - * -------------------------------------- - * - * Portable code to test performance on the different platforms we support. - * This is what the report should look like: - * - * nprocs = Max#Procs - * IO API = POSIXIO - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * IO API = MPIO - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * IO API = PHDF5 - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * nprocs = Max#Procs / 2 - * - * . . . - * - */ - -/* system header files */ -#include -#include -#include - -#include "hdf5.h" - -#ifdef H5_HAVE_PARALLEL - -/* library header files */ -#include - -/* our header files */ -#include "pio_perf.h" - -/* useful macros */ -#define TAB_SPACE 4 - -#define ONE_KB 1024 -#define ONE_MB (ONE_KB * ONE_KB) -#define ONE_GB (ONE_MB * ONE_KB) - -#define PIO_POSIX 0x1 -#define PIO_MPI 0x2 -#define PIO_HDF5 0x4 - -#ifdef STANDALONE -#define DBL_EPSILON 2.2204460492503131e-16 -#define H5_DBL_ABS_EQUAL(X, Y) (fabs((X) - (Y)) < DBL_EPSILON) -#endif - -/* report 0.0 in case t is zero too */ -#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL((t), 0.0) ? 0.0 : ((((double)bytes) / ONE_MB) / (t))) - -#ifndef TRUE -#define TRUE 1 -#endif /* TRUE */ -#ifndef FALSE -#define FALSE (!TRUE) -#endif /* FALSE */ - -/* global variables */ -FILE * output; /* output file */ -int comm_world_rank_g; /* my rank in MPI_COMM_RANK */ -int comm_world_nprocs_g; /* num. of processes of MPI_COMM_WORLD */ -MPI_Comm pio_comm_g; /* Communicator to run the PIO */ -int pio_mpi_rank_g; /* MPI rank of pio_comm_g */ -int pio_mpi_nprocs_g; /* Number of processes of pio_comm_g */ -int pio_debug_level = 0; /* The debug level: - * 0 - Off - * 1 - Minimal - * 2 - Some more - * 3 - Maximal - * 4 - Maximal & then some - */ - -/* local variables */ -static const char *progname = "h5perf"; - -/* - * Command-line options: The user can specify short or long-named - * parameters. The long-named ones can be partially spelled. When - * adding more, make sure that they don't clash with each other. - */ -#if 1 -static const char *s_opts = "a:A:B:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; -#else -static const char *s_opts = "a:A:bB:cCd:D:e:F:ghi:Imno:p:P:stT:wx:X:"; -#endif /* 1 */ -static struct long_options l_opts[] = {{"align", require_arg, 'a'}, - {"alig", require_arg, 'a'}, - {"ali", require_arg, 'a'}, - {"al", require_arg, 'a'}, - {"api", require_arg, 'A'}, - {"ap", require_arg, 'A'}, -#if 0 - /* a sighting of the elusive binary option */ - { "binary", no_arg, 'b' }, - { "binar", no_arg, 'b' }, - { "bina", no_arg, 'b' }, - { "bin", no_arg, 'b' }, - { "bi", no_arg, 'b' }, -#endif /* 0 */ - {"block-size", require_arg, 'B'}, - {"block-siz", require_arg, 'B'}, - {"block-si", require_arg, 'B'}, - {"block-s", require_arg, 'B'}, - {"block-", require_arg, 'B'}, - {"block", require_arg, 'B'}, - {"bloc", require_arg, 'B'}, - {"blo", require_arg, 'B'}, - {"bl", require_arg, 'B'}, - {"chunk", no_arg, 'c'}, - {"chun", no_arg, 'c'}, - {"chu", no_arg, 'c'}, - {"ch", no_arg, 'c'}, - {"collective", no_arg, 'C'}, - {"collectiv", no_arg, 'C'}, - {"collecti", no_arg, 'C'}, - {"collect", no_arg, 'C'}, - {"collec", no_arg, 'C'}, - {"colle", no_arg, 'C'}, - {"coll", no_arg, 'C'}, - {"col", no_arg, 'C'}, - {"co", no_arg, 'C'}, - {"debug", require_arg, 'D'}, - {"debu", require_arg, 'D'}, - {"deb", require_arg, 'D'}, - {"de", require_arg, 'D'}, - {"geometry", no_arg, 'g'}, - {"geometr", no_arg, 'g'}, - {"geomet", no_arg, 'g'}, - {"geome", no_arg, 'g'}, - {"geom", no_arg, 'g'}, - {"geo", no_arg, 'g'}, - {"ge", no_arg, 'g'}, - {"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, - {"interleaved", require_arg, 'I'}, - {"interleave", require_arg, 'I'}, - {"interleav", require_arg, 'I'}, - {"interlea", require_arg, 'I'}, - {"interle", require_arg, 'I'}, - {"interl", require_arg, 'I'}, - {"inter", require_arg, 'I'}, - {"inte", require_arg, 'I'}, - {"int", require_arg, 'I'}, - {"in", require_arg, 'I'}, - {"max-num-processes", require_arg, 'P'}, - {"max-num-processe", require_arg, 'P'}, - {"max-num-process", require_arg, 'P'}, - {"max-num-proces", require_arg, 'P'}, - {"max-num-proce", require_arg, 'P'}, - {"max-num-proc", require_arg, 'P'}, - {"max-num-pro", require_arg, 'P'}, - {"max-num-pr", require_arg, 'P'}, - {"max-num-p", require_arg, 'P'}, - {"min-num-processes", require_arg, 'p'}, - {"min-num-processe", require_arg, 'p'}, - {"min-num-process", require_arg, 'p'}, - {"min-num-proces", require_arg, 'p'}, - {"min-num-proce", require_arg, 'p'}, - {"min-num-proc", require_arg, 'p'}, - {"min-num-pro", require_arg, 'p'}, - {"min-num-pr", require_arg, 'p'}, - {"min-num-p", require_arg, 'p'}, - {"max-xfer-size", require_arg, 'X'}, - {"max-xfer-siz", require_arg, 'X'}, - {"max-xfer-si", require_arg, 'X'}, - {"max-xfer-s", require_arg, 'X'}, - {"max-xfer", require_arg, 'X'}, - {"max-xfe", require_arg, 'X'}, - {"max-xf", require_arg, 'X'}, - {"max-x", require_arg, 'X'}, - {"min-xfer-size", require_arg, 'x'}, - {"min-xfer-siz", require_arg, 'x'}, - {"min-xfer-si", require_arg, 'x'}, - {"min-xfer-s", require_arg, 'x'}, - {"min-xfer", require_arg, 'x'}, - {"min-xfe", require_arg, 'x'}, - {"min-xf", require_arg, 'x'}, - {"min-x", require_arg, 'x'}, - {"num-bytes", require_arg, 'e'}, - {"num-byte", require_arg, 'e'}, - {"num-byt", require_arg, 'e'}, - {"num-by", require_arg, 'e'}, - {"num-b", require_arg, 'e'}, - {"num-dsets", require_arg, 'd'}, - {"num-dset", require_arg, 'd'}, - {"num-dse", require_arg, 'd'}, - {"num-ds", require_arg, 'd'}, - {"num-d", require_arg, 'd'}, - {"num-files", require_arg, 'F'}, - {"num-file", require_arg, 'F'}, - {"num-fil", require_arg, 'F'}, - {"num-fi", require_arg, 'F'}, - {"num-f", require_arg, 'F'}, - {"num-iterations", require_arg, 'i'}, - {"num-iteration", require_arg, 'i'}, - {"num-iteratio", require_arg, 'i'}, - {"num-iterati", require_arg, 'i'}, - {"num-iterat", require_arg, 'i'}, - {"num-itera", require_arg, 'i'}, - {"num-iter", require_arg, 'i'}, - {"num-ite", require_arg, 'i'}, - {"num-it", require_arg, 'i'}, - {"num-i", require_arg, 'i'}, - {"output", require_arg, 'o'}, - {"outpu", require_arg, 'o'}, - {"outp", require_arg, 'o'}, - {"out", require_arg, 'o'}, - {"ou", require_arg, 'o'}, - {"threshold", require_arg, 'T'}, - {"threshol", require_arg, 'T'}, - {"thresho", require_arg, 'T'}, - {"thresh", require_arg, 'T'}, - {"thres", require_arg, 'T'}, - {"thre", require_arg, 'T'}, - {"thr", require_arg, 'T'}, - {"th", require_arg, 'T'}, - {"write-only", require_arg, 'w'}, - {"write-onl", require_arg, 'w'}, - {"write-on", require_arg, 'w'}, - {"write-o", require_arg, 'w'}, - {"write", require_arg, 'w'}, - {"writ", require_arg, 'w'}, - {"wri", require_arg, 'w'}, - {"wr", require_arg, 'w'}, - {NULL, 0, '\0'}}; - -struct options { - long io_types; /* bitmask of which I/O types to test */ - const char *output_file; /* file to print report to */ - long num_dsets; /* number of datasets */ - long num_files; /* number of files */ - off_t num_bpp; /* number of bytes per proc per dset */ - int num_iters; /* number of iterations */ - int max_num_procs; /* maximum number of processes to use */ - int min_num_procs; /* minimum number of processes to use */ - size_t max_xfer_size; /* maximum transfer buffer size */ - size_t min_xfer_size; /* minimum transfer buffer size */ - size_t blk_size; /* Block size */ - unsigned interleaved; /* Interleaved vs. contiguous blocks */ - unsigned collective; /* Collective vs. independent I/O */ - unsigned dim2d; /* 1D vs. 2D geometry */ - int print_times; /* print times as well as throughputs */ - int print_raw; /* print raw data throughput info */ - off_t h5_alignment; /* alignment in HDF5 file */ - off_t h5_threshold; /* threshold for alignment in HDF5 file */ - int h5_use_chunks; /* Make HDF5 dataset chunked */ - int h5_write_only; /* Perform the write tests only */ - int verify; /* Verify data correctness */ -}; - -typedef struct _minmax { - double min; - double max; - double sum; - int num; -} minmax; - -/* local functions */ -static off_t parse_size_directive(const char *size); -static struct options *parse_command_line(int argc, char *argv[]); -static void run_test_loop(struct options *options); -static int run_test(iotype iot, parameters parms, struct options *opts); -static void output_all_info(minmax *mm, int count, int indent_level); -static void get_minmax(minmax *mm, double val); -static minmax accumulate_minmax_stuff(minmax *mm, int count); -static int create_comm_world(int num_procs, int *doing_pio); -static int destroy_comm_world(void); -static void output_results(const struct options *options, const char *name, minmax *table, int table_size, - off_t data_size); -static void output_times(const struct options *options, const char *name, minmax *table, int table_size); -static void output_report(const char *fmt, ...); -static void print_indent(register int indent); -static void usage(const char *prog); -static void report_parameters(struct options *opts); -static off_t squareo(off_t); - -/* - * Function: main - * Purpose: Start things up. Initialize MPI and then call the test looping - * function. - * Return: EXIT_SUCCESS or EXIT_FAILURE - * Programmer: Bill Wendling, 30. October 2001 - * Modifications: - */ -int -main(int argc, char *argv[]) -{ - int ret; - int exit_value = EXIT_SUCCESS; - struct options *opts = NULL; - -#ifndef STANDALONE - /* Initialize h5tools lib */ - h5tools_init(); -#endif - - output = stdout; - - /* initialize MPI and get the maximum num of processors we started with */ - MPI_Init(&argc, &argv); - ret = MPI_Comm_size(MPI_COMM_WORLD, &comm_world_nprocs_g); - - if (ret != MPI_SUCCESS) { - HDfprintf(stderr, "%s: MPI_Comm_size call failed\n", progname); - - if (ret == MPI_ERR_COMM) - HDfprintf(stderr, "invalid MPI communicator\n"); - else - HDfprintf(stderr, "invalid argument\n"); - - exit_value = EXIT_FAILURE; - goto finish; - } - - ret = MPI_Comm_rank(MPI_COMM_WORLD, &comm_world_rank_g); - - if (ret != MPI_SUCCESS) { - HDfprintf(stderr, "%s: MPI_Comm_rank call failed\n", progname); - - if (ret == MPI_ERR_COMM) - HDfprintf(stderr, "invalid MPI communicator\n"); - else - HDfprintf(stderr, "invalid argument\n"); - - exit_value = EXIT_FAILURE; - goto finish; - } - - pio_comm_g = MPI_COMM_WORLD; - - h5_set_info_object(); - opts = parse_command_line(argc, argv); - - if (!opts) { - exit_value = EXIT_FAILURE; - goto finish; - } - - if (opts->output_file) { - if ((output = HDfopen(opts->output_file, "w")) == NULL) { - HDfprintf(stderr, "%s: cannot open output file\n", progname); - perror(opts->output_file); - goto finish; - } - } - - if ((pio_debug_level == 0 && comm_world_rank_g == 0) || pio_debug_level > 0) - report_parameters(opts); - - run_test_loop(opts); - -finish: - MPI_Finalize(); - free(opts); - return exit_value; -} - -off_t -squareo(off_t x) -{ - return x * x; -} - -/* - * Function: run_test_loop - * Purpose: Run the I/O tests. Write the results to OUTPUT. - * - * - The slowest changing part of the test is the number of - * processors to use. For each loop iteration, we divide that - * number by 2 and rerun the test. - * - * - The second slowest is what type of IO API to perform. We have - * three choices: POSIXIO, MPI-IO, and PHDF5. - * - * - Then we change the size of the buffer. This information is - * inferred from the number of datasets to create and the number - * of integers to put into each dataset. The backend code figures - * this out. - * - * Return: Nothing - * Programmer: Bill Wendling, 30. October 2001 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static void -run_test_loop(struct options *opts) -{ - parameters parms; - int num_procs; - int doing_pio; /* if this process is doing PIO */ - - parms.num_files = opts->num_files; - parms.num_dsets = opts->num_dsets; - parms.num_iters = opts->num_iters; - parms.blk_size = opts->blk_size; - parms.interleaved = opts->interleaved; - parms.collective = opts->collective; - parms.dim2d = opts->dim2d; - parms.h5_align = (hsize_t)opts->h5_alignment; - parms.h5_thresh = (hsize_t)opts->h5_threshold; - parms.h5_use_chunks = opts->h5_use_chunks; - parms.h5_write_only = opts->h5_write_only; - parms.verify = opts->verify; - - /* start with max_num_procs and decrement it by half for each loop. */ - /* if performance needs restart, fewer processes may be needed. */ - for (num_procs = opts->max_num_procs; num_procs >= opts->min_num_procs; num_procs >>= 1) { - register size_t buf_size; - - parms.num_procs = num_procs; - - if (create_comm_world(parms.num_procs, &doing_pio) != SUCCESS) { - /* do something harsh */ - } - - /* only processes doing PIO will run the tests */ - if (doing_pio) { - output_report("Number of processors = %ld\n", parms.num_procs); - - /* multiply the xfer buffer size by 2 for each loop iteration */ - for (buf_size = opts->min_xfer_size; buf_size <= opts->max_xfer_size; buf_size <<= 1) { - parms.buf_size = buf_size; - - if (parms.dim2d) { - parms.num_bytes = squareo(opts->num_bpp * parms.num_procs); - if (parms.interleaved) - output_report("Transfer Buffer Size: %ldx%ld bytes, File size: %.2f MB\n", buf_size, - opts->blk_size, - ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); - else - output_report("Transfer Buffer Size: %ldx%ld bytes, File size: %.2f MB\n", - opts->blk_size, buf_size, - ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); - - print_indent(1); - output_report(" # of files: %ld, # of datasets: %ld, dataset size: %.2fx%.2f KB\n", - parms.num_files, parms.num_dsets, - (double)(opts->num_bpp * parms.num_procs) / ONE_KB, - (double)(opts->num_bpp * parms.num_procs) / ONE_KB); - } - else { - parms.num_bytes = (off_t)opts->num_bpp * parms.num_procs; - output_report("Transfer Buffer Size: %ld bytes, File size: %.2f MB\n", buf_size, - ((double)parms.num_dsets * (double)parms.num_bytes) / ONE_MB); - - print_indent(1); - output_report(" # of files: %ld, # of datasets: %ld, dataset size: %.2f MB\n", - parms.num_files, parms.num_dsets, - (double)(opts->num_bpp * parms.num_procs) / ONE_MB); - } - - if (opts->io_types & PIO_POSIX) - run_test(POSIXIO, parms, opts); - - if (opts->io_types & PIO_MPI) - run_test(MPIO, parms, opts); - - if (opts->io_types & PIO_HDF5) - run_test(PHDF5, parms, opts); - - /* Run the tests once if buf_size==0, but then break out */ - if (buf_size == 0) - break; - } - - if (destroy_comm_world() != SUCCESS) { - /* do something harsh */ - } - } - } -} - -/* - * Function: run_test - * Purpose: Inner loop call to actually run the I/O test. - * Return: Nothing - * Programmer: Bill Wendling, 18. December 2001 - * Modifications: - */ -static int -run_test(iotype iot, parameters parms, struct options *opts) -{ - results res; - register int i, ret_value = SUCCESS; - int comm_size; - off_t raw_size; - minmax * write_mpi_mm_table = NULL; - minmax * write_mm_table = NULL; - minmax * write_gross_mm_table = NULL; - minmax * write_raw_mm_table = NULL; - minmax * read_mpi_mm_table = NULL; - minmax * read_mm_table = NULL; - minmax * read_gross_mm_table = NULL; - minmax * read_raw_mm_table = NULL; - minmax * read_open_mm_table = NULL; - minmax * read_close_mm_table = NULL; - minmax * write_open_mm_table = NULL; - minmax * write_close_mm_table = NULL; - minmax write_mpi_mm = {0.0, 0.0, 0.0, 0}; - minmax write_mm = {0.0, 0.0, 0.0, 0}; - minmax write_gross_mm = {0.0, 0.0, 0.0, 0}; - minmax write_raw_mm = {0.0, 0.0, 0.0, 0}; - minmax read_mpi_mm = {0.0, 0.0, 0.0, 0}; - minmax read_mm = {0.0, 0.0, 0.0, 0}; - minmax read_gross_mm = {0.0, 0.0, 0.0, 0}; - minmax read_raw_mm = {0.0, 0.0, 0.0, 0}; - minmax read_open_mm = {0.0, 0.0, 0.0, 0}; - minmax read_close_mm = {0.0, 0.0, 0.0, 0}; - minmax write_open_mm = {0.0, 0.0, 0.0, 0}; - minmax write_close_mm = {0.0, 0.0, 0.0, 0}; - - raw_size = parms.num_files * (off_t)parms.num_dsets * (off_t)parms.num_bytes; - parms.io_type = iot; - print_indent(2); - output_report("IO API = "); - - switch (iot) { - case POSIXIO: - output_report("POSIX\n"); - break; - case MPIO: - output_report("MPIO\n"); - break; - case PHDF5: - output_report("PHDF5 (w/MPI-IO driver)\n"); - break; - default: - break; - } - - MPI_Comm_size(pio_comm_g, &comm_size); - - /* allocate space for tables minmax and that it is sufficient */ - /* to initialize all elements to zeros by calloc. */ - write_mpi_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_gross_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_raw_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_open_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - write_close_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - if (!parms.h5_write_only) { - read_mpi_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_gross_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_raw_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_open_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - read_close_mm_table = calloc((size_t)parms.num_iters, sizeof(minmax)); - } - - /* Do IO iteration times, collecting statistics each time */ - for (i = 0; i < parms.num_iters; ++i) { - double t; - - MPI_Barrier(pio_comm_g); - res = do_pio(parms); - - /* gather all of the "mpi write" times */ - t = io_time_get(res.timers, HDF5_MPI_WRITE); - get_minmax(&write_mpi_mm, t); - - write_mpi_mm_table[i] = write_mpi_mm; - - /* gather all of the "write" times */ - t = io_time_get(res.timers, HDF5_FINE_WRITE_FIXED_DIMS); - get_minmax(&write_mm, t); - - write_mm_table[i] = write_mm; - - /* gather all of the "write" times from open to close */ - t = io_time_get(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS); - get_minmax(&write_gross_mm, t); - - write_gross_mm_table[i] = write_gross_mm; - - /* gather all of the raw "write" times */ - t = io_time_get(res.timers, HDF5_RAW_WRITE_FIXED_DIMS); - get_minmax(&write_raw_mm, t); - - write_raw_mm_table[i] = write_raw_mm; - - /* gather all of the file open times (time from open to first write) */ - t = io_time_get(res.timers, HDF5_FILE_WRITE_OPEN); - get_minmax(&write_open_mm, t); - - write_open_mm_table[i] = write_open_mm; - - /* gather all of the file close times (time from last write to close) */ - t = io_time_get(res.timers, HDF5_FILE_WRITE_CLOSE); - get_minmax(&write_close_mm, t); - - write_close_mm_table[i] = write_close_mm; - - if (!parms.h5_write_only) { - /* gather all of the "mpi read" times */ - t = io_time_get(res.timers, HDF5_MPI_READ); - get_minmax(&read_mpi_mm, t); - - read_mpi_mm_table[i] = read_mpi_mm; - - /* gather all of the "read" times */ - t = io_time_get(res.timers, HDF5_FINE_READ_FIXED_DIMS); - get_minmax(&read_mm, t); - - read_mm_table[i] = read_mm; - - /* gather all of the "read" times from open to close */ - t = io_time_get(res.timers, HDF5_GROSS_READ_FIXED_DIMS); - get_minmax(&read_gross_mm, t); - - read_gross_mm_table[i] = read_gross_mm; - - /* gather all of the raw "read" times */ - t = io_time_get(res.timers, HDF5_RAW_READ_FIXED_DIMS); - get_minmax(&read_raw_mm, t); - - read_raw_mm_table[i] = read_raw_mm; - - /* gather all of the file open times (time from open to first read) */ - t = io_time_get(res.timers, HDF5_FILE_READ_OPEN); - get_minmax(&read_open_mm, t); - - read_open_mm_table[i] = read_open_mm; - - /* gather all of the file close times (time from last read to close) */ - t = io_time_get(res.timers, HDF5_FILE_READ_CLOSE); - get_minmax(&read_close_mm, t); - - read_close_mm_table[i] = read_close_mm; - } - - io_time_destroy(res.timers); - } - - /* - * Show various statistics - */ - /* Write statistics */ - /* Print the raw data throughput if desired */ - if (opts->print_raw) { - /* accumulate and output the max, min, and average "raw write" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Raw Data Write details:\n"); - output_all_info(write_raw_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); - } /* end if */ - - /* show mpi write statics */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("MPI Write details:\n"); - output_all_info(write_mpi_mm_table, parms.num_iters, 4); - } - - /* We don't currently output the MPI write results */ - - /* accumulate and output the max, min, and average "write" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write details:\n"); - output_all_info(write_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Write", write_mm_table, parms.num_iters, raw_size); - - /* accumulate and output the max, min, and average "gross write" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write Open-Close details:\n"); - output_all_info(write_gross_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Write Open-Close", write_gross_mm_table, parms.num_iters, raw_size); - - if (opts->print_times) { - output_times(opts, "Write File Open", write_open_mm_table, parms.num_iters); - output_times(opts, "Write File Close", write_close_mm_table, parms.num_iters); - } - - /* Print out time from open to first write */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write file open details:\n"); - output_all_info(write_open_mm_table, parms.num_iters, 4); - } - - /* Print out time from last write to close */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write file close details:\n"); - output_all_info(write_close_mm_table, parms.num_iters, 4); - } - - if (!parms.h5_write_only) { - /* Read statistics */ - /* Print the raw data throughput if desired */ - if (opts->print_raw) { - /* accumulate and output the max, min, and average "raw read" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Raw Data Read details:\n"); - output_all_info(read_raw_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); - } /* end if */ - - /* show mpi read statics */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("MPI Read details:\n"); - output_all_info(read_mpi_mm_table, parms.num_iters, 4); - } - - /* We don't currently output the MPI read results */ - - /* accumulate and output the max, min, and average "read" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read details:\n"); - output_all_info(read_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Read", read_mm_table, parms.num_iters, raw_size); - - /* accumulate and output the max, min, and average "gross read" times */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read Open-Close details:\n"); - output_all_info(read_gross_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Read Open-Close", read_gross_mm_table, parms.num_iters, raw_size); - - if (opts->print_times) { - output_times(opts, "Read File Open", read_open_mm_table, parms.num_iters); - output_times(opts, "Read File Close", read_close_mm_table, parms.num_iters); - } - - /* Print out time from open to first read */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read file open details:\n"); - output_all_info(read_open_mm_table, parms.num_iters, 4); - } - - /* Print out time from last read to close */ - if (pio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read file close details:\n"); - output_all_info(read_close_mm_table, parms.num_iters, 4); - } - } - - /* clean up our mess */ - free(write_mpi_mm_table); - free(write_mm_table); - free(write_gross_mm_table); - free(write_raw_mm_table); - free(write_open_mm_table); - free(write_close_mm_table); - - if (!parms.h5_write_only) { - free(read_mpi_mm_table); - free(read_mm_table); - free(read_gross_mm_table); - free(read_raw_mm_table); - free(read_open_mm_table); - free(read_close_mm_table); - } - - return ret_value; -} - -/* - * Function: output_all_info - * Purpose: - * Return: Nothing - * Programmer: Bill Wendling, 29. January 2002 - * Modifications: - */ -static void -output_all_info(minmax *mm, int count, int indent_level) -{ - int i; - - for (i = 0; i < count; ++i) { - print_indent(indent_level); - output_report("Iteration %d:\n", i + 1); - print_indent(indent_level + 1); - output_report("Minimum Time: %.2fs\n", mm[i].min); - print_indent(indent_level + 1); - output_report("Maximum Time: %.2fs\n", mm[i].max); - } -} - -/* - * Function: get_minmax - * Purpose: Gather all the min, max and total of val. - * Return: Nothing - * Programmer: Bill Wendling, 21. December 2001 - * Modifications: - * Use MPI_Allreduce to do it. -akc, 2002/01/11 - */ -static void -get_minmax(minmax *mm, double val) -{ - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - MPI_Comm_size(pio_comm_g, &mm->num); - - MPI_Allreduce(&val, &mm->max, 1, MPI_DOUBLE, MPI_MAX, pio_comm_g); - MPI_Allreduce(&val, &mm->min, 1, MPI_DOUBLE, MPI_MIN, pio_comm_g); - MPI_Allreduce(&val, &mm->sum, 1, MPI_DOUBLE, MPI_SUM, pio_comm_g); -} - -/* - * Function: accumulate_minmax_stuff - * Purpose: Accumulate the minimum, maximum, and average of the times - * across all processes. - * Return: TOTAL_MM - the total of all of these. - * Programmer: Bill Wendling, 21. December 2001 - * Modifications: - * Changed to use seconds instead of MB/s - QAK, 5/9/02 - */ -static minmax -accumulate_minmax_stuff(minmax *mm, int count) -{ - int i; - minmax total_mm; - - total_mm.sum = 0.0f; - total_mm.max = -DBL_MAX; - total_mm.min = DBL_MAX; - total_mm.num = count; - - for (i = 0; i < count; ++i) { - double m = mm[i].max; - - total_mm.sum += m; - - if (m < total_mm.min) - total_mm.min = m; - - if (m > total_mm.max) - total_mm.max = m; - } - - return total_mm; -} - -/* - * Function: create_comm_world - * Purpose: Create an MPI Comm world and store it in pio_comm_g, which - * is a global variable. - * Return: SUCCESS on success. - * FAIL otherwise. - * Programmer: Bill Wendling, 19. December 2001 - * Modifications: - */ -static int -create_comm_world(int num_procs, int *doing_pio) -{ - /* MPI variables */ - int mrc; /* return values */ - int color; /* for communicator creation */ - int myrank, nprocs; - - pio_comm_g = MPI_COMM_NULL; - - /* - * Create a sub communicator for this PIO run. Easier to use the first N - * processes. - */ - MPI_Comm_size(MPI_COMM_WORLD, &nprocs); - - if (num_procs > nprocs) { - HDfprintf(stderr, "number of process(%d) must be <= number of processes in MPI_COMM_WORLD(%d)\n", - num_procs, nprocs); - goto error_done; - } - - MPI_Comm_rank(MPI_COMM_WORLD, &myrank); - color = (myrank < num_procs); - mrc = MPI_Comm_split(MPI_COMM_WORLD, color, myrank, &pio_comm_g); - - if (mrc != MPI_SUCCESS) { - HDfprintf(stderr, "MPI_Comm_split failed\n"); - goto error_done; - } - - if (!color) { - /* not involved in this run */ - mrc = destroy_comm_world(); - goto done; - } - - /* determine the MPI rank in the PIO communicator */ - MPI_Comm_size(pio_comm_g, &pio_mpi_nprocs_g); - MPI_Comm_rank(pio_comm_g, &pio_mpi_rank_g); - -done: - *doing_pio = color; - return SUCCESS; - -error_done: - destroy_comm_world(); - return FAIL; -} - -/* - * Function: destroy_comm_world - * Purpose: Destroy the created MPI Comm world which is stored in the - * pio_comm_g global variable. - * Return: SUCCESS on success. - * FAIL otherwise. - * Programmer: Bill Wendling, 19. December 2001 - * Modifications: - */ -static int -destroy_comm_world(void) -{ - int mrc = SUCCESS; /* return code */ - - /* release MPI resources */ - if (pio_comm_g != MPI_COMM_NULL) - mrc = (MPI_Comm_free(&pio_comm_g) == MPI_SUCCESS ? SUCCESS : FAIL); - - return mrc; -} - -/* - * Function: output_results - * Purpose: Print information about the time & bandwidth for a given - * minmax & # of iterations. - * Return: Nothing - * Programmer: Quincey Koziol, 9. May 2002 - * Modifications: - */ -static void -output_results(const struct options *opts, const char *name, minmax *table, int table_size, off_t data_size) -{ - minmax total_mm; - - total_mm = accumulate_minmax_stuff(table, table_size); - - print_indent(3); - output_report("%s (%d iteration(s)):\n", name, table_size); - - /* Note: The maximum throughput uses the minimum amount of time & vice versa */ - - print_indent(4); - output_report("Maximum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.min)); - if (opts->print_times) - output_report(" (%7.3f s)\n", total_mm.min); - else - output_report("\n"); - - print_indent(4); - output_report("Average Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.sum / total_mm.num)); - if (opts->print_times) - output_report(" (%7.3f s)\n", (total_mm.sum / total_mm.num)); - else - output_report("\n"); - - print_indent(4); - output_report("Minimum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.max)); - if (opts->print_times) - output_report(" (%7.3f s)\n", total_mm.max); - else - output_report("\n"); -} - -static void -output_times(const struct options *opts, const char *name, minmax *table, int table_size) -{ - minmax total_mm; - - total_mm = accumulate_minmax_stuff(table, table_size); - - print_indent(3); - output_report("%s (%d iteration(s)):\n", name, table_size); - - /* Note: The maximum throughput uses the minimum amount of time & vice versa */ - - print_indent(4); - output_report("Minimum Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, (total_mm.min)); - - print_indent(4); - output_report("Average Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, - (total_mm.sum / total_mm.num)); - - print_indent(4); - output_report("Maximum Accumulated Time using %d file(s): %7.5f s\n", opts->num_files, (total_mm.max)); -} - -/* - * Function: output_report - * Purpose: Print a line of the report. Only do so if I'm the 0 process. - * Return: Nothing - * Programmer: Bill Wendling, 19. December 2001 - * Modifications: - */ -static void -output_report(const char *fmt, ...) -{ - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - - if (myrank == 0) { - va_list ap; - - HDva_start(ap, fmt); - HDvfprintf(output, fmt, ap); - HDva_end(ap); - } -} - -/* - * Function: print_indent - * Purpose: Print spaces to indent a new line of text for pretty printing - * things. - * Return: Nothing - * Programmer: Bill Wendling, 29. October 2001 - * Modifications: - */ -static void -print_indent(register int indent) -{ - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - - if (myrank == 0) { - indent *= TAB_SPACE; - - for (; indent > 0; --indent) - HDfputc(' ', output); - } -} - -static void -recover_size_and_print(long long val, const char *end) -{ - if (val >= ONE_KB && (val % ONE_KB) == 0) { - if (val >= ONE_MB && (val % ONE_MB) == 0) { - if (val >= ONE_GB && (val % ONE_GB) == 0) - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "GB%s", - val / ONE_GB, end); - else - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "MB%s", - val / ONE_MB, end); - } - else { - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "KB%s", - val / ONE_KB, end); - } - } - else { - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "%s", - val, end); - } -} - -static void -print_io_api(long io_types) -{ - if (io_types & PIO_POSIX) - HDfprintf(output, "posix "); - if (io_types & PIO_MPI) - HDfprintf(output, "mpiio "); - if (io_types & PIO_HDF5) - HDfprintf(output, "phdf5 "); - HDfprintf(output, "\n"); -} - -static void -report_parameters(struct options *opts) -{ - int rank = comm_world_rank_g; - - print_version("HDF5 Library"); /* print library version */ - HDfprintf(output, "rank %d: ==== Parameters ====\n", rank); - - HDfprintf(output, "rank %d: IO API=", rank); - print_io_api(opts->io_types); - - HDfprintf(output, "rank %d: Number of files=%ld\n", rank, opts->num_files); - HDfprintf(output, "rank %d: Number of datasets=%ld\n", rank, opts->num_dsets); - HDfprintf(output, "rank %d: Number of iterations=%d\n", rank, opts->num_iters); - HDfprintf(output, "rank %d: Number of processes=%d:%d\n", rank, opts->min_num_procs, opts->max_num_procs); - - if (opts->dim2d) { - HDfprintf(output, "rank %d: Number of bytes per process per dataset=", rank); - recover_size_and_print((long long)(opts->num_bpp * opts->num_bpp * opts->min_num_procs), ":"); - recover_size_and_print((long long)(opts->num_bpp * opts->num_bpp * opts->max_num_procs), "\n"); - - HDfprintf(output, "rank %d: Size of dataset(s)=", rank); - recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), "x"); - recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), ":"); - recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "x"); - recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "\n"); - - HDfprintf(output, "rank %d: File size=", rank); - recover_size_and_print((long long)(squareo(opts->num_bpp * opts->min_num_procs) * opts->num_dsets), - ":"); - recover_size_and_print((long long)(squareo(opts->num_bpp * opts->max_num_procs) * opts->num_dsets), - "\n"); - - HDfprintf(output, "rank %d: Transfer buffer size=", rank); - if (opts->interleaved) { - recover_size_and_print((long long)opts->min_xfer_size, "x"); - recover_size_and_print((long long)opts->blk_size, ":"); - recover_size_and_print((long long)opts->max_xfer_size, "x"); - recover_size_and_print((long long)opts->blk_size, "\n"); - } - else { - recover_size_and_print((long long)opts->blk_size, "x"); - recover_size_and_print((long long)opts->min_xfer_size, ":"); - recover_size_and_print((long long)opts->blk_size, "x"); - recover_size_and_print((long long)opts->max_xfer_size, "\n"); - } - HDfprintf(output, "rank %d: Block size=", rank); - recover_size_and_print((long long)opts->blk_size, "x"); - recover_size_and_print((long long)opts->blk_size, "\n"); - } - else { - HDfprintf(output, "rank %d: Number of bytes per process per dataset=", rank); - recover_size_and_print((long long)opts->num_bpp, "\n"); - - HDfprintf(output, "rank %d: Size of dataset(s)=", rank); - recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs), ":"); - recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs), "\n"); - - HDfprintf(output, "rank %d: File size=", rank); - recover_size_and_print((long long)(opts->num_bpp * opts->min_num_procs * opts->num_dsets), ":"); - recover_size_and_print((long long)(opts->num_bpp * opts->max_num_procs * opts->num_dsets), "\n"); - - HDfprintf(output, "rank %d: Transfer buffer size=", rank); - recover_size_and_print((long long)opts->min_xfer_size, ":"); - recover_size_and_print((long long)opts->max_xfer_size, "\n"); - HDfprintf(output, "rank %d: Block size=", rank); - recover_size_and_print((long long)opts->blk_size, "\n"); - } - - HDfprintf(output, "rank %d: Block Pattern in Dataset=", rank); - if (opts->interleaved) - HDfprintf(output, "Interleaved\n"); - else - HDfprintf(output, "Contiguous\n"); - - HDfprintf(output, "rank %d: I/O Method for MPI and HDF5=", rank); - if (opts->collective) - HDfprintf(output, "Collective\n"); - else - HDfprintf(output, "Independent\n"); - - HDfprintf(output, "rank %d: Geometry=", rank); - if (opts->dim2d) - HDfprintf(output, "2D\n"); - else - HDfprintf(output, "1D\n"); - - HDfprintf(output, "rank %d: VFL used for HDF5 I/O=%s\n", rank, "MPI-IO driver"); - - HDfprintf(output, "rank %d: Data storage method in HDF5=", rank); - if (opts->h5_use_chunks) - HDfprintf(output, "Chunked\n"); - else - HDfprintf(output, "Contiguous\n"); - - { - char *prefix = HDgetenv("HDF5_PARAPREFIX"); - - HDfprintf(output, "rank %d: Env HDF5_PARAPREFIX=%s\n", rank, (prefix ? prefix : "not set")); - } - - HDfprintf(output, "rank %d: ", rank); - h5_dump_info_object(h5_io_info_g); - - HDfprintf(output, "rank %d: ==== End of Parameters ====\n", rank); - HDfprintf(output, "\n"); -} - -/* - * Function: parse_command_line - * Purpose: Parse the command line options and return a STRUCT OPTIONS - * structure which will need to be freed by the calling function. - * Return: Pointer to an OPTIONS structure - * Programmer: Bill Wendling, 31. October 2001 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static struct options * -parse_command_line(int argc, char *argv[]) -{ - register int opt; - struct options *cl_opts; - - cl_opts = (struct options *)malloc(sizeof(struct options)); - - cl_opts->output_file = NULL; - cl_opts->io_types = 0; /* will set default after parsing options */ - cl_opts->num_dsets = 1; - cl_opts->num_files = 1; - cl_opts->num_bpp = 0; - cl_opts->num_iters = 1; - cl_opts->max_num_procs = comm_world_nprocs_g; - cl_opts->min_num_procs = 1; - cl_opts->max_xfer_size = 0; - cl_opts->min_xfer_size = 0; - cl_opts->blk_size = 0; - cl_opts->interleaved = 0; /* Default to contiguous blocks in dataset */ - cl_opts->collective = 0; /* Default to independent I/O access */ - cl_opts->dim2d = 0; /* Default to 1D */ - cl_opts->print_times = FALSE; /* Printing times is off by default */ - cl_opts->print_raw = FALSE; /* Printing raw data throughput is off by default */ - cl_opts->h5_alignment = 1; /* No alignment for HDF5 objects by default */ - cl_opts->h5_threshold = 1; /* No threshold for aligning HDF5 objects by default */ - cl_opts->h5_use_chunks = FALSE; /* Don't chunk the HDF5 dataset by default */ - cl_opts->h5_write_only = FALSE; /* Do both read and write by default */ - cl_opts->verify = FALSE; /* No Verify data correctness by default */ - - while ((opt = get_option(argc, (const char **)argv, s_opts, l_opts)) != EOF) { - switch ((char)opt) { - case 'a': - cl_opts->h5_alignment = parse_size_directive(opt_arg); - break; - case 'A': { - const char *end = opt_arg; - - while (end && *end != '\0') { - char buf[10]; - int i; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (isalnum(*end) && i < 10) - buf[i++] = *end; - - if (!HDstrcasecmp(buf, "phdf5")) { - cl_opts->io_types |= PIO_HDF5; - } - else if (!HDstrcasecmp(buf, "mpiio")) { - cl_opts->io_types |= PIO_MPI; - } - else if (!HDstrcasecmp(buf, "posix")) { - cl_opts->io_types |= PIO_POSIX; - } - else { - HDfprintf(stderr, "pio_perf: invalid --api option %s\n", buf); - HDexit(EXIT_FAILURE); - } - - if (*end == '\0') - break; - - end++; - } - } - - break; -#if 0 - case 'b': - /* the future "binary" option */ - break; -#endif /* 0 */ - case 'B': - cl_opts->blk_size = (size_t)parse_size_directive(opt_arg); - break; - case 'c': - /* Turn on chunked HDF5 dataset creation */ - cl_opts->h5_use_chunks = TRUE; - break; - case 'C': - cl_opts->collective = 1; - break; - case 'd': - cl_opts->num_dsets = atoi(opt_arg); - break; - case 'D': { - const char *end = opt_arg; - - while (end && *end != '\0') { - char buf[10]; - int i; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - if (HDstrlen(buf) > 1 || HDisdigit(buf[0])) { - size_t j; - - for (j = 0; j < 10 && buf[j] != '\0'; ++j) - if (!isdigit(buf[j])) { - HDfprintf(stderr, "pio_perf: invalid --debug option %s\n", buf); - HDexit(EXIT_FAILURE); - } - - pio_debug_level = atoi(buf); - - if (pio_debug_level > 4) - pio_debug_level = 4; - else if (pio_debug_level < 0) - pio_debug_level = 0; - } - else { - switch (*buf) { - case 'r': - /* Turn on raw data throughput info */ - cl_opts->print_raw = TRUE; - break; - case 't': - /* Turn on time printing */ - cl_opts->print_times = TRUE; - break; - case 'v': - /* Turn on verify data correctness*/ - cl_opts->verify = TRUE; - break; - default: - HDfprintf(stderr, "pio_perf: invalid --debug option %s\n", buf); - HDexit(EXIT_FAILURE); - } - } - - if (*end == '\0') - break; - - end++; - } - } - - break; - case 'e': - cl_opts->num_bpp = parse_size_directive(opt_arg); - break; - case 'F': - cl_opts->num_files = HDatoi(opt_arg); - break; - case 'g': - cl_opts->dim2d = 1; - break; - case 'i': - cl_opts->num_iters = HDatoi(opt_arg); - break; - case 'I': - cl_opts->interleaved = 1; - break; - case 'o': - cl_opts->output_file = opt_arg; - break; - case 'p': - cl_opts->min_num_procs = HDatoi(opt_arg); - break; - case 'P': - cl_opts->max_num_procs = HDatoi(opt_arg); - break; - case 'T': - cl_opts->h5_threshold = parse_size_directive(opt_arg); - break; - case 'w': - cl_opts->h5_write_only = TRUE; - break; - case 'x': - cl_opts->min_xfer_size = (size_t)parse_size_directive(opt_arg); - break; - case 'X': - cl_opts->max_xfer_size = (size_t)parse_size_directive(opt_arg); - break; - case 'h': - case '?': - default: - usage(progname); - HDfree(cl_opts); - return NULL; - } - } - - if (cl_opts->num_bpp == 0) { - if (cl_opts->dim2d == 0) - cl_opts->num_bpp = 256 * ONE_KB; - else - cl_opts->num_bpp = 8 * ONE_KB; - } - - if (cl_opts->max_xfer_size == 0) - cl_opts->max_xfer_size = (size_t)cl_opts->num_bpp; - - if (cl_opts->min_xfer_size == 0) - cl_opts->min_xfer_size = (size_t)(cl_opts->num_bpp) / 2; - - if (cl_opts->blk_size == 0) - cl_opts->blk_size = (size_t)(cl_opts->num_bpp) / 2; - - /* set default if none specified yet */ - if (!cl_opts->io_types) - cl_opts->io_types = PIO_HDF5 | PIO_MPI | PIO_POSIX; /* run all API */ - - /* verify parameters sanity. Adjust if needed. */ - /* cap xfer_size with bytes per process */ - if (!cl_opts->dim2d) { - if (cl_opts->min_xfer_size > (size_t)cl_opts->num_bpp) - cl_opts->min_xfer_size = (size_t)cl_opts->num_bpp; - if (cl_opts->max_xfer_size > (size_t)cl_opts->num_bpp) - cl_opts->max_xfer_size = (size_t)cl_opts->num_bpp; - } - if (cl_opts->min_xfer_size > cl_opts->max_xfer_size) - cl_opts->min_xfer_size = cl_opts->max_xfer_size; - if (cl_opts->blk_size > (size_t)cl_opts->num_bpp) - cl_opts->blk_size = (size_t)cl_opts->num_bpp; - /* check range of number of processes */ - if (cl_opts->min_num_procs <= 0) - cl_opts->min_num_procs = 1; - if (cl_opts->max_num_procs <= 0) - cl_opts->max_num_procs = 1; - if (cl_opts->min_num_procs > cl_opts->max_num_procs) - cl_opts->min_num_procs = cl_opts->max_num_procs; - /* check iteration */ - if (cl_opts->num_iters <= 0) - cl_opts->num_iters = 1; - - return cl_opts; -} - -/* - * Function: parse_size_directive - * Purpose: Parse the size directive passed on the commandline. The size - * directive is an integer followed by a size indicator: - * - * K, k - Kilobyte - * M, m - Megabyte - * G, g - Gigabyte - * - * Return: The size as a off_t because this is related to file size. - * If an unknown size indicator is used, then the program will - * exit with EXIT_FAILURE as the return value. - * Programmer: Bill Wendling, 18. December 2001 - * Modifications: - */ -static off_t -parse_size_directive(const char *size) -{ - off_t s; - char *endptr; - - s = HDstrtol(size, &endptr, 10); - - if (endptr && *endptr) { - while (*endptr != '\0' && (*endptr == ' ' || *endptr == '\t')) - ++endptr; - - switch (*endptr) { - case 'K': - case 'k': - s *= ONE_KB; - break; - case 'M': - case 'm': - s *= ONE_MB; - break; - case 'G': - case 'g': - s *= ONE_GB; - break; - default: - HDfprintf(stderr, "Illegal size specifier '%c'\n", *endptr); - HDexit(EXIT_FAILURE); - } - } - - return s; -} - -/* - * Function: usage - * Purpose: Print a usage message and then exit. - * Return: Nothing - * Programmer: Bill Wendling, 31. October 2001 - * Modifications: - * Added 2D testing (Christian Chilan, 10. August 2005) - */ -static void -usage(const char *prog) -{ - int myrank; - - MPI_Comm_rank(pio_comm_g, &myrank); - - if (myrank == 0) { - print_version(prog); - HDprintf("usage: %s [OPTIONS]\n", prog); - HDprintf(" OPTIONS\n"); - HDprintf(" -h, --help Print a usage message and exit\n"); - HDprintf(" -a S, --align=S Alignment of objects in HDF5 file [default: 1]\n"); - HDprintf(" -A AL, --api=AL Which APIs to test [default: all of them]\n"); -#if 0 - HDprintf(" -b, --binary The elusive binary option\n"); -#endif /* 0 */ - HDprintf(" -B S, --block-size=S Block size within transfer buffer\n"); - HDprintf(" (see below for description)\n"); - HDprintf(" [default: half the number of bytes per process\n"); - HDprintf(" per dataset]\n"); - HDprintf(" -c, --chunk Create HDF5 datasets using chunked storage\n"); - HDprintf(" [default: contiguous storage]\n"); - HDprintf(" -C, --collective Use collective I/O for MPI and HDF5 APIs\n"); - HDprintf(" [default: independent I/O)\n"); - HDprintf(" -d N, --num-dsets=N Number of datasets per file [default: 1]\n"); - HDprintf(" -D DL, --debug=DL Indicate the debugging level\n"); - HDprintf(" [default: no debugging]\n"); - HDprintf(" -e S, --num-bytes=S Number of bytes per process per dataset\n"); - HDprintf(" (see below for description)\n"); - HDprintf(" [default: 256K for 1D, 8K for 2D]\n"); - HDprintf(" -F N, --num-files=N Number of files [default: 1]\n"); - HDprintf(" -g, --geometry Use 2D geometry [default: 1D geometry]\n"); - HDprintf(" -i N, --num-iterations=N Number of iterations to perform [default: 1]\n"); - HDprintf(" -I, --interleaved Interleaved access pattern\n"); - HDprintf(" (see below for example)\n"); - HDprintf(" [default: Contiguous access pattern]\n"); - HDprintf(" -o F, --output=F Output raw data into file F [default: none]\n"); - HDprintf(" -p N, --min-num-processes=N Minimum number of processes to use [default: 1]\n"); - HDprintf(" -P N, --max-num-processes=N Maximum number of processes to use\n"); - HDprintf(" [default: all MPI_COMM_WORLD processes ]\n"); - HDprintf(" -T S, --threshold=S Threshold for alignment of objects in HDF5 file\n"); - HDprintf(" [default: 1]\n"); - HDprintf(" -w, --write-only Perform write tests not the read tests\n"); - HDprintf(" -x S, --min-xfer-size=S Minimum transfer buffer size\n"); - HDprintf(" (see below for description)\n"); - HDprintf(" [default: half the number of bytes per process\n"); - HDprintf(" per dataset]\n"); - HDprintf(" -X S, --max-xfer-size=S Maximum transfer buffer size\n"); - HDprintf(" [default: the number of bytes per process per\n"); - HDprintf(" dataset]\n"); - HDprintf("\n"); - HDprintf(" F - is a filename.\n"); - HDprintf(" N - is an integer >=0.\n"); - HDprintf(" S - is a size specifier, an integer >=0 followed by a size indicator:\n"); - HDprintf(" K - Kilobyte (%d)\n", ONE_KB); - HDprintf(" M - Megabyte (%d)\n", ONE_MB); - HDprintf(" G - Gigabyte (%d)\n", ONE_GB); - HDprintf("\n"); - HDprintf(" Example: '37M' is 37 megabytes or %d bytes\n", 37 * ONE_MB); - HDprintf("\n"); - HDprintf(" AL - is an API list. Valid values are:\n"); - HDprintf(" phdf5 - Parallel HDF5\n"); - HDprintf(" mpiio - MPI-I/O\n"); - HDprintf(" posix - POSIX\n"); - HDprintf("\n"); - HDprintf(" Example: --api=mpiio,phdf5\n"); - HDprintf("\n"); - HDprintf(" Dataset size:\n"); - HDprintf(" Depending on the selected geometry, each test dataset is either a linear\n"); - HDprintf(" array of size bytes-per-process * num-processes, or a square array of size\n"); - HDprintf(" (bytes-per-process * num-processes) x (bytes-per-process * num-processes).\n"); - HDprintf("\n"); - HDprintf(" Block size vs. Transfer buffer size:\n"); - HDprintf(" buffer-size controls the size of the memory buffer, which is broken into\n"); - HDprintf(" blocks and written to the file. Depending on the selected geometry, each\n"); - HDprintf(" block can be a linear array of size block-size or a square array of size\n"); - HDprintf(" block-size x block-size. The arrangement in which blocks are written is\n"); - HDprintf(" determined by the access pattern.\n"); - HDprintf("\n"); - HDprintf(" In 1D geometry, the transfer buffer is a linear array of size buffer-size.\n"); - HDprintf(" In 2D geometry, it is a rectangular array of size block-size x buffer-size\n"); - HDprintf(" or buffer-size x block-size if interleaved pattern is selected.\n"); - HDprintf("\n"); - HDprintf(" Interleaved and Contiguous patterns in 1D geometry:\n"); - HDprintf(" When contiguous access pattern is chosen, the dataset is evenly divided\n"); - HDprintf(" into num-processes regions and each process writes data to its own region.\n"); - HDprintf(" When interleaved blocks are written to a dataset, space for the first\n"); - HDprintf(" block of the first process is allocated in the dataset, then space is\n"); - HDprintf(" allocated for the first block of the second process, etc. until space is\n"); - HDprintf(" allocated for the first block of each process, then space is allocated for\n"); - HDprintf(" the second block of the first process, the second block of the second\n"); - HDprintf(" process, etc.\n"); - HDprintf("\n"); - HDprintf(" For example, with a 3 process run, 512KB bytes-per-process, 256KB transfer\n"); - HDprintf(" buffer size, and 64KB block size, each process must issue 2 transfer\n"); - HDprintf(" requests to complete access to the dataset.\n"); - HDprintf(" Contiguous blocks of the first transfer request are written like so:\n"); - HDprintf(" 1111----2222----3333----\n"); - HDprintf(" Interleaved blocks of the first transfer request are written like so:\n"); - HDprintf(" 123123123123------------\n"); - HDprintf(" The actual number of I/O operations involved in a transfer request\n"); - HDprintf(" depends on the access pattern and communication mode.\n"); - HDprintf(" When using independent I/O with interleaved pattern, each process\n"); - HDprintf(" performs 4 small non-contiguous I/O operations per transfer request.\n"); - HDprintf(" If collective I/O is turned on, the combined content of the buffers of\n"); - HDprintf(" the 3 processes will be written using one collective I/O operation\n"); - HDprintf(" per transfer request.\n"); - HDprintf("\n"); - HDprintf(" For information about access patterns in 2D geometry, please refer to the\n"); - HDprintf(" HDF5 Reference Manual.\n"); - HDprintf("\n"); - HDprintf(" DL - is a list of debugging flags. Valid values are:\n"); - HDprintf(" 1 - Minimal\n"); - HDprintf(" 2 - Not quite everything\n"); - HDprintf(" 3 - Everything\n"); - HDprintf(" 4 - The kitchen sink\n"); - HDprintf(" r - Raw data I/O throughput information\n"); - HDprintf(" t - Times as well as throughputs\n"); - HDprintf(" v - Verify data correctness\n"); - HDprintf("\n"); - HDprintf(" Example: --debug=2,r,t\n"); - HDprintf("\n"); - HDprintf(" Environment variables:\n"); - HDprintf(" HDF5_NOCLEANUP Do not remove data files if set [default remove]\n"); - HDprintf(" HDF5_MPI_INFO MPI INFO object key=value separated by ;\n"); - HDprintf(" HDF5_PARAPREFIX Paralllel data files prefix\n"); - fflush(stdout); - } /* end if */ -} /* end usage() */ - -#else /* H5_HAVE_PARALLEL */ - -/* - * Function: main - * Purpose: Dummy main() function for if HDF5 was configured without - * parallel stuff. - * Return: EXIT_SUCCESS - * Programmer: Bill Wendling, 14. November 2001 - */ -int -main(void) -{ - HDprintf("No parallel IO performance because parallel is not configured\n"); - return EXIT_SUCCESS; -} /* end main */ - -#endif /* !H5_HAVE_PARALLEL */ diff --git a/tools/test/perform/pio_perf.h b/tools/test/perform/pio_perf.h deleted file mode 100644 index 24621da..0000000 --- a/tools/test/perform/pio_perf.h +++ /dev/null @@ -1,100 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef PIO_PERF_H -#define PIO_PERF_H - -#ifndef STANDALONE -#include "io_timer.h" -#include "h5test.h" -#include "h5tools.h" -#include "h5tools_utils.h" -#else -#include "io_timer.h" -#include "pio_standalone.h" -#endif - -/* setup the dataset no fill option if this is v1.5 or more */ -#if H5_VERS_MAJOR > 1 || H5_VERS_MINOR > 4 -#define H5_HAVE_NOFILL 1 -#endif - -typedef enum iotype_ { - POSIXIO, - MPIO, - PHDF5 - /*NUM_TYPES*/ -} iotype; - -typedef struct parameters_ { - iotype io_type; /* The type of IO test to perform */ - int num_procs; /* Maximum number of processes to use */ - long num_files; /* Number of files to create */ - long num_dsets; /* Number of datasets to create */ - off_t num_bytes; /* Number of bytes in each dset */ - int num_iters; /* Number of times to loop doing the IO */ - size_t buf_size; /* Buffer size */ - size_t blk_size; /* Block size */ - unsigned interleaved; /* Interleaved vs. contiguous blocks */ - unsigned collective; /* Collective vs. independent I/O */ - unsigned dim2d; /* 1D vs. 2D */ - hsize_t h5_align; /* HDF5 object alignment */ - hsize_t h5_thresh; /* HDF5 object alignment threshold */ - int h5_use_chunks; /* Make HDF5 dataset chunked */ - int h5_write_only; /* Perform the write tests only */ - int verify; /* Verify data correctness */ -} parameters; - -typedef struct results_ { - herr_t ret_code; - io_time_t *timers; -} results; - -#ifndef SUCCESS -#define SUCCESS 0 -#endif /* !SUCCESS */ - -#ifndef FAIL -#define FAIL -1 -#endif /* !FAIL */ - -extern FILE * output; /* output file */ -extern io_time_t *timer_g; /* timer: global for stub functions */ -extern int comm_world_rank_g; /* my rank in MPI_COMM_RANK */ -extern int comm_world_nprocs_g; /* num. of processes of MPI_COMM_WORLD */ -extern MPI_Comm pio_comm_g; /* Communicator to run the PIO */ -extern int pio_mpi_rank_g; /* MPI rank of pio_comm_g */ -extern int pio_mpi_nprocs_g; /* number of processes of pio_comm_g */ -extern int pio_debug_level; /* The debug level: - * 0 - Off - * 1 - Minimal - * 2 - Some more - * 3 - Maximal - * 4 - Even More Debugging (timer stuff) - */ - -#define HDprint_rank(f) /* print rank in MPI_COMM_WORLD */ HDfprintf(f, "%d: ", comm_world_rank_g); -#define HDprint_size(f) /* print size of MPI_COMM_WORLD */ HDfprintf(f, "%d", comm_world_nprocs_g); -#define HDprint_rank_size(f) /* print rank/size of MPI_COMM_WORLD */ \ - HDfprintf(f, "%d/%d: ", comm_world_rank_g, comm_world_nprocs_g); - -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -extern results do_pio(parameters param); - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* PIO_PERF_H */ diff --git a/tools/test/perform/pio_standalone.c b/tools/test/perform/pio_standalone.c index 0a9bfc0..032bfba 100644 --- a/tools/test/perform/pio_standalone.c +++ b/tools/test/perform/pio_standalone.c @@ -17,40 +17,31 @@ #include "pio_perf.h" -#ifdef STANDALONE -MPI_Info h5_io_info_g = MPI_INFO_NULL; /* MPI INFO object for IO */ -#endif - /** From h5tools_utils.c **/ /* global variables */ int nCols = 80; -/* ``get_option'' variables */ -int opt_err = 1; /*get_option prints errors if this is on */ -int opt_ind = 1; /*token pointer */ -const char *opt_arg; /*flag argument (or value) */ - int -get_option(int argc, const char **argv, const char *opts, const struct long_options *l_opts) +get_option(int argc, const char **argv, const char *opts, const struct h5_long_options *l_opts) { static int sp = 1; /* character index in current token */ int opt_opt = '?'; /* option character passed back to user */ if (sp == 1) { /* check for more flag-like tokens */ - if (opt_ind >= argc || argv[opt_ind][0] != '-' || argv[opt_ind][1] == '\0') { + if (H5_optind >= argc || argv[H5_optind][0] != '-' || argv[H5_optind][1] == '\0') { return EOF; } - else if (HDstrcmp(argv[opt_ind], "--") == 0) { - opt_ind++; + else if (HDstrcmp(argv[H5_optind], "--") == 0) { + H5_optind++; return EOF; } } - if (sp == 1 && argv[opt_ind][0] == '-' && argv[opt_ind][1] == '-') { + if (sp == 1 && argv[H5_optind][0] == '-' && argv[H5_optind][1] == '-') { /* long command line option */ - const char *arg = &argv[opt_ind][2]; + const char *arg = &argv[H5_optind][2]; int i; for (i = 0; l_opts && l_opts[i].name; i++) { @@ -62,13 +53,13 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti if (l_opts[i].has_arg != no_arg) { if (arg[len] == '=') { - opt_arg = &arg[len + 1]; + H5_optarg = &arg[len + 1]; } - else if (opt_ind < (argc - 1) && argv[opt_ind + 1][0] != '-') { - opt_arg = argv[++opt_ind]; + else if (H5_optind < (argc - 1) && argv[H5_optind + 1][0] != '-') { + H5_optarg = argv[++H5_optind]; } else if (l_opts[i].has_arg == require_arg) { - if (opt_err) + if (H5_opterr) HDfprintf(stderr, "%s: option required for \"--%s\" flag\n", argv[0], arg); opt_opt = '?'; @@ -76,13 +67,13 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti } else { if (arg[len] == '=') { - if (opt_err) + if (H5_opterr) HDfprintf(stderr, "%s: no option required for \"%s\" flag\n", argv[0], arg); opt_opt = '?'; } - opt_arg = NULL; + H5_optarg = NULL; } break; @@ -91,29 +82,29 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti if (l_opts[i].name == NULL) { /* exhausted all of the l_opts we have and still didn't match */ - if (opt_err) + if (H5_opterr) HDfprintf(stderr, "%s: unknown option \"%s\"\n", argv[0], arg); opt_opt = '?'; } - opt_ind++; + H5_optind++; sp = 1; } else { register char *cp; /* pointer into current token */ /* short command line option */ - opt_opt = argv[opt_ind][sp]; + opt_opt = argv[H5_optind][sp]; if (opt_opt == ':' || (cp = strchr(opts, opt_opt)) == 0) { - if (opt_err) + if (H5_opterr) HDfprintf(stderr, "%s: unknown option \"%c\"\n", argv[0], opt_opt); /* if no chars left in this token, move to next token */ - if (argv[opt_ind][++sp] == '\0') { - opt_ind++; + if (argv[H5_optind][++sp] == '\0') { + H5_optind++; sp = 1; } @@ -122,32 +113,32 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti if (*++cp == ':') { /* if a value is expected, get it */ - if (argv[opt_ind][sp + 1] != '\0') { + if (argv[H5_optind][sp + 1] != '\0') { /* flag value is rest of current token */ - opt_arg = &argv[opt_ind++][sp + 1]; + H5_optarg = &argv[H5_optind++][sp + 1]; } - else if (++opt_ind >= argc) { - if (opt_err) + else if (++H5_optind >= argc) { + if (H5_opterr) HDfprintf(stderr, "%s: value expected for option \"%c\"\n", argv[0], opt_opt); opt_opt = '?'; } else { /* flag value is next token */ - opt_arg = argv[opt_ind++]; + H5_optarg = argv[H5_optind++]; } sp = 1; } else { /* set up to look at next char in token, next time */ - if (argv[opt_ind][++sp] == '\0') { + if (argv[H5_optind][++sp] == '\0') { /* no more in current token, so setup next token */ - opt_ind++; + H5_optind++; sp = 1; } - opt_arg = NULL; + H5_optarg = NULL; } } @@ -161,134 +152,3 @@ print_version(const char *progname) printf("%s: Version %u.%u.%u%s%s\n", progname, H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE, H5_VERS_SUBRELEASE[0] ? "-" : "", H5_VERS_SUBRELEASE); } - -#ifdef STANDALONE -/* - * Function: h5_set_info_object - * Purpose: Process environment variables setting to set up MPI Info - * object. - * Return: 0 if all is fine; otherwise non-zero. - * Programmer: Albert Cheng, 2002/05/21. - * Modifications: - * Bill Wendling, 2002/05/31 - * Modified so that the HDF5_MPI_INFO environment variable can - * be a semicolon separated list of "key=value" pairings. Most - * of the code is to remove any whitespaces which might be - * surrounding the "key=value" pairs. - */ -int -h5_set_info_object(void) -{ - char *envp; /* environment pointer */ - int ret_value = 0; - - /* handle any MPI INFO hints via $HDF5_MPI_INFO */ - if ((envp = HDgetenv("HDF5_MPI_INFO")) != NULL) { - char *next, *valp; - - valp = envp = next = HDstrdup(envp); - - if (!valp) - return 0; - - /* create an INFO object if not created yet */ - if (h5_io_info_g == MPI_INFO_NULL) - MPI_Info_create(&h5_io_info_g); - - do { - size_t len; - char * key_val, *endp, *namep; - - if (*valp == ';') - valp++; - - /* copy key/value pair into temporary buffer */ - len = strcspn(valp, ";"); - next = &valp[len]; - key_val = (char *)HDcalloc(1, len + 1); - - /* increment the next pointer past the terminating semicolon */ - if (*next == ';') - ++next; - - namep = HDstrncpy(key_val, valp, len); - - /* pass up any beginning whitespaces */ - while (*namep && (*namep == ' ' || *namep == '\t')) - namep++; - - if (!*namep) - continue; /* was all white space, so move to next k/v pair */ - - /* eat up any ending white spaces */ - endp = &namep[HDstrlen(namep) - 1]; - - while (endp && (*endp == ' ' || *endp == '\t')) - *endp-- = '\0'; - - /* find the '=' */ - valp = HDstrchr(namep, '='); - - if (valp != NULL) { /* it's a valid key/value pairing */ - char *tmp_val = valp + 1; - - /* change '=' to \0, move valp down one */ - *valp-- = '\0'; - - /* eat up ending whitespace on the "key" part */ - while (*valp == ' ' || *valp == '\t') - *valp-- = '\0'; - - valp = tmp_val; - - /* eat up beginning whitespace on the "value" part */ - while (*valp == ' ' || *valp == '\t') - *valp++ = '\0'; - - /* actually set the darned thing */ - if (MPI_SUCCESS != MPI_Info_set(h5_io_info_g, namep, valp)) { - HDprintf("MPI_Info_set failed\n"); - ret_value = -1; - } - } - - valp = next; - HDfree(key_val); - } while (next && *next); - - HDfree(envp); - } - - return ret_value; -} - -/* - * Function: h5_dump_info_object - * Purpose: Display content of an MPI Info object - * Return: void - * Programmer: Albert Cheng 2002/05/21 - * Modifications: - */ -void -h5_dump_info_object(MPI_Info info) -{ - char key[MPI_MAX_INFO_KEY + 1]; - char value[MPI_MAX_INFO_VAL + 1]; - int flag; - int i, nkeys; - - HDprintf("Dumping MPI Info Object (up to %d bytes per item):\n", MPI_MAX_INFO_VAL); - if (info == MPI_INFO_NULL) { - HDprintf("object is MPI_INFO_NULL\n"); - } - else { - MPI_Info_get_nkeys(info, &nkeys); - HDprintf("object has %d items\n", nkeys); - for (i = 0; i < nkeys; i++) { - MPI_Info_get_nthkey(info, i, key); - MPI_Info_get(info, key, MPI_MAX_INFO_VAL, value, &flag); - HDprintf("%s=%s\n", key, value); - } - } -} -#endif /* STANDALONE */ diff --git a/tools/test/perform/pio_standalone.h b/tools/test/perform/pio_standalone.h index 35a5217..23cd0e8 100644 --- a/tools/test/perform/pio_standalone.h +++ b/tools/test/perform/pio_standalone.h @@ -463,37 +463,27 @@ extern char * strdup(const char *s); #define TRUE true #endif -/** From h5test.h **/ - -#ifdef H5_HAVE_PARALLEL -extern MPI_Info h5_io_info_g; /* MPI INFO object for IO */ -#endif - -#ifdef H5_HAVE_PARALLEL -int h5_set_info_object(void); -void h5_dump_info_object(MPI_Info info); -#endif - /** From h5tools_utils.h **/ -extern int opt_err; /* getoption prints errors if this is on */ -extern int opt_ind; /* token pointer */ -extern const char *opt_arg; /* flag argument (or value) */ +H5_DLLVAR int H5_opterr; /* getoption prints errors if this is on */ +H5_DLLVAR int H5_optind; /* token pointer */ +H5_DLLVAR const char *H5_optarg; /* flag argument (or value) */ -enum { +enum h5_arg_level { no_arg = 0, /* doesn't take an argument */ require_arg, /* requires an argument */ optional_arg /* argument is optional */ }; -typedef struct long_options { - const char *name; /* name of the long option */ - int has_arg; /* whether we should look for an arg */ - char shortval; /* the shortname equivalent of long arg - * this gets returned from get_option */ -} long_options; +struct h5_long_options { + const char * name; /* Name of the long option */ + enum h5_arg_level has_arg; /* Whether we should look for an arg */ + char shortval; /* The shortname equivalent of long arg + * this gets returned from get_option + */ +}; -extern int get_option(int argc, const char **argv, const char *opt, const struct long_options *l_opt); +extern int H5_get_option(int argc, const char **argv, const char *opt, const struct h5_long_options *l_opt); extern int nCols; /*max number of columns for outputting */ diff --git a/tools/test/perform/sio_engine.c b/tools/test/perform/sio_engine.c deleted file mode 100644 index e5a0ec8..0000000 --- a/tools/test/perform/sio_engine.c +++ /dev/null @@ -1,1338 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Author: Christian Chilan, April 2008 - */ - -#include "hdf5.h" - -#ifdef H5_STDC_HEADERS -#include -#include -#include -#include -#endif - -#ifdef H5_HAVE_UNISTD_H -#include -#include -#endif - -#ifdef H5_HAVE_SYS_STAT_H -#include -#endif - -#include "sio_perf.h" - -/* Macro definitions */ - -/* sizes of various items. these sizes won't change during program execution */ -#define ELMT_H5_TYPE H5T_NATIVE_UCHAR - -#define GOTOERROR(errcode) \ - { \ - ret_code = errcode; \ - goto done; \ - } -#define ERRMSG(mesg) \ - { \ - HDfprintf(stderr, "*** Assertion failed (%s) at line %4d in %s\n", mesg, (int)__LINE__, __FILE__); \ - } - -/* verify: if val is false (0), print mesg. */ -#define VRFY(val, mesg) \ - do { \ - if (!val) { \ - ERRMSG(mesg); \ - GOTOERROR(FAIL); \ - } \ - } while (0) - -/* POSIX I/O macros */ -#ifdef H5_HAVE_WIN32_API -/* Can't link against the library, so this test will use the older, non-Unicode - * _open() call on Windows. - */ -#define HDopen(S, F, ...) _open(S, F | _O_BINARY, __VA_ARGS__) -#endif /* H5_HAVE_WIN32_API */ -#define POSIXCREATE(fn) HDopen(fn, O_CREAT | O_TRUNC | O_RDWR, 0600) -#define POSIXOPEN(fn, F) HDopen(fn, F, 0600) -#define POSIXCLOSE(F) HDclose(F) -#define POSIXSEEK(F, L) HDlseek(F, L, SEEK_SET) -#define POSIXWRITE(F, B, S) HDwrite(F, B, S) -#define POSIXREAD(F, B, S) HDread(F, B, S) - -enum { SIO_CREATE = 1, SIO_WRITE = 2, SIO_READ = 4 }; - -/* Global variables */ -static int clean_file_g = -1; /*whether to cleanup temporary test */ -/*files. -1 is not defined; */ -/*0 is no cleanup; 1 is do cleanup */ - -/* the different types of file descriptors we can expect */ -typedef union { - int posixfd; /* POSIX file handle*/ - hid_t h5fd; /* HDF5 file */ -} file_descr; - -/* local functions */ -static char * sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size, - parameters *param); -static herr_t do_write(results *res, file_descr *fd, parameters *parms, void *buffer); -static herr_t do_read(results *res, file_descr *fd, parameters *parms, void *buffer); -static herr_t dset_write(int local_dim, file_descr *fd, parameters *parms, void *buffer); -static herr_t posix_buffer_write(int local_dim, file_descr *fd, parameters *parms, void *buffer); -static herr_t dset_read(int localrank, file_descr *fd, parameters *parms, void *buffer, const char *buffer2); -static herr_t posix_buffer_read(int local_dim, file_descr *fd, parameters *parms, void *buffer); -static herr_t do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags); -hid_t set_vfd(parameters *param); -static herr_t do_fclose(iotype iot, file_descr *fd); -static void do_cleanupfile(iotype iot, char *fname); - -/* global variables */ -static HDoff_t offset[MAX_DIMS]; /* dataset size in bytes */ -static size_t buf_offset[MAX_DIMS]; /* dataset size in bytes */ -static int order[MAX_DIMS]; /* dimension access order */ -static size_t linear_buf_size; /* linear buffer size */ -static int cont_dim; /* lowest dimension for contiguous POSIX - access */ -static size_t cont_size; /* size of contiguous POSIX access */ -static hid_t fapl; /* file access list */ -static unsigned char *buf_p; /* buffer pointer */ -static const char * multi_letters = "msbrglo"; /* string for multi driver */ - -/* HDF5 global variables */ -static hsize_t h5count[MAX_DIMS]; /*selection count */ -static hssize_t h5offset[MAX_DIMS]; /* Selection offset within dataspace */ -static hid_t h5dset_space_id = H5I_INVALID_HID; /*dataset space ID */ -static hid_t h5mem_space_id = H5I_INVALID_HID; /*memory dataspace ID */ -static hid_t h5ds_id = H5I_INVALID_HID; /*dataset handle */ -static hid_t h5dcpl = H5I_INVALID_HID; /* Dataset creation property list */ -static hid_t h5dxpl = H5I_INVALID_HID; /* Dataset transfer property list */ - -/* - * Function: do_sio - * Purpose: SIO Engine where IO are executed. - * Return: results - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ -void -do_sio(parameters param, results *res) -{ - char * buffer = NULL; /*data buffer pointer */ - size_t buf_size[MAX_DIMS]; /* general buffer size in bytes */ - file_descr fd; /* file handles */ - iotype iot; /* API type */ - char base_name[256]; /* test file base name */ - /* return codes */ - herr_t ret_code = 0; /*return code */ - - char fname[FILENAME_MAX]; /* test file name */ - int i; - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - - /* Sanity check parameters */ - - /* IO type */ - iot = param.io_type; - - switch (iot) { - case POSIXIO: - fd.posixfd = -1; - res->timers = io_time_new(SYS_CLOCK); - break; - case HDF5: - fd.h5fd = -1; - res->timers = io_time_new(SYS_CLOCK); - break; - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - GOTOERROR(FAIL); - } - - linear_buf_size = 1; - - for (i = 0; i < param.rank; i++) { - buf_size[i] = param.buf_size[i]; - order[i] = param.order[i]; - linear_buf_size *= buf_size[i]; - buf_offset[i] = 0; - offset[i] = 0; - - /* Validate transfer buffer size */ - if (param.buf_size[i] <= 0) { - HDfprintf(stderr, "Transfer buffer size[%d] (%zu) must be > 0\n", i, buf_size[i]); - GOTOERROR(FAIL); - } - - if ((param.dset_size[i] % param.buf_size[i]) != 0) { - HDfprintf(stderr, - "Dataset size[%d] (%" H5_PRINTF_LL_WIDTH "d) must be a multiple of the " - "trasfer buffer size[%d] (%zu)\n", - param.rank, (long long)param.dset_size[i], param.rank, param.buf_size[i]); - GOTOERROR(FAIL); - } - } - - /* Allocate transfer buffer */ - if ((buffer = (char *)malloc(linear_buf_size)) == NULL) { - HDfprintf(stderr, "malloc for transfer buffer size (%zu) failed\n", linear_buf_size); - GOTOERROR(FAIL); - } - - if (sio_debug_level >= 4) - - /* output all of the times for all iterations */ - HDfprintf(output, "Timer details:\n"); - - /* - * Write performance measurement - */ - /* Open file for write */ - - HDstrcpy(base_name, "#sio_tmp"); - sio_create_filename(iot, base_name, fname, sizeof(fname), ¶m); - - if (sio_debug_level > 0) - HDfprintf(output, "data filename=%s\n", fname); - - io_time_set(res->timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTART); - hrc = do_fopen(¶m, fname, &fd, SIO_CREATE | SIO_WRITE); - VRFY((hrc == SUCCESS), "do_fopen failed"); - - io_time_set(res->timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTART); - hrc = do_write(res, &fd, ¶m, buffer); - io_time_set(res->timers, HDF5_FINE_WRITE_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_write failed"); - - /* Close file for write */ - hrc = do_fclose(iot, &fd); - io_time_set(res->timers, HDF5_GROSS_WRITE_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_fclose failed"); - - if (!param.h5_write_only) { - /* - * Read performance measurement - */ - - /* Open file for read */ - io_time_set(res->timers, HDF5_GROSS_READ_FIXED_DIMS, TSTART); - hrc = do_fopen(¶m, fname, &fd, SIO_READ); - VRFY((hrc == SUCCESS), "do_fopen failed"); - - io_time_set(res->timers, HDF5_FINE_READ_FIXED_DIMS, TSTART); - hrc = do_read(res, &fd, ¶m, buffer); - io_time_set(res->timers, HDF5_FINE_READ_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_read failed"); - - /* Close file for read */ - hrc = do_fclose(iot, &fd); - - io_time_set(res->timers, HDF5_GROSS_READ_FIXED_DIMS, TSTOP); - VRFY((hrc == SUCCESS), "do_fclose failed"); - } - - do_cleanupfile(iot, fname); - -done: - /* clean up */ - /* release HDF5 objects */ - - /* close any opened files */ - /* no remove(fname) because that should have happened normally. */ - switch (iot) { - case POSIXIO: - if (fd.posixfd != -1) - hrc = do_fclose(iot, &fd); - break; - case HDF5: - if (fd.h5fd != -1) - hrc = do_fclose(iot, &fd); - break; - default: - /* unknown request */ - HDassert(0 && "Unknown IO type"); - break; - } - - /* release generic resources */ - if (buffer) - free(buffer); - - res->ret_code = ret_code; -} - -/* - * Function: sio_create_filename - * Purpose: Create a new filename to write to. Determine the correct - * suffix to append to the filename by the type of I/O we're - * doing. Also, place in the /tmp/{$USER,$LOGIN} directory if - * USER or LOGIN are specified in the environment. - * Return: Pointer to filename or NULL - * Programmer: Bill Wendling, 21. November 2001 - * Modifications: Support for file drivers. Christian Chilan, April, 2008 - */ -static char * -sio_create_filename(iotype iot, const char *base_name, char *fullname, size_t size, parameters *param) -{ - const char *prefix, *suffix = ""; - char * ptr, last = '\0'; - size_t i, j; - vfdtype vfd; - vfd = param->vfd; - - if (!base_name || !fullname || size < 1) - return NULL; - - memset(fullname, 0, size); - - switch (iot) { - case POSIXIO: - suffix = ".posix"; - break; - case HDF5: - suffix = ".h5"; - if (vfd == family) - suffix = "%05d.h5"; - else if (vfd == multi) - suffix = NULL; - break; - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - HDassert(0 && "Unknown IO type"); - break; - } - - /* First use the environment variable and then try the constant */ - prefix = HDgetenv("HDF5_PREFIX"); - -#ifdef HDF5_PREFIX - if (!prefix) - prefix = HDF5_PREFIX; -#endif /* HDF5_PREFIX */ - - /* Prepend the prefix value to the base name */ - if (prefix && *prefix) { - /* If the prefix specifies the HDF5_PREFIX directory, then - * default to using the "/tmp/$USER" or "/tmp/$LOGIN" - * directory instead. */ - register char *user, *login, *subdir; - - user = HDgetenv("USER"); - login = HDgetenv("LOGIN"); - subdir = (user ? user : login); - - if (subdir) { - for (i = 0; i < size - 1 && prefix[i]; i++) - fullname[i] = prefix[i]; - - fullname[i++] = '/'; - - for (j = 0; i < size && subdir[j]; i++, j++) - fullname[i] = subdir[j]; - } - else { - /* We didn't append the prefix yet */ - HDstrncpy(fullname, prefix, size); - fullname[size - 1] = '\0'; - } - - if ((HDstrlen(fullname) + HDstrlen(base_name) + 1) < size) { - /* Append the base_name with a slash first. Multiple slashes are - * handled below. */ - h5_stat_t buf; - - if (HDstat(fullname, &buf) < 0) - /* The directory doesn't exist just yet */ - if (HDmkdir(fullname, 0755) < 0 && errno != EEXIST) { - /* We couldn't make the "/tmp/${USER,LOGIN}" subdirectory. - * Default to PREFIX's original prefix value. */ - HDstrcpy(fullname, prefix); - } - - HDstrcat(fullname, "/"); - HDstrcat(fullname, base_name); - } - else { - /* Buffer is too small */ - return NULL; - } - } - else if (strlen(base_name) >= size) { - /* Buffer is too small */ - return NULL; - } - else { - HDstrcpy(fullname, base_name); - } - - /* Append a suffix */ - if (suffix) { - if (HDstrlen(fullname) + HDstrlen(suffix) >= size) - return NULL; - - HDstrcat(fullname, suffix); - } - - /* Remove any double slashes in the filename */ - for (ptr = fullname, i = j = 0; ptr && (i < size); i++, ptr++) { - if (*ptr != '/' || last != '/') - fullname[j++] = *ptr; - - last = *ptr; - } - - return fullname; -} - -/* - * Function: do_write - * Purpose: Write the required amount of data to the file. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ -static herr_t -do_write(results *res, file_descr *fd, parameters *parms, void *buffer) -{ - int ret_code = SUCCESS; - char dname[64]; - int i; - size_t u; - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - hsize_t h5dims[MAX_DIMS]; /*dataset dim sizes */ - hsize_t h5chunk[MAX_DIMS]; /*dataset dim sizes */ - hsize_t h5block[MAX_DIMS]; /*dataspace selection */ - hsize_t h5stride[MAX_DIMS]; /*selection stride */ - hsize_t h5start[MAX_DIMS]; /*selection start */ - hsize_t h5maxdims[MAX_DIMS]; - int rank; /*rank of dataset */ - - /* Prepare buffer for verifying data */ - /* if (parms->verify) - memset(buffer,1,linear_buf_size); */ - - buf_p = (unsigned char *)buffer; - - for (u = 0; u < linear_buf_size; u++) - buf_p[u] = u % 128; - - rank = parms->rank; - - for (i = 0; i < rank; i++) - h5offset[i] = offset[i] = 0; - - /* I/O Access specific setup */ - switch (parms->io_type) { - case POSIXIO: - - /* determine lowest dimension for contiguous POSIX access */ - cont_dim = rank; - - for (i = rank - 1; i >= 0; i--) { - if (parms->buf_size[i] == parms->dset_size[i]) - cont_dim = i; - else - break; - } - - /* determine size of the contiguous POSIX access */ - cont_size = (!cont_dim) ? 1 : parms->buf_size[cont_dim - 1]; - for (i = cont_dim; i < rank; i++) - cont_size *= parms->buf_size[i]; - - break; - - case HDF5: /* HDF5 setup */ - - for (i = 0; i < rank; i++) { - h5dims[i] = parms->dset_size[i]; - h5start[i] = 0; - h5stride[i] = 1; - h5block[i] = 1; - h5count[i] = parms->buf_size[i]; - h5chunk[i] = parms->chk_size[i]; - h5maxdims[i] = H5S_UNLIMITED; - } - - if (parms->h5_use_chunks && parms->h5_extendable) { - h5dset_space_id = H5Screate_simple(rank, h5count, h5maxdims); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - } - else { - h5dset_space_id = H5Screate_simple(rank, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - } - - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - - /* Create the memory dataspace that corresponds to the xfer buffer */ - h5mem_space_id = H5Screate_simple(rank, h5count, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - - /* Create the dataset transfer property list */ - h5dxpl = H5Pcreate(H5P_DATASET_XFER); - if (h5dxpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - break; - - default: - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - GOTOERROR(FAIL); - break; - } /* end switch */ - - /* create dataset */ - switch (parms->io_type) { - case POSIXIO: - break; - - case HDF5: - h5dcpl = H5Pcreate(H5P_DATASET_CREATE); - - if (h5dcpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - if (parms->h5_use_chunks) { - /* Set the chunk size to be the same as the buffer size */ - hrc = H5Pset_chunk(h5dcpl, rank, h5chunk); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Set failed\n"); - GOTOERROR(FAIL); - } /* end if */ - } /* end if */ - - HDsprintf(dname, "Dataset_%ld", (unsigned long)parms->num_bytes); - h5ds_id = - H5Dcreate2(fd->h5fd, dname, ELMT_H5_TYPE, h5dset_space_id, H5P_DEFAULT, h5dcpl, H5P_DEFAULT); - - if (h5ds_id < 0) { - HDfprintf(stderr, "HDF5 Dataset Create failed\n"); - GOTOERROR(FAIL); - } - - hrc = H5Pclose(h5dcpl); - /* verifying the close of the dcpl */ - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Property List Close failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - GOTOERROR(FAIL); - break; - } - - /* Start "raw data" write timer */ - io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTART); - - /* Perform write */ - hrc = dset_write(rank - 1, fd, parms, buffer); - - if (hrc < 0) { - HDfprintf(stderr, "Error in dataset write\n"); - GOTOERROR(FAIL); - } - - /* Stop "raw data" write timer */ - io_time_set(res->timers, HDF5_RAW_WRITE_FIXED_DIMS, TSTOP); - - /* Calculate write time */ - - /* Close dataset. Only HDF5 needs to do an explicit close. */ - if (parms->io_type == HDF5) { - hrc = H5Dclose(h5ds_id); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Close failed\n"); - GOTOERROR(FAIL); - } - - h5ds_id = H5I_INVALID_HID; - } /* end if */ - -done: - - /* release HDF5 objects */ - if (h5dset_space_id != -1) { - hrc = H5Sclose(h5dset_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); - ret_code = FAIL; - } - else { - h5dset_space_id = H5I_INVALID_HID; - } - } - - if (h5mem_space_id != -1) { - hrc = H5Sclose(h5mem_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); - ret_code = FAIL; - } - else { - h5mem_space_id = H5I_INVALID_HID; - } - } - - if (h5dxpl != -1) { - hrc = H5Pclose(h5dxpl); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); - ret_code = FAIL; - } - else { - h5dxpl = H5I_INVALID_HID; - } - } - - return ret_code; -} - -/* - * Function: dset_write - * Purpose: Write buffer into the dataset. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ -static herr_t -dset_write(int local_dim, file_descr *fd, parameters *parms, void *buffer) -{ - int cur_dim = order[local_dim] - 1; - int ret_code = SUCCESS; - int k; - hsize_t dims[MAX_DIMS], maxdims[MAX_DIMS]; - hsize_t i; - int j; - herr_t hrc; - - /* iterates according to the dimensions in order array */ - for (i = 0; i < parms->dset_size[cur_dim]; i += parms->buf_size[cur_dim]) { - - h5offset[cur_dim] = (hssize_t)i; - offset[cur_dim] = (HDoff_t)i; - - if (local_dim > 0) { - - dset_write(local_dim - 1, fd, parms, buffer); - } - else { - - switch (parms->io_type) { - - case POSIXIO: - /* initialize POSIX offset in the buffer */ - for (j = 0; j < parms->rank; j++) - buf_offset[j] = 0; - buf_p = (unsigned char *)buffer; - /* write POSIX buffer */ - posix_buffer_write(0, fd, parms, buffer); - break; - - case HDF5: - /* if dimensions are extendable, extend them as needed during access */ - if (parms->h5_use_chunks && parms->h5_extendable) { - - hrc = H5Sget_simple_extent_dims(h5dset_space_id, dims, maxdims); - VRFY((hrc >= 0), "H5Sget_simple_extent_dims"); - - for (k = 0; k < parms->rank; k++) { - - HDassert(h5offset[k] >= 0); - if (dims[k] <= (hsize_t)h5offset[k]) { - dims[k] = dims[k] + h5count[k]; - hrc = H5Sset_extent_simple(h5dset_space_id, parms->rank, dims, maxdims); - VRFY((hrc >= 0), "H5Sset_extent_simple"); - hrc = H5Dset_extent(h5ds_id, dims); - VRFY((hrc >= 0), "H5Dextend"); - } - } - } - /* applies offset */ - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - - /* Write the buffer out */ - hrc = H5Sget_simple_extent_dims(h5dset_space_id, dims, maxdims); - hrc = H5Dwrite(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dwrite"); - - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - HDassert(0 && "Unknown IO type"); - break; - } /* switch (parms->io_type) */ - } - } -done: - return ret_code; -} - -/* - * Function: posix_buffer_write - * Purpose: Write buffer into the POSIX file considering contiguity. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ - -static herr_t -posix_buffer_write(int local_dim, file_descr *fd, parameters *parms, void *buffer) -{ - int ret_code = SUCCESS; - - /* if dimension is not contiguous, call recursively */ - if (local_dim < parms->rank - 1 && local_dim != cont_dim) { - size_t u; - - for (u = 0; u < parms->buf_size[local_dim]; u++) { - buf_offset[local_dim] = u; - posix_buffer_write(local_dim + 1, fd, parms, buffer); - - /* if next dimension is cont_dim, it will fill out the buffer - traversing the entire dimension local_dim without the need - of performing iteration */ - if (local_dim + 1 == cont_dim) - break; - } - /* otherwise, perform contiguous POSIX access */ - } - else { - HDoff_t d_offset; - HDoff_t linear_dset_offset = 0; - int i, j, rc; - - buf_offset[local_dim] = 0; - - /* determine offset in the buffer */ - for (i = 0; i < parms->rank; i++) { - d_offset = 1; - - for (j = i + 1; j < parms->rank; j++) - d_offset *= (HDoff_t)parms->dset_size[j]; - - linear_dset_offset += (offset[i] + (HDoff_t)buf_offset[i]) * d_offset; - } - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, linear_dset_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - /* check if all bytes are written */ - rc = ((ssize_t)cont_size == POSIXWRITE(fd->posixfd, buf_p, cont_size)); - VRFY((rc != 0), "POSIXWRITE"); - - /* Advance location in buffer */ - buf_p += cont_size; - } - -done: - return ret_code; -} - -/* - * Function: do_read - * Purpose: Read the required amount of data to the file. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ -static herr_t -do_read(results *res, file_descr *fd, parameters *parms, void *buffer) -{ - char * buffer2 = NULL; /* Buffer for data verification */ - int ret_code = SUCCESS; - char dname[64]; - int i; - size_t u; - /* HDF5 variables */ - herr_t hrc; /*HDF5 return code */ - hsize_t h5dims[MAX_DIMS]; /*dataset dim sizes */ - hsize_t h5block[MAX_DIMS]; /*dataspace selection */ - hsize_t h5stride[MAX_DIMS]; /*selection stride */ - hsize_t h5start[MAX_DIMS]; /*selection start */ - int rank; - - /* Allocate data verification buffer */ - if (NULL == (buffer2 = (char *)malloc(linear_buf_size))) { - HDfprintf(stderr, "malloc for data verification buffer size (%zu) failed\n", linear_buf_size); - GOTOERROR(FAIL); - } /* end if */ - - /* Prepare buffer for verifying data */ - for (u = 0; u < linear_buf_size; u++) - buffer2[u] = (char)(u % 128); - - rank = parms->rank; - for (i = 0; i < rank; i++) - h5offset[i] = offset[i] = 0; - - /* I/O Access specific setup */ - switch (parms->io_type) { - case POSIXIO: - cont_dim = rank; - - for (i = rank - 1; i >= 0; i--) { - if (parms->buf_size[i] == parms->dset_size[i]) - cont_dim = i; - else - break; - } - cont_size = (!cont_dim) ? 1 : parms->buf_size[cont_dim - 1]; - for (i = cont_dim; i < rank; i++) - cont_size *= parms->buf_size[i]; - - break; - - case HDF5: /* HDF5 setup */ - for (i = 0; i < rank; i++) { - h5dims[i] = parms->dset_size[i]; - h5start[i] = 0; - h5stride[i] = 1; - h5block[i] = 1; - h5count[i] = parms->buf_size[i]; - } - - h5dset_space_id = H5Screate_simple(rank, h5dims, NULL); - VRFY((h5dset_space_id >= 0), "H5Screate_simple"); - - hrc = H5Sselect_hyperslab(h5dset_space_id, H5S_SELECT_SET, h5start, h5stride, h5count, h5block); - VRFY((hrc >= 0), "H5Sselect_hyperslab"); - - /* Create the memory dataspace that corresponds to the xfer buffer */ - h5mem_space_id = H5Screate_simple(rank, h5count, NULL); - VRFY((h5mem_space_id >= 0), "H5Screate_simple"); - - /* Create the dataset transfer property list */ - h5dxpl = H5Pcreate(H5P_DATASET_XFER); - if (h5dxpl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - GOTOERROR(FAIL); - break; - } /* end switch */ - - /* create dataset */ - switch (parms->io_type) { - case POSIXIO: - break; - - case HDF5: - HDsprintf(dname, "Dataset_%ld", (long)parms->num_bytes); - h5ds_id = H5Dopen2(fd->h5fd, dname, H5P_DEFAULT); - if (h5ds_id < 0) { - HDfprintf(stderr, "HDF5 Dataset open failed\n"); - GOTOERROR(FAIL); - } - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - GOTOERROR(FAIL); - break; - } /* end switch */ - - /* Start "raw data" read timer */ - io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTART); - hrc = dset_read(rank - 1, fd, parms, buffer, buffer2); - - if (hrc < 0) { - HDfprintf(stderr, "Error in dataset read\n"); - GOTOERROR(FAIL); - } - - /* Stop "raw data" read timer */ - io_time_set(res->timers, HDF5_RAW_READ_FIXED_DIMS, TSTOP); - - /* Calculate read time */ - - /* Close dataset. Only HDF5 needs to do an explicit close. */ - if (parms->io_type == HDF5) { - hrc = H5Dclose(h5ds_id); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Close failed\n"); - GOTOERROR(FAIL); - } - - h5ds_id = H5I_INVALID_HID; - } /* end if */ - -done: - - /* release HDF5 objects */ - if (h5dset_space_id != -1) { - hrc = H5Sclose(h5dset_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Space Close failed\n"); - ret_code = FAIL; - } - else { - h5dset_space_id = H5I_INVALID_HID; - } - } - - if (h5mem_space_id != -1) { - hrc = H5Sclose(h5mem_space_id); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Memory Space Close failed\n"); - ret_code = FAIL; - } - else { - h5mem_space_id = H5I_INVALID_HID; - } - } - - if (h5dxpl != -1) { - hrc = H5Pclose(h5dxpl); - if (hrc < 0) { - HDfprintf(stderr, "HDF5 Dataset Transfer Property List Close failed\n"); - ret_code = FAIL; - } - else { - h5dxpl = H5I_INVALID_HID; - } - } - - /* release generic resources */ - if (buffer2) - free(buffer2); - - return ret_code; -} - -/* - * Function: dset_read - * Purpose: Read buffer into the dataset. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ - -static herr_t -dset_read(int local_dim, file_descr *fd, parameters *parms, void *buffer, const char *buffer2) -{ - int cur_dim = order[local_dim] - 1; - hsize_t i; - int j; - herr_t hrc; - int ret_code = SUCCESS; - - /* iterate on the current dimension */ - for (i = 0; i < parms->dset_size[cur_dim]; i += parms->buf_size[cur_dim]) { - - h5offset[cur_dim] = (hssize_t)i; - offset[cur_dim] = (HDoff_t)i; - - /* if traverse in order array is incomplete, recurse */ - if (local_dim > 0) { - - ret_code = dset_read(local_dim - 1, fd, parms, buffer, buffer2); - - /* otherwise, write buffer into dataset */ - } - else { - - switch (parms->io_type) { - - case POSIXIO: - for (j = 0; j < parms->rank; j++) { - buf_offset[j] = 0; - } - buf_p = (unsigned char *)buffer; - posix_buffer_read(0, fd, parms, buffer); - break; - - case HDF5: - hrc = H5Soffset_simple(h5dset_space_id, h5offset); - VRFY((hrc >= 0), "H5Soffset_simple"); - /* Read the buffer out */ - hrc = H5Dread(h5ds_id, ELMT_H5_TYPE, h5mem_space_id, h5dset_space_id, h5dxpl, buffer); - VRFY((hrc >= 0), "H5Dread"); - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)parms->io_type); - HDassert(0 && "Unknown IO type"); - break; - } /* switch (parms->io_type) */ - } - } -done: - return ret_code; -} - -/* - * Function: posix_buffer_read - * Purpose: Read buffer into the POSIX file considering contiguity. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ - -static herr_t -posix_buffer_read(int local_dim, file_descr *fd, parameters *parms, void *buffer) -{ - int ret_code = SUCCESS; - - /* if local dimension is not contiguous, recurse */ - if (local_dim < parms->rank - 1 && local_dim != cont_dim) { - size_t u; - - for (u = 0; u < parms->buf_size[local_dim]; u++) { - buf_offset[local_dim] = u; - ret_code = posix_buffer_read(local_dim + 1, fd, parms, buffer); - if (local_dim + 1 == cont_dim) - break; - } - /* otherwise, perform contiguous POSIX access */ - } - else { - HDoff_t d_offset; - HDoff_t linear_dset_offset = 0; - int i, j, rc; - - buf_offset[local_dim] = 0; - /* determine offset in buffer */ - for (i = 0; i < parms->rank; i++) { - d_offset = 1; - - for (j = i + 1; j < parms->rank; j++) - d_offset *= (HDoff_t)parms->dset_size[j]; - - linear_dset_offset += (offset[i] + (HDoff_t)buf_offset[i]) * d_offset; - } - - /* only care if seek returns error */ - rc = POSIXSEEK(fd->posixfd, linear_dset_offset) < 0 ? -1 : 0; - VRFY((rc == 0), "POSIXSEEK"); - /* check if all bytes are read */ - rc = ((ssize_t)cont_size == POSIXREAD(fd->posixfd, buf_p, cont_size)); - VRFY((rc != 0), "POSIXREAD"); - - /* Advance location in buffer */ - buf_p += cont_size; - } -done: - return ret_code; -} - -/* - * Function: do_fopen - * Purpose: Open the specified file. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: Support for file drivers, Christian Chilan, April, 2008 - */ -static herr_t -do_fopen(parameters *param, char *fname, file_descr *fd /*out*/, int flags) -{ - int ret_code = SUCCESS; - hid_t fcpl; - - switch (param->io_type) { - case POSIXIO: - if (flags & (SIO_CREATE | SIO_WRITE)) - fd->posixfd = POSIXCREATE(fname); - else - fd->posixfd = POSIXOPEN(fname, O_RDONLY); - - if (fd->posixfd < 0) { - HDfprintf(stderr, "POSIX File Open failed(%s)\n", fname); - GOTOERROR(FAIL); - } - - break; - - case HDF5: - - fapl = set_vfd(param); - - if (fapl < 0) { - HDfprintf(stderr, "HDF5 Property List Create failed\n"); - GOTOERROR(FAIL); - } - - fcpl = H5Pcreate(H5P_FILE_CREATE); - if (param->page_size) { - H5Pset_file_space_strategy(fcpl, H5F_FSPACE_STRATEGY_PAGE, 0, (hsize_t)1); - H5Pset_file_space_page_size(fcpl, param->page_size); - if (param->page_buffer_size) - H5Pset_page_buffer_size(fapl, param->page_buffer_size, 0, 0); - } - - /* create the parallel file */ - if (flags & (SIO_CREATE | SIO_WRITE)) { - fd->h5fd = H5Fcreate(fname, H5F_ACC_TRUNC, fcpl, fapl); - } - else { - fd->h5fd = H5Fopen(fname, H5F_ACC_RDONLY, fapl); - } - - if (fd->h5fd < 0) { - HDfprintf(stderr, "HDF5 File Create failed(%s)\n", fname); - GOTOERROR(FAIL); - } - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)param->io_type); - GOTOERROR(FAIL); - break; - } - -done: - return ret_code; -} - -/* - * Function: set_vfd - * Purpose: Sets file driver. - * Return: SUCCESS or FAIL - * Programmer: Christian Chilan, April, 2008 - * Modifications: - */ - -hid_t -set_vfd(parameters *param) -{ - hid_t my_fapl = H5I_INVALID_HID; - vfdtype vfd; - - vfd = param->vfd; - - if ((my_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0) - return -1; - - if (vfd == sec2) { - /* Unix read() and write() system calls */ - if (H5Pset_fapl_sec2(my_fapl) < 0) - return -1; - } - else if (vfd == stdio) { - /* Standard C fread() and fwrite() system calls */ - if (H5Pset_fapl_stdio(my_fapl) < 0) - return -1; - } - else if (vfd == core) { - /* In-core temporary file with 1MB increment */ - if (H5Pset_fapl_core(my_fapl, (size_t)1024 * 1024, TRUE) < 0) - return -1; - } - else if (vfd == split) { - /* Split meta data and raw data each using default driver */ - if (H5Pset_fapl_split(my_fapl, "-m.h5", H5P_DEFAULT, "-r.h5", H5P_DEFAULT) < 0) - return -1; - } - else if (vfd == multi) { - /* Multi-file driver, general case of the split driver */ - H5FD_mem_t memb_map[H5FD_MEM_NTYPES]; - hid_t memb_fapl[H5FD_MEM_NTYPES]; - const char *memb_name[H5FD_MEM_NTYPES]; - char sv[H5FD_MEM_NTYPES][1024]; - haddr_t memb_addr[H5FD_MEM_NTYPES]; - H5FD_mem_t mt; - - HDmemset(memb_map, 0, sizeof memb_map); - HDmemset(memb_fapl, 0, sizeof memb_fapl); - HDmemset(memb_name, 0, sizeof memb_name); - HDmemset(memb_addr, 0, sizeof memb_addr); - - HDassert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { - memb_fapl[mt] = H5P_DEFAULT; - HDsprintf(sv[mt], "%%s-%c.h5", multi_letters[mt]); - memb_name[mt] = sv[mt]; - memb_addr[mt] = (haddr_t)MAX(mt - 1, 0) * (HADDR_MAX / 10); - } - - if (H5Pset_fapl_multi(my_fapl, memb_map, memb_fapl, memb_name, memb_addr, FALSE) < 0) { - return -1; - } - } - else if (vfd == family) { - hsize_t fam_size = 1 * 1024 * 1024; /*100 MB*/ - - /* Family of files, each 1MB and using the default driver */ - /* if ((val=HDstrtok(NULL, " \t\n\r"))) - fam_size = (hsize_t)(HDstrtod(val, NULL) * 1024*1024); */ - if (H5Pset_fapl_family(my_fapl, fam_size, H5P_DEFAULT) < 0) - return -1; - } - else if (vfd == direct) { -#ifdef H5_HAVE_DIRECT - /* Linux direct read() and write() system calls. Set memory boundary, file block size, - * and copy buffer size to the default values. */ - if (H5Pset_fapl_direct(my_fapl, 1024, 4096, 8 * 4096) < 0) - return -1; -#endif - } - else { - /* Unknown driver */ - return -1; - } - - return my_fapl; -} - -/* - * Function: do_fclose - * Purpose: Close the specified file descriptor. - * Return: SUCCESS or FAIL - * Programmer: Albert Cheng, Bill Wendling, 2001/12/13 - * Modifications: - */ -static herr_t -do_fclose(iotype iot, file_descr *fd /*out*/) -{ - herr_t ret_code = SUCCESS, hrc; - int rc = 0; - - switch (iot) { - case POSIXIO: - rc = POSIXCLOSE(fd->posixfd); - - if (rc != 0) { - HDfprintf(stderr, "POSIX File Close failed\n"); - GOTOERROR(FAIL); - } - - fd->posixfd = -1; - break; - - case HDF5: - hrc = H5Fclose(fd->h5fd); - - if (hrc < 0) { - HDfprintf(stderr, "HDF5 File Close failed\n"); - GOTOERROR(FAIL); - } - - fd->h5fd = -1; - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - GOTOERROR(FAIL); - break; - } - -done: - return ret_code; -} - -/* - * Function: do_cleanupfile - * Purpose: Cleanup temporary file unless HDF5_NOCLEANUP is set. - * Return: void - * Programmer: Albert Cheng 2001/12/12 - * Modifications: Support for file drivers. Christian Chilan, April, 2008 - */ -/* Disable warning for "format not a string literal" here -QAK */ -/* - * This pragma only needs to surround the snprintf() calls with - * 'temp' in the code below, but early (4.4.7, at least) gcc only - * allows diagnostic pragmas to be toggled outside of functions. - */ -H5_GCC_DIAG_OFF("format-nonliteral") -static void -do_cleanupfile(iotype iot, char *filename) -{ - char temp[2048]; - int j; - hid_t driver; - - if (clean_file_g == -1) - clean_file_g = (HDgetenv("HDF5_NOCLEANUP") == NULL) ? 1 : 0; - - if (clean_file_g) { - - switch (iot) { - case POSIXIO: - HDremove(filename); - break; - - case HDF5: - driver = H5Pget_driver(fapl); - - if (driver == H5FD_FAMILY) { - for (j = 0; /*void*/; j++) { - HDsnprintf(temp, sizeof temp, filename, j); - - if (HDaccess(temp, F_OK) < 0) - break; - - HDremove(temp); - } - } - else if (driver == H5FD_CORE) { - hbool_t backing; /* Whether the core file has backing store */ - - H5Pget_fapl_core(fapl, NULL, &backing); - - /* If the file was stored to disk with bacing store, remove it */ - if (backing) - HDremove(filename); - } - else if (driver == H5FD_MULTI) { - H5FD_mem_t mt; - assert(HDstrlen(multi_letters) == H5FD_MEM_NTYPES); - - for (mt = H5FD_MEM_DEFAULT; mt < H5FD_MEM_NTYPES; mt++) { - HDsnprintf(temp, sizeof temp, "%s-%c.h5", filename, multi_letters[mt]); - HDremove(temp); /*don't care if it fails*/ - } - } - else { - HDremove(filename); - } - H5Pclose(fapl); - break; - - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - HDassert(0 && "Unknown IO type"); - break; - } - } -} -H5_GCC_DIAG_ON("format-nonliteral") diff --git a/tools/test/perform/sio_perf.c b/tools/test/perform/sio_perf.c deleted file mode 100644 index fe4e7e7..0000000 --- a/tools/test/perform/sio_perf.c +++ /dev/null @@ -1,1437 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -/* - * Serial HDF5 Performance Testing Code - * -------------------------------------- - * - * Portable code to test performance on the different platforms we support. - * This is what the report should look like: - * - * nprocs = Max#Procs - * IO API = POSIXIO - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * - * IO API = HDF5 - * # Files = 1, # of dsets = 1000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * # Files = 1, # of dsets = 3000, Elements per dset = 37000 - * Write Results = x MB/s - * Read Results = x MB/s - * - * . . . - * - * - * . . . - * - */ - -/* system header files */ -#include -#include -#include - -#include "hdf5.h" - -/* our header files */ -#include "sio_perf.h" - -/* useful macros */ -#define TAB_SPACE 4 - -#define ONE_KB 1024 -#define ONE_MB (ONE_KB * ONE_KB) -#define ONE_GB (ONE_MB * ONE_KB) - -#define SIO_POSIX 0x1 -#define SIO_HDF5 0x4 - -/* report 0.0 in case t is zero too */ -#define MB_PER_SEC(bytes, t) (H5_DBL_ABS_EQUAL(t, 0.0) ? 0.0 : ((((double)bytes) / (double)ONE_MB) / (t))) - -#ifndef TRUE -#define TRUE 1 -#endif /* TRUE */ -#ifndef FALSE -#define FALSE (!TRUE) -#endif /* FALSE */ - -/* global variables */ -FILE *output; /* output file */ -int sio_debug_level = 0; /* The debug level: - * 0 - Off - * 1 - Minimal - * 2 - Some more - * 3 - Maximal - * 4 - Maximal & then some - */ - -/* local variables */ -static const char *progname = "h5perf_serial"; - -/* - * Command-line options: The user can specify short or long-named - * parameters. The long-named ones can be partially spelled. When - * adding more, make sure that they don't clash with each other. - */ - -/* - * It seems that only the options that accept additional information - * such as dataset size (-e) require the colon next to it. - */ -static const char * s_opts = "a:A:B:c:Cd:D:e:F:ghi:Imno:p:P:r:stT:v:wx:X:"; -static struct long_options l_opts[] = {{"align", require_arg, 'a'}, - {"alig", require_arg, 'a'}, - {"ali", require_arg, 'a'}, - {"al", require_arg, 'a'}, - {"api", require_arg, 'A'}, - {"ap", require_arg, 'A'}, -#if 0 - /* a sighting of the elusive binary option */ - { "binary", no_arg, 'b' }, - { "binar", no_arg, 'b' }, - { "bina", no_arg, 'b' }, - { "bin", no_arg, 'b' }, - { "bi", no_arg, 'b' }, -#endif /* 0 */ - {"block-size", require_arg, 'B'}, - {"block-siz", require_arg, 'B'}, - {"block-si", require_arg, 'B'}, - {"block-s", require_arg, 'B'}, - {"block-", require_arg, 'B'}, - {"block", require_arg, 'B'}, - {"bloc", require_arg, 'B'}, - {"blo", require_arg, 'B'}, - {"bl", require_arg, 'B'}, - {"chunk", no_arg, 'c'}, - {"chun", no_arg, 'c'}, - {"chu", no_arg, 'c'}, - {"ch", no_arg, 'c'}, - {"collective", no_arg, 'C'}, - {"collectiv", no_arg, 'C'}, - {"collecti", no_arg, 'C'}, - {"collect", no_arg, 'C'}, - {"collec", no_arg, 'C'}, - {"colle", no_arg, 'C'}, - {"coll", no_arg, 'C'}, - {"col", no_arg, 'C'}, - {"co", no_arg, 'C'}, - {"debug", require_arg, 'D'}, - {"debu", require_arg, 'D'}, - {"deb", require_arg, 'D'}, - {"de", require_arg, 'D'}, - {"file-driver", require_arg, 'v'}, - {"file-drive", require_arg, 'v'}, - {"file-driv", require_arg, 'v'}, - {"file-dri", require_arg, 'v'}, - {"file-dr", require_arg, 'v'}, - {"file-d", require_arg, 'v'}, - {"file-", require_arg, 'v'}, - {"file", require_arg, 'v'}, - {"fil", require_arg, 'v'}, - {"fi", require_arg, 'v'}, - {"geometry", no_arg, 'g'}, - {"geometr", no_arg, 'g'}, - {"geomet", no_arg, 'g'}, - {"geome", no_arg, 'g'}, - {"geom", no_arg, 'g'}, - {"geo", no_arg, 'g'}, - {"ge", no_arg, 'g'}, - {"help", no_arg, 'h'}, - {"hel", no_arg, 'h'}, - {"he", no_arg, 'h'}, - {"interleaved", require_arg, 'I'}, - {"interleave", require_arg, 'I'}, - {"interleav", require_arg, 'I'}, - {"interlea", require_arg, 'I'}, - {"interle", require_arg, 'I'}, - {"interl", require_arg, 'I'}, - {"inter", require_arg, 'I'}, - {"inte", require_arg, 'I'}, - {"int", require_arg, 'I'}, - {"in", require_arg, 'I'}, - {"max-num-processes", require_arg, 'P'}, - {"max-num-processe", require_arg, 'P'}, - {"max-num-process", require_arg, 'P'}, - {"max-num-proces", require_arg, 'P'}, - {"max-num-proce", require_arg, 'P'}, - {"max-num-proc", require_arg, 'P'}, - {"max-num-pro", require_arg, 'P'}, - {"max-num-pr", require_arg, 'P'}, - {"max-num-p", require_arg, 'P'}, - {"min-num-processes", require_arg, 'p'}, - {"min-num-processe", require_arg, 'p'}, - {"min-num-process", require_arg, 'p'}, - {"min-num-proces", require_arg, 'p'}, - {"min-num-proce", require_arg, 'p'}, - {"min-num-proc", require_arg, 'p'}, - {"min-num-pro", require_arg, 'p'}, - {"min-num-pr", require_arg, 'p'}, - {"min-num-p", require_arg, 'p'}, - {"max-xfer-size", require_arg, 'X'}, - {"max-xfer-siz", require_arg, 'X'}, - {"max-xfer-si", require_arg, 'X'}, - {"max-xfer-s", require_arg, 'X'}, - {"max-xfer", require_arg, 'X'}, - {"max-xfe", require_arg, 'X'}, - {"max-xf", require_arg, 'X'}, - {"max-x", require_arg, 'X'}, - {"min-xfer-size", require_arg, 'x'}, - {"min-xfer-siz", require_arg, 'x'}, - {"min-xfer-si", require_arg, 'x'}, - {"min-xfer-s", require_arg, 'x'}, - {"min-xfer", require_arg, 'x'}, - {"min-xfe", require_arg, 'x'}, - {"min-xf", require_arg, 'x'}, - {"min-x", require_arg, 'x'}, - {"num-bytes", require_arg, 'e'}, - {"num-byte", require_arg, 'e'}, - {"num-byt", require_arg, 'e'}, - {"num-by", require_arg, 'e'}, - {"num-b", require_arg, 'e'}, - {"num-dsets", require_arg, 'd'}, - {"num-dset", require_arg, 'd'}, - {"num-dse", require_arg, 'd'}, - {"num-ds", require_arg, 'd'}, - {"num-d", require_arg, 'd'}, - {"num-files", require_arg, 'F'}, - {"num-file", require_arg, 'F'}, - {"num-fil", require_arg, 'F'}, - {"num-fi", require_arg, 'F'}, - {"num-f", require_arg, 'F'}, - {"num-iterations", require_arg, 'i'}, - {"num-iteration", require_arg, 'i'}, - {"num-iteratio", require_arg, 'i'}, - {"num-iterati", require_arg, 'i'}, - {"num-iterat", require_arg, 'i'}, - {"num-itera", require_arg, 'i'}, - {"num-iter", require_arg, 'i'}, - {"num-ite", require_arg, 'i'}, - {"num-it", require_arg, 'i'}, - {"num-i", require_arg, 'i'}, - {"order", require_arg, 'r'}, - {"orde", require_arg, 'r'}, - {"ord", require_arg, 'r'}, - {"or", require_arg, 'r'}, - {"output", require_arg, 'o'}, - {"outpu", require_arg, 'o'}, - {"outp", require_arg, 'o'}, - {"out", require_arg, 'o'}, - {"ou", require_arg, 'o'}, - {"extendable", no_arg, 't'}, - {"extendabl", no_arg, 't'}, - {"extendab", no_arg, 't'}, - {"extenda", no_arg, 't'}, - {"extend", no_arg, 't'}, - {"exten", no_arg, 't'}, - {"exte", no_arg, 't'}, - {"ext", no_arg, 't'}, - {"ex", no_arg, 't'}, - {"threshold", require_arg, 'T'}, - {"threshol", require_arg, 'T'}, - {"thresho", require_arg, 'T'}, - {"thresh", require_arg, 'T'}, - {"thres", require_arg, 'T'}, - {"thre", require_arg, 'T'}, - {"thr", require_arg, 'T'}, - {"th", require_arg, 'T'}, - {"write-only", require_arg, 'w'}, - {"write-onl", require_arg, 'w'}, - {"write-on", require_arg, 'w'}, - {"write-o", require_arg, 'w'}, - {"write", require_arg, 'w'}, - {"writ", require_arg, 'w'}, - {"wri", require_arg, 'w'}, - {"wr", require_arg, 'w'}, - {NULL, 0, '\0'}}; - -struct options { - long io_types; /* bitmask of which I/O types to test */ - const char *output_file; /* file to print report to */ - long num_dsets; /* number of datasets */ - long num_files; /* number of files */ - off_t num_bpp; /* number of bytes per proc per dset */ - int num_iters; /* number of iterations */ - hsize_t dset_size[MAX_DIMS]; /* Dataset size */ - size_t buf_size[MAX_DIMS]; /* Buffer size */ - size_t chk_size[MAX_DIMS]; /* Chunk size */ - int order[MAX_DIMS]; /* Dimension access order */ - int dset_rank; /* Rank */ - int buf_rank; /* Rank */ - int order_rank; /* Rank */ - int chk_rank; /* Rank */ - int print_times; /* print times as well as throughputs */ - int print_raw; /* print raw data throughput info */ - hsize_t h5_alignment; /* alignment in HDF5 file */ - hsize_t h5_threshold; /* threshold for alignment in HDF5 file */ - int h5_use_chunks; /* Make HDF5 dataset chunked */ - int h5_write_only; /* Perform the write tests only */ - int h5_extendable; /* Perform the write tests only */ - int verify; /* Verify data correctness */ - vfdtype vfd; /* File driver */ - size_t page_buffer_size; - size_t page_size; -}; - -typedef struct { - double min; - double max; - double sum; - int num; -} minmax; - -/* local functions */ -static hsize_t parse_size_directive(const char *size); -static struct options *parse_command_line(int argc, const char *argv[]); -static void run_test_loop(struct options *options); -static int run_test(iotype iot, parameters parms, struct options *opts); -static void output_all_info(minmax *mm, int count, int indent_level); -static void get_minmax(minmax *mm, double val); -static void accumulate_minmax_stuff(const minmax *mm, int count, minmax *total_mm); -static void output_results(const struct options *options, const char *name, minmax *table, int table_size, - off_t data_size); -static void output_report(const char *fmt, ...); -static void print_indent(register int indent); -static void usage(const char *prog); -static void report_parameters(struct options *opts); - -/* - * Function: main - * Purpose: Start things up. - * Return: EXIT_SUCCESS or EXIT_FAILURE - * Programmer: Bill Wendling, 30. October 2001 - * Modifications: - */ -int -main(int argc, const char *argv[]) -{ - int exit_value = EXIT_SUCCESS; - struct options *opts = NULL; - -#ifndef STANDALONE - /* Initialize h5tools lib */ - h5tools_init(); -#endif - - output = stdout; - - opts = parse_command_line(argc, argv); - - if (!opts) { - exit_value = EXIT_FAILURE; - goto finish; - } - - if (opts->output_file) { - if ((output = HDfopen(opts->output_file, "w")) == NULL) { - HDfprintf(stderr, "%s: cannot open output file\n", progname); - HDperror(opts->output_file); - goto finish; - } - } - - report_parameters(opts); - - run_test_loop(opts); - -finish: - HDfree(opts); - return exit_value; -} - -/* - * Function: run_test_loop - * Purpose: Run the I/O tests. Write the results to OUTPUT. - * - * - The slowest changing part of the test is the number of - * processors to use. For each loop iteration, we divide that - * number by 2 and rerun the test. - * - * - The second slowest is what type of IO API to perform. We have - * three choices: POSIXIO, and HDF5. - * - * - Then we change the size of the buffer. This information is - * inferred from the number of datasets to create and the number - * of integers to put into each dataset. The backend code figures - * this out. - * - * Return: Nothing - * Programmer: Bill Wendling, 30. October 2001 - * Modifications: - * Added multidimensional testing (Christian Chilan, April, 2008) - */ -static void -run_test_loop(struct options *opts) -{ - parameters parms; - int i; - size_t buf_bytes; - - /* load options into parameter structure */ - parms.num_files = opts->num_files; - parms.num_dsets = opts->num_dsets; - parms.num_iters = opts->num_iters; - parms.rank = opts->dset_rank; - parms.h5_align = opts->h5_alignment; - parms.h5_thresh = opts->h5_threshold; - parms.h5_use_chunks = opts->h5_use_chunks; - parms.h5_extendable = opts->h5_extendable; - parms.h5_write_only = opts->h5_write_only; - parms.verify = opts->verify; - parms.vfd = opts->vfd; - parms.page_buffer_size = opts->page_buffer_size; - parms.page_size = opts->page_size; - - /* load multidimensional options */ - parms.num_bytes = 1; - buf_bytes = 1; - for (i = 0; i < parms.rank; i++) { - parms.buf_size[i] = opts->buf_size[i]; - parms.dset_size[i] = opts->dset_size[i]; - parms.chk_size[i] = opts->chk_size[i]; - parms.order[i] = opts->order[i]; - parms.num_bytes *= opts->dset_size[i]; - buf_bytes *= opts->buf_size[i]; - } - - /* print size information */ - output_report("Transfer Buffer Size (bytes): %d\n", buf_bytes); - output_report("File Size(MB): %.2f\n", ((double)parms.num_bytes) / ONE_MB); - - print_indent(0); - if (opts->io_types & SIO_POSIX) - run_test(POSIXIO, parms, opts); - - print_indent(0); - if (opts->io_types & SIO_HDF5) - run_test(HDF5, parms, opts); -} - -/* - * Function: run_test - * Purpose: Inner loop call to actually run the I/O test. - * Return: Nothing - * Programmer: Bill Wendling, 18. December 2001 - * Modifications: - */ -static int -run_test(iotype iot, parameters parms, struct options *opts) -{ - results res; - register int i, ret_value = SUCCESS; - off_t raw_size; - minmax * write_sys_mm_table = NULL; - minmax * write_mm_table = NULL; - minmax * write_gross_mm_table = NULL; - minmax * write_raw_mm_table = NULL; - minmax * read_sys_mm_table = NULL; - minmax * read_mm_table = NULL; - minmax * read_gross_mm_table = NULL; - minmax * read_raw_mm_table = NULL; - minmax write_sys_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax write_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax write_gross_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax write_raw_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_sys_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_gross_mm = {0.0F, 0.0F, 0.0F, 0}; - minmax read_raw_mm = {0.0F, 0.0F, 0.0F, 0}; - - raw_size = (off_t)parms.num_bytes; - parms.io_type = iot; - print_indent(2); - output_report("IO API = "); - - switch (iot) { - case POSIXIO: - output_report("POSIX\n"); - break; - case HDF5: - output_report("HDF5\n"); - break; - default: - /* unknown request */ - HDfprintf(stderr, "Unknown IO type request (%d)\n", (int)iot); - HDassert(0 && "Unknown IO tpe"); - break; - } - - /* allocate space for tables minmax and that it is sufficient */ - /* to initialize all elements to zeros by calloc. */ - write_sys_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - write_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - write_gross_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - write_raw_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - - if (!parms.h5_write_only) { - read_sys_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - read_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - read_gross_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - read_raw_mm_table = (minmax *)calloc((size_t)parms.num_iters, sizeof(minmax)); - } - - /* Do IO iteration times, collecting statistics each time */ - for (i = 0; i < parms.num_iters; ++i) { - double t; - - do_sio(parms, &res); - - /* gather all of the "sys write" times */ - t = io_time_get(res.timers, HDF5_MPI_WRITE); - get_minmax(&write_sys_mm, t); - - write_sys_mm_table[i] = write_sys_mm; - - /* gather all of the "write" times */ - t = io_time_get(res.timers, HDF5_FINE_WRITE_FIXED_DIMS); - get_minmax(&write_mm, t); - - write_mm_table[i] = write_mm; - - /* gather all of the "write" times from open to close */ - t = io_time_get(res.timers, HDF5_GROSS_WRITE_FIXED_DIMS); - get_minmax(&write_gross_mm, t); - - write_gross_mm_table[i] = write_gross_mm; - - /* gather all of the raw "write" times */ - t = io_time_get(res.timers, HDF5_RAW_WRITE_FIXED_DIMS); - get_minmax(&write_raw_mm, t); - - write_raw_mm_table[i] = write_raw_mm; - - if (!parms.h5_write_only) { - /* gather all of the "mpi read" times */ - t = io_time_get(res.timers, HDF5_MPI_READ); - get_minmax(&read_sys_mm, t); - - read_sys_mm_table[i] = read_sys_mm; - - /* gather all of the "read" times */ - t = io_time_get(res.timers, HDF5_FINE_READ_FIXED_DIMS); - get_minmax(&read_mm, t); - - read_mm_table[i] = read_mm; - - /* gather all of the "read" times from open to close */ - t = io_time_get(res.timers, HDF5_GROSS_READ_FIXED_DIMS); - get_minmax(&read_gross_mm, t); - - read_gross_mm_table[i] = read_gross_mm; - - /* gather all of the raw "read" times */ - t = io_time_get(res.timers, HDF5_RAW_READ_FIXED_DIMS); - get_minmax(&read_raw_mm, t); - - read_raw_mm_table[i] = read_gross_mm; - } - io_time_destroy(res.timers); - } - - /* - * Show various statistics - */ - /* Write statistics */ - /* Print the raw data throughput if desired */ - if (opts->print_raw) { - /* accumulate and output the max, min, and average "raw write" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Raw Data Write details:\n"); - output_all_info(write_raw_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Raw Data Write", write_raw_mm_table, parms.num_iters, raw_size); - } /* end if */ - - /* show sys write statics */ -#if 0 - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("MPI Write details:\n"); - output_all_info(write_sys_mm_table, parms.num_iters, 4); - } -#endif - /* We don't currently output the MPI write results */ - - /* accumulate and output the max, min, and average "write" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write details:\n"); - output_all_info(write_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Write", write_mm_table, parms.num_iters, raw_size); - - /* accumulate and output the max, min, and average "gross write" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Write Open-Close details:\n"); - output_all_info(write_gross_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Write Open-Close", write_gross_mm_table, parms.num_iters, raw_size); - - if (!parms.h5_write_only) { - /* Read statistics */ - /* Print the raw data throughput if desired */ - if (opts->print_raw) { - /* accumulate and output the max, min, and average "raw read" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Raw Data Read details:\n"); - output_all_info(read_raw_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Raw Data Read", read_raw_mm_table, parms.num_iters, raw_size); - } /* end if */ - - /* show mpi read statics */ -#if 0 - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("MPI Read details:\n"); - output_all_info(read_sys_mm_table, parms.num_iters, 4); - } -#endif - /* We don't currently output the MPI read results */ - - /* accumulate and output the max, min, and average "read" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read details:\n"); - output_all_info(read_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Read", read_mm_table, parms.num_iters, raw_size); - - /* accumulate and output the max, min, and average "gross read" times */ - if (sio_debug_level >= 3) { - /* output all of the times for all iterations */ - print_indent(3); - output_report("Read Open-Close details:\n"); - output_all_info(read_gross_mm_table, parms.num_iters, 4); - } - - output_results(opts, "Read Open-Close", read_gross_mm_table, parms.num_iters, raw_size); - } - - /* clean up our mess */ - HDfree(write_sys_mm_table); - HDfree(write_mm_table); - HDfree(write_gross_mm_table); - HDfree(write_raw_mm_table); - - if (!parms.h5_write_only) { - HDfree(read_sys_mm_table); - HDfree(read_mm_table); - HDfree(read_gross_mm_table); - HDfree(read_raw_mm_table); - } - - return ret_value; -} - -/* - * Function: output_all_info - * Purpose: - * Return: Nothing - * Programmer: Bill Wendling, 29. January 2002 - * Modifications: - */ -static void -output_all_info(minmax *mm, int count, int indent_level) -{ - int i; - - for (i = 0; i < count; ++i) { - print_indent(indent_level); - output_report("Iteration %d:\n", i + 1); - print_indent(indent_level + 1); - output_report("Minimum Time: %.2fs\n", mm[i].min); - print_indent(indent_level + 1); - output_report("Maximum Time: %.2fs\n", mm[i].max); - } -} - -/* - * Function: get_minmax - * Purpose: Gather all the min, max and total of val. - * Return: Nothing - * Programmer: Bill Wendling, 21. December 2001 - * Modifications: - * Use MPI_Allreduce to do it. -akc, 2002/01/11 - */ - -static void -get_minmax(minmax *mm, double val) -{ - mm->max = val; - mm->min = val; - mm->sum = val; -} - -/* - * Function: accumulate_minmax_stuff - * Purpose: Accumulate the minimum, maximum, and average of the times - * across all processes. - * Return: TOTAL_MM - the total of all of these. - * Programmer: Bill Wendling, 21. December 2001 - * Modifications: - * Changed to use seconds instead of MB/s - QAK, 5/9/02 - */ -static void -accumulate_minmax_stuff(const minmax *mm, int count, minmax *total_mm) -{ - int i; - - total_mm->sum = 0.0F; - total_mm->max = -DBL_MAX; - total_mm->min = DBL_MAX; - total_mm->num = count; - - for (i = 0; i < count; ++i) { - double m = mm[i].max; - - total_mm->sum += m; - - if (m < total_mm->min) - total_mm->min = m; - - if (m > total_mm->max) - total_mm->max = m; - } -} - -/* - * Function: output_results - * Purpose: Print information about the time & bandwidth for a given - * minmax & # of iterations. - * Return: Nothing - * Programmer: Quincey Koziol, 9. May 2002 - * Modifications: - */ -static void -output_results(const struct options *opts, const char *name, minmax *table, int table_size, off_t data_size) -{ - minmax total_mm; - - accumulate_minmax_stuff(table, table_size, &total_mm); - - print_indent(3); - output_report("%s (%d iteration(s)):\n", name, table_size); - - /* Note: The maximum throughput uses the minimum amount of time & vice versa */ - - print_indent(4); - output_report("Maximum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.min)); - if (opts->print_times) - output_report(" (%7.3f s)\n", total_mm.min); - else - output_report("\n"); - - print_indent(4); - output_report("Average Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.sum / total_mm.num)); - if (opts->print_times) - output_report(" (%7.3f s)\n", (total_mm.sum / total_mm.num)); - else - output_report("\n"); - - print_indent(4); - output_report("Minimum Throughput: %6.2f MB/s", MB_PER_SEC(data_size, total_mm.max)); - if (opts->print_times) - output_report(" (%7.3f s)\n", total_mm.max); - else - output_report("\n"); -} - -/* - * Function: output_report - * Purpose: Print a line of the report. Only do so if I'm the 0 process. - * Return: Nothing - * Programmer: Bill Wendling, 19. December 2001 - * Modifications: - */ -static void -output_report(const char *fmt, ...) -{ - va_list ap; - - HDva_start(ap, fmt); - HDvfprintf(output, fmt, ap); - HDva_end(ap); -} - -/* - * Function: print_indent - * Purpose: Print spaces to indent a new line of text for pretty printing - * things. - * Return: Nothing - * Programmer: Bill Wendling, 29. October 2001 - * Modifications: - */ -static void -print_indent(register int indent) -{ - indent *= TAB_SPACE; - - for (; indent > 0; --indent) - HDfputc(' ', output); -} - -static void -recover_size_and_print(long long val, const char *end) -{ - if (val >= ONE_KB && (val % ONE_KB) == 0) { - if (val >= ONE_MB && (val % ONE_MB) == 0) { - if (val >= ONE_GB && (val % ONE_GB) == 0) - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "GB%s", - val / ONE_GB, end); - else - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "MB%s", - val / ONE_MB, end); - } - else { - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "KB%s", - val / ONE_KB, end); - } - } - else { - HDfprintf(output, - "%" H5_PRINTF_LL_WIDTH "d" - "%s", - val, end); - } -} - -static void -print_io_api(long io_types) -{ - if (io_types & SIO_POSIX) - HDfprintf(output, "posix "); - if (io_types & SIO_HDF5) - HDfprintf(output, "hdf5 "); - HDfprintf(output, "\n"); -} - -static void -report_parameters(struct options *opts) -{ - int i, rank; - rank = opts->dset_rank; - - print_version("HDF5 Library"); /* print library version */ - HDfprintf(output, "==== Parameters ====\n"); - - HDfprintf(output, "IO API="); - print_io_api(opts->io_types); - - HDfprintf(output, "Number of iterations=%d\n", opts->num_iters); - - HDfprintf(output, "Dataset size="); - - for (i = 0; i < rank; i++) - recover_size_and_print((long long)opts->dset_size[i], " "); - HDfprintf(output, "\n"); - - HDfprintf(output, "Transfer buffer size="); - for (i = 0; i < rank; i++) - recover_size_and_print((long long)opts->buf_size[i], " "); - HDfprintf(output, "\n"); - - if (opts->page_size) { - HDfprintf(output, "Page Aggregation Enabled. Page size = %zu\n", opts->page_size); - if (opts->page_buffer_size) - HDfprintf(output, "Page Buffering Enabled. Page Buffer size = %zu\n", opts->page_buffer_size); - else - HDfprintf(output, "Page Buffering Disabled\n"); - } - else - HDfprintf(output, "Page Aggregation Disabled\n"); - - HDfprintf(output, "Dimension access order="); - for (i = 0; i < rank; i++) - recover_size_and_print((long long)opts->order[i], " "); - HDfprintf(output, "\n"); - - if (opts->io_types & SIO_HDF5) { - - HDfprintf(output, "HDF5 data storage method="); - - if (opts->h5_use_chunks) { - - HDfprintf(output, "Chunked\n"); - HDfprintf(output, "HDF5 chunk size="); - for (i = 0; i < rank; i++) - recover_size_and_print((long long)opts->chk_size[i], " "); - HDfprintf(output, "\n"); - - HDfprintf(output, "HDF5 dataset dimensions="); - if (opts->h5_extendable) { - HDfprintf(output, "Extendable\n"); - } - else { - HDfprintf(output, "Fixed\n"); - } - } - else { - HDfprintf(output, "Contiguous\n"); - } - - HDfprintf(output, "HDF5 file driver="); - if (opts->vfd == sec2) { - HDfprintf(output, "sec2\n"); - } - else if (opts->vfd == stdio) { - HDfprintf(output, "stdio\n"); - } - else if (opts->vfd == core) { - HDfprintf(output, "core\n"); - } - else if (opts->vfd == split) { - HDfprintf(output, "split\n"); - } - else if (opts->vfd == multi) { - HDfprintf(output, "multi\n"); - } - else if (opts->vfd == family) { - HDfprintf(output, "family\n"); - } - else if (opts->vfd == direct) { - HDfprintf(output, "direct\n"); - } - } - - { - char *prefix = HDgetenv("HDF5_PREFIX"); - - HDfprintf(output, "Env HDF5_PREFIX=%s\n", (prefix ? prefix : "not set")); - } - - HDfprintf(output, "==== End of Parameters ====\n"); - HDfprintf(output, "\n"); -} - -/* - * Function: parse_command_line - * Purpose: Parse the command line options and return a STRUCT OPTIONS - * structure which will need to be freed by the calling function. - * Return: Pointer to an OPTIONS structure - * Programmer: Bill Wendling, 31. October 2001 - * Modifications: - * Added multidimensional testing (Christian Chilan, April, 2008) - */ -static struct options * -parse_command_line(int argc, const char *argv[]) -{ - int opt; - struct options *cl_opts; - int i, default_rank, actual_rank, ranks[4]; - - cl_opts = (struct options *)HDmalloc(sizeof(struct options)); - - cl_opts->page_buffer_size = 0; - cl_opts->page_size = 0; - - cl_opts->output_file = NULL; - cl_opts->io_types = 0; /* will set default after parsing options */ - cl_opts->num_iters = 1; - - default_rank = 2; - - cl_opts->dset_rank = 0; - cl_opts->buf_rank = 0; - cl_opts->chk_rank = 0; - cl_opts->order_rank = 0; - - for (i = 0; i < MAX_DIMS; i++) { - cl_opts->buf_size[i] = (size_t)((i + 1) * 10); - cl_opts->dset_size[i] = (hsize_t)((i + 1) * 100); - cl_opts->chk_size[i] = (size_t)((i + 1) * 10); - cl_opts->order[i] = i + 1; - } - - cl_opts->vfd = sec2; - - cl_opts->print_times = FALSE; /* Printing times is off by default */ - cl_opts->print_raw = FALSE; /* Printing raw data throughput is off by default */ - cl_opts->h5_alignment = 1; /* No alignment for HDF5 objects by default */ - cl_opts->h5_threshold = 1; /* No threshold for aligning HDF5 objects by default */ - cl_opts->h5_use_chunks = FALSE; /* Don't chunk the HDF5 dataset by default */ - cl_opts->h5_write_only = FALSE; /* Do both read and write by default */ - cl_opts->h5_extendable = FALSE; /* Use extendable dataset */ - cl_opts->verify = FALSE; /* No Verify data correctness by default */ - - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { - switch ((char)opt) { - case 'a': - cl_opts->h5_alignment = parse_size_directive(opt_arg); - break; - case 'G': - cl_opts->page_size = parse_size_directive(opt_arg); - break; - case 'b': - cl_opts->page_buffer_size = parse_size_directive(opt_arg); - break; - case 'A': { - const char *end = opt_arg; - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - if (!HDstrcasecmp(buf, "hdf5")) { - cl_opts->io_types |= SIO_HDF5; - } - else if (!HDstrcasecmp(buf, "posix")) { - cl_opts->io_types |= SIO_POSIX; - } - else { - HDfprintf(stderr, "sio_perf: invalid --api option %s\n", buf); - HDexit(EXIT_FAILURE); - } - - if (*end == '\0') - break; - - end++; - } - } - - break; -#if 0 - case 'b': - /* the future "binary" option */ - break; -#endif /* 0 */ - case 'c': - /* Turn on chunked HDF5 dataset creation */ - cl_opts->h5_use_chunks = 1; - { - const char *end = opt_arg; - int j = 0; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - cl_opts->chk_size[j] = parse_size_directive(buf); - - j++; - - if (*end == '\0') - break; - - end++; - } - cl_opts->chk_rank = j; - } - - break; - - case 'D': { - const char *end = opt_arg; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - if (HDstrlen(buf) > 1 || HDisdigit(buf[0])) { - size_t j; - - for (j = 0; j < 10 && buf[j] != '\0'; ++j) - if (!HDisdigit(buf[j])) { - HDfprintf(stderr, "sio_perf: invalid --debug option %s\n", buf); - HDexit(EXIT_FAILURE); - } - - sio_debug_level = atoi(buf); - - if (sio_debug_level > 4) - sio_debug_level = 4; - else if (sio_debug_level < 0) - sio_debug_level = 0; - } - else { - switch (*buf) { - case 'r': - /* Turn on raw data throughput info */ - cl_opts->print_raw = TRUE; - break; - case 't': - /* Turn on time printing */ - cl_opts->print_times = TRUE; - break; - case 'v': - /* Turn on verify data correctness*/ - cl_opts->verify = TRUE; - break; - default: - HDfprintf(stderr, "sio_perf: invalid --debug option %s\n", buf); - HDexit(EXIT_FAILURE); - } - } - - if (*end == '\0') - break; - - end++; - } - } - - break; - case 'e': { - const char *end = opt_arg; - int j = 0; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - cl_opts->dset_size[j] = parse_size_directive(buf); - - j++; - - if (*end == '\0') - break; - - end++; - } - cl_opts->dset_rank = j; - } - - break; - - case 'i': - cl_opts->num_iters = HDatoi(opt_arg); - break; - case 'o': - cl_opts->output_file = opt_arg; - break; - case 'T': - cl_opts->h5_threshold = parse_size_directive(opt_arg); - break; - case 'v': - if (!HDstrcasecmp(opt_arg, "sec2")) { - cl_opts->vfd = sec2; - } - else if (!HDstrcasecmp(opt_arg, "stdio")) { - cl_opts->vfd = stdio; - } - else if (!HDstrcasecmp(opt_arg, "core")) { - cl_opts->vfd = core; - } - else if (!HDstrcasecmp(opt_arg, "split")) { - cl_opts->vfd = split; - } - else if (!HDstrcasecmp(opt_arg, "multi")) { - cl_opts->vfd = multi; - } - else if (!HDstrcasecmp(opt_arg, "family")) { - cl_opts->vfd = family; - } - else if (!HDstrcasecmp(opt_arg, "direct")) { - cl_opts->vfd = direct; - } - else { - HDfprintf(stderr, "sio_perf: invalid --api option %s\n", opt_arg); - HDexit(EXIT_FAILURE); - } - break; - case 'w': - cl_opts->h5_write_only = TRUE; - break; - case 't': - cl_opts->h5_extendable = TRUE; - break; - case 'x': { - const char *end = opt_arg; - int j = 0; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - cl_opts->buf_size[j] = parse_size_directive(buf); - - j++; - - if (*end == '\0') - break; - - end++; - } - cl_opts->buf_rank = j; - } - - break; - - case 'r': { - const char *end = opt_arg; - int j = 0; - - while (end && *end != '\0') { - char buf[10]; - - HDmemset(buf, '\0', sizeof(buf)); - - for (i = 0; *end != '\0' && *end != ','; ++end) - if (HDisalnum(*end) && i < 10) - buf[i++] = *end; - - cl_opts->order[j] = (int)parse_size_directive(buf); - - j++; - - if (*end == '\0') - break; - - end++; - } - - cl_opts->order_rank = j; - } - - break; - - case 'h': - case '?': - default: - usage(progname); - HDfree(cl_opts); - return NULL; - } - } - - /* perform rank consistency analysis */ - actual_rank = 0; - - ranks[0] = cl_opts->dset_rank; - ranks[1] = cl_opts->buf_rank; - ranks[2] = cl_opts->order_rank; - ranks[3] = cl_opts->chk_rank; - - for (i = 0; i < 4; i++) { - if (ranks[i] > 0) { - if (!actual_rank) { - actual_rank = ranks[i]; - } - else { - if (actual_rank != ranks[i]) - exit(EXIT_FAILURE); - } - } - } - - if (!actual_rank) - actual_rank = default_rank; - - cl_opts->dset_rank = actual_rank; - cl_opts->buf_rank = actual_rank; - cl_opts->order_rank = actual_rank; - cl_opts->chk_rank = actual_rank; - - for (i = 0; i < actual_rank; i++) { - if (cl_opts->order[i] > actual_rank) { - exit(EXIT_FAILURE); - } - } - - /* set default if none specified yet */ - if (!cl_opts->io_types) - cl_opts->io_types = SIO_HDF5 | SIO_POSIX; /* run all API */ - - /* verify parameters sanity. Adjust if needed. */ - /* cap xfer_size with bytes per process */ - if (cl_opts->num_iters <= 0) - cl_opts->num_iters = 1; - - return cl_opts; -} - -/* - * Function: parse_size_directive - * Purpose: Parse the size directive passed on the commandline. The size - * directive is an integer followed by a size indicator: - * - * K, k - Kilobyte - * M, m - Megabyte - * G, g - Gigabyte - * - * Return: The size as a off_t because this is related to file size. - * If an unknown size indicator is used, then the program will - * exit with EXIT_FAILURE as the return value. - * Programmer: Bill Wendling, 18. December 2001 - * Modifications: - */ - -static hsize_t -parse_size_directive(const char *size) -{ - hsize_t s; - char * endptr; - - s = HDstrtoull(size, &endptr, 10); - - if (endptr && *endptr) { - while (*endptr != '\0' && (*endptr == ' ' || *endptr == '\t')) - ++endptr; - - switch (*endptr) { - case 'K': - case 'k': - s *= ONE_KB; - break; - - case 'M': - case 'm': - s *= ONE_MB; - break; - - case 'G': - case 'g': - s *= ONE_GB; - break; - - default: - HDfprintf(stderr, "Illegal size specifier '%c'\n", *endptr); - HDexit(EXIT_FAILURE); - } - } - - return s; -} - -/* - * Function: usage - * Purpose: Print a usage message and then exit. - * Return: Nothing - * Programmer: Bill Wendling, 31. October 2001 - * Modifications: - */ -static void -usage(const char *prog) -{ - print_version(prog); - HDprintf("usage: %s [OPTIONS]\n", prog); - HDprintf(" OPTIONS\n"); - HDprintf(" -h Print an usage message and exit\n"); - HDprintf(" -A AL Which APIs to test\n"); - HDprintf(" [default: all of them]\n"); - HDprintf(" -c SL Selects chunked storage and defines chunks dimensions\n"); - HDprintf(" and sizes\n"); - HDprintf(" [default: Off]\n"); - HDprintf(" -e SL Dimensions and sizes of dataset\n"); - HDprintf(" [default: 100,200]\n"); - HDprintf(" -i N Number of iterations to perform\n"); - HDprintf(" [default: 1]\n"); - HDprintf(" -r NL Dimension access order (see below for description)\n"); - HDprintf(" [default: 1,2]\n"); - HDprintf(" -t Selects extendable dimensions for HDF5 dataset\n"); - HDprintf(" [default: Off]\n"); - HDprintf(" -v VFD Selects file driver for HDF5 access\n"); - HDprintf(" [default: sec2]\n"); - HDprintf(" -w Perform write tests, not the read tests\n"); - HDprintf(" [default: Off]\n"); - HDprintf(" -x SL Dimensions and sizes of the transfer buffer\n"); - HDprintf(" [default: 10,20]\n"); - HDprintf("\n"); - HDprintf(" N - is an integer > 0.\n"); - HDprintf("\n"); - HDprintf(" S - is a size specifier, an integer > 0 followed by a size indicator:\n"); - HDprintf(" K - Kilobyte (%d)\n", ONE_KB); - HDprintf(" M - Megabyte (%d)\n", ONE_MB); - HDprintf(" G - Gigabyte (%d)\n", ONE_GB); - HDprintf("\n"); - HDprintf(" Example: '37M' is 37 megabytes or %d bytes\n", 37 * ONE_MB); - HDprintf("\n"); - HDprintf(" AL - is an API list. Valid values are:\n"); - HDprintf(" hdf5 - HDF5\n"); - HDprintf(" posix - POSIX\n"); - HDprintf("\n"); - HDprintf(" Example: -A posix,hdf5\n"); - HDprintf("\n"); - HDprintf(" NL - is list of integers (N) separated by commas.\n"); - HDprintf("\n"); - HDprintf(" Example: 1,2,3\n"); - HDprintf("\n"); - HDprintf(" SL - is list of size specifiers (S) separated by commas.\n"); - HDprintf("\n"); - HDprintf(" Example: 2K,2K,3K\n"); - HDprintf("\n"); - HDprintf(" The example defines an object (dataset, tranfer buffer) with three\n"); - HDprintf(" dimensions. Be aware that as the number of dimensions increases, the\n"); - HDprintf(" the total size of the object increases exponentially.\n"); - HDprintf("\n"); - HDprintf(" VFD - is an HDF5 file driver specifier. Valid values are:\n"); - HDprintf(" sec2, stdio, core, split, multi, family, direct\n"); - HDprintf("\n"); - HDprintf(" Dimension access order:\n"); - HDprintf(" Data access starts at the cardinal origin of the dataset using the\n"); - HDprintf(" transfer buffer. The next access occurs on a dataset region next to\n"); - HDprintf(" the previous one. For a multidimensional dataset, there are several\n"); - HDprintf(" directions as to where to proceed. This can be specified in the dimension\n"); - HDprintf(" access order. For example, -r 1,2 states that the tool should traverse\n"); - HDprintf(" dimension 1 first, and then dimension 2.\n"); - HDprintf("\n"); - HDprintf(" Environment variables:\n"); - HDprintf(" HDF5_NOCLEANUP Do not remove data files if set [default remove]\n"); - HDprintf(" HDF5_PREFIX Data file prefix\n"); - HDprintf("\n"); - HDfflush(stdout); -} /* end usage() */ diff --git a/tools/test/perform/sio_perf.h b/tools/test/perform/sio_perf.h deleted file mode 100644 index 6242782..0000000 --- a/tools/test/perform/sio_perf.h +++ /dev/null @@ -1,104 +0,0 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * - * Copyright by The HDF Group. * - * All rights reserved. * - * * - * This file is part of HDF5. The full HDF5 copyright notice, including * - * terms governing use, modification, and redistribution, is contained in * - * the COPYING file, which can be found at the root of the source code * - * distribution tree, or in https://www.hdfgroup.org/licenses. * - * If you do not have access to either file, you may request a copy from * - * help@hdfgroup.org. * - * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - -#ifndef SIO_PERF_H -#define SIO_PERF_H - -#ifndef STANDALONE -#include "io_timer.h" -#include "h5test.h" -#include "h5tools.h" -#include "h5tools_utils.h" -#else -#include "io_timer.h" -#include "sio_standalone.h" -#endif - -/* setup the dataset no fill option if this is v1.5 or more */ -#if H5_VERS_MAJOR > 1 || H5_VERS_MINOR > 4 -#define H5_HAVE_NOFILL 1 -#endif - -#define MAX_DIMS 32 - -typedef enum iotype_ { - POSIXIO, - HDF5 - /*NUM_TYPES*/ -} iotype; - -typedef enum vfdtype_ { - sec2, - stdio, - core, - split, - multi, - family, - direct - /*NUM_TYPES*/ -} vfdtype; - -typedef struct parameters_ { - iotype io_type; /* The type of IO test to perform */ - vfdtype vfd; - long num_files; /* Number of files to create */ - long num_dsets; /* Number of datasets to create */ - hsize_t num_bytes; /* Number of bytes in each dset */ - int num_iters; /* Number of times to loop doing the IO */ - int rank; /* Rank of dataset */ - hsize_t dset_size[MAX_DIMS]; /* Dataset size */ - size_t buf_size[MAX_DIMS]; /* Buffer size */ - size_t chk_size[MAX_DIMS]; /* Chunk size */ - int order[MAX_DIMS]; /* Buffer size */ - hsize_t h5_align; /* HDF5 object alignment */ - hsize_t h5_thresh; /* HDF5 object alignment threshold */ - int h5_use_chunks; /* Make HDF5 dataset chunked */ - int h5_extendable; /* Make HDF5 dataset chunked */ - int h5_write_only; /* Perform the write tests only */ - int verify; /* Verify data correctness */ - size_t page_size; - size_t page_buffer_size; -} parameters; - -typedef struct results_ { - herr_t ret_code; - io_time_t *timers; -} results; - -#ifndef SUCCESS -#define SUCCESS 0 -#endif /* !SUCCESS */ - -#ifndef FAIL -#define FAIL -1 -#endif /* !FAIL */ - -extern FILE * output; /* output file */ -extern io_time_t *timer_g; /* timer: global for stub functions */ -extern int sio_debug_level; /* The debug level: - * 0 - Off - * 1 - Minimal - * 2 - Some more - * 3 - Maximal - * 4 - Even More Debugging (timer stuff) - */ -#ifdef __cplusplus -extern "C" { -#endif /* __cplusplus */ - -extern void do_sio(parameters param, results *res); - -#ifdef __cplusplus -} -#endif /* __cplusplus */ - -#endif /* SIO_PERF_H */ diff --git a/tools/test/perform/sio_standalone.c b/tools/test/perform/sio_standalone.c index abc0777..7c22b53 100644 --- a/tools/test/perform/sio_standalone.c +++ b/tools/test/perform/sio_standalone.c @@ -22,31 +22,26 @@ /* global variables */ int nCols = 80; -/* ``get_option'' variables */ -int opt_err = 1; /*get_option prints errors if this is on */ -int opt_ind = 1; /*token pointer */ -const char *opt_arg; /*flag argument (or value) */ - int -get_option(int argc, const char **argv, const char *opts, const struct long_options *l_opts) +get_option(int argc, const char **argv, const char *opts, const struct h5_long_options *l_opts) { static int sp = 1; /* character index in current token */ int opt_opt = '?'; /* option character passed back to user */ if (sp == 1) { /* check for more flag-like tokens */ - if (opt_ind >= argc || argv[opt_ind][0] != '-' || argv[opt_ind][1] == '\0') { + if (H5_optind >= argc || argv[H5_optind][0] != '-' || argv[H5_optind][1] == '\0') { return EOF; } - else if (HDstrcmp(argv[opt_ind], "--") == 0) { - opt_ind++; + else if (HDstrcmp(argv[H5_optind], "--") == 0) { + H5_optind++; return EOF; } } - if (sp == 1 && argv[opt_ind][0] == '-' && argv[opt_ind][1] == '-') { + if (sp == 1 && argv[H5_optind][0] == '-' && argv[H5_optind][1] == '-') { /* long command line option */ - const char *arg = &argv[opt_ind][2]; + const char *arg = &argv[H5_optind][2]; int i; for (i = 0; l_opts && l_opts[i].name; i++) { @@ -58,13 +53,13 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti if (l_opts[i].has_arg != no_arg) { if (arg[len] == '=') { - opt_arg = &arg[len + 1]; + H5_optarg = &arg[len + 1]; } - else if (opt_ind < (argc - 1) && argv[opt_ind + 1][0] != '-') { - opt_arg = argv[++opt_ind]; + else if (H5_optind < (argc - 1) && argv[H5_optind + 1][0] != '-') { + H5_optarg = argv[++H5_optind]; } else if (l_opts[i].has_arg == require_arg) { - if (opt_err) + if (H5_opterr) HDfprintf(stderr, "%s: option required for \"--%s\" flag\n", argv[0], arg); opt_opt = '?'; @@ -72,13 +67,13 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti } else { if (arg[len] == '=') { - if (opt_err) + if (H5_opterr) HDfprintf(stderr, "%s: no option required for \"%s\" flag\n", argv[0], arg); opt_opt = '?'; } - opt_arg = NULL; + H5_optarg = NULL; } break; @@ -87,29 +82,29 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti if (l_opts[i].name == NULL) { /* exhausted all of the l_opts we have and still didn't match */ - if (opt_err) + if (H5_opterr) HDfprintf(stderr, "%s: unknown option \"%s\"\n", argv[0], arg); opt_opt = '?'; } - opt_ind++; + H5_optind++; sp = 1; } else { register char *cp; /* pointer into current token */ /* short command line option */ - opt_opt = argv[opt_ind][sp]; + opt_opt = argv[H5_optind][sp]; if (opt_opt == ':' || (cp = strchr(opts, opt_opt)) == 0) { - if (opt_err) + if (H5_opterr) HDfprintf(stderr, "%s: unknown option \"%c\"\n", argv[0], opt_opt); /* if no chars left in this token, move to next token */ - if (argv[opt_ind][++sp] == '\0') { - opt_ind++; + if (argv[H5_optind][++sp] == '\0') { + H5_optind++; sp = 1; } @@ -118,32 +113,32 @@ get_option(int argc, const char **argv, const char *opts, const struct long_opti if (*++cp == ':') { /* if a value is expected, get it */ - if (argv[opt_ind][sp + 1] != '\0') { + if (argv[H5_optind][sp + 1] != '\0') { /* flag value is rest of current token */ - opt_arg = &argv[opt_ind++][sp + 1]; + H5_optarg = &argv[H5_optind++][sp + 1]; } - else if (++opt_ind >= argc) { - if (opt_err) + else if (++H5_optind >= argc) { + if (H5_opterr) HDfprintf(stderr, "%s: value expected for option \"%c\"\n", argv[0], opt_opt); opt_opt = '?'; } else { /* flag value is next token */ - opt_arg = argv[opt_ind++]; + H5_optarg = argv[H5_optind++]; } sp = 1; } else { /* set up to look at next char in token, next time */ - if (argv[opt_ind][++sp] == '\0') { + if (argv[H5_optind][++sp] == '\0') { /* no more in current token, so setup next token */ - opt_ind++; + H5_optind++; sp = 1; } - opt_arg = NULL; + H5_optarg = NULL; } } diff --git a/tools/test/perform/sio_standalone.h b/tools/test/perform/sio_standalone.h index d8b6412..bf925f1 100644 --- a/tools/test/perform/sio_standalone.h +++ b/tools/test/perform/sio_standalone.h @@ -478,37 +478,27 @@ extern char * strdup(const char *s); #define TRUE true #endif -/** From h5test.h **/ - -#ifdef H5_HAVE_PARALLEL -extern MPI_Info h5_io_info_g; /* MPI INFO object for IO */ -#endif - -#ifdef H5_HAVE_PARALLEL -int h5_set_info_object(void); -void h5_dump_info_object(MPI_Info info); -#endif - /** From h5tools_utils.h **/ -extern int opt_err; /* getoption prints errors if this is on */ -extern int opt_ind; /* token pointer */ -extern const char *opt_arg; /* flag argument (or value) */ +H5_DLLVAR int H5_opterr; /* getoption prints errors if this is on */ +H5_DLLVAR int H5_optind; /* token pointer */ +H5_DLLVAR const char *H5_optarg; /* flag argument (or value) */ -enum { +enum h5_arg_level { no_arg = 0, /* doesn't take an argument */ require_arg, /* requires an argument */ optional_arg /* argument is optional */ }; -typedef struct long_options { - const char *name; /* name of the long option */ - int has_arg; /* whether we should look for an arg */ - char shortval; /* the shortname equivalent of long arg - * this gets returned from get_option */ -} long_options; +struct h5_long_options { + const char * name; /* Name of the long option */ + enum h5_arg_level has_arg; /* Whether we should look for an arg */ + char shortval; /* The shortname equivalent of long arg + * this gets returned from get_option + */ +}; -extern int get_option(int argc, const char **argv, const char *opt, const struct long_options *l_opt); +extern int H5_get_option(int argc, const char **argv, const char *opt, const struct h5_long_options *l_opt); extern int nCols; /*max number of columns for outputting */ diff --git a/tools/test/perform/zip_perf.c b/tools/test/perform/zip_perf.c index 517866b..05ad083 100644 --- a/tools/test/perform/zip_perf.c +++ b/tools/test/perform/zip_perf.c @@ -64,66 +64,15 @@ static void error(const char *fmt, ...); static void compress_buffer(Bytef *dest, uLongf *destLen, const Bytef *source, uLong sourceLen); /* commandline options : long and short form */ -static const char * s_opts = "hB:b:c:p:rs:0123456789"; -static struct long_options l_opts[] = {{"help", no_arg, 'h'}, - {"compressability", require_arg, 'c'}, - {"compressabilit", require_arg, 'c'}, - {"compressabili", require_arg, 'c'}, - {"compressabil", require_arg, 'c'}, - {"compressabi", require_arg, 'c'}, - {"compressab", require_arg, 'c'}, - {"compressa", require_arg, 'c'}, - {"compress", require_arg, 'c'}, - {"compres", require_arg, 'c'}, - {"compre", require_arg, 'c'}, - {"compr", require_arg, 'c'}, - {"comp", require_arg, 'c'}, - {"com", require_arg, 'c'}, - {"co", require_arg, 'c'}, - {"file-size", require_arg, 's'}, - {"file-siz", require_arg, 's'}, - {"file-si", require_arg, 's'}, - {"file-s", require_arg, 's'}, - {"file", require_arg, 's'}, - {"fil", require_arg, 's'}, - {"fi", require_arg, 's'}, - {"max-buffer-size", require_arg, 'B'}, - {"max-buffer-siz", require_arg, 'B'}, - {"max-buffer-si", require_arg, 'B'}, - {"max-buffer-s", require_arg, 'B'}, - {"max-buffer", require_arg, 'B'}, - {"max-buffe", require_arg, 'B'}, - {"max-buff", require_arg, 'B'}, - {"max-buf", require_arg, 'B'}, - {"max-bu", require_arg, 'B'}, - {"max-b", require_arg, 'B'}, - {"max", require_arg, 'B'}, - {"min-buffer-size", require_arg, 'b'}, - {"min-buffer-siz", require_arg, 'b'}, - {"min-buffer-si", require_arg, 'b'}, - {"min-buffer-s", require_arg, 'b'}, - {"min-buffer", require_arg, 'b'}, - {"min-buffe", require_arg, 'b'}, - {"min-buff", require_arg, 'b'}, - {"min-buf", require_arg, 'b'}, - {"min-bu", require_arg, 'b'}, - {"min-b", require_arg, 'b'}, - {"min", require_arg, 'b'}, - {"prefix", require_arg, 'p'}, - {"prefi", require_arg, 'p'}, - {"pref", require_arg, 'p'}, - {"pre", require_arg, 'p'}, - {"pr", require_arg, 'p'}, - {"random-test", no_arg, 'r'}, - {"random-tes", no_arg, 'r'}, - {"random-te", no_arg, 'r'}, - {"random-t", no_arg, 'r'}, - {"random", no_arg, 'r'}, - {"rando", no_arg, 'r'}, - {"rand", no_arg, 'r'}, - {"ran", no_arg, 'r'}, - {"ra", no_arg, 'r'}, - {NULL, 0, '\0'}}; +static const char * s_opts = "hB:b:c:p:rs:0123456789"; +static struct h5_long_options l_opts[] = {{"help", no_arg, 'h'}, + {"compressability", require_arg, 'c'}, + {"file-size", require_arg, 's'}, + {"max-buffer-size", require_arg, 'B'}, + {"min-buffer-size", require_arg, 'b'}, + {"prefix", require_arg, 'p'}, + {"random-test", no_arg, 'r'}, + {NULL, 0, '\0'}}; /* * Function: error @@ -551,7 +500,7 @@ main(int argc, const char *argv[]) /* Initialize h5tools lib */ h5tools_init(); - while ((opt = get_option(argc, argv, s_opts, l_opts)) > 0) { + while ((opt = H5_get_option(argc, argv, s_opts, l_opts)) > 0) { switch ((char)opt) { case '0': case '1': @@ -566,13 +515,13 @@ main(int argc, const char *argv[]) compress_level = opt - '0'; break; case 'B': - max_buf_size = parse_size_directive(opt_arg); + max_buf_size = parse_size_directive(H5_optarg); break; case 'b': - min_buf_size = parse_size_directive(opt_arg); + min_buf_size = parse_size_directive(H5_optarg); break; case 'c': - compress_percent = (int)HDstrtol(opt_arg, NULL, 10); + compress_percent = (int)HDstrtol(H5_optarg, NULL, 10); if (compress_percent < 0) compress_percent = 0; @@ -581,13 +530,13 @@ main(int argc, const char *argv[]) break; case 'p': - option_prefix = opt_arg; + option_prefix = H5_optarg; break; case 'r': random_test = TRUE; break; case 's': - file_size = parse_size_directive(opt_arg); + file_size = parse_size_directive(H5_optarg); break; case '?': usage(); -- cgit v0.12