diff options
92 files changed, 5765 insertions, 1438 deletions
@@ -799,8 +799,11 @@ ./src/H5PBpkg.h ./src/H5PBprivate.h ./src/H5PL.c +./src/H5PLint.c ./src/H5PLmodule.h +./src/H5PLpath.c ./src/H5PLpkg.h +./src/H5PLplugin_cache.c ./src/H5PLprivate.h ./src/H5PLpublic.h ./src/H5PLextern.h @@ -1222,6 +1225,7 @@ ./testpar/COPYING ./testpar/Makefile.am +./testpar/t_bigio.c ./testpar/t_cache.c ./testpar/t_cache_image.c ./testpar/t_chunk_alloc.c @@ -2090,6 +2094,8 @@ ./tools/testfiles/tdset2.h5.xml ./tools/testfiles/tempty.h5.xml ./tools/testfiles/tenum.h5.xml +./tools/testfiles/test35.nc +./tools/testfiles/test35.nc.xml ./tools/testfiles/tfpformat.h5.xml ./tools/testfiles/tgroup.h5.xml ./tools/testfiles/thlink.h5.xml @@ -2399,6 +2405,7 @@ ./tools/test/h5diff/testfiles/h5diff_udfail.txt ./tools/test/h5diff/testfiles/diff_strings1.h5 ./tools/test/h5diff/testfiles/diff_strings2.h5 +./tools/test/h5diff/testfiles/h5diff_vlstr.txt #vds ./tools/test/h5diff/testfiles/h5diff_v1.txt ./tools/test/h5diff/testfiles/h5diff_v2.txt @@ -2406,6 +2413,7 @@ #test files for h5repack ./tools/test/h5repack/testfiles/README +./tools/test/h5repack/testfiles/h5repack_aggr.h5 ./tools/test/h5repack/testfiles/h5repack_attr.h5 ./tools/test/h5repack/testfiles/h5repack_attr_refs.h5 ./tools/test/h5repack/testfiles/h5repack_deflate.h5 @@ -2415,6 +2423,8 @@ ./tools/test/h5repack/testfiles/h5repack_fill.h5 ./tools/test/h5repack/testfiles/h5repack_filters.h5 ./tools/test/h5repack/testfiles/h5repack_fletcher.h5 +./tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 +./tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 ./tools/test/h5repack/testfiles/h5repack_hlink.h5 ./tools/test/h5repack/testfiles/h5repack.info ./tools/test/h5repack/testfiles/h5repack_layout.h5 @@ -2426,7 +2436,10 @@ ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum_deflated.h5 ./tools/test/h5repack/testfiles/h5repack_nested_8bit_enum.h5 ./tools/test/h5repack/testfiles/h5repack_nbit.h5 +./tools/test/h5repack/testfiles/h5repack_none.h5 ./tools/test/h5repack/testfiles/h5repack_objs.h5 +./tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 +./tools/test/h5repack/testfiles/h5repack_paged_persist.h5 ./tools/test/h5repack/testfiles/h5repack_refs.h5 ./tools/test/h5repack/testfiles/h5repack_shuffle.h5 ./tools/test/h5repack/testfiles/h5repack_soffset.h5 @@ -2445,6 +2458,12 @@ ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_version_test.ddl ./tools/test/h5repack/testfiles/h5repack_layout.h5-plugin_zero.tst +./tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl +./tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl +./tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl +./tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl +./tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl +./tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl ./tools/test/h5repack/testfiles/1_vds.h5-vds_dset_chunk20x10x5-v.ddl ./tools/test/h5repack/testfiles/2_vds.h5-vds_chunk3x6x9-v.ddl ./tools/test/h5repack/testfiles/3_1_vds.h5-vds_chunk2x5x8-v.ddl @@ -76,7 +76,7 @@ $Source = ""; "off_t" => "o", "H5O_type_t" => "Ot", "H5P_class_t" => "p", - "hobj_ref_t" => "r", + "hobj_ref_t" => "r", "H5R_type_t" => "Rt", "char" => "s", "unsigned char" => "s", @@ -124,7 +124,7 @@ $Source = ""; "H5G_iterate_t" => "x", "H5G_info_t" => "x", "H5I_free_t" => "x", - "H5I_search_func_t" => "x", + "H5I_search_func_t" => "x", "H5L_class_t" => "x", "H5L_elink_traverse_t" => "x", "H5L_iterate_t" => "x", diff --git a/config/cmake_ext_mod/HDFLibMacros.cmake b/config/cmake_ext_mod/HDFLibMacros.cmake index f2e03d7..54e408b 100644 --- a/config/cmake_ext_mod/HDFLibMacros.cmake +++ b/config/cmake_ext_mod/HDFLibMacros.cmake @@ -74,14 +74,14 @@ macro (EXTERNAL_JPEG_LIBRARY compress_type jpeg_pic) HDF_IMPORT_SET_LIB_OPTIONS (jpeg-static "jpeg" STATIC "") add_dependencies (JPEG jpeg-static) set (JPEG_STATIC_LIBRARY "jpeg-static") - set (JPEG_LIBRARIES ${JPEG_static_LIBRARY}) + set (JPEG_LIBRARIES ${JPEG_STATIC_LIBRARY}) if (BUILD_SHARED_LIBS) # Create imported target jpeg-shared add_library(jpeg-shared SHARED IMPORTED) HDF_IMPORT_SET_LIB_OPTIONS (jpeg-shared "jpeg" SHARED "") add_dependencies (JPEG jpeg-shared) set (JPEG_SHARED_LIBRARY "jpeg-shared") - set (JPEG_LIBRARIES ${JPEG_LIBRARIES} ${JPEG_shared_LIBRARY}) + set (JPEG_LIBRARIES ${JPEG_LIBRARIES} ${JPEG_SHARED_LIBRARY}) endif () set (JPEG_INCLUDE_DIR_GEN "${BINARY_DIR}") @@ -167,14 +167,14 @@ macro (EXTERNAL_SZIP_LIBRARY compress_type encoding) HDF_IMPORT_SET_LIB_OPTIONS (szip-static "szip" STATIC "") add_dependencies (SZIP szip-static) set (SZIP_STATIC_LIBRARY "szip-static") - set (SZIP_LIBRARIES ${SZIP_static_LIBRARY}) + set (SZIP_LIBRARIES ${SZIP_STATIC_LIBRARY}) if (BUILD_SHARED_LIBS) # Create imported target szip-shared add_library(szip-shared SHARED IMPORTED) HDF_IMPORT_SET_LIB_OPTIONS (szip-shared "szip" SHARED "") add_dependencies (SZIP szip-shared) set (SZIP_SHARED_LIBRARY "szip-shared") - set (SZIP_LIBRARIES ${SZIP_LIBRARIES} ${SZIP_shared_LIBRARY}) + set (SZIP_LIBRARIES ${SZIP_LIBRARIES} ${SZIP_SHARED_LIBRARY}) endif () set (SZIP_INCLUDE_DIR_GEN "${BINARY_DIR}") @@ -262,7 +262,7 @@ macro (EXTERNAL_ZLIB_LIBRARY compress_type) HDF_IMPORT_SET_LIB_OPTIONS (zlib-static ${ZLIB_LIB_NAME} STATIC "") add_dependencies (ZLIB zlib-static) set (ZLIB_STATIC_LIBRARY "zlib-static") - set (ZLIB_LIBRARIES ${ZLIB_static_LIBRARY}) + set (ZLIB_LIBRARIES ${ZLIB_STATIC_LIBRARY}) if (BUILD_SHARED_LIBS) # Create imported target zlib-shared add_library(zlib-shared SHARED IMPORTED) diff --git a/config/linux-gnulibc1 b/config/linux-gnulibc1 index 1785e0e..39225e7 100644 --- a/config/linux-gnulibc1 +++ b/config/linux-gnulibc1 @@ -84,7 +84,20 @@ else $RM $tmpfile fc_version_info=`$FC -V | grep Absoft` ;; - + # The NAG compiler + nagfor*|nagftn*) + RM='rm -f' + tmpfile=/tmp/cmpver.$$ + $FC -V >& $tmpfile + if test -s "$tmpfile"; then + if( grep -s 'NAG Fortran' $tmpfile > /dev/null) then + FC_BASENAME=nagfor + fi + fi + fc_version_info=`grep "NAG Fortran" $tmpfile` + echo "compiler '$FC' is $fc_version_info" + $RM $tmpfile + ;; *) ;; esac @@ -108,10 +121,10 @@ case $FC_BASENAME in # f95) # Set required flag for compiling C stubs - H5_CFLAGS="$H5_CFLAGS" + H5_CFLAGS="$H5_CFLAGS" F9XSUFFIXFLAG="" -# We force compiler to use upper case for external names +# We force compiler to use upper case for external names # (just in case since this should be a default EIP) H5_FCFLAGS="$H5_FCFLAGS" FSEARCH_DIRS="" @@ -136,6 +149,37 @@ case $FC_BASENAME in f9x_flags_set=yes ;; +# +# NAG compiler +# + nagfor) + + F9XSUFFIXFLAG="" +# We force compiler to use upper case for external names +# (just in case since this should be a default EIP) + H5_FCFLAGS="$H5_FCFLAGS" + FSEARCH_DIRS="" + + # Production + PROD_FCFLAGS= + + # Debug + DEBUG_FCFLAGS="-C" + + # Symbols + SYMBOLS_FCFLAGS="-g" + NO_SYMBOLS_FCFLAGS="-s" + + # Profiling + PROFILE_FCFLAGS="-pg" + + # Optimization + HIGH_OPT_FCFLAGS="-O" + DEBUG_OPT_FCFLAGS="-O0" + NO_OPT_FCFLAGS="-O0" + + f9x_flags_set=yes + ;; esac @@ -230,6 +274,19 @@ case $FC in *pgf90*) fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'pgf90'` ;; + *nagfor*|*nagftn*) + RM='rm -f' + tmpfile=/tmp/cmpver.$$ + $FC -V >& $tmpfile + if test -s "$tmpfile"; then + if( grep -s 'NAG Fortran' $tmpfile > /dev/null) then + FC_BASENAME=nagfor + fi + fi + fc_version_info=`grep "NAG Fortran" $tmpfile` + $RM $tmpfile + echo "compiler '$FC' is $fc_version_info" + ;; *) echo "No match to get fc_version_info for $FC" diff --git a/configure.ac b/configure.ac index 163f6d3..91800e7 100644 --- a/configure.ac +++ b/configure.ac @@ -3338,6 +3338,27 @@ else STATIC_SHARED="none" fi +if test "X$HDF_FORTRAN" = "Xyes"; then + + ### libtool does not pass the correct argument linking (-Wl,-Wl,,) for the NAG Fortran compiler + ### on Linux (other OSs have not been tested). + ### Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to + ### wl="-Wl,-Wl,," in the libtool file. (HDFFV-10037) + case "`uname`" in + Linux*) + + fortran_linux_linker_option="-Wl," + if test "X$FC_BASENAME" = "Xnagfor"; then + fortran_linux_linker_option="-Wl,-Wl,," + fi + + ## Set the correct linker option for use in h5fc.in markup + AC_SUBST([fortran_linux_linker_option]) + ;; + esac + +fi + ## ---------------------------------------------------------------------- ## Set a macro if shared library is enabled. ## @@ -3473,6 +3494,10 @@ AC_CONFIG_COMMANDS([.classes], [], [$MKDIR_P java/src/.classes; AC_OUTPUT chmod 755 tools/src/misc/h5cc +if test "X$HDF_CXX" = "Xyes"; then + chmod 755 c++/src/h5c++ +fi + if test "X$HDF_FORTRAN" = "Xyes"; then chmod 755 fortran/src/h5fc @@ -3484,10 +3509,18 @@ if test "X$HDF_FORTRAN" = "Xyes"; then cat libtool | awk '/wl=\"/{c++;if(c==3){sub("wl=\"\"","wl=\"-Wl,\"");c=0}}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool ;; esac -fi -if test "X$HDF_CXX" = "Xyes"; then - chmod 755 c++/src/h5c++ + ### libtool does not pass the correct argument linking (-Wl,-Wl,,) for the NAG Fortran compiler + ### on Linux (other OSs have not been tested). + ### Therefore, detect if we are using the NAG Fortran compiler, and replace the wl="-Wl," for Fortran to + ### wl="-Wl,-Wl,," in the libtool file. (HDFFV-10037) + case "`uname`" in + Linux*) + if test "X$FC_BASENAME" = "Xnagfor"; then + cat libtool | awk '/NAG_Fortran/{flag=1}flag&&/wl=/{$NF="wl=\"-Wl,Wl,,\"";flag=0}1' > libtool.tmp && mv -f libtool.tmp libtool && chmod 755 libtool + fi + ;; + esac fi ## HDF5 configure code created by autotools with gcc 4.9.2 is adding problematic diff --git a/fortran/examples/Makefile.am b/fortran/examples/Makefile.am index db85c09..6bf2edb 100644 --- a/fortran/examples/Makefile.am +++ b/fortran/examples/Makefile.am @@ -66,10 +66,10 @@ h5_crtatt.chkexe_: h5_rdwt.chkexe_ # Additional dependencies for the examples are listed below if BUILD_PARALLEL_CONDITIONAL $(EXTRA_PROG): $(H5FC_PP) - $(H5FC_PP) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90; + $(H5FC_PP) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90; else $(EXTRA_PROG): $(H5FC) - $(H5FC) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90; + $(H5FC) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90; endif # Tell automake how to install examples diff --git a/fortran/src/h5fc.in b/fortran/src/h5fc.in index 47642c9..29ef83f 100644 --- a/fortran/src/h5fc.in +++ b/fortran/src/h5fc.in @@ -307,7 +307,7 @@ if test "x$do_link" = "xyes"; then link_args="$link_args -L${libdir}" case "$host_os" in - linux*) flag="-Wl,-rpath -Wl," ;; + linux*) flag="@fortran_linux_linker_option@-rpath -Wl," ;; hpux*) flag="-Wl,+b -Wl," ;; freebsd*|solaris*) flag="-R" ;; rs6000*|aix*) flag="-L" ;; diff --git a/hl/fortran/examples/Makefile.am b/hl/fortran/examples/Makefile.am index 6709fb7..d383f9a 100644 --- a/hl/fortran/examples/Makefile.am +++ b/hl/fortran/examples/Makefile.am @@ -43,10 +43,10 @@ FORTRAN_API=yes # Tell automake how to build examples using h5fc if BUILD_PARALLEL_CONDITIONAL $(EXTRA_PROG): $(H5FC_PP) - $(H5FC_PP) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90; + $(H5FC_PP) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90; else $(EXTRA_PROG): $(H5FC) - $(H5FC) $(H5CCFLAGS) $(FCFLAGS) -o $@ $(srcdir)/$@.f90; + $(H5FC) $(H5CCFLAGS) -o $@ $(srcdir)/$@.f90; endif # Tell automake how to install examples diff --git a/hl/tools/h5watch/CMakeTests.cmake b/hl/tools/h5watch/CMakeTests.cmake index 35e7829..0b7b4d4 100644 --- a/hl/tools/h5watch/CMakeTests.cmake +++ b/hl/tools/h5watch/CMakeTests.cmake @@ -56,18 +56,11 @@ set (H5WATCH_TEST_FILES # make test dir file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") -add_custom_target(h5watch-files ALL COMMENT "Copying files needed by h5watch tests") foreach (h5watch_file ${H5WATCH_TEST_FILES}) - set (dest "${PROJECT_BINARY_DIR}/testfiles/${h5watch_file}") - #message (STATUS " Copying ${h5watch_file}") - add_custom_command ( - TARGET h5watch-files - POST_BUILD - COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different ${HDF5_HL_TOOLS_DIR}/testfiles/${h5watch_file} ${dest} - ) + HDFTEST_COPY_FILE("${HDF5_HL_TOOLS_DIR}/testfiles/${h5watch_file}" "${PROJECT_BINARY_DIR}/testfiles/${h5watch_file}" "H5WATCH_files") endforeach () +add_custom_target(H5WATCH_files ALL COMMENT "Copying files needed by H5WATCH tests" DEPENDS ${H5WATCH_files_list}) ############################################################################## ############################################################################## diff --git a/java/src/jni/h5Constants.c b/java/src/jni/h5Constants.c index d4511e1..f6f8bfa 100644 --- a/java/src/jni/h5Constants.c +++ b/java/src/jni/h5Constants.c @@ -26,6 +26,7 @@ extern "C" { #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wmissing-prototypes" +#pragma GCC diagnostic ignored "-Wunused-parameter" JNIEXPORT jlong JNICALL Java_hdf_hdf5lib_HDF5Constants_H5_1QUARTER_1HADDR_1MAX(JNIEnv *env, jclass cls) { return (hsize_t)HADDR_MAX/4; } diff --git a/java/test/TestH5PL.java b/java/test/TestH5PL.java index aa59478..8ce708b 100644 --- a/java/test/TestH5PL.java +++ b/java/test/TestH5PL.java @@ -70,26 +70,74 @@ public class TestH5PL { @Test public void TestH5PLpaths() { try { - int original_entries = H5.H5PLsize(); - H5.H5PLappend("path_one"); - int plugin_entries = H5.H5PLsize(); - assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+1) == plugin_entries); - H5.H5PLprepend("path_two"); - plugin_entries = H5.H5PLsize(); - assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+2) == plugin_entries); - H5.H5PLinsert("path_three", original_entries); - plugin_entries = H5.H5PLsize(); - assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+3) == plugin_entries); - String first_path = H5.H5PLget(original_entries); - assertTrue("First path was : "+first_path + " ",first_path.compareToIgnoreCase("path_three")==0); - H5.H5PLreplace("path_four", original_entries); - first_path = H5.H5PLget(original_entries); - assertTrue("First path changed to : "+first_path + " ",first_path.compareToIgnoreCase("path_four")==0); - H5.H5PLremove(original_entries); - first_path = H5.H5PLget(original_entries); - assertTrue("First path now : "+first_path + " ",first_path.compareToIgnoreCase("path_two")==0); - plugin_entries = H5.H5PLsize(); - assertTrue("H5.H5PLsize: "+plugin_entries, (original_entries+2) == plugin_entries); + // Get the original number of paths + int nStartPaths = H5.H5PLsize(); + + int nPaths; /* # paths from H5PLSize() */ + int nTruePaths = nStartPaths; /* What the # paths should be */ + int index; /* Path table index */ + String path; /* Path from H5PLget() */ + + // APPEND a path and ensure it was added correctly + String pathAppend = "path_append"; + H5.H5PLappend(pathAppend); + + nPaths = H5.H5PLsize(); + nTruePaths++; + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + index = nTruePaths - 1; + path = H5.H5PLget(index); + assertTrue("Path should be " + pathAppend + " but was " + path, path.compareToIgnoreCase(pathAppend) == 0); + + // PREPEND a path and ensure it was added correctly + String pathPrepend = "path_prepend"; + H5.H5PLprepend(pathPrepend); + + nPaths = H5.H5PLsize(); + nTruePaths++; + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + index = 0; + path = H5.H5PLget(index); + assertTrue("Path should be " + pathPrepend + " but was " + path, path.compareToIgnoreCase(pathPrepend) == 0); + + // INSERT a path and ensure it was added correctly + // Inserting at the index == # of start paths ensures we're in the middle + String pathInsert = "path_insert"; + index = nStartPaths; + H5.H5PLinsert(pathInsert, index); + + nPaths = H5.H5PLsize(); + nTruePaths++; + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + path = H5.H5PLget(index); + assertTrue("Path should be " + pathInsert + " but was " + path, path.compareToIgnoreCase(pathInsert) == 0); + + // REPLACE the path we just added and ensure it updated correctly + String pathReplace = "path_replace"; + index = nStartPaths; + H5.H5PLreplace(pathReplace, index); + + nPaths = H5.H5PLsize(); + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + path = H5.H5PLget(index); + assertTrue("Path should be " + pathReplace + " but was " + path, path.compareToIgnoreCase(pathReplace) == 0); + + // REMOVE the path we just replaced and check that the table was compacted + // The (index+1) path should move down to fill the space when the path is removed. + index = nStartPaths; + String pathRemove = H5.H5PLget(index + 1); + H5.H5PLremove(index); + + nPaths = H5.H5PLsize(); + nTruePaths--; + assertTrue("# paths should be " + nTruePaths + " but was " + nPaths, nTruePaths == nPaths); + + path = H5.H5PLget(index); + assertTrue("Path should be " + pathRemove + " but was " + path, path.compareToIgnoreCase(pathRemove) == 0); } catch (Throwable err) { err.printStackTrace(); diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 20d58b3..2dbe7e9 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -62,7 +62,29 @@ New Features Parallel Library: ----------------- - - + - Large MPI-IO transfers + + Previous releases of PHDF5 would fail when attempting to + read or write greater than 2GB of data in a single IO operation. + This issue stems principally from an MPI API whose definitions + utilize 32 bit integers to describe the number of data elements + and datatype that MPI should use to effect a data transfer. + Historically, HDF5 has invoked MPI-IO with the number of + elements in a contiguous buffer represented as the length + of that buffer in bytes. + + Resolving the issue and thus enabling larger MPI-IO transfers + is accomplished first, by detecting when a user IO request would + exceed the 2GB limit as described above. Once a transfer request + is identified as requiring special handling, PHDF5 now creates a + derived datatype consisting of a vector of fixed sized blocks + which is in turn wrapped within a single MPI_Type_struct to + contain the vector and any remaining data. The newly created + datatype is then used in place of MPI_BYTE and can be used to + fulfill the original user request without encountering API + errors. + + (RAW – 2017/07/11, HDFFV-8839) Fortran Library: ---------------- @@ -105,7 +127,15 @@ Bug Fixes since HDF5-1.10.1 release Configuration ------------- - - + - cmake + + Too many commands for POST_BUILD step caused command line to be + too big on windows. + + Changed foreach of copy command to use a custom command with the + use of the HDFTEST_COPY_FILE macro. + + (ADB - 2017/07/12, HDFFV-10254) Performance ------------- @@ -117,6 +147,25 @@ Bug Fixes since HDF5-1.10.1 release Tools ----- + - h5dump + + h5dump segfaulted on output of XML file. + + Function that escape'd strings used the full buffer length + instead of just the length of the replacement string in a + strncpy call. Using the correct length fixed the issue. + + (ADB - 2017/08/01, HDFFV-10256) + + - h5diff + + h5diff segfaulted on compare of a NULL variable length string. + + Improved h5diff compare of strings by adding a check for + NULL strings and setting the lengths to zero. + + (ADB - 2017/07/25, HDFFV-10246) + - h5import h5import crashed trying to import data from a subset of a dataset. @@ -127,7 +176,7 @@ Bug Fixes since HDF5-1.10.1 release The import from h5dump function expects the binary files to use native types (FILE '-b' option) in the binary file. - (ADB - 2017/06/15, HDFFV-102191) + (ADB - 2017/06/15, HDFFV-10219) - h5repack diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 178c954..96ea589 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -516,6 +516,9 @@ IDE_GENERATED_PROPERTIES ("H5PB" "${H5PB_HDRS}" "${H5PB_SOURCES}" ) set (H5PL_SOURCES ${HDF5_SRC_DIR}/H5PL.c + ${HDF5_SRC_DIR}/H5PLint.c + ${HDF5_SRC_DIR}/H5PLpath.c + ${HDF5_SRC_DIR}/H5PLplugin_cache.c ) set (H5PL_HDRS diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index e26581c..af6599a 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -1016,11 +1016,16 @@ H5D__chunk_init(H5F_t *f, hid_t dxpl_id, const H5D_t *dset, hid_t dapl_id) unsigned u; /* Local index value */ for(u = 0; u < dset->shared->ndims; u++) { + hsize_t scaled_power2up; /* Scaled value, rounded to next power of 2 */ + /* Initial scaled dimension sizes */ rdcc->scaled_dims[u] = dset->shared->curr_dims[u] / dset->shared->layout.u.chunk.dim[u]; + if( !(scaled_power2up = H5VM_power2up(rdcc->scaled_dims[u])) ) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") + /* Inital 'power2up' values for scaled dimensions */ - rdcc->scaled_power2up[u] = H5VM_power2up(rdcc->scaled_dims[u]); + rdcc->scaled_power2up[u] = scaled_power2up; /* Number of bits required to encode scaled dimension size */ rdcc->scaled_encode_bits[u] = H5VM_log2_gen(rdcc->scaled_power2up[u]); diff --git a/src/H5Ddeprec.c b/src/H5Ddeprec.c index 8d9461c..0807048 100644 --- a/src/H5Ddeprec.c +++ b/src/H5Ddeprec.c @@ -329,8 +329,11 @@ H5D__extend(H5D_t *dataset, const hsize_t *size, hid_t dxpl_id) dataset->shared->cache.chunk.scaled_dims[u] > dataset->shared->cache.chunk.nslots)) update_chunks = TRUE; + if( !(scaled_power2up = H5VM_power2up(scaled)) ) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") + /* Check if the number of bits required to encode the scaled size value changed */ - if(dataset->shared->cache.chunk.scaled_power2up[u] != (scaled_power2up = H5VM_power2up(scaled))) { + if(dataset->shared->cache.chunk.scaled_power2up[u] != scaled_power2up) { /* Update the 'power2up' & 'encode_bits' values for the current dimension */ dataset->shared->cache.chunk.scaled_power2up[u] = scaled_power2up; dataset->shared->cache.chunk.scaled_encode_bits[u] = H5VM_log2_gen(scaled_power2up); diff --git a/src/H5Dint.c b/src/H5Dint.c index 0928afe..bdedd1e 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -745,8 +745,13 @@ H5D__cache_dataspace_info(const H5D_t *dset) dset->shared->ndims = (unsigned)sndims; /* Compute the inital 'power2up' values */ - for(u = 0; u < dset->shared->ndims; u++) - dset->shared->curr_power2up[u] = H5VM_power2up(dset->shared->curr_dims[u]); + for(u = 0; u < dset->shared->ndims; u++) { + hsize_t scaled_power2up; /* Scaled value, rounded to next power of 2 */ + + if( !(scaled_power2up = H5VM_power2up(dset->shared->curr_dims[u])) ) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") + dset->shared->curr_power2up[u] = scaled_power2up; + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -2805,8 +2810,11 @@ H5D__set_extent(H5D_t *dset, const hsize_t *size, hid_t dxpl_id) dset->shared->cache.chunk.scaled_dims[u] > dset->shared->cache.chunk.nslots)) update_chunks = TRUE; + if( !(scaled_power2up = H5VM_power2up(scaled)) ) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get the next power of 2") + /* Check if the number of bits required to encode the scaled size value changed */ - if(dset->shared->cache.chunk.scaled_power2up[u] != (scaled_power2up = H5VM_power2up(scaled))) { + if(dset->shared->cache.chunk.scaled_power2up[u] != scaled_power2up) { /* Update the 'power2up' & 'encode_bits' values for the current dimension */ dset->shared->cache.chunk.scaled_power2up[u] = scaled_power2up; dset->shared->cache.chunk.scaled_encode_bits[u] = H5VM_log2_gen(scaled_power2up); diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 6f68a62..eba48de 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -549,9 +549,11 @@ #define H5F_FILE_SPACE_PAGE_SIZE_DEF 4096 /* For paged aggregation: minimum value for file space page size */ #define H5F_FILE_SPACE_PAGE_SIZE_MIN 512 +/* For paged aggregation: maxiumum value for file space page size: 1 gigabyte */ +#define H5F_FILE_SPACE_PAGE_SIZE_MAX 1024*1024*1024 /* For paged aggregation: drop free-space with size <= this threshold for small meta section */ -#define H5F_FILE_SPACE_PGEND_META_THRES 10 +#define H5F_FILE_SPACE_PGEND_META_THRES 0 /* Default for threshold for alignment (can be set via H5Pset_alignment()) */ #define H5F_ALIGN_DEF 1 @@ -1519,7 +1519,7 @@ H5MF_try_shrink(H5F_t *f, H5FD_mem_t alloc_type, hid_t dxpl_id, haddr_t addr, H5AC_ring_t fsm_ring = H5AC_RING_INV; /* Ring of fsm */ H5F_mem_page_t fs_type; /* Free space type */ hbool_t reset_ring = FALSE; /* Whether the ring was set */ - htri_t ret_value = FAIL; /* Return value */ + htri_t ret_value = FALSE; /* Return value */ FUNC_ENTER_NOAPI_TAG(dxpl_id, H5AC__FREESPACE_TAG, FAIL) #ifdef H5MF_ALLOC_DEBUG diff --git a/src/H5MFsection.c b/src/H5MFsection.c index 02fc2d9..02e3218 100644 --- a/src/H5MFsection.c +++ b/src/H5MFsection.c @@ -130,27 +130,27 @@ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SIMPLE[1] = {{ /* Class info for "small" free space sections */ H5FS_section_class_t H5MF_FSPACE_SECT_CLS_SMALL[1] = {{ /* Class variables */ - H5MF_FSPACE_SECT_SMALL, /* Section type */ - 0, /* Extra serialized size */ - H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ - NULL, /* Class private info */ + H5MF_FSPACE_SECT_SMALL, /* Section type */ + 0, /* Extra serialized size */ + H5FS_CLS_MERGE_SYM | H5FS_CLS_ADJUST_OK, /* Class flags */ + NULL, /* Class private info */ /* Class methods */ - NULL, /* Initialize section class */ - NULL, /* Terminate section class */ + NULL, /* Initialize section class */ + NULL, /* Terminate section class */ /* Object methods */ - H5MF_sect_small_add, /* Add section */ - NULL, /* Serialize section */ + H5MF_sect_small_add, /* Add section */ + NULL, /* Serialize section */ H5MF_sect_deserialize, /* Deserialize section */ - H5MF_sect_small_can_merge, /* Can sections merge? */ - H5MF_sect_small_merge, /* Merge sections */ - H5MF_sect_small_can_shrink, /* Can section shrink container?*/ - H5MF_sect_small_shrink, /* Shrink container w/section */ - H5MF_sect_free, /* Free section */ - H5MF_sect_valid, /* Check validity of section */ - H5MF_sect_split, /* Split section node for alignment */ - NULL, /* Dump debugging for section */ + H5MF_sect_small_can_merge, /* Can sections merge? */ + H5MF_sect_small_merge, /* Merge sections */ + NULL, /* Can section shrink container?*/ + NULL, /* Shrink container w/section */ + H5MF_sect_free, /* Free section */ + H5MF_sect_valid, /* Check validity of section */ + H5MF_sect_split, /* Split section node for alignment */ + NULL, /* Dump debugging for section */ }}; /* Class info for "large" free space sections */ @@ -674,7 +674,7 @@ HDfprintf(stderr, "%s: Entering, section {%a, %Hu}\n", FUNC, (*sect)->sect_info. HDfprintf(stderr, "%s: section is dropped\n", FUNC); #endif /* H5MF_ALLOC_DEBUG_MORE */ } /* end if */ - /* Adjust the section if it is not at page end but its size + pgend threshold is at page end */ + /* Adjust the section if it is not at page end but its size + prem is at page end */ else if(prem <= H5F_PGEND_META_THRES(udata->f)) { (*sect)->sect_info.size += prem; @@ -22,135 +22,28 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5Eprivate.h" /* Error handling */ -#include "H5MMprivate.h" /* Memory management */ #include "H5PLpkg.h" /* Plugin */ -#include "H5Zprivate.h" /* Filter pipeline */ /****************/ /* Local Macros */ /****************/ -#ifdef H5_HAVE_WIN32_API -#define H5PL_EXPAND_ENV_VAR { \ - long bufCharCount; \ - char *tempbuf; \ - if(NULL == (tempbuf = (char *)H5MM_malloc(H5PL_EXPAND_BUFFER_SIZE))) \ - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path") \ - if((bufCharCount = ExpandEnvironmentStringsA(dl_path, tempbuf, H5PL_EXPAND_BUFFER_SIZE)) > H5PL_EXPAND_BUFFER_SIZE) { \ - tempbuf = (char *)H5MM_xfree(tempbuf); \ - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long") \ - } \ - if(bufCharCount == 0) { \ - tempbuf = (char *)H5MM_xfree(tempbuf); \ - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path") \ - } \ - dl_path = (char *)H5MM_xfree(dl_path); \ - dl_path = tempbuf; \ - } -#else -#define H5PL_EXPAND_ENV_VAR -#endif /* H5_HAVE_WIN32_API */ - -/****************************/ -/* Macros for supporting - * both Windows and Unix */ -/****************************/ -/* Windows support - * - * SPECIAL WINDOWS NOTE - * - * Some of the Win32 API functions expand to fooA or fooW depending on - * whether UNICODE or _UNICODE are defined. You MUST explicitly use - * the A version of the functions to force char * behavior until we - * work out a scheme for proper Windows Unicode support. - * - * If you do not do this, people will be unable to incorporate our - * source code into their own CMake builds if they define UNICODE. - */ -#ifdef H5_HAVE_WIN32_API - -#define H5PL_PATH_SEPARATOR ";" - -/* Handle for dynamic library */ -#define H5PL_HANDLE HINSTANCE - -/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ -#define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) - -/* Get the address of a symbol in dynamic library */ -#define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N) - -/* Close dynamic library */ -#define H5PL_CLOSE_LIB(H) FreeLibrary(H) - -/* Clear error - nothing to do */ -#define H5PL_CLR_ERROR - -/* maximum size for expanding env vars */ -#define H5PL_EXPAND_BUFFER_SIZE 32767 - -typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void); - -/* Unix support */ -#else /* H5_HAVE_WIN32_API */ - -#define H5PL_PATH_SEPARATOR ":" - -/* Handle for dynamic library */ -#define H5PL_HANDLE void * - -/* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ -#define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY) - -/* Get the address of a symbol in dynamic library */ -#define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N) - -/* Close dynamic library */ -#define H5PL_CLOSE_LIB(H) dlclose(H) - -/* Clear error */ -#define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror()) - -typedef const void *(*H5PL_get_plugin_info_t)(void); -#endif /* H5_HAVE_WIN32_API */ - -/* Whether to preload pathnames for plugin libraries */ -#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR - -/* Special symbol to indicate no plugin loading */ -#define H5PL_NO_PLUGIN "::" /******************/ /* Local Typedefs */ /******************/ -/* Type for the list of info for opened plugin libraries */ -typedef struct H5PL_table_t { - H5PL_type_t pl_type; /* plugin type */ - int pl_id; /* ID for the plugin */ - H5PL_HANDLE handle; /* plugin handle */ -} H5PL_table_t; - /********************/ /* Local Prototypes */ /********************/ -static herr_t H5PL__init_path_table(void); -static htri_t H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info); -static htri_t H5PL__open(H5PL_type_t pl_type, char *libname, int plugin_id, const void **pl_info); -static htri_t H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info); -static herr_t H5PL__close(H5PL_HANDLE handle); - /*********************/ /* Package Variables */ /*********************/ -/* Package initialization variable */ -hbool_t H5_PKG_INIT_VAR = FALSE; - /*****************************/ /* Library Private Variables */ @@ -161,145 +54,43 @@ hbool_t H5_PKG_INIT_VAR = FALSE; /* Local Variables */ /*******************/ -/* Table for opened plugin libraries */ -static size_t H5PL_table_alloc_g = 0; -static size_t H5PL_table_used_g = 0; -static H5PL_table_t *H5PL_table_g = NULL; - -/* Table of location paths for plugin libraries */ -static char *H5PL_path_table_g[H5PL_MAX_PATH_NUM]; -static size_t H5PL_num_paths_g = 0; -static hbool_t H5PL_path_found_g = FALSE; - -/* Enable all plugin libraries */ -static unsigned int H5PL_plugin_g = H5PL_ALL_PLUGIN; - - - -/*-------------------------------------------------------------------------- -NAME - H5PL__init_package -- Initialize interface-specific information -USAGE - herr_t H5PL__init_package() -RETURNS - Non-negative on success/Negative on failure -DESCRIPTION - Initializes any interface-specific data or routines. - ---------------------------------------------------------------------------*/ -herr_t -H5PL__init_package(void) -{ - char *preload_path; - - FUNC_ENTER_PACKAGE_NOERR - - /* Retrieve pathnames from HDF5_PLUGIN_PRELOAD if the user sets it - * to tell the library to load plugin libraries without search. - */ - if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD"))) - /* Special symbol "::" means no plugin during data reading. */ - if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN)) - H5PL_plugin_g = 0; - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5PL__init_package() */ /*------------------------------------------------------------------------- - * Function: H5PL_term_package - * - * Purpose: Terminate the H5PL interface: release all memory, reset all - * global variables to initial values. This only happens if all - * types have been destroyed from other interfaces. + * Function: H5PLset_loading_state * - * Return: Success: Positive if any action was taken that might - * affect some other interface; zero otherwise. - * Failure: Negative. - * - * Programmer: Raymond Lu - * 20 February 2013 - * - *------------------------------------------------------------------------- - */ -int -H5PL_term_package(void) -{ - int n = 0; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - if(H5_PKG_INIT_VAR) { - size_t u; /* Local index variable */ - - /* Close opened dynamic libraries */ - if(H5PL_table_g) { - for(u = 0; u < H5PL_table_used_g; u++) - H5PL__close((H5PL_table_g[u]).handle); - - /* Free the table of dynamic libraries */ - H5PL_table_g = (H5PL_table_t *)H5MM_xfree(H5PL_table_g); - H5PL_table_used_g = H5PL_table_alloc_g = 0; - - n++; - } /* end if */ - - /* Free the table of search paths */ - if(H5PL_num_paths_g > 0) { - for(u = 0; u < H5PL_num_paths_g; u++) - if(H5PL_path_table_g[u]) - H5PL_path_table_g[u] = (char *)H5MM_xfree(H5PL_path_table_g[u]); - H5PL_num_paths_g = 0; - H5PL_path_found_g = FALSE; - - n++; - } /* end if */ - - /* Mark the interface as uninitialized */ - if(0 == n) - H5_PKG_INIT_VAR = FALSE; - } /* end if */ - - FUNC_LEAVE_NOAPI(n) -} /* end H5PL_term_package() */ - - -/*------------------------------------------------------------------------- - * Function: H5PLset_loading_state + * Purpose: Control the loading of dynamic plugin types. * - * Purpose: Control the loading of dynamic plugin types. + * The plugin_control_mask parameter is a bitfield that controls + * whether certain classes of plugins (e.g.: filters, + * VOL drivers) will be loaded by the library. * - * This function will not allow plugin types if the pathname from the HDF5_PLUGIN_PRELOAD - * environment variable is set to the special "::" string. + * plugin bit = 0, will prevent the use of that dynamic plugin type. + * plugin bit = 1, will allow the use of that dynamic plugin type. * - * plugin bit = 0, will prevent the use of that dynamic plugin type. - * plugin bit = 1, will allow the use of that dynamic plugin type. + * A list of pre-defined masks can be found in H5PLpublic.h. + * Set the mask to 0 to disable all plugins. * - * H5PL_TYPE_FILTER changes just dynamic filters - * A H5PL_ALL_PLUGIN will enable all dynamic plugin types - * A zero value will disable all dynamic plugin types + * This function will not allow plugin types if the pathname + * from the HDF5_PLUGIN_PRELOAD environment variable is set to + * the special "::" string. * - * Return: Non-negative or success + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLset_loading_state(unsigned int plugin_type) +H5PLset_loading_state(unsigned int plugin_control_mask) { - char *preload_path; - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE1("e", "Iu", plugin_type); - - /* change the bit value of the requested plugin type(s) */ - H5PL_plugin_g = plugin_type; + H5TRACE1("e", "Iu", plugin_control_mask); - /* check if special ENV variable is set and disable all plugin types */ - if(NULL != (preload_path = HDgetenv("HDF5_PLUGIN_PRELOAD"))) - /* Special symbol "::" means no plugin during data reading. */ - if(!HDstrcmp(preload_path, H5PL_NO_PLUGIN)) - H5PL_plugin_g = 0; + /* Set the plugin control mask */ + if(H5PL__set_plugin_control_mask(plugin_control_mask) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTSET, FAIL, "error setting plugin control mask") done: FUNC_LEAVE_API(ret_value) @@ -307,27 +98,35 @@ done: /*------------------------------------------------------------------------- - * Function: H5PLget_loading_state + * Function: H5PLget_loading_state * - * Purpose: Query state of the loading of dynamic plugin types. + * Purpose: Get the bitmask that controls whether certain classes + * of plugins (e.g.: filters, VOL drivers) will be loaded + * by the library. * - * This function will return the state of the global flag. + * Zero if all plugin types are disabled + * Negative if all plugin types are enabled + * Positive if one or more of the plugin types are enabled * - * Return: Zero if all plugin types are disabled, negative if all - * plugin types are enabled, positive if one or more of the plugin types are enabled. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLget_loading_state(unsigned int *plugin_type) +H5PLget_loading_state(unsigned int *plugin_control_mask) { - herr_t ret_value = SUCCEED; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*Iu", plugin_type); + H5TRACE1("e", "*Iu", plugin_control_mask); + + if (NULL == plugin_control_mask) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_control_mask parameter cannot be NULL") - if(plugin_type) - *plugin_type = H5PL_plugin_g; + /* Set the plugin control mask */ + if(H5PL__get_plugin_control_mask(plugin_control_mask) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_CANTGET, FAIL, "error getting plugin control mask") done: FUNC_LEAVE_API(ret_value) @@ -335,674 +134,282 @@ done: /*------------------------------------------------------------------------- - * Function: H5PL_load - * - * Purpose: Given the plugin type and identifier, this function searches - * and/or loads a dynamic plugin library first among the already - * opened libraries then in the designated location paths. - * - * Return: Non-NULL on success/NULL on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -const void * -H5PL_load(H5PL_type_t type, int id) -{ - htri_t found; /* Whether the plugin was found */ - const void *plugin_info = NULL; - const void *ret_value = NULL; - - FUNC_ENTER_NOAPI(NULL) - - switch(type) { - case H5PL_TYPE_FILTER: - if((H5PL_plugin_g & H5PL_FILTER_PLUGIN) == 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id) - break; - - case H5PL_TYPE_ERROR: - case H5PL_TYPE_NONE: - default: - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id) - } /* end switch */ - - /* Initialize the location paths for dynamic libraries, if they aren't - * already set up. - */ - if(FALSE == H5PL_path_found_g) - if(H5PL__init_path_table() < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, NULL, "can't initialize search path table") - - /* Search in the table of already loaded plugin libraries */ - if((found = H5PL__search_table(type, id, &plugin_info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in table failed") - - /* If not found, iterate through the path table to find the right dynamic library */ - if(!found) { - size_t i; /* Local index variable */ - - for(i = 0; i < H5PL_num_paths_g; i++) { - if((found = H5PL__find(type, id, H5PL_path_table_g[i], &plugin_info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in paths failed") - - /* Break out if found */ - if(found) { - HDassert(plugin_info); - break; - } /* end if */ - } /* end for */ - } /* end if */ - - /* Check if we found the plugin */ - if(found) - ret_value = plugin_info; - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL_load() */ - - -/*------------------------------------------------------------------------- - * Function: H5PLappend + * Function: H5PLappend * - * Purpose: Insert a plugin path at the end of the list. + * Purpose: Insert a plugin search path at the end of the list. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLappend(const char *plugin_path) +H5PLappend(const char *search_path) { herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*s", plugin_path); - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + H5TRACE1("e", "*s", search_path); - H5PL_EXPAND_ENV_VAR + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") - H5PL_path_table_g[H5PL_num_paths_g] = dl_path; - H5PL_num_paths_g++; + /* Append the search path to the path table */ + if (H5PL__append_path(search_path) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTAPPEND, FAIL, "unable to append search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLappend() */ - + /*------------------------------------------------------------------------- - * Function: H5PLprepend + * Function: H5PLprepend * - * Purpose: Insert a plugin path at the beginning of the list. + * Purpose: Insert a plugin search path at the beginning of the list. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLprepend(const char *plugin_path) +H5PLprepend(const char *search_path) { herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; - unsigned int plindex; FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*s", plugin_path); - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + H5TRACE1("e", "*s", search_path); - H5PL_EXPAND_ENV_VAR + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") - for (plindex = (unsigned int)H5PL_num_paths_g; plindex > 0; plindex--) - H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1]; - H5PL_path_table_g[0] = dl_path; - H5PL_num_paths_g++; + /* Prepend the search path to the path table */ + if (H5PL__prepend_path(search_path) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLprepend() */ - + /*------------------------------------------------------------------------- - * Function: H5PLreplace + * Function: H5PLreplace * - * Purpose: Replace the path at the specified index. + * Purpose: Replace the path at the specified index. The path at the + * index must exist. * - * Return: Non-negative or success. + * Return: Non-negative or success. * *------------------------------------------------------------------------- */ herr_t -H5PLreplace(const char *plugin_path, unsigned int index) +H5PLreplace(const char *search_path, unsigned int index) { - herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; + unsigned num_paths; /* Current number of stored paths */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE2("e", "*sIu", plugin_path, index); - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + H5TRACE2("e", "*sIu", search_path, index); - H5PL_EXPAND_ENV_VAR + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") - if(H5PL_path_table_g[index]) - H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]); - H5PL_path_table_g[index] = dl_path; + /* Check index */ + num_paths = H5PL__get_num_paths(); + if (0 == num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty") + else if (index >= num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Insert the search path into the path table */ + if (H5PL__replace_path(search_path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLreplace() */ - + /*------------------------------------------------------------------------- - * Function: H5PLinsert + * Function: H5PLinsert * - * Purpose: Insert a plugin path at the specified index, moving other paths after the index. + * Purpose: Insert a plugin search path at the specified index, moving + * other paths after the index. * - * Return: Non-negative or success. + * Return: Success: Non-negative + * Failture: Negative * *------------------------------------------------------------------------- */ herr_t -H5PLinsert(const char *plugin_path, unsigned int index) +H5PLinsert(const char *search_path, unsigned int index) { - herr_t ret_value = SUCCEED; /* Return value */ - char *dl_path = NULL; - unsigned int plindex; + unsigned num_paths; /* Current number of stored paths */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE2("e", "*sIu", plugin_path, index); - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == plugin_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no path provided") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == (dl_path = H5MM_strdup(plugin_path))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - - H5PL_EXPAND_ENV_VAR - - for(plindex = (unsigned int)H5PL_num_paths_g; plindex > index; plindex--) - H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex - 1]; - H5PL_path_table_g[index] = dl_path; - H5PL_num_paths_g++; + H5TRACE2("e", "*sIu", search_path, index); + + /* Check args */ + if (NULL == search_path) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot be NULL") + if (0 == HDstrlen(search_path)) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "plugin_path parameter cannot have length zero") + + /* Check index */ + num_paths = H5PL__get_num_paths(); + if ((0 != num_paths) && (index >= num_paths)) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Insert the search path into the path table */ + if (H5PL__insert_path(search_path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLinsert() */ - + /*------------------------------------------------------------------------- - * Function: H5PLremove + * Function: H5PLremove + * + * Purpose: Remove the plugin path at the specifed index and compact + * the list. * - * Purpose: Remove the plugin path at the specifed index and compacting the list. + * Return: Success: Non-negative + * Failture: Negative * - * Return: Non-negative or success. + * Return: Non-negative or success. * *------------------------------------------------------------------------- */ herr_t H5PLremove(unsigned int index) { - herr_t ret_value = SUCCEED; /* Return value */ - unsigned int plindex; + unsigned num_paths; /* Current number of stored paths */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE1("e", "Iu", index); - if(H5PL_num_paths_g == 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == H5PL_path_table_g[index]) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index") - H5PL_path_table_g[index] = (char *)H5MM_xfree(H5PL_path_table_g[index]); - - H5PL_num_paths_g--; - for(plindex = index; plindex < (unsigned int)H5PL_num_paths_g; plindex++) - H5PL_path_table_g[plindex] = H5PL_path_table_g[plindex + 1]; - H5PL_path_table_g[H5PL_num_paths_g] = NULL; + + /* Check index */ + num_paths = H5PL__get_num_paths(); + if (0 == num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty") + else if (index >= num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Delete the search path from the path table */ + if (H5PL__remove_path(index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "unable to remove search path") done: FUNC_LEAVE_API(ret_value) } /* end H5PLremove() */ - + /*------------------------------------------------------------------------- - * Function: H5PLget + * Function: H5PLget + * + * Purpose: Query the plugin path at a specified index. + * + * If 'path_buf' is non-NULL then up to 'buf_size' bytes will be written into + * that buffer and the length of the path name will be returned. * - * Purpose: Query the plugin path at the specified index. + * If 'path_buf' is NULL, this function will simply return the number of + * characters required to store the path name, ignoring 'path_buf' and + * 'buf_size' * - * Return: Success: The length of path. + * If an error occurs then the buffer pointed to by 'path_buf' + * (NULL or non-NULL) will be unchanged and the function will return a + * negative value. * - * If `pathname' is non-NULL then write up to `size' bytes into that - * buffer and always return the length of the pathname. - * Otherwise `size' is ignored and the function does not store the pathname, - * just returning the number of characters required to store the pathname. - * If an error occurs then the buffer pointed to by `pathname' (NULL or non-NULL) - * is unchanged and the function returns a negative value. - * If a zero is returned for the name's length, then there is no pathname - * associated with the index. + * If a zero is returned for the name's length, then there is no path name + * associated with the index and the 'path_buf' buffer will be unchanged. + * + * Return: Success: The length of path + * Failure: A negative value * *------------------------------------------------------------------------- */ ssize_t -H5PLget(unsigned int index, char *pathname/*out*/, size_t size) +H5PLget(unsigned int index, char *path_buf, size_t buf_size) { - ssize_t ret_value = 0; /* Return value */ - size_t len = 0; /* Length of pathname */ - char *dl_path = NULL; + unsigned num_paths; /* Current number of stored paths */ + const char *path = NULL; /* path from table */ + size_t path_len = 0; /* Length of path */ + ssize_t ret_value = 0; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE3("Zs", "Iuxz", index, pathname, size); - if(H5PL_num_paths_g == 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "no directories in table") - if(index >= H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "index path out of bounds for table") - if(NULL == (dl_path = H5PL_path_table_g[index])) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "no directory path at index") - len = HDstrlen(dl_path); - if(pathname) { - HDstrncpy(pathname, dl_path, MIN((size_t)(len + 1), size)); - if((size_t)len >= size) - pathname[size - 1] = '\0'; + H5TRACE3("Zs", "Iu*sz", index, path_buf, buf_size); + + /* Check index */ + num_paths = H5PL__get_num_paths(); + if (0 == num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "path table is empty") + else if (index >= num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "index path out of bounds for table - can't be more than %u", (num_paths - 1)) + + /* Check if the search table is empty */ + if (H5PL__get_num_paths() == 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, (-1), "plugin search path table is empty") + + /* Get the path at the specified index and its length */ + if (NULL == (path = H5PL__get_path(index))) + HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, (-1), "no path stored at that index") + path_len = HDstrlen(path); + + /* If the path buffer is not NULL, copy the path to the buffer */ + if (path_buf) { + HDstrncpy(path_buf, path, MIN((size_t)(path_len + 1), buf_size)); + if ((size_t)path_len >= buf_size) + path_buf[buf_size - 1] = '\0'; } /* end if */ /* Set return value */ - ret_value = (ssize_t)len; + ret_value = (ssize_t)path_len; done: FUNC_LEAVE_API(ret_value) } /* end H5PLget() */ - + /*------------------------------------------------------------------------- - * Function: H5PLsize + * Function: H5PLsize * - * Purpose: Query the size of the current list of plugin paths. + * Purpose: Get the number of stored plugin paths. + * XXX: This is a terrible name. Can it be changed? * - * Return: Plugin path size + * Return: SUCCEED/FAIL * *------------------------------------------------------------------------- */ herr_t -H5PLsize(unsigned int *listsize) +H5PLsize(unsigned int *num_paths) { herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) - H5TRACE1("e", "*Iu", listsize); + H5TRACE1("e", "*Iu", num_paths); - *listsize = (unsigned int)H5PL_num_paths_g; + /* Check arguments */ + if (!num_paths) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "num_paths parameter cannot be NULL") + + /* Get the number of stored plugin paths */ + *num_paths = H5PL__get_num_paths(); done: FUNC_LEAVE_API(ret_value) } /* end H5PLsize() */ - -/*------------------------------------------------------------------------- - * Function: H5PL__init_path_table - * - * Purpose: Initialize the path table. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Quincey Koziol - * 18 March 2013 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5PL__init_path_table(void) -{ - char *dl_path = NULL; - char *origin_dl_path; - char *dir; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it - * or from the default paths if it isn't set. - */ - origin_dl_path = HDgetenv("HDF5_PLUGIN_PATH"); - if(NULL == origin_dl_path) - dl_path = H5MM_strdup(H5PL_DEFAULT_PATH); - else - dl_path = H5MM_strdup(origin_dl_path); - if(NULL == dl_path) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - - H5PL_EXPAND_ENV_VAR - - /* Put paths in the path table. They are separated by ":" */ - dir = HDstrtok(dl_path, H5PL_PATH_SEPARATOR); - while(dir) { - /* Check for too many directories in path */ - if(H5PL_num_paths_g == H5PL_MAX_PATH_NUM) - HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "too many directories in path for table") - if(NULL == (H5PL_path_table_g[H5PL_num_paths_g] = H5MM_strdup(dir))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - H5PL_num_paths_g++; - dir = HDstrtok(NULL, H5PL_PATH_SEPARATOR); - } /* end while */ - - H5PL_path_found_g = TRUE; - -done: - if(dl_path) - dl_path = (char *)H5MM_xfree(dl_path); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__init_path_table() */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__find - * - * Purpose: Given a path, this function opens the directory and envokes - * another function to go through all files to find the right - * plugin library. Two function definitions are for Unix and - * Windows. - * - * Return: TRUE on success, - * FALSE on not found, - * negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -#ifndef H5_HAVE_WIN32_API -static htri_t -H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info) -{ - char *pathname = NULL; - DIR *dirp = NULL; - struct dirent *dp; - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* Open the directory */ - if(!(dirp = HDopendir(dir))) - HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir) - - /* Iterates through all entries in the directory to find the right plugin library */ - while(NULL != (dp = HDreaddir(dirp))) { - /* The library we are looking for should be called libxxx.so... on Unix - * or libxxx.xxx.dylib on Mac. - */ -#ifndef __CYGWIN__ - if(!HDstrncmp(dp->d_name, "lib", (size_t)3) && - (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) { -#else - if(!HDstrncmp(dp->d_name, "cyg", (size_t)3) && - HDstrstr(dp->d_name, ".dll") ) { - -#endif - h5_stat_t my_stat; - size_t pathname_len; - htri_t found_in_dir; - - /* Allocate & initialize the path name */ - pathname_len = HDstrlen(dir) + HDstrlen(dp->d_name) + 2; - if(NULL == (pathname = (char *)H5MM_malloc(pathname_len))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - HDsnprintf(pathname, pathname_len, "%s/%s", dir, dp->d_name); - - /* Get info for directory entry */ - if(HDstat(pathname, &my_stat) == -1) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file: %s", HDstrerror(errno)) - - /* If it is a directory, skip it */ - if(S_ISDIR(my_stat.st_mode)) - continue; - - /* Attempt to open the dynamic library as a filter library */ - if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") - if(found_in_dir) - HGOTO_DONE(TRUE) /* Indicate success */ - pathname = (char *)H5MM_xfree(pathname); - } /* end if */ - } /* end while */ - -done: - if(dirp) - if(HDclosedir(dirp) < 0) - HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno)) - pathname = (char *)H5MM_xfree(pathname); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__find() */ -#else /* H5_HAVE_WIN32_API */ -static htri_t -H5PL__find(H5PL_type_t plugin_type, int type_id, char *dir, const void **info) -{ - WIN32_FIND_DATAA fdFile; - HANDLE hFind; - char *pathname = NULL; - char service[2048]; - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* Specify a file mask. *.* = We want everything! */ - sprintf(service, "%s\\*.dll", dir); - if((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE) - HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory") - - do { - /* Find first file will always return "." - * and ".." as the first two directories. - */ - if(HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) { - size_t pathname_len; - htri_t found_in_dir; - - /* Allocate & initialize the path name */ - pathname_len = HDstrlen(dir) + HDstrlen(fdFile.cFileName) + 2; - if(NULL == (pathname = (char *)H5MM_malloc(pathname_len))) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") - HDsnprintf(pathname, pathname_len, "%s\\%s", dir, fdFile.cFileName); - - /* Is the entity a File or Folder? */ - if(fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) - continue; - - if((found_in_dir = H5PL__open(plugin_type, pathname, type_id, info)) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") - if(found_in_dir) - HGOTO_DONE(TRUE) /* Indicate success */ - pathname = (char *)H5MM_xfree(pathname); - } /* end if */ - } while(FindNextFileA(hFind, &fdFile)); /* Find the next file. */ - -done: - if(hFind) - FindClose(hFind); - if(pathname) - pathname = (char *)H5MM_xfree(pathname); - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__find() */ -#endif /* H5_HAVE_WIN32_API */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__open - * - * Purpose: Iterates through all files to find the right plugin library. - * It loads the dynamic plugin library and keeps it on the list - * of loaded libraries. - * - * Return: TRUE on success, - * FALSE on not found, - * negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -static htri_t -H5PL__open(H5PL_type_t pl_type, char *libname, int pl_id, const void **pl_info) -{ - H5PL_HANDLE handle = NULL; - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* There are different reasons why a library can't be open, e.g. wrong architecture. - * simply continue if we can't open it. - */ - if(NULL == (handle = H5PL_OPEN_DLIB(libname))) { - H5PL_CLR_ERROR; /* clear error */ - } /* end if */ - else { - H5PL_get_plugin_info_t get_plugin_info = NULL; - - /* Return a handle for the function H5PLget_plugin_info in the dynamic library. - * The plugin library is suppose to define this function. - */ - if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) { - if(H5PL__close(handle) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") - } /* end if */ - else { - const H5Z_class2_t *plugin_info; - - /* Invoke H5PLget_plugin_info to verify this is the right library we are looking for. - * Move on if it isn't. - */ - if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)())) { - if(H5PL__close(handle) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") - HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info") - } /* end if */ - - /* Successfully found plugin library, check if it's the right one */ - if(plugin_info->id == pl_id) { - /* Expand the table if it is too small */ - if(H5PL_table_used_g >= H5PL_table_alloc_g) { - size_t n = MAX(H5Z_MAX_NFILTERS, 2 * H5PL_table_alloc_g); - H5PL_table_t *table = (H5PL_table_t *)H5MM_realloc(H5PL_table_g, n * sizeof(H5PL_table_t)); - - if(!table) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, FAIL, "unable to extend dynamic library table") - - H5PL_table_g = table; - H5PL_table_alloc_g = n; - } /* end if */ - - (H5PL_table_g[H5PL_table_used_g]).handle = handle; - (H5PL_table_g[H5PL_table_used_g]).pl_type = pl_type; - (H5PL_table_g[H5PL_table_used_g]).pl_id = plugin_info->id; - H5PL_table_used_g++; - - /* Set the plugin info to return */ - *pl_info = (const void *)plugin_info; - - /* Indicate success */ - ret_value = TRUE; - } /* end if */ - else - if(H5PL__close(handle) < 0) - HGOTO_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") - } /* end if */ - } /* end else */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__open() */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__search_table - * - * Purpose: Search in the list of already opened dynamic libraries - * to see if the one we are looking for is already opened. - * - * Return: TRUE on success, - * FALSE on not found, - * Negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -static htri_t -H5PL__search_table(H5PL_type_t plugin_type, int type_id, const void **info) -{ - htri_t ret_value = FALSE; - - FUNC_ENTER_STATIC - - /* Search in the table of already opened dynamic libraries */ - if(H5PL_table_used_g > 0) { - size_t i; - - for(i = 0; i < H5PL_table_used_g; i++) { - if((plugin_type == (H5PL_table_g[i]).pl_type) && (type_id == (H5PL_table_g[i]).pl_id)) { - H5PL_get_plugin_info_t get_plugin_info; - const H5Z_class2_t *plugin_info; - - if(NULL == (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_table_g[i]).handle, "H5PLget_plugin_info"))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info") - - if(NULL == (plugin_info = (const H5Z_class2_t *)(*get_plugin_info)())) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info") - - *info = plugin_info; - HGOTO_DONE(TRUE) - } /* end if */ - } /* end for */ - } /* end if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5PL__search_table() */ - - -/*------------------------------------------------------------------------- - * Function: H5PL__close - * - * Purpose: Closes the handle for dynamic library - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Raymond Lu - * 13 February 2013 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5PL__close(H5PL_HANDLE handle) -{ - FUNC_ENTER_STATIC_NOERR - - H5PL_CLOSE_LIB(handle); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5PL__close() */ - diff --git a/src/H5PLextern.h b/src/H5PLextern.h index 7547ad7..cd5464d 100644 --- a/src/H5PLextern.h +++ b/src/H5PLextern.h @@ -11,9 +11,9 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu <songyulu@hdfgroup.org> - * 13 February 2013 + * Purpose: Header file for writing external HDF5 plugins. */ + #ifndef _H5PLextern_H #define _H5PLextern_H diff --git a/src/H5PLint.c b/src/H5PLint.c new file mode 100644 index 0000000..c887f86 --- /dev/null +++ b/src/H5PLint.c @@ -0,0 +1,384 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Internal routines for managing plugins. + * + */ + + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5PLmodule.h" /* This source code file is part of the H5PL module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5PLpkg.h" /* Plugin */ +#include "H5Zprivate.h" /* Filter pipeline */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Package initialization variable */ +hbool_t H5_PKG_INIT_VAR = FALSE; + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Bitmask that controls whether classes of plugins + * (e.g.: filters, VOL drivers) can be loaded. + */ +static unsigned int H5PL_plugin_control_mask_g = H5PL_ALL_PLUGIN; + +/* This flag will be set to FALSE if the HDF5_PLUGIN_PRELOAD + * environment variable was set to H5PL_NO_PLUGIN at + * package initialization. + */ +static hbool_t H5PL_allow_plugins_g = TRUE; + + + +/*------------------------------------------------------------------------- + * Function: H5PL__get_plugin_control_mask + * + * Purpose: Gets the internal plugin control mask value. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__get_plugin_control_mask(unsigned int *mask /*out*/) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Check args - Just assert on package functions */ + HDassert(mask); + + /* Return the mask */ + *mask = H5PL_plugin_control_mask_g; + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5PL__get_plugin_control_mask() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__set_plugin_control_mask + * + * Purpose: Sets the internal plugin control mask value. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__set_plugin_control_mask(unsigned int mask) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Only allow setting this if plugins have not been disabled. + * XXX: Note that we don't consider this an error, but instead + * silently ignore it. We may want to consider this behavior + * more carefully. + */ + if (H5PL_allow_plugins_g) + H5PL_plugin_control_mask_g = mask; + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5PL__set_plugin_control_mask() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__init_package + * + * Purpose: Initialize any package-specific data and call any init + * routines for the package. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__init_package(void) +{ + char *env_var = NULL; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check the environment variable to determine if the user wants + * to ignore plugins. The special symbol H5PL_NO_PLUGIN (defined in + * H5PLpublic.h) means we don't want to load plugins. + */ + if (NULL != (env_var = HDgetenv("HDF5_PLUGIN_PRELOAD"))) + if (!HDstrcmp(env_var, H5PL_NO_PLUGIN)) { + H5PL_plugin_control_mask_g = 0; + H5PL_allow_plugins_g = FALSE; + } + + /* Create the table of previously-loaded plugins */ + if (H5PL__create_plugin_cache() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin cache") + + /* Create the table of search paths for dynamic libraries */ + if (H5PL__create_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINIT, FAIL, "can't create plugin search path table") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__init_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL_term_package + * + * Purpose: Terminate the H5PL interface: release all memory, reset all + * global variables to initial values. This only happens if all + * types have been destroyed from other interfaces. + * + * Return: Success: Positive if any action was taken that might + * affect some other interface; zero otherwise + * Failure: Negative + * + *------------------------------------------------------------------------- + */ +int +H5PL_term_package(void) +{ + hbool_t already_closed = FALSE; + int ret_value = 0; + + FUNC_ENTER_NOAPI_NOINIT + + if (H5_PKG_INIT_VAR) { + + /* Close the plugin cache. + * We need to bump the return value if we did any real work here. + */ + if (H5PL__close_plugin_cache(&already_closed) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing plugin cache") + if (!already_closed) + ret_value++; + + /* Close the search path table and free the paths */ + if (H5PL__close_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, (-1), "problem closing search path table") + + /* Mark the interface as uninitialized */ + if (0 == ret_value) + H5_PKG_INIT_VAR = FALSE; + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL_term_package() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL_load + * + * Purpose: Given the plugin type and identifier, this function searches + * for and, if found, loads a dynamic plugin library. + * + * The function searches first in the cached plugins and then + * in the paths listed in the path table. + * + * Return: Success: A pointer to the plugin info + * Failure: NULL + * + *------------------------------------------------------------------------- + */ +const void * +H5PL_load(H5PL_type_t type, int id) +{ + H5PL_search_params_t search_params; + hbool_t found = FALSE; /* Whether the plugin was found */ + const void *plugin_info = NULL; + const void *ret_value = NULL; + + FUNC_ENTER_NOAPI(NULL) + + /* Check if plugins can be loaded for this plugin type */ + switch (type) { + case H5PL_TYPE_FILTER: + if ((H5PL_plugin_control_mask_g & H5PL_FILTER_PLUGIN) == 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin filter '%d' is not available", id) + break; + + case H5PL_TYPE_ERROR: + case H5PL_TYPE_NONE: + default: + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTLOAD, NULL, "required dynamically loaded plugin '%d' is not valid", id) + } + + /* Set up the search parameters */ + search_params.type = type; + search_params.id = id; + + /* Search in the table of already loaded plugin libraries */ + if(H5PL__find_plugin_in_cache(&search_params, &found, &plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in plugin cache failed") + + /* If not found, try iterating through the path table to find an appropriate plugin */ + if (!found) + if (H5PL__find_plugin_in_path_table(&search_params, &found, &plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, NULL, "search in path table failed") + + /* Set the return value we found the plugin */ + if (found) + ret_value = plugin_info; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL_load() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__open + * + * Purpose: Opens a plugin. + * + * The success parameter will be set to TRUE and the plugin_info + * parameter will be filled in on success. Otherwise, they + * will be FALSE and NULL, respectively. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +/* NOTE: We turn off -Wpedantic in gcc to quiet a warning about converting + * object pointers to function pointers, which is undefined in ANSI C. + * This is basically unavoidable due to the nature of dlsym() and *is* + * defined in POSIX, so it's fine. + * + * This pragma only needs to surround the assignment of the + * get_plugin_info function pointer, but early (4.4.7, at least) gcc + * only allows diagnostic pragmas to be toggled outside of functions. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +herr_t +H5PL__open(const char *path, H5PL_type_t type, int id, hbool_t *success, const void **plugin_info) +{ + H5PL_HANDLE handle = NULL; + H5PL_get_plugin_info_t get_plugin_info = NULL; + htri_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(success); + HDassert(plugin_info); + + /* Initialize out parameters */ + *success = FALSE; + *plugin_info = NULL; + + /* There are different reasons why a library can't be open, e.g. wrong architecture. + * If we can't open the library, just return. + */ + if (NULL == (handle = H5PL_OPEN_DLIB(path))) { + H5PL_CLR_ERROR; /* clear error */ + HGOTO_DONE(SUCCEED); + } + + /* Return a handle for the function H5PLget_plugin_info in the dynamic library. + * The plugin library is suppose to define this function. + */ + if (NULL != (get_plugin_info = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC(handle, "H5PLget_plugin_info"))) { + + const H5Z_class2_t *info; + + /* Get the plugin info */ + if (NULL == (info = (const H5Z_class2_t *)(*get_plugin_info)())) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "can't get plugin info") + + /* Check if the filter IDs match */ + if (info->id == id) { + + /* Store the plugin in the cache */ + if (H5PL__add_plugin(type, id, handle)) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to add new plugin to plugin cache") + + /* Set output parameters */ + *success = TRUE; + *plugin_info = (const void *)info; + } + } + +done: + if (!success && handle) + if (H5PL__close(handle) < 0) + HDONE_ERROR(H5E_PLUGIN, H5E_CLOSEERROR, FAIL, "can't close dynamic library") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__open() */ +#pragma GCC diagnostic pop + + +/*------------------------------------------------------------------------- + * Function: H5PL__close + * + * Purpose: Closes the handle for dynamic library + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__close(H5PL_HANDLE handle) +{ + FUNC_ENTER_PACKAGE_NOERR + + H5PL_CLOSE_LIB(handle); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5PL__close() */ + diff --git a/src/H5PLmodule.h b/src/H5PLmodule.h index b441aed..945441e 100644 --- a/src/H5PLmodule.h +++ b/src/H5PLmodule.h @@ -11,13 +11,11 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Quincey Koziol <koziol@hdfgroup.org> - * Saturday, September 12, 2015 - * - * Purpose: This file contains declarations which define macros for the - * H5PL package. Including this header means that the source file - * is part of the H5PL package. + * Purpose: This file contains declarations which define macros for the + * H5PL package. Including this header means that the source file + * is part of the H5PL package. */ + #ifndef _H5PLmodule_H #define _H5PLmodule_H diff --git a/src/H5PLpath.c b/src/H5PLpath.c new file mode 100644 index 0000000..435802a --- /dev/null +++ b/src/H5PLpath.c @@ -0,0 +1,776 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Code to implement a path table which stores plugin search paths. + * + * The path table is implemented as a dynamic, global array which + * will grow as new paths are inserted. The capacity of the path + * table never shrinks (though given the low number of paths + * expected and the low likelihood of paths being removed, this + * seems unlikely to be a problem). Inserts and removals rework + * the array so that there are no 'holes' in the in-use part + * of the array. + * + * Note that it's basically up to the user to manage the indexes + * when a complicated series of insert, overwrite, and, remove + * operations take place. + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5PLmodule.h" /* This source code file is part of the H5PL module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5PLpkg.h" /* Plugin */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* Initial capacity of the path table */ +#define H5PL_INITIAL_PATH_CAPACITY 16 + +/* The amount to add to the capacity when the table is full */ +#define H5PL_PATH_CAPACITY_ADD 16 + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + +static herr_t H5PL__insert_at(const char *path, unsigned int index); +static herr_t H5PL__make_space_at(unsigned int index); +static herr_t H5PL__replace_at(const char *path, unsigned int index); +static herr_t H5PL__expand_path_table(void); +static herr_t H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info); + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Stored plugin paths to search */ +static char **H5PL_paths_g = NULL; + +/* The number of stored paths */ +static unsigned H5PL_num_paths_g = 0; + +/* The capacity of the path table */ +static unsigned H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY; + + + +/*------------------------------------------------------------------------- + * Function: H5PL__insert_at() + * + * Purpose: Insert a path at a particular index in the path table. + * Does not clobber! Will move existing paths up to make + * room. Use H5PL__replace_at(index) if you want to clobber. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__insert_at(const char *path, unsigned int index) +{ + char *path_copy = NULL; /* copy of path string (for storing) */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Expand the table if it is full */ + if (H5PL_num_paths_g == H5PL_path_capacity_g) + if (H5PL__expand_path_table() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand path table") + + /* Copy the path for storage so the caller can dispose of theirs */ + if (NULL == (path_copy = H5MM_strdup(path))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path") + +#ifdef H5_HAVE_WIN32_API + /* Clean up Microsoft Windows environment variables in the path string */ + if(H5_expand_windows_env_vars(&path_copy)) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string") +#endif /* H5_HAVE_WIN32_API */ + + /* If the table entry is in use, make some space */ + if (H5PL_paths_g[index]) + if (H5PL__make_space_at(index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "unable to make space in the table for the new entry") + + /* Insert the copy of the search path into the table at the specified index */ + H5PL_paths_g[index] = path_copy; + H5PL_num_paths_g++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__insert_at() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__make_space_at() + * + * Purpose: Free up a slot in the path table, moving existing path + * entries as necessary. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__make_space_at(unsigned int index) +{ + unsigned u; /* iterator */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check args - Just assert on package functions */ + HDassert(index < H5PL_path_capacity_g); + + /* Copy the paths back to make a space */ + for (u = H5PL_num_paths_g; u > index; u--) + H5PL_paths_g[u] = H5PL_paths_g[u-1]; + + H5PL_paths_g[index] = NULL; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__make_space_at() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__replace_at() + * + * Purpose: Replace a path at a particular index in the path table. + * The path in the table must exist and will be freed by this + * function. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__replace_at(const char *path, unsigned int index) +{ + char *path_copy = NULL; /* copy of path string (for storing) */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Check that the table entry is in use */ + if (!H5PL_paths_g[index]) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTFREE, FAIL, "path entry at index %u in the table is NULL", index) + + /* Copy the path for storage so the caller can dispose of theirs */ + if (NULL == (path_copy = H5MM_strdup(path))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't make internal copy of path") + +#ifdef H5_HAVE_WIN32_API + /* Clean up Microsoft Windows environment variables in the path string */ + if (H5_expand_windows_env_vars(&path_copy)) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTCONVERT, FAIL, "can't expand environment variable string") +#endif /* H5_HAVE_WIN32_API */ + + /* Free the existing path entry */ + H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]); + + /* Copy the search path into the table at the specified index */ + H5PL_paths_g[index] = path_copy; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__replace_at() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__create_path_table + * + * Purpose: Create the collection of paths that will be searched + * when loading plugins. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__create_path_table(void) +{ + char *env_var= NULL; /* Path string from environment variable */ + char *paths = NULL; /* Delimited paths string. Either from the + * environment variable or the default. + */ + char *next_path = NULL; /* A path tokenized from the paths string */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Allocate memory for the path table */ + H5PL_num_paths_g = 0; + H5PL_path_capacity_g = H5PL_INITIAL_PATH_CAPACITY; + if (NULL == (H5PL_paths_g = (char **)H5MM_calloc((size_t)H5PL_path_capacity_g * sizeof(char *)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path table") + + /* Retrieve paths from HDF5_PLUGIN_PATH if the user sets it + * or from the default paths if it isn't set. + */ + env_var = HDgetenv("HDF5_PLUGIN_PATH"); + if (NULL == env_var) + paths = H5MM_strdup(H5PL_DEFAULT_PATH); + else + paths = H5MM_strdup(env_var); + + if (NULL == paths) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path copy") + + /* Separate the paths and store them */ + /* XXX: strtok() is not thread-safe */ + next_path = HDstrtok(paths, H5PL_PATH_SEPARATOR); + while (next_path) { + + /* Insert the path into the table */ + if (H5PL__append_path(next_path) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't insert path: %s", next_path) + + /* Get the next path from the environment string */ + next_path = HDstrtok(NULL, H5PL_PATH_SEPARATOR); + } /* end while */ + +done: + if (paths) + paths = (char *)H5MM_xfree(paths); + + /* Try to clean up on errors */ + if (FAIL == ret_value) { + if (H5PL_paths_g) + H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g); + H5PL_path_capacity_g = 0; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__create_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__close_path_table + * + * Purpose: Close the collection of paths that will be searched + * when loading plugins. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__close_path_table(void) +{ + unsigned u; /* iterator */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE_NOERR + + /* Free paths */ + for (u = 0; u < H5PL_num_paths_g; u++) + if (H5PL_paths_g[u]) + H5PL_paths_g[u] = (char *)H5MM_xfree(H5PL_paths_g[u]); + + /* Free path table */ + H5PL_paths_g = (char **)H5MM_xfree(H5PL_paths_g); + + /* Reset values */ + H5PL_num_paths_g = 0; + + FUNC_LEAVE_NOAPI(ret_value) + +} /* end H5PL__close_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__get_num_paths + * + * Purpose: Gets the number of plugin paths that have been stored. + * + * Return: Success: The number of paths + * Failture: Can't fail + *------------------------------------------------------------------------- + */ +unsigned +H5PL__get_num_paths(void) +{ + FUNC_ENTER_PACKAGE_NOERR + + FUNC_LEAVE_NOAPI(H5PL_num_paths_g) + +} /* end H5PL__get_num_paths() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__expand_path_table + * + * Purpose: Expand the path table when it's full. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__expand_path_table(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Update the capacity */ + H5PL_path_capacity_g += H5PL_PATH_CAPACITY_ADD; + + /* Resize the array */ + if(NULL == (H5PL_paths_g = (char **)H5MM_realloc(H5PL_paths_g, (size_t)H5PL_path_capacity_g * sizeof(char *)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for path table failed") + + /* Initialize the new memory */ + HDmemset(H5PL_paths_g + H5PL_num_paths_g, 0, (size_t)H5PL_PATH_CAPACITY_ADD * sizeof(char *)); + +done: + /* Set the path capacity back if there were problems */ + if (FAIL == ret_value) + H5PL_path_capacity_g -= H5PL_PATH_CAPACITY_ADD; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__expand_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__append_path + * + * Purpose: Insert a path at the end of the table. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__append_path(const char *path) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Insert the path at the end of the table */ + if (H5PL__insert_at(path, H5PL_num_paths_g) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to append search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__append_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__prepend_path + * + * Purpose: Insert a path at the beginning of the table. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__prepend_path(const char *path) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + + /* Insert the path at the beginning of the table */ + if (H5PL__insert_at(path, 0) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to prepend search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__prepend_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__replace_path + * + * Purpose: Replace a path at particular index in the table. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__replace_path(const char *path, unsigned int index) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + HDassert(index < H5PL_path_capacity_g); + + /* Insert the path at the requested index */ + if (H5PL__replace_at(path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to replace search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__replace_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__insert_path + * + * Purpose: Insert a path at particular index in the table, moving + * any existing paths back to make space. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__insert_path(const char *path, unsigned int index) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(path); + HDassert(HDstrlen(path)); + HDassert(index < H5PL_path_capacity_g); + + /* Insert the path at the requested index */ + if (H5PL__insert_at(path, index) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTINSERT, FAIL, "unable to insert search path") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__insert_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__remove_path + * + * Purpose: Remove a path at particular index in the table, freeing + * the path string and moving the paths down to close the gap. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__remove_path(unsigned int index) +{ + unsigned u; /* iterator */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(index < H5PL_path_capacity_g); + + /* Check if the path at that index is set */ + if (!H5PL_paths_g[index]) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTDELETE, FAIL, "search path at index %u is NULL", index) + + /* Delete the path */ + H5PL_num_paths_g--; + H5PL_paths_g[index] = (char *)H5MM_xfree(H5PL_paths_g[index]); + + /* Shift the paths down to close the gap */ + for (u = index; u < H5PL_num_paths_g; u++) + H5PL_paths_g[u] = H5PL_paths_g[u+1]; + + /* Set the (former) last path to NULL */ + H5PL_paths_g[H5PL_num_paths_g] = NULL; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__remove_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__get_path + * + * Purpose: Get a pointer to a path at particular index in the table. + * + * Return: Success: A pointer to a path string stored in the table + * Failure: NULL + * + *------------------------------------------------------------------------- + */ +const char * +H5PL__get_path(unsigned int index) +{ + char *ret_value = NULL; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Get the path at the requested index */ + if (index >= H5PL_num_paths_g) + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, NULL, "path index %u is out of range in table", index) + + return H5PL_paths_g[index]; +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__replace_path() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__find_plugin_in_path_table + * + * Purpose: Attempts to find a matching plugin in the file system + * using the paths stored in the path table. + *. + * The 'found' parameter will be set appropriately. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info) +{ + unsigned int u; /* iterator */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(plugin_info); + + /* Initialize output parameters */ + *found = FALSE; + *plugin_info = NULL; + + /* Loop over the paths in the table, checking for an appropriate plugin */ + for (u = 0; u < H5PL_num_paths_g; u++) { + + /* Search for the plugin in this path */ + if (H5PL__find_plugin_in_path(search_params, found, H5PL_paths_g[u], plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in path %s encountered an error", H5PL_paths_g[u]) + + /* Break out if found */ + if (*found) { + if (!plugin_info) + HGOTO_ERROR(H5E_PLUGIN, H5E_BADVALUE, FAIL, "plugin info should not be NULL") + break; + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_path_table() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__find_plugin_in_path + * + * Purpose: Given a path, this function opens the directory and envokes + * another function to go through all files to find the right + * plugin library. Two function definitions are for Unix and + * Windows. + * + * The found parameter will be set to TRUE and the info + * parameter will be filled in on success. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +#ifndef H5_HAVE_WIN32_API +static herr_t +H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info) +{ + char *path = NULL; + DIR *dirp = NULL; /* Directory stream */ + struct dirent *dp = NULL; /* Directory entry */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(dir); + HDassert(plugin_info); + + /* Initialize the found parameter */ + *found = FALSE; + + /* Open the directory */ + if (!(dirp = HDopendir(dir))) + HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory: %s", dir) + + /* Iterate through all entries in the directory */ + while (NULL != (dp = HDreaddir(dirp))) { + + /* The library we are looking for should be called libxxx.so... on Unix + * or libxxx.xxx.dylib on Mac. + */ +#ifndef __CYGWIN__ + if (!HDstrncmp(dp->d_name, "lib", (size_t)3) && + (HDstrstr(dp->d_name, ".so") || HDstrstr(dp->d_name, ".dylib"))) { +#else + if (!HDstrncmp(dp->d_name, "cyg", (size_t)3) && + HDstrstr(dp->d_name, ".dll") ) { +#endif + + h5_stat_t my_stat; + size_t len; + + /* Allocate & initialize the path name */ + len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(dp->d_name) + 1 /*\0*/; + + if (NULL == (path = (char *)H5MM_calloc(len))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + + HDsnprintf(path, len, "%s/%s", dir, dp->d_name); + + /* Get info for directory entry */ + if (HDstat(path, &my_stat) == -1) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't stat file %s -- error was: %s", path, HDstrerror(errno)) + + /* If it is a directory, skip it */ + if (S_ISDIR(my_stat.st_mode)) + continue; + + /* attempt to open the dynamic library as a filter library */ + if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") + if (*found) + HGOTO_DONE(SUCCEED) + + path = (char *)H5MM_xfree(path); + } /* end if */ + } /* end while */ + +done: + if (dirp) + if (HDclosedir(dirp) < 0) + HDONE_ERROR(H5E_FILE, H5E_CLOSEERROR, FAIL, "can't close directory: %s", HDstrerror(errno)) + + path = (char *)H5MM_xfree(path); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_path() */ +#else /* H5_HAVE_WIN32_API */ +static herr_t +H5PL__find_plugin_in_path(const H5PL_search_params_t *search_params, hbool_t *found, const char *dir, const void **plugin_info) +{ + WIN32_FIND_DATAA fdFile; + HANDLE hFind = INVALID_HANDLE_VALUE; + char *path = NULL; + char service[2048]; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(dir); + HDassert(plugin_info); + + /* Initialize the found parameter */ + *found = FALSE; + + /* Specify a file mask. *.* = We want everything! */ + HDsprintf(service, "%s\\*.dll", dir); + if ((hFind = FindFirstFileA(service, &fdFile)) == INVALID_HANDLE_VALUE) + HGOTO_ERROR(H5E_PLUGIN, H5E_OPENERROR, FAIL, "can't open directory") + + /* Loop over all the files */ + do { + /* Ignore '.' and '..' */ + if (HDstrcmp(fdFile.cFileName, ".") != 0 && HDstrcmp(fdFile.cFileName, "..") != 0) { + + /* XXX: Probably just continue here and move the code below over one tab */ + + size_t len; + + /* Allocate & initialize the path name */ + len = HDstrlen(dir) + HDstrlen(H5PL_PATH_SEPARATOR) + HDstrlen(fdFile.cFileName) + 1; + + if (NULL == (path = (char *)H5MM_calloc(len))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for path") + + HDsnprintf(path, len, "%s\\%s", dir, fdFile.cFileName); + + /* Ignore directories */ + if (fdFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) + continue; + + /* attempt to open the dynamic library as a filter library */ + if (H5PL__open(path, search_params->type, search_params->id, found, plugin_info) < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "search in directory failed") + if (*found) + HGOTO_DONE(SUCCEED) + + path = (char *)H5MM_xfree(path); + } + } while (FindNextFileA(hFind, &fdFile)); + +done: + if (hFind != INVALID_HANDLE_VALUE) + FindClose(hFind); + if (path) + path = (char *)H5MM_xfree(path); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_path() */ +#endif /* H5_HAVE_WIN32_API */ + diff --git a/src/H5PLpkg.h b/src/H5PLpkg.h index e356893..0d1c271 100644 --- a/src/H5PLpkg.h +++ b/src/H5PLpkg.h @@ -11,6 +11,12 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ +/* + * Purpose: This file contains declarations which are visible only within + * the H5PL package. Source files outside the H5PL package should + * include H5PLprivate.h instead. + */ + #if !(defined H5PL_FRIEND || defined H5PL_MODULE) #error "Do not include this file outside the H5PL package!" #endif @@ -27,13 +33,92 @@ /* Package Private Macros */ /**************************/ -#define H5PL_MAX_PATH_NUM 16 +/* Whether to pre-load pathnames for plugin libraries */ +#define H5PL_DEFAULT_PATH H5_DEFAULT_PLUGINDIR + + +/****************************/ +/* Macros for supporting */ +/* both Windows and POSIX */ +/****************************/ + +/*******************/ +/* Windows support */ +/*******************/ +/* + * SPECIAL WINDOWS NOTE + * + * Some of the Win32 API functions expand to fooA or fooW depending on + * whether UNICODE or _UNICODE are defined. You MUST explicitly use + * the A version of the functions to force char * behavior until we + * work out a scheme for proper Windows Unicode support. + * + * If you do not do this, people will be unable to incorporate our + * source code into their own CMake builds if they define UNICODE. + */ +#ifdef H5_HAVE_WIN32_API + + /* The path separator on this platform */ +# define H5PL_PATH_SEPARATOR ";" + + /* Handle for dynamic library */ +# define H5PL_HANDLE HINSTANCE + + /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ +# define H5PL_OPEN_DLIB(S) LoadLibraryExA(S, NULL, LOAD_WITH_ALTERED_SEARCH_PATH) + + /* Get the address of a symbol in dynamic library */ +# define H5PL_GET_LIB_FUNC(H,N) GetProcAddress(H,N) + + /* Close dynamic library */ +# define H5PL_CLOSE_LIB(H) FreeLibrary(H) + + /* Clear error - nothing to do */ +# define H5PL_CLR_ERROR + + /* maximum size for expanding env vars */ +# define H5PL_EXPAND_BUFFER_SIZE 32767 + + typedef const void *(__cdecl *H5PL_get_plugin_info_t)(void); + +#else /* H5_HAVE_WIN32_API */ + + /*****************/ + /* POSIX support */ + /*****************/ + + /* The path separator on this platform */ +# define H5PL_PATH_SEPARATOR ":" + + /* Handle for dynamic library */ +# define H5PL_HANDLE void * + + /* Get a handle to a plugin library. Windows: TEXT macro handles Unicode strings */ +# define H5PL_OPEN_DLIB(S) dlopen(S, RTLD_LAZY) + + /* Get the address of a symbol in dynamic library */ +# define H5PL_GET_LIB_FUNC(H,N) dlsym(H,N) + + /* Close dynamic library */ +# define H5PL_CLOSE_LIB(H) dlclose(H) + + /* Clear error */ +# define H5PL_CLR_ERROR HERROR(H5E_PLUGIN, H5E_CANTGET, "can't dlopen:%s", dlerror()) + + typedef const void *(*H5PL_get_plugin_info_t)(void); +#endif /* H5_HAVE_WIN32_API */ /****************************/ /* Package Private Typedefs */ /****************************/ +/* Data used to search for plugins */ +typedef struct H5PL_search_params_t { + H5PL_type_t type; + int id; +} H5PL_search_params_t; + /*****************************/ /* Package Private Variables */ @@ -44,5 +129,31 @@ /* Package Private Prototypes */ /******************************/ +/* Accessors to global variables and flags */ +H5_DLL herr_t H5PL__get_plugin_control_mask(unsigned int *mask /*out*/); +H5_DLL herr_t H5PL__set_plugin_control_mask(unsigned int mask); + +/* Plugin search and manipulation */ +H5_DLL herr_t H5PL__open(const char *libname, H5PL_type_t type, int id, hbool_t *success /*out*/, const void **plugin_info /*out*/); +H5_DLL herr_t H5PL__close(H5PL_HANDLE handle); + +/* Plugin cache calls */ +H5_DLL herr_t H5PL__create_plugin_cache(void); +H5_DLL herr_t H5PL__close_plugin_cache(hbool_t *already_closed /*out*/); +H5_DLL herr_t H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle); +H5_DLL herr_t H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/); + +/* Plugin search path calls */ +H5_DLL herr_t H5PL__create_path_table(void); +H5_DLL herr_t H5PL__close_path_table(void); +H5_DLL unsigned H5PL__get_num_paths(void); +H5_DLL herr_t H5PL__append_path(const char *path); +H5_DLL herr_t H5PL__prepend_path(const char *path); +H5_DLL herr_t H5PL__replace_path(const char *path, unsigned int index); +H5_DLL herr_t H5PL__insert_path(const char *path, unsigned int index); +H5_DLL herr_t H5PL__remove_path(unsigned int index); +H5_DLL const char *H5PL__get_path(unsigned int index); +H5_DLL herr_t H5PL__find_plugin_in_path_table(const H5PL_search_params_t *search_params, hbool_t *found /*out*/, const void **plugin_info /*out*/); + #endif /* _H5PLpkg_H */ diff --git a/src/H5PLplugin_cache.c b/src/H5PLplugin_cache.c new file mode 100644 index 0000000..d826ba0 --- /dev/null +++ b/src/H5PLplugin_cache.c @@ -0,0 +1,307 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the COPYING file, which can be found at the root of the source code * + * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. * + * If you do not have access to either file, you may request a copy from * + * help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* + * Purpose: Code to implement a plugin cache which stores information + * about plugins which have already been loaded. + * + * The plugin cache is implemented as a dynamic, global array which + * will grow as new plugins are added. The capacity of the cache + * never shrinks since plugins stay in memory once loaded. + * + * Note that this functionality has absolutely nothing to do with + * the metadata or chunk caches. + */ + +/****************/ +/* Module Setup */ +/****************/ + +#include "H5PLmodule.h" /* This source code file is part of the H5PL module */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5PLpkg.h" /* Plugin */ +#include "H5Zprivate.h" /* Filter pipeline */ + + +/****************/ +/* Local Macros */ +/****************/ + +/* Initial capacity of the plugin cache */ +#define H5PL_INITIAL_CACHE_CAPACITY 16 + +/* The amount to add to the capacity when the cache is full */ +#define H5PL_CACHE_CAPACITY_ADD 16 + + +/******************/ +/* Local Typedefs */ +/******************/ + +/* Type for the list of info for opened plugin libraries */ +typedef struct H5PL_plugin_t { + H5PL_type_t type; /* Plugin type */ + int id; /* ID for the plugin */ + H5PL_HANDLE handle; /* Plugin handle */ +} H5PL_plugin_t; + + +/********************/ +/* Local Prototypes */ +/********************/ + +static herr_t H5PL__expand_cache(void); + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Cache for storing opened plugin libraries */ +static H5PL_plugin_t *H5PL_cache_g = NULL; + +/* The number of stored plugins */ +static unsigned int H5PL_num_plugins_g = 0; + +/* The capacity of the plugin cache */ +static unsigned int H5PL_cache_capacity_g = 0; + + + +/*------------------------------------------------------------------------- + * Function: H5PL__create_plugin_cache + * + * Purpose: Create the cache that will store plugins that have already + * been loaded. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__create_plugin_cache(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Allocate memory for the plugin cache */ + H5PL_num_plugins_g = 0; + + H5PL_cache_capacity_g = H5PL_INITIAL_CACHE_CAPACITY; + + if (NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_calloc((size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for plugin cache") + +done: + /* Try to clean up on errors */ + if (FAIL == ret_value) { + if (H5PL_cache_g) + H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g); + H5PL_cache_capacity_g = 0; + } + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__create_plugin_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__close_plugin_cache + * + * Purpose: Close the cache of plugins that have already been loaded, + * closing all the plugins contained inside. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__close_plugin_cache(hbool_t *already_closed /*out*/) +{ + unsigned int u; /* iterator */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE_NOERR + + /* Close opened dynamic libraries */ + if (H5PL_cache_g) { + + /* Close any cached plugins */ + for (u = 0; u < H5PL_num_plugins_g; u++) + H5PL__close((H5PL_cache_g[u]).handle); + + /* Free the cache array */ + H5PL_cache_g = (H5PL_plugin_t *)H5MM_xfree(H5PL_cache_g); + H5PL_num_plugins_g = 0; + H5PL_cache_capacity_g = 0; + + /* Note that actually closed the table (needed by package close call) */ + *already_closed = FALSE; + } + else + *already_closed = TRUE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__close_plugin_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__expand_cache + * + * Purpose: Expand the plugin cache when it's full. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +static herr_t +H5PL__expand_cache(void) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_STATIC + + /* Update the capacity */ + H5PL_cache_capacity_g += H5PL_CACHE_CAPACITY_ADD; + + /* Resize the array */ + if(NULL == (H5PL_cache_g = (H5PL_plugin_t *)H5MM_realloc(H5PL_cache_g, (size_t)H5PL_cache_capacity_g * sizeof(H5PL_plugin_t)))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "allocating additional memory for plugin cache failed") + + /* Initialize the new memory */ + HDmemset(H5PL_cache_g + H5PL_num_plugins_g, 0, (size_t)H5PL_CACHE_CAPACITY_ADD * sizeof(H5PL_plugin_t)); + +done: + /* Set the cache capacity back if there were problems */ + if (FAIL == ret_value) + H5PL_cache_capacity_g -= H5PL_CACHE_CAPACITY_ADD; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__expand_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__add_plugin + * + * Purpose: Add a plugin to the plugin cached. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5PL__add_plugin(H5PL_type_t type, int id, H5PL_HANDLE handle) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Expand the cache if it is too small */ + if (H5PL_num_plugins_g >= H5PL_cache_capacity_g) + if (H5PL__expand_cache() < 0) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't expand plugin cache") + + /* Store the plugin info and bump the # of plugins */ + H5PL_cache_g[H5PL_num_plugins_g].type = type; + H5PL_cache_g[H5PL_num_plugins_g].id = id; + H5PL_cache_g[H5PL_num_plugins_g].handle = handle; + + H5PL_num_plugins_g++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__add_plugin() */ + + +/*------------------------------------------------------------------------- + * Function: H5PL__find_plugin_in_cache + * + * Purpose: Attempts to find a matching plugin from the cache. + * + * The 'found' parameter will be set appropriately. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +/* See the other use of H5PL_GET_LIB_FUNC() for an explanation + * for why we disable -Wpedantic here. + */ +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +herr_t +H5PL__find_plugin_in_cache(const H5PL_search_params_t *search_params, hbool_t *found, const void **plugin_info) +{ + unsigned int u; /* iterator */ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_PACKAGE + + /* Check args - Just assert on package functions */ + HDassert(search_params); + HDassert(found); + HDassert(plugin_info); + + /* Initialize output parameters */ + *found = FALSE; + *plugin_info = NULL; + + /* Loop over all the plugins, looking for one that matches */ + for (u = 0; u < H5PL_num_plugins_g; u++) { + + /* If the plugin type (filter, etc.) and ID match, query the plugin for its info */ + if ((search_params->type == (H5PL_cache_g[u]).type) && (search_params->id == (H5PL_cache_g[u]).id)) { + + H5PL_get_plugin_info_t get_plugin_info_function; + const H5Z_class2_t *filter_info; + + /* Get the "get plugin info" function from the plugin. */ + if (NULL == (get_plugin_info_function = (H5PL_get_plugin_info_t)H5PL_GET_LIB_FUNC((H5PL_cache_g[u]).handle, "H5PLget_plugin_info"))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get function for H5PLget_plugin_info") + + /* Call the "get plugin info" function */ + if (NULL == (filter_info = (const H5Z_class2_t *)(*get_plugin_info_function)())) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "can't get plugin info") + + /* Set output parameters */ + *found = TRUE; + *plugin_info = filter_info; + + /* No need to continue processing */ + break; + + } /* end if */ + + } /* end for */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5PL__find_plugin_in_cache() */ +#pragma GCC diagnostic pop + diff --git a/src/H5PLprivate.h b/src/H5PLprivate.h index 0ab8f8c..bc12e64 100644 --- a/src/H5PLprivate.h +++ b/src/H5PLprivate.h @@ -10,8 +10,8 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Raymond Lu <songyulu@hdfgroup.org> - * 13 February 2013 +/* + * This file contains private information about the H5PL module */ #ifndef _H5PLprivate_H diff --git a/src/H5PLpublic.h b/src/H5PLpublic.h index 9ce1fca..3b36ccd 100644 --- a/src/H5PLpublic.h +++ b/src/H5PLpublic.h @@ -10,8 +10,8 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -/* Programmer: Raymond Lu <songyulu@hdfgroup.org> - * 13 February 2013 +/* + * This file contains public declarations for the H5PL module. */ #ifndef _H5PLpublic_H @@ -24,31 +24,35 @@ /* Public Typedefs */ /*******************/ +/* Special string to indicate no plugin loading. + */ +#define H5PL_NO_PLUGIN "::" + /* Plugin type used by the plugin library */ typedef enum H5PL_type_t { - H5PL_TYPE_ERROR = -1, /*error */ - H5PL_TYPE_FILTER = 0, /*filter */ - H5PL_TYPE_NONE = 1 /*this must be last! */ + H5PL_TYPE_ERROR = -1, /* Error */ + H5PL_TYPE_FILTER = 0, /* Filter */ + H5PL_TYPE_NONE = 1 /* This must be last! */ } H5PL_type_t; /* Common dynamic plugin type flags used by the set/get_loading_state functions */ -#define H5PL_FILTER_PLUGIN 0x0001 -#define H5PL_ALL_PLUGIN 0xFFFF +#define H5PL_FILTER_PLUGIN 0x0001 +#define H5PL_ALL_PLUGIN 0xFFFF #ifdef __cplusplus extern "C" { #endif /* plugin state */ -H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_type); -H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_type/*out*/); -H5_DLL herr_t H5PLappend(const char *plugin_path); -H5_DLL herr_t H5PLprepend(const char *plugin_path); -H5_DLL herr_t H5PLreplace(const char *plugin_path, unsigned int index); -H5_DLL herr_t H5PLinsert(const char *plugin_path, unsigned int index); +H5_DLL herr_t H5PLset_loading_state(unsigned int plugin_control_mask); +H5_DLL herr_t H5PLget_loading_state(unsigned int *plugin_control_mask /*out*/); +H5_DLL herr_t H5PLappend(const char *search_path); +H5_DLL herr_t H5PLprepend(const char *search_path); +H5_DLL herr_t H5PLreplace(const char *search_path, unsigned int index); +H5_DLL herr_t H5PLinsert(const char *search_path, unsigned int index); H5_DLL herr_t H5PLremove(unsigned int index); -H5_DLL ssize_t H5PLget(unsigned int index, char *pathname/*out*/, size_t size); -H5_DLL herr_t H5PLsize(unsigned int *listsize/*out*/); +H5_DLL ssize_t H5PLget(unsigned int index, char *path_buf /*out*/, size_t buf_size); +H5_DLL herr_t H5PLsize(unsigned int *num_paths /*out*/); #ifdef __cplusplus } diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c index 5383aae..6b0d2c0 100644 --- a/src/H5Pfcpl.c +++ b/src/H5Pfcpl.c @@ -1472,6 +1472,9 @@ H5Pset_file_space_page_size(hid_t plist_id, hsize_t fsp_size) if(fsp_size < H5F_FILE_SPACE_PAGE_SIZE_MIN) HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to less than 512") + if(fsp_size > H5F_FILE_SPACE_PAGE_SIZE_MAX) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "cannot set file space page size to more than 1GB") + /* Set the value*/ if(H5P_set(plist, H5F_CRT_FILE_SPACE_PAGE_SIZE_NAME, &fsp_size) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't set file space page size") diff --git a/src/H5Smpio.c b/src/H5Smpio.c index c24c455..46f7a59 100644 --- a/src/H5Smpio.c +++ b/src/H5Smpio.c @@ -33,7 +33,7 @@ #include "H5Oprivate.h" /* Object headers */ #include "H5Pprivate.h" /* Property lists */ #include "H5Spkg.h" /* Dataspaces */ -#include "H5VMprivate.h" /* Vector and array functions */ +#include "H5VMprivate.h" /* Vector and array functions */ #ifdef H5_HAVE_PARALLEL @@ -55,9 +55,42 @@ static herr_t H5S_mpio_span_hyper_type(const H5S_t *space, size_t elmt_size, MPI_Datatype *new_type, int *count, hbool_t *is_derived_type); static herr_t H5S_obtain_datatype(const hsize_t down[], H5S_hyper_span_t* span, const MPI_Datatype *elmt_type, MPI_Datatype *span_type, size_t elmt_size); +static herr_t H5S_mpio_create_large_type (hsize_t, MPI_Aint, MPI_Datatype , MPI_Datatype *); + #define H5S_MPIO_INITIAL_ALLOC_COUNT 256 +#define TWO_GIG_LIMIT 2147483648 + +#ifndef H5S_MAX_MPI_COUNT +#define H5S_MAX_MPI_COUNT 536870911 /* (2^29)-1 */ +#endif + +static hsize_t bigio_count = H5S_MAX_MPI_COUNT; + +/*------------------------------------------------------------------------- + * Function: H5S_mpio_set_bigio_count + * + * Purpose: Allow us to programatically change the switch point + * when we utilize derived datatypes. This is of + * particular interest for allowing nightly testing + * + * Return: the current/previous value of bigio_count. + * + * Programmer: Richard Warren, March 10, 2017 + * + *------------------------------------------------------------------------- + */ +hsize_t +H5S_mpio_set_bigio_count(hsize_t new_count) +{ + hsize_t orig_count = bigio_count; + if ((new_count > 0) && (new_count < TWO_GIG_LIMIT)) { + bigio_count = new_count; + } + return orig_count; +} + /*------------------------------------------------------------------------- * Function: H5S_mpio_all_type @@ -72,6 +105,11 @@ static herr_t H5S_obtain_datatype(const hsize_t down[], H5S_hyper_span_t* span, * *is_derived_type 0 if MPI primitive type, 1 if derived * * Programmer: rky 980813 + * Modifications: + * Mohamad Chaarawi + * Adding support for large datatypes (beyond the limit of a + * 32 bit integer. + * * *------------------------------------------------------------------------- */ @@ -95,11 +133,22 @@ H5S_mpio_all_type(const H5S_t *space, size_t elmt_size, H5_CHECKED_ASSIGN(nelmts, hsize_t, snelmts, hssize_t); total_bytes = (hsize_t)elmt_size * nelmts; - - /* fill in the return values */ - *new_type = MPI_BYTE; - H5_CHECKED_ASSIGN(*count, int, total_bytes, hsize_t); - *is_derived_type = FALSE; + /* Verify that the size can be expressed as a 32 bit integer */ + if(bigio_count >= total_bytes) { + /* fill in the return values */ + *new_type = MPI_BYTE; + H5_CHECKED_ASSIGN(*count, int, total_bytes, hsize_t); + *is_derived_type = FALSE; + } + else { + /* Create a LARGE derived datatype for this transfer */ + if (H5S_mpio_create_large_type (total_bytes, 0, MPI_BYTE, new_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't create a large datatype from the all selection") + } + *count = 1; + *is_derived_type = TRUE; + } done: FUNC_LEAVE_NOAPI(ret_value) @@ -167,27 +216,103 @@ H5S_mpio_create_point_datatype (size_t elmt_size, hsize_t num_points, HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) elmt_type_created = TRUE; + /* Check whether standard or BIGIO processing will be employeed */ + if(bigio_count >= num_points) { #if MPI_VERSION >= 3 - /* Create an MPI datatype for the whole point selection */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block((int)num_points, 1, disp, elmt_type, new_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code) + /* Create an MPI datatype for the whole point selection */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block((int)num_points, 1, disp, elmt_type, new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code) #else - /* Allocate block sizes for MPI datatype call */ - if(NULL == (blocks = (int *)H5MM_malloc(sizeof(int) * num_points))) - HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") + /* Allocate block sizes for MPI datatype call */ + if(NULL == (blocks = (int *)H5MM_malloc(sizeof(int) * num_points))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") - for(u = 0; u < num_points; u++) - blocks[u] = 1; + for(u = 0; u < num_points; u++) + blocks[u] = 1; - /* Create an MPI datatype for the whole point selection */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_points, blocks, disp, elmt_type, new_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_indexed_block failed", mpi_code) + /* Create an MPI datatype for the whole point selection */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)num_points, blocks, disp, elmt_type, new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) #endif - /* Commit MPI datatype for later use */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + /* Commit MPI datatype for later use */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + } + else { + /* use LARGE_DATATYPE:: + * We'll create an hindexed_block type for every 2G point count and then combine + * those and any remaining points into a single large datatype. + */ + int total_types, i; + int remaining_points; + int num_big_types; + hsize_t leftover; + + int *inner_blocks; + MPI_Aint *inner_disps; + MPI_Datatype *inner_types = NULL; + + /* Calculate how many Big MPI datatypes are needed to represent the buffer */ + num_big_types = (int)(num_points/bigio_count); + + leftover = (hsize_t)num_points - (hsize_t)num_big_types * (hsize_t)bigio_count; + H5_CHECKED_ASSIGN(remaining_points, int, leftover, hsize_t); + + total_types = (int)(remaining_points) ? (num_big_types + 1) : num_big_types; + + /* Allocate array if MPI derived types needed */ + if(NULL == (inner_types = (MPI_Datatype *)H5MM_malloc((sizeof(MPI_Datatype) * (size_t)total_types)))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") + + if(NULL == (inner_blocks = (int *)H5MM_malloc(sizeof(int) * (size_t)total_types))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") + + if(NULL == (inner_disps = (MPI_Aint *)H5MM_malloc(sizeof(MPI_Aint) * (size_t)total_types))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTALLOC, FAIL, "can't allocate array of blocks") + + for(i=0 ; i<num_big_types ; i++) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block(bigio_count, + 1, + &disp[i*bigio_count], + elmt_type, + &inner_types[i]))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code); + } + inner_blocks[i] = 1; + inner_disps[i] = 0; + } + + if(remaining_points) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed_block(remaining_points, + 1, + &disp[num_big_types*bigio_count], + elmt_type, + &inner_types[num_big_types]))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed_block failed", mpi_code); + } + inner_blocks[num_big_types] = 1; + inner_disps[num_big_types] = 0; + } + + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_struct(total_types, + inner_blocks, + inner_disps, + inner_types, + new_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct", mpi_code); + } + for(i=0 ; i<total_types ; i++) + MPI_Type_free(&inner_types[i]); + + H5MM_free(inner_types); + H5MM_free(inner_blocks); + H5MM_free(inner_disps); + /* Commit MPI datatype for later use */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + } done: if(elmt_type_created) MPI_Type_free(&elmt_type); @@ -481,7 +606,10 @@ done: * *is_derived_type 0 if MPI primitive type, 1 if derived * * Programmer: rky 980813 - * + * Modifications: + * Mohamad Chaarawi + * Adding support for large datatypes (beyond the limit of a + * 32 bit integer. *------------------------------------------------------------------------- */ static herr_t @@ -636,8 +764,25 @@ H5S_mpio_hyper_type(const H5S_t *space, size_t elmt_size, HDfprintf(H5DEBUG(S), "d[%d].xtent=%Hu \n", i, d[i].xtent); } #endif - if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &inner_type))) + + /* LARGE_DATATYPE:: + * Check if the number of elements to form the inner type fits into a 32 bit integer. + * If yes then just create the innertype with MPI_Type_contiguous. + * Otherwise create a compound datatype by iterating as many times as needed + * for the innertype to be created. + */ + if(bigio_count >= elmt_size) { + /* Use a single MPI datatype that has a 32 bit size */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &inner_type))) HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + else { + /* Create the compound datatype for this operation (> 2GB) */ + if (H5S_mpio_create_large_type (elmt_size, 0, MPI_BYTE, &inner_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't ccreate a large inner datatype in hyper selection") + } + } /******************************************************* * Construct the type by walking the hyperslab dims @@ -645,30 +790,93 @@ H5S_mpio_hyper_type(const H5S_t *space, size_t elmt_size, *******************************************************/ for(i = ((int)rank) - 1; i >= 0; --i) { #ifdef H5S_DEBUG - if(H5DEBUG(S)) - HDfprintf(H5DEBUG(S), "%s: Dimension i=%d \n" - "start=%Hd count=%Hu block=%Hu stride=%Hu, xtent=%Hu max_xtent=%d\n", - FUNC, i, d[i].start, d[i].count, d[i].block, d[i].strid, d[i].xtent, max_xtent[i]); + if(H5DEBUG(S)) + HDfprintf(H5DEBUG(S), "%s: Dimension i=%d \n" + "start=%Hd count=%Hu block=%Hu stride=%Hu, xtent=%Hu max_xtent=%d\n", + FUNC, i, d[i].start, d[i].count, d[i].block, d[i].strid, d[i].xtent, max_xtent[i]); #endif #ifdef H5S_DEBUG - if(H5DEBUG(S)) - HDfprintf(H5DEBUG(S), "%s: i=%d Making vector-type \n", FUNC,i); + if(H5DEBUG(S)) + HDfprintf(H5DEBUG(S), "%s: i=%d Making vector-type \n", FUNC,i); #endif /**************************************** * Build vector type of the selection. ****************************************/ - mpi_code = MPI_Type_vector((int)(d[i].count), /* count */ - (int)(d[i].block), /* blocklength */ - (int)(d[i].strid), /* stride */ - inner_type, /* old type */ - &outer_type); /* new type */ + if (bigio_count >= d[i].count && + bigio_count >= d[i].block && + bigio_count >= d[i].strid) { + + /* All the parameters fit into 32 bit integers so create the vector type normally */ + mpi_code = MPI_Type_vector((int)(d[i].count), /* count */ + (int)(d[i].block), /* blocklength */ + (int)(d[i].strid), /* stride */ + inner_type, /* old type */ + &outer_type); /* new type */ + + MPI_Type_free(&inner_type); + if(mpi_code != MPI_SUCCESS) + HMPI_GOTO_ERROR(FAIL, "couldn't create MPI vector type", mpi_code) + } + else { + /* Things get a bit more complicated and require LARGE_DATATYPE processing + * There are two MPI datatypes that need to be created: + * 1) an internal contiguous block; and + * 2) a collection of elements where an element is a contiguous block(1). + * Remember that the input arguments to the MPI-IO functions use integer + * values to represent element counts. We ARE allowed however, in the + * more recent MPI implementations to use constructed datatypes whereby + * the total number of bytes in a transfer could be : + * (2GB-1)number_of_blocks * the_datatype_extent. + */ + + MPI_Aint stride_in_bytes, inner_extent; + MPI_Datatype block_type; + + /* create a contiguous datatype inner_type x number of BLOCKS. + * Again we need to check that the number of BLOCKS can fit into + * a 32 bit integer */ + if (bigio_count < d[i].block) { + if (H5S_mpio_create_large_type(d[i].block, 0, inner_type, + &block_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't ccreate a large block datatype in hyper selection") + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)d[i].block, + inner_type, + &block_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } - MPI_Type_free(&inner_type); - if(mpi_code != MPI_SUCCESS) - HMPI_GOTO_ERROR(FAIL, "couldn't create MPI vector type", mpi_code) + MPI_Type_extent (inner_type, &inner_extent); + stride_in_bytes = inner_extent * (MPI_Aint)d[i].strid; - /**************************************** + /* If the element count is larger than what a 32 bit integer can hold, + * we call the large type creation function to handle that + */ + if (bigio_count < d[i].count) { + if (H5S_mpio_create_large_type (d[i].count, stride_in_bytes, block_type, + &outer_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't create a large outer datatype in hyper selection") + } + } + /* otherwise a regular create_hvector will do */ + else { + mpi_code = MPI_Type_create_hvector((int)d[i].count, /* count */ + 1, /* blocklength */ + stride_in_bytes, /* stride in bytes*/ + block_type, /* old type */ + &outer_type); /* new type */ + if(MPI_SUCCESS != mpi_code) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) + } + MPI_Type_free(&block_type); + MPI_Type_free(&inner_type); + } + /**************************************** * Then build the dimension type as (start, vector type, xtent). ****************************************/ /* calculate start and extent values of this dimension */ @@ -752,6 +960,10 @@ done: * * Programmer: kyang * + * Modifications: + * Mohamad Chaarawi + * Adding support for large datatypes (beyond the limit of a + * 32 bit integer. *------------------------------------------------------------------------- */ static herr_t @@ -774,8 +986,17 @@ H5S_mpio_span_hyper_type(const H5S_t *space, size_t elmt_size, HDassert(space->select.sel_info.hslab->span_lst->head); /* Create the base type for an element */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &elmt_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + if (bigio_count >= elmt_size) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)elmt_size, MPI_BYTE, &elmt_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + else { + if (H5S_mpio_create_large_type (elmt_size, 0, MPI_BYTE, &elmt_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't create a large element datatype in span_hyper selection") + } + } elmt_type_is_derived = TRUE; /* Compute 'down' sizes for each dimension */ @@ -821,14 +1042,15 @@ static herr_t H5S_obtain_datatype(const hsize_t *down, H5S_hyper_span_t *span, const MPI_Datatype *elmt_type, MPI_Datatype *span_type, size_t elmt_size) { - size_t alloc_count; /* Number of span tree nodes allocated at this level */ - size_t outercount; /* Number of span tree nodes at this level */ + size_t alloc_count = 0; /* Number of span tree nodes allocated at this level */ + size_t outercount = 0; /* Number of span tree nodes at this level */ MPI_Datatype *inner_type = NULL; hbool_t inner_types_freed = FALSE; /* Whether the inner_type MPI datatypes have been freed */ hbool_t span_type_valid = FALSE; /* Whether the span_type MPI datatypes is valid */ + hbool_t large_block = FALSE; /* Wether the block length is larger than 32 bit integer */ int *blocklen = NULL; MPI_Aint *disp = NULL; - H5S_hyper_span_t *tspan; /* Temporary pointer to span tree node */ + H5S_hyper_span_t *tspan = NULL; /* Temporary pointer to span tree node */ int mpi_code; /* MPI return status code */ herr_t ret_value = SUCCEED; /* Return value */ @@ -870,14 +1092,70 @@ H5S_obtain_datatype(const hsize_t *down, H5S_hyper_span_t *span, disp[outercount] = (MPI_Aint)elmt_size * tspan->low; H5_CHECK_OVERFLOW(tspan->nelem, hsize_t, int) blocklen[outercount] = (int)tspan->nelem; - tspan = tspan->next; + + if (bigio_count < blocklen[outercount]) { + large_block = TRUE; /* at least one block type is large, so set this flag to true */ + } + outercount++; } /* end while */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)outercount, blocklen, disp, *elmt_type, span_type))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) - span_type_valid = TRUE; + /* Everything fits into integers, so cast them and use hindexed */ + if (bigio_count >= outercount && large_block == FALSE) { + + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed((int)outercount, blocklen, disp, *elmt_type, span_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + span_type_valid = TRUE; + } + else { /* LARGE_DATATYPE:: Something doesn't fit into a 32 bit integer */ + size_t i; + + for (i=0 ; i<outercount ; i++) { + MPI_Datatype temp_type = MPI_DATATYPE_NULL, outer_type = MPI_DATATYPE_NULL; + /* create the block type from elmt_type while checking the 32 bit int limit */ + if (blocklen[i] > bigio_count) { + if (H5S_mpio_create_large_type (blocklen[i], 0, *elmt_type, &temp_type) < 0) { + HGOTO_ERROR(H5E_DATASPACE, H5E_BADTYPE, FAIL, + "couldn't create a large element datatype in span_hyper selection") + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous((int)blocklen[i], + *elmt_type, + &temp_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + + /* combine the current datatype that is created with this current block type */ + if (0 == i) { /* first iteration, there is no combined datatype yet */ + *span_type = temp_type; + } + else { + int bl[2] = {1,1}; + MPI_Aint ds[2] = {disp[i-1],disp[i]}; + MPI_Datatype dt[2] = {*span_type, temp_type}; + + if (MPI_SUCCESS != (mpi_code = MPI_Type_create_struct (2, /* count */ + bl, /* blocklength */ + ds, /* stride in bytes*/ + dt, /* old type */ + &outer_type))){ /* new type */ + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + } + *span_type = outer_type; + } + + if (outer_type != MPI_DATATYPE_NULL) + MPI_Type_free(&outer_type); + /* temp_type shouldn't be freed here... + * Note that we have simply copied it above (not MPI_Type_dup) + * into the 'span_type' argument of the caller. + * The caller needs to deal with it there! + */ + } + } /* end (LARGE_DATATYPE::) */ + } /* end if */ else { size_t u; /* Local index variable */ @@ -1091,5 +1369,139 @@ H5S_mpio_space_type(const H5S_t *space, size_t elmt_size, MPI_Datatype *new_type done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5S_mpio_space_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5S_mpio_create_large_type + * + * Purpose: Create a large datatype of size larger than what a 32 bit integer + * can hold. + * + * Return: non-negative on success, negative on failure. + * + * *new_type the new datatype created + * + * Programmer: Mohamad Chaarawi + * + *------------------------------------------------------------------------- + */ +static herr_t H5S_mpio_create_large_type (hsize_t num_elements, + MPI_Aint stride_bytes, + MPI_Datatype old_type, + MPI_Datatype *new_type) +{ + int num_big_types; /* num times the 2G datatype will be repeated */ + int remaining_bytes; /* the number of bytes left that can be held in an int value */ + hsize_t leftover; + int block_len[2]; + int mpi_code; /* MPI return code */ + MPI_Datatype inner_type, outer_type, leftover_type, type[2]; + MPI_Aint disp[2], old_extent; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Calculate how many Big MPI datatypes are needed to represent the buffer */ + num_big_types = (int)(num_elements/bigio_count); + leftover = num_elements - num_big_types * (hsize_t)bigio_count; + H5_CHECKED_ASSIGN(remaining_bytes, int, leftover, hsize_t); + + /* Create a contiguous datatype of size equal to the largest + * number that a 32 bit integer can hold x size of old type. + * If the displacement is 0, then the type is contiguous, otherwise + * use type_hvector to create the type with the displacement provided + */ + if (0 == stride_bytes) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(bigio_count, + old_type, + &inner_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector (bigio_count, + 1, + stride_bytes, + old_type, + &inner_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) + } + } + + /* Create a contiguous datatype of the buffer (minus the remaining < 2GB part) + * If a stride is present, use hvector type + */ + if (0 == stride_bytes) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous(num_big_types, + inner_type, + &outer_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector (num_big_types, + 1, + stride_bytes, + inner_type, + &outer_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) + } + } + + MPI_Type_free(&inner_type); + + /* If there is a remaining part create a contiguous/vector datatype and then + * use a struct datatype to encapsulate everything. + */ + if(remaining_bytes) { + if (stride_bytes == 0) { + if(MPI_SUCCESS != (mpi_code = MPI_Type_contiguous (remaining_bytes, + old_type, + &leftover_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_contiguous failed", mpi_code) + } + } + else { + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hvector + ((int)(num_elements - (hsize_t)num_big_types*bigio_count), + 1, + stride_bytes, + old_type, + &leftover_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hvector failed", mpi_code) + } + } + + MPI_Type_extent (old_type, &old_extent); + + /* Set up the arguments for MPI_Type_struct constructor */ + type[0] = outer_type; + type[1] = leftover_type; + block_len[0] = 1; + block_len[1] = 1; + disp[0] = 0; + disp[1] = (old_extent+stride_bytes)*num_big_types*(MPI_Aint)bigio_count; + + if(MPI_SUCCESS != (mpi_code = + MPI_Type_create_struct(2, block_len, disp, type, new_type))) { + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_struct failed", mpi_code) + } + + MPI_Type_free(&outer_type); + MPI_Type_free(&leftover_type); + } + else { + /* There are no remaining bytes so just set the new type to + * the outer type created */ + *new_type = outer_type; + } + + if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(new_type))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5S_mpio_create_large_type */ + #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5VMprivate.h b/src/H5VMprivate.h index 4d71b29..decac7e 100644 --- a/src/H5VMprivate.h +++ b/src/H5VMprivate.h @@ -460,7 +460,11 @@ H5VM_power2up(hsize_t n) { hsize_t ret_value = 1; /* Return value */ - while(ret_value < n) + /* Returns 0 when n exceeds 2^63 */ + if(n >= (hsize_t)1 << ((sizeof(hsize_t) * CHAR_BIT) - 1)) + ret_value = 0; + + while(ret_value && ret_value < n) ret_value <<= 1; return(ret_value); diff --git a/src/H5system.c b/src/H5system.c index 7e25540..a1cdf19 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -1237,3 +1237,49 @@ H5_get_time(void) } /* end H5_get_time() */ +#ifdef H5_HAVE_WIN32_API + +#define H5_WIN32_ENV_VAR_BUFFER_SIZE 32767 + + +/*------------------------------------------------------------------------- + * Function: H5_expand_windows_env_vars() + * + * Purpose: Replaces windows environment variables of the form %foo% + * with user-specific values. + * + * Return: SUCCEED/FAIL + * + *------------------------------------------------------------------------- + */ +herr_t +H5_expand_windows_env_vars(char **env_var) +{ + long n_chars = 0; + char *temp_buf = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT + + /* Allocate buffer for expanded environment variable string */ + if (NULL == (temp_buf = (char *)H5MM_calloc((size_t)H5_WIN32_ENV_VAR_BUFFER_SIZE))) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTALLOC, FAIL, "can't allocate memory for expanded path") + + /* Expand the environment variable string */ + if ((n_chars = ExpandEnvironmentStringsA(*env_var, temp_buf, H5_WIN32_ENV_VAR_BUFFER_SIZE)) > H5_WIN32_ENV_VAR_BUFFER_SIZE) + HGOTO_ERROR(H5E_PLUGIN, H5E_NOSPACE, FAIL, "expanded path is too long") + + if (0 == n_chars) + HGOTO_ERROR(H5E_PLUGIN, H5E_CANTGET, FAIL, "failed to expand path") + + *env_var = (char *)H5MM_xfree(*env_var); + *env_var = temp_buf; + +done: + if (FAIL == ret_value && temp_buf) + temp_buf = (char *)H5MM_xfree(temp_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_expand_windows_env_vars() */ +#endif /* H5_HAVE_WIN32_API */ + diff --git a/src/H5win32defs.h b/src/H5win32defs.h index 97f7179..4522228 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -119,6 +119,7 @@ extern "C" { H5_DLL int c99_snprintf(char* str, size_t size, const char* format, ...); H5_DLL int c99_vsnprintf(char* str, size_t size, const char* format, va_list ap); H5_DLL int Wnanosleep(const struct timespec *req, struct timespec *rem); + H5_DLL herr_t H5_expand_windows_env_vars(char **env_var); /* Round functions only needed for VS2012 and earlier. * They are always built to ensure they don't go stale and diff --git a/src/Makefile.am b/src/Makefile.am index 0b664a7..9a64717 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -97,7 +97,7 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5Pgcpl.c H5Pint.c \ H5Plapl.c H5Plcpl.c H5Pocpl.c H5Pocpypl.c H5Pstrcpl.c H5Ptest.c \ H5PB.c \ - H5PL.c \ + H5PL.c H5PLint.c H5PLpath.c H5PLplugin_cache.c \ H5R.c H5Rdeprec.c \ H5UC.c \ H5RS.c \ diff --git a/test/dsets.c b/test/dsets.c index c29d18e..0ca08e4 100644 --- a/test/dsets.c +++ b/test/dsets.c @@ -51,14 +51,15 @@ const char *FILENAME[] = { "copy_dcpl_newfile",/* 13 */ "partial_chunks", /* 14 */ "layout_extend", /* 15 */ - "zero_chunk", /* 16 */ + "zero_chunk", /* 16 */ "chunk_single", /* 17 */ "swmr_non_latest", /* 18 */ "earray_hdr_fd", /* 19 */ "farray_hdr_fd", /* 20 */ "bt2_hdr_fd", /* 21 */ - "storage_size", /* 22 */ + "storage_size", /* 22 */ "dls_01_strings", /* 23 */ + "power2up", /* 24 */ NULL }; #define FILENAME_BUF_SIZE 1024 @@ -11345,6 +11346,97 @@ error: /*------------------------------------------------------------------------- + * Function: test_power2up + * + * Purpose: Tests that the H5VM_power2up(n) function does not result in an + * infinite loop when input n exceeds 2^63. (HDFFV-10217) + * H5VM_power2up() is used to calculate the next power of 2 for + * a dataset's scaled dimension sizes. + * + * Return: Success: 0 + * Failure: -1 + * + * Programmer: Vailin Choi; June 2017 + * + *------------------------------------------------------------------------- + */ +static herr_t +test_power2up(hid_t fapl) +{ + char filename[FILENAME_BUF_SIZE]; + hid_t fid = -1; /* File ID */ + hid_t dcpl = -1; /* Dataset creation property list */ + hid_t sid = -1; /* Dataspace ID */ + hid_t did = -1; /* Dataset ID */ + hsize_t dims[2]; /* Dataset dimension sizes */ + hsize_t max_dims[2]; /* Maximum dimension sizes */ + hsize_t chunk_dims[2]; /* Chunk dimensions */ + hsize_t ext_dims[2]; /* Extended dimension sizes */ + herr_t status; /* Error status */ + + TESTING("the next power of 2"); + + h5_fixname(FILENAME[24], fapl, filename, sizeof filename); + + /* Create file */ + if((fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) FAIL_STACK_ERROR + + /* Set dims[1] to ((2^63) -1) */ + dims[0] = 0; + dims[1] = ((hsize_t)1 << ((sizeof(hsize_t) * CHAR_BIT) -1)) - 1; + max_dims[0] = max_dims[1] = H5S_UNLIMITED; + sid = H5Screate_simple(2, dims, max_dims); + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + TEST_ERROR + + /* Set chunk size */ + chunk_dims[0] = chunk_dims[1] = 1; + if(H5Pset_chunk(dcpl, 2, chunk_dims) < 0) + TEST_ERROR + + /* Create chunked dataset */ + if((did = H5Dcreate2(fid, "dset", H5T_NATIVE_INT64, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) + TEST_ERROR + + ext_dims[0] = 1; + ext_dims[1] = dims[1] + 5; + + /* Extend to (2^63)+ */ + H5E_BEGIN_TRY { + status = H5Dset_extent(did, ext_dims); + } H5E_END_TRY; + if(status >= 0) + TEST_ERROR + + /* Closing */ + if(H5Dclose(did) < 0) + TEST_ERROR + if(H5Sclose(sid) < 0) + TEST_ERROR + if(H5Pclose(dcpl) < 0) + TEST_ERROR + if(H5Fclose(fid) < 0) + TEST_ERROR + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Dclose(did); + H5Sclose(sid); + H5Pclose(dcpl); + H5Fclose(fid); + } H5E_END_TRY; + return -1; +} /* end test_power2up() */ + + +/*------------------------------------------------------------------------- * Function: test_scatter * * Purpose: Tests H5Dscatter with a variety of different selections @@ -12928,6 +13020,7 @@ main(void) nerrors += (test_large_chunk_shrink(my_fapl) < 0 ? 1 : 0); nerrors += (test_zero_dim_dset(my_fapl) < 0 ? 1 : 0); nerrors += (test_storage_size(my_fapl) < 0 ? 1 : 0); + nerrors += (test_power2up(my_fapl) < 0 ? 1 : 0); nerrors += (test_swmr_non_latest(envval, my_fapl) < 0 ? 1 : 0); nerrors += (test_earray_hdr_fd(envval, my_fapl) < 0 ? 1 : 0); diff --git a/test/fheap.c b/test/fheap.c index 4be6cb9..6c3a8ac 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -16377,6 +16377,16 @@ main(void) unsigned nerrors = 0; /* Cumulative error count */ unsigned num_pb_fs = 1; /* The number of settings to test for page buffering and file space handling */ int ExpressMode; /* Express testing level */ + const char *envval; /* Environment variable */ + hbool_t contig_addr_vfd; /* Whether VFD used has a contigous address space */ + + /* Don't run this test using certain file drivers */ + envval = HDgetenv("HDF5_DRIVER"); + if(envval == NULL) + envval = "nomatch"; + + /* Current VFD that does not support contigous address space */ + contig_addr_vfd = (hbool_t)(HDstrcmp(envval, "split") && HDstrcmp(envval, "multi")); /* Reset library */ h5_reset(); @@ -16428,6 +16438,12 @@ main(void) shared_wobj_g[u] = (unsigned char)u; for(v = 0; v < num_pb_fs; v++) { + /* Skip test when: + a) multi/split drivers and + b) persisting free-space or using paged aggregation strategy + because the library will fail file creation (temporary) for the above conditions */ + if(!contig_addr_vfd && v) + break; if((fcpl = H5Pcopy(def_fcpl)) < 0) TEST_ERROR diff --git a/test/plugin.c b/test/plugin.c index 1254e9a..ea199f6 100644 --- a/test/plugin.c +++ b/test/plugin.c @@ -10,9 +10,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu - * 13 February 2013 - * * Purpose: Tests the plugin module (H5PL) */ @@ -660,13 +657,10 @@ error: /*------------------------------------------------------------------------- * Function: test_groups_with_filters * - * Purpose: Tests opening group with dynamically loaded filters - * - * Return: Success: 0 - * Failure: -1 + * Purpose: Tests opening group with dynamically loaded filters * - * Programmer: Raymond Lu - * 1 April 2013 + * Return: Success: 0 + * Failure: -1 * *------------------------------------------------------------------------- */ @@ -710,261 +704,388 @@ error: /*------------------------------------------------------------------------- - * Function: test_filter_path_apis + * Function: test_path_api_calls * - * Purpose: Tests accessing the path table for dynamically loaded filters + * Purpose: Tests the H5PL API calls that manipulate the plugin search + * paths. * - * Return: Success: 0 - * Failure: -1 + * Return: SUCCEED/FAIL + * *------------------------------------------------------------------------- */ static herr_t -test_filter_path_apis(void) +test_path_api_calls(void) { - herr_t ret_value = -1; - unsigned int i; - unsigned int ndx; + unsigned int n_starting_paths; + unsigned int u; + unsigned int n_paths; herr_t ret; - ssize_t pathlen = -1; - char pathname[256]; - char tempname[256]; + ssize_t path_len = -1; + char path[256]; + char temp_name[256]; HDputs("Testing access to the filter path table"); - if(H5Zfilter_avail(H5Z_FILTER_DYNLIB1) != TRUE) TEST_ERROR + if(H5Zfilter_avail(H5Z_FILTER_DYNLIB1) != TRUE) + TEST_ERROR + + /* Set the number of paths to create for this test. + * + * This should be set high enough to ensure that at least one array + * expansion will take place. See H5PLpath.c for details. + */ + n_starting_paths = 42; + + /* Check that initialization is correct */ + TESTING(" initialize"); - H5PLsize(&ndx); + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != 2) + TEST_ERROR + + PASSED(); + /****************/ + /* H5PLremove() */ + /****************/ + + /* Remove all the current paths */ TESTING(" remove"); - /* Remove all existing paths*/ - for(i=ndx; i > 0; i--) - if(H5PLremove(i-1) < 0) { - HDfprintf(stderr," at %d: %s\n", i, pathname); + + /* Get the current size */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + + /* Remove all existing paths */ + for(u = n_paths; u > 0; u--) + if(H5PLremove(u-1) < 0) { + HDfprintf(stderr," at %u: %s\n", u, path); TEST_ERROR - } /* end if */ + } + /* Verify the table is empty */ - H5PLsize(&ndx); - if(ndx > 0) TEST_ERROR + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths > 0) + TEST_ERROR + PASSED(); - TESTING(" remove (exceed min)"); - /* Exceed the min path removal */ + + TESTING(" remove (index 0 in empty table)"); + + /* Try to remove index zero in an empty list (SHOULD FAIL) */ H5E_BEGIN_TRY { ret = H5PLremove(0); } H5E_END_TRY - if(ret >= 0) TEST_ERROR + if(ret >= 0) + TEST_ERROR + PASSED(); + + /****************/ + /* H5PLappend() */ + /****************/ + TESTING(" append"); - /* Create multiple paths to fill table */ - for(i=0; i < H5PL_MAX_PATH_NUM; i++) { - HDsprintf(pathname, "a_path_%d", i); - if(H5PLappend(pathname) < 0) { - HDfprintf(stderr," at %d: %s\n", i, pathname); + + /* Add a bunch of paths to the path table */ + for(u = 0; u < n_starting_paths; u++) { + HDsprintf(path, "a_path_%u", u); + if(H5PLappend(path) < 0) { + HDfprintf(stderr," at %u: %s\n", u, path); TEST_ERROR } } - /* Verify the table is full */ - H5PLsize(&ndx); - if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR - PASSED(); - TESTING(" append (exceed)"); - /* Exceed the max path append */ - H5E_BEGIN_TRY { - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM); - ret = H5PLappend(pathname); - } H5E_END_TRY - if(ret >= 0) TEST_ERROR PASSED(); - TESTING(" remove (exceed max)"); - /* Exceed the max path removal */ + + /**********************/ + /* H5PLremove() again */ + /**********************/ + + TESTING(" remove (index too high)"); + + /* Try to remove a path where the index is beyond the table capacity (SHOULD FAIL) */ H5E_BEGIN_TRY { - ret = H5PLremove(H5PL_MAX_PATH_NUM); + ret = H5PLremove(n_starting_paths); } H5E_END_TRY - if(ret >= 0) TEST_ERROR + + if(ret >= 0) + TEST_ERROR + PASSED(); + + /*************/ + /* H5PLget() */ + /*************/ + TESTING(" get (path name)"); - if((pathlen = H5PLget(0, NULL, 0)) <= 0) { + + /* Get the path length by passing in NULL */ + if((path_len = H5PLget(0, NULL, 0)) <= 0) { HDfprintf(stderr," get path 0 length failed\n"); TEST_ERROR } - if(pathlen != 8) TEST_ERROR + if(path_len != 8) + TEST_ERROR - if((pathlen = H5PLget(0, pathname, 256)) <= 0) { - HDfprintf(stderr," get 0 len: %d : %s\n", pathlen, pathname); + /* Get the path */ + if((path_len = H5PLget(0, path, 256)) <= 0) { + HDfprintf(stderr," get 0 len: %u : %s\n", path_len, path); TEST_ERROR } - if(HDstrcmp(pathname, "a_path_0") != 0) { - HDfprintf(stderr," get 0: %s\n", pathname); + if(HDstrcmp(path, "a_path_0") != 0) { + HDfprintf(stderr," get 0: %s\n", path); TEST_ERROR } + PASSED(); - TESTING(" get (bounds)"); - if((pathlen = H5PLget(1, pathname, 256)) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_1") != 0) { - HDfprintf(stderr," get 1: %s\n", pathname); + + TESTING(" get (high and low indices)"); + + /* Get path at index 1 */ + if((path_len = H5PLget(1, path, 256)) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_1") != 0) { + HDfprintf(stderr," get 1: %s\n", path); TEST_ERROR } - if((pathlen = H5PLget(H5PL_MAX_PATH_NUM - 1, pathname, 256)) <= 0) TEST_ERROR - HDsprintf(tempname, "a_path_%d", H5PL_MAX_PATH_NUM - 1); - if(HDstrcmp(pathname, tempname) != 0) { - HDfprintf(stderr," get %d: %s\n", H5PL_MAX_PATH_NUM - 1, pathname); + + /* Get path at the last index */ + if((path_len = H5PLget(n_starting_paths - 1, path, 256)) <= 0) + TEST_ERROR + HDsprintf(temp_name, "a_path_%u", n_starting_paths - 1); + if(HDstrcmp(path, temp_name) != 0) { + HDfprintf(stderr," get %u: %s\n", n_starting_paths - 1, path); TEST_ERROR } + PASSED(); - TESTING(" get (bounds exceed)"); + + TESTING(" get (index too high)"); + + /* Get path at the last + 1 index (SHOULD FAIL) */ H5E_BEGIN_TRY { - pathlen = H5PLget(H5PL_MAX_PATH_NUM, NULL, 0); + path_len = H5PLget(n_starting_paths, NULL, 0); } H5E_END_TRY - if(pathlen > 0) TEST_ERROR + if(path_len > 0) + TEST_ERROR + PASSED(); - TESTING(" remove (verify for prepend)"); - /* Remove one path*/ - if(H5PLremove(8) < 0) TEST_ERROR + + /*****************/ + /* H5PLprepend() */ + /*****************/ + + /* We'll remove a path at an arbitrary index and then + * prepend a new path. + */ + + TESTING(" remove (arbitrary index 1)"); + + /* Remove one path */ + if(H5PLremove(8) < 0) + TEST_ERROR /* Verify that the entries were moved */ - if((pathlen = H5PLget(8, pathname, 256)) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_9") != 0) { - HDfprintf(stderr," get 8: %s\n", pathname); + if((path_len = H5PLget(8, path, 256)) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_9") != 0) { + HDfprintf(stderr," get 8: %s\n", path); TEST_ERROR } + + /* Verify the table shrank */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths - 1) + TEST_ERROR + PASSED(); - /* Verify the table is not full */ - H5PLsize(&ndx); - if (ndx != H5PL_MAX_PATH_NUM - 1) TEST_ERROR TESTING(" prepend"); - /* Prepend one path*/ - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 1); - if(H5PLprepend(pathname) < 0) { - HDfprintf(stderr," prepend %d: %s\n", H5PL_MAX_PATH_NUM + 1, pathname); + + /* Prepend one path */ + HDsprintf(path, "a_path_%d", n_starting_paths + 1); + if(H5PLprepend(path) < 0) { + HDfprintf(stderr," prepend %u: %s\n", n_starting_paths + 1, path); TEST_ERROR } - /* Verify the table is full */ - H5PLsize(&ndx); - if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR + /* Verify the table increased */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths) + TEST_ERROR /* Verify that the entries were moved */ - if(H5PLget(8, pathname, 256) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_7") != 0) { - HDfprintf(stderr," get 8: %s\n", pathname); + if(H5PLget(8, path, 256) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_7") != 0) { + HDfprintf(stderr," get 8: %s\n", path); TEST_ERROR } - if(H5PLget(0, pathname, 256) <= 0) TEST_ERROR - HDsprintf(tempname, "a_path_%d", H5PL_MAX_PATH_NUM + 1); - if(HDstrcmp(pathname, tempname) != 0) { - HDfprintf(stderr," get 0: %s\n", pathname); + + /* Verify that the path was inserted at index zero */ + if(H5PLget(0, path, 256) <= 0) + TEST_ERROR + HDsprintf(temp_name, "a_path_%d", n_starting_paths + 1); + if(HDstrcmp(path, temp_name) != 0) { + HDfprintf(stderr," get 0: %s\n", path); TEST_ERROR } - PASSED(); - TESTING(" prepend (exceed)"); - /* Exceed the max path prepend */ - H5E_BEGIN_TRY { - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 2); - ret = H5PLprepend(pathname); - } H5E_END_TRY - if(ret >= 0) TEST_ERROR PASSED(); + + /*****************/ + /* H5PLreplace() */ + /*****************/ + TESTING(" replace"); - /* Replace one path*/ - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 4); - if(H5PLreplace(pathname, 1) < 0) { - HDfprintf(stderr," replace 1: %s\n", pathname); + + /* Replace one path at index 1 */ + HDsprintf(path, "a_path_%u", n_starting_paths + 4); + if(H5PLreplace(path, 1) < 0) { + HDfprintf(stderr," replace 1: %s\n", path); TEST_ERROR } - /* Verify the table is full */ - H5PLsize(&ndx); - if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR + /* Verify the table size remained the same */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths) + TEST_ERROR + + /* Verify that the entries were not moved by + * inspecting the paths at indices +/- 1. + */ - /* Verify that the entries were not moved */ - if(H5PLget(0, pathname, 256) <= 0) TEST_ERROR - HDsprintf(tempname, "a_path_%d", H5PL_MAX_PATH_NUM + 1); - if(HDstrcmp(pathname, tempname) != 0) { - HDfprintf(stderr," get 0: %s\n", pathname); + /* Check path at index 0 */ + if(H5PLget(0, path, 256) <= 0) + TEST_ERROR + HDsprintf(temp_name, "a_path_%u", n_starting_paths + 1); + if(HDstrcmp(path, temp_name) != 0) { + HDfprintf(stderr," get 0: %s\n", path); TEST_ERROR } - if(H5PLget(2, pathname, 256) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_1") != 0) { - HDfprintf(stderr," get 2: %s\n", pathname); + + /* Check path at index 2 */ + if(H5PLget(2, path, 256) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_1") != 0) { + HDfprintf(stderr," get 2: %s\n", path); TEST_ERROR } + PASSED(); - TESTING(" remove (verify for insert)"); - /* Remove one path*/ - if(H5PLremove(4) < 0) TEST_ERROR + + /****************/ + /* H5PLinsert() */ + /****************/ + + /* We'll remove a path at an arbitrary index and then + * insert a new path. + */ + + TESTING(" remove (arbitrary index 2)"); + + /* Remove one path */ + if(H5PLremove(4) < 0) + TEST_ERROR /* Verify that the entries were moved */ - if(H5PLget(4, pathname, 256) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_4") != 0) { - HDfprintf(stderr," get 4: %s\n", pathname); + if(H5PLget(4, path, 256) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_4") != 0) { + HDfprintf(stderr," get 4: %s\n", path); TEST_ERROR } + + /* Verify the table size */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths - 1) + TEST_ERROR PASSED(); - /* Verify the table is not full */ - H5PLsize(&ndx); - if(ndx != 15) TEST_ERROR TESTING(" insert"); - /* Insert one path*/ - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 5); - if(H5PLinsert(pathname, 3) < 0) { - HDfprintf(stderr," insert 3: %s\n", pathname); + + /* Insert one path at index 3*/ + HDsprintf(path, "a_path_%d", n_starting_paths + 5); + if(H5PLinsert(path, 3) < 0) { + HDfprintf(stderr," insert 3: %s\n", path); TEST_ERROR } /* Verify that the entries were moved */ - if(H5PLget(4, pathname, 256) <= 0) TEST_ERROR - if(HDstrcmp(pathname, "a_path_2") != 0) { - HDfprintf(stderr," get 4: %s\n", pathname); + if(H5PLget(4, path, 256) <= 0) + TEST_ERROR + if(HDstrcmp(path, "a_path_2") != 0) { + HDfprintf(stderr," get 4: %s\n", path); TEST_ERROR } + + /* Verify the table size increased */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths != n_starting_paths) + TEST_ERROR + PASSED(); - /* Verify the table is full */ - H5PLsize(&ndx); - if(ndx != H5PL_MAX_PATH_NUM) TEST_ERROR - TESTING(" insert (exceed)"); - /* Exceed the max path insert */ - H5E_BEGIN_TRY { - HDsprintf(pathname, "a_path_%d", H5PL_MAX_PATH_NUM + 6); - ret = H5PLinsert(pathname, 12); - } H5E_END_TRY - if(ret >= 0) TEST_ERROR + /****************/ + /* H5PLremove() */ + /****************/ + + /* Remove all the current paths */ + TESTING(" remove (all)"); + + /* Get the current size */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + + /* Remove all existing paths */ + for(u = n_paths; u > 0; u--) + if(H5PLremove(u-1) < 0) { + HDfprintf(stderr," at %u: %s\n", u, path); + TEST_ERROR + } + + /* Verify the table is empty */ + if(H5PLsize(&n_paths) < 0) + TEST_ERROR + if(n_paths > 0) + TEST_ERROR PASSED(); - ret_value = 0; + + return SUCCEED; error: - return ret_value; -} + return FAIL; +} /* end test_path_api_calls() */ /*------------------------------------------------------------------------- * Function: main * - * Purpose: Tests the plugin module (H5PL) + * Purpose: Tests the plugin module (H5PL) * - * Return: Success: exit(EXIT_SUCCESS) - * - * Failure: exit(EXIT_FAILURE) - * - * Programmer: Raymond Lu - * 14 March 2013 + * Return: EXIT_SUCCESS/EXIT_FAILURE * *------------------------------------------------------------------------- */ @@ -984,37 +1105,44 @@ main(void) /* Testing setup */ h5_reset(); - fapl = h5_fileaccess(); + + if ((fapl = h5_fileaccess()) < 0) + TEST_ERROR /* Turn off the chunk cache, so all the chunks are immediately written to disk */ - if(H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0) TEST_ERROR + if (H5Pget_cache(fapl, &mdc_nelmts, &rdcc_nelmts, &rdcc_nbytes, &rdcc_w0) < 0) + TEST_ERROR rdcc_nbytes = 0; - if(H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0) TEST_ERROR + if (H5Pset_cache(fapl, mdc_nelmts, rdcc_nelmts, rdcc_nbytes, rdcc_w0) < 0) + TEST_ERROR /* Copy the file access property list */ - if((fapl2 = H5Pcopy(fapl)) < 0) TEST_ERROR + if ((fapl2 = H5Pcopy(fapl)) < 0) + TEST_ERROR /* Set the "use the latest version of the format" bounds for creating objects in the file */ - if(H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) TEST_ERROR + if (H5Pset_libver_bounds(fapl2, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0) + TEST_ERROR - h5_fixname(FILENAME[0], fapl, filename, sizeof filename); + h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); /* Test with old & new format groups */ - for(new_format = FALSE; new_format <= TRUE; new_format++) { + for (new_format = FALSE; new_format <= TRUE; new_format++) { hid_t my_fapl; /* Set the FAPL for the type of format */ - if(new_format) { + if (new_format) { HDputs("\nTesting with new file format:"); my_fapl = fapl2; - } /* end if */ + } else { HDputs("Testing with old file format:"); my_fapl = fapl; - } /* end else */ + } /* Create the file for this test */ - if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) TEST_ERROR + if ((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) + TEST_ERROR /* Test dynamically loaded filters for chunked dataset */ nerrors += (test_filters_for_datasets(file) < 0 ? 1 : 0); @@ -1022,12 +1150,15 @@ main(void) /* Test dynamically loaded filters for groups */ nerrors += (test_filters_for_groups(file) < 0 ? 1 : 0); - if(H5Fclose(file) < 0) TEST_ERROR + if (H5Fclose(file) < 0) + TEST_ERROR } /* end for */ /* Close FAPL */ - if(H5Pclose(fapl2) < 0) TEST_ERROR - if(H5Pclose(fapl) < 0) TEST_ERROR + if (H5Pclose(fapl2) < 0) + TEST_ERROR + if (H5Pclose(fapl) < 0) + TEST_ERROR /* Restore the default error handler (set in h5_reset()) */ h5_restore_err(); @@ -1036,10 +1167,12 @@ main(void) /* Close the library so that all loaded plugin libraries are unloaded */ h5_reset(); - fapl = h5_fileaccess(); + if ((fapl = h5_fileaccess()) < 0) + TEST_ERROR /* Reopen the file for testing data reading */ - if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR + if ((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) + TEST_ERROR /* Read the data with filters */ nerrors += (test_read_with_filters(file) < 0 ? 1 : 0); @@ -1052,29 +1185,33 @@ main(void) /* Close the library so that all loaded plugin libraries are unloaded */ h5_reset(); - fapl = h5_fileaccess(); + if ((fapl = h5_fileaccess()) < 0) + TEST_ERROR /* Reopen the file for testing data reading */ - if((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR + if ((file = H5Fopen(filename, H5F_ACC_RDONLY, fapl)) < 0) + TEST_ERROR /* Read the data with disabled filters */ nerrors += (test_noread_with_filters(file) < 0 ? 1 : 0); - if(H5Fclose(file) < 0) TEST_ERROR + if (H5Fclose(file) < 0) + TEST_ERROR /* Test the APIs for access to the filter plugin path table */ - nerrors += (test_filter_path_apis() < 0 ? 1 : 0); + nerrors += (test_path_api_calls() < 0 ? 1 : 0); - if(nerrors) TEST_ERROR + if (nerrors) + TEST_ERROR HDprintf("All plugin tests passed.\n"); h5_cleanup(FILENAME, fapl); - return 0; + HDexit(EXIT_SUCCESS); error: nerrors = MAX(1, nerrors); HDprintf("***** %d PLUGIN TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S"); - return 1; -} + HDexit(EXIT_FAILURE); +} /* end main() */ diff --git a/test/tfile.c b/test/tfile.c index 533bb24..027ad62 100644 --- a/test/tfile.c +++ b/test/tfile.c @@ -109,6 +109,7 @@ #define TEST_THRESHOLD10 10 /* Free space section threshold */ #define FSP_SIZE_DEF 4096 /* File space page size default */ #define FSP_SIZE512 512 /* File space page size */ +#define FSP_SIZE1G 1024*1024*1024 /* File space page size */ /* Declaration for test_libver_macros2() */ #define FILE6 "tfile6.h5" /* Test file */ @@ -3581,6 +3582,9 @@ test_filespace_info(const char *env_h5_drvr) * Setting value less than 512 will return an error; * --setting file space page size to 0 * --setting file space page size to 511 + * + * File space page size has a maximum size of 1 gigabyte. + * Setting value greater than 1 gigabyte will return an error. */ /* Create file creation property list template */ fcpl = H5Pcreate(H5P_FILE_CREATE); @@ -3598,6 +3602,12 @@ test_filespace_info(const char *env_h5_drvr) } H5E_END_TRY; VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); + /* Setting to 1GB+1: should fail */ + H5E_BEGIN_TRY { + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE1G+1); + } H5E_END_TRY; + VERIFY(ret, FAIL, "H5Pset_file_space_page_size"); + /* Setting to 512: should succeed */ ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE512); CHECK(ret, FAIL, "H5Pset_file_space_page_size"); @@ -3605,6 +3615,13 @@ test_filespace_info(const char *env_h5_drvr) CHECK(ret, FAIL, "H5Pget_file_space_page_size"); VERIFY(fsp_size, FSP_SIZE512, "H5Pget_file_space_page_size"); + /* Setting to 1GB: should succeed */ + ret = H5Pset_file_space_page_size(fcpl, FSP_SIZE1G); + CHECK(ret, FAIL, "H5Pset_file_space_page_size"); + ret = H5Pget_file_space_page_size(fcpl, &fsp_size); + CHECK(ret, FAIL, "H5Pget_file_space_page_size"); + VERIFY(fsp_size, FSP_SIZE1G, "H5Pget_file_space_page_size"); + /* Close property list */ H5Pclose(fcpl); diff --git a/test/tmisc.c b/test/tmisc.c index 22ae558..708ca9b 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -30,7 +30,6 @@ #include "testhdf5.h" #include "H5srcdir.h" #include "H5Dpkg.h" /* Datasets */ -#include "H5Iprivate.h" /* IDs, can be removed when H5I_REFERENCE is gone */ /* Definitions for misc. test #1 */ #define MISC1_FILE "tmisc1.h5" @@ -179,17 +178,12 @@ typedef struct #define MISC13_GROUP1_NAME "Group1" #define MISC13_GROUP2_NAME "Group2" #define MISC13_DTYPE_NAME "Datatype" -#define MISC13_RANK 2 -#define MISC13_DIM1 20 -#define MISC13_DIM2 30 +#define MISC13_RANK 1 +#define MISC13_DIM1 600 #define MISC13_CHUNK_DIM1 10 -#define MISC13_CHUNK_DIM2 15 #define MISC13_USERBLOCK_SIZE 512 #define MISC13_COPY_BUF_SIZE 4096 -unsigned m13_data[MISC13_DIM1][MISC13_DIM2]; /* Data to write to dataset */ -unsigned m13_rdata[MISC13_DIM1][MISC13_DIM2]; /* Data read from dataset */ - /* Definitions for misc. test #14 */ #define MISC14_FILE "tmisc14.h5" #define MISC14_DSET1_NAME "Dataset1" @@ -2084,38 +2078,36 @@ test_misc12(void) /* Various routines for misc. 13 test */ static void -init_data(void) +misc13_init_data(unsigned *original_data) { - unsigned u,v; /* Local index variables */ + unsigned u; - for(u=0; u<MISC13_DIM1; u++) - for(v=0; v<MISC13_DIM2; v++) - m13_data[u][v]=(u*MISC13_DIM2)+v; + for(u = 0; u < MISC13_DIM1; u++) + original_data[u] = u; } -static int -verify_data(void) +static hbool_t +misc13_verify_data_match(const unsigned *original_data, const unsigned *read_data) { - unsigned u,v; /* Local index variables */ + unsigned u; - for(u=0; u<MISC13_DIM1; u++) - for(v=0; v<MISC13_DIM2; v++) - if(m13_data[u][v]!=m13_rdata[u][v]) - return(-1); - return(0); + for(u = 0; u < MISC13_DIM1; u++) + if(original_data[u] != read_data[u]) + return FALSE; + + return TRUE; } static void -create_dataset(hid_t loc_id, const char *name, hid_t dcpl) +misc13_create_dataset(hid_t loc_id, const char *name, hid_t dcpl, const unsigned *data) { - hid_t dsid; /* Dataset ID */ - hid_t sid; /* Dataspace ID */ - hsize_t dims[MISC13_RANK]; /* Dataset dimensions */ - herr_t ret; /* Generic return value */ + hid_t dsid = -1; /* Dataset ID */ + hid_t sid = -1; /* Dataspace ID */ + hsize_t dims[MISC13_RANK]; /* Dataset dimensions */ + herr_t ret; /* Generic return value */ /* Create dataspace for use with dataset */ dims[0] = MISC13_DIM1; - dims[1] = MISC13_DIM2; sid = H5Screate_simple(MISC13_RANK, dims, NULL); CHECK(sid, FAIL, "H5Screate_simple"); @@ -2124,7 +2116,7 @@ create_dataset(hid_t loc_id, const char *name, hid_t dcpl) CHECK(dsid, FAIL, "H5Dcreate2"); /* Write some data to dataset */ - ret = H5Dwrite(dsid, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, m13_data); + ret = H5Dwrite(dsid, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, data); CHECK(ret, FAIL, "H5Dwrite"); /* Close the contiguous dataset */ @@ -2134,64 +2126,74 @@ create_dataset(hid_t loc_id, const char *name, hid_t dcpl) /* Close the dataspace */ ret = H5Sclose(sid); CHECK(ret, FAIL, "H5Sclose"); -} + +} /* end misc13_create_dataset() */ static void -verify_dataset(hid_t loc_id, const char *name) +misc13_verify_dataset(hid_t loc_id, const char *name, const unsigned *data) { - hid_t dsid; /* Dataset ID */ - herr_t ret; /* Generic return value */ + unsigned *read_data = NULL; /* Data to write to dataset */ + hid_t dsid = -1; /* Dataset ID */ + herr_t ret; /* Generic return value */ + + /* Create a data buffer for the dataset read */ + read_data = (unsigned *)HDcalloc(MISC13_DIM1, sizeof(unsigned)); + CHECK(read_data, NULL, "HDcalloc"); /* Open the contiguous dataset in the root group */ dsid = H5Dopen2(loc_id, name, H5P_DEFAULT); CHECK(dsid, FAIL, "H5Dopen2"); /* Read the data */ - ret = H5Dread(dsid, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, m13_rdata); + ret = H5Dread(dsid, H5T_NATIVE_UINT, H5S_ALL, H5S_ALL, H5P_DEFAULT, read_data); CHECK(ret, FAIL, "H5Dread"); - /* Verify that the data is correct */ - ret=verify_data(); - CHECK(ret, FAIL, "verify_data"); + /* Verify that the data are correct */ + ret = misc13_verify_data_match(data, read_data); + CHECK(ret, FAIL, "misc13_verify_data_match"); /* Close the contiguous dataset */ ret = H5Dclose(dsid); CHECK(ret, FAIL, "H5Dclose"); -} + + /* Free the dataset read buffer */ + HDfree(read_data); + +} /* end misc13_verify_dataset() */ static void -create_hdf_file(const char *name) +misc13_create_hdf_file(const char *name, const unsigned *data) { - hid_t fid; /* File ID */ - hid_t gid,gid2; /* Group IDs */ - hid_t tid; /* Datatype ID */ - hid_t dcpl; /* Dataset creation property list ID */ - hsize_t chunk_dims[MISC13_RANK]; /* Chunk dimensions */ - herr_t ret; /* Generic return value */ + hid_t fid = -1; /* File ID */ + hid_t gid1 = -1; /* Group ID (level 1) */ + hid_t gid2 = -1; /* Group ID (level 2) */ + hid_t tid = -1; /* Datatype ID */ + hid_t dcplid = -1; /* Dataset creation property list ID */ + hsize_t chunk_dims[MISC13_RANK]; /* Chunk dimensions */ + herr_t ret; /* Generic return value */ /* Create file */ - fid=H5Fcreate(name, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); + fid = H5Fcreate(name, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fcreate"); /* Create DCPL for use with datasets */ - dcpl = H5Pcreate(H5P_DATASET_CREATE); - CHECK(dcpl, FAIL, "H5Pcreate"); + dcplid = H5Pcreate(H5P_DATASET_CREATE); + CHECK(dcplid, FAIL, "H5Pcreate"); /* Set the DCPL to be chunked */ - ret = H5Pset_layout(dcpl, H5D_CHUNKED); + ret = H5Pset_layout(dcplid, H5D_CHUNKED); CHECK(ret, FAIL, "H5Pset_layout"); /* Use chunked storage for this DCPL */ chunk_dims[0] = MISC13_CHUNK_DIM1; - chunk_dims[1] = MISC13_CHUNK_DIM2; - ret = H5Pset_chunk(dcpl, MISC13_RANK, chunk_dims); + ret = H5Pset_chunk(dcplid, MISC13_RANK, chunk_dims); CHECK(ret, FAIL, "H5Pset_chunk"); /* Create contiguous dataset in root group */ - create_dataset(fid, MISC13_DSET1_NAME, H5P_DEFAULT); + misc13_create_dataset(fid, MISC13_DSET1_NAME, H5P_DEFAULT, data); /* Create chunked dataset in root group */ - create_dataset(fid, MISC13_DSET2_NAME, dcpl); + misc13_create_dataset(fid, MISC13_DSET2_NAME, dcplid, data); /* Create a datatype to commit to the file */ tid = H5Tcopy(H5T_NATIVE_INT); @@ -2206,11 +2208,11 @@ create_hdf_file(const char *name) CHECK(ret, FAIL, "H5Tclose"); /* Create a group in the root group */ - gid = H5Gcreate2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); - CHECK(gid, FAIL, "H5Gcreate2"); + gid1 = H5Gcreate2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + CHECK(gid1, FAIL, "H5Gcreate2"); /* Create another group in the new group */ - gid2 = H5Gcreate2(gid, MISC13_GROUP2_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + gid2 = H5Gcreate2(gid1, MISC13_GROUP2_NAME, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(gid2, FAIL, "H5Gcreate2"); /* Close the second group */ @@ -2218,17 +2220,17 @@ create_hdf_file(const char *name) CHECK(ret, FAIL, "H5Gclose"); /* Create contiguous dataset in new group */ - create_dataset(gid, MISC13_DSET1_NAME, H5P_DEFAULT); + misc13_create_dataset(gid1, MISC13_DSET1_NAME, H5P_DEFAULT, data); /* Create chunked dataset in new group */ - create_dataset(gid, MISC13_DSET2_NAME, dcpl); + misc13_create_dataset(gid1, MISC13_DSET2_NAME, dcplid, data); /* Create a datatype to commit to the new group */ tid = H5Tcopy(H5T_NATIVE_INT); CHECK(tid, FAIL, "H5Tcopy"); /* Create a named datatype in the new group */ - ret = H5Tcommit2(gid, MISC13_DTYPE_NAME, tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + ret = H5Tcommit2(gid1, MISC13_DTYPE_NAME, tid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); CHECK(ret, FAIL, "H5Tcommit2"); /* Close named datatype */ @@ -2236,25 +2238,26 @@ create_hdf_file(const char *name) CHECK(ret, FAIL, "H5Tclose"); /* Close the first group */ - ret = H5Gclose(gid); + ret = H5Gclose(gid1); CHECK(ret, FAIL, "H5Gclose"); /* Close the DCPL */ - ret = H5Pclose(dcpl); + ret = H5Pclose(dcplid); CHECK(ret, FAIL, "H5Pclose"); /* Close the file */ ret = H5Fclose(fid); - HDassert(ret >= 0); CHECK(ret, FAIL, "H5Fclose"); -} + +} /* end misc13_create_hdf_file() */ static void -insert_user_block(const char *old_name, const char *new_name,const char *str,size_t size) +misc13_insert_user_block(const char *old_name, const char *new_name, const char *str, size_t size) { - FILE *new_fp, *old_fp; /* Pointers to new & old files */ - void *user_block; /* Pointer to user block to write to file */ - void *copy_buf; /* Pointer to buffer for copying data */ + FILE *new_fp = NULL; /* Pointers to new & old files */ + FILE *old_fp = NULL; + void *user_block = NULL; /* Pointer to user block to write to file */ + void *copy_buf = NULL; /* Pointer to buffer for copying data */ size_t written; /* Amount of data written to new file */ size_t read_in; /* Amount of data read in from old file */ int ret; /* Generic status value */ @@ -2264,10 +2267,10 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz CHECK(user_block, NULL, "HDcalloc"); /* Copy in the user block data */ - HDmemcpy(user_block,str,strlen(str)); + HDmemcpy(user_block, str, strlen(str)); /* Open the new file */ - new_fp=HDfopen(new_name,"wb"); + new_fp = HDfopen(new_name,"wb"); CHECK(new_fp, NULL, "HDfopen"); /* Write the user block to the new file */ @@ -2275,7 +2278,7 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz VERIFY(written, size, "HDfwrite"); /* Open the old file */ - old_fp=HDfopen(old_name,"rb"); + old_fp = HDfopen(old_name,"rb"); CHECK(old_fp, NULL, "HDfopen"); /* Allocate space for the copy buffer */ @@ -2287,14 +2290,14 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz /* Write the data to the new file */ written = HDfwrite(copy_buf, (size_t)1, read_in, new_fp); VERIFY(written, read_in, "HDfwrite"); - } /* end while */ + } /* Close the old file */ - ret=HDfclose(old_fp); + ret = HDfclose(old_fp); VERIFY(ret, 0, "HDfclose"); /* Close the new file */ - ret=HDfclose(new_fp); + ret = HDfclose(new_fp); VERIFY(ret, 0, "HDfclose"); /* Free the copy buffer */ @@ -2302,81 +2305,84 @@ insert_user_block(const char *old_name, const char *new_name,const char *str,siz /* Free the user block */ HDfree(user_block); -} + +} /* end misc13_insert_user_block() */ static void -verify_file(const char *name, hsize_t blk_size, unsigned check_new_data) +misc13_verify_file(const char *name, const unsigned *data, hsize_t userblock_size, + hbool_t check_for_new_dataset) { - hid_t fid; /* File ID */ - hid_t gid,gid2; /* Group IDs */ - hid_t tid; /* Datatype ID */ - hid_t fcpl; /* File creation property list ID */ - hsize_t userblock; /* Userblock size retrieved from FCPL */ - herr_t ret; /* Generic return value */ + hid_t fid = -1; /* File ID */ + hid_t gid1 = -1; /* Group IDs */ + hid_t gid2 = -1; /* Group IDs */ + hid_t tid = -1; /* Datatype ID */ + hid_t fcplid = -1; /* File creation property list ID */ + hsize_t ub_size_out; /* Userblock size retrieved from FCPL */ + herr_t ret; /* Generic return value */ /* Open the file */ - fid=H5Fopen(name, H5F_ACC_RDONLY, H5P_DEFAULT); + fid = H5Fopen(name, H5F_ACC_RDONLY, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); /* Get the file's FCPL */ - fcpl=H5Fget_create_plist(fid); - CHECK(fcpl, FAIL, "H5Fget_create_plist"); + fcplid = H5Fget_create_plist(fid); + CHECK(fcplid, FAIL, "H5Fget_create_plist"); /* Get the user block size for the file */ - ret=H5Pget_userblock(fcpl,&userblock); + ret = H5Pget_userblock(fcplid, &ub_size_out); CHECK(ret, FAIL, "H5Pget_userblock"); /* Check the userblock size */ - VERIFY(userblock, blk_size, "H5Pget_userblock"); + VERIFY(userblock_size, ub_size_out, "H5Pget_userblock"); /* Close the FCPL */ - ret = H5Pclose(fcpl); + ret = H5Pclose(fcplid); CHECK(ret, FAIL, "H5Pclose"); /* Verify the contiguous dataset in the root group */ - verify_dataset(fid,MISC13_DSET1_NAME); + misc13_verify_dataset(fid, MISC13_DSET1_NAME, data); /* Verify the chunked dataset in the root group */ - verify_dataset(fid,MISC13_DSET2_NAME); + misc13_verify_dataset(fid, MISC13_DSET2_NAME, data); /* Verify the "new" contiguous dataset in the root group, if asked */ - if(check_new_data) - verify_dataset(fid,MISC13_DSET3_NAME); + if(check_for_new_dataset) + misc13_verify_dataset(fid, MISC13_DSET3_NAME, data); /* Open the named datatype in the root group */ tid = H5Topen2(fid, MISC13_DTYPE_NAME, H5P_DEFAULT); CHECK(tid, FAIL, "H5Topen2"); /* Verify the type is correct */ - VERIFY(H5Tequal(tid,H5T_NATIVE_INT), TRUE, "H5Tequal"); + VERIFY(H5Tequal(tid, H5T_NATIVE_INT), TRUE, "H5Tequal"); /* Close named datatype */ - ret=H5Tclose(tid); + ret = H5Tclose(tid); CHECK(ret, FAIL, "H5Tclose"); /* Open the first group */ - gid = H5Gopen2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT); - CHECK(gid, FAIL, "H5Gopen2"); + gid1 = H5Gopen2(fid, MISC13_GROUP1_NAME, H5P_DEFAULT); + CHECK(gid1, FAIL, "H5Gopen2"); /* Verify the contiguous dataset in the first group */ - verify_dataset(gid,MISC13_DSET1_NAME); + misc13_verify_dataset(gid1, MISC13_DSET1_NAME, data); /* Verify the chunked dataset in the first group */ - verify_dataset(gid,MISC13_DSET2_NAME); + misc13_verify_dataset(gid1, MISC13_DSET2_NAME, data); /* Open the named datatype in the first group */ - tid = H5Topen2(gid,MISC13_DTYPE_NAME, H5P_DEFAULT); + tid = H5Topen2(gid1, MISC13_DTYPE_NAME, H5P_DEFAULT); CHECK(tid, FAIL, "H5Topen2"); /* Verify the type is correct */ - VERIFY(H5Tequal(tid,H5T_NATIVE_INT), TRUE, "H5Tequal"); + VERIFY(H5Tequal(tid, H5T_NATIVE_INT), TRUE, "H5Tequal"); /* Close named datatype */ - ret=H5Tclose(tid); + ret = H5Tclose(tid); CHECK(ret, FAIL, "H5Tclose"); /* Open the second group */ - gid2 = H5Gopen2(gid, MISC13_GROUP2_NAME, H5P_DEFAULT); + gid2 = H5Gopen2(gid1, MISC13_GROUP2_NAME, H5P_DEFAULT); CHECK(gid2, FAIL, "H5Gopen2"); /* Close the second group */ @@ -2384,31 +2390,33 @@ verify_file(const char *name, hsize_t blk_size, unsigned check_new_data) CHECK(ret, FAIL, "H5Gclose"); /* Close the first group */ - ret = H5Gclose(gid); + ret = H5Gclose(gid1); CHECK(ret, FAIL, "H5Gclose"); /* Close the file */ ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); -} + +} /* end misc13_verify_file() */ static void -add_to_new_file(const char *name) +misc13_add_to_new_file(const char *name, const unsigned *data) { - hid_t fid; /* File ID */ + hid_t fid = -1; /* File ID */ herr_t ret; /* Generic return value */ /* Open the file */ - fid=H5Fopen(name, H5F_ACC_RDWR, H5P_DEFAULT); + fid = H5Fopen(name, H5F_ACC_RDWR, H5P_DEFAULT); CHECK(fid, FAIL, "H5Fopen"); /* Create new contiguous dataset in root group */ - create_dataset(fid, MISC13_DSET3_NAME, H5P_DEFAULT); + misc13_create_dataset(fid, MISC13_DSET3_NAME, H5P_DEFAULT, data); /* Close the file */ ret = H5Fclose(fid); CHECK(ret, FAIL, "H5Fclose"); -} + +} /* end misc13_add_to_new_file() */ /**************************************************************** ** @@ -2419,26 +2427,44 @@ add_to_new_file(const char *name) static void test_misc13(void) { + unsigned *data = NULL; /* Data to write to dataset */ + hsize_t userblock_size; /* Correct size of userblock */ + hbool_t check_for_new_dataset; /* Whether to check for the post-userblock-creation dataset */ + + /* Create a data buffer for the datasets */ + data = (unsigned *)HDcalloc(MISC13_DIM1, sizeof(unsigned)); + CHECK(data, NULL, "HDcalloc"); + /* Initialize data to write */ - init_data(); + misc13_init_data(data); /* Create first file, with no user block */ - create_hdf_file(MISC13_FILE_1); + misc13_create_hdf_file(MISC13_FILE_1, data); /* Verify file contents are correct */ - verify_file(MISC13_FILE_1,(hsize_t)0,0); + userblock_size = 0; + check_for_new_dataset = FALSE; + misc13_verify_file(MISC13_FILE_1, data, userblock_size, check_for_new_dataset); /* Create a new file by inserting a user block in front of the first file */ - insert_user_block(MISC13_FILE_1, MISC13_FILE_2, "Test String", (size_t)MISC13_USERBLOCK_SIZE); + misc13_insert_user_block(MISC13_FILE_1, MISC13_FILE_2, "Test String", (size_t)MISC13_USERBLOCK_SIZE); /* Verify file contents are still correct */ - verify_file(MISC13_FILE_2,(hsize_t)MISC13_USERBLOCK_SIZE,0); + userblock_size = MISC13_USERBLOCK_SIZE; + check_for_new_dataset = FALSE; + misc13_verify_file(MISC13_FILE_2, data, userblock_size, check_for_new_dataset); /* Make certain we can modify the new file */ - add_to_new_file(MISC13_FILE_2); + misc13_add_to_new_file(MISC13_FILE_2, data); /* Verify file contents are still correct */ - verify_file(MISC13_FILE_2,(hsize_t)MISC13_USERBLOCK_SIZE,1); + userblock_size = MISC13_USERBLOCK_SIZE; + check_for_new_dataset = TRUE; + misc13_verify_file(MISC13_FILE_2, data, userblock_size, check_for_new_dataset); + + /* Free the dataset buffer */ + HDfree(data); + } /* end test_misc13() */ /**************************************************************** @@ -3448,34 +3474,6 @@ test_misc19(void) HDfree(vfd_cls); -/* Check H5I operations on references */ - - /* Reference IDs are not used by the library so there's no - * way of testing if incr/decr, etc. work. Instead, just - * do a quick smoke check to ensure that a couple of basic - * calls return sane values. - * - * H5I_REFERENCE has been declared deprecated with a tentative - * removal version of HDF5 1.12.0. - * - * Delete this entire block when H5I_REFERENCE no longer exists. - * - * The H5Iprivate.h header was included to support H5I_nmembers() - * so that can also probably be removed as well. - */ -{ - htri_t tf; /* Boolean generic return */ - int64_t num_members; /* Number of members in type */ - - tf = H5Itype_exists(H5I_REFERENCE); - VERIFY(tf, TRUE, "H5Itype_exists"); - - num_members = 999; - num_members = H5I_nmembers(H5I_REFERENCE); - VERIFY(num_members, 0, "H5Inmembers"); - -} /* end block */ - } /* end test_misc19() */ /**************************************************************** diff --git a/testpar/CMakeLists.txt b/testpar/CMakeLists.txt index 298d326..e994b65 100644 --- a/testpar/CMakeLists.txt +++ b/testpar/CMakeLists.txt @@ -43,6 +43,7 @@ ENDMACRO (ADD_H5P_EXE file) set (H5P_TESTS t_mpi + t_bigio t_cache t_pflush1 t_pflush2 diff --git a/testpar/Makefile.am b/testpar/Makefile.am index 90bdd04..b0fe0cd 100644 --- a/testpar/Makefile.am +++ b/testpar/Makefile.am @@ -23,7 +23,7 @@ AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test # Test programs. These are our main targets. # -TEST_PROG_PARA=t_mpi testphdf5 t_cache t_cache_image t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame t_filters_parallel +TEST_PROG_PARA=t_mpi t_bigio testphdf5 t_cache t_cache_image t_pflush1 t_pflush2 t_pshutdown t_prestart t_init_term t_shapesame t_filters_parallel check_PROGRAMS = $(TEST_PROG_PARA) diff --git a/testpar/t_bigio.c b/testpar/t_bigio.c new file mode 100644 index 0000000..a4a1323 --- /dev/null +++ b/testpar/t_bigio.c @@ -0,0 +1,2153 @@ + +#include "hdf5.h" +#include "testphdf5.h" +#include "H5Dprivate.h" /* For Chunk tests */ + +// int TestVerbosity = VERBO_LO; /* Default Verbosity is Low */ + +/* Constants definitions */ +#define MAX_ERR_REPORT 10 /* Maximum number of errors reported */ + +/* Define some handy debugging shorthands, routines, ... */ +/* debugging tools */ + +#define MAINPROCESS (!mpi_rank) /* define process 0 as main process */ + +/* Constants definitions */ +#define RANK 2 + +#define IN_ORDER 1 +#define OUT_OF_ORDER 2 + +#define DATASET1 "DSET1" +#define DATASET2 "DSET2" +#define DATASET3 "DSET3" +#define DATASET4 "DSET4" +#define DATASET5 "DSET5" +#define DXFER_COLLECTIVE_IO 0x1 /* Collective IO*/ +#define DXFER_INDEPENDENT_IO 0x2 /* Independent IO collectively */ +#define DXFER_BIGCOUNT 536870916 + +#define HYPER 1 +#define POINT 2 +#define ALL 3 + +/* Dataset data type. Int's can be easily octo dumped. */ +typedef hsize_t B_DATATYPE; + +int facc_type = FACC_MPIO; /*Test file access type */ +int dxfer_coll_type = DXFER_COLLECTIVE_IO; +size_t bigcount = DXFER_BIGCOUNT; +char filename[20] = "bigio_test.h5"; +int nerrors = 0; +int mpi_size, mpi_rank; + +hsize_t space_dim1 = SPACE_DIM1 * 256; // 4096 +hsize_t space_dim2 = SPACE_DIM2; + +static void coll_chunktest(const char* filename, int chunk_factor, int select_factor, + int api_option, int file_selection, int mem_selection, int mode); +hid_t create_faccess_plist(MPI_Comm comm, MPI_Info info, int l_facc_type); + +/* + * Setup the coordinates for point selection. + */ +static void +set_coords(hsize_t start[], + hsize_t count[], + hsize_t stride[], + hsize_t block[], + size_t num_points, + hsize_t coords[], + int order) +{ + hsize_t i,j, k = 0, m ,n, s1 ,s2; + + if(OUT_OF_ORDER == order) + k = (num_points * RANK) - 1; + else if(IN_ORDER == order) + k = 0; + + s1 = start[0]; + s2 = start[1]; + + for(i = 0 ; i < count[0]; i++) + for(j = 0 ; j < count[1]; j++) + for(m = 0 ; m < block[0]; m++) + for(n = 0 ; n < block[1]; n++) + if(OUT_OF_ORDER == order) { + coords[k--] = s2 + (stride[1] * j) + n; + coords[k--] = s1 + (stride[0] * i) + m; + } + else if(IN_ORDER == order) { + coords[k++] = s1 + stride[0] * i + m; + coords[k++] = s2 + stride[1] * j + n; + } +} + +/* + * Fill the dataset with trivial data for testing. + * Assume dimension rank is 2 and data is stored contiguous. + */ +static void +fill_datasets(hsize_t start[], hsize_t block[], B_DATATYPE * dataset) +{ + B_DATATYPE *dataptr = dataset; + hsize_t i, j; + + /* put some trivial data in the data_array */ + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + *dataptr = (B_DATATYPE)((i+start[0])*100 + (j+start[1]+1)); + dataptr++; + } + } +} + +/* + * Setup the coordinates for point selection. + */ +void point_set(hsize_t start[], + hsize_t count[], + hsize_t stride[], + hsize_t block[], + size_t num_points, + hsize_t coords[], + int order) +{ + hsize_t i,j, k = 0, m ,n, s1 ,s2; + + HDcompile_assert(RANK == 2); + + if(OUT_OF_ORDER == order) + k = (num_points * RANK) - 1; + else if(IN_ORDER == order) + k = 0; + + s1 = start[0]; + s2 = start[1]; + + for(i = 0 ; i < count[0]; i++) + for(j = 0 ; j < count[1]; j++) + for(m = 0 ; m < block[0]; m++) + for(n = 0 ; n < block[1]; n++) + if(OUT_OF_ORDER == order) { + coords[k--] = s2 + (stride[1] * j) + n; + coords[k--] = s1 + (stride[0] * i) + m; + } + else if(IN_ORDER == order) { + coords[k++] = s1 + stride[0] * i + m; + coords[k++] = s2 + stride[1] * j + n; + } + + if(VERBOSE_MED) { + printf("start[]=(%lu, %lu), count[]=(%lu, %lu), stride[]=(%lu, %lu), block[]=(%lu, %lu), total datapoints=%lu\n", + (unsigned long)start[0], (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1], + (unsigned long)stride[0], (unsigned long)stride[1], (unsigned long)block[0], (unsigned long)block[1], + (unsigned long)(block[0] * block[1] * count[0] * count[1])); + k = 0; + for(i = 0; i < num_points ; i++) { + printf("(%d, %d)\n", (int)coords[k], (int)coords[k + 1]); + k += 2; + } + } +} + +/* + * Print the content of the dataset. + */ +static void +dataset_print(hsize_t start[], hsize_t block[], B_DATATYPE * dataset) +{ + B_DATATYPE *dataptr = dataset; + hsize_t i, j; + + /* print the column heading */ + printf("%-8s", "Cols:"); + for (j=0; j < block[1]; j++){ + printf("%3lu ", (unsigned long)(start[1]+j)); + } + printf("\n"); + + /* print the slab data */ + for (i=0; i < block[0]; i++){ + printf("Row %2lu: ", (unsigned long)(i+start[0])); + for (j=0; j < block[1]; j++){ + printf("%llu ", *dataptr++); + } + printf("\n"); + } +} + + +/* + * Print the content of the dataset. + */ +static int +verify_data(hsize_t start[], hsize_t count[], hsize_t stride[], hsize_t block[], B_DATATYPE *dataset, B_DATATYPE *original) +{ + hsize_t i, j; + int vrfyerrs; + + /* print it if VERBOSE_MED */ + if(VERBOSE_MED) { + printf("verify_data dumping:::\n"); + printf("start(%lu, %lu), count(%lu, %lu), stride(%lu, %lu), block(%lu, %lu)\n", + (unsigned long)start[0], (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1], + (unsigned long)stride[0], (unsigned long)stride[1], (unsigned long)block[0], (unsigned long)block[1]); + printf("original values:\n"); + dataset_print(start, block, original); + printf("compared values:\n"); + dataset_print(start, block, dataset); + } + + vrfyerrs = 0; + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(*dataset != *original){ + if(vrfyerrs++ < MAX_ERR_REPORT || VERBOSE_MED){ + printf("Dataset Verify failed at [%lu][%lu](row %lu, col %lu): expect %llu, got %llu\n", + (unsigned long)i, (unsigned long)j, + (unsigned long)(i+start[0]), (unsigned long)(j+start[1]), + *(original), *(dataset)); + } + dataset++; + original++; + } + } + } + if(vrfyerrs > MAX_ERR_REPORT && !VERBOSE_MED) + printf("[more errors ...]\n"); + if(vrfyerrs) + printf("%d errors found in verify_data\n", vrfyerrs); + return(vrfyerrs); +} + +/* Set up the selection */ +static void +ccslab_set(int mpi_rank, + int mpi_size, + hsize_t start[], + hsize_t count[], + hsize_t stride[], + hsize_t block[], + int mode) +{ + + switch (mode){ + + case BYROW_CONT: + /* Each process takes a slabs of rows. */ + block[0] = 1; + block[1] = 1; + stride[0] = 1; + stride[1] = 1; + count[0] = space_dim1; + count[1] = space_dim2; + start[0] = mpi_rank*count[0]; + start[1] = 0; + + break; + + case BYROW_DISCONT: + /* Each process takes several disjoint blocks. */ + block[0] = 1; + block[1] = 1; + stride[0] = 3; + stride[1] = 3; + count[0] = space_dim1/(stride[0]*block[0]); + count[1] = (space_dim2)/(stride[1]*block[1]); + start[0] = space_dim1*mpi_rank; + start[1] = 0; + + break; + + case BYROW_SELECTNONE: + /* Each process takes a slabs of rows, there are + no selections for the last process. */ + block[0] = 1; + block[1] = 1; + stride[0] = 1; + stride[1] = 1; + count[0] = ((mpi_rank >= MAX(1,(mpi_size-2)))?0:space_dim1); + count[1] = space_dim2; + start[0] = mpi_rank*count[0]; + start[1] = 0; + + break; + + case BYROW_SELECTUNBALANCE: + /* The first one-third of the number of processes only + select top half of the domain, The rest will select the bottom + half of the domain. */ + + block[0] = 1; + count[0] = 2; + stride[0] = space_dim1*mpi_size/4+1; + block[1] = space_dim2; + count[1] = 1; + start[1] = 0; + stride[1] = 1; + if((mpi_rank *3)<(mpi_size*2)) start[0] = mpi_rank; + else start[0] = 1 + space_dim1*mpi_size/2 + (mpi_rank-2*mpi_size/3); + break; + + case BYROW_SELECTINCHUNK: + /* Each process will only select one chunk */ + + block[0] = 1; + count[0] = 1; + start[0] = mpi_rank*space_dim1; + stride[0]= 1; + block[1] = space_dim2; + count[1] = 1; + stride[1]= 1; + start[1] = 0; + + break; + + default: + /* Unknown mode. Set it to cover the whole dataset. */ + block[0] = space_dim1*mpi_size; + block[1] = space_dim2; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = 0; + + break; + } + if (VERBOSE_MED){ + printf("start[]=(%lu,%lu), count[]=(%lu,%lu), stride[]=(%lu,%lu), block[]=(%lu,%lu), total datapoints=%lu\n", + (unsigned long)start[0], (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1], + (unsigned long)stride[0], (unsigned long)stride[1], (unsigned long)block[0], (unsigned long)block[1], + (unsigned long)(block[0]*block[1]*count[0]*count[1])); + } +} + + +/* + * Fill the dataset with trivial data for testing. + * Assume dimension rank is 2. + */ +static void +ccdataset_fill(hsize_t start[], + hsize_t stride[], + hsize_t count[], + hsize_t block[], + DATATYPE * dataset, + int mem_selection) +{ + DATATYPE *dataptr = dataset; + DATATYPE *tmptr; + hsize_t i,j,k1,k2,k=0; + /* put some trivial data in the data_array */ + tmptr = dataptr; + + /* assign the disjoint block (two-dimensional)data array value + through the pointer */ + + for (k1 = 0; k1 < count[0]; k1++) { + for(i = 0; i < block[0]; i++) { + for(k2 = 0; k2 < count[1]; k2++) { + for(j = 0;j < block[1]; j++) { + + if (ALL != mem_selection) { + dataptr = tmptr + ((start[0]+k1*stride[0]+i)*space_dim2+ + start[1]+k2*stride[1]+j); + } + else { + dataptr = tmptr + k; + k++; + } + + *dataptr = (DATATYPE)(k1+k2+i+j); + } + } + } + } +} + +/* + * Print the first block of the content of the dataset. + */ +static void +ccdataset_print(hsize_t start[], + hsize_t block[], + DATATYPE * dataset) + +{ + DATATYPE *dataptr = dataset; + hsize_t i, j; + + /* print the column heading */ + printf("Print only the first block of the dataset\n"); + printf("%-8s", "Cols:"); + for (j=0; j < block[1]; j++){ + printf("%3lu ", (unsigned long)(start[1]+j)); + } + printf("\n"); + + /* print the slab data */ + for (i=0; i < block[0]; i++){ + printf("Row %2lu: ", (unsigned long)(i+start[0])); + for (j=0; j < block[1]; j++){ + printf("%03d ", *dataptr++); + } + printf("\n"); + } +} + +/* + * Print the content of the dataset. + */ +static int +ccdataset_vrfy(hsize_t start[], + hsize_t count[], + hsize_t stride[], + hsize_t block[], + DATATYPE *dataset, + DATATYPE *original, + int mem_selection) +{ + hsize_t i, j,k1,k2,k=0; + int vrfyerrs; + DATATYPE *dataptr,*oriptr; + + /* print it if VERBOSE_MED */ + if (VERBOSE_MED) { + printf("dataset_vrfy dumping:::\n"); + printf("start(%lu, %lu), count(%lu, %lu), stride(%lu, %lu), block(%lu, %lu)\n", + (unsigned long)start[0], (unsigned long)start[1], (unsigned long)count[0], (unsigned long)count[1], + (unsigned long)stride[0], (unsigned long)stride[1], (unsigned long)block[0], (unsigned long)block[1]); + printf("original values:\n"); + ccdataset_print(start, block, original); + printf("compared values:\n"); + ccdataset_print(start, block, dataset); + } + + vrfyerrs = 0; + + for (k1=0;k1<count[0];k1++) { + for(i=0;i<block[0];i++) { + for(k2=0; k2<count[1];k2++) { + for(j=0;j<block[1];j++) { + if (ALL != mem_selection) { + dataptr = dataset + ((start[0]+k1*stride[0]+i)*space_dim2+ + start[1]+k2*stride[1]+j); + oriptr = original + ((start[0]+k1*stride[0]+i)*space_dim2+ + start[1]+k2*stride[1]+j); + } + else { + dataptr = dataset + k; + oriptr = original + k; + k++; + } + if (*dataptr != *oriptr){ + if (vrfyerrs++ < MAX_ERR_REPORT || VERBOSE_MED){ + printf("Dataset Verify failed at [%lu][%lu]: expect %d, got %d\n", + (unsigned long)i, (unsigned long)j, + *(oriptr), *(dataptr)); + } + } + } + } + } + } + if (vrfyerrs > MAX_ERR_REPORT && !VERBOSE_MED) + printf("[more errors ...]\n"); + if (vrfyerrs) + printf("%d errors found in ccdataset_vrfy\n", vrfyerrs); + return(vrfyerrs); +} + +/* + * Example of using the parallel HDF5 library to create two datasets + * in one HDF5 file with collective parallel access support. + * The Datasets are of sizes (number-of-mpi-processes x dim0) x dim1. + * Each process controls only a slab of size dim0 x dim1 within each + * dataset. [Note: not so yet. Datasets are of sizes dim0xdim1 and + * each process controls a hyperslab within.] + */ + +static void +dataset_big_write(void) +{ + + hid_t xfer_plist; /* Dataset transfer properties list */ + hid_t sid; /* Dataspace ID */ + hid_t file_dataspace; /* File dataspace ID */ + hid_t mem_dataspace; /* memory dataspace ID */ + hid_t dataset; + hid_t datatype; /* Datatype ID */ + hsize_t dims[RANK]; /* dataset dim sizes */ + hsize_t start[RANK]; /* for hyperslab setting */ + hsize_t count[RANK], stride[RANK]; /* for hyperslab setting */ + hsize_t block[RANK]; /* for hyperslab setting */ + hsize_t *coords = NULL; + int i; + herr_t ret; /* Generic return value */ + hid_t fid; /* HDF5 file ID */ + hid_t acc_tpl; /* File access templates */ + hsize_t h; + size_t num_points; + B_DATATYPE * wdata; + + + /* allocate memory for data buffer */ + wdata = (B_DATATYPE *)malloc(bigcount*sizeof(B_DATATYPE)); + VRFY((wdata != NULL), "wdata malloc succeeded"); + + /* setup file access template */ + acc_tpl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "H5P_FILE_ACCESS"); + H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* create the file collectively */ + fid = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, acc_tpl); + VRFY((fid >= 0), "H5Fcreate succeeded"); + + /* Release file-access template */ + ret = H5Pclose(acc_tpl); + VRFY((ret >= 0), ""); + + + /* Each process takes a slabs of rows. */ + printf("\nTesting Dataset1 write by ROW\n"); + /* Create a large dataset */ + dims[0] = bigcount; + dims[1] = mpi_size; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET1, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + block[0] = dims[0]/mpi_size; + block[1] = dims[1]; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = mpi_rank*block[0]; + start[1] = 0; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, block, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* fill the local slab with some trivial data */ + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + dataset_print(start, block, wdata); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* write data collectively */ + MESG("writeAll by Row"); + { + int j,k =0; + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", wdata[k]); + k++; + } + } + } + printf("\n"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + + + /* Each process takes a slabs of cols. */ + printf("\nTesting Dataset2 write by COL\n"); + /* Create a large dataset */ + dims[0] = bigcount; + dims[1] = mpi_size; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET2, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + block[0] = dims[0]; + block[1] = dims[1]/mpi_size; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = mpi_rank*block[1]; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, block, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* fill the local slab with some trivial data */ + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + dataset_print(start, block, wdata); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* write data collectively */ + MESG("writeAll by Col"); + { + int j,k =0; + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", wdata[k]); + k++; + } + } + } + printf("\n"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + + + /* ALL selection */ + printf("\nTesting Dataset3 write select ALL proc 0, NONE others\n"); + /* Create a large dataset */ + dims[0] = bigcount; + dims[1] = 1; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET3, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + if(MAINPROCESS) { + ret = H5Sselect_all(file_dataspace); + VRFY((ret >= 0), "H5Sset_all succeeded"); + } + else { + ret = H5Sselect_none(file_dataspace); + VRFY((ret >= 0), "H5Sset_none succeeded"); + } + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, dims, NULL); + VRFY((mem_dataspace >= 0), ""); + if(!MAINPROCESS) { + ret = H5Sselect_none(mem_dataspace); + VRFY((ret >= 0), "H5Sset_none succeeded"); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* fill the local slab with some trivial data */ + fill_datasets(start, dims, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + /* write data collectively */ + MESG("writeAll by process 0"); + { + int j,k =0; + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", wdata[k]); + k++; + } + } + } + printf("\n"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + /* Point selection */ + printf("\nTesting Dataset4 write point selection\n"); + /* Create a large dataset */ + dims[0] = bigcount; + dims[1] = mpi_size * 4; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET4, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + block[0] = dims[0]/2; + block[1] = 2; + stride[0] = dims[0]/2; + stride[1] = 2; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = dims[1]/mpi_size * mpi_rank; + + num_points = bigcount; + + coords = (hsize_t *)malloc(num_points * RANK * sizeof(hsize_t)); + VRFY((coords != NULL), "coords malloc succeeded"); + + set_coords (start, count, stride, block, num_points, coords, IN_ORDER); + /* create a file dataspace */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((ret >= 0), "H5Sselect_elements succeeded"); + + if(coords) free(coords); + + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + dataset_print(start, block, wdata); + } + + /* create a memory dataspace */ + mem_dataspace = H5Screate_simple (1, &bigcount, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + /* Irregular selection */ + /* Need larger memory for data buffer */ + free(wdata); +#if 0 + wdata = (B_DATATYPE *)malloc(bigcount*4*sizeof(B_DATATYPE)); + VRFY((wdata != NULL), "wdata malloc succeeded"); + + printf("\nTesting Dataset5 write irregular selection\n"); + /* Create a large dataset */ + dims[0] = bigcount/6; + dims[1] = mpi_size * 4; + sid = H5Screate_simple (RANK, dims, NULL); + VRFY((sid >= 0), "H5Screate_simple succeeded"); + dataset = H5Dcreate2(fid, DATASET5, H5T_NATIVE_LLONG, sid, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dcreate2 succeeded"); + H5Sclose(sid); + + /* first select 1 col in this procs splice */ + block[0] = dims[0]; + block[1] = 1; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = mpi_rank * 4; + + /* create a file dataspace */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + + dims[1] = 4; + /* create a memory dataspace */ + mem_dataspace = H5Screate_simple (RANK, dims, NULL); + VRFY((mem_dataspace >= 0), ""); + + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + start[1] = 0; + ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* select every other row in the process splice and OR it with + the col selection to create an irregular selection */ + for(h=0 ; h<dims[0] ; h+=2) { + block[0] = 1; + block[1] = 4; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = h; + start[1] = mpi_rank * 4; + + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_OR, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + start[1] = 0; + ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_OR, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + } + printf("Setting up for collective transfer\n"); + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), "H5Pcreate xfer succeeded"); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pset_dxpl_mpio succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* fill the local slab with some trivial data */ + fill_datasets(start, dims, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + ret = H5Dwrite(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, wdata); + VRFY((ret >= 0), "H5Dwrite dataset1 succeeded"); + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + free(wdata); +#endif + H5Fclose(fid); +} + +/* + * Example of using the parallel HDF5 library to read two datasets + * in one HDF5 file with collective parallel access support. + * The Datasets are of sizes (number-of-mpi-processes x dim0) x dim1. + * Each process controls only a slab of size dim0 x dim1 within each + * dataset. [Note: not so yet. Datasets are of sizes dim0xdim1 and + * each process controls a hyperslab within.] + */ + +static void +dataset_big_read(void) +{ + hid_t fid; /* HDF5 file ID */ + hid_t acc_tpl; /* File access templates */ + hid_t xfer_plist; /* Dataset transfer properties list */ + hid_t file_dataspace; /* File dataspace ID */ + hid_t mem_dataspace; /* memory dataspace ID */ + hid_t dataset; + B_DATATYPE *rdata = NULL; /* data buffer */ + B_DATATYPE *wdata = NULL; /* expected data buffer */ + hsize_t dims[RANK]; /* dataset dim sizes */ + hsize_t start[RANK]; /* for hyperslab setting */ + hsize_t count[RANK], stride[RANK]; /* for hyperslab setting */ + hsize_t block[RANK]; /* for hyperslab setting */ + int i,j,k; + hsize_t h; + size_t num_points; + hsize_t *coords = NULL; + herr_t ret; /* Generic return value */ + + /* allocate memory for data buffer */ + rdata = (B_DATATYPE *)malloc(bigcount*sizeof(B_DATATYPE)); + VRFY((rdata != NULL), "rdata malloc succeeded"); + wdata = (B_DATATYPE *)malloc(bigcount*sizeof(B_DATATYPE)); + VRFY((wdata != NULL), "wdata malloc succeeded"); + + memset(rdata, 0, bigcount*sizeof(B_DATATYPE)); + + /* setup file access template */ + acc_tpl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((acc_tpl >= 0), "H5P_FILE_ACCESS"); + H5Pset_fapl_mpio(acc_tpl, MPI_COMM_WORLD, MPI_INFO_NULL); + + /* open the file collectively */ + fid=H5Fopen(filename,H5F_ACC_RDONLY,acc_tpl); + VRFY((fid >= 0), "H5Fopen succeeded"); + + /* Release file-access template */ + ret = H5Pclose(acc_tpl); + VRFY((ret >= 0), ""); + + + printf("\nRead Testing Dataset1 by COL\n"); + dataset = H5Dopen2(fid, DATASET1, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = mpi_size; + /* Each process takes a slabs of cols. */ + block[0] = dims[0]; + block[1] = dims[1]/mpi_size; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = mpi_rank*block[1]; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, block, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* fill dataset with test data */ + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset1 succeeded"); + + { + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", rdata[k]); + k++; + } + } + } + printf("\n"); + } + + /* verify the read data with original expected data */ + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + + printf("\nRead Testing Dataset2 by ROW\n"); + memset(rdata, 0, bigcount*sizeof(B_DATATYPE)); + dataset = H5Dopen2(fid, DATASET2, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = mpi_size; + /* Each process takes a slabs of rows. */ + block[0] = dims[0]/mpi_size; + block[1] = dims[1]; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = mpi_rank*block[0]; + start[1] = 0; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, block, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* fill dataset with test data */ + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset2 succeeded"); + + { + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", rdata[k]); + k++; + } + } + } + printf("\n"); + } + + /* verify the read data with original expected data */ + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + + printf("\nRead Testing Dataset3 read select ALL proc 0, NONE others\n"); + memset(rdata, 0, bigcount*sizeof(B_DATATYPE)); + dataset = H5Dopen2(fid, DATASET3, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = 1; + + /* create a file dataspace independently */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + if(MAINPROCESS) { + ret = H5Sselect_all(file_dataspace); + VRFY((ret >= 0), "H5Sset_all succeeded"); + } + else { + ret = H5Sselect_none(file_dataspace); + VRFY((ret >= 0), "H5Sset_none succeeded"); + } + + /* create a memory dataspace independently */ + mem_dataspace = H5Screate_simple (RANK, dims, NULL); + VRFY((mem_dataspace >= 0), ""); + if(!MAINPROCESS) { + ret = H5Sselect_none(mem_dataspace); + VRFY((ret >= 0), "H5Sset_none succeeded"); + } + + /* fill dataset with test data */ + fill_datasets(start, dims, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset3 succeeded"); + + { + for (i=0; i < block[0]; i++){ + for (j=0; j < block[1]; j++){ + if(k < 10) { + printf("%lld ", rdata[k]); + k++; + } + } + } + printf("\n"); + } + + if(MAINPROCESS) { + /* verify the read data with original expected data */ + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + } + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + printf("\nRead Testing Dataset4 with Point selection\n"); + dataset = H5Dopen2(fid, DATASET4, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = mpi_size * 4; + + block[0] = dims[0]/2; + block[1] = 2; + stride[0] = dims[0]/2; + stride[1] = 2; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = dims[1]/mpi_size * mpi_rank; + + fill_datasets(start, block, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + dataset_print(start, block, wdata); + } + + num_points = bigcount; + + coords = (hsize_t *)malloc(num_points * RANK * sizeof(hsize_t)); + VRFY((coords != NULL), "coords malloc succeeded"); + + set_coords (start, count, stride, block, num_points, coords, IN_ORDER); + /* create a file dataspace */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + ret = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((ret >= 0), "H5Sselect_elements succeeded"); + + if(coords) free(coords); + + /* create a memory dataspace */ + mem_dataspace = H5Screate_simple (1, &bigcount, NULL); + VRFY((mem_dataspace >= 0), ""); + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset1 succeeded"); + + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + printf("\nRead Testing Dataset5 with Irregular selection\n"); + /* Need larger memory for data buffer */ + free(wdata); + free(rdata); +#if 0 + wdata = (B_DATATYPE *)malloc(bigcount*4*sizeof(B_DATATYPE)); + VRFY((wdata != NULL), "wdata malloc succeeded"); + rdata = (B_DATATYPE *)malloc(bigcount*4*sizeof(B_DATATYPE)); + VRFY((rdata != NULL), "rdata malloc succeeded"); + + dataset = H5Dopen2(fid, DATASET5, H5P_DEFAULT); + VRFY((dataset >= 0), "H5Dopen2 succeeded"); + + dims[0] = bigcount; + dims[1] = mpi_size * 4; + + /* first select 1 col in this proc splice */ + block[0] = dims[0]; + block[1] = 1; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = mpi_rank * 4; + + /* get file dataspace */ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), "H5Dget_space succeeded"); + + /* create a memory dataspace */ + dims[1] = 4; + mem_dataspace = H5Screate_simple (RANK, dims, NULL); + VRFY((mem_dataspace >= 0), ""); + + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + start[1] = 0; + ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + /* select every other row in the process splice and OR it with + the col selection to create an irregular selection */ + for(h=0 ; h<dims[0] ; h+=2) { + block[0] = 1; + block[1] = 4; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = h; + start[1] = mpi_rank * 4; + + ret = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_OR, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + start[1] = 0; + ret = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_OR, start, stride, count, block); + VRFY((ret >= 0), "H5Sset_hyperslab succeeded"); + + //fprintf(stderr, "%d: %d - %d\n", mpi_rank, (int)h, (int)H5Sget_select_npoints(mem_dataspace)); + } + + /* set up the collective transfer properties list */ + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + ret = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((ret >= 0), "H5Pcreate xfer succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + ret = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((ret>= 0),"set independent IO collectively succeeded"); + } + + /* read data collectively */ + ret = H5Dread(dataset, H5T_NATIVE_LLONG, mem_dataspace, file_dataspace, + xfer_plist, rdata); + VRFY((ret >= 0), "H5Dread dataset1 succeeded"); + + /* fill dataset with test data */ + fill_datasets(start, dims, wdata); + MESG("data_array initialized"); + if(VERBOSE_MED){ + MESG("data_array created"); + } + + + + /* verify the read data with original expected data */ + block[0] = dims[0]; + block[1] = 1; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = 0; + start[1] = 0; + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + + for(h=0 ; h<dims[0] ; h+=2) { + block[0] = 1; + block[1] = 4; + stride[0] = block[0]; + stride[1] = block[1]; + count[0] = 1; + count[1] = 1; + start[0] = h; + start[1] = 0; + ret = verify_data(start, count, stride, block, rdata, wdata); + if(ret) {fprintf(stderr, "verify failed\n"); exit(1);} + } + + /* release all temporary handles. */ + H5Sclose(file_dataspace); + H5Sclose(mem_dataspace); + H5Pclose(xfer_plist); + ret = H5Dclose(dataset); + VRFY((ret >= 0), "H5Dclose1 succeeded"); + + H5Fclose(fid); + + /* release data buffers */ + if(rdata) free(rdata); + if(wdata) free(wdata); +#endif +} /* dataset_large_readAll */ + + +/* + * Create the appropriate File access property list + */ +hid_t +create_faccess_plist(MPI_Comm comm, MPI_Info info, int l_facc_type) +{ + hid_t ret_pl = -1; + herr_t ret; /* generic return value */ + int mpi_rank; /* mpi variables */ + + /* need the rank for error checking macros */ + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + + ret_pl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((ret_pl >= 0), "H5P_FILE_ACCESS"); + + if (l_facc_type == FACC_DEFAULT) + return (ret_pl); + + if (l_facc_type == FACC_MPIO){ + /* set Parallel access with communicator */ + ret = H5Pset_fapl_mpio(ret_pl, comm, info); + VRFY((ret >= 0), ""); + ret = H5Pset_all_coll_metadata_ops(ret_pl, TRUE); + VRFY((ret >= 0), ""); + ret = H5Pset_coll_metadata_write(ret_pl, TRUE); + VRFY((ret >= 0), ""); + return(ret_pl); + } + + if (l_facc_type == (FACC_MPIO | FACC_SPLIT)){ + hid_t mpio_pl; + + mpio_pl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((mpio_pl >= 0), ""); + /* set Parallel access with communicator */ + ret = H5Pset_fapl_mpio(mpio_pl, comm, info); + VRFY((ret >= 0), ""); + + /* setup file access template */ + ret_pl = H5Pcreate (H5P_FILE_ACCESS); + VRFY((ret_pl >= 0), ""); + /* set Parallel access with communicator */ + ret = H5Pset_fapl_split(ret_pl, ".meta", mpio_pl, ".raw", mpio_pl); + VRFY((ret >= 0), "H5Pset_fapl_split succeeded"); + H5Pclose(mpio_pl); + return(ret_pl); + } + + /* unknown file access types */ + return (ret_pl); +} + + +/*------------------------------------------------------------------------- + * Function: coll_chunk1 + * + * Purpose: Wrapper to test the collective chunk IO for regular JOINT + selection with a single chunk + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Unknown + * July 12th, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +/* ------------------------------------------------------------------------ + * Descriptions for the selection: One big singluar selection inside one chunk + * Two dimensions, + * + * dim1 = space_dim1(5760)*mpi_size + * dim2 = space_dim2(3) + * chunk_dim1 = dim1 + * chunk_dim2 = dim2 + * block = 1 for all dimensions + * stride = 1 for all dimensions + * count0 = space_dim1(5760) + * count1 = space_dim2(3) + * start0 = mpi_rank*space_dim1 + * start1 = 0 + * ------------------------------------------------------------------------ + */ + +void +coll_chunk1(void) +{ + if (MAINPROCESS) + printf("coll_chunk1\n"); + + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, HYPER, HYPER, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, HYPER, POINT, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, ALL, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, POINT, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, HYPER, OUT_OF_ORDER); + + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, ALL, IN_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, POINT, IN_ORDER); + coll_chunktest(filename, 1, BYROW_CONT, API_NONE, POINT, HYPER, IN_ORDER); +} + + +/*------------------------------------------------------------------------- + * Function: coll_chunk2 + * + * Purpose: Wrapper to test the collective chunk IO for regular DISJOINT + selection with a single chunk + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Unknown + * July 12th, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + + /* ------------------------------------------------------------------------ + * Descriptions for the selection: many disjoint selections inside one chunk + * Two dimensions, + * + * dim1 = space_dim1*mpi_size(5760) + * dim2 = space_dim2(3) + * chunk_dim1 = dim1 + * chunk_dim2 = dim2 + * block = 1 for all dimensions + * stride = 3 for all dimensions + * count0 = space_dim1/stride0(5760/3) + * count1 = space_dim2/stride(3/3 = 1) + * start0 = mpi_rank*space_dim1 + * start1 = 0 + * + * ------------------------------------------------------------------------ + */ +void +coll_chunk2(void) +{ + if (MAINPROCESS) + printf("coll_chunk2\n"); + + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, HYPER, HYPER, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, HYPER, POINT, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, ALL, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, POINT, OUT_OF_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, HYPER, OUT_OF_ORDER); + + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, ALL, IN_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, POINT, IN_ORDER); + coll_chunktest(filename, 1, BYROW_DISCONT, API_NONE, POINT, HYPER, IN_ORDER); +} + + +/*------------------------------------------------------------------------- + * Function: coll_chunk3 + * + * Purpose: Wrapper to test the collective chunk IO for regular JOINT + selection with at least number of 2*mpi_size chunks + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Unknown + * July 12th, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +/* ------------------------------------------------------------------------ + * Descriptions for the selection: one singular selection accross many chunks + * Two dimensions, Num of chunks = 2* mpi_size + * + * dim1 = space_dim1*mpi_size + * dim2 = space_dim2(3) + * chunk_dim1 = space_dim1 + * chunk_dim2 = dim2/2 + * block = 1 for all dimensions + * stride = 1 for all dimensions + * count0 = space_dim1 + * count1 = space_dim2(3) + * start0 = mpi_rank*space_dim1 + * start1 = 0 + * + * ------------------------------------------------------------------------ + */ + +void +coll_chunk3(void) +{ + if (MAINPROCESS) + printf("coll_chunk3\n"); + + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, HYPER, HYPER, OUT_OF_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, HYPER, POINT, OUT_OF_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, ALL, OUT_OF_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, POINT, OUT_OF_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, HYPER, OUT_OF_ORDER); + + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, ALL, IN_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, POINT, IN_ORDER); + coll_chunktest(filename, mpi_size, BYROW_CONT, API_NONE, POINT, HYPER, IN_ORDER); +} + + +//------------------------------------------------------------------------- +// Borrowed/Modified (slightly) from t_coll_chunk.c +/*------------------------------------------------------------------------- + * Function: coll_chunktest + * + * Purpose: The real testing routine for regular selection of collective + chunking storage + testing both write and read, + If anything fails, it may be read or write. There is no + separation test between read and write. + * + * Return: Success: 0 + * + * Failure: -1 + * + * Modifications: + * Remove invalid temporary property checkings for API_LINK_HARD and + * API_LINK_TRUE cases. + * Programmer: Jonathan Kim + * Date: 2012-10-10 + * + * Programmer: Unknown + * July 12th, 2004 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ + +static void +coll_chunktest(const char* filename, + int chunk_factor, + int select_factor, + int api_option, + int file_selection, + int mem_selection, + int mode) +{ + hid_t file, dataset, file_dataspace, mem_dataspace; + hid_t acc_plist,xfer_plist,crp_plist; + + hsize_t dims[RANK], chunk_dims[RANK]; + int* data_array1 = NULL; + int* data_origin1 = NULL; + + hsize_t start[RANK],count[RANK],stride[RANK],block[RANK]; + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + unsigned prop_value; +#endif /* H5_HAVE_INSTRUMENTED_LIBRARY */ + + herr_t status; + MPI_Comm comm = MPI_COMM_WORLD; + MPI_Info info = MPI_INFO_NULL; + + size_t num_points; /* for point selection */ + hsize_t *coords = NULL; /* for point selection */ + hsize_t current_dims; /* for point selection */ + int i; + + /* Create the data space */ + + acc_plist = create_faccess_plist(comm,info,facc_type); + VRFY((acc_plist >= 0),""); + + file = H5Fcreate(filename,H5F_ACC_TRUNC,H5P_DEFAULT,acc_plist); + VRFY((file >= 0),"H5Fcreate succeeded"); + + status = H5Pclose(acc_plist); + VRFY((status >= 0),""); + + /* setup dimensionality object */ + dims[0] = space_dim1*mpi_size; + dims[1] = space_dim2; + + /* allocate memory for data buffer */ + data_array1 = (int *)HDmalloc(dims[0] * dims[1] * sizeof(int)); + VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); + + /* set up dimensions of the slab this process accesses */ + ccslab_set(mpi_rank, mpi_size, start, count, stride, block, select_factor); + + /* set up the coords array selection */ + num_points = block[0] * block[1] * count[0] * count[1]; + coords = (hsize_t *)HDmalloc(num_points * RANK * sizeof(hsize_t)); + VRFY((coords != NULL), "coords malloc succeeded"); + point_set(start, count, stride, block, num_points, coords, mode); + + file_dataspace = H5Screate_simple(2, dims, NULL); + VRFY((file_dataspace >= 0), "file dataspace created succeeded"); + + if(ALL != mem_selection) { + mem_dataspace = H5Screate_simple(2, dims, NULL); + VRFY((mem_dataspace >= 0), "mem dataspace created succeeded"); + } + else { + current_dims = num_points; + mem_dataspace = H5Screate_simple (1, ¤t_dims, NULL); + VRFY((mem_dataspace >= 0), "mem_dataspace create succeeded"); + } + + crp_plist = H5Pcreate(H5P_DATASET_CREATE); + VRFY((crp_plist >= 0),""); + + /* Set up chunk information. */ + chunk_dims[0] = dims[0]/chunk_factor; + + /* to decrease the testing time, maintain bigger chunk size */ + (chunk_factor == 1) ? (chunk_dims[1] = space_dim2) : (chunk_dims[1] = space_dim2/2); + status = H5Pset_chunk(crp_plist, 2, chunk_dims); + VRFY((status >= 0),"chunk creation property list succeeded"); + + dataset = H5Dcreate2(file, DSET_COLLECTIVE_CHUNK_NAME, H5T_NATIVE_INT, + file_dataspace, H5P_DEFAULT, crp_plist, H5P_DEFAULT); + VRFY((dataset >= 0),"dataset created succeeded"); + + status = H5Pclose(crp_plist); + VRFY((status >= 0), ""); + + /*put some trivial data in the data array */ + ccdataset_fill(start, stride, count,block, data_array1, mem_selection); + + MESG("data_array initialized"); + + switch (file_selection) { + case HYPER: + status = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((status >= 0),"hyperslab selection succeeded"); + break; + + case POINT: + if (num_points) { + status = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((status >= 0),"Element selection succeeded"); + } + else { + status = H5Sselect_none(file_dataspace); + VRFY((status >= 0),"none selection succeeded"); + } + break; + + case ALL: + status = H5Sselect_all(file_dataspace); + VRFY((status >= 0), "H5Sselect_all succeeded"); + break; + } + + switch (mem_selection) { + case HYPER: + status = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((status >= 0),"hyperslab selection succeeded"); + break; + + case POINT: + if (num_points) { + status = H5Sselect_elements(mem_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((status >= 0),"Element selection succeeded"); + } + else { + status = H5Sselect_none(mem_dataspace); + VRFY((status >= 0),"none selection succeeded"); + } + break; + + case ALL: + status = H5Sselect_all(mem_dataspace); + VRFY((status >= 0), "H5Sselect_all succeeded"); + break; + } + + /* set up the collective transfer property list */ + xfer_plist = H5Pcreate(H5P_DATASET_XFER); + VRFY((xfer_plist >= 0), ""); + + status = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((status>= 0),"MPIO collective transfer property succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + status = H5Pset_dxpl_mpio_collective_opt(xfer_plist, H5FD_MPIO_INDIVIDUAL_IO); + VRFY((status>= 0),"set independent IO collectively succeeded"); + } + + switch(api_option){ + case API_LINK_HARD: + status = H5Pset_dxpl_mpio_chunk_opt(xfer_plist,H5FD_MPIO_CHUNK_ONE_IO); + VRFY((status>= 0),"collective chunk optimization succeeded"); + break; + + case API_MULTI_HARD: + status = H5Pset_dxpl_mpio_chunk_opt(xfer_plist,H5FD_MPIO_CHUNK_MULTI_IO); + VRFY((status>= 0),"collective chunk optimization succeeded "); + break; + + case API_LINK_TRUE: + status = H5Pset_dxpl_mpio_chunk_opt_num(xfer_plist,2); + VRFY((status>= 0),"collective chunk optimization set chunk number succeeded"); + break; + + case API_LINK_FALSE: + status = H5Pset_dxpl_mpio_chunk_opt_num(xfer_plist,6); + VRFY((status>= 0),"collective chunk optimization set chunk number succeeded"); + break; + + case API_MULTI_COLL: + status = H5Pset_dxpl_mpio_chunk_opt_num(xfer_plist,8);/* make sure it is using multi-chunk IO */ + VRFY((status>= 0),"collective chunk optimization set chunk number succeeded"); + status = H5Pset_dxpl_mpio_chunk_opt_ratio(xfer_plist,50); + VRFY((status>= 0),"collective chunk optimization set chunk ratio succeeded"); + break; + + case API_MULTI_IND: + status = H5Pset_dxpl_mpio_chunk_opt_num(xfer_plist,8);/* make sure it is using multi-chunk IO */ + VRFY((status>= 0),"collective chunk optimization set chunk number succeeded"); + status = H5Pset_dxpl_mpio_chunk_opt_ratio(xfer_plist,100); + VRFY((status>= 0),"collective chunk optimization set chunk ratio succeeded"); + break; + + default: + ; + } + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if(facc_type == FACC_MPIO) { + switch(api_option) { + case API_LINK_HARD: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_LINK_HARD_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_MULTI_HARD: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_MULTI_HARD_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_LINK_TRUE: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_LINK_NUM_TRUE_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_LINK_FALSE: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_LINK_NUM_FALSE_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_MULTI_COLL: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_MULTI_RATIO_COLL_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + case API_MULTI_IND: + prop_value = H5D_XFER_COLL_CHUNK_DEF; + status = H5Pinsert2(xfer_plist, H5D_XFER_COLL_CHUNK_MULTI_RATIO_IND_NAME, H5D_XFER_COLL_CHUNK_SIZE, &prop_value, + NULL, NULL, NULL, NULL, NULL, NULL); + VRFY((status >= 0),"testing property list inserted succeeded"); + break; + + default: + ; + } + } +#endif + + /* write data collectively */ + status = H5Dwrite(dataset, H5T_NATIVE_INT, mem_dataspace, file_dataspace, + xfer_plist, data_array1); + VRFY((status >= 0),"dataset write succeeded"); + +#ifdef H5_HAVE_INSTRUMENTED_LIBRARY + if(facc_type == FACC_MPIO) { + switch(api_option){ + case API_LINK_HARD: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_LINK_HARD_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set LINK COLLECTIVE IO directly succeeded"); + break; + + case API_MULTI_HARD: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_MULTI_HARD_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set MULTI-CHUNK COLLECTIVE IO optimization succeeded"); + break; + + case API_LINK_TRUE: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_LINK_NUM_TRUE_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set LINK COLLECTIVE IO succeeded"); + break; + + case API_LINK_FALSE: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_LINK_NUM_FALSE_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set LINK IO transferring to multi-chunk IO succeeded"); + break; + + case API_MULTI_COLL: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_MULTI_RATIO_COLL_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set MULTI-CHUNK COLLECTIVE IO with optimization succeeded"); + break; + + case API_MULTI_IND: + status = H5Pget(xfer_plist,H5D_XFER_COLL_CHUNK_MULTI_RATIO_IND_NAME,&prop_value); + VRFY((status >= 0),"testing property list get succeeded"); + VRFY((prop_value == 0),"API to set MULTI-CHUNK IO transferring to independent IO succeeded"); + break; + + default: + ; + } + } +#endif + + status = H5Dclose(dataset); + VRFY((status >= 0),""); + + status = H5Pclose(xfer_plist); + VRFY((status >= 0),"property list closed"); + + status = H5Sclose(file_dataspace); + VRFY((status >= 0),""); + + status = H5Sclose(mem_dataspace); + VRFY((status >= 0),""); + + + status = H5Fclose(file); + VRFY((status >= 0),""); + + if (data_array1) HDfree(data_array1); + + /* Use collective read to verify the correctness of collective write. */ + + /* allocate memory for data buffer */ + data_array1 = (int *)HDmalloc(dims[0]*dims[1]*sizeof(int)); + VRFY((data_array1 != NULL), "data_array1 malloc succeeded"); + + /* allocate memory for data buffer */ + data_origin1 = (int *)HDmalloc(dims[0]*dims[1]*sizeof(int)); + VRFY((data_origin1 != NULL), "data_origin1 malloc succeeded"); + + acc_plist = create_faccess_plist(comm, info, facc_type); + VRFY((acc_plist >= 0),"MPIO creation property list succeeded"); + + file = H5Fopen(filename,H5F_ACC_RDONLY,acc_plist); + VRFY((file >= 0),"H5Fcreate succeeded"); + + status = H5Pclose(acc_plist); + VRFY((status >= 0),""); + + /* open the collective dataset*/ + dataset = H5Dopen2(file, DSET_COLLECTIVE_CHUNK_NAME, H5P_DEFAULT); + VRFY((dataset >= 0), ""); + + /* set up dimensions of the slab this process accesses */ + ccslab_set(mpi_rank, mpi_size, start, count, stride, block, select_factor); + + /* obtain the file and mem dataspace*/ + file_dataspace = H5Dget_space (dataset); + VRFY((file_dataspace >= 0), ""); + + if (ALL != mem_selection) { + mem_dataspace = H5Dget_space (dataset); + VRFY((mem_dataspace >= 0), ""); + } + else { + current_dims = num_points; + mem_dataspace = H5Screate_simple (1, ¤t_dims, NULL); + VRFY((mem_dataspace >= 0), "mem_dataspace create succeeded"); + } + + switch (file_selection) { + case HYPER: + status = H5Sselect_hyperslab(file_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((status >= 0),"hyperslab selection succeeded"); + break; + + case POINT: + if (num_points) { + status = H5Sselect_elements(file_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((status >= 0),"Element selection succeeded"); + } + else { + status = H5Sselect_none(file_dataspace); + VRFY((status >= 0),"none selection succeeded"); + } + break; + + case ALL: + status = H5Sselect_all(file_dataspace); + VRFY((status >= 0), "H5Sselect_all succeeded"); + break; + } + + switch (mem_selection) { + case HYPER: + status = H5Sselect_hyperslab(mem_dataspace, H5S_SELECT_SET, start, stride, count, block); + VRFY((status >= 0),"hyperslab selection succeeded"); + break; + + case POINT: + if (num_points) { + status = H5Sselect_elements(mem_dataspace, H5S_SELECT_SET, num_points, coords); + VRFY((status >= 0),"Element selection succeeded"); + } + else { + status = H5Sselect_none(mem_dataspace); + VRFY((status >= 0),"none selection succeeded"); + } + break; + + case ALL: + status = H5Sselect_all(mem_dataspace); + VRFY((status >= 0), "H5Sselect_all succeeded"); + break; + } + + /* fill dataset with test data */ + ccdataset_fill(start, stride,count,block, data_origin1, mem_selection); + xfer_plist = H5Pcreate (H5P_DATASET_XFER); + VRFY((xfer_plist >= 0),""); + + status = H5Pset_dxpl_mpio(xfer_plist, H5FD_MPIO_COLLECTIVE); + VRFY((status>= 0),"MPIO collective transfer property succeeded"); + if(dxfer_coll_type == DXFER_INDEPENDENT_IO) { + status = H5Pset_dxpl_mpio_collective_opt(xfer_plist,H5FD_MPIO_INDIVIDUAL_IO); + VRFY((status>= 0),"set independent IO collectively succeeded"); + } + + status = H5Dread(dataset, H5T_NATIVE_INT, mem_dataspace, file_dataspace, + xfer_plist, data_array1); + VRFY((status >=0),"dataset read succeeded"); + + /* verify the read data with original expected data */ + status = ccdataset_vrfy(start, count, stride, block, data_array1, data_origin1, mem_selection); + if (status) nerrors++; + + status = H5Pclose(xfer_plist); + VRFY((status >= 0),"property list closed"); + + /* close dataset collectively */ + status=H5Dclose(dataset); + VRFY((status >= 0), "H5Dclose"); + + /* release all IDs created */ + status = H5Sclose(file_dataspace); + VRFY((status >= 0),"H5Sclose"); + + status = H5Sclose(mem_dataspace); + VRFY((status >= 0),"H5Sclose"); + + /* close the file collectively */ + status = H5Fclose(file); + VRFY((status >= 0),"H5Fclose"); + + /* release data buffers */ + if(coords) HDfree(coords); + if(data_array1) HDfree(data_array1); + if(data_origin1) HDfree(data_origin1); + +} + + + +/***************************************************************************** + * + * Function: do_express_test() + * + * Purpose: Do an MPI_Allreduce to obtain the maximum value returned + * by GetTestExpress() across all processes. Return this + * value. + * + * Envirmoment variables can be different across different + * processes. This function ensures that all processes agree + * on whether to do an express test. + * + * Return: Success: Maximum of the values returned by + * GetTestExpress() across all processes. + * + * Failure: -1 + * + * Programmer: JRM -- 4/25/06 + * + *****************************************************************************/ +static int +do_express_test(int world_mpi_rank) +{ + int express_test; + int max_express_test; + int result; + + express_test = GetTestExpress(); + + result = MPI_Allreduce((void *)&express_test, + (void *)&max_express_test, + 1, + MPI_INT, + MPI_MAX, + MPI_COMM_WORLD); + + if ( result != MPI_SUCCESS ) { + nerrors++; + max_express_test = -1; + if ( VERBOSE_MED && (world_mpi_rank == 0)) { + HDfprintf(stdout, "%d:%s: MPI_Allreduce() failed.\n", + world_mpi_rank, FUNC ); + } + } + + return(max_express_test); + +} /* do_express_test() */ + + +int main(int argc, char **argv) +{ + int ExpressMode = 0; + hsize_t newsize = 1048576; + hsize_t oldsize = H5S_mpio_set_bigio_count(newsize); + + if (newsize != oldsize) { + bigcount = newsize * 2; + } + + MPI_Init(&argc, &argv); + MPI_Comm_size(MPI_COMM_WORLD,&mpi_size); + MPI_Comm_rank(MPI_COMM_WORLD,&mpi_rank); + + ExpressMode = do_express_test(mpi_rank); + + dataset_big_write(); + MPI_Barrier(MPI_COMM_WORLD); + + dataset_big_read(); + MPI_Barrier(MPI_COMM_WORLD); + + if (ExpressMode > 0) { + printf("***Express test mode on. Several tests are skipped\n"); + } + else { + coll_chunk1(); + MPI_Barrier(MPI_COMM_WORLD); + coll_chunk2(); + MPI_Barrier(MPI_COMM_WORLD); + coll_chunk3(); + } + + /* close HDF5 library */ + H5close(); + + MPI_Finalize(); + + return 0; +} + diff --git a/tools/lib/h5diff.c b/tools/lib/h5diff.c index afb36d9..9da5b6b 100644 --- a/tools/lib/h5diff.c +++ b/tools/lib/h5diff.c @@ -11,8 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdlib.h> - #include "H5private.h" #include "h5tools.h" #include "h5tools_utils.h" diff --git a/tools/lib/h5diff_array.c b/tools/lib/h5diff_array.c index 0b1a96e..7236ad1 100644 --- a/tools/lib/h5diff_array.c +++ b/tools/lib/h5diff_array.c @@ -673,21 +673,33 @@ static hsize_t diff_datum(void *_mem1, h5difftrace("diff_datum H5T_STRING variable\n"); /* Get pointer to first string */ s1 = *(char**) mem1; - size1 = HDstrlen(s1); + if(s1) + size1 = HDstrlen(s1); + else + size1 = 0; /* Get pointer to second string */ s2 = *(char**) mem2; - size2 = HDstrlen(s2); + if(s2) + size2 = HDstrlen(s2); + else + size2 = 0; } else if (H5T_STR_NULLTERM == pad) { h5difftrace("diff_datum H5T_STRING null term\n"); /* Get pointer to first string */ s1 = (char*) mem1; - size1 = HDstrlen(s1); + if(s1) + size1 = HDstrlen(s1); + else + size1 = 0; if (size1 > size_mtype) size1 = size_mtype; /* Get pointer to second string */ s2 = (char*) mem2; - size2 = HDstrlen(s2); + if(s2) + size2 = HDstrlen(s2); + else + size2 = 0; if (size2 > size_mtype) size2 = size_mtype; } diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 3729cac..5031e44 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -12,16 +12,10 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> - * Thursday, July 23, 1998 - * * Purpose: A library for displaying the values of a dataset in a human * readable format. */ -#include <stdio.h> -#include <stdlib.h> - #include "h5tools.h" #include "h5tools_dump.h" #include "h5tools_ref.h" diff --git a/tools/lib/h5tools_dump.c b/tools/lib/h5tools_dump.c index 1a57512..fb79b77 100644 --- a/tools/lib/h5tools_dump.c +++ b/tools/lib/h5tools_dump.c @@ -12,16 +12,10 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Robb Matzke <matzke@llnl.gov> - * Thursday, July 23, 1998 - * * Purpose: A library for displaying the values of a dataset in a human * readable format. */ -#include <stdio.h> -#include <stdlib.h> - #include "h5tools.h" #include "h5tools_dump.h" #include "h5tools_ref.h" diff --git a/tools/lib/h5tools_ref.c b/tools/lib/h5tools_ref.c index 85850e3..e000080 100644 --- a/tools/lib/h5tools_ref.c +++ b/tools/lib/h5tools_ref.c @@ -11,8 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdio.h> -#include <stdlib.h> #include "h5tools_ref.h" #include "H5private.h" #include "H5SLprivate.h" diff --git a/tools/lib/h5tools_str.c b/tools/lib/h5tools_str.c index fa15785..a66cfe5 100644 --- a/tools/lib/h5tools_str.c +++ b/tools/lib/h5tools_str.c @@ -12,16 +12,8 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Bill Wendling <wendling@ncsa.uiuc.edu> - * Monday, 19. February 2001 - * * Purpose: These are string functions for us to use and abuse. */ -#include <stdarg.h> -#include <stdio.h> -#include <stdlib.h> -#include <string.h> - #include "H5private.h" #include "h5tools.h" /* for h5tool_format_t structure */ #include "h5tools_ref.h" diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index c361e25..08213df 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -12,20 +12,10 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Bill Wendling <wendling@ncsa.uiuc.edu> - * Tuesday, 6. March 2001 - */ - -/* * Portions of this work are derived from _Obfuscated C and Other Mysteries_, * by Don Libes, copyright (c) 1993 by John Wiley & Sons, Inc. */ -#include <ctype.h> -#include <stdarg.h> -#include <stdio.h> -#include <stdlib.h> - #include "h5tools.h" #include "h5tools_utils.h" #include "H5private.h" diff --git a/tools/lib/io_timer.c b/tools/lib/io_timer.c index e3318e9..a6885df 100644 --- a/tools/lib/io_timer.c +++ b/tools/lib/io_timer.c @@ -22,16 +22,9 @@ * This is a module of useful timing functions for performance testing. */ -#include <stdio.h> -#include <stdlib.h> - #include "H5private.h" #include "hdf5.h" -#ifdef H5_HAVE_PARALLEL -#include <mpi.h> -#endif - #include "io_timer.h" /* diff --git a/tools/lib/ph5diff.h b/tools/lib/ph5diff.h index 9628d45..996a611 100644 --- a/tools/lib/ph5diff.h +++ b/tools/lib/ph5diff.h @@ -42,9 +42,5 @@ struct diffs_found int not_cmp; }; -#ifdef H5_HAVE_PARALLEL -#include <mpi.h> -#endif - #endif /* _PH5DIFF_H__ */ diff --git a/tools/src/h5copy/h5copy.c b/tools/src/h5copy/h5copy.c index 390b93e..3f91fce 100644 --- a/tools/src/h5copy/h5copy.c +++ b/tools/src/h5copy/h5copy.c @@ -14,8 +14,6 @@ #include "H5private.h" #include "h5tools.h" #include "h5tools_utils.h" -#include <string.h> -#include <stdlib.h> /* Name of tool */ #define PROGRAMNAME "h5copy" diff --git a/tools/src/h5diff/h5diff_common.c b/tools/src/h5diff/h5diff_common.c index 0537b9f..1069a31 100644 --- a/tools/src/h5diff/h5diff_common.c +++ b/tools/src/h5diff/h5diff_common.c @@ -11,8 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdlib.h> -#include <string.h> #include "H5private.h" #include "h5diff.h" #include "h5diff_common.h" diff --git a/tools/src/h5diff/h5diff_main.c b/tools/src/h5diff/h5diff_main.c index 92a1610..66ff71e 100644 --- a/tools/src/h5diff/h5diff_main.c +++ b/tools/src/h5diff/h5diff_main.c @@ -11,9 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdlib.h> -#include <assert.h> -#include <memory.h> #include "H5private.h" #include "h5diff.h" #include "h5diff_common.h" diff --git a/tools/src/h5diff/ph5diff_main.c b/tools/src/h5diff/ph5diff_main.c index bfeb408..192067f 100644 --- a/tools/src/h5diff/ph5diff_main.c +++ b/tools/src/h5diff/ph5diff_main.c @@ -11,9 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdlib.h> -#include <string.h> -#include <assert.h> #include "H5private.h" #include "h5diff.h" #include "ph5diff.h" diff --git a/tools/src/h5dump/h5dump.c b/tools/src/h5dump/h5dump.c index b53c212..bf2e127 100644 --- a/tools/src/h5dump/h5dump.c +++ b/tools/src/h5dump/h5dump.c @@ -10,8 +10,6 @@ * If you do not have access to either file, you may request a copy from * * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdio.h> -#include <stdlib.h> #include "h5dump.h" #include "h5dump_ddl.h" diff --git a/tools/src/h5dump/h5dump_ddl.c b/tools/src/h5dump/h5dump_ddl.c index 8ce6cd6..0a45840 100644 --- a/tools/src/h5dump/h5dump_ddl.c +++ b/tools/src/h5dump/h5dump_ddl.c @@ -10,8 +10,6 @@ * If you do not have access to either file, you may request a copy from * * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdio.h> -#include <stdlib.h> #include "H5private.h" #include "h5tools.h" diff --git a/tools/src/h5dump/h5dump_xml.c b/tools/src/h5dump/h5dump_xml.c index 1c3978d..49d4f96 100644 --- a/tools/src/h5dump/h5dump_xml.c +++ b/tools/src/h5dump/h5dump_xml.c @@ -10,8 +10,6 @@ * If you do not have access to either file, you may request a copy from * * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdio.h> -#include <stdlib.h> #include "H5private.h" #include "h5tools.h" @@ -782,20 +780,20 @@ xml_escape_the_string(const char *str, int slen) esc_len = 1; } else if (*cp == '\'') { - HDstrncpy(ncp, apos, ncp_len); esc_len = HDstrlen(apos); + HDstrncpy(ncp, apos, esc_len); } else if (*cp == '<') { - HDstrncpy(ncp, lt, ncp_len); esc_len = HDstrlen(lt); + HDstrncpy(ncp, lt, esc_len); } else if (*cp == '>') { - HDstrncpy(ncp, gt, ncp_len); esc_len = HDstrlen(gt); + HDstrncpy(ncp, gt, esc_len); } else if (*cp == '&') { - HDstrncpy(ncp, amp, ncp_len); esc_len = HDstrlen(amp); + HDstrncpy(ncp, amp, esc_len); } else { *ncp = *cp; diff --git a/tools/src/h5repack/h5repack.c b/tools/src/h5repack/h5repack.c index bc8527b..4860d9e 100644 --- a/tools/src/h5repack/h5repack.c +++ b/tools/src/h5repack/h5repack.c @@ -11,10 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdlib.h> -#include <string.h> -#include <ctype.h> - #include "H5private.h" #include "h5repack.h" #include "h5tools.h" diff --git a/tools/src/h5stat/h5stat.c b/tools/src/h5stat/h5stat.c index 6aee7a8..6f196b4 100644 --- a/tools/src/h5stat/h5stat.c +++ b/tools/src/h5stat/h5stat.c @@ -11,8 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdlib.h> -#include <string.h> #include "H5private.h" /* Generic Functions */ #include "h5tools.h" #include "h5tools_utils.h" diff --git a/tools/src/misc/h5mkgrp.c b/tools/src/misc/h5mkgrp.c index 597b6b3..43d6bfe 100644 --- a/tools/src/misc/h5mkgrp.c +++ b/tools/src/misc/h5mkgrp.c @@ -15,8 +15,6 @@ #include "H5private.h" #include "h5tools.h" #include "h5tools_utils.h" -#include <string.h> -#include <stdlib.h> /* Name of tool */ #define PROGRAMNAME "h5mkgrp" diff --git a/tools/src/misc/h5repart.c b/tools/src/misc/h5repart.c index 4432621..cdc554f 100644 --- a/tools/src/misc/h5repart.c +++ b/tools/src/misc/h5repart.c @@ -25,39 +25,9 @@ /* See H5private.h for how to include system headers */ #include "hdf5.h" #include "H5private.h" -#ifdef H5_STDC_HEADERS -# include <ctype.h> -# include <errno.h> -# include <fcntl.h> -# include <stdio.h> -# include <stdlib.h> -# include <string.h> -#endif - -#ifdef H5_HAVE_UNISTD_H -# include <sys/types.h> -# include <unistd.h> -#endif - -#ifdef H5_HAVE_SYS_STAT_H -# include <sys/stat.h> -#endif - -#ifndef FALSE -# define FALSE 0 -#endif -#ifndef TRUE -# define TRUE 1 -#endif -# define NAMELEN 4096 -#define GB *1024*1024*1024 -#ifndef MIN -# define MIN(X,Y) ((X)<(Y)?(X):(Y)) -#endif -#ifndef MIN3 -# define MIN3(X,Y,Z) MIN(MIN(X,Y),Z) -#endif +#define NAMELEN 4096 +#define GB *1024*1024*1024 /*Make these 2 private properties(defined in H5Fprivate.h) available to h5repart. *The first one updates the member file size in the superblock. The second one diff --git a/tools/test/h5copy/h5copygentest.c b/tools/test/h5copy/h5copygentest.c index d4d6a08..35f9132 100644 --- a/tools/test/h5copy/h5copygentest.c +++ b/tools/test/h5copy/h5copygentest.c @@ -14,7 +14,6 @@ /* * Generate the binary hdf5 file for the h5copy tests */ -#include <stdlib.h> #include "hdf5.h" #include "H5private.h" diff --git a/tools/test/h5diff/CMakeTests.cmake b/tools/test/h5diff/CMakeTests.cmake index 72dda6b..608795c 100644 --- a/tools/test/h5diff/CMakeTests.cmake +++ b/tools/test/h5diff/CMakeTests.cmake @@ -95,6 +95,8 @@ ${HDF5_TOOLS_DIR}/testfiles/vds/5_b.h5 ${HDF5_TOOLS_DIR}/testfiles/vds/5_c.h5 ${HDF5_TOOLS_DIR}/testfiles/vds/5_vds.h5 + # tools/testfiles + ${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5 ) set (LIST_OTHER_TEST_FILES @@ -286,6 +288,7 @@ ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v1.txt ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v2.txt ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_v3.txt + ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_vlstr.txt ) set (LIST_WIN_TEST_FILES @@ -311,6 +314,11 @@ HDFTEST_COPY_FILE("${h5_tstfiles}" "${PROJECT_BINARY_DIR}/PAR/testfiles/${fname}" "h5diff_files") endif () endforeach () + # copy second version of tvlstr.h5 + HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5" "${PROJECT_BINARY_DIR}/testfiles/tvlstr2.h5" "h5diff_files") + if (H5_HAVE_PARALLEL) + HDFTEST_COPY_FILE("${HDF5_TOOLS_DIR}/testfiles/tvlstr.h5" "${PROJECT_BINARY_DIR}/PAR/testfiles/tvlstr2.h5" "h5diff_files") + endif () # @@ -888,6 +896,8 @@ h5diff_v2.out.err h5diff_v3.out h5diff_v3.out.err + h5diff_vlstr.out + h5diff_vlstr.out.err ) set_tests_properties (H5DIFF-clearall-objects PROPERTIES WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles") if (NOT "${last_test}" STREQUAL "") @@ -1465,6 +1475,7 @@ ADD_H5_TEST (h5diff_487 1 -v --exclude-path "/group1/dset" h5diff_exclude3-1.h5 # # diff various multiple vlen and fixed strings in a compound type dataset # ############################################################################## ADD_H5_TEST (h5diff_530 0 -v ${COMP_VL_STRS_FILE} ${COMP_VL_STRS_FILE} /group /group_copy) +ADD_H5_TEST (h5diff_vlstr 0 -v tvlstr.h5 tvlstr2.h5) # ############################################################################## # # Test container types (array,vlen) with multiple nested compound types diff --git a/tools/test/h5diff/testfiles/h5diff_vlstr.txt b/tools/test/h5diff/testfiles/h5diff_vlstr.txt new file mode 100644 index 0000000..67141f1 --- /dev/null +++ b/tools/test/h5diff/testfiles/h5diff_vlstr.txt @@ -0,0 +1,16 @@ + +file1 file2 +--------------------------------------- + x x / + x x /Dataset1 + x x /vl_string_type + +group : </> and </> +0 differences found +attribute: <test_scalar of </>> and <test_scalar of </>> +0 differences found +dataset: </Dataset1> and </Dataset1> +0 differences found +datatype: </vl_string_type> and </vl_string_type> +0 differences found +EXIT CODE: 0 diff --git a/tools/test/h5diff/testh5diff.sh.in b/tools/test/h5diff/testh5diff.sh.in index d769c23..df472d4 100644 --- a/tools/test/h5diff/testh5diff.sh.in +++ b/tools/test/h5diff/testh5diff.sh.in @@ -124,6 +124,7 @@ $SRC_H5DIFF_TESTFILES/tmpSingleSiteBethe.reference.h5 $SRC_H5DIFF_TESTFILES/tmpSingleSiteBethe.output.h5 $SRC_H5DIFF_TESTFILES/diff_strings1.h5 $SRC_H5DIFF_TESTFILES/diff_strings2.h5 +$SRC_TOOLS_TESTFILES/tvlstr.h5 " LIST_HDF5_VDS_TEST_FILES=" @@ -341,6 +342,7 @@ $SRC_H5DIFF_TESTFILES/h5diff_tmp2.txt $SRC_H5DIFF_TESTFILES/h5diff_v1.txt $SRC_H5DIFF_TESTFILES/h5diff_v2.txt $SRC_H5DIFF_TESTFILES/h5diff_v3.txt +$SRC_H5DIFF_TESTFILES/h5diff_vlstr.txt " # @@ -566,6 +568,8 @@ SKIP() { ############################################################################## # prepare for test COPY_TESTFILES_TO_TESTDIR +# second copy of tvlstr.h5 +$CP -f $SRC_TOOLS_TESTFILES/tvlstr.h5 $TESTDIR/tvlstr2.h5 # ############################################################################ # # Common usage @@ -1141,6 +1145,7 @@ TOOLTEST h5diff_487.txt -v --exclude-path "/group1/dset" h5diff_exclude3-1.h5 h5 # # diff various multiple vlen and fixed strings in a compound type dataset # ############################################################################## TOOLTEST h5diff_530.txt -v h5diff_comp_vl_strs.h5 h5diff_comp_vl_strs.h5 /group /group_copy +TOOLTEST h5diff_vlstr.txt -v tvlstr.h5 tvlstr2.h5 # ############################################################################## # # Test container types (array,vlen) with multiple nested compound types diff --git a/tools/test/h5dump/CMakeTestsXML.cmake b/tools/test/h5dump/CMakeTestsXML.cmake index 92e08a0..bdc6755 100644 --- a/tools/test/h5dump/CMakeTestsXML.cmake +++ b/tools/test/h5dump/CMakeTestsXML.cmake @@ -50,6 +50,7 @@ ${HDF5_TOOLS_DIR}/testfiles/tname-quot.h5 ${HDF5_TOOLS_DIR}/testfiles/tname-sp.h5 ${HDF5_TOOLS_DIR}/testfiles/tnamed_dtype_attr.h5 + ${HDF5_TOOLS_DIR}/testfiles/test35.nc ${HDF5_TOOLS_DIR}/testfiles/tnestedcomp.h5 ${HDF5_TOOLS_DIR}/testfiles/tnodata.h5 ${HDF5_TOOLS_DIR}/testfiles/tobjref.h5 @@ -99,6 +100,7 @@ ${HDF5_TOOLS_DIR}/testfiles/tempty-ns.h5.xml ${HDF5_TOOLS_DIR}/testfiles/tempty-ns-2.h5.xml ${HDF5_TOOLS_DIR}/testfiles/tenum.h5.xml + ${HDF5_TOOLS_DIR}/testfiles/test35.nc.xml ${HDF5_TOOLS_DIR}/testfiles/textlink.h5.xml ${HDF5_TOOLS_DIR}/testfiles/tfpformat.h5.xml ${HDF5_TOOLS_DIR}/testfiles/tgroup.h5.xml @@ -258,6 +260,8 @@ tempty.h5.out.err tenum.h5.out tenum.h5.out.err + test35.nc.out + test35.nc.out.err textlink.h5.out textlink.h5.out.err tfpformat.h5.out @@ -430,3 +434,6 @@ # tests for floating point user defined printf format ADD_XML_H5_TEST (tfpformat.h5 0 -u -m %.7f tfpformat.h5) + # test for HDFFV-10256 issue + ADD_XML_H5_TEST (test35.nc 0 test35.nc) + diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index 44c4369..2128acb 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -20,7 +20,6 @@ * trying it on a new platform, ...), you need to verify the correctness * of the expected output and update the corresponding *.ddl files. */ -#include <limits.h> #include "hdf5.h" #include "H5private.h" diff --git a/tools/test/h5dump/testh5dumpxml.sh.in b/tools/test/h5dump/testh5dumpxml.sh.in index 5f62946..b3b9e14 100644 --- a/tools/test/h5dump/testh5dumpxml.sh.in +++ b/tools/test/h5dump/testh5dumpxml.sh.in @@ -80,6 +80,7 @@ $SRC_H5DUMP_TESTFILES/tdset.h5 $SRC_H5DUMP_TESTFILES/tdset2.h5 $SRC_H5DUMP_TESTFILES/tempty.h5 $SRC_H5DUMP_TESTFILES/tenum.h5 +$SRC_H5DUMP_TESTFILES/test35.nc $SRC_H5DUMP_TESTFILES/textlink.h5 $SRC_H5DUMP_TESTFILES/tfpformat.h5 $SRC_H5DUMP_TESTFILES/tgroup.h5 @@ -144,6 +145,7 @@ $SRC_H5DUMP_TESTFILES/tempty-nons-uri.h5.xml $SRC_H5DUMP_TESTFILES/tempty-ns.h5.xml $SRC_H5DUMP_TESTFILES/tempty-ns-2.h5.xml $SRC_H5DUMP_TESTFILES/tenum.h5.xml +$SRC_H5DUMP_TESTFILES/test35.nc.xml $SRC_H5DUMP_TESTFILES/textlink.h5.xml $SRC_H5DUMP_TESTFILES/tfpformat.h5.xml $SRC_H5DUMP_TESTFILES/tgroup.h5.xml @@ -383,6 +385,9 @@ TOOLTEST torderattr4.h5.xml --xml -H --sort_by=creation_order --sort_order=desce # tests for floating point user defined printf format TOOLTEST tfpformat.h5.xml -u -m %.7f tfpformat.h5 +# test for HDFFV-10256 issue +TOOLTEST test35.nc.xml --xml test35.nc + # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/test/h5import/h5importtest.c b/tools/test/h5import/h5importtest.c index 00ae2e7..489bc01 100644 --- a/tools/test/h5import/h5importtest.c +++ b/tools/test/h5import/h5importtest.c @@ -11,7 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdio.h> #include "H5private.h" #ifdef H5_HAVE_WIN32_API diff --git a/tools/test/h5jam/h5jamgentest.c b/tools/test/h5jam/h5jamgentest.c index d713bb9..9f3d000 100644 --- a/tools/test/h5jam/h5jamgentest.c +++ b/tools/test/h5jam/h5jamgentest.c @@ -20,8 +20,6 @@ * trying it on a new platform, ...), you need to verify the correctness * of the expected output and update the corresponding *.ddl files. */ -#include <assert.h> -#include <limits.h> #include "hdf5.h" #include "H5private.h" diff --git a/tools/test/h5jam/tellub.c b/tools/test/h5jam/tellub.c index fad14b7..8e4b3ac 100644 --- a/tools/test/h5jam/tellub.c +++ b/tools/test/h5jam/tellub.c @@ -11,12 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdio.h> - -#ifdef H5_HAVE_UNISTD_H -#include <unistd.h> -#endif - #include "hdf5.h" #include "H5private.h" #include "h5tools.h" diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake index 897209c..225f6a8 100644 --- a/tools/test/h5repack/CMakeTests.cmake +++ b/tools/test/h5repack/CMakeTests.cmake @@ -76,6 +76,12 @@ ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_shuffle.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_soffset.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_szip.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_aggr.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fsm_aggr_nopersist.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fsm_aggr_persist.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_none.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_paged_nopersist.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_paged_persist.h5 # h5diff/testfile ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr1.h5 # tools/testfiles @@ -139,6 +145,12 @@ ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/4_vds.h5-vds_conti-v.ddl ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5-plugin_zero.tst ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/crtorder.tordergr.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/STG.h5repack_none.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/GS.h5repack_paged_nopersist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SP.h5repack_paged_persist.h5.ddl + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/SPT.h5repack_aggr.h5.ddl ) foreach (h5_file ${LIST_HDF5_TEST_FILES} ${LIST_OTHER_TEST_FILES}) @@ -238,7 +250,8 @@ if (HDF5_ENABLE_USING_MEMCHECKER) add_test ( NAME H5REPACK_CMP-${testname} - COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) else () add_test ( NAME H5REPACK_CMP-${testname} @@ -272,7 +285,8 @@ if (HDF5_ENABLE_USING_MEMCHECKER) add_test ( NAME H5REPACK_MASK-${testname} - COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) else (HDF5_ENABLE_USING_MEMCHECKER) add_test ( NAME H5REPACK_MASK-${testname} @@ -305,7 +319,8 @@ # If using memchecker add tests without using scripts add_test ( NAME H5REPACK_DMP-${testname} - COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile}) + COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${resultfile} + ) if (NOT "${last_test}" STREQUAL "") set_tests_properties (H5REPACK_DMP-${testname} PROPERTIES DEPENDS ${last_test}) endif () @@ -326,6 +341,40 @@ endif () endmacro () + macro (ADD_H5_STAT_TEST testname testtype resultcode statarg resultfile) + if ("${testtype}" STREQUAL "SKIP") + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_STAT-${testname}-SKIPPED + COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${statarg}.${resultfile}" + ) + endif () + else () + # If using memchecker add tests without using scripts + add_test ( + NAME H5REPACK_STAT-${testname} + COMMAND $<TARGET_FILE:h5repack> ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${resultfile} ${PROJECT_BINARY_DIR}/testfiles/out-${statarg}.${resultfile} + ) + if (NOT "${last_test}" STREQUAL "") + set_tests_properties (H5REPACK_STAT-${testname} PROPERTIES DEPENDS ${last_test}) + endif () + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_STAT-h5stat-${testname} + COMMAND "${CMAKE_COMMAND}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5stat>" + -D "TEST_ARGS:STRING=-S;-s;out-${statarg}.${resultfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=${resultfile}-${testname}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${statarg}.${resultfile}.ddl" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5REPACK_STAT-h5stat-${testname} PROPERTIES DEPENDS "H5REPACK_STAT-${testname}") + endif () + endif () + endmacro () + macro (ADD_H5_VERIFY_TEST testname testtype resultcode testfile testdset testfilter) if ("${testtype}" STREQUAL "SKIP") if (NOT HDF5_ENABLE_USING_MEMCHECKER) @@ -972,6 +1021,48 @@ set (TESTTYPE "TEST") ADD_H5_DMP_TEST (crtorder ${TESTTYPE} 0 ${arg}) +################################################################################################### +# Testing paged aggregation related options: +# -G pagesize +# -P 1 or 0 +# -S strategy +# -T threshold +# +# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_": +# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ +# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ +# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ +# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ +# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */ +# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ +# +##################################################################################################### +# + set (arg h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (SP_PAGE ${TESTTYPE} 0 SP ${arg}) + + set (arg h5repack_fsm_aggr_persist.h5 -S AGGR) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (S_AGGR ${TESTTYPE} 0 S ${arg}) + + set (arg h5repack_none.h5 -S PAGE -T 10 -G 2048) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (STG_PAGE ${TESTTYPE} 0 STG ${arg}) + + set (arg h5repack_paged_nopersist.h5 -G 512 -S AGGR) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (GS_AGGR ${TESTTYPE} 0 GS ${arg}) + + set (arg h5repack_paged_persist.h5 -S NONE -P 1) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (SP_NONE ${TESTTYPE} 0 SP ${arg}) + + set (arg h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5) + set (TESTTYPE "TEST") + ADD_H5_STAT_TEST (SPT_FSM_AGGR ${TESTTYPE} 0 SPT ${arg}) + + ######################################################### # layout options (these files have no filters) ######################################################### diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index bc6b527..e474bc3 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -36,6 +36,9 @@ H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary H5DUMP=../../src/h5dump/h5dump # The h5dump tool name H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary +H5STAT=../../src/h5stat/h5stat # The h5stat tool name +H5STAT_BIN=`pwd`/$H5STAT # The path of the h5stat tool binary + RM='rm -rf' CMP='cmp' DIFF='diff -c' @@ -101,6 +104,12 @@ $SRC_H5REPACK_TESTFILES/h5repack_refs.h5 $SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5 $SRC_H5REPACK_TESTFILES/h5repack_soffset.h5 $SRC_H5REPACK_TESTFILES/h5repack_szip.h5 +$SRC_H5REPACK_TESTFILES/h5repack_aggr.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_none.h5 +$SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5 +$SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5 $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 $SRC_TOOLS_TESTFILES/tfamily00000.h5 $SRC_TOOLS_TESTFILES/tfamily00001.h5 @@ -156,6 +165,12 @@ $SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl $SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl $SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl +$SRC_H5REPACK_TESTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl +$SRC_H5REPACK_TESTFILES/S.h5repack_fsm_aggr_persist.h5.ddl +$SRC_H5REPACK_TESTFILES/STG.h5repack_none.h5.ddl +$SRC_H5REPACK_TESTFILES/GS.h5repack_paged_nopersist.h5.ddl +$SRC_H5REPACK_TESTFILES/SP.h5repack_paged_persist.h5.ddl +$SRC_H5REPACK_TESTFILES/SPT.h5repack_aggr.h5.ddl " # @@ -707,6 +722,58 @@ TOOLTEST_DUMP() rm -f $outfile } +# This is similar to TOOLTEST_DUMP(). +# Test h5repack with options added for paged aggregation. +# h5stat is used on the repacked file and the expected output +# is compared for correctness. +# +TOOLTEST_STAT() +{ + infile=$2 + outfile=out-$1.$2 + expect="$TESTDIR/$1.$2.ddl" + actual="$TESTDIR/out-$1.$2.out" + actual_err="$TESTDIR/out-$1.$2.err" + + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + VERIFY h5stat output $@ + ( + cd $TESTDIR + $RUNSERIAL $H5STAT_BIN -S -s $outfile + ) >$actual 2>$actual_err + cat $actual_err >> $actual + + RET=$? + + fi + + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.ddl) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + # TOOLTEST_META: # Test metadata block size option. # Reason to create a function here is to localize all special steps related to @@ -1057,6 +1124,42 @@ fi arg="tordergr.h5 -L" TOOLTEST_DUMP crtorder $arg +################################################################################################### +# Testing paged aggregation related options: +# -G pagesize +# -P 1 or 0 +# -S strategy +# -T threshold +# +# The testfiles used are generated by test/gen_filespace.c and the file names are prepended with "h5repack_": +# (1) "fsm_aggr_nopersist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + not persisting free-space */ +# (2) "fsm_aggr_persist.h5" /* H5F_FSPACE_STRATEGY_FSM_AGGR + persisting free-space */ +# (3) "paged_nopersist.h5" /* H5F_FSPACE_STRATEGY_PAGE + not persisting free-space */ +# (4) "paged_persist.h5" /* H5F_FSPACE_STRATEGY_PAGE + persisting free-space */ +# (5) "aggr.h5" /* H5F_FSPACE_STRATEGY_AGGR */ +# (6) "none.h5" /* H5F_FSPACE_STRATEGY_NONE */ +# +##################################################################################################### +# +arg="h5repack_fsm_aggr_nopersist.h5 -S PAGE -P 1" +TOOLTEST_STAT SP $arg +# +arg="h5repack_fsm_aggr_persist.h5 -S AGGR" +TOOLTEST_STAT S $arg +# +arg="h5repack_none.h5 -S PAGE -T 10 -G 2048" +TOOLTEST_STAT STG $arg +# +arg="h5repack_paged_nopersist.h5 -G 512 -S AGGR" +TOOLTEST_STAT GS $arg +# +arg="h5repack_paged_persist.h5 -S NONE -P 1" +TOOLTEST_STAT SP $arg +# +arg="h5repack_aggr.h5 -S FSM_AGGR -P 1 -T 5" +TOOLTEST_STAT SPT $arg + + ######################################################### # layout options (these files have no filters) ######################################################### diff --git a/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl b/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl new file mode 100644 index 0000000..d8ca992 --- /dev/null +++ b/tools/test/h5repack/testfiles/GS.h5repack_paged_nopersist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-GS.h5repack_paged_nopersist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_AGGR +File space page size: 512 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 776 bytes +Total space: 2448 bytes diff --git a/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl b/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl new file mode 100644 index 0000000..87ccbc4 --- /dev/null +++ b/tools/test/h5repack/testfiles/S.h5repack_fsm_aggr_persist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-S.h5repack_fsm_aggr_persist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_AGGR +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 776 bytes +Total space: 2448 bytes diff --git a/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl new file mode 100644 index 0000000..affa93b --- /dev/null +++ b/tools/test/h5repack/testfiles/SP.h5repack_fsm_aggr_nopersist.h5.ddl @@ -0,0 +1,16 @@ +Filename: out-SP.h5repack_fsm_aggr_nopersist.h5 +Free-space persist: TRUE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + # of sections of size 1000 - 9999: 2 + Total # of sections: 2 +File space management strategy: H5F_FSPACE_STRATEGY_PAGE +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1602 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 6307 bytes/38.5% + Unaccounted space: 8075 bytes +Total space: 16384 bytes diff --git a/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl b/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl new file mode 100644 index 0000000..1084090 --- /dev/null +++ b/tools/test/h5repack/testfiles/SP.h5repack_paged_persist.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-SP.h5repack_paged_persist.h5 +Free-space persist: FALSE +Free-space section threshold: 1 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_NONE +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 0 bytes +Total space: 1672 bytes diff --git a/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl new file mode 100644 index 0000000..1ce06b2 --- /dev/null +++ b/tools/test/h5repack/testfiles/SPT.h5repack_aggr.h5.ddl @@ -0,0 +1,16 @@ +Filename: out-SPT.h5repack_aggr.h5 +Free-space persist: TRUE +Free-space section threshold: 5 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + # of sections of size 100 - 999: 1 + Total # of sections: 1 +File space management strategy: H5F_FSPACE_STRATEGY_FSM_AGGR +File space page size: 4096 bytes +Summary of file space information: + File metadata: 1485 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 680 bytes/26.5% + Unaccounted space: 0 bytes +Total space: 2565 bytes diff --git a/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl b/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl new file mode 100644 index 0000000..51cd7e5 --- /dev/null +++ b/tools/test/h5repack/testfiles/STG.h5repack_none.h5.ddl @@ -0,0 +1,15 @@ +Filename: out-STG.h5repack_none.h5 +Free-space persist: FALSE +Free-space section threshold: 10 bytes +Small size free-space sections (< 10 bytes): + Total # of small size sections: 0 +Free-space section bins: + Total # of sections: 0 +File space management strategy: H5F_FSPACE_STRATEGY_PAGE +File space page size: 2048 bytes +Summary of file space information: + File metadata: 1272 bytes + Raw data: 400 bytes + Amount/Percent of tracked free space: 0 bytes/0.0% + Unaccounted space: 2424 bytes +Total space: 4096 bytes diff --git a/tools/test/h5repack/testfiles/h5repack_aggr.h5 b/tools/test/h5repack/testfiles/h5repack_aggr.h5 Binary files differnew file mode 100644 index 0000000..8da0751 --- /dev/null +++ b/tools/test/h5repack/testfiles/h5repack_aggr.h5 diff --git a/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 Binary files differnew file mode 100644 index 0000000..1cd5f3f --- /dev/null +++ b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_nopersist.h5 diff --git a/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 Binary files differnew file mode 100644 index 0000000..469dbce --- /dev/null +++ b/tools/test/h5repack/testfiles/h5repack_fsm_aggr_persist.h5 diff --git a/tools/test/h5repack/testfiles/h5repack_none.h5 b/tools/test/h5repack/testfiles/h5repack_none.h5 Binary files differnew file mode 100644 index 0000000..2e4d789 --- /dev/null +++ b/tools/test/h5repack/testfiles/h5repack_none.h5 diff --git a/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 b/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 Binary files differnew file mode 100644 index 0000000..352e586 --- /dev/null +++ b/tools/test/h5repack/testfiles/h5repack_paged_nopersist.h5 diff --git a/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 b/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 Binary files differnew file mode 100644 index 0000000..64c3b23 --- /dev/null +++ b/tools/test/h5repack/testfiles/h5repack_paged_persist.h5 diff --git a/tools/test/h5repack/testh5repack_detect_szip.c b/tools/test/h5repack/testh5repack_detect_szip.c index e08d5ab..6e7a24e 100644 --- a/tools/test/h5repack/testh5repack_detect_szip.c +++ b/tools/test/h5repack/testh5repack_detect_szip.c @@ -11,7 +11,6 @@ * help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ -#include <stdio.h> #include "h5repack.h" #include "h5tools.h" #include "h5tools_utils.h" diff --git a/tools/test/misc/repart_test.c b/tools/test/misc/repart_test.c index 372f46a..4016ee8 100644 --- a/tools/test/misc/repart_test.c +++ b/tools/test/misc/repart_test.c @@ -12,19 +12,16 @@ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ /* - * Programmer: Raymond Lu <slu@ncsa.uiuc.edu> - * June 1, 2005 - * * Purpose: This program tests family files after being repartitioned * by h5repart. It simply tries to reopen the files with * correct family driver and member size. */ #include "hdf5.h" +#include "H5private.h" #define KB 1024 #define FAMILY_H5REPART_SIZE1 20000 #define FAMILY_H5REPART_SIZE2 (5*KB) -#define MAX(a,b) (a>b ? a:b) const char *FILENAME[] = { "fst_family%05d.h5", @@ -42,52 +39,54 @@ herr_t test_sec2_h5repart_opens(void); * * Purpose: Tries to reopen family files. * - * Return: Success: exit(0) - * - * Failure: exit(1) + * Return: SUCCEED/FAIL * - * Programmer: Raymond Lu - * June 1, 2005 - * - * Modifications: *------------------------------------------------------------------------- */ herr_t test_family_h5repart_opens(void) { - hid_t file=(-1), fapl=(-1); + hid_t fid = -1; + hid_t fapl_id = -1; /* open 1st file(single member file) with correct family size(20000 byte) */ - if ((fapl=H5Pcreate(H5P_FILE_ACCESS))<0) + if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0) goto error; - if(H5Pset_fapl_family(fapl, (hsize_t)FAMILY_H5REPART_SIZE1, H5P_DEFAULT)<0) + if (H5Pset_fapl_family(fapl_id, (hsize_t)FAMILY_H5REPART_SIZE1, H5P_DEFAULT) < 0) goto error; - if((file=H5Fopen(FILENAME[0], H5F_ACC_RDWR, fapl))<0) + if ((fid = H5Fopen(FILENAME[0], H5F_ACC_RDWR, fapl_id))<0) goto error; - if(H5Fclose(file)<0) + if (H5Fclose(fid) < 0) goto error; /* open 2nd file(multiple member files) with correct family size(5KB) */ - if(H5Pset_fapl_family(fapl, (hsize_t)FAMILY_H5REPART_SIZE2, H5P_DEFAULT)<0) + if (H5Pset_fapl_family(fapl_id, (hsize_t)FAMILY_H5REPART_SIZE2, H5P_DEFAULT) < 0) + goto error; + + if ((fid = H5Fopen(FILENAME[1], H5F_ACC_RDWR, fapl_id)) < 0) goto error; - if((file=H5Fopen(FILENAME[1], H5F_ACC_RDWR, fapl))<0) + if (H5Pclose(fapl_id) < 0) goto error; - if(H5Fclose(file)<0) + if (H5Fclose(fid) < 0) goto error; - return 0; + return SUCCEED; error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Pclose(fapl_id); + H5Fclose(fid); } H5E_END_TRY; - return -1; -} + + return FAIL; + +} /* end test_family_h5repart_opens() */ + /*------------------------------------------------------------------------- @@ -95,36 +94,32 @@ error: * * Purpose: Tries to reopen a sec2 file. * - * Return: Success: exit(0) + * Return: SUCCEED/FAIL * - * Failure: exit(1) - * - * Programmer: Raymond Lu - * June 21, 2005 - * - * Modifications: *------------------------------------------------------------------------- */ herr_t test_sec2_h5repart_opens(void) { - hid_t file=(-1); + hid_t fid = -1; /* open the sec2 file */ - if((file=H5Fopen(FILENAME[2], H5F_ACC_RDWR, H5P_DEFAULT))<0) + if ((fid = H5Fopen(FILENAME[2], H5F_ACC_RDWR, H5P_DEFAULT)) < 0) goto error; - if(H5Fclose(file)<0) + if (H5Fclose(fid) < 0) goto error; - return 0; + return SUCCEED; error: H5E_BEGIN_TRY { - H5Fclose(file); + H5Fclose(fid); } H5E_END_TRY; - return -1; -} + + return FAIL; + +} /* end test_sec2_h5repart_opens() */ /*------------------------------------------------------------------------- @@ -132,32 +127,26 @@ error: * * Purpose: Tests h5repart-ed family files * - * Return: Success: exit(0) - * - * Failure: exit(1) - * - * Programmer: Raymond Lu - * June 1, 2005 - * - * Modifications: + * Return: EXIT_SUCCESS/EXIT_FAILURE * *------------------------------------------------------------------------- */ int main(void) { - int nerrors=0; + int nerrors = 0; - nerrors += test_family_h5repart_opens()<0 ?1:0; - nerrors += test_sec2_h5repart_opens()<0 ?1:0; + nerrors += test_family_h5repart_opens() < 0 ? 1 : 0; + nerrors += test_sec2_h5repart_opens() < 0 ? 1 : 0; - if (nerrors) goto error; + if (nerrors) + goto error; - return 0; + HDexit(EXIT_SUCCESS); error: nerrors = MAX(1, nerrors); - printf("***** %d FAMILY FILE TEST%s FAILED! *****\n", + HDprintf("***** %d FAMILY FILE TEST%s FAILED! *****\n", nerrors, 1 == nerrors ? "" : "S"); - return 1; -} + HDexit(EXIT_FAILURE); +} /* end main() */ diff --git a/tools/test/misc/talign.c b/tools/test/misc/talign.c index 9a72557..ce866b4 100644 --- a/tools/test/misc/talign.c +++ b/tools/test/misc/talign.c @@ -15,9 +15,6 @@ * Small program to illustrate the "misalignment" of members within a compound * datatype, in a datatype fixed by H5Tget_native_type(). */ -#include <string.h> -#include <stdlib.h> -/*#include <unistd.h> *//* Required for unlink() */ #include "hdf5.h" #include "H5private.h" diff --git a/tools/test/perform/CMakeTests.cmake b/tools/test/perform/CMakeTests.cmake index 2933563..39faa73 100644 --- a/tools/test/perform/CMakeTests.cmake +++ b/tools/test/perform/CMakeTests.cmake @@ -146,6 +146,7 @@ else () -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" ) endif () +set_tests_properties (PERFORM_zip_perf PROPERTIES DEPENDS PERFORM_zip_perf_help) if (H5_HAVE_PARALLEL) add_test (NAME PERFORM_h5perf COMMAND ${MPIEXEC} ${MPIEXEC_PREFLAGS} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_MAX_NUMPROCS} ${MPIEXEC_POSTFLAGS} $<TARGET_FILE:h5perf>) diff --git a/tools/testfiles/test35.nc b/tools/testfiles/test35.nc Binary files differnew file mode 100644 index 0000000..4bd5d7f --- /dev/null +++ b/tools/testfiles/test35.nc diff --git a/tools/testfiles/test35.nc.xml b/tools/testfiles/test35.nc.xml new file mode 100644 index 0000000..a38ed8a --- /dev/null +++ b/tools/testfiles/test35.nc.xml @@ -0,0 +1,20 @@ +<?xml version="1.0" encoding="UTF-8"?> +<hdf5:HDF5-File xmlns:hdf5="http://hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://hdfgroup.org/HDF5/XML/schema/HDF5-File http://www.hdfgroup.org/HDF5/XML/schema/HDF5-File.xsd"> +<hdf5:RootGroup OBJ-XID="xid_48" H5Path="/"> + <hdf5:Attribute Name="aaa"> + <hdf5:Dataspace> + <hdf5:ScalarDataspace /> + </hdf5:Dataspace> + <hdf5:DataType> + <hdf5:AtomicType> + <hdf5:StringType Cset="H5T_CSET_ASCII" StrSize="14108" StrPad="H5T_STR_NULLTERM"/> + </hdf5:AtomicType> + </hdf5:DataType> + <hdf5:Data> + <hdf5:DataFromFile> + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaa>aaaa<aaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>a<aaaaaa>aaaaaaaa<aaaaaa>a<aaaaaa>aaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaa>aaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaa<aaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"aaaaa\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaa<aaaa>aaaa<aaaa>>aaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaa>aaaaaaaaaaaaaaaaaa<aaaa>aaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<a>aaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaa<>aaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaa<aa>aaaaaaaaaaaaaaaaaaa<aaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaa<aa>aaaaaaaaaaaaaaaaaaaa<aaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaa<aa>aaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaaaaaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaa>aaaaa<aaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa<aaaaaaa>aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + </hdf5:DataFromFile> + </hdf5:Data> + </hdf5:Attribute> +</hdf5:RootGroup> +</hdf5:HDF5-File> |