From 63ebb100e4aa4d2cc1117ca7ebc67a2f071119d1 Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Tue, 19 Oct 2010 14:11:23 -0500 Subject: [svn-r19641] Description: Bring r19542:19639 from trunk to revise_chunks branch. Tested on: FreeBSD/32 6.3 (duty) in debug mode FreeBSD/64 6.3 (liberty) w/C++ & FORTRAN, in debug mode Linux/32 2.6 (jam) w/PGI compilers, w/default API=1.8.x, w/C++ & FORTRAN, w/threadsafe, in debug mode Linux/64-amd64 2.6 (amani) w/Intel compilers, w/default API=1.6.x, w/C++ & FORTRAN, in production mode Solaris/32 2.10 (linew) w/deprecated symbols disabled, w/C++ & FORTRAN, w/szip filter, w/threadsafe, in production mode Linux/PPC 2.6 (heiwa) w/C++ & FORTRAN, w/threadsafe, in debug mode Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN, in production mode Linux/64-amd64 2.6 (abe) w/parallel, w/FORTRAN, in debug mode Mac OS X/32 10.6.4 (amazon) in debug mode Mac OS X/32 10.6.4 (amazon) w/C++ & FORTRAN, w/threadsafe, in production mode Mac OS X/32 10.6.4 (amazon) w/parallel, in debug mode --- CMakeLists.txt | 76 +- MANIFEST | 3 + README.txt | 2 +- c++/src/Makefile.in | 2 +- config/apple | 21 + config/ibm-aix | 12 +- config/ibm-aix6.x | 21 + config/ibm-flags | 2 +- config/intel-flags | 14 +- config/lt_vers.am | 2 +- config/powerpc-ibm-aix5.x | 151 +-- configure | 25 +- configure.in | 2 +- fortran/src/Makefile.in | 2 +- hl/c++/src/Makefile.in | 2 +- hl/fortran/src/Makefile.in | 2 +- hl/src/Makefile.in | 2 +- release_docs/CMake.txt | 15 +- release_docs/RELEASE.txt | 5 +- src/H5AC.c | 34 + src/H5ACprivate.h | 2 + src/H5B.c | 373 ++++--- src/H5C.c | 133 ++- src/H5Cprivate.h | 3 + src/H5D.c | 14 + src/H5Dbtree.c | 4 +- src/H5Dchunk.c | 22 +- src/H5Dint.c | 4 +- src/H5Dpkg.h | 3 + src/H5E.c | 52 +- src/H5Edeprec.c | 26 +- src/H5Eint.c | 18 +- src/H5Epkg.h | 19 +- src/H5F.c | 2 +- src/H5Faccum.c | 736 ++++++++----- src/H5Fio.c | 24 +- src/H5Fpkg.h | 11 +- src/H5Fsuper.c | 61 +- src/H5Fsuper_cache.c | 4 +- src/H5G.c | 16 + src/H5Gobj.c | 5 +- src/H5Groot.c | 4 + src/H5Gtraverse.c | 10 +- src/H5L.c | 22 +- src/H5O.c | 57 +- src/H5Ocache.c | 74 ++ src/H5Opkg.h | 1 + src/H5Oprivate.h | 4 +- src/H5Otest.c | 49 + src/H5Tcommit.c | 17 +- src/H5public.h | 4 +- src/Makefile.in | 2 +- test/CMakeLists.txt | 1 + test/Makefile.am | 6 +- test/Makefile.in | 69 +- test/accum.c | 1809 +++++++++++++++++++++++++++++++ test/err_compat.c | 394 +++++-- test/ohdr.c | 175 ++- test/testerror.sh.in | 2 + test/testfiles/err_compat_1 | 40 +- testpar/t_mdset.c | 512 +++++++++ testpar/testphdf5.c | 7 + testpar/testphdf5.h | 1 + tools/h5copy/CMakeLists.txt | 3 - tools/h5copy/h5copy.c | 605 ++++++----- tools/h5copy/testfiles/h5copy_misc1.out | 3 + tools/h5copy/testh5copy.sh | 110 +- tools/h5stat/testfiles/h5stat_tsohm.ddl | 8 +- tools/h5stat/testfiles/h5stat_tsohm.h5 | Bin 3887 -> 3850 bytes tools/h5stat/testh5stat.sh.in | 2 +- tools/testfiles/file_space.h5 | Bin 792 -> 792 bytes vms/src/h5pubconf.h | 6 +- windows/src/H5pubconf.h | 6 +- windows/tools/h5copy/testh5copy.bat | 12 +- 74 files changed, 4709 insertions(+), 1233 deletions(-) create mode 100644 config/ibm-aix6.x create mode 100644 test/accum.c create mode 100644 tools/h5copy/testfiles/h5copy_misc1.out diff --git a/CMakeLists.txt b/CMakeLists.txt index 49a4d5f..10503ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -445,6 +445,18 @@ HDF5_SETUP_FILTERS (SCALEOFFSET) INCLUDE (ExternalProject) OPTION (HDF5_ALLOW_EXTERNAL_SUPPORT "Allow External Library Building" "NO") +OPTION (ZLIB_USE_EXTERNAL "Use External Library Building for ZLIB" 0) +OPTION (SZIP_USE_EXTERNAL "Use External Library Building for SZIP" 0) +IF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") + SET (ZLIB_USE_EXTERNAL 1) + SET (SZIP_USE_EXTERNAL 1) + IF (NOT ZLIB_SVN_URL) + SET (ZLIB_SVN_URL "http://svn.hdfgroup.uiuc.edu/zlib/trunk") + ENDIF (NOT ZLIB_SVN_URL) + IF (NOT SZIP_SVN_URL) + SET (SZIP_SVN_URL "http://svn.hdfgroup.uiuc.edu/szip/trunk") + ENDIF (NOT SZIP_SVN_URL) +ENDIF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") #----------------------------------------------------------------------------- # Option for ZLib support @@ -452,7 +464,9 @@ OPTION (HDF5_ALLOW_EXTERNAL_SUPPORT "Allow External Library Building" "NO") OPTION (HDF5_ENABLE_Z_LIB_SUPPORT "Enable Zlib Filters" OFF) IF (HDF5_ENABLE_Z_LIB_SUPPORT) IF (NOT H5_ZLIB_HEADER) - FIND_PACKAGE (ZLIB) + IF (NOT ZLIB_USE_EXTERNAL) + FIND_PACKAGE (ZLIB) + ENDIF (NOT ZLIB_USE_EXTERNAL) IF (ZLIB_FOUND) SET (H5_HAVE_FILTER_DEFLATE 1) SET (H5_HAVE_ZLIB_H 1) @@ -461,22 +475,35 @@ IF (HDF5_ENABLE_Z_LIB_SUPPORT) ELSE (ZLIB_FOUND) IF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") EXTERNALPROJECT_ADD (ZLIB - SVN_REPOSITORY http://svn.hdfgroup.uiuc.edu/zlib/trunk + SVN_REPOSITORY ${ZLIB_SVN_URL} # [SVN_REVISION rev] INSTALL_COMMAND "" CMAKE_ARGS + -DBLDTYPE:STRING="Release" -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS} ) EXTERNALPROJECT_GET_PROPERTY (ZLIB BINARY_DIR SOURCE_DIR) IF (BUILD_SHARED_LIBS) - SET (ZLIB_LIBRARY - "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}zlib1${CMAKE_IMPORT_LIBRARY_SUFFIX}" - ) + IF (WIN32 AND NOT MINGW) + SET (ZLIB_LIBRARY + "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}zlib1${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + ELSE (WIN32 AND NOT MINGW) + SET (ZLIB_LIBRARY + "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}z${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + ENDIF (WIN32 AND NOT MINGW) ELSE (BUILD_SHARED_LIBS) - SET (ZLIB_LIBRARY - "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/libzlib${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) + IF (WIN32 AND NOT MINGW) + SET (ZLIB_LIBRARY + "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/zlib${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + ELSE (WIN32 AND NOT MINGW) + SET (ZLIB_LIBRARY + "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/libz${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) + ENDIF (WIN32 AND NOT MINGW) ENDIF (BUILD_SHARED_LIBS) SET (ZLIB_INCLUDE_DIR_GEN "${BINARY_DIR}" @@ -491,6 +518,7 @@ IF (HDF5_ENABLE_Z_LIB_SUPPORT) SET (H5_HAVE_FILTER_DEFLATE 1) SET (H5_HAVE_ZLIB_H 1) SET (H5_HAVE_LIBZ 1) + MESSAGE (STATUS "Filter ZLIB is built") ELSE (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") MESSAGE (FATAL " ZLib is Required for ZLib support in HDF5") ENDIF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") @@ -513,7 +541,9 @@ SET (CMAKE_MODULE_PATH ${HDF5_RESOURCES_DIR} ${CMAKE_MODULE_PATH}) OPTION (HDF5_ENABLE_SZIP_SUPPORT "Use SZip Filter" OFF) IF (HDF5_ENABLE_SZIP_SUPPORT) OPTION (HDF5_ENABLE_SZIP_ENCODING "Use SZip Encoding" OFF) - FIND_PACKAGE (SZIP) + IF (NOT SZIP_USE_EXTERNAL) + FIND_PACKAGE (SZIP) + ENDIF (NOT SZIP_USE_EXTERNAL) IF (SZIP_FOUND) SET (H5_HAVE_FILTER_SZIP 1) SET (H5_HAVE_SZLIB_H 1) @@ -521,10 +551,11 @@ IF (HDF5_ENABLE_SZIP_SUPPORT) ELSE (SZIP_FOUND) IF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") EXTERNALPROJECT_ADD (SZIP - SVN_REPOSITORY http://svn.hdfgroup.uiuc.edu/szip/trunk + SVN_REPOSITORY ${SZIP_SVN_URL} # [SVN_REVISION rev] INSTALL_COMMAND "" CMAKE_ARGS + -DBLDTYPE:STRING="Release" -DBUILD_SHARED_LIBS:BOOL=${BUILD_SHARED_LIBS} -DSZIP_ENABLE_ENCODING:BOOL=${HDF5_ENABLE_SZIP_ENCODING} ) @@ -532,13 +563,19 @@ IF (HDF5_ENABLE_SZIP_SUPPORT) IF (BUILD_SHARED_LIBS) - SET (SZIP_LIBRARY - "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}szip${CMAKE_IMPORT_LIBRARY_SUFFIX}" - ) + IF (WIN32 AND NOT MINGW) + SET (SZIP_LIBRARY + "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/${CMAKE_IMPORT_LIBRARY_PREFIX}szip${CMAKE_IMPORT_LIBRARY_SUFFIX}" + ) + ELSE (WIN32 AND NOT MINGW) + SET (SZIP_LIBRARY + "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/${CMAKE_SHARED_LIBRARY_PREFIX}szip${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + ENDIF (WIN32 AND NOT MINGW) ELSE (BUILD_SHARED_LIBS) - SET (SZIP_LIBRARY - "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/libszip${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) + SET (SZIP_LIBRARY + "${BINARY_DIR}/bin/${CMAKE_CFG_INTDIR}/libszip${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) ENDIF (BUILD_SHARED_LIBS) SET (SZIP_INCLUDE_DIR_GEN "${BINARY_DIR}" @@ -553,6 +590,7 @@ IF (HDF5_ENABLE_SZIP_SUPPORT) SET (H5_HAVE_FILTER_SZIP 1) SET (H5_HAVE_SZLIB_H 1) SET (H5_HAVE_LIBSZ 1) + MESSAGE (STATUS "Filter SZIP is built") ELSE (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") MESSAGE (FATAL_ERROR "SZIP is Required for SZIP support in HDF5") ENDIF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") @@ -582,6 +620,9 @@ IF (WIN32 AND NOT CYGWIN) COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ZLIB_BIN_PATH}/${ZLIB_DLL_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INTDIR}/ COMMENT "Copying ${ZLIB_BIN_PATH}/${ZLIB_DLL_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} to ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INTDIR}/" ) + IF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") + ADD_DEPENDENCIES (ZLIB-Release-Copy ZLIB) + ENDIF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") ENDIF (HDF5_ENABLE_Z_LIB_SUPPORT AND ZLIB_FOUND) IF (HDF5_ENABLE_SZIP_SUPPORT AND SZIP_FOUND) @@ -593,6 +634,9 @@ IF (WIN32 AND NOT CYGWIN) COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SZIP_BIN_PATH}/${SZIP_DLL_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INTDIR}/ COMMENT "Copying ${SZIP_BIN_PATH}/${SZIP_DLL_NAME}${CMAKE_SHARED_LIBRARY_SUFFIX} to ${CMAKE_RUNTIME_OUTPUT_DIRECTORY}/${CMAKE_CFG_INTDIR}/" ) + IF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") + ADD_DEPENDENCIES (SZIP-Release-Copy SZIP) + ENDIF (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN") ENDIF (HDF5_ENABLE_SZIP_SUPPORT AND SZIP_FOUND) ENDIF (BUILD_SHARED_LIBS) diff --git a/MANIFEST b/MANIFEST index b1f4ee2..b01cffe 100644 --- a/MANIFEST +++ b/MANIFEST @@ -96,6 +96,7 @@ ./config/i686-pc-cygwin ./config/ia64-linux-gnu ./config/ibm-aix +./config/ibm-aix6.x ./config/ibm-flags ./config/intel-fflags ./config/intel-flags @@ -810,6 +811,7 @@ ./test/H5srcdir_str.h.in ./test/Makefile.am ./test/Makefile.in +./test/accum.c ./test/app_ref.c ./test/be_data.h5 ./test/be_extlink1.h5 @@ -1738,6 +1740,7 @@ ./tools/h5copy/testfiles/h5copy_extlinks_src.h5 ./tools/h5copy/testfiles/h5copy_extlinks_trg.h5 ./tools/h5copy/testfiles/h5copy_extlinks_src.out.ls +./tools/h5copy/testfiles/h5copy_misc1.out ./tools/h5copy/testfiles/h5copytst_new.h5 ./tools/h5copy/testfiles/h5copytst_new.out.ls diff --git a/README.txt b/README.txt index e117db2..be67850 100644 --- a/README.txt +++ b/README.txt @@ -1,4 +1,4 @@ -HDF5 version 1.9.76-FA_a5 currently under development +HDF5 version 1.9.77-FA_a5 currently under development Please refer to the release_docs/INSTALL file for installation instructions. ------------------------------------------------------------------------------ diff --git a/c++/src/Makefile.in b/c++/src/Makefile.in index 27505d5..70d71db 100644 --- a/c++/src/Makefile.in +++ b/c++/src/Makefile.in @@ -402,7 +402,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog # Add libtool shared library version numbers to the HDF5 library # See libtool versioning documentation online. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 66 +LT_VERS_REVISION = 67 LT_VERS_AGE = 0 # Include src directory diff --git a/config/apple b/config/apple index 10674fb..babcdad 100644 --- a/config/apple +++ b/config/apple @@ -62,6 +62,11 @@ case $CC in grep 'GCC' | sed 's/.*\((GCC) [-a-z0-9\. ]*.*\)/\1/'` ;; + *icc*) + cc_version_info=`$CC $CCFLAGS $H5_CCFLAGS -V 2>&1 | grep 'Version' |\ + sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` + ;; + *) echo "No match to get cc_version_info for $CC" ;; @@ -72,6 +77,16 @@ case $FC in fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS --version 2>&1 |\ grep 'GCC' | sed 's/\(.*(GCC) [-a-z0-9\. ]*\).*/\1/'` ;; + + *ifc*|*ifort*) + fc_version_info=`$FC $FCFLAGS $H5_FCFLAGS -V 2>&1 | grep 'Version' |\ + sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` + ;; + + *) + echo "No match to get fc_version_info for $FC" + ;; + esac # get c++ version info @@ -80,6 +95,12 @@ case $CXX in cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS --version 2>&1 |\ grep 'GCC' | sed 's/.*\((GCC) [-a-z0-9\. ]*.*\)/\1/'` ;; + + *icpc*) + cxx_version_info=`$CXX $CXXFLAGS $H5_CXXFLAGS -V 2>&1 | grep 'Version' |\ + sed 's/\(Intel.* Compiler\).*\( Version [a-z0-9\.]*\).*\( Build [0-9]*\)/\1\2\3/'` + ;; + *) echo "No match to get cxx_version_info for $CXX" ;; diff --git a/config/ibm-aix b/config/ibm-aix index 134ad8e..2fb6bc4 100644 --- a/config/ibm-aix +++ b/config/ibm-aix @@ -31,15 +31,17 @@ if test "X-" = "X-$CC"; then fi # Define RUNPARALLEL if parallel mode is enabled or a parallel compiler used. +# Define RUNSERIAL: Temporary patch for Bug 1678. -q32 binary default to run +# with smaller memory. +# Ask for more memory so that "make check" will pass. Not necessary for -q64 +# mode but it does no harm. if test "X-$enable_parallel" = "X-yes" -o X-$CC_BASENAME = X-mpcc_r; then RUNPARALLEL=${RUNPARALLEL="env MP_PROCS=\$\${NPROCS:=3} MP_TASKS_PER_NODE=\$\${NPROCS:=3} poe"} + RUNSERIAL=${RUNSERIAL="env LDR_CNTRL=MAXDATA=0x20000000@DSA MP_PROCS=1 MP_TASKS_PER_NODE=1 poe"} +else + RUNSERIAL=${RUNSERIAL="env LDR_CNTRL=MAXDATA=0x20000000@DSA"} fi -# Temporary patch for Bug 1678. -q32 binary default to run with smaller memory. -# Ask for more memory so that "make check" will pass. Not necessary for -q64 -# mode but it does no harm. -RUNSERIAL=${RUNSERIAL="env LDR_CNTRL=MAXDATA=0x20000000@DSA"} - #---------------------------------------------------------------------------- # Compiler flags. The CPPFLAGS values should not include package debug diff --git a/config/ibm-aix6.x b/config/ibm-aix6.x new file mode 100644 index 0000000..104008a --- /dev/null +++ b/config/ibm-aix6.x @@ -0,0 +1,21 @@ +# -*- shell-script -*- +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the files COPYING and Copyright.html. COPYING can be found at the root +# of the source code distribution tree; Copyright.html can be found at the +# root level of an installed copy of the electronic HDF5 document set and +# is linked from the top-level documents page. It can also be found at +# http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have +# access to either file, you may request a copy from help@hdfgroup.org. + +# Configuration file for building on the IBM AIX 6.X platforms. +# This file is part of the HDF5 build script. It is processed shortly +# after configure starts and defines, among other things, flags for +# the various compile modes. + +# Use the generic ibm-aix. +. $srcdir/config/ibm-aix diff --git a/config/ibm-flags b/config/ibm-flags index 9e63f79..5e092b8 100644 --- a/config/ibm-flags +++ b/config/ibm-flags @@ -28,7 +28,7 @@ # if test X = "X$cc_flags_set"; then # Verify this is an IBM XL compiler - cc_version="`$CC $CFLAGS -qversion 2>&1 | grep '^IBM XL C/C++'`" + cc_version="`$CC $CFLAGS -qversion 2>&1 | grep 'IBM XL C/C++'`" if test X != "X$cc_version"; then cc_vendor="XL" cc_version="`$CC $CFLAGS -qversion 2>&1 | sed -n 's/Version: \([0-9\.]*\).*/\1/p'`" diff --git a/config/intel-flags b/config/intel-flags index 5bf06d4..e6a9c31 100644 --- a/config/intel-flags +++ b/config/intel-flags @@ -70,13 +70,8 @@ if test "X-icc" = "X-$cc_vendor"; then # Default to C99 standard. H5_CFLAGS="${H5_CFLAGS:--std=c99 $arch}" - # Production - # -Wl,-s to remove all symbols for smaller file - # O3 optimization causes compilation failures on many platforms; - # the problem exists in all versions of the icc compiler up to the latest 9.1 - # I changed optimization flag to default -O2. EIP, 2006-08-15 - #PROD_CFLAGS="-O3 -Wl,-s" - PROD_CFLAGS="-O2 -Wl,-s" + # Production is set to default; see settings for specific version further down + PROD_CFLAGS="-O" PROD_CPPFLAGS= # Debug @@ -98,6 +93,11 @@ fi # Please follow the pattern below by adding new versions at the top, copying # the information from the previous version and adding modifications to that. case "$cc_vendor-$cc_version" in + icc-11*) + # -s became obsolete; we also fixed bugs that allow us to enable higher level + # of optimization starting with 1.8.7 + PROD_CFLAGS="-O3" + ;; icc-10*) PROD_CFLAGS="-O1 -Wl,-s" ;; diff --git a/config/lt_vers.am b/config/lt_vers.am index 0d5db8d..b28eed0 100644 --- a/config/lt_vers.am +++ b/config/lt_vers.am @@ -17,7 +17,7 @@ # Add libtool shared library version numbers to the HDF5 library # See libtool versioning documentation online. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 66 +LT_VERS_REVISION = 67 LT_VERS_AGE = 0 ## If the API changes *at all*, increment LT_VERS_INTERFACE and diff --git a/config/powerpc-ibm-aix5.x b/config/powerpc-ibm-aix5.x index b39fb47..50a4e8c 100644 --- a/config/powerpc-ibm-aix5.x +++ b/config/powerpc-ibm-aix5.x @@ -1,7 +1,6 @@ # -*- shell-script -*- # # Copyright by The HDF Group. -# Copyright by the Board of Trustees of the University of Illinois. # All rights reserved. # # This file is part of HDF5. The full HDF5 copyright notice, including @@ -18,151 +17,5 @@ # after configure starts and defines, among other things, flags for # the various compile modes. -# Use AIX supplied C compiler by default, xlc for serial, mpcc_r for parallel. -# Make sure this is applied to other API compile options such as C++. -if test "X-" = "X-$CC"; then - if test "X-$enable_parallel" = "X-yes"; then - CC=mpcc_r - CC_BASENAME=mpcc_r - else - CC=xlc - CC_BASENAME=xlc - fi -fi - -# Define RUNPARALLEL if parallel mode is enabled or a parallel compiler used. -if test "X-$enable_parallel" = "X-yes" -o X-$CC_BASENAME = X-mpcc_r; then - RUNPARALLEL=${RUNPARALLEL="env MP_PROCS=\$\${NPROCS:=3} MP_TASKS_PER_NODE=\$\${NPROCS:=3} poe"} -fi - -# Temporary patch for Bug 1678. -q32 binary default to run with smaller memory. -# Ask for more memory so that "make check" will pass. Not necessary for -q64 -# mode but it does no harm. -RUNSERIAL=${RUNSERIAL="env LDR_CNTRL=MAXDATA=0x20000000@DSA"} - - -#---------------------------------------------------------------------------- -# Compiler flags. The CPPFLAGS values should not include package debug -# flags like `-DH5G_DEBUG' since these are added with the -# `--enable-debug' switch of configure. - -case $CC_BASENAME in - xlc|xlc-*|mpcc_r|mpcc_r-*) - # Turn off shared lib option. It causes some test suite to fail. - enable_shared="${enable_shared:-no}" - # Make sure this is applied to other API compile options such as C++. - AM_CFLAGS="$AM_CFLAGS" - H5_CFLAGS="-qlanglvl=stdc99 $H5_CFLAGS" - DEBUG_CFLAGS="-g -qfullpath" - DEBUG_CPPFLAGS= - # -O causes test/dtypes to fail badly. Turn it off for now. - PROD_CFLAGS="" - PROD_CPPFLAGS= - PROFILE_CFLAGS="-g -qfullpath -pg" - PROFILE_CPPFLAGS= - ;; - - gcc) - . $srcdir/config/gnu-flags - ;; - - *) - H5_CFLAGS="$H5_CFLAGS -ansi" - DEBUG_CFLAGS="-g" - DEBUG_CPPFLAGS= - PROD_CFLAGS="-O" - PROD_CPPFLAGS= - PROFILE_CFLAGS="-pg" - PROFILE_CPPFLAGS= - ;; -esac - -#---------------------------------------------------------------------------- -# Values for overriding configuration tests when cross compiling. -# This includes compiling on some machines where the serial front end -# compiles for a parallel back end. - -# Set this to `yes' or `no' depending on whether the target is big -# endian or little endian. -hdf5_cv_printf_ll=${hdf5_cv_printf_ll='ll'} -ac_cv_c_bigendian=${ac_cv_c_bigendian='yes'} -ac_cv_header_stdc=${ac_cv_header_stdc='yes'} -ac_cv_header_sys_ioctl_h=${ac_cv_header_sys_ioctl_h=yes} - -# cache the sizeof of "standard C types" so that configure can run faster. -ac_cv_sizeof_char=${ac_cv_sizeof_char=1} -ac_cv_sizeof_short=${ac_cv_sizeof_short=2} -ac_cv_sizeof_int=${ac_cv_sizeof_int=4} -ac_cv_sizeof_long_long=${ac_cv_sizeof_long_long=8} -ac_cv_sizeof___int64=${ac_cv_sizeof___int64=8} -ac_cv_sizeof_float=${ac_cv_sizeof_float=4} -ac_cv_sizeof_double=${ac_cv_sizeof_double=8} -ac_cv_sizeof_long_double=${ac_cv_sizeof_long_double=8} -ac_cv_sizeof_int8_t=${ac_cv_sizeof_int8_t=1} -ac_cv_sizeof_uint8_t=${ac_cv_sizeof_uint8_t=1} -ac_cv_sizeof_int_least8_t=${ac_cv_sizeof_int_least8_t=1} -ac_cv_sizeof_uint_least8_t=${ac_cv_sizeof_uint_least8_t=1} -# Do not cache int_fast8_t since the vendor changes often. -ac_cv_sizeof_int16_t=${ac_cv_sizeof_int16_t=2} -ac_cv_sizeof_uint16_t=${ac_cv_sizeof_uint16_t=2} -ac_cv_sizeof_int_least16_t=${ac_cv_sizeof_int_least16_t=2} -ac_cv_sizeof_uint_least16_t=${ac_cv_sizeof_uint_least16_t=2} -# Do not cache int_fast16_t since the vendor changes often. -ac_cv_sizeof_int32_t=${ac_cv_sizeof_int32_t=4} -ac_cv_sizeof_uint32_t=${ac_cv_sizeof_uint32_t=4} -ac_cv_sizeof_int_least32_t=${ac_cv_sizeof_int_least32_t=4} -ac_cv_sizeof_uint_least32_t=${ac_cv_sizeof_uint_least32_t=4} -ac_cv_sizeof_int_fast32_t=${ac_cv_sizeof_int_fast32_t=4} -ac_cv_sizeof_uint_fast32_t=${ac_cv_sizeof_uint_fast32_t=4} -ac_cv_sizeof_int64_t=${ac_cv_sizeof_int64_t=8} -ac_cv_sizeof_uint64_t=${ac_cv_sizeof_uint64_t=8} -ac_cv_sizeof_int_least64_t=${ac_cv_sizeof_int_least64_t=8} -ac_cv_sizeof_uint_least64_t=${ac_cv_sizeof_uint_least64_t=8} -ac_cv_sizeof_int_fast64_t=${ac_cv_sizeof_int_fast64_t=8} -ac_cv_sizeof_uint_fast64_t=${ac_cv_sizeof_uint_fast64_t=8} - -# Don't cache long since it varies between 32 and 64 bits -#ac_cv_sizeof_long=${ac_cv_sizeof_long=4} - -# The default Fortran 90 compiler - -if test "X-" = "X-$FC"; then - if test "X-$enable_parallel" = "X-yes"; then - FC=mpxlf90_r - else - FC=xlf90 - fi -fi - -# While we try to avoid setting FCFLAGS directly for use in compilation, in -# this case we need the -k flag present for some configure checks. As such, -# the configure script saves the user's set FCFLAGS before running, and -# restores them when complete. We must then set up both FCFLAGS and H5_FCFLAGS -# to ensure the flag is present for both configure as well as for the build. -if test "X-" = "X-$f9x_flags_set"; then - F9XSUFFIXFLAG="-qsuffix=f=f90" - FCFLAGS="$FCFLAGS -O ${F9XSUFFIXFLAG}" - H5_FCFLAGS="$H5_FCFLAGS -O ${F9XSUFFIXFLAG}" - FSEARCH_DIRS="-I./ -I../src" - DEBUG_FCFLAGS="-O" - PROD_FCFLAGS="-O" - PROFILE_FCFLAGS="-O" - f9x_flags_set=yes -fi - -# With poe version 3.2.0.19 or lower(using lpp -l all | grep ppe.poe to check the version number, -# IBM MPI-IO implementation has a bug, -#it cannot generate correct MPI derived datatype. Please uncomment the following line: -#hdf5_cv_mpi_complex_derived_datatype_works=${hdf5_cv_mpi_complex_derived_datatype_works='no'} - -# The default C++ compiler - -# Use AIX supplied C++ compiler by default. -CXX=${CXX=xlC} - -# Added -qweaksymbol to suppress linker messages warning of duplicate -# symbols; these warnings are harmless. - BMR -H5_CXXFLAGS="$H5_CXXFLAGS -qweaksymbol" -AM_CXXFLAGS="$AM_CXXFLAGS" - - +# Use the generic ibm-aix. +. $srcdir/config/ibm-aix diff --git a/configure b/configure index fd85f7e..40df2ad 100755 --- a/configure +++ b/configure @@ -1,7 +1,7 @@ #! /bin/sh -# From configure.in Id: configure.in 19350 2010-09-04 02:57:04Z koziol . +# From configure.in Id: configure.in 19543 2010-10-08 01:55:21Z koziol . # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.67 for HDF5 1.9.75-FA_a5. +# Generated by GNU Autoconf 2.67 for HDF5 1.9.77-FA_a5. # # Report bugs to . # @@ -563,8 +563,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='HDF5' PACKAGE_TARNAME='hdf5' -PACKAGE_VERSION='1.9.75-FA_a5' -PACKAGE_STRING='HDF5 1.9.75-FA_a5' +PACKAGE_VERSION='1.9.77-FA_a5' +PACKAGE_STRING='HDF5 1.9.77-FA_a5' PACKAGE_BUGREPORT='help@hdfgroup.org' PACKAGE_URL='' @@ -1437,7 +1437,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures HDF5 1.9.75-FA_a5 to adapt to many kinds of systems. +\`configure' configures HDF5 1.9.77-FA_a5 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1507,7 +1507,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of HDF5 1.9.75-FA_a5:";; + short | recursive ) echo "Configuration of HDF5 1.9.77-FA_a5:";; esac cat <<\_ACEOF @@ -1693,7 +1693,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -HDF5 configure 1.9.75-FA_a5 +HDF5 configure 1.9.77-FA_a5 generated by GNU Autoconf 2.67 Copyright (C) 2010 Free Software Foundation, Inc. @@ -2782,7 +2782,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by HDF5 $as_me 1.9.75-FA_a5, which was +It was created by HDF5 $as_me 1.9.77-FA_a5, which was generated by GNU Autoconf 2.67. Invocation command line was $ $0 $@ @@ -3603,7 +3603,7 @@ fi # Define the identity of the package. PACKAGE='hdf5' - VERSION='1.9.75-FA_a5' + VERSION='1.9.77-FA_a5' cat >>confdefs.h <<_ACEOF @@ -3859,6 +3859,9 @@ case $host_os in aix5.*) host_os_novers=aix5.x ;; + aix6.*) + host_os_novers=aix6.x + ;; freebsd*) host_os_novers=freebsd ;; @@ -28797,7 +28800,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by HDF5 $as_me 1.9.75-FA_a5, which was +This file was extended by HDF5 $as_me 1.9.77-FA_a5, which was generated by GNU Autoconf 2.67. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -28863,7 +28866,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -HDF5 config.status 1.9.75-FA_a5 +HDF5 config.status 1.9.77-FA_a5 configured by $0, generated by GNU Autoconf 2.67, with options \\"\$ac_cs_config\\" diff --git a/configure.in b/configure.in index 2819a55..4dd6192 100644 --- a/configure.in +++ b/configure.in @@ -26,7 +26,7 @@ dnl dnl NOTE: Don't forget to change the version number here when we do a dnl release!!! dnl -AC_INIT([HDF5], [1.9.76-FA_a5], [help@hdfgroup.org]) +AC_INIT([HDF5], [1.9.77-FA_a5], [help@hdfgroup.org]) AC_CONFIG_SRCDIR([src/H5.c]) AM_CONFIG_HEADER([src/H5config.h]) diff --git a/fortran/src/Makefile.in b/fortran/src/Makefile.in index cbb002b..d38d654 100644 --- a/fortran/src/Makefile.in +++ b/fortran/src/Makefile.in @@ -436,7 +436,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog # Add libtool shared library version numbers to the HDF5 library # See libtool versioning documentation online. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 66 +LT_VERS_REVISION = 67 LT_VERS_AGE = 0 # Include src directory in both Fortran and C flags (C compiler is used diff --git a/hl/c++/src/Makefile.in b/hl/c++/src/Makefile.in index e20c7e9..5354a78 100644 --- a/hl/c++/src/Makefile.in +++ b/hl/c++/src/Makefile.in @@ -392,7 +392,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog # Add libtool shared library version numbers to the HDF5 library # See libtool versioning documentation online. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 66 +LT_VERS_REVISION = 67 LT_VERS_AGE = 0 # Include src directory diff --git a/hl/fortran/src/Makefile.in b/hl/fortran/src/Makefile.in index cbcfa47..b1257f3 100644 --- a/hl/fortran/src/Makefile.in +++ b/hl/fortran/src/Makefile.in @@ -401,7 +401,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog # Add libtool shared library version numbers to the HDF5 library # See libtool versioning documentation online. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 66 +LT_VERS_REVISION = 67 LT_VERS_AGE = 0 INCLUDES = -I$(top_srcdir)/src -I$(top_srcdir)/hl/src -I$(top_builddir)/hl/src \ -I$(top_srcdir)/fortran/src -I$(top_builddir)/fortran/src diff --git a/hl/src/Makefile.in b/hl/src/Makefile.in index 9c0b74c..d5da363 100644 --- a/hl/src/Makefile.in +++ b/hl/src/Makefile.in @@ -392,7 +392,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog # Add libtool shared library version numbers to the HDF5 library # See libtool versioning documentation online. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 66 +LT_VERS_REVISION = 67 LT_VERS_AGE = 0 # This library is our main target. diff --git a/release_docs/CMake.txt b/release_docs/CMake.txt index e9c764e..1220066 100755 --- a/release_docs/CMake.txt +++ b/release_docs/CMake.txt @@ -146,7 +146,16 @@ Notes: This short instruction is written for users who want to quickly build 3.1 If you wish to use the Visual Studio environment, open the solution file in your build directory. Be sure to select either Debug or Release and build the solution. - + + 3.2 The external libraries (zlib, szip, and jpeg) can be configured + to allow building the libraries by downloading from a SVN repository. + The option is 'HDF5_ALLOW_EXTERNAL_SUPPORT'; by adding the following + configuration option: + -DHDF5_ALLOW_EXTERNAL_SUPPORT:STRING="SVN" + + The options to control the SVN URL are by defualt: + ZLIB_SVN_URL:STRING="http://svn.hdfgroup.uiuc.edu/zlib/trunk" + SZIP_SVN_URL:STRING="http://svn.hdfgroup.uiuc.edu/szip/trunk" 4. Test HDF5. @@ -170,13 +179,13 @@ Notes: This short instruction is written for users who want to quickly build NOTE: We have just introduced the packaging capability and it has not been extensively tested. Please send us comments on how it can be improved. - 5. The files that support building HDF5 with CMake are all the files in the + 6. The files that support building HDF5 with CMake are all the files in the config/cmake folder, the CMakeLists.txt files in each source folder, and CTestConfig.cmake. CTestConfig.cmake is specific to the internal testing performed by The HDF Group. It should be altered for the users installation and needs. - 6. More information about using CMake can be found at the KitWare site, + 7. More information about using CMake can be found at the KitWare site, www.cmake.org. diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 1c65a15..b714052 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1,4 +1,4 @@ -HDF5 version 1.9.76-FA_a5 currently under development +HDF5 version 1.9.77-FA_a5 currently under development ================================================================================ @@ -240,6 +240,7 @@ New Features Support for new platforms, languages and compilers. ======================================= + - Intel V11.1 uses now -O3 optimization in production mode (EIP - 2010/10/08) - PathScale compilers are recognized and can build the HDF5 library properly. AKC - 2009/7/28 - @@ -473,6 +474,8 @@ Bug Fixes since HDF5-1.8.0 release Tools ----- + - Fixed h5copy to fail gracefully when copying object to non-exist + group without -p option. Bug#2040 (JKM 2010/10/18) - Fixed to compare member objects and groups recursively when two files or groups are specified to be compared. Bug#1975 (JKM 2010/9/16) diff --git a/src/H5AC.c b/src/H5AC.c index 1c92833..f2a728f 100644 --- a/src/H5AC.c +++ b/src/H5AC.c @@ -1825,6 +1825,40 @@ done: /*------------------------------------------------------------------------- + * Function: H5AC_dump_cache + * + * Purpose: Dumps a summary of the contents of the metadata cache + * to stdout. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * Sunday, October 10, 2010 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_dump_cache(const H5F_t *f) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5AC_dump_cache, FAIL) + + HDassert(f); + HDassert(f->shared); + HDassert(f->shared->cache); + + if ( H5C_dump_cache(f->shared->cache, H5F_OPEN_NAME(f)) < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_dump_cache() failed.") + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_dump_cache() */ + + +/*------------------------------------------------------------------------- * Function: H5AC_get_cache_auto_resize_config * * Purpose: Wrapper function for H5C_get_cache_auto_resize_config(). diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 6780b72..2a817a0 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -373,6 +373,8 @@ H5_DLL herr_t H5AC_set_write_done_callback(H5C_t * cache_ptr, void (* write_done)(void)); H5_DLL herr_t H5AC_stats(const H5F_t *f); +H5_DLL herr_t H5AC_dump_cache(const H5F_t *f); + H5_DLL herr_t H5AC_get_cache_auto_resize_config(const H5AC_t * cache_ptr, H5AC_cache_config_t *config_ptr); diff --git a/src/H5B.c b/src/H5B.c index f15eedb..389dbb5 100644 --- a/src/H5B.c +++ b/src/H5B.c @@ -120,6 +120,8 @@ 4 + /*type, level, num entries */ \ 2*H5F_SIZEOF_ADDR(F)) /*left and right sibling addresses */ +/* Default initializer for H5B_ins_ud_t */ +#define H5B_INS_UD_T_NULL {NULL, HADDR_UNDEF, H5AC__NO_FLAGS_SET} /******************/ /* Local Typedefs */ @@ -131,24 +133,32 @@ typedef struct H5B_iter_ud_t { void *udata; /* Node type's 'udata' for loading & iterator callback */ } H5B_info_ud_t; +/* Convenience struct for the arguments needed to unprotect a b-tree after a + * call to H5B_iterate_helper() or H5B_split() */ +typedef struct H5B_ins_ud_t { + H5B_t *bt; /* B-tree */ + haddr_t addr; /* B-tree address */ + unsigned cache_flags; /* Cache flags for H5AC_unprotect() */ +} H5B_ins_ud_t; + /********************/ /* Local Prototypes */ /********************/ -static H5B_ins_t H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, +static H5B_ins_t H5B_insert_helper(H5F_t *f, hid_t dxpl_id, H5B_ins_ud_t *bt_ud, const H5B_class_t *type, uint8_t *lt_key, hbool_t *lt_key_changed, uint8_t *md_key, void *udata, uint8_t *rt_key, hbool_t *rt_key_changed, - haddr_t *retval); + H5B_ins_ud_t *split_bt_ud/*out*/); static herr_t H5B_insert_child(H5B_t *bt, unsigned *bt_flags, unsigned idx, haddr_t child, H5B_ins_t anchor, const void *md_key); -static herr_t H5B_split(H5F_t *f, hid_t dxpl_id, H5B_t *old_bt, - unsigned *old_bt_flags, haddr_t old_addr, - unsigned idx, void *udata, haddr_t *new_addr/*out*/); +static herr_t H5B_split(H5F_t *f, hid_t dxpl_id, H5B_ins_ud_t *bt_ud, + unsigned idx, void *udata, + H5B_ins_ud_t *split_bt_ud/*out*/); static H5B_t * H5B_copy(const H5B_t *old_bt); @@ -374,7 +384,7 @@ done: * The UDATA pointer is passed to the sizeof_rkey() method but is * otherwise unused. * - * The OLD_BT argument is a pointer to a protected B-tree + * The BT_UD argument is a pointer to a protected B-tree * node. * * Return: Non-negative on success (The address of the new node is @@ -387,14 +397,12 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B_split(H5F_t *f, hid_t dxpl_id, H5B_t *old_bt, unsigned *old_bt_flags, - haddr_t old_addr, unsigned idx, void *udata, haddr_t *new_addr_p/*out*/) +H5B_split(H5F_t *f, hid_t dxpl_id, H5B_ins_ud_t *bt_ud, unsigned idx, + void *udata, H5B_ins_ud_t *split_bt_ud/*out*/) { H5P_genplist_t *dx_plist; /* Data transfer property list */ - H5B_shared_t *shared; /* Pointer to shared B-tree info */ + H5B_shared_t *shared; /* Pointer to shared B-tree info */ H5B_cache_ud_t cache_udata; /* User-data for metadata cache callback */ - unsigned new_bt_flags = H5AC__NO_FLAGS_SET; - H5B_t *new_bt = NULL; unsigned nleft, nright; /* Number of keys in left & right halves */ double split_ratios[3]; /* B-tree split ratios */ herr_t ret_value = SUCCEED; /* Return value */ @@ -405,16 +413,18 @@ H5B_split(H5F_t *f, hid_t dxpl_id, H5B_t *old_bt, unsigned *old_bt_flags, * Check arguments. */ HDassert(f); - HDassert(old_bt); - HDassert(old_bt_flags); - HDassert(H5F_addr_defined(old_addr)); + HDassert(bt_ud); + HDassert(bt_ud->bt); + HDassert(H5F_addr_defined(bt_ud->addr)); + HDassert(split_bt_ud); + HDassert(!split_bt_ud->bt); /* * Initialize variables. */ - shared = (H5B_shared_t *)H5RC_GET_OBJ(old_bt->rc_shared); + shared = (H5B_shared_t *)H5RC_GET_OBJ(bt_ud->bt->rc_shared); HDassert(shared); - HDassert(old_bt->nchildren == shared->two_k); + HDassert(bt_ud->bt->nchildren == shared->two_k); /* Get the dataset transfer property list */ if(NULL == (dx_plist = (H5P_genplist_t *)H5I_object(dxpl_id))) @@ -428,11 +438,11 @@ H5B_split(H5F_t *f, hid_t dxpl_id, H5B_t *old_bt, unsigned *old_bt_flags, if(H5DEBUG(B)) { const char *side; - if(!H5F_addr_defined(old_bt->left) && !H5F_addr_defined(old_bt->right)) + if(!H5F_addr_defined(bt_ud->bt->left) && !H5F_addr_defined(bt_ud->bt->right)) side = "ONLY"; - else if(!H5F_addr_defined(old_bt->right)) + else if(!H5F_addr_defined(bt_ud->bt->right)) side = "RIGHT"; - else if(!H5F_addr_defined(old_bt->left)) + else if(!H5F_addr_defined(bt_ud->bt->left)) side = "LEFT"; else side = "MIDDLE"; @@ -445,9 +455,9 @@ H5B_split(H5F_t *f, hid_t dxpl_id, H5B_t *old_bt, unsigned *old_bt_flags, * Decide how to split the children of the old node among the old node * and the new node. */ - if(!H5F_addr_defined(old_bt->right)) + if(!H5F_addr_defined(bt_ud->bt->right)) nleft = (unsigned)((double)shared->two_k * split_ratios[2]); /*right*/ - else if(!H5F_addr_defined(old_bt->left)) + else if(!H5F_addr_defined(bt_ud->bt->left)) nleft = (unsigned)((double)shared->two_k * split_ratios[0]); /*left*/ else nleft = (unsigned)((double)shared->two_k * split_ratios[1]); /*middle*/ @@ -470,69 +480,66 @@ H5B_split(H5F_t *f, hid_t dxpl_id, H5B_t *old_bt, unsigned *old_bt_flags, /* * Create the new B-tree node. */ - if(H5B_create(f, dxpl_id, shared->type, udata, new_addr_p/*out*/) < 0) + if(H5B_create(f, dxpl_id, shared->type, udata, &split_bt_ud->addr/*out*/) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to create B-tree") cache_udata.f = f; cache_udata.type = shared->type; - cache_udata.rc_shared = old_bt->rc_shared; - if(NULL == (new_bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, *new_addr_p, &cache_udata, H5AC_WRITE))) + cache_udata.rc_shared = bt_ud->bt->rc_shared; + if(NULL == (split_bt_ud->bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, split_bt_ud->addr, &cache_udata, H5AC_WRITE))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree") - new_bt->level = old_bt->level; + split_bt_ud->bt->level = bt_ud->bt->level; /* * Copy data from the old node to the new node. */ - /* this function didn't used to mark the new bt entry as dirty. Since - * we just inserted the entry, this doesn't matter unless the entry - * somehow gets flushed between the insert and the protect. At present, - * I don't think this can happen, but it doesn't hurt to mark the entry - * dirty again. - * -- JRM - */ - new_bt_flags |= H5AC__DIRTIED_FLAG; - HDmemcpy(new_bt->native, - old_bt->native + nleft * shared->type->sizeof_nkey, + split_bt_ud->cache_flags = H5AC__DIRTIED_FLAG; + HDmemcpy(split_bt_ud->bt->native, + bt_ud->bt->native + nleft * shared->type->sizeof_nkey, (nright + 1) * shared->type->sizeof_nkey); - HDmemcpy(new_bt->child, - &old_bt->child[nleft], + HDmemcpy(split_bt_ud->bt->child, + &bt_ud->bt->child[nleft], nright * sizeof(haddr_t)); - new_bt->nchildren = nright; + split_bt_ud->bt->nchildren = nright; /* * Truncate the old node. */ - *old_bt_flags |= H5AC__DIRTIED_FLAG; - old_bt->nchildren = nleft; + bt_ud->cache_flags |= H5AC__DIRTIED_FLAG; + bt_ud->bt->nchildren = nleft; /* * Update sibling pointers. */ - new_bt->left = old_addr; - new_bt->right = old_bt->right; + split_bt_ud->bt->left = bt_ud->addr; + split_bt_ud->bt->right = bt_ud->bt->right; - if(H5F_addr_defined(old_bt->right)) { + if(H5F_addr_defined(bt_ud->bt->right)) { H5B_t *tmp_bt; H5B_cache_ud_t cache_udata2; /* User-data for metadata cache callback */ cache_udata2.f = f; cache_udata2.type = shared->type; - cache_udata2.rc_shared = old_bt->rc_shared; - if(NULL == (tmp_bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, old_bt->right, &cache_udata2, H5AC_WRITE))) + cache_udata2.rc_shared = bt_ud->bt->rc_shared; + if(NULL == (tmp_bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, bt_ud->bt->right, &cache_udata2, H5AC_WRITE))) HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load right sibling") - tmp_bt->left = *new_addr_p; + tmp_bt->left = split_bt_ud->addr; - if(H5AC_unprotect(f, dxpl_id, H5AC_BT, old_bt->right, tmp_bt, H5AC__DIRTIED_FLAG) < 0) + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, bt_ud->bt->right, tmp_bt, H5AC__DIRTIED_FLAG) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") } /* end if */ - old_bt->right = *new_addr_p; + bt_ud->bt->right = split_bt_ud->addr; done: - if(new_bt && H5AC_unprotect(f, dxpl_id, H5AC_BT, *new_addr_p, new_bt, new_bt_flags) < 0) - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + if(ret_value < 0) { + if(split_bt_ud->bt && H5AC_unprotect(f, dxpl_id, H5AC_BT, split_bt_ud->addr, split_bt_ud->bt, split_bt_ud->cache_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + split_bt_ud->bt = NULL; + split_bt_ud->addr = HADDR_UNDEF; + } /* end if */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5B_split() */ @@ -541,8 +548,7 @@ done: /*------------------------------------------------------------------------- * Function: H5B_insert * - * Purpose: Adds a new item to the B-tree. If the root node of - * the B-tree splits then the B-tree gets a new address. + * Purpose: Adds a new item to the B-tree. * * Return: Non-negative on success/Negative on failure * @@ -565,10 +571,11 @@ H5B_insert(H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, haddr_t addr, uint8_t *rt_key=(uint8_t*)_rt_key; hbool_t lt_key_changed = FALSE, rt_key_changed = FALSE; - haddr_t child, old_root; + haddr_t old_root_addr = HADDR_UNDEF; unsigned level; - H5B_t *bt; - H5B_t *new_bt; /* Copy of B-tree info */ + H5B_ins_ud_t bt_ud = H5B_INS_UD_T_NULL; /* (Old) root node */ + H5B_ins_ud_t split_bt_ud = H5B_INS_UD_T_NULL; /* Split B-tree node */ + H5B_t *new_root_bt = NULL; /* New root node */ H5RC_t *rc_shared; /* Ref-counted shared info */ H5B_shared_t *shared; /* Pointer to shared B-tree info */ H5B_cache_ud_t cache_udata; /* User-data for metadata cache callback */ @@ -583,118 +590,110 @@ H5B_insert(H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, haddr_t addr, HDassert(type->sizeof_nkey <= sizeof _lt_key); HDassert(H5F_addr_defined(addr)); - if((int)(my_ins = H5B_insert_helper(f, dxpl_id, addr, type, lt_key, - <_key_changed, md_key, udata, rt_key, &rt_key_changed, &child/*out*/)) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to insert key") - if(H5B_INS_NOOP == my_ins) - HGOTO_DONE(SUCCEED) - HDassert(H5B_INS_RIGHT == my_ins); - /* Get shared info for B-tree */ if(NULL == (rc_shared = (type->get_shared)(f, udata))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree's shared ref. count object") + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree's shared ref. count object") shared = (H5B_shared_t *)H5RC_GET_OBJ(rc_shared); HDassert(shared); - /* the current root */ + /* Protect the root node */ cache_udata.f = f; cache_udata.type = type; cache_udata.rc_shared = rc_shared; - if(NULL == (bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, addr, &cache_udata, H5AC_READ))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to locate root of B-tree") + bt_ud.addr = addr; + if(NULL == (bt_ud.bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, addr, &cache_udata, H5AC_WRITE))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to locate root of B-tree") + + /* Insert the object */ + if((int)(my_ins = H5B_insert_helper(f, dxpl_id, &bt_ud, type, lt_key, + <_key_changed, md_key, udata, rt_key, &rt_key_changed, + &split_bt_ud/*out*/)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to insert key") + if(H5B_INS_NOOP == my_ins) { + HDassert(!split_bt_ud.bt); + HGOTO_DONE(SUCCEED) + } /* end if */ + HDassert(H5B_INS_RIGHT == my_ins); + HDassert(split_bt_ud.bt); + HDassert(H5F_addr_defined(split_bt_ud.addr)); - level = bt->level; + /* Get level of old root */ + level = bt_ud.bt->level; + /* update left and right keys */ if(!lt_key_changed) - HDmemcpy(lt_key, H5B_NKEY(bt,shared,0), type->sizeof_nkey); - - if(H5AC_unprotect(f, dxpl_id, H5AC_BT, addr, bt, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release new child") - bt = NULL; - - /* the new node */ - if(NULL == (bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, child, &cache_udata, H5AC_READ))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load new node") - + HDmemcpy(lt_key, H5B_NKEY(bt_ud.bt,shared,0), type->sizeof_nkey); if(!rt_key_changed) - HDmemcpy(rt_key, H5B_NKEY(bt,shared,bt->nchildren), type->sizeof_nkey); - - if(H5AC_unprotect(f, dxpl_id, H5AC_BT, child, bt, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release new child") - bt = NULL; + HDmemcpy(rt_key, H5B_NKEY(split_bt_ud.bt,shared,split_bt_ud.bt->nchildren), type->sizeof_nkey); /* - * Copy the old root node to some other file location and make the new - * root at the old root's previous address. This prevents the B-tree - * from "moving". + * Copy the old root node to some other file location and make the new root + * at the old root's previous address. This prevents the B-tree from + * "moving". */ H5_CHECK_OVERFLOW(shared->sizeof_rnode,size_t,hsize_t); - if(HADDR_UNDEF == (old_root = H5MF_alloc(f, H5FD_MEM_BTREE, dxpl_id, (hsize_t)shared->sizeof_rnode))) + if(HADDR_UNDEF == (old_root_addr = H5MF_alloc(f, H5FD_MEM_BTREE, dxpl_id, (hsize_t)shared->sizeof_rnode))) HGOTO_ERROR(H5E_BTREE, H5E_CANTALLOC, FAIL, "unable to allocate file space to move root") - /* update the new child's left pointer */ - if(NULL == (bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, child, &cache_udata, H5AC_WRITE))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load new child") - - bt->left = old_root; - - if(H5AC_unprotect(f, dxpl_id, H5AC_BT, child, bt, H5AC__DIRTIED_FLAG) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release new child") - bt = NULL; /* Make certain future references will be caught */ - /* - * Move the node to the new location by checking it out & checking it in - * at the new location -QAK + * Move the node to the new location */ - /* Bring the old root into the cache if it's not already */ - if(NULL == (bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, addr, &cache_udata, H5AC_WRITE))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load new child") - - /* Make certain the old root info is marked as dirty before moving it, */ - /* so it is certain to be written out at the new location */ /* Make a copy of the old root information */ - if(NULL == (new_bt = H5B_copy(bt))) { - HCOMMON_ERROR(H5E_BTREE, H5E_CANTCOPY, "unable to copy old root"); - - if(H5AC_unprotect(f, dxpl_id, H5AC_BT, addr, bt, H5AC__DIRTIED_FLAG) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release new child") - - HGOTO_DONE(FAIL) - } /* end if */ + if(NULL == (new_root_bt = H5B_copy(bt_ud.bt))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTCOPY, FAIL, "unable to copy old root"); - if(H5AC_unprotect(f, dxpl_id, H5AC_BT, addr, bt, H5AC__DIRTIED_FLAG) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release new child") - bt = NULL; /* Make certain future references will be caught */ + /* Unprotect the old root so we can move it. Also force it to be marked + * dirty so it is written to the new location. */ + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, bt_ud.addr, bt_ud.bt, H5AC__DIRTIED_FLAG) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release old root") + bt_ud.bt = NULL; /* Make certain future references will be caught */ /* Move the location of the old root on the disk */ - if(H5AC_move_entry(f, H5AC_BT, addr, old_root) < 0) + if(H5AC_move_entry(f, H5AC_BT, bt_ud.addr, old_root_addr) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, FAIL, "unable to move B-tree root node") + bt_ud.addr = old_root_addr; + + /* Update the split b-tree's left pointer to point to the new location */ + split_bt_ud.bt->left = bt_ud.addr; + split_bt_ud.cache_flags |= H5AC__DIRTIED_FLAG; /* clear the old root info at the old address (we already copied it) */ - new_bt->left = HADDR_UNDEF; - new_bt->right = HADDR_UNDEF; + new_root_bt->left = HADDR_UNDEF; + new_root_bt->right = HADDR_UNDEF; /* Set the new information for the copy */ - new_bt->level = level + 1; - new_bt->nchildren = 2; + new_root_bt->level = level + 1; + new_root_bt->nchildren = 2; - new_bt->child[0] = old_root; - HDmemcpy(H5B_NKEY(new_bt, shared, 0), lt_key, shared->type->sizeof_nkey); + new_root_bt->child[0] = bt_ud.addr; + HDmemcpy(H5B_NKEY(new_root_bt, shared, 0), lt_key, shared->type->sizeof_nkey); - new_bt->child[1] = child; - HDmemcpy(H5B_NKEY(new_bt, shared, 1), md_key, shared->type->sizeof_nkey); - HDmemcpy(H5B_NKEY(new_bt, shared, 2), rt_key, shared->type->sizeof_nkey); + new_root_bt->child[1] = split_bt_ud.addr; + HDmemcpy(H5B_NKEY(new_root_bt, shared, 1), md_key, shared->type->sizeof_nkey); + HDmemcpy(H5B_NKEY(new_root_bt, shared, 2), rt_key, shared->type->sizeof_nkey); /* Insert the modified copy of the old root into the file again */ - if(H5AC_insert_entry(f, dxpl_id, H5AC_BT, addr, new_bt, H5AC__NO_FLAGS_SET) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_CANTFLUSH, FAIL, "unable to flush old B-tree root node") + if(H5AC_insert_entry(f, dxpl_id, H5AC_BT, addr, new_root_bt, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTFLUSH, FAIL, "unable to add old B-tree root node to cache") #ifdef H5B_DEBUG H5B_assert(f, dxpl_id, addr, type, udata); #endif done: + if(ret_value < 0) + if(new_root_bt && H5B_node_dest(new_root_bt) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTRELEASE, FAIL, "unable to free B-tree root node"); + + if(bt_ud.bt) + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, bt_ud.addr, bt_ud.bt, bt_ud.cache_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to unprotect old root") + + if(split_bt_ud.bt) + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, split_bt_ud.addr, split_bt_ud.bt, split_bt_ud.cache_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to unprotect new child") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5B_insert() */ @@ -725,6 +724,7 @@ H5B_insert_child(H5B_t *bt, unsigned *bt_flags, unsigned idx, HDassert(bt); HDassert(bt_flags); + HDassert(H5F_addr_defined(child)); shared = (H5B_shared_t *)H5RC_GET_OBJ(bt->rc_shared); HDassert(shared); HDassert(bt->nchildren < shared->two_k); @@ -803,20 +803,21 @@ H5B_insert_child(H5B_t *bt, unsigned *bt_flags, unsigned idx, *------------------------------------------------------------------------- */ static H5B_ins_t -H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type, +H5B_insert_helper(H5F_t *f, hid_t dxpl_id, H5B_ins_ud_t *bt_ud, + const H5B_class_t *type, uint8_t *lt_key, hbool_t *lt_key_changed, uint8_t *md_key, void *udata, uint8_t *rt_key, hbool_t *rt_key_changed, - haddr_t *new_node_p/*out*/) + H5B_ins_ud_t *split_bt_ud/*out*/) { - unsigned bt_flags = H5AC__NO_FLAGS_SET, twin_flags = H5AC__NO_FLAGS_SET; - H5B_t *bt = NULL, *twin = NULL; + H5B_t *bt; /* Convenience pointer to B-tree */ H5RC_t *rc_shared; /* Ref-counted shared info */ - H5B_shared_t *shared; /* Pointer to shared B-tree info */ + H5B_shared_t *shared; /* Pointer to shared B-tree info */ H5B_cache_ud_t cache_udata; /* User-data for metadata cache callback */ unsigned lt = 0, idx = 0, rt; /* Left, final & right index values */ int cmp = -1; /* Key comparison value */ - haddr_t child_addr = HADDR_UNDEF; + H5B_ins_ud_t child_bt_ud = H5B_INS_UD_T_NULL; /* Child B-tree */ + H5B_ins_ud_t new_child_bt_ud = H5B_INS_UD_T_NULL; /* Newly split child B-tree */ H5B_ins_t my_ins = H5B_INS_ERROR; H5B_ins_t ret_value = H5B_INS_ERROR; /* Return value */ @@ -826,7 +827,9 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type * Check arguments */ HDassert(f); - HDassert(H5F_addr_defined(addr)); + HDassert(bt_ud); + HDassert(bt_ud->bt); + HDassert(H5F_addr_defined(bt_ud->addr)); HDassert(type); HDassert(type->decode); HDassert(type->cmp3); @@ -835,7 +838,12 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type HDassert(lt_key_changed); HDassert(rt_key); HDassert(rt_key_changed); - HDassert(new_node_p); + HDassert(split_bt_ud); + HDassert(!split_bt_ud->bt); + HDassert(!H5F_addr_defined(split_bt_ud->addr)); + HDassert(split_bt_ud->cache_flags == H5AC__NO_FLAGS_SET); + + bt = bt_ud->bt; *lt_key_changed = FALSE; *rt_key_changed = FALSE; @@ -851,11 +859,6 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type * data. When the search completes IDX points to the child that * should get the new data. */ - cache_udata.f = f; - cache_udata.type = type; - cache_udata.rc_shared = rc_shared; - if(NULL == (bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, addr, &cache_udata, H5AC_WRITE))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, H5B_INS_ERROR, "unable to load node") rt = bt->nchildren; while(lt < rt && cmp) { @@ -866,6 +869,11 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type lt = idx + 1; } /* end while */ + /* Set up user data for cache callbacks */ + cache_udata.f = f; + cache_udata.type = type; + cache_udata.rc_shared = rc_shared; + if(0 == bt->nchildren) { /* * The value being inserted will be the only value in this tree. We @@ -876,13 +884,13 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type H5B_NKEY(bt, shared, 1), bt->child + 0/*out*/) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, H5B_INS_ERROR, "unable to create leaf node") bt->nchildren = 1; - bt_flags |= H5AC__DIRTIED_FLAG; + bt_ud->cache_flags |= H5AC__DIRTIED_FLAG; idx = 0; if(type->follow_min) { if((int)(my_ins = (type->insert)(f, dxpl_id, bt->child[idx], H5B_NKEY(bt, shared, idx), lt_key_changed, md_key, udata, H5B_NKEY(bt, shared, idx + 1), - rt_key_changed, &child_addr/*out*/)) < 0) + rt_key_changed, &new_child_bt_ud.addr/*out*/)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "unable to insert first leaf node") } /* end if */ else @@ -893,10 +901,14 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type * The value being inserted is less than any value in this tree. * Follow the minimum branch out of this node to a subtree. */ - if((int)(my_ins = H5B_insert_helper(f, dxpl_id, bt->child[idx], type, + child_bt_ud.addr = bt->child[idx]; + if(NULL == (child_bt_ud.bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, child_bt_ud.addr, &cache_udata, H5AC_WRITE))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, H5B_INS_ERROR, "unable to load node") + + if((int)(my_ins = H5B_insert_helper(f, dxpl_id, &child_bt_ud, type, H5B_NKEY(bt,shared,idx), lt_key_changed, md_key, udata, H5B_NKEY(bt, shared, idx + 1), rt_key_changed, - &child_addr/*out*/)) < 0) + &new_child_bt_ud/*out*/)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert minimum subtree") } else if(type->follow_min) { /* @@ -906,7 +918,7 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type */ if((int)(my_ins = (type->insert)(f, dxpl_id, bt->child[idx], H5B_NKEY(bt, shared, idx), lt_key_changed, md_key, udata, H5B_NKEY(bt, shared, idx + 1), - rt_key_changed, &child_addr/*out*/)) < 0) + rt_key_changed, &new_child_bt_ud.addr/*out*/)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert minimum leaf node") } else { /* @@ -917,7 +929,7 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type my_ins = H5B_INS_LEFT; HDmemcpy(md_key, H5B_NKEY(bt,shared,idx), type->sizeof_nkey); if((type->new_node)(f, dxpl_id, H5B_INS_LEFT, H5B_NKEY(bt, shared, idx), udata, - md_key, &child_addr/*out*/) < 0) + md_key, &new_child_bt_ud.addr/*out*/) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert minimum leaf node") *lt_key_changed = TRUE; } /* end else */ @@ -935,9 +947,14 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type * Follow the maximum branch out of this node to a subtree. */ idx = bt->nchildren - 1; - if((int)(my_ins = H5B_insert_helper(f, dxpl_id, bt->child[idx], type, + child_bt_ud.addr = bt->child[idx]; + if(NULL == (child_bt_ud.bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, child_bt_ud.addr, &cache_udata, H5AC_WRITE))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, H5B_INS_ERROR, "unable to load node") + + if((int)(my_ins = H5B_insert_helper(f, dxpl_id, &child_bt_ud, type, H5B_NKEY(bt, shared, idx), lt_key_changed, md_key, udata, - H5B_NKEY(bt, shared, idx + 1), rt_key_changed, &child_addr/*out*/)) < 0) + H5B_NKEY(bt, shared, idx + 1), rt_key_changed, + &new_child_bt_ud/*out*/)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert maximum subtree") } else if(type->follow_max) { /* @@ -948,7 +965,7 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type idx = bt->nchildren - 1; if((int)(my_ins = (type->insert)(f, dxpl_id, bt->child[idx], H5B_NKEY(bt, shared, idx), lt_key_changed, md_key, udata, H5B_NKEY(bt, shared, idx + 1), - rt_key_changed, &child_addr/*out*/)) < 0) + rt_key_changed, &new_child_bt_ud.addr/*out*/)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert maximum leaf node") } else { /* @@ -960,7 +977,7 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type my_ins = H5B_INS_RIGHT; HDmemcpy(md_key, H5B_NKEY(bt, shared, idx + 1), type->sizeof_nkey); if((type->new_node)(f, dxpl_id, H5B_INS_RIGHT, md_key, udata, - H5B_NKEY(bt, shared, idx + 1), &child_addr/*out*/) < 0) + H5B_NKEY(bt, shared, idx + 1), &new_child_bt_ud.addr/*out*/) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert maximum leaf node") *rt_key_changed = TRUE; } /* end else */ @@ -985,9 +1002,13 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type * Follow a branch out of this node to another subtree. */ HDassert(idx < bt->nchildren); - if((int)(my_ins = H5B_insert_helper(f, dxpl_id, bt->child[idx], type, + child_bt_ud.addr = bt->child[idx]; + if(NULL == (child_bt_ud.bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, child_bt_ud.addr, &cache_udata, H5AC_WRITE))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, H5B_INS_ERROR, "unable to load node") + + if((int)(my_ins = H5B_insert_helper(f, dxpl_id, &child_bt_ud, type, H5B_NKEY(bt, shared, idx), lt_key_changed, md_key, udata, - H5B_NKEY(bt, shared, idx + 1), rt_key_changed, &child_addr/*out*/)) < 0) + H5B_NKEY(bt, shared, idx + 1), rt_key_changed, &new_child_bt_ud/*out*/)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert subtree") } else { /* @@ -996,7 +1017,7 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type HDassert(idx < bt->nchildren); if((int)(my_ins = (type->insert)(f, dxpl_id, bt->child[idx], H5B_NKEY(bt, shared, idx), lt_key_changed, md_key, udata, H5B_NKEY(bt, shared, idx + 1), - rt_key_changed, &child_addr/*out*/)) < 0) + rt_key_changed, &new_child_bt_ud.addr/*out*/)) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert leaf node") } HDassert((int)my_ins >= 0); @@ -1005,18 +1026,20 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type * Update the left and right keys of the current node. */ if(*lt_key_changed) { - bt_flags |= H5AC__DIRTIED_FLAG; + bt_ud->cache_flags |= H5AC__DIRTIED_FLAG; if(idx > 0) { HDassert(type->critical_key == H5B_LEFT); + HDassert(!(H5B_INS_LEFT == my_ins || H5B_INS_RIGHT == my_ins)); *lt_key_changed = FALSE; } /* end if */ else HDmemcpy(lt_key, H5B_NKEY(bt, shared, idx), type->sizeof_nkey); } /* end if */ if(*rt_key_changed) { - bt_flags |= H5AC__DIRTIED_FLAG; + bt_ud->cache_flags |= H5AC__DIRTIED_FLAG; if(idx + 1 < bt->nchildren) { HDassert(type->critical_key == H5B_RIGHT); + HDassert(!(H5B_INS_LEFT == my_ins || H5B_INS_RIGHT == my_ins)); *rt_key_changed = FALSE; } /* end if */ else @@ -1026,9 +1049,10 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type /* * The insertion simply changed the address for the child. */ - bt->child[idx] = child_addr; - bt_flags |= H5AC__DIRTIED_FLAG; - ret_value = H5B_INS_NOOP; + HDassert(!child_bt_ud.bt); + HDassert(bt->level == 0); + bt->child[idx] = new_child_bt_ud.addr; + bt_ud->cache_flags |= H5AC__DIRTIED_FLAG; } else if(H5B_INS_LEFT == my_ins || H5B_INS_RIGHT == my_ins) { hbool_t *tmp_bt_flags_ptr = NULL; H5B_t *tmp_bt; @@ -1037,26 +1061,24 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type * If this node is full then split it before inserting the new child. */ if(bt->nchildren == shared->two_k) { - if(H5B_split(f, dxpl_id, bt, &bt_flags, addr, idx, udata, new_node_p/*out*/) < 0) + if(H5B_split(f, dxpl_id, bt_ud, idx, udata, split_bt_ud/*out*/) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTSPLIT, H5B_INS_ERROR, "unable to split node") - if(NULL == (twin = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, *new_node_p, &cache_udata, H5AC_WRITE))) - HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, H5B_INS_ERROR, "unable to load node") if(idx < bt->nchildren) { tmp_bt = bt; - tmp_bt_flags_ptr = &bt_flags; + tmp_bt_flags_ptr = &bt_ud->cache_flags; } else { idx -= bt->nchildren; - tmp_bt = twin; - tmp_bt_flags_ptr = &twin_flags; + tmp_bt = split_bt_ud->bt; + tmp_bt_flags_ptr = &split_bt_ud->cache_flags; } } /* end if */ else { tmp_bt = bt; - tmp_bt_flags_ptr = &bt_flags; + tmp_bt_flags_ptr = &bt_ud->cache_flags; } /* end else */ /* Insert the child */ - if(H5B_insert_child(tmp_bt, tmp_bt_flags_ptr, idx, child_addr, my_ins, md_key) < 0) + if(H5B_insert_child(tmp_bt, tmp_bt_flags_ptr, idx, new_child_bt_ud.addr, my_ins, md_key) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTINSERT, H5B_INS_ERROR, "can't insert child") } @@ -1064,8 +1086,8 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type * If this node split, return the mid key (the one that is shared * by the left and right node). */ - if(twin) { - HDmemcpy(md_key, H5B_NKEY(twin, shared, 0), type->sizeof_nkey); + if(split_bt_ud->bt) { + HDmemcpy(md_key, H5B_NKEY(split_bt_ud->bt, shared, 0), type->sizeof_nkey); ret_value = H5B_INS_RIGHT; #ifdef H5B_DEBUG /* @@ -1073,7 +1095,7 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type * in the new node. */ cmp = (type->cmp2)(H5B_NKEY(bt, shared, bt->nchildren), udata, - H5B_NKEY(twin, shared, 0)); + H5B_NKEY(split_bt_ud->bt, shared, 0)); HDassert(0 == cmp); #endif } /* end if */ @@ -1081,12 +1103,13 @@ H5B_insert_helper(H5F_t *f, hid_t dxpl_id, haddr_t addr, const H5B_class_t *type ret_value = H5B_INS_NOOP; done: - { - herr_t e1 = (bt && H5AC_unprotect(f, dxpl_id, H5AC_BT, addr, bt, bt_flags) < 0); - herr_t e2 = (twin && H5AC_unprotect(f, dxpl_id, H5AC_BT, *new_node_p, twin, twin_flags) < 0); - if(e1 || e2) /*use vars to prevent short-circuit of side effects */ - HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, H5B_INS_ERROR, "unable to release node(s)") - } + if(child_bt_ud.bt) + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, child_bt_ud.addr, child_bt_ud.bt, child_bt_ud.cache_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, H5B_INS_ERROR, "unable to unprotect child") + + if(new_child_bt_ud.bt) + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, new_child_bt_ud.addr, new_child_bt_ud.bt, new_child_bt_ud.cache_flags) < 0) + HDONE_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, H5B_INS_ERROR, "unable to unprotect new child") FUNC_LEAVE_NOAPI(ret_value) } diff --git a/src/H5C.c b/src/H5C.c index 5b7984b..9a7ba81 100644 --- a/src/H5C.c +++ b/src/H5C.c @@ -5018,6 +5018,137 @@ H5C_stats__reset(H5C_t UNUSED * cache_ptr) /*------------------------------------------------------------------------- + * Function: H5C_dump_cache + * + * Purpose: Print a summary of the contents of the metadata cache for + * debugging purposes. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 10/10/10 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_dump_cache(H5C_t * cache_ptr, + const char * cache_name) +{ + herr_t ret_value = SUCCEED; /* Return value */ + int i; + H5C_cache_entry_t * entry_ptr = NULL; + H5SL_t * slist_ptr = NULL; + H5SL_node_t * node_ptr = NULL; + + FUNC_ENTER_NOAPI(H5C_dump_cache, FAIL) + + HDassert(cache_ptr != NULL); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(cache_name != NULL ); + + /* First, create a skip list */ + slist_ptr = H5SL_create(H5SL_TYPE_HADDR); + + if ( slist_ptr == NULL ) { + + HGOTO_ERROR(H5E_CACHE, H5E_CANTCREATE, FAIL, "can't create skip list.") + } + + /* Next, scan the index, and insert all entries in the skip list. + * Do this, as we want to display cache entries in increasing address + * order. + */ + for ( i = 0; i < H5C__HASH_TABLE_LEN; i++ ) { + + entry_ptr = cache_ptr->index[i]; + + while ( entry_ptr != NULL ) { + + HDassert( entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC ); + + if ( H5SL_insert(slist_ptr, entry_ptr, &(entry_ptr->addr)) < 0 ) { + + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, \ + "Can't insert entry in skip list") + } + + entry_ptr = entry_ptr->ht_next; + } + } + + /* If we get this far, all entries in the cache are listed in the + * skip list -- scan the skip list generating the desired output. + */ + + HDfprintf(stdout, "\n\nDump of metadata cache \"%s\".\n", cache_name); + HDfprintf(stdout, + "Num: Addr: Len: Type: Prot: Pinned: Dirty:\n"); + + i = 0; + + node_ptr = H5SL_first(slist_ptr); + + if ( node_ptr != NULL ) { + + entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + + } else { + + entry_ptr = NULL; + } + + while ( entry_ptr != NULL ) { + + HDassert( entry_ptr->magic == H5C__H5C_CACHE_ENTRY_T_MAGIC ); + + HDfprintf(stdout, + "%s%d 0x%08llx 0x%3llx %2d %d %d %d\n", + cache_ptr->prefix, i, + (long long)(entry_ptr->addr), + (long long)(entry_ptr->size), + (int)(entry_ptr->type->id), + (int)(entry_ptr->is_protected), + (int)(entry_ptr->is_pinned), + (int)(entry_ptr->is_dirty)); + + /* increment node_ptr before we delete its target */ + node_ptr = H5SL_next(node_ptr); + + /* remove the first item in the skip list */ + if ( H5SL_remove(slist_ptr, &(entry_ptr->addr)) != entry_ptr ) { + + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, \ + "Can't delete entry from skip list.") + } + + if ( node_ptr != NULL ) { + + entry_ptr = (H5C_cache_entry_t *)H5SL_item(node_ptr); + + } else { + + entry_ptr = NULL; + } + + i++; + } + + HDfprintf(stdout, "\n\n"); + + /* Finally, discard the skip list */ + + HDassert( H5SL_count(slist_ptr) == 0 ); + + H5SL_close(slist_ptr); + +done: + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5C_dump_cache() */ + + +/*------------------------------------------------------------------------- * Function: H5C_unpin_entry_from_client() * * Purpose: Internal routine to unpin a cache entry from a client action. @@ -7792,7 +7923,7 @@ end_of_inner_loop: */ HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "Pinned entry count not decreasing.") + "Pinned entry count not decreasing, cur_pel_len = %d, old_pel_len = %d", (int)cur_pel_len, (int)old_pel_len) } else if ( ( cur_pel_len == 0 ) && ( old_pel_len == 0 ) ) { diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 9f10409..0c7631a 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -1180,6 +1180,9 @@ H5_DLL herr_t H5C_stats(H5C_t * cache_ptr, H5_DLL void H5C_stats__reset(H5C_t * cache_ptr); +H5_DLL herr_t H5C_dump_cache(H5C_t * cache_ptr, + const char * cache_name); + H5_DLL herr_t H5C_unpin_entry(void *thing); H5_DLL herr_t H5C_destroy_flush_dependency(void *parent_thing, void *child_thing); diff --git a/src/H5D.c b/src/H5D.c index 91a7c3e..667fe1fb 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -256,6 +256,20 @@ H5Dcreate_anon(hid_t loc_id, hid_t type_id, hid_t space_id, hid_t dcpl_id, HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, "unable to register dataset") done: + /* Release the dataset's object header, if it was created */ + if(dset) { + H5O_loc_t *oloc; /* Object location for dataset */ + + /* Get the new dataset's object location */ + if(NULL == (oloc = H5D_oloc(dset))) + HDONE_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "unable to get object location of dataset") + + /* Decrement refcount on dataset's object header in memory */ + if(H5O_dec_rc_by_loc(oloc, H5AC_dxpl_id) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object") + } /* end if */ + + /* Cleanup on failure */ if(ret_value < 0) if(dset && H5D_close(dset) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, FAIL, "unable to release dataset") diff --git a/src/H5Dbtree.c b/src/H5Dbtree.c index 9fcf1fd..e61811d 100644 --- a/src/H5Dbtree.c +++ b/src/H5Dbtree.c @@ -825,7 +825,7 @@ done: */ static herr_t H5D_btree_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t UNUSED *space, - haddr_t UNUSED dset_ohdr_addr) + haddr_t dset_ohdr_addr) { herr_t ret_value = SUCCEED; /* Return value */ @@ -839,6 +839,8 @@ H5D_btree_idx_init(const H5D_chk_idx_info_t *idx_info, const H5S_t UNUSED *space HDassert(idx_info->storage); HDassert(H5F_addr_defined(dset_ohdr_addr)); + idx_info->storage->u.btree.dset_ohdr_addr = dset_ohdr_addr; + /* Allocate the shared structure */ if(H5D_btree_shared_create(idx_info->f, idx_info->storage, idx_info->layout->ndims) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index b14d28d..c0ce03d 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -108,7 +108,7 @@ typedef struct H5D_chunk_it_ud1_t { const H5D_chk_idx_info_t *idx_info; /* Chunked index info */ const H5D_io_info_t *io_info; /* I/O info for dataset operation */ const hsize_t *space_dim; /* New dataset dimensions */ - const hbool_t *shrunk_dim; /* Dimensions which have been shrunk */ + const hbool_t *shrunk_dim; /* Dimensions which have been shrunk */ H5S_t *chunk_space; /* Dataspace for a chunk */ uint32_t elmts_per_chunk;/* Elements in chunk */ hsize_t *hyper_start; /* Starting location of hyperslab */ @@ -2298,6 +2298,7 @@ H5D_chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *chunk_offset, udata->common.layout = &(dset->shared->layout.u.chunk); udata->common.storage = &(dset->shared->layout.storage.u.chunk); udata->common.offset = chunk_offset; + udata->common.rdcc = &(dset->shared->cache.chunk); /* Reset information about the chunk we are looking for */ udata->nbytes = 0; @@ -2311,7 +2312,8 @@ H5D_chunk_lookup(const H5D_t *dset, hid_t dxpl_id, const hsize_t *chunk_offset, ent = dset->shared->cache.chunk.slot[udata->idx_hint]; if(ent) - for(u = 0, found = TRUE; u < dset->shared->layout.u.chunk.ndims; u++) + for(u = 0, found = TRUE; u < dset->shared->layout.u.chunk.ndims - 1; + u++) if(chunk_offset[u] != ent->offset[u]) { found = FALSE; break; @@ -2398,6 +2400,7 @@ H5D_chunk_flush_entry(const H5D_t *dset, hid_t dxpl_id, const H5D_dxpl_cache_t * udata.common.layout = &dset->shared->layout.u.chunk; udata.common.storage = &dset->shared->layout.storage.u.chunk; udata.common.offset = ent->offset; + udata.common.rdcc = &(dset->shared->cache.chunk); udata.filter_mask = 0; udata.nbytes = dset->shared->layout.u.chunk.size; udata.addr = ent->chunk_addr; @@ -3616,7 +3619,7 @@ H5D_chunk_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, } /* end if */ carry = FALSE; - } /* end if */ + } /* end else */ while(!carry) { #ifndef NDEBUG @@ -3697,6 +3700,7 @@ H5D_chunk_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, udata.common.layout = &layout->u.chunk; udata.common.storage = &layout->storage.u.chunk; udata.common.offset = chunk_offset; + udata.common.rdcc = NULL; H5_ASSIGN_OVERFLOW(udata.nbytes, chunk_size, size_t, uint32_t); udata.filter_mask = filter_mask; udata.addr = HADDR_UNDEF; @@ -3780,9 +3784,9 @@ H5D_chunk_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite, } /* end for */ } /* end while(!carry) */ - /* Adjust max_unalloc_dim_idx so we don't allocate the same chunk twice. - * Also check if this dimension started from 0 (and hence allocated all - * of the chunks. */ + /* Adjust max_unalloc so we don't allocate the same chunk twice. Also + * check if this dimension started from 0 (and hence allocated all of + * the chunks. */ if(min_unalloc[op_dim] == 0) break; else @@ -4349,6 +4353,7 @@ H5D_chunk_prune_by_extent(H5D_t *dset, hid_t dxpl_id, const hsize_t *old_dim) HDmemset(&udata, 0, sizeof udata); udata.common.layout = &layout->u.chunk; udata.common.storage = &layout->storage.u.chunk; + udata.common.rdcc = rdcc; udata.io_info = &chk_io_info; udata.idx_info = &idx_info; udata.space_dim = space_dim; @@ -4679,7 +4684,8 @@ H5D_chunk_addrmap(const H5D_io_info_t *io_info, haddr_t chunk_addr[]) HDmemset(&udata, 0, sizeof(udata)); udata.common.layout = &dset->shared->layout.u.chunk; udata.common.storage = &dset->shared->layout.storage.u.chunk; - udata.chunk_addr = chunk_addr; + udata.common.rdcc = &(dset->shared->cache.chunk); + udata.chunk_addr = chunk_addr; /* Compose chunked index info struct */ idx_info.f = dset->oloc.file; @@ -5026,6 +5032,7 @@ H5D_chunk_copy_cb(const H5D_chunk_rec_t *chunk_rec, void *_udata) udata_dst.common.layout = udata->idx_info_dst->layout; udata_dst.common.storage = udata->idx_info_dst->storage; udata_dst.common.offset = chunk_rec->offset; + udata_dst.common.rdcc = NULL; udata_dst.nbytes = chunk_rec->nbytes; udata_dst.filter_mask = chunk_rec->filter_mask; udata_dst.addr = HADDR_UNDEF; @@ -5289,6 +5296,7 @@ H5D_chunk_copy(H5F_t *f_src, H5O_storage_chunk_t *storage_src, HDmemset(&udata, 0, sizeof udata); udata.common.layout = layout_src; udata.common.storage = storage_src; + udata.common.rdcc = NULL; udata.file_src = f_src; udata.idx_info_dst = &idx_info_dst; udata.buf = buf; diff --git a/src/H5Dint.c b/src/H5Dint.c index 46f15ea..2b2715b 100644 --- a/src/H5Dint.c +++ b/src/H5Dint.c @@ -890,7 +890,7 @@ H5D_update_oh_info(H5F_t *file, hid_t dxpl_id, H5D_t *dset, hid_t dapl_id) ohdr_size += layout->storage.u.compact.size; /* Create an object header for the dataset */ - if(H5O_create(file, dxpl_id, ohdr_size, dset->shared->dcpl_id, oloc/*out*/) < 0) + if(H5O_create(file, dxpl_id, ohdr_size, (size_t)1, dset->shared->dcpl_id, oloc/*out*/) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to create dataset object header") HDassert(file == dset->oloc.file); @@ -1177,6 +1177,8 @@ done: if(new_dset->shared->type && H5I_dec_ref(new_dset->shared->type_id) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, NULL, "unable to release datatype") if(H5F_addr_defined(new_dset->oloc.addr)) { + if(H5O_dec_rc_by_loc(&(new_dset->oloc), dxpl_id) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTDEC, NULL, "unable to decrement refcount on newly created object") if(H5O_close(&(new_dset->oloc)) < 0) HDONE_ERROR(H5E_DATASET, H5E_CLOSEERROR, NULL, "unable to release object header") if(file) { diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 45e9870..673dac3 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -274,6 +274,9 @@ typedef struct H5D_chunk_common_ud_t { const H5O_layout_chunk_t *layout; /* Chunk layout description */ const H5O_storage_chunk_t *storage; /* Chunk storage description */ const hsize_t *offset; /* Logical offset of chunk */ + const struct H5D_rdcc_t *rdcc; /* Chunk cache. Only necessary if the index may + * be modified, and if any chunks in the dset + * may be cached */ } H5D_chunk_common_ud_t; /* B-tree callback info for various operations */ diff --git a/src/H5E.c b/src/H5E.c index 01dd35b..7a81d32 100644 --- a/src/H5E.c +++ b/src/H5E.c @@ -170,13 +170,20 @@ H5E_set_default_auto(H5E_t *stk) { FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5E_set_default_auto) -#ifdef H5_USE_16_API +#ifndef H5_NO_DEPRECATED_SYMBOLS +#ifdef H5_USE_16_API_DEFAULT stk->auto_op.vers = 1; - stk->auto_op.u.func1 = (H5E_auto1_t)H5Eprint1; #else /* H5_USE_16_API */ stk->auto_op.vers = 2; - stk->auto_op.u.func2 = (H5E_auto2_t)H5Eprint2; -#endif /* H5_USE_16_API */ +#endif /* H5_USE_16_API_DEFAULT */ + + stk->auto_op.func1 = stk->auto_op.func1_default = (H5E_auto1_t)H5Eprint1; + stk->auto_op.func2 = stk->auto_op.func2_default = (H5E_auto2_t)H5Eprint2; + stk->auto_op.is_default = TRUE; +#else /* H5_NO_DEPRECATED_SYMBOLS */ + stk->auto_op.func2 = (H5E_auto2_t)H5Eprint2; +#endif /* H5_NO_DEPRECATED_SYMBOLS */ + stk->auto_data = NULL; FUNC_LEAVE_NOAPI(SUCCEED) @@ -1555,6 +1562,11 @@ done: * Programmer: Robb Matzke * Saturday, February 28, 1998 * + * Modification:Raymond Lu + * 4 October 2010 + * If the printing function isn't the default H5Eprint1 or 2, + * and H5Eset_auto1 has been called to set the old style + * printing function, a call to H5Eget_auto2 should fail. *------------------------------------------------------------------------- */ herr_t @@ -1578,8 +1590,15 @@ H5Eget_auto2(hid_t estack_id, H5E_auto2_t *func, void **client_data) /* Get the automatic error reporting information */ if(H5E_get_auto(estack, &op, client_data) < 0) HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "can't get automatic error info") + +#ifndef H5_NO_DEPRECATED_SYMBOLS + /* Fail if the printing function isn't the default(user-set) and set through H5Eset_auto1 */ + if(!op.is_default && op.vers == 1) + HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "wrong API function, H5Eset_auto1 has been called") +#endif /* H5_NO_DEPRECATED_SYMBOLS */ + if(func) - *func = op.u.func2; + *func = op.func2; done: FUNC_LEAVE_API(ret_value) @@ -1606,6 +1625,9 @@ done: * Programmer: Robb Matzke * Friday, February 27, 1998 * + * Modification:Raymond Lu + * 4 October 2010 + * If the FUNC is H5Eprint2, put the IS_DEFAULT flag on. *------------------------------------------------------------------------- */ herr_t @@ -1627,9 +1649,23 @@ H5Eset_auto2(hid_t estack_id, H5E_auto2_t func, void *client_data) if(NULL == (estack = (H5E_t *)H5I_object_verify(estack_id, H5I_ERROR_STACK))) HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a error stack ID") +#ifndef H5_NO_DEPRECATED_SYMBOLS + /* Get the automatic error reporting information */ + if(H5E_get_auto(estack, &op, NULL) < 0) + HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "can't get automatic error info") + /* Set the automatic error reporting information */ + if(func != op.func2_default) + op.is_default = FALSE; + else + op.is_default = TRUE; + op.vers = 2; - op.u.func2 = func; +#endif /* H5_NO_DEPRECATED_SYMBOLS */ + + /* Set the automatic error reporting function */ + op.func2 = func; + if(H5E_set_auto(estack, &op, client_data) < 0) HGOTO_ERROR(H5E_ERROR, H5E_CANTSET, FAIL, "can't set automatic error info") @@ -1672,7 +1708,11 @@ H5Eauto_is_v2(hid_t estack_id, unsigned *is_stack) /* Check if the error stack reporting function is the "newer" stack type */ if(is_stack) +#ifndef H5_NO_DEPRECATED_SYMBOLS *is_stack = estack->auto_op.vers > 1; +#else + *is_stack = 1; +#endif done: FUNC_LEAVE_API(ret_value) diff --git a/src/H5Edeprec.c b/src/H5Edeprec.c index 30f3ae9..0a028d9 100644 --- a/src/H5Edeprec.c +++ b/src/H5Edeprec.c @@ -370,6 +370,11 @@ done: * Programmer: Raymond Lu * Sep 16, 2003 * + * Modification:Raymond Lu + * 4 October 2010 + * If the printing function isn't the default H5Eprint1 or 2, + * and H5Eset_auto2 has been called to set the new style + * printing function, a call to H5Eget_auto1 should fail. *------------------------------------------------------------------------- */ herr_t @@ -389,8 +394,13 @@ H5Eget_auto1(H5E_auto1_t *func, void **client_data) /* Get the automatic error reporting information */ if(H5E_get_auto(estack, &auto_op, client_data) < 0) HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "can't get automatic error info") + + /* Fail if the printing function isn't the default(user-set) and set through H5Eset_auto2 */ + if(!auto_op.is_default && auto_op.vers == 2) + HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "wrong API function, H5Eset_auto2 has been called") + if(func) - *func = auto_op.u.func1; + *func = auto_op.func1; done: FUNC_LEAVE_API(ret_value) @@ -418,6 +428,9 @@ done: * Programmer: Raymond Lu * Sep 16, 2003 * + * Modification:Raymond Lu + * 4 October 2010 + * If the FUNC is H5Eprint2, put the IS_DEFAULT flag on. *------------------------------------------------------------------------- */ herr_t @@ -434,9 +447,18 @@ H5Eset_auto1(H5E_auto1_t func, void *client_data) if(NULL == (estack = H5E_get_my_stack())) /*lint !e506 !e774 Make lint 'constant value Boolean' in non-threaded case */ HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "can't get current error stack") + /* Get the automatic error reporting information */ + if(H5E_get_auto(estack, &auto_op, NULL) < 0) + HGOTO_ERROR(H5E_ERROR, H5E_CANTGET, FAIL, "can't get automatic error info") + /* Set the automatic error reporting information */ auto_op.vers = 1; - auto_op.u.func1 = func; + if(func != auto_op.func1_default) + auto_op.is_default = FALSE; + else + auto_op.is_default = TRUE; + auto_op.func1 = func; + if(H5E_set_auto(estack, &auto_op, client_data) < 0) HGOTO_ERROR(H5E_ERROR, H5E_CANTSET, FAIL, "can't set automatic error info") diff --git a/src/H5Eint.c b/src/H5Eint.c index 584ba40..f650581 100644 --- a/src/H5Eint.c +++ b/src/H5Eint.c @@ -1011,18 +1011,20 @@ H5E_dump_api_stack(hbool_t is_api) H5E_t *estack = H5E_get_my_stack(); HDassert(estack); + +#ifdef H5_NO_DEPRECATED_SYMBOLS + if(estack->auto_op.func2) + (void)((estack->auto_op.func2)(H5E_DEFAULT, estack->auto_data)); +#else /* H5_NO_DEPRECATED_SYMBOLS */ if(estack->auto_op.vers == 1) { -#ifndef H5_NO_DEPRECATED_SYMBOLS - if(estack->auto_op.u.func1) - (void)((estack->auto_op.u.func1)(estack->auto_data)); -#else /* H5_NO_DEPRECATED_SYMBOLS */ - HDassert(0 && "version 1 error stack dump without deprecated symbols!"); -#endif /* H5_NO_DEPRECATED_SYMBOLS */ + if(estack->auto_op.func1) + (void)((estack->auto_op.func1)(estack->auto_data)); } /* end if */ else { - if(estack->auto_op.u.func2) - (void)((estack->auto_op.u.func2)(H5E_DEFAULT, estack->auto_data)); + if(estack->auto_op.func2) + (void)((estack->auto_op.func2)(H5E_DEFAULT, estack->auto_data)); } /* end else */ +#endif /* H5_NO_DEPRECATED_SYMBOLS */ } /* end if */ done: diff --git a/src/H5Epkg.h b/src/H5Epkg.h index a85ddc9..9a1163a 100644 --- a/src/H5Epkg.h +++ b/src/H5Epkg.h @@ -68,15 +68,20 @@ /****************************/ /* Some syntactic sugar to make the compiler happy with two different kinds of callbacks */ -typedef struct { - unsigned vers; /* Which version callback to use */ - union { #ifndef H5_NO_DEPRECATED_SYMBOLS - H5E_auto1_t func1; /* Old-style callback, NO error stack param. */ -#endif /* H5_NO_DEPRECATED_SYMBOLS */ - H5E_auto2_t func2; /* New-style callback, with error stack param. */ - }u; +typedef struct { + unsigned vers; /* Which version callback to use */ + hbool_t is_default; /* If the printing function is the library's own. */ + H5E_auto1_t func1; /* Old-style callback, NO error stack param. */ + H5E_auto2_t func2; /* New-style callback, with error stack param. */ + H5E_auto1_t func1_default; /* The saved library's default function - old style. */ + H5E_auto2_t func2_default; /* The saved library's default function - new style. */ +} H5E_auto_op_t; +#else /* H5_NO_DEPRECATED_SYMBOLS */ +typedef struct { + H5E_auto_t func2; /* Only the new style callback function is available. */ } H5E_auto_op_t; +#endif /* H5_NO_DEPRECATED_SYMBOLS */ /* Some syntactic sugar to make the compiler happy with two different kinds of callbacks */ typedef struct { diff --git a/src/H5F.c b/src/H5F.c index a465759..c32400f 100644 --- a/src/H5F.c +++ b/src/H5F.c @@ -1048,7 +1048,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) } /* end if */ /* Destroy other components of the file */ - if(H5F_accum_reset(f, dxpl_id) < 0) + if(H5F_accum_reset(f, dxpl_id, TRUE) < 0) /* Push error, but keep going*/ HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "problems closing file") if(H5FO_dest(f) < 0) diff --git a/src/H5Faccum.c b/src/H5Faccum.c index 2fc53ea..c89ff33 100644 --- a/src/H5Faccum.c +++ b/src/H5Faccum.c @@ -111,11 +111,11 @@ H5FL_BLK_DEFINE_STATIC(meta_accum); * *------------------------------------------------------------------------- */ -htri_t +herr_t H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/) { - htri_t ret_value = FALSE; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5F_accum_read, FAIL) @@ -124,90 +124,135 @@ H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, HDassert(buf); /* Check if this information is in the metadata accumulator */ - if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && type != H5FD_MEM_DRAW - && size < H5F_ACCUM_MAX_SIZE) { - /* Sanity check */ - HDassert(!f->shared->accum.buf || (f->shared->accum.alloc_size >= f->shared->accum.size)); - - /* Current read adjoins or overlaps with metadata accumulator */ - if(H5F_addr_overlap(addr, size, f->shared->accum.loc, f->shared->accum.size) - || ((addr + size) == f->shared->accum.loc) - || (f->shared->accum.loc + f->shared->accum.size) == addr) { - size_t amount_before; /* Amount to read before current accumulator */ - haddr_t new_addr; /* New address of the accumulator buffer */ - size_t new_size; /* New size of the accumulator buffer */ - - /* Compute new values for accumulator */ - new_addr = MIN(addr, f->shared->accum.loc); - new_size = (size_t)(MAX((addr + size), (f->shared->accum.loc + f->shared->accum.size)) - - new_addr); - - /* Check if we need more buffer space */ - if(new_size > f->shared->accum.alloc_size) { - size_t new_alloc_size; /* New size of accumulator */ - - /* Adjust the buffer size to be a power of 2 that is large enough to hold data */ - new_alloc_size = (size_t)1 << (1 + H5V_log2_gen((uint64_t)(new_size - 1))); - - /* Reallocate the metadata accumulator buffer */ - if(NULL == (f->shared->accum.buf = H5FL_BLK_REALLOC(meta_accum, f->shared->accum.buf, new_alloc_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer") - - /* Note the new buffer size */ - f->shared->accum.alloc_size = new_alloc_size; + if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && type != H5FD_MEM_DRAW) { + if(size < H5F_ACCUM_MAX_SIZE) { + /* Sanity check */ + HDassert(!f->shared->accum.buf || (f->shared->accum.alloc_size >= f->shared->accum.size)); + + /* Current read adjoins or overlaps with metadata accumulator */ + if(H5F_addr_overlap(addr, size, f->shared->accum.loc, f->shared->accum.size) + || ((addr + size) == f->shared->accum.loc) + || (f->shared->accum.loc + f->shared->accum.size) == addr) { + size_t amount_before; /* Amount to read before current accumulator */ + haddr_t new_addr; /* New address of the accumulator buffer */ + size_t new_size; /* New size of the accumulator buffer */ + + /* Compute new values for accumulator */ + new_addr = MIN(addr, f->shared->accum.loc); + new_size = (size_t)(MAX((addr + size), (f->shared->accum.loc + f->shared->accum.size)) + - new_addr); + + /* Check if we need more buffer space */ + if(new_size > f->shared->accum.alloc_size) { + size_t new_alloc_size; /* New size of accumulator */ + + /* Adjust the buffer size to be a power of 2 that is large enough to hold data */ + new_alloc_size = (size_t)1 << (1 + H5V_log2_gen((uint64_t)(new_size - 1))); + + /* Reallocate the metadata accumulator buffer */ + if(NULL == (f->shared->accum.buf = H5FL_BLK_REALLOC(meta_accum, f->shared->accum.buf, new_alloc_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer") + + /* Note the new buffer size */ + f->shared->accum.alloc_size = new_alloc_size; #ifdef H5_CLEAR_MEMORY -HDmemset(f->shared->accum.buf + f->shared->accum.size, 0, (f->shared->accum.alloc_size - f->shared->accum.size)); + HDmemset(f->shared->accum.buf + f->shared->accum.size, 0, (f->shared->accum.alloc_size - f->shared->accum.size)); #endif /* H5_CLEAR_MEMORY */ - } /* end if */ + } /* end if */ - /* Read the part before the metadata accumulator */ - if(addr < f->shared->accum.loc) { - /* Set the amount to read */ - H5_ASSIGN_OVERFLOW(amount_before, (f->shared->accum.loc - addr), hsize_t, size_t); + /* Read the part before the metadata accumulator */ + if(addr < f->shared->accum.loc) { + /* Set the amount to read */ + H5_ASSIGN_OVERFLOW(amount_before, (f->shared->accum.loc - addr), hsize_t, size_t); - /* Make room for the metadata to read in */ - HDmemmove(f->shared->accum.buf + amount_before, f->shared->accum.buf, f->shared->accum.size); + /* Make room for the metadata to read in */ + HDmemmove(f->shared->accum.buf + amount_before, f->shared->accum.buf, f->shared->accum.size); - /* Adjust dirty region tracking info, if present */ - if(f->shared->accum.dirty) - f->shared->accum.dirty_off += amount_before; + /* Adjust dirty region tracking info, if present */ + if(f->shared->accum.dirty) + f->shared->accum.dirty_off += amount_before; - /* Dispatch to driver */ - if(H5FD_read(f->shared->lf, dxpl_id, type, addr, amount_before, f->shared->accum.buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") - } /* end if */ - else - amount_before = 0; + /* Dispatch to driver */ + if(H5FD_read(f->shared->lf, dxpl_id, type, addr, amount_before, f->shared->accum.buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") + } /* end if */ + else + amount_before = 0; - /* Read the part after the metadata accumulator */ - if((addr + size) > (f->shared->accum.loc + f->shared->accum.size)) { - size_t amount_after; /* Amount to read at a time */ + /* Read the part after the metadata accumulator */ + if((addr + size) > (f->shared->accum.loc + f->shared->accum.size)) { + size_t amount_after; /* Amount to read at a time */ - /* Set the amount to read */ - H5_ASSIGN_OVERFLOW(amount_after, ((addr + size) - (f->shared->accum.loc + f->shared->accum.size)), hsize_t, size_t); + /* Set the amount to read */ + H5_ASSIGN_OVERFLOW(amount_after, ((addr + size) - (f->shared->accum.loc + f->shared->accum.size)), hsize_t, size_t); - /* Dispatch to driver */ - if(H5FD_read(f->shared->lf, dxpl_id, type, (f->shared->accum.loc + f->shared->accum.size), amount_after, (f->shared->accum.buf + f->shared->accum.size + amount_before)) < 0) - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") - } /* end if */ + /* Dispatch to driver */ + if(H5FD_read(f->shared->lf, dxpl_id, type, (f->shared->accum.loc + f->shared->accum.size), amount_after, (f->shared->accum.buf + f->shared->accum.size + amount_before)) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") + } /* end if */ - /* Copy the data out of the buffer */ - HDmemcpy(buf, f->shared->accum.buf + (addr - new_addr), size); + /* Copy the data out of the buffer */ + HDmemcpy(buf, f->shared->accum.buf + (addr - new_addr), size); - /* Adjust the accumulator address & size */ - f->shared->accum.loc = new_addr; - f->shared->accum.size = new_size; + /* Adjust the accumulator address & size */ + f->shared->accum.loc = new_addr; + f->shared->accum.size = new_size; + } /* end if */ + /* Current read doesn't overlap with metadata accumulator, read it from file */ + else { + /* Dispatch to driver */ + if(H5FD_read(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") + } /* end else */ } /* end if */ - /* Current read doesn't overlap with metadata accumulator, read it from file */ else { - /* Dispatch to driver */ + /* Read the data */ if(H5FD_read(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") - } /* end else */ - /* Indicate success */ - HGOTO_DONE(TRUE); + /* Check for overlap w/dirty accumulator */ + /* (Note that this could be improved by updating the non-dirty + * information in the accumulator with [some of] the information + * just read in. -QAK) + */ + if(f->shared->accum.dirty && + H5F_addr_overlap(addr, size, f->shared->accum.loc + f->shared->accum.dirty_off, f->shared->accum.dirty_len)) { + haddr_t dirty_loc = f->shared->accum.loc + f->shared->accum.dirty_off; /* File offset of dirty information */ + size_t buf_off; /* Offset of dirty region in buffer */ + size_t dirty_off; /* Offset within dirty region */ + size_t overlap_size; /* Size of overlap with dirty region */ + + /* Check for read starting before beginning dirty region */ + if(H5F_addr_le(addr, dirty_loc)) { + /* Compute offset of dirty region within buffer */ + buf_off = (size_t)(dirty_loc - addr); + + /* Compute offset within dirty region */ + dirty_off = 0; + + /* Check for read ending within dirty region */ + if(H5F_addr_lt(addr + size, dirty_loc + f->shared->accum.dirty_len)) + overlap_size = (size_t)((addr + size) - buf_off); + else /* Access covers whole dirty region */ + overlap_size = f->shared->accum.dirty_len; + } /* end if */ + else { /* Read starts after beginning of dirty region */ + /* Compute dirty offset within buffer and overlap size */ + buf_off = 0; + dirty_off = (size_t)(addr - dirty_loc); + overlap_size = (size_t)((dirty_loc + f->shared->accum.dirty_len) - addr); + } /* end else */ + + /* Copy the dirty region to buffer */ + HDmemcpy((unsigned char *)buf + buf_off, (unsigned char *)f->shared->accum.buf + f->shared->accum.dirty_off + dirty_off, overlap_size); + } /* end if */ + } /* end else */ } /* end if */ + else { + /* Read the data */ + if(H5FD_read(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -312,12 +357,6 @@ H5F_accum_adjust(H5F_meta_accum_t *accum, H5FD_t *lf, hid_t dxpl_id, accum->dirty = FALSE; } /* end if */ - /* Move remnant of accumulator down */ - HDmemmove(accum->buf, (accum->buf + shrink_size), remnant_size); - - /* Adjust accumulator's location */ - accum->loc += shrink_size; - /* Adjust dirty region tracking info */ accum->dirty_off -= shrink_size; } /* end else */ @@ -325,6 +364,15 @@ H5F_accum_adjust(H5F_meta_accum_t *accum, H5FD_t *lf, hid_t dxpl_id, /* Trim the accumulator's use of its buffer */ accum->size = remnant_size; + + /* When appending, need to adjust location of accumulator */ + if(H5F_ACCUM_APPEND == adjust) { + /* Move remnant of accumulator down */ + HDmemmove(accum->buf, (accum->buf + shrink_size), remnant_size); + + /* Adjust accumulator's location */ + accum->loc += shrink_size; + } /* end if */ } /* end if */ /* Check for accumulator needing to be reallocated */ @@ -352,8 +400,8 @@ done: /*------------------------------------------------------------------------- * Function: H5F_accum_write * - * Purpose: Attempts to read some data from the metadata accumulator for - * a file into a buffer. + * Purpose: Attempts to write some data to the metadata accumulator for + * a file from a buffer. * * Return: Non-negative on success/Negative on failure * @@ -363,11 +411,11 @@ done: * *------------------------------------------------------------------------- */ -htri_t +herr_t H5F_accum_write(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf) { - htri_t ret_value = FALSE; /* Return value */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5F_accum_write, FAIL) @@ -377,248 +425,284 @@ H5F_accum_write(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, HDassert(buf); /* Check for accumulating metadata */ - if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && type != H5FD_MEM_DRAW - && size < H5F_ACCUM_MAX_SIZE) { - /* Sanity check */ - HDassert(!f->shared->accum.buf || (f->shared->accum.alloc_size >= f->shared->accum.size)); - - /* Check if there is already metadata in the accumulator */ - if(f->shared->accum.size > 0) { - /* Check if the new metadata adjoins the beginning of the current accumulator */ - if((addr + size) == f->shared->accum.loc) { - /* Check if we need to adjust accumulator size */ - if(H5F_accum_adjust(&f->shared->accum, f->shared->lf, dxpl_id, H5F_ACCUM_PREPEND, size) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") - - /* Move the existing metadata to the proper location */ - HDmemmove(f->shared->accum.buf + size, f->shared->accum.buf, f->shared->accum.size); - - /* Copy the new metadata at the front */ - HDmemcpy(f->shared->accum.buf, buf, size); - - /* Set the new size & location of the metadata accumulator */ - f->shared->accum.loc = addr; - f->shared->accum.size += size; - - /* Adjust the dirty region and mark accumulator dirty */ - if(f->shared->accum.dirty) - f->shared->accum.dirty_len = size + f->shared->accum.dirty_off - + f->shared->accum.dirty_len; - else { - f->shared->accum.dirty_len = size; - f->shared->accum.dirty = TRUE; - } /* end else */ - f->shared->accum.dirty_off = 0; - } /* end if */ - /* Check if the new metadata adjoins the end of the current accumulator */ - else if(addr == (f->shared->accum.loc + f->shared->accum.size)) { - /* Check if we need to adjust accumulator size */ - if(H5F_accum_adjust(&f->shared->accum, f->shared->lf, dxpl_id, H5F_ACCUM_APPEND, size) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") + if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && type != H5FD_MEM_DRAW) { + if(size < H5F_ACCUM_MAX_SIZE) { + /* Sanity check */ + HDassert(!f->shared->accum.buf || (f->shared->accum.alloc_size >= f->shared->accum.size)); + + /* Check if there is already metadata in the accumulator */ + if(f->shared->accum.size > 0) { + /* Check if the new metadata adjoins the beginning of the current accumulator */ + if((addr + size) == f->shared->accum.loc) { + /* Check if we need to adjust accumulator size */ + if(H5F_accum_adjust(&f->shared->accum, f->shared->lf, dxpl_id, H5F_ACCUM_PREPEND, size) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") - /* Copy the new metadata to the end */ - HDmemcpy(f->shared->accum.buf + f->shared->accum.size, buf, size); + /* Move the existing metadata to the proper location */ + HDmemmove(f->shared->accum.buf + size, f->shared->accum.buf, f->shared->accum.size); - /* Adjust the dirty region and mark accumulator dirty */ - if(f->shared->accum.dirty) - f->shared->accum.dirty_len = size + (f->shared->accum.size - - f->shared->accum.dirty_off); - else { - f->shared->accum.dirty_off = f->shared->accum.size; - f->shared->accum.dirty_len = size; - f->shared->accum.dirty = TRUE; - } /* end else */ + /* Copy the new metadata at the front */ + HDmemcpy(f->shared->accum.buf, buf, size); - /* Set the new size of the metadata accumulator */ - f->shared->accum.size += size; - } /* end if */ - /* Check if the piece of metadata being written overlaps the metadata accumulator */ - else if(H5F_addr_overlap(addr, size, f->shared->accum.loc, f->shared->accum.size)) { - size_t add_size; /* New size of the accumulator buffer */ + /* Set the new size & location of the metadata accumulator */ + f->shared->accum.loc = addr; + f->shared->accum.size += size; - /* Check if the new metadata is entirely within the current accumulator */ - if(addr >= f->shared->accum.loc && (addr + size) <= (f->shared->accum.loc + f->shared->accum.size)) { - size_t dirty_off = (size_t)(addr - f->shared->accum.loc); + /* Adjust the dirty region and mark accumulator dirty */ + if(f->shared->accum.dirty) + f->shared->accum.dirty_len = size + f->shared->accum.dirty_off + + f->shared->accum.dirty_len; + else { + f->shared->accum.dirty_len = size; + f->shared->accum.dirty = TRUE; + } /* end else */ + f->shared->accum.dirty_off = 0; + } /* end if */ + /* Check if the new metadata adjoins the end of the current accumulator */ + else if(addr == (f->shared->accum.loc + f->shared->accum.size)) { + /* Check if we need to adjust accumulator size */ + if(H5F_accum_adjust(&f->shared->accum, f->shared->lf, dxpl_id, H5F_ACCUM_APPEND, size) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") - /* Copy the new metadata to the proper location within the accumulator */ - HDmemcpy(f->shared->accum.buf + dirty_off, buf, size); + /* Copy the new metadata to the end */ + HDmemcpy(f->shared->accum.buf + f->shared->accum.size, buf, size); /* Adjust the dirty region and mark accumulator dirty */ - if(f->shared->accum.dirty) { - /* Check for new metadata starting before current dirty region */ - if(dirty_off <= f->shared->accum.dirty_off) { - if((dirty_off + size) <= (f->shared->accum.dirty_off + f->shared->accum.dirty_len)) - f->shared->accum.dirty_len = (f->shared->accum.dirty_off + f->shared->accum.dirty_len) - dirty_off; - else - f->shared->accum.dirty_len = size; - f->shared->accum.dirty_off = dirty_off; - } /* end if */ - else { - if((dirty_off + size) <= (f->shared->accum.dirty_off + f->shared->accum.dirty_len)) - ; /* f->shared->accum.dirty_len doesn't change */ - else - f->shared->accum.dirty_len = (dirty_off + size) - f->shared->accum.dirty_off; - } /* end else */ - } /* end if */ + if(f->shared->accum.dirty) + f->shared->accum.dirty_len = size + (f->shared->accum.size - + f->shared->accum.dirty_off); else { - f->shared->accum.dirty_off = dirty_off; + f->shared->accum.dirty_off = f->shared->accum.size; f->shared->accum.dirty_len = size; f->shared->accum.dirty = TRUE; } /* end else */ + + /* Set the new size of the metadata accumulator */ + f->shared->accum.size += size; } /* end if */ - /* Check if the new metadata overlaps the beginning of the current accumulator */ - else if(addr < f->shared->accum.loc && (addr + size) <= (f->shared->accum.loc + f->shared->accum.size)) { - size_t old_offset; /* Offset of old data within the accumulator buffer */ + /* Check if the piece of metadata being written overlaps the metadata accumulator */ + else if(H5F_addr_overlap(addr, size, f->shared->accum.loc, f->shared->accum.size)) { + size_t add_size; /* New size of the accumulator buffer */ + + /* Check if the new metadata is entirely within the current accumulator */ + if(addr >= f->shared->accum.loc && (addr + size) <= (f->shared->accum.loc + f->shared->accum.size)) { + size_t dirty_off = (size_t)(addr - f->shared->accum.loc); + + /* Copy the new metadata to the proper location within the accumulator */ + HDmemcpy(f->shared->accum.buf + dirty_off, buf, size); + + /* Adjust the dirty region and mark accumulator dirty */ + if(f->shared->accum.dirty) { + /* Check for new metadata starting before current dirty region */ + if(dirty_off <= f->shared->accum.dirty_off) { + if((dirty_off + size) <= (f->shared->accum.dirty_off + f->shared->accum.dirty_len)) + f->shared->accum.dirty_len = (f->shared->accum.dirty_off + f->shared->accum.dirty_len) - dirty_off; + else + f->shared->accum.dirty_len = size; + f->shared->accum.dirty_off = dirty_off; + } /* end if */ + else { + if((dirty_off + size) <= (f->shared->accum.dirty_off + f->shared->accum.dirty_len)) + ; /* f->shared->accum.dirty_len doesn't change */ + else + f->shared->accum.dirty_len = (dirty_off + size) - f->shared->accum.dirty_off; + } /* end else */ + } /* end if */ + else { + f->shared->accum.dirty_off = dirty_off; + f->shared->accum.dirty_len = size; + f->shared->accum.dirty = TRUE; + } /* end else */ + } /* end if */ + /* Check if the new metadata overlaps the beginning of the current accumulator */ + else if(addr < f->shared->accum.loc && (addr + size) <= (f->shared->accum.loc + f->shared->accum.size)) { + size_t old_offset; /* Offset of old data within the accumulator buffer */ - /* Calculate the amount we will need to add to the accumulator size, based on the amount of overlap */ - H5_ASSIGN_OVERFLOW(add_size, (f->shared->accum.loc - addr), hsize_t, size_t); + /* Calculate the amount we will need to add to the accumulator size, based on the amount of overlap */ + H5_ASSIGN_OVERFLOW(add_size, (f->shared->accum.loc - addr), hsize_t, size_t); - /* Check if we need to adjust accumulator size */ - if(H5F_accum_adjust(&f->shared->accum, f->shared->lf, dxpl_id, H5F_ACCUM_PREPEND, add_size) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") + /* Check if we need to adjust accumulator size */ + if(H5F_accum_adjust(&f->shared->accum, f->shared->lf, dxpl_id, H5F_ACCUM_PREPEND, add_size) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") - /* Calculate the proper offset of the existing metadata */ - H5_ASSIGN_OVERFLOW(old_offset, (addr + size) - f->shared->accum.loc, hsize_t, size_t); + /* Calculate the proper offset of the existing metadata */ + H5_ASSIGN_OVERFLOW(old_offset, (addr + size) - f->shared->accum.loc, hsize_t, size_t); - /* Move the existing metadata to the proper location */ - HDmemmove(f->shared->accum.buf + size, f->shared->accum.buf + old_offset, (f->shared->accum.size - old_offset)); + /* Move the existing metadata to the proper location */ + HDmemmove(f->shared->accum.buf + size, f->shared->accum.buf + old_offset, (f->shared->accum.size - old_offset)); - /* Copy the new metadata at the front */ - HDmemcpy(f->shared->accum.buf, buf, size); + /* Copy the new metadata at the front */ + HDmemcpy(f->shared->accum.buf, buf, size); - /* Set the new size & location of the metadata accumulator */ - f->shared->accum.loc = addr; - f->shared->accum.size += add_size; + /* Set the new size & location of the metadata accumulator */ + f->shared->accum.loc = addr; + f->shared->accum.size += add_size; - /* Adjust the dirty region and mark accumulator dirty */ - if(f->shared->accum.dirty) { - size_t curr_dirty_end = add_size + f->shared->accum.dirty_off + f->shared->accum.dirty_len; + /* Adjust the dirty region and mark accumulator dirty */ + if(f->shared->accum.dirty) { + size_t curr_dirty_end = add_size + f->shared->accum.dirty_off + f->shared->accum.dirty_len; - f->shared->accum.dirty_off = 0; - if(size <= curr_dirty_end) - f->shared->accum.dirty_len = curr_dirty_end; - else + f->shared->accum.dirty_off = 0; + if(size <= curr_dirty_end) + f->shared->accum.dirty_len = curr_dirty_end; + else + f->shared->accum.dirty_len = size; + } /* end if */ + else { + f->shared->accum.dirty_off = 0; f->shared->accum.dirty_len = size; + f->shared->accum.dirty = TRUE; + } /* end else */ } /* end if */ - else { - f->shared->accum.dirty_off = 0; - f->shared->accum.dirty_len = size; - f->shared->accum.dirty = TRUE; - } /* end else */ - } /* end if */ - /* Check if the new metadata overlaps the end of the current accumulator */ - else if(addr >= f->shared->accum.loc && (addr + size) > (f->shared->accum.loc + f->shared->accum.size)) { - size_t dirty_off = (size_t)(addr - f->shared->accum.loc); + /* Check if the new metadata overlaps the end of the current accumulator */ + else if(addr >= f->shared->accum.loc && (addr + size) > (f->shared->accum.loc + f->shared->accum.size)) { + size_t dirty_off; /* Offset of dirty region */ - /* Calculate the amount we will need to add to the accumulator size, based on the amount of overlap */ - H5_ASSIGN_OVERFLOW(add_size, (addr + size) - (f->shared->accum.loc + f->shared->accum.size), hsize_t, size_t); + /* Calculate the amount we will need to add to the accumulator size, based on the amount of overlap */ + H5_ASSIGN_OVERFLOW(add_size, (addr + size) - (f->shared->accum.loc + f->shared->accum.size), hsize_t, size_t); - /* Check if we need to adjust accumulator size */ - if(H5F_accum_adjust(&f->shared->accum, f->shared->lf, dxpl_id, H5F_ACCUM_APPEND, add_size) < 0) - HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") + /* Check if we need to adjust accumulator size */ + if(H5F_accum_adjust(&f->shared->accum, f->shared->lf, dxpl_id, H5F_ACCUM_APPEND, add_size) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTRESIZE, FAIL, "can't adjust metadata accumulator") - /* Copy the new metadata to the end */ - HDmemcpy(f->shared->accum.buf + dirty_off, buf, size); + /* Compute offset of dirty region (after adjusting accumulator) */ + dirty_off = (size_t)(addr - f->shared->accum.loc); - /* Set the new size of the metadata accumulator */ - f->shared->accum.size += add_size; + /* Copy the new metadata to the end */ + HDmemcpy(f->shared->accum.buf + dirty_off, buf, size); - /* Adjust the dirty region and mark accumulator dirty */ - if(f->shared->accum.dirty) { - /* Check for new metadata starting before current dirty region */ - if(dirty_off <= f->shared->accum.dirty_off) { - f->shared->accum.dirty_off = dirty_off; - f->shared->accum.dirty_len = size; + /* Set the new size of the metadata accumulator */ + f->shared->accum.size += add_size; + + /* Adjust the dirty region and mark accumulator dirty */ + if(f->shared->accum.dirty) { + /* Check for new metadata starting before current dirty region */ + if(dirty_off <= f->shared->accum.dirty_off) { + f->shared->accum.dirty_off = dirty_off; + f->shared->accum.dirty_len = size; + } /* end if */ + else { + f->shared->accum.dirty_len = (dirty_off + size) - f->shared->accum.dirty_off; + } /* end else */ } /* end if */ else { - f->shared->accum.dirty_len = (dirty_off + size) - f->shared->accum.dirty_off; + f->shared->accum.dirty_off = dirty_off; + f->shared->accum.dirty_len = size; + f->shared->accum.dirty = TRUE; } /* end else */ } /* end if */ + /* New metadata overlaps both ends of the current accumulator */ else { - f->shared->accum.dirty_off = dirty_off; + /* Check if we need more buffer space */ + if(size > f->shared->accum.alloc_size) { + size_t new_alloc_size; /* New size of accumulator */ + + /* Adjust the buffer size to be a power of 2 that is large enough to hold data */ + new_alloc_size = (size_t)1 << (1 + H5V_log2_gen((uint64_t)(size - 1))); + + /* Reallocate the metadata accumulator buffer */ + if(NULL == (f->shared->accum.buf = H5FL_BLK_REALLOC(meta_accum, f->shared->accum.buf, new_alloc_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer") + + /* Note the new buffer size */ + f->shared->accum.alloc_size = new_alloc_size; +#ifdef H5_CLEAR_MEMORY +HDmemset(f->shared->accum.buf + size, 0, (f->shared->accum.alloc_size - size)); +#endif /* H5_CLEAR_MEMORY */ + } /* end if */ + + /* Copy the new metadata to the buffer */ + HDmemcpy(f->shared->accum.buf, buf, size); + + /* Set the new size & location of the metadata accumulator */ + f->shared->accum.loc = addr; + f->shared->accum.size = size; + + /* Adjust the dirty region and mark accumulator dirty */ + f->shared->accum.dirty_off = 0; f->shared->accum.dirty_len = size; f->shared->accum.dirty = TRUE; } /* end else */ } /* end if */ - /* New metadata overlaps both ends of the current accumulator */ + /* New piece of metadata doesn't adjoin or overlap the existing accumulator */ else { - /* Check if we need more buffer space */ + /* Write out the existing metadata accumulator, with dispatch to driver */ + if(f->shared->accum.dirty) { + if(H5FD_write(f->shared->lf, dxpl_id, H5FD_MEM_DEFAULT, f->shared->accum.loc + f->shared->accum.dirty_off, f->shared->accum.dirty_len, f->shared->accum.buf + f->shared->accum.dirty_off) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") + + /* Reset accumulator dirty flag */ + f->shared->accum.dirty = FALSE; + } /* end if */ + + /* Cache the new piece of metadata */ + /* Check if we need to resize the buffer */ if(size > f->shared->accum.alloc_size) { - size_t new_alloc_size; /* New size of accumulator */ + size_t new_size; /* New size of accumulator */ /* Adjust the buffer size to be a power of 2 that is large enough to hold data */ - new_alloc_size = (size_t)1 << (1 + H5V_log2_gen((uint64_t)(size - 1))); + new_size = (size_t)1 << (1 + H5V_log2_gen((uint64_t)(size - 1))); - /* Reallocate the metadata accumulator buffer */ - if(NULL == (f->shared->accum.buf = H5FL_BLK_REALLOC(meta_accum, f->shared->accum.buf, new_alloc_size))) + /* Grow the metadata accumulator buffer */ + if(NULL == (f->shared->accum.buf = H5FL_BLK_REALLOC(meta_accum, f->shared->accum.buf, new_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer") /* Note the new buffer size */ - f->shared->accum.alloc_size = new_alloc_size; + f->shared->accum.alloc_size = new_size; #ifdef H5_CLEAR_MEMORY -HDmemset(f->shared->accum.buf + size, 0, (f->shared->accum.alloc_size - size)); +{ +size_t clear_size = MAX(f->shared->accum.size, size); +HDmemset(f->shared->accum.buf + clear_size, 0, (f->shared->accum.alloc_size - clear_size)); +} #endif /* H5_CLEAR_MEMORY */ } /* end if */ + else { + /* Check if we should shrink the accumulator buffer */ + if(size < (f->shared->accum.alloc_size / H5F_ACCUM_THROTTLE) && + f->shared->accum.alloc_size > H5F_ACCUM_THRESHOLD) { + size_t tmp_size = (f->shared->accum.alloc_size / H5F_ACCUM_THROTTLE); /* New size of accumulator buffer */ - /* Copy the new metadata to the buffer */ - HDmemcpy(f->shared->accum.buf, buf, size); + /* Shrink the accumulator buffer */ + if(NULL == (f->shared->accum.buf = H5FL_BLK_REALLOC(meta_accum, f->shared->accum.buf, tmp_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer") - /* Set the new size & location of the metadata accumulator */ + /* Note the new buffer size */ + f->shared->accum.alloc_size = tmp_size; + } /* end if */ + } /* end else */ + + /* Update the metadata accumulator information */ f->shared->accum.loc = addr; f->shared->accum.size = size; + /* Store the piece of metadata in the accumulator */ + HDmemcpy(f->shared->accum.buf, buf, size); + /* Adjust the dirty region and mark accumulator dirty */ f->shared->accum.dirty_off = 0; f->shared->accum.dirty_len = size; f->shared->accum.dirty = TRUE; } /* end else */ } /* end if */ - /* New piece of metadata doesn't adjoin or overlap the existing accumulator */ + /* No metadata in the accumulator, grab this piece and keep it */ else { - /* Write out the existing metadata accumulator, with dispatch to driver */ - if(f->shared->accum.dirty) { - if(H5FD_write(f->shared->lf, dxpl_id, H5FD_MEM_DEFAULT, f->shared->accum.loc + f->shared->accum.dirty_off, f->shared->accum.dirty_len, f->shared->accum.buf + f->shared->accum.dirty_off) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") - - /* Reset accumulator dirty flag */ - f->shared->accum.dirty = FALSE; - } /* end if */ - - /* Cache the new piece of metadata */ - /* Check if we need to resize the buffer */ + /* Check if we need to reallocate the buffer */ if(size > f->shared->accum.alloc_size) { size_t new_size; /* New size of accumulator */ /* Adjust the buffer size to be a power of 2 that is large enough to hold data */ new_size = (size_t)1 << (1 + H5V_log2_gen((uint64_t)(size - 1))); - /* Grow the metadata accumulator buffer */ + /* Reallocate the metadata accumulator buffer */ if(NULL == (f->shared->accum.buf = H5FL_BLK_REALLOC(meta_accum, f->shared->accum.buf, new_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer") /* Note the new buffer size */ f->shared->accum.alloc_size = new_size; #ifdef H5_CLEAR_MEMORY -{ -size_t clear_size = MAX(f->shared->accum.size, size); -HDmemset(f->shared->accum.buf + clear_size, 0, (f->shared->accum.alloc_size - clear_size)); -} +HDmemset(f->shared->accum.buf + size, 0, (f->shared->accum.alloc_size - size)); #endif /* H5_CLEAR_MEMORY */ } /* end if */ - else { - /* Check if we should shrink the accumulator buffer */ - if(size < (f->shared->accum.alloc_size / H5F_ACCUM_THROTTLE) && - f->shared->accum.alloc_size > H5F_ACCUM_THRESHOLD) { - size_t tmp_size = (f->shared->accum.alloc_size / H5F_ACCUM_THROTTLE); /* New size of accumulator buffer */ - - /* Shrink the accumulator buffer */ - if(NULL == (f->shared->accum.buf = H5FL_BLK_REALLOC(meta_accum, f->shared->accum.buf, tmp_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer") - - /* Note the new buffer size */ - f->shared->accum.alloc_size = tmp_size; - } /* end if */ - } /* end else */ /* Update the metadata accumulator information */ f->shared->accum.loc = addr; @@ -633,42 +717,96 @@ HDmemset(f->shared->accum.buf + clear_size, 0, (f->shared->accum.alloc_size - cl f->shared->accum.dirty = TRUE; } /* end else */ } /* end if */ - /* No metadata in the accumulator, grab this piece and keep it */ else { - /* Check if we need to reallocate the buffer */ - if(size > f->shared->accum.alloc_size) { - size_t new_size; /* New size of accumulator */ + /* Write the data */ + if(H5FD_write(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") + + /* Check for overlap w/accumulator */ + /* (Note that this could be improved by updating the accumulator + * with [some of] the information just read in. -QAK) + */ + if(H5F_addr_overlap(addr, size, f->shared->accum.loc, f->shared->accum.size)) { + /* Check for write starting before beginning of accumulator */ + if(H5F_addr_le(addr, f->shared->accum.loc)) { + /* Check for write ending within accumulator */ + if(H5F_addr_le(addr + size, f->shared->accum.loc + f->shared->accum.size)) { + size_t overlap_size; /* Size of overlapping region */ + + /* Compute overlap size */ + overlap_size = (size_t)((addr + size) - f->shared->accum.loc); + + /* Check for dirty region */ + if(f->shared->accum.dirty) { + haddr_t dirty_start = f->shared->accum.loc + f->shared->accum.dirty_off; /* File address of start of dirty region */ + haddr_t dirty_end = dirty_start + f->shared->accum.dirty_len; /* File address of end of dirty region */ + + /* Check if entire dirty region is overwritten */ + if(H5F_addr_le(dirty_end, addr + size)) { + f->shared->accum.dirty = FALSE; + f->shared->accum.dirty_len = 0; + } /* end if */ + else { + /* Check for dirty region falling after write */ + if(H5F_addr_le(addr + size, dirty_start)) + f->shared->accum.dirty_off = overlap_size; + else { /* Dirty region overlaps w/written region */ + f->shared->accum.dirty_off = 0; + f->shared->accum.dirty_len -= (size_t)((addr + size) - dirty_start); + } /* end else */ + } /* end if */ + } /* end if */ - /* Adjust the buffer size to be a power of 2 that is large enough to hold data */ - new_size = (size_t)1 << (1 + H5V_log2_gen((uint64_t)(size - 1))); + /* Trim bottom of accumulator off */ + f->shared->accum.loc += overlap_size; + f->shared->accum.size -= overlap_size; + HDmemmove(f->shared->accum.buf, f->shared->accum.buf + overlap_size, f->shared->accum.size); + } /* end if */ + else { /* Access covers whole accumulator */ + /* Reset accumulator, but don't flush */ + if(H5F_accum_reset(f, dxpl_id, FALSE) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTRESET, FAIL, "can't reset accumulator") + } /* end else */ + } /* end if */ + else { /* Write starts after beginning of accumulator */ + size_t overlap_size; /* Size of overlapping region */ - /* Reallocate the metadata accumulator buffer */ - if(NULL == (f->shared->accum.buf = H5FL_BLK_REALLOC(meta_accum, f->shared->accum.buf, new_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate metadata accumulator buffer") + /* Sanity check */ + HDassert(H5F_addr_gt(addr + size, f->shared->accum.loc + f->shared->accum.size)); - /* Note the new buffer size */ - f->shared->accum.alloc_size = new_size; -#ifdef H5_CLEAR_MEMORY -HDmemset(f->shared->accum.buf + size, 0, (f->shared->accum.alloc_size - size)); -#endif /* H5_CLEAR_MEMORY */ - } /* end if */ + /* Compute overlap size */ + overlap_size = (size_t)((f->shared->accum.loc + f->shared->accum.size) - addr); - /* Update the metadata accumulator information */ - f->shared->accum.loc = addr; - f->shared->accum.size = size; + /* Check for dirty region */ + if(f->shared->accum.dirty) { + haddr_t dirty_start = f->shared->accum.loc + f->shared->accum.dirty_off; /* File address of start of dirty region */ + haddr_t dirty_end = dirty_start + f->shared->accum.dirty_len; /* File address of end of dirty region */ - /* Store the piece of metadata in the accumulator */ - HDmemcpy(f->shared->accum.buf, buf, size); + /* Check if entire dirty region is overwritten */ + if(H5F_addr_ge(dirty_start, addr)) { + f->shared->accum.dirty = FALSE; + f->shared->accum.dirty_len = 0; + } /* end if */ + else { + /* Check for dirty region falling before write */ + if(H5F_addr_le(dirty_end, addr)) + ; /* noop */ + else /* Dirty region overlaps w/written region */ + f->shared->accum.dirty_len = (size_t)(addr - dirty_start); + } /* end if */ + } /* end if */ - /* Adjust the dirty region and mark accumulator dirty */ - f->shared->accum.dirty_off = 0; - f->shared->accum.dirty_len = size; - f->shared->accum.dirty = TRUE; + /* Trim top of accumulator off */ + f->shared->accum.size -= overlap_size; + } /* end else */ + } /* end if */ } /* end else */ - - /* Indicate success */ - HGOTO_DONE(TRUE); } /* end if */ + else { + /* Write the data */ + if(H5FD_write(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") + } /* end else */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -775,22 +913,20 @@ H5F_accum_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t UNUSED type, haddr_t addr, HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end if */ /* Block to free overlaps with some/all of dirty region */ - else { + /* Check for unfreed dirty region to write */ + else if(H5F_addr_lt(tail_addr, dirty_end)) { size_t write_size; + size_t dirty_delta; write_size = (size_t)(dirty_end - tail_addr); + dirty_delta = f->shared->accum.dirty_len - write_size; - /* Check for unfreed dirty region to write */ - if(write_size > 0) { - size_t dirty_delta; - - dirty_delta = f->shared->accum.dirty_len - write_size; + HDassert(write_size > 0); - /* Write out the unfreed dirty region of the accumulator */ - if(H5FD_write(f->shared->lf, dxpl_id, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, f->shared->accum.buf + f->shared->accum.dirty_off + dirty_delta) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") - } /* end if */ - } /* end else */ + /* Write out the unfreed dirty region of the accumulator */ + if(H5FD_write(f->shared->lf, dxpl_id, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, f->shared->accum.buf + f->shared->accum.dirty_off + dirty_delta) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") + } /* end if */ /* Reset dirty flag */ f->shared->accum.dirty = FALSE; @@ -800,19 +936,16 @@ H5F_accum_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t UNUSED type, haddr_t addr, /* Check if block to free ends before end of dirty region */ if(H5F_addr_lt(tail_addr, dirty_end)) { size_t write_size; + size_t dirty_delta; write_size = (size_t)(dirty_end - tail_addr); + dirty_delta = f->shared->accum.dirty_len - write_size; - /* Check for unfreed dirty region to write */ - if(write_size > 0) { - size_t dirty_delta; + HDassert(write_size > 0); - dirty_delta = f->shared->accum.dirty_len - write_size; - - /* Write out the unfreed end of the dirty region of the accumulator */ - if(H5FD_write(f->shared->lf, dxpl_id, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, f->shared->accum.buf + f->shared->accum.dirty_off + dirty_delta) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") - } /* end if */ + /* Write out the unfreed end of the dirty region of the accumulator */ + if(H5FD_write(f->shared->lf, dxpl_id, H5FD_MEM_DEFAULT, dirty_start + dirty_delta, write_size, f->shared->accum.buf + f->shared->accum.dirty_off + dirty_delta) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end if */ /* Check for block to free beginning at same location as dirty region */ @@ -822,7 +955,7 @@ H5F_accum_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t UNUSED type, haddr_t addr, } /* end if */ /* Block to free eliminates end of dirty region */ else { - f->shared->accum.dirty_len = (addr - dirty_start); + f->shared->accum.dirty_len = (size_t)(addr - dirty_start); } /* end else */ } /* end else */ @@ -852,7 +985,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_accum_flush(H5F_t *f, hid_t dxpl_id) +H5F_accum_flush(const H5F_t *f, hid_t dxpl_id) { herr_t ret_value = SUCCEED; /* Return value */ @@ -890,7 +1023,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_accum_reset(H5F_t *f, hid_t dxpl_id) +H5F_accum_reset(const H5F_t *f, hid_t dxpl_id, hbool_t flush) { herr_t ret_value = SUCCEED; /* Return value */ @@ -899,9 +1032,10 @@ H5F_accum_reset(H5F_t *f, hid_t dxpl_id) HDassert(f); HDassert(f->shared); - /* Flush any dirty data in accumulator */ - if(H5F_accum_flush(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "can't flush metadata accumulator") + /* Flush any dirty data in accumulator, if requested */ + if(flush) + if(H5F_accum_flush(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "can't flush metadata accumulator") /* Check if we need to reset the metadata accumulator information */ if(f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) { diff --git a/src/H5Fio.c b/src/H5Fio.c index 407f950..231c4c9 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -95,7 +95,6 @@ herr_t H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, hid_t dxpl_id, void *buf/*out*/) { - htri_t accumulated; /* Whether the data was accepted by the metadata accumulator */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5F_block_read, FAIL) @@ -108,14 +107,9 @@ H5F_block_read(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, if(H5F_addr_le(f->shared->tmp_addr, (addr + size))) HGOTO_ERROR(H5E_IO, H5E_BADRANGE, FAIL, "attempting I/O in temporary file space") - /* Check if this I/O can be satisfied by the metadata accumulator */ - if((accumulated = H5F_accum_read(f, dxpl_id, type, addr, size, buf)) < 0) - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read from metadata accumulator failed") - else if(accumulated == FALSE) { - /* Read the data */ - if(H5FD_read(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") - } /* end else */ + /* Pass through metadata accumulator layer */ + if(H5F_accum_read(f, dxpl_id, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "read through metadata accumulator failed") done: FUNC_LEAVE_NOAPI(ret_value) @@ -141,7 +135,6 @@ herr_t H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr, size_t size, hid_t dxpl_id, const void *buf) { - htri_t accumulated; /* Whether the data was accepted by the metadata accumulator */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5F_block_write, FAIL) @@ -158,14 +151,9 @@ HDfprintf(stderr, "%s: write to addr = %a, size = %Zu\n", FUNC, addr, size); if(H5F_addr_le(f->shared->tmp_addr, (addr + size))) HGOTO_ERROR(H5E_IO, H5E_BADRANGE, FAIL, "attempting I/O in temporary file space") - /* Check for accumulating metadata */ - if((accumulated = H5F_accum_write(f, dxpl_id, type, addr, size, buf)) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write to metadata accumulator failed") - else if(accumulated == FALSE) { - /* Write the data */ - if(H5FD_write(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") - } /* end else */ + /* Pass through metadata accumulator layer */ + if(H5F_accum_write(f, dxpl_id, type, addr, size, buf) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "write through metadata accumulator failed") done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 0d30e53..5bff8c6 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -310,17 +310,18 @@ H5_DLL herr_t H5F_super_free(H5F_super_t *sblock); H5_DLL herr_t H5F_super_ext_open(H5F_t *f, haddr_t ext_addr, H5O_loc_t *ext_ptr); H5_DLL herr_t H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, void *mesg, unsigned id, hbool_t may_create); H5_DLL herr_t H5F_super_ext_remove_msg(H5F_t *f, hid_t dxpl_id, unsigned id); -H5_DLL herr_t H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr); +H5_DLL herr_t H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr, hid_t dxpl_id, + hbool_t was_created); /* Metadata accumulator routines */ -H5_DLL htri_t H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, +H5_DLL herr_t H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, size_t size, void *buf); -H5_DLL htri_t H5F_accum_write(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, +H5_DLL herr_t H5F_accum_write(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf); H5_DLL herr_t H5F_accum_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, hsize_t size); -H5_DLL herr_t H5F_accum_flush(H5F_t *f, hid_t dxpl_id); -H5_DLL herr_t H5F_accum_reset(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5F_accum_flush(const H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5F_accum_reset(const H5F_t *f, hid_t dxpl_id, hbool_t flush); /* Shared file list related routines */ H5_DLL herr_t H5F_sfile_add(H5F_file_t *shared); diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index a259dde..2510487 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -204,7 +204,7 @@ H5F_super_ext_create(H5F_t *f, hid_t dxpl_id, H5O_loc_t *ext_ptr) * extension. */ H5O_loc_reset(ext_ptr); - if(H5O_create(f, dxpl_id, 0, H5P_GROUP_CREATE_DEFAULT, ext_ptr) < 0) + if(H5O_create(f, dxpl_id, 0, (size_t)1, H5P_GROUP_CREATE_DEFAULT, ext_ptr) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, FAIL, "unable to create superblock extension") /* Record the address of the superblock extension */ @@ -267,7 +267,8 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr) +H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr, hid_t dxpl_id, + hbool_t was_created) { herr_t ret_value = SUCCEED; /* Return value */ @@ -277,6 +278,17 @@ H5F_super_ext_close(H5F_t *f, H5O_loc_t *ext_ptr) HDassert(f); HDassert(ext_ptr); + /* Check if extension was created */ + if(was_created) { + /* Increment link count on superblock extension's object header */ + if(H5O_link(ext_ptr, 1, dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_LINKCOUNT, FAIL, "unable to increment hard link count") + + /* Decrement refcount on superblock extension's object header in memory */ + if(H5O_dec_rc_by_loc(ext_ptr, dxpl_id) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "unable to decrement refcount on superblock extension") + } /* end if */ + /* Twiddle the number of open objects to avoid closing the file. */ f->nopen_objs++; if(H5O_close(ext_ptr) < 0) @@ -384,7 +396,9 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) hsize_t superblock_size; /* Size of superblock, in bytes */ size_t driver_size; /* Size of driver info block (bytes) */ unsigned super_vers = HDF5_SUPERBLOCK_VERSION_DEF; /* Superblock version for file */ + H5O_loc_t ext_loc; /* Superblock extension object location */ hbool_t need_ext; /* Whether the superblock extension is needed */ + hbool_t ext_created = FALSE; /* Whether the extension has been created */ herr_t ret_value = SUCCEED; /* Return Value */ FUNC_ENTER_NOAPI_TAG(H5F_super_init, dxpl_id, H5AC__SUPERBLOCK_TAG, FAIL) @@ -545,8 +559,6 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) /* Create the superblock extension for "extra" superblock data, if necessary. */ if(need_ext) { - H5O_loc_t ext_loc; /* Superblock extension object location */ - /* The superblock extension isn't actually a group, but the * default group creation list should work fine. * If we don't supply a size for the object header, HDF5 will @@ -557,6 +569,7 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) */ if(H5F_super_ext_create(f, dxpl_id, &ext_loc) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCREATE, FAIL, "unable to create superblock extension") + ext_created = TRUE; /* Create the Shared Object Header Message table and register it with * the metadata cache, if this file supports shared messages. @@ -615,13 +628,13 @@ H5F_super_init(H5F_t *f, hid_t dxpl_id) if(H5O_msg_create(&ext_loc, H5O_FSINFO_ID, H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, &fsinfo, dxpl_id) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to update free-space info header message") } /* end if */ - - /* Close superblock extension */ - if(H5F_super_ext_close(f, &ext_loc) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to close file's superblock extension") } /* end if */ done: + /* Close superblock extension, if it was created */ + if(ext_created && H5F_super_ext_close(f, &ext_loc, dxpl_id, ext_created) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to close file's superblock extension") + /* Cleanup on failure */ if(ret_value < 0) { /* Check if the superblock has been allocated yet */ @@ -786,7 +799,8 @@ done: herr_t H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, void *mesg, unsigned id, hbool_t may_create) { - hbool_t sblock_dirty = FALSE; /* Whether superblock was dirtied */ + hbool_t ext_created = FALSE; /* Whether superblock extension was created */ + hbool_t ext_opened = FALSE; /* Whether superblock extension was opened */ H5O_loc_t ext_loc; /* "Object location" for superblock extension */ htri_t status; /* Indicate whether the message exists or not */ herr_t ret_value = SUCCEED; /* Return value */ @@ -807,9 +821,10 @@ H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, void *mesg, unsigned id, hbool_ HDassert(may_create); if(H5F_super_ext_create(f, dxpl_id, &ext_loc) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCREATE, FAIL, "unable to create file's superblock extension") - sblock_dirty = TRUE; + ext_created = TRUE; } /* end else */ HDassert(H5F_addr_defined(ext_loc.addr)); + ext_opened = TRUE; /* Check if message with ID does not exist in the object header */ if((status = H5O_msg_exists(&ext_loc, id, dxpl_id)) < 0) @@ -833,15 +848,14 @@ H5F_super_ext_write_msg(H5F_t *f, hid_t dxpl_id, void *mesg, unsigned id, hbool_ HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to write the message in object header") } /* end else */ - /* Close the superblock extension object header */ - if(H5F_super_ext_close(f, &ext_loc) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to close file's superblock extension") - done: - /* Mark superblock dirty in cache, if necessary */ - if(sblock_dirty) - if(H5AC_mark_entry_dirty(f->shared->sblock) < 0) - HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") + /* Close the superblock extension, if it was opened */ + if(ext_opened && H5F_super_ext_close(f, &ext_loc, dxpl_id, ext_created) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to close file's superblock extension") + + /* Mark superblock dirty in cache, if superblock extension was created */ + if(ext_created && H5AC_mark_entry_dirty(f->shared->sblock) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTMARKDIRTY, FAIL, "unable to mark superblock as dirty") FUNC_LEAVE_NOAPI(ret_value) } /* H5F_super_ext_write_msg() */ @@ -861,9 +875,10 @@ done: herr_t H5F_super_ext_remove_msg(H5F_t *f, hid_t dxpl_id, unsigned id) { - htri_t status; /* Indicate whether the message exists or not */ H5O_loc_t ext_loc; /* "Object location" for superblock extension */ + hbool_t ext_opened = FALSE; /* Whether the superblock extension was opened */ int null_count = 0; /* # of null messages */ + htri_t status; /* Indicate whether the message exists or not */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5F_super_ext_remove_msg, FAIL) @@ -874,6 +889,7 @@ H5F_super_ext_remove_msg(H5F_t *f, hid_t dxpl_id, unsigned id) /* Open superblock extension object header */ if(H5F_super_ext_open(f, f->shared->sblock->ext_addr, &ext_loc) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "error in starting file's superblock extension") + ext_opened = TRUE; /* Check if message with ID exists in the object header */ if((status = H5O_msg_exists(&ext_loc, id, dxpl_id)) < 0) @@ -902,10 +918,11 @@ H5F_super_ext_remove_msg(H5F_t *f, hid_t dxpl_id, unsigned id) } /* end if */ } /* end if */ - /* Close superblock extension object header */ - if(H5F_super_ext_close(f, &ext_loc) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to close file's superblock extension") done: + /* Close superblock extension object header, if opened */ + if(ext_opened && H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) + HDONE_ERROR(H5E_FILE, H5E_CANTRELEASE, FAIL, "unable to close file's superblock extension") + FUNC_LEAVE_NOAPI(ret_value) } /* H5F_super_ext_remove_msg() */ diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index 4212f97..946fcad 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -601,7 +601,7 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t UNUSED addr, void *_udata) } /* end if */ /* Close superblock extension */ - if(H5F_super_ext_close(f, &ext_loc) < 0) + if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTRELEASE, NULL, "unable to close file's superblock extension") } /* end if */ @@ -800,7 +800,7 @@ H5F_sblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, HGOTO_ERROR(H5E_FILE, H5E_WRITEERROR, FAIL, "unable to update driver info header message") /* Close the superblock extension object header */ - if(H5F_super_ext_close(f, &ext_loc) < 0) + if(H5F_super_ext_close(f, &ext_loc, dxpl_id, FALSE) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTCLOSEOBJ, FAIL, "unable to close file's superblock extension") } /* end if */ } /* end if */ diff --git a/src/H5G.c b/src/H5G.c index 18e6403..6da8ec6 100644 --- a/src/H5G.c +++ b/src/H5G.c @@ -353,6 +353,20 @@ H5Gcreate_anon(hid_t loc_id, hid_t gcpl_id, hid_t gapl_id) HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register group") done: + /* Release the group's object header, if it was created */ + if(grp) { + H5O_loc_t *oloc; /* Object location for group */ + + /* Get the new group's object location */ + if(NULL == (oloc = H5G_oloc(grp))) + HDONE_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "unable to get object location of group") + + /* Decrement refcount on group's object header in memory */ + if(H5O_dec_rc_by_loc(oloc, H5AC_dxpl_id) < 0) + HDONE_ERROR(H5E_SYM, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object") + } /* end if */ + + /* Cleanup on failure */ if(ret_value < 0) if(grp && H5G_close(grp) < 0) HDONE_ERROR(H5E_SYM, H5E_CLOSEERROR, FAIL, "unable to release group") @@ -901,6 +915,8 @@ done: if(ret_value == NULL) { /* Check if we need to release the file-oriented symbol table info */ if(oloc_init) { + if(H5O_dec_rc_by_loc(&(grp->oloc), dxpl_id) < 0) + HDONE_ERROR(H5E_SYM, H5E_CANTDEC, NULL, "unable to decrement refcount on newly created object") if(H5O_close(&(grp->oloc)) < 0) HDONE_ERROR(H5E_SYM, H5E_CLOSEERROR, NULL, "unable to release object header") if(H5O_delete(file, dxpl_id, grp->oloc.addr) < 0) diff --git a/src/H5Gobj.c b/src/H5Gobj.c index ce11990..b0add06 100644 --- a/src/H5Gobj.c +++ b/src/H5Gobj.c @@ -160,8 +160,7 @@ H5G_obj_create(H5F_t *f, hid_t dxpl_id, H5G_obj_create_t *gcrt_info, HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "can't get group info") /* Call the "real" group creation routine now */ - if(H5G_obj_create_real(f, dxpl_id, &ginfo, &linfo, &pline, gcrt_info, oloc) - < 0) + if(H5G_obj_create_real(f, dxpl_id, &ginfo, &linfo, &pline, gcrt_info, oloc) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTCREATE, FAIL, "unable to create group") done: @@ -264,7 +263,7 @@ H5G_obj_create_real(H5F_t *f, hid_t dxpl_id, const H5O_ginfo_t *ginfo, * since nothing refers to it yet. The link count will be * incremented if the object is added to the group directed graph. */ - if(H5O_create(f, dxpl_id, hdr_size, gcpl_id, oloc/*out*/) < 0) + if(H5O_create(f, dxpl_id, hdr_size, (size_t)1, gcpl_id, oloc/*out*/) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "can't create header") /* Check for format of group to create */ diff --git a/src/H5Groot.c b/src/H5Groot.c index a79a5dd..b8ba0fd 100644 --- a/src/H5Groot.c +++ b/src/H5Groot.c @@ -148,6 +148,10 @@ H5G_mkroot(H5F_t *f, hid_t dxpl_id, hbool_t create_root) if(1 != H5O_link(root_loc.oloc, 1, dxpl_id)) HGOTO_ERROR(H5E_SYM, H5E_LINKCOUNT, FAIL, "internal error (wrong link count)") + /* Decrement refcount on root group's object header in memory */ + if(H5O_dec_rc_by_loc(root_loc.oloc, dxpl_id) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTDEC, FAIL, "unable to decrement refcount on root group's object header") + /* Mark superblock dirty, so root group info is flushed */ sblock_dirty = TRUE; diff --git a/src/H5Gtraverse.c b/src/H5Gtraverse.c index 4016068..bb8e590 100644 --- a/src/H5Gtraverse.c +++ b/src/H5Gtraverse.c @@ -792,15 +792,17 @@ H5G_traverse_real(const H5G_loc_t *_loc, const char *name, unsigned target, gcrt_info.gcpl_id = H5P_GROUP_CREATE_DEFAULT; gcrt_info.cache_type = H5G_NOTHING_CACHED; HDmemset(&gcrt_info.cache, 0, sizeof(gcrt_info.cache)); - if(H5G_obj_create_real(grp_oloc.file, dxpl_id, ginfo, linfo, - pline, &gcrt_info, obj_loc.oloc/*out*/) < 0) + if(H5G_obj_create_real(grp_oloc.file, dxpl_id, ginfo, linfo, pline, &gcrt_info, obj_loc.oloc/*out*/) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to create group entry") /* Insert new group into current group's symbol table */ - if(H5G_loc_insert(&grp_loc, H5G_comp_g, &obj_loc, - H5O_TYPE_GROUP, &gcrt_info, dxpl_id) < 0) + if(H5G_loc_insert(&grp_loc, H5G_comp_g, &obj_loc, H5O_TYPE_GROUP, &gcrt_info, dxpl_id) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINSERT, FAIL, "unable to insert intermediate group") + /* Decrement refcount on intermediate group's object header in memory */ + if(H5O_dec_rc_by_loc(obj_loc.oloc, dxpl_id) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object") + /* Close new group */ if(H5O_close(obj_loc.oloc) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTINIT, FAIL, "unable to close") diff --git a/src/H5L.c b/src/H5L.c index 64e194a..9319e2c 100644 --- a/src/H5L.c +++ b/src/H5L.c @@ -1664,8 +1664,9 @@ H5L_link_cb(H5G_loc_t *grp_loc/*in*/, const char *name, const H5O_link_t UNUSED H5G_t *grp = NULL; /* H5G_t for this group, opened to pass to user callback */ hid_t grp_id = FAIL; /* Id for this group (passed to user callback */ H5G_loc_t temp_loc; /* For UD callback */ - hbool_t temp_loc_init = FALSE; - herr_t ret_value = SUCCEED; /* Return value */ + hbool_t temp_loc_init = FALSE; /* Temporary location for UD callback (temp_loc) has been initialized */ + hbool_t obj_created = FALSE; /* Whether an object was created (through a hard link) */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5L_link_cb) @@ -1690,6 +1691,9 @@ H5L_link_cb(H5G_loc_t *grp_loc/*in*/, const char *name, const H5O_link_t UNUSED /* Set object path to use for setting object name (below) */ udata->path = new_loc.path; + + /* Indicate that an object was created */ + obj_created = TRUE; } /* end if */ else { /* Check that both objects are in same file */ @@ -1763,6 +1767,20 @@ H5L_link_cb(H5G_loc_t *grp_loc/*in*/, const char *name, const H5O_link_t UNUSED } /* end if */ done: + /* Check if an object was created */ + if(obj_created) { + H5O_loc_t oloc; /* Object location for created object */ + + /* Set up object location */ + HDmemset(&oloc, 0, sizeof(oloc)); + oloc.file = grp_loc->oloc->file; + oloc.addr = udata->lnk->u.hard.addr; + + /* Decrement refcount on superblock extension's object header in memory */ + if(H5O_dec_rc_by_loc(&oloc, udata->dxpl_id) < 0) + HDONE_ERROR(H5E_LINK, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object") + } /* end if */ + /* Close the location given to the user callback if it was created */ if(grp_id >= 0) { if(H5I_dec_app_ref(grp_id) < 0) diff --git a/src/H5O.c b/src/H5O.c index 91fad55..2b26963 100644 --- a/src/H5O.c +++ b/src/H5O.c @@ -1107,14 +1107,15 @@ done: *------------------------------------------------------------------------- */ herr_t -H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, hid_t ocpl_id, - H5O_loc_t *loc/*out*/) +H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, size_t initial_rc, + hid_t ocpl_id, H5O_loc_t *loc/*out*/) { H5P_genplist_t *oc_plist; /* Object creation property list */ H5O_t *oh = NULL; /* Object header created */ haddr_t oh_addr; /* Address of initial object header */ size_t oh_size; /* Size of initial object header */ uint8_t oh_flags; /* Object header's initial status flags */ + unsigned insert_flags = H5AC__NO_FLAGS_SET; /* Flags for inserting object header into cache */ hbool_t store_msg_crt_idx; /* Whether to always store message creation indices for this file */ herr_t ret_value = SUCCEED; /* return value */ @@ -1244,11 +1245,18 @@ H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, hid_t ocpl_id, oh->mesg[0].raw_size = size_hint - (size_t)H5O_SIZEOF_MSGHDR_OH(oh); oh->mesg[0].chunkno = 0; + /* Check for non-zero initial refcount on the object header */ + if(initial_rc > 0) { + /* Set the initial refcount & pin the header when its inserted */ + oh->rc = initial_rc; + insert_flags |= H5AC__PIN_ENTRY_FLAG; + } /* end if */ + /* Set metadata tag in dxpl_id */ H5_BEGIN_TAG(dxpl_id, oh_addr, FAIL); /* Cache object header */ - if(H5AC_insert_entry(f, dxpl_id, H5AC_OHDR, oh_addr, oh, H5AC__NO_FLAGS_SET) < 0) + if(H5AC_insert_entry(f, dxpl_id, H5AC_OHDR, oh_addr, oh, insert_flags) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "unable to cache object header") oh = NULL; @@ -3423,6 +3431,49 @@ done: /*------------------------------------------------------------------------- + * Function: H5O_dec_rc_by_loc + * + * Purpose: Decrement the refcount of an object header, using its + * object location information. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Oct 08 2010 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_dec_rc_by_loc(const H5O_loc_t *loc, hid_t dxpl_id) +{ + H5O_t *oh = NULL; /* Object header */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5O_dec_rc_by_loc, FAIL) + + /* check args */ + HDassert(loc); + + /* Get header */ + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC_READ))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") + + /* Decrement the reference count on the object header */ + /* (which will unpin it, if appropriate) */ + if(H5O_dec_rc(oh) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDEC, FAIL, "unable to decrement reference count on object header") + +done: + /* Release the object header from the cache */ + if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_dec_rc_by_loc() */ + + +/*------------------------------------------------------------------------- * Function: H5O_free * * Purpose: Destroys an object header. diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 9e909a6..e64262b 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -560,6 +560,21 @@ done: * koziol@ncsa.uiuc.edu * Mar 20 2003 * + * Changes: In the parallel case, there is the possibility that the + * the object header may be flushed by different processes + * over the life of the computation. Thus we must ensure + * that the chunk images are up to date before we mark the + * messages clean -- as otherwise we may overwrite valid + * data with a blank section of a chunk image. + * + * To deal with this, I have added code to call + * H5O_chunk_serialize() for all chunks before we + * mark all messages as clean if we are not destroying the + * object. Do this in the parallel case only, as the problem + * can only occur in this context. + * + * JRM -- 10/12/10 + * *------------------------------------------------------------------------- */ static herr_t @@ -573,6 +588,30 @@ H5O_clear(H5F_t *f, H5O_t *oh, hbool_t destroy) /* check args */ HDassert(oh); +#ifdef H5_HAVE_PARALLEL + if ( ( oh->cache_info.is_dirty ) && ( ! destroy ) ) { + + size_t i; + + /* scan through all chunks associated with the object header, + * and cause them to update their images for all entries currently + * marked dirty. Must do this in the parallel case, as it is possible + * that this processor may clear this object header several times + * before flushing it -- thus causing undefined sections of the image + * to be written to disk overwriting valid data. + */ + + for ( i = 0; i < oh->nchunks; i++ ) { + + if ( H5O_chunk_serialize(f, oh, i) < 0 ) { + + HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, + "unable to serialize object header chunk") + } + } + } +#endif /* H5_HAVE_PARALLEL */ + /* Mark messages as clean */ for(u = 0; u < oh->nmesgs; u++) oh->mesg[u].dirty = FALSE; @@ -830,6 +869,30 @@ done: * koziol@hdfgroup.org * July 12, 2008 * + * Changes: In the parallel case, there is the possibility that the + * the object header chunk may be flushed by different + * processes over the life of the computation. Thus we must + * ensure that the chunk image is up to date before we mark its + * messages clean -- as otherwise we may overwrite valid + * data with a blank section of a chunk image. + * + * To deal with this, I have added code to call + * H5O_chunk_serialize() for this chunk before we + * mark all messages as clean if we are not destroying the + * chunk. + * + * Do this in the parallel case only, as the problem + * can only occur in this context. + * + * Note that at present at least, it seems that this fix + * is not necessary, as we don't seem to be able to + * generate a dirty chunk without creating a dirty object + * header. However, the object header code will be changing + * a lot in the near future, so I'll leave this fix in + * for now, unless Quincey requests otherwise. + * + * JRM -- 10/12/10 + * *------------------------------------------------------------------------- */ static herr_t @@ -843,6 +906,17 @@ H5O_cache_chk_clear(H5F_t *f, H5O_chunk_proxy_t *chk_proxy, hbool_t destroy) /* check args */ HDassert(chk_proxy); +#ifdef H5_HAVE_PARALLEL + if ( ( chk_proxy->oh->cache_info.is_dirty ) && ( ! destroy ) ) { + + if ( H5O_chunk_serialize(f, chk_proxy->oh, chk_proxy->chunkno) < 0 ) { + + HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, + "unable to serialize object header chunk") + } + } +#endif /* H5_HAVE_PARALLEL */ + /* Mark messages in chunk as clean */ for(u = 0; u < chk_proxy->oh->nmesgs; u++) if(chk_proxy->oh->mesg[u].chunkno == chk_proxy->chunkno) diff --git a/src/H5Opkg.h b/src/H5Opkg.h index 68b70a5..49cecfd 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -613,6 +613,7 @@ H5_DLL herr_t H5O_num_attrs_test(hid_t oid, hsize_t *nattrs); H5_DLL herr_t H5O_attr_dense_info_test(hid_t oid, hsize_t *name_count, hsize_t *corder_count); H5_DLL herr_t H5O_check_msg_marked_test(hid_t oid, hbool_t flag_val); H5_DLL herr_t H5O_expunge_chunks_test(const H5O_loc_t *oloc, hid_t dxpl_id); +H5_DLL herr_t H5O_get_rc(const H5O_loc_t *oloc, hid_t dxpl_id, unsigned *rc); #endif /* H5O_TESTING */ /* Object header debugging routines */ diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 0d10fb9..b4d0e8d 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -392,6 +392,7 @@ typedef struct H5O_storage_contig_t { } H5O_storage_contig_t; typedef struct H5O_storage_chunk_btree_t { + haddr_t dset_ohdr_addr; /* File address dataset's object header */ H5RC_t *shared; /* Ref-counted shared info for B-tree nodes */ } H5O_storage_chunk_btree_t; @@ -714,13 +715,14 @@ struct H5P_genplist_t; /* Object header routines */ H5_DLL herr_t H5O_init(void); H5_DLL herr_t H5O_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, - hid_t ocpl_id, H5O_loc_t *loc/*out*/); + size_t initial_rc, hid_t ocpl_id, H5O_loc_t *loc/*out*/); H5_DLL herr_t H5O_open(H5O_loc_t *loc); H5_DLL herr_t H5O_close(H5O_loc_t *loc); H5_DLL int H5O_link(const H5O_loc_t *loc, int adjust, hid_t dxpl_id); H5_DLL H5O_t *H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, H5AC_protect_t prot); H5_DLL H5O_t *H5O_pin(const H5O_loc_t *loc, hid_t dxpl_id); H5_DLL herr_t H5O_unpin(H5O_t *oh); +H5_DLL herr_t H5O_dec_rc_by_loc(const H5O_loc_t *loc, hid_t dxpl_id); H5_DLL herr_t H5O_unprotect(const H5O_loc_t *loc, hid_t dxpl_id, H5O_t *oh, unsigned oh_flags); H5_DLL herr_t H5O_touch(const H5O_loc_t *loc, hbool_t force, hid_t dxpl_id); diff --git a/src/H5Otest.c b/src/H5Otest.c index 557ac9e..883bfcd 100644 --- a/src/H5Otest.c +++ b/src/H5Otest.c @@ -536,3 +536,52 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5O_expunge_chunks_test() */ + +/*-------------------------------------------------------------------------- + NAME + H5O_get_rc + PURPOSE + Retrieve the refcount for the object header + USAGE + herr_t H5O_expunge_chunks_test(loc, dxpl_id, rc) + const H5O_loc_t *loc; IN: Object location for object header to query + hid_t dxpl_id; IN: DXPL to use for operation + unsigned *rc; OUT: Pointer to refcount for object header + RETURNS + Non-negative on success, negative on failure + DESCRIPTION + Protects object header, retrieves the object header's refcount, and + unprotects object header. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + DO NOT USE THIS FUNCTION FOR ANYTHING EXCEPT TESTING + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5O_get_rc(const H5O_loc_t *loc, hid_t dxpl_id, unsigned *rc) +{ + H5O_t *oh = NULL; /* Object header */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5O_get_rc, FAIL) + + /* Sanity check */ + HDassert(loc); + HDassert(rc); + + /* Get the object header */ + if(NULL == (oh = H5O_protect(loc, dxpl_id, H5AC_READ))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, FAIL, "unable to protect object header") + + /* Save the refcount for the object header */ + *rc = oh->nlink; + +done: + /* Release the object header */ + if(oh && H5O_unprotect(loc, dxpl_id, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_expunge_chunks_test() */ + diff --git a/src/H5Tcommit.c b/src/H5Tcommit.c index b924b61..e109f95 100644 --- a/src/H5Tcommit.c +++ b/src/H5Tcommit.c @@ -302,6 +302,19 @@ H5Tcommit_anon(hid_t loc_id, hid_t type_id, hid_t tcpl_id, hid_t tapl_id) if(H5T_commit(loc.oloc->file, type, tcpl_id, H5AC_dxpl_id) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to commit datatype") + /* Release the datatype's object header */ + { + H5O_loc_t *oloc; /* Object location for datatype */ + + /* Get the new committed datatype's object location */ + if(NULL == (oloc = H5T_oloc(type))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTGET, FAIL, "unable to get object location of committed datatype") + + /* Decrement refcount on committed datatype's object header in memory */ + if(H5O_dec_rc_by_loc(oloc, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object") + } /* end if */ + done: FUNC_LEAVE_API(ret_value) } /* end H5Tcommit_anon() */ @@ -379,7 +392,7 @@ H5T_commit(H5F_t *file, H5T_t *type, hid_t tcpl_id, hid_t dxpl_id) * Create the object header and open it for write access. Insert the data * type message and then give the object header a name. */ - if(H5O_create(file, dxpl_id, dtype_size, tcpl_id, &temp_oloc) < 0) + if(H5O_create(file, dxpl_id, dtype_size, (size_t)1, tcpl_id, &temp_oloc) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to create datatype object header") if(H5O_msg_create(&temp_oloc, H5O_DTYPE_ID, H5O_MSG_FLAG_CONSTANT | H5O_MSG_FLAG_DONTSHARE, H5O_UPDATE_TIME, type, dxpl_id) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to update type header message") @@ -415,6 +428,8 @@ done: H5G_name_free(&temp_path); } /* end if */ if((type->shared->state == H5T_STATE_TRANSIENT || type->shared->state == H5T_STATE_RDONLY) && (type->sh_loc.type == H5O_SHARE_TYPE_COMMITTED)) { + if(H5O_dec_rc_by_loc(&(type->oloc), dxpl_id) < 0) + HDONE_ERROR(H5E_DATATYPE, H5E_CANTDEC, FAIL, "unable to decrement refcount on newly created object") if(H5O_close(&(type->oloc)) < 0) HDONE_ERROR(H5E_DATATYPE, H5E_CLOSEERROR, FAIL, "unable to release object header") if(H5O_delete(file, dxpl_id, type->sh_loc.u.loc.oh_addr) < 0) diff --git a/src/H5public.h b/src/H5public.h index 4fc8c7e..b536e0e 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -75,10 +75,10 @@ extern "C" { /* Version numbers */ #define H5_VERS_MAJOR 1 /* For major interface/format changes */ #define H5_VERS_MINOR 9 /* For minor interface/format changes */ -#define H5_VERS_RELEASE 76 /* For tweaks, bug-fixes, or development */ +#define H5_VERS_RELEASE 77 /* For tweaks, bug-fixes, or development */ #define H5_VERS_SUBRELEASE "FA_a5" /* For pre-releases like snap0 */ /* Empty string for real releases. */ -#define H5_VERS_INFO "HDF5 library version: 1.9.76-FA_a5" /* Full version string */ +#define H5_VERS_INFO "HDF5 library version: 1.9.77-FA_a5" /* Full version string */ #define H5check() H5check_version(H5_VERS_MAJOR,H5_VERS_MINOR, \ H5_VERS_RELEASE) diff --git a/src/Makefile.in b/src/Makefile.in index d668dc0..4d3e088 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -457,7 +457,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog # Add libtool shared library version numbers to the HDF5 library # See libtool versioning documentation online. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 66 +LT_VERS_REVISION = 67 LT_VERS_AGE = 0 H5detect_CFLAGS = -g $(AM_CFLAGS) diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 7e12026..2bed576 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -241,6 +241,7 @@ ADD_TEST ( ) SET (H5_TESTS + accum lheap ohdr stab diff --git a/test/Makefile.am b/test/Makefile.am index 42545d1..1a29873 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -36,8 +36,8 @@ SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) # These tests (fheap, btree2) are under development and are not used by # the library yet. Move them to the end so that their failure do not block # other current library code tests. -TEST_PROG=testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ - pool hyperslab istore bittests dt_arith \ +TEST_PROG= testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ + pool accum hyperslab istore bittests dt_arith \ dtypes dsets cmpd_dset filter_fail extend external objcopy links unlink \ big mtime fillval mount flush1 flush2 app_ref enum \ set_extent ttsafe \ @@ -106,7 +106,7 @@ flush2.chkexe_: flush1.chkexe_ # specifying a file prefix or low-level driver. Changing the file # prefix or low-level driver with environment variables will influence # the temporary file name in ways that the makefile is not aware of. -CHECK_CLEANFILES+=cmpd_dset.h5 compact_dataset.h5 dataset.h5 \ +CHECK_CLEANFILES+=accum.h5 cmpd_dset.h5 compact_dataset.h5 dataset.h5 \ dset_offset.h5 \ max_compact_dataset.h5 simple.h5 set_local.h5 random_chunks.h5 \ huge_chunks.h5 chunk_cache.h5 big_chunk.h5 chunk_fast.h5 \ diff --git a/test/Makefile.in b/test/Makefile.in index 89a2e30..1d42982 100644 --- a/test/Makefile.in +++ b/test/Makefile.in @@ -81,18 +81,18 @@ libh5test_la_OBJECTS = $(am_libh5test_la_OBJECTS) am__installdirs = "$(DESTDIR)$(bindir)" am__EXEEXT_1 = testhdf5$(EXEEXT) lheap$(EXEEXT) ohdr$(EXEEXT) \ stab$(EXEEXT) gheap$(EXEEXT) cache$(EXEEXT) cache_api$(EXEEXT) \ - cache_tagging$(EXEEXT) pool$(EXEEXT) hyperslab$(EXEEXT) \ - istore$(EXEEXT) bittests$(EXEEXT) dt_arith$(EXEEXT) \ - dtypes$(EXEEXT) dsets$(EXEEXT) cmpd_dset$(EXEEXT) \ - filter_fail$(EXEEXT) extend$(EXEEXT) external$(EXEEXT) \ - objcopy$(EXEEXT) links$(EXEEXT) unlink$(EXEEXT) big$(EXEEXT) \ - mtime$(EXEEXT) fillval$(EXEEXT) mount$(EXEEXT) flush1$(EXEEXT) \ - flush2$(EXEEXT) app_ref$(EXEEXT) enum$(EXEEXT) \ - set_extent$(EXEEXT) ttsafe$(EXEEXT) getname$(EXEEXT) \ - vfd$(EXEEXT) ntypes$(EXEEXT) dangle$(EXEEXT) \ - dtransform$(EXEEXT) reserved$(EXEEXT) cross_read$(EXEEXT) \ - freespace$(EXEEXT) mf$(EXEEXT) farray$(EXEEXT) earray$(EXEEXT) \ - btree2$(EXEEXT) fheap$(EXEEXT) + cache_tagging$(EXEEXT) pool$(EXEEXT) accum$(EXEEXT) \ + hyperslab$(EXEEXT) istore$(EXEEXT) bittests$(EXEEXT) \ + dt_arith$(EXEEXT) dtypes$(EXEEXT) dsets$(EXEEXT) \ + cmpd_dset$(EXEEXT) filter_fail$(EXEEXT) extend$(EXEEXT) \ + external$(EXEEXT) objcopy$(EXEEXT) links$(EXEEXT) \ + unlink$(EXEEXT) big$(EXEEXT) mtime$(EXEEXT) fillval$(EXEEXT) \ + mount$(EXEEXT) flush1$(EXEEXT) flush2$(EXEEXT) \ + app_ref$(EXEEXT) enum$(EXEEXT) set_extent$(EXEEXT) \ + ttsafe$(EXEEXT) getname$(EXEEXT) vfd$(EXEEXT) ntypes$(EXEEXT) \ + dangle$(EXEEXT) dtransform$(EXEEXT) reserved$(EXEEXT) \ + cross_read$(EXEEXT) freespace$(EXEEXT) mf$(EXEEXT) \ + farray$(EXEEXT) earray$(EXEEXT) btree2$(EXEEXT) fheap$(EXEEXT) am__EXEEXT_2 = gen_bad_ohdr$(EXEEXT) gen_bogus$(EXEEXT) \ gen_cross$(EXEEXT) gen_deflate$(EXEEXT) gen_filters$(EXEEXT) \ gen_idx$(EXEEXT) gen_new_array$(EXEEXT) gen_new_fill$(EXEEXT) \ @@ -102,6 +102,10 @@ am__EXEEXT_2 = gen_bad_ohdr$(EXEEXT) gen_bogus$(EXEEXT) \ space_overflow$(EXEEXT) gen_filespace$(EXEEXT) \ gen_specmetaread$(EXEEXT) gen_sizes_lheap$(EXEEXT) PROGRAMS = $(bin_PROGRAMS) $(noinst_PROGRAMS) +accum_SOURCES = accum.c +accum_OBJECTS = accum.$(OBJEXT) +accum_LDADD = $(LDADD) +accum_DEPENDENCIES = libh5test.la $(LIBHDF5) app_ref_SOURCES = app_ref.c app_ref_OBJECTS = app_ref.$(OBJEXT) app_ref_LDADD = $(LDADD) @@ -405,22 +409,7 @@ CCLD = $(CC) LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \ $(LDFLAGS) -o $@ -SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c btree2.c \ - cache.c cache_api.c cache_tagging.c cmpd_dset.c cross_read.c \ - dangle.c dsets.c dt_arith.c dtransform.c dtypes.c earray.c \ - enum.c err_compat.c error_test.c extend.c external.c farray.c \ - fheap.c fillval.c filter_fail.c flush1.c flush2.c freespace.c \ - gen_bad_ohdr.c gen_bogus.c gen_cross.c gen_deflate.c \ - gen_filespace.c gen_filters.c gen_idx.c gen_new_array.c \ - gen_new_fill.c gen_new_group.c gen_new_mtime.c gen_new_super.c \ - gen_noencoder.c gen_nullspace.c gen_sizes_lheap.c \ - gen_specmetaread.c gen_udlinks.c getname.c gheap.c hyperslab.c \ - istore.c lheap.c links.c mf.c mount.c mtime.c ntypes.c \ - objcopy.c ohdr.c pool.c reserved.c set_extent.c \ - space_overflow.c stab.c swmr_generator.c swmr_reader.c \ - swmr_writer.c tcheck_version.c $(testhdf5_SOURCES) testmeta.c \ - $(ttsafe_SOURCES) unlink.c vfd.c -DIST_SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c \ +SOURCES = $(libh5test_la_SOURCES) accum.c app_ref.c big.c bittests.c \ btree2.c cache.c cache_api.c cache_tagging.c cmpd_dset.c \ cross_read.c dangle.c dsets.c dt_arith.c dtransform.c dtypes.c \ earray.c enum.c err_compat.c error_test.c extend.c external.c \ @@ -436,6 +425,22 @@ DIST_SOURCES = $(libh5test_la_SOURCES) app_ref.c big.c bittests.c \ swmr_reader.c swmr_writer.c tcheck_version.c \ $(testhdf5_SOURCES) testmeta.c $(ttsafe_SOURCES) unlink.c \ vfd.c +DIST_SOURCES = $(libh5test_la_SOURCES) accum.c app_ref.c big.c \ + bittests.c btree2.c cache.c cache_api.c cache_tagging.c \ + cmpd_dset.c cross_read.c dangle.c dsets.c dt_arith.c \ + dtransform.c dtypes.c earray.c enum.c err_compat.c \ + error_test.c extend.c external.c farray.c fheap.c fillval.c \ + filter_fail.c flush1.c flush2.c freespace.c gen_bad_ohdr.c \ + gen_bogus.c gen_cross.c gen_deflate.c gen_filespace.c \ + gen_filters.c gen_idx.c gen_new_array.c gen_new_fill.c \ + gen_new_group.c gen_new_mtime.c gen_new_super.c \ + gen_noencoder.c gen_nullspace.c gen_sizes_lheap.c \ + gen_specmetaread.c gen_udlinks.c getname.c gheap.c hyperslab.c \ + istore.c lheap.c links.c mf.c mount.c mtime.c ntypes.c \ + objcopy.c ohdr.c pool.c reserved.c set_extent.c \ + space_overflow.c stab.c swmr_generator.c swmr_reader.c \ + swmr_writer.c tcheck_version.c $(testhdf5_SOURCES) testmeta.c \ + $(ttsafe_SOURCES) unlink.c vfd.c ETAGS = etags CTAGS = ctags am__tty_colors = \ @@ -715,7 +720,7 @@ TRACE = perl $(top_srcdir)/bin/trace # specifying a file prefix or low-level driver. Changing the file # prefix or low-level driver with environment variables will influence # the temporary file name in ways that the makefile is not aware of. -CHECK_CLEANFILES = *.chkexe *.chklog *.clog cmpd_dset.h5 \ +CHECK_CLEANFILES = *.chkexe *.chklog *.clog accum.h5 cmpd_dset.h5 \ compact_dataset.h5 dataset.h5 dset_offset.h5 \ max_compact_dataset.h5 simple.h5 set_local.h5 random_chunks.h5 \ huge_chunks.h5 chunk_cache.h5 big_chunk.h5 chunk_fast.h5 \ @@ -753,7 +758,7 @@ SCRIPT_DEPEND = error_test$(EXEEXT) err_compat$(EXEEXT) # the library yet. Move them to the end so that their failure do not block # other current library code tests. TEST_PROG = testhdf5 lheap ohdr stab gheap cache cache_api cache_tagging \ - pool hyperslab istore bittests dt_arith \ + pool accum hyperslab istore bittests dt_arith \ dtypes dsets cmpd_dset filter_fail extend external objcopy links unlink \ big mtime fillval mount flush1 flush2 app_ref enum \ set_extent ttsafe \ @@ -927,6 +932,9 @@ clean-noinstPROGRAMS: list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ echo " rm -f" $$list; \ rm -f $$list +accum$(EXEEXT): $(accum_OBJECTS) $(accum_DEPENDENCIES) + @rm -f accum$(EXEEXT) + $(LINK) $(accum_OBJECTS) $(accum_LDADD) $(LIBS) app_ref$(EXEEXT): $(app_ref_OBJECTS) $(app_ref_DEPENDENCIES) @rm -f app_ref$(EXEEXT) $(LINK) $(app_ref_OBJECTS) $(app_ref_LDADD) $(LIBS) @@ -1144,6 +1152,7 @@ mostlyclean-compile: distclean-compile: -rm -f *.tab.c +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/accum.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/app_ref.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/big.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/bittests.Po@am__quote@ diff --git a/test/accum.c b/test/accum.c new file mode 100644 index 0000000..c5f6610 --- /dev/null +++ b/test/accum.c @@ -0,0 +1,1809 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/* Programmer: Mike McGreevy + * October 7, 2010 + */ +#include "h5test.h" + +#define H5F_PACKAGE +#include "H5Fpkg.h" +#include "H5FDprivate.h" +#include "H5Iprivate.h" + +/* Filename */ +#define FILENAME "accum.h5" + +/* "big" I/O test values */ +#define BIG_BUF_SIZE (6 * 1024 * 1024) + +/* Random I/O test values */ +#define RANDOM_BUF_SIZE (1 * 1024 * 1024) +#define MAX_RANDOM_SEGMENTS (5 * 1024) +#define RAND_SEG_LEN (1024) +#define RANDOM_BASE_OFF (1024 * 1024) + +/* Make file global to all tests */ +H5F_t * f = NULL; + +/* Function Prototypes */ +unsigned test_write_read(void); +unsigned test_write_read_nonacc_front(void); +unsigned test_write_read_nonacc_end(void); +unsigned test_accum_overlap(void); +unsigned test_accum_overlap_clean(void); +unsigned test_accum_overlap_size(void); +unsigned test_accum_non_overlap_size(void); +unsigned test_accum_adjust(void); +unsigned test_read_after(void); +unsigned test_free(void); +unsigned test_big(void); +unsigned test_random_write(void); + +/* Helper Function Prototypes */ +void accum_printf(void); + +/* Private Test H5Faccum Function Wrappers */ +#define accum_write(a,s,b) H5F_block_write(f, H5FD_MEM_DEFAULT, (haddr_t)(a), (size_t)(s), H5P_DATASET_XFER_DEFAULT, (b)) +#define accum_read(a,s,b) H5F_block_read(f, H5FD_MEM_DEFAULT, (haddr_t)(a), (size_t)(s), H5P_DATASET_XFER_DEFAULT, (b)) +#define accum_free(a,s) H5F_accum_free(f, H5P_DATASET_XFER_DEFAULT, H5FD_MEM_DEFAULT, (haddr_t)(a), (hsize_t)(s)) +#define accum_flush() H5F_accum_flush(f, H5P_DATASET_XFER_DEFAULT) +#define accum_reset() H5F_accum_reset(f, H5P_DATASET_XFER_DEFAULT, TRUE) + +/* ================= */ +/* Main Test Routine */ +/* ================= */ + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Test the metadata accumulator code + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mike McGreevy + * October 7, 2010 + * + *------------------------------------------------------------------------- + */ +int +main(void) +{ + unsigned nerrors = 0; /* track errors */ + hid_t fid = -1; + + /* Test Setup */ + puts("Testing the metadata accumulator"); + + /* Create a test file */ + if((fid = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) FAIL_STACK_ERROR + + /* Get H5F_t * to internal file structure */ + if(NULL == (f = (H5F_t *)H5I_object(fid))) FAIL_STACK_ERROR + + /* We'll be writing lots of garbage data, so extend the + file a ways. 10MB should do. */ + if(H5FD_set_eoa(f->shared->lf, H5FD_MEM_DEFAULT, (haddr_t)(1024*1024*10)) < 0) FAIL_STACK_ERROR + + /* Reset metadata accumulator for the file */ + if(accum_reset() < 0) FAIL_STACK_ERROR + + /* Test Functions */ + nerrors += test_write_read(); + nerrors += test_write_read_nonacc_front(); + nerrors += test_write_read_nonacc_end(); + nerrors += test_accum_overlap(); + nerrors += test_accum_overlap_clean(); + nerrors += test_accum_overlap_size(); + nerrors += test_accum_non_overlap_size(); + nerrors += test_accum_adjust(); + nerrors += test_read_after(); + nerrors += test_free(); + nerrors += test_big(); + nerrors += test_random_write(); + + /* End of test code, close and delete file */ + if(H5Fclose(fid) < 0) TEST_ERROR + HDremove(FILENAME); + + if(nerrors) + goto error; + puts("All metadata accumulator tests passed."); + + return 0; + +error: + puts("*** TESTS FAILED ***"); + return 1; +} /* end main() */ + +/* ============================= */ +/* Individual Unit Test Routines */ +/* ============================= */ + + +/*------------------------------------------------------------------------- + * Function: test_write_read + * + * Purpose: Simple test to write to then read from metadata accumulator. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mike McGreevy + * October 7, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_write_read(void) +{ + int i = 0; + int *write_buf, *read_buf; + + TESTING("simple write/read to/from metadata accumulator"); + + /* Allocate buffers */ + write_buf = (int *)HDmalloc(1024 * sizeof(int)); + HDassert(write_buf); + read_buf = (int *)HDcalloc(1024, sizeof(int)); + HDassert(read_buf); + + /* Fill buffer with data, zero out read buffer */ + for(i = 0; i < 1024; i++) + write_buf[i] = i + 1; + + /* Do a simple write/read/verify of data */ + /* Write 1KB at Address 0 */ + if(accum_write(0, 1024, write_buf) < 0) FAIL_STACK_ERROR; + if(accum_read(0, 1024, read_buf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(write_buf, read_buf, 1024) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(write_buf); + HDfree(read_buf); + + return 0; + +error: + /* Release memory */ + HDfree(write_buf); + HDfree(read_buf); + + return 1; +} /* test_write_read */ + + +/*------------------------------------------------------------------------- + * Function: test_write_read_nonacc_front + * + * Purpose: Simple test to write to then read from before metadata accumulator. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Allen Byrne + * October 8, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_write_read_nonacc_front(void) +{ + int i = 0; + int *write_buf, *read_buf; + + TESTING("simple write/read to/from before metadata accumulator"); + + /* Allocate buffers */ + write_buf = (int *)HDmalloc(2048 * sizeof(int)); + HDassert(write_buf); + read_buf = (int *)HDcalloc(2048, sizeof(int)); + HDassert(read_buf); + + /* Fill buffer with data, zero out read buffer */ + for(i = 0; i < 2048; i++) + write_buf[i] = i + 1; + + /* Do a simple write/read/verify of data */ + /* Write 1KB at Address 0 */ + if(accum_write(0, 1024, write_buf) < 0) FAIL_STACK_ERROR; + if(accum_flush() < 0) FAIL_STACK_ERROR; + if(accum_reset() < 0) FAIL_STACK_ERROR; + if(accum_write(1024, 1024, write_buf) < 0) FAIL_STACK_ERROR; + if(accum_read(0, 1024, read_buf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(write_buf, read_buf, 1024) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(write_buf); + HDfree(read_buf); + + return 0; + +error: + /* Release memory */ + HDfree(write_buf); + HDfree(read_buf); + + return 1; +} /* test_write_read */ + + +/*------------------------------------------------------------------------- + * Function: test_write_read_nonacc_end + * + * Purpose: Simple test to write to then read from after metadata accumulator. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Allen Byrne + * October 8, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_write_read_nonacc_end(void) +{ + int i = 0; + int *write_buf, *read_buf; + + TESTING("simple write/read to/from after metadata accumulator"); + + /* Allocate buffers */ + write_buf = (int *)HDmalloc(2048 * sizeof(int)); + HDassert(write_buf); + read_buf = (int *)HDcalloc(2048, sizeof(int)); + HDassert(read_buf); + + /* Fill buffer with data, zero out read buffer */ + for(i = 0; i < 2048; i++) + write_buf[i] = i + 1; + + /* Do a simple write/read/verify of data */ + /* Write 1KB at Address 0 */ + if(accum_write(1024, 1024, write_buf) < 0) FAIL_STACK_ERROR; + if(accum_flush() < 0) FAIL_STACK_ERROR; + if(accum_reset() < 0) FAIL_STACK_ERROR; + if(accum_write(0, 1024, write_buf) < 0) FAIL_STACK_ERROR; + if(accum_read(1024, 1024, read_buf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(write_buf, read_buf, 1024) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(write_buf); + HDfree(read_buf); + + return 0; + +error: + /* Release memory */ + HDfree(write_buf); + HDfree(read_buf); + + return 1; +} /* test_write_read */ + + +/*------------------------------------------------------------------------- + * Function: test_free + * + * Purpose: Simple test to free metadata accumulator. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Raymond Lu + * October 8, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_free(void) +{ + int i = 0; + int32_t *wbuf = NULL; + int32_t *rbuf = NULL; + int32_t *expect = NULL; + + TESTING("simple freeing metadata accumulator"); + + /* Write and free the whole accumulator. */ + wbuf = (int32_t *)HDmalloc(256 * sizeof(int32_t)); + HDassert(wbuf); + rbuf = (int32_t *)HDmalloc(256 * sizeof(int32_t)); + HDassert(rbuf); + expect = (int32_t *)HDmalloc(256 * sizeof(int32_t)); + HDassert(expect); + + /* Fill buffer with data */ + for(i = 0; i < 256; i++) + wbuf[i] = (int32_t)(i + 1); + + if(accum_write(0, 256 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + + if(accum_free(0, 256 * sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Free an empty accumulator */ + if(accum_free(0, 256 * 1024 * sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Write second quarter of the accumulator */ + if(accum_write(64 * sizeof(int32_t), 64 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + + /* Free the second quarter of the accumulator, the requested area + * is bigger than the data region on the right side. */ + if(accum_free(64 * sizeof(int32_t), 65 * sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + + /* Write half of the accumulator. */ + if(accum_write(0, 128 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + + /* Free the first block of 4B */ + if(accum_free(0, sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Check that the accumulator still contains the correct data */ + if(accum_read(1 * sizeof(int32_t), 127 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf + 1, rbuf, 127 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Free the block of 4B at 127*4B */ + if(accum_free(127 * sizeof(int32_t), sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Check that the accumulator still contains the correct data */ + if(accum_read(1 * sizeof(int32_t), 126 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf + 1, rbuf, 126 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Free the block of 4B at 2*4B */ + if(accum_free(2 * sizeof(int32_t), sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Check that the accumulator still contains the correct data */ + if(accum_read(1 * sizeof(int32_t), 1 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf + 1, rbuf, 1 * sizeof(int32_t)) != 0) TEST_ERROR; + if(accum_read(3 * sizeof(int32_t), 124 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf + 3, rbuf, 124 * sizeof(int32_t)) != 0) TEST_ERROR; + + + /* Test freeing section that overlaps the start of the accumulator and is + * entirely before dirty section */ + if(accum_write(64 * sizeof(int32_t), 128 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 64, wbuf, 128 * sizeof(int32_t)); + if(accum_flush() < 0) FAIL_STACK_ERROR; + if(accum_write(68 * sizeof(int32_t), 4 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 68, wbuf, 4 * sizeof(int32_t)); + if(accum_free(62 * sizeof(int32_t), 4 * sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Check that the accumulator still contains the correct data */ + if(accum_read(66 * sizeof(int32_t), 126 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(expect + 66, rbuf, 126 * sizeof(int32_t)) != 0) TEST_ERROR; + + + /* Test freeing section that overlaps the start of the accumulator and + * completely contains dirty section */ + if(accum_write(64 * sizeof(int32_t), 128 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 64, wbuf, 128 * sizeof(int32_t)); + if(accum_flush() < 0) FAIL_STACK_ERROR; + if(accum_write(68 * sizeof(int32_t), 4 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 68, wbuf, 4 * sizeof(int32_t)); + if(accum_free(62 * sizeof(int32_t), 16 * sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Check that the accumulator still contains the correct data */ + if(accum_read(78 * sizeof(int32_t), 114 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(expect + 78, rbuf, 114 * sizeof(int32_t)) != 0) TEST_ERROR; + + + /* Test freeing section completely contained in accumulator and is entirely + * before dirty section */ + if(accum_write(64 * sizeof(int32_t), 128 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 64, wbuf, 128 * sizeof(int32_t)); + if(accum_flush() < 0) FAIL_STACK_ERROR; + if(accum_write(72 * sizeof(int32_t), 4 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 72, wbuf, 4 * sizeof(int32_t)); + if(accum_free(66 * sizeof(int32_t), 4 * sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Check that the accumulator still contains the correct data */ + if(accum_read(70 * sizeof(int32_t), 122 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(expect + 70, rbuf, 122 * sizeof(int32_t)) != 0) TEST_ERROR; + + + /* Test freeing section completely contained in accumulator, starts before + * dirty section, and ends in dirty section */ + if(accum_write(64 * sizeof(int32_t), 128 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 64, wbuf, 128 * sizeof(int32_t)); + if(accum_flush() < 0) FAIL_STACK_ERROR; + if(accum_write(72 * sizeof(int32_t), 4 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 72, wbuf, 4 * sizeof(int32_t)); + if(accum_free(70 * sizeof(int32_t), 4 * sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Check that the accumulator still contains the correct data */ + if(accum_read(74 * sizeof(int32_t), 118 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(expect + 74, rbuf, 118 * sizeof(int32_t)) != 0) TEST_ERROR; + + + /* Test freeing section completely contained in accumulator and completely + * contains dirty section */ + if(accum_write(64 * sizeof(int32_t), 128 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 64, wbuf, 128 * sizeof(int32_t)); + if(accum_flush() < 0) FAIL_STACK_ERROR; + if(accum_write(72 * sizeof(int32_t), 4 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 72, wbuf, 4 * sizeof(int32_t)); + if(accum_free(70 * sizeof(int32_t), 8 * sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Check that the accumulator still contains the correct data */ + if(accum_read(78 * sizeof(int32_t), 114 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(expect + 78, rbuf, 114 * sizeof(int32_t)) != 0) TEST_ERROR; + + + /* Test freeing section completely contained in accumulator, starts at start + * of dirty section, and ends in dirty section */ + if(accum_write(64 * sizeof(int32_t), 128 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 64, wbuf, 128 * sizeof(int32_t)); + if(accum_flush() < 0) FAIL_STACK_ERROR; + if(accum_write(72 * sizeof(int32_t), 8 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + HDmemcpy(expect + 72, wbuf, 8 * sizeof(int32_t)); + if(accum_free(72 * sizeof(int32_t), 4 * sizeof(int32_t)) < 0) FAIL_STACK_ERROR; + + /* Check that the accumulator still contains the correct data */ + if(accum_read(76 * sizeof(int32_t), 116 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(expect + 76, rbuf, 116 * sizeof(int32_t)) != 0) TEST_ERROR; + + HDfree(wbuf); + HDfree(rbuf); + HDfree(expect); + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + return 0; + +error: + HDfree(wbuf); + HDfree(rbuf); + HDfree(expect); + + return 1; +} /* test_free */ + + +/*------------------------------------------------------------------------- + * Function: test_accum_overlap + * + * Purpose: This test will write a series of pieces of data + * to the accumulator with the goal of overlapping + * the writes in various different ways. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mike McGreevy + * October 7, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_accum_overlap(void) +{ + int i = 0; + int32_t *wbuf, *rbuf; + + TESTING("overlapping write to metadata accumulator"); + + /* Allocate buffers */ + wbuf = (int32_t *)HDmalloc(4096 * sizeof(int32_t)); + HDassert(wbuf); + rbuf = (int32_t *)HDcalloc(4096, sizeof(int32_t)); + HDassert(rbuf); + + /* Case 1: No metadata in accumulator */ + /* Write 10 1's at address 40 */ + /* @0:| 1111111111| */ + /* Put some data in the accumulator initially */ + for(i = 0; i < 10; i++) + wbuf[i] = 1; + if(accum_write(40, 10 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(40, 10 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 10 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 2: End of new piece aligns with start of accumulated data */ + /* Write 5 2's at address 20 */ + /* @0:| 222221111111111| */ + for(i = 0; i < 5; i++) + wbuf[i] = 2; + if(accum_write(20, 5 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(20, 5 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 5 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 3: Start of new piece aligns with start of accumulated data */ + /* Write 3 3's at address 20 */ + /* @0:| 333221111111111| */ + for(i = 0; i < 3; i++) + wbuf[i] = 3; + if(accum_write(20, 3 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(20, 3 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 3 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 4: New piece overlaps start of accumulated data */ + /* Write 5 4's at address 8 */ + /* @0:| 444443221111111111| */ + for(i = 0; i < 5; i++) + wbuf[i] = 4; + if(accum_write(8, 5 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(8, 5 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 5 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 5: New piece completely within accumulated data */ + /* Write 4 5's at address 48 */ + /* @0:| 444443221155551111| */ + for(i = 0; i < 4; i++) + wbuf[i] = 5; + if(accum_write(48, 4 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(48, 4 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 4 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 6: End of new piece aligns with end of accumulated data */ + /* Write 3 6's at address 68 */ + /* @0:| 444443221155551666| */ + for(i = 0; i < 3; i++) + wbuf[i] = 6; + if(accum_write(68, 3 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(68, 3 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 3 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 7: New piece overlaps end of accumulated data */ + /* Write 5 7's at address 76 */ + /* @0:| 4444432211555516677777| */ + for(i = 0; i < 5; i++) + wbuf[i] = 7; + if(accum_write(76, 5 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(76, 5 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 5 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 8: Start of new piece aligns with end of accumulated data */ + /* Write 3 8's at address 96 */ + /* @0:| 4444432211555516677777888| */ + for(i = 0; i < 3; i++) + wbuf[i] = 8; + if(accum_write(96, 3 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(96, 3 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 3 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Set up expected data buffer and verify contents of + accumulator as constructed by cases 1-8, above */ + for(i = 0; i < 5; i++) + wbuf[i] = 4; + for(i = 5; i < 6; i++) + wbuf[i] = 3; + for(i = 6; i < 8; i++) + wbuf[i] = 2; + for(i = 8; i < 10; i++) + wbuf[i] = 1; + for(i = 10; i < 14; i++) + wbuf[i] = 5; + for(i = 14; i < 15; i++) + wbuf[i] = 1; + for(i = 15; i < 17; i++) + wbuf[i] = 6; + for(i = 17; i < 22; i++) + wbuf[i] = 7; + for(i = 22; i < 25; i++) + wbuf[i] = 8; + if(accum_read(8, 25 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 25 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 9: New piece completely before accumulated data */ + /* Write 1 9 at address 0 */ + /* @0:|9 4444432211555516677777888| */ + for(i = 0; i < 1; i++) + wbuf[i] = 9; + if(accum_write(0, 1 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(0, 1 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 1 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 10: New piece completely after accumulated data */ + /* Write 4 3's at address 116 */ + /* @0:|9 4444432211555516677777888 3333| */ + for(i = 0; i < 4; i++) + wbuf[i] = 3; + if(accum_write(116, 4 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(116, 4 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 4 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 11: New piece completely overlaps accumulated data */ + /* Write 6 4's at address 112 */ + /* @0:|9 4444432211555516677777888 444444| */ + for(i = 0; i < 6; i++) + wbuf[i] = 4; + if(accum_write(112, 6 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(112, 6 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 6 * sizeof(int32_t)) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 0; + +error: + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 1; +} /* test_accum_overlap */ + + +/*------------------------------------------------------------------------- + * Function: test_accum_overlap_clean + * + * Purpose: This test will write a series of pieces of data + * to the accumulator with the goal of overlapping + * the writes in various different ways, with clean + * areas in the accumulator. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Neil Fortner + * October 8, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_accum_overlap_clean(void) +{ + int i = 0; + int32_t *wbuf, *rbuf; + + TESTING("overlapping write to partially clean metadata accumulator"); + + /* Allocate buffers */ + wbuf = (int32_t *)HDmalloc(4096 * sizeof(int32_t)); + HDassert(wbuf); + rbuf = (int32_t *)HDcalloc(4096, sizeof(int32_t)); + HDassert(rbuf); + + /* Case 1: No metadata in accumulator */ + /* Write 10 1's at address 40 */ + /* @0:| 1111111111| */ + /* Put some data in the accumulator initially */ + for(i = 0; i < 10; i++) + wbuf[i] = 1; + if(accum_write(40, 10 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(40, 10 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 10 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 2: End of new piece aligns with start of clean accumulated data */ + /* Write 5 2's at address 20 */ + /* @0:| 222221111111111| */ + if(accum_flush() < 0) FAIL_STACK_ERROR; + for(i = 0; i < 5; i++) + wbuf[i] = 2; + if(accum_write(20, 5 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(20, 5 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 5 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 3: Start of new piece aligns with start of accumulated data, + * completely encloses dirty section of accumulator */ + /* Write 6 3's at address 20 */ + /* @0:| 333333111111111| */ + for(i = 0; i < 6; i++) + wbuf[i] = 3; + if(accum_write(20, 6 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(20, 6 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 6 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 4: New piece completely within accumulated data, overlaps + * end of dirty section of accumulator */ + /* Write 2 4's at address 40 */ + /* @0:| 333334411111111| */ + for(i = 0; i < 2; i++) + wbuf[i] = 4; + if(accum_write(40, 2 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(40, 2 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 2 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 5: New piece completely within accumulated data, completely + * after dirty section of accumulator */ + /* Write 2 5's at address 52 */ + /* @0:| 333334415511111| */ + for(i = 0; i < 2; i++) + wbuf[i] = 5; + if(accum_write(52, 2 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(52, 2 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 2 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 6: New piece completely within clean accumulated data */ + /* Write 3 6's at address 44 */ + /* @0:| 333334666511111| */ + if(accum_flush() < 0) FAIL_STACK_ERROR; + for(i = 0; i < 3; i++) + wbuf[i] = 6; + if(accum_write(44, 3 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(44, 3 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 3 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 7: New piece overlaps start of clean accumulated data */ + /* Write 2 7's at address 16 */ + /* @0:| 7733334666511111| */ + if(accum_flush() < 0) FAIL_STACK_ERROR; + for(i = 0; i < 2; i++) + wbuf[i] = 7; + if(accum_write(16, 2 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(16, 2 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 2 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 8: New piece overlaps start of accumulated data, completely + * encloses dirty section of accumulator */ + /* Write 4 8's at address 12 */ + /* @0:| 88883334666511111| */ + for(i = 0; i < 4; i++) + wbuf[i] = 8; + if(accum_write(12, 4 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(12, 4 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 4 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 9: Start of new piece aligns with end of clean accumulated data */ + /* Write 3 9's at address 80 */ + /* @0:| 88883334666511111999| */ + if(accum_flush() < 0) FAIL_STACK_ERROR; + for(i = 0; i < 3; i++) + wbuf[i] = 9; + if(accum_write(80, 3 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(80, 3 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 3 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 10: New piece overlaps end of clean accumulated data */ + /* Write 3 2's at address 88 */ + /* @0:| 888833346665111119922| */ + if(accum_flush() < 0) FAIL_STACK_ERROR; + for(i = 0; i < 2; i++) + wbuf[i] = 2; + if(accum_write(88, 2 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(88, 2 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 2 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 11: New piece overlaps end of accumulated data, completely encloses + * dirty section of accumulator */ + /* Write 4 7's at address 84 */ + /* @0:| 8888333466651111197777| */ + for(i = 0; i < 4; i++) + wbuf[i] = 7; + if(accum_write(84, 4 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(84, 4 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 4 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Set up expected data buffer and verify contents of + accumulator as constructed by cases 1-11, above */ + for(i = 0; i < 4; i++) + wbuf[i] = 8; + for(i = 4; i < 7; i++) + wbuf[i] = 3; + for(i = 7; i < 8; i++) + wbuf[i] = 4; + for(i = 8; i < 11; i++) + wbuf[i] = 6; + for(i = 11; i < 12; i++) + wbuf[i] = 5; + for(i = 12; i < 17; i++) + wbuf[i] = 1; + for(i = 17; i < 18; i++) + wbuf[i] = 9; + for(i = 18; i < 22; i++) + wbuf[i] = 7; + if(accum_read(12, 22 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 22 * sizeof(int32_t)) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 0; + +error: + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 1; +} /* test_accum_overlap_clean */ + + +/*------------------------------------------------------------------------- + * Function: test_accum_non_overlap_size + * + * Purpose: This test will write a series of pieces of data + * to the accumulator with the goal of not overlapping + * the writes with a data size larger then the accum size. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Allen Byrne + * October 8, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_accum_non_overlap_size(void) +{ + int i = 0; + int32_t *wbuf, *rbuf; + + TESTING("non-overlapping write to accumulator larger then accum_size"); + + /* Allocate buffers */ + wbuf = (int *)HDmalloc(4096 * sizeof(int32_t)); + HDassert(wbuf); + rbuf = (int *)HDcalloc(4096, sizeof(int32_t)); + HDassert(rbuf); + + /* Case 1: No metadata in accumulator */ + /* Write 10 1's at address 140 */ + /* @0:| 1111111111| */ + /* Put some data in the accumulator initially */ + for(i = 0; i < 10; i++) + wbuf[i] = 1; + if(accum_write(140, 10 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(140, 10 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 10 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 9: New piece completely before accumulated data */ + /* Write 20 9 at address 0 */ + /* @0:|9 1111111111| */ + for(i = 0; i < 20; i++) + wbuf[i] = 9; + if(accum_write(0, 20 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(0, 20 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 20 * sizeof(int32_t)) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 0; + +error: + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 1; +} /* test_accum_non_overlap_size */ + +/*------------------------------------------------------------------------- + * Function: test_accum_overlap_size + * + * Purpose: This test will write a series of pieces of data + * to the accumulator with the goal of overlapping + * the writes with a data size completely overlapping + * the accumulator at both ends. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Allen Byrne + * October 8, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_accum_overlap_size(void) +{ + int i = 0; + int32_t *wbuf, *rbuf; + + TESTING("overlapping write to accumulator larger then accum_size"); + + /* Allocate buffers */ + wbuf = (int32_t *)HDmalloc(4096 * sizeof(int32_t)); + HDassert(wbuf); + rbuf = (int32_t *)HDcalloc(4096, sizeof(int32_t)); + HDassert(rbuf); + + /* Case 1: No metadata in accumulator */ + /* Write 10 1's at address 64 */ + /* @0:| 1111111111| */ + /* Put some data in the accumulator initially */ + for(i = 0; i < 10; i++) + wbuf[i] = 1; + if(accum_write(64, 10 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(64, 10 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 10 * sizeof(int32_t)) != 0) TEST_ERROR; + + /* Case 9: New piece completely before accumulated data */ + /* Write 72 9 at address 60 */ + /* @0:|9 1111111111| */ + for(i = 0; i < 72; i++) + wbuf[i] = 9; + if(accum_write(60, 72 * sizeof(int32_t), wbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(60, 72 * sizeof(int32_t), rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 72 * sizeof(int32_t)) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 0; + +error: + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 1; +} /* test_accum_overlap_size */ + + +/*------------------------------------------------------------------------- + * Function: test_accum_adjust + * + * Purpose: This test examines the various ways the accumulator might + * adjust itself as a result of data appending or prepending + * to it. + * + * This test program covers all the code in H5F_accum_adjust, + * but NOT all possible paths through said code. It only covers + * six potential paths through the function. (Again, though, each + * piece of code within an if/else statement in H5F_accum_adjust is + * covered by one of the paths in this test function). Since there + * are a ridiculous number of total possible paths through this + * function due to its large number of embedded if/else statements, + * that's certainly a lot of different test cases to write by hand. + * (Though if someone comes across this code and has some free + * time, go for it). + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mike McGreevy + * October 11, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_accum_adjust(void) +{ + int i = 0; + int s = 1048576; /* size of buffer */ + int32_t *wbuf, *rbuf; + + TESTING("accumulator adjustments after append/prepend of data"); + + /* Allocate buffers */ + wbuf = (int32_t *)HDmalloc((size_t)s * sizeof(int32_t)); + HDassert(wbuf); + rbuf = (int32_t *)HDcalloc((size_t)s, sizeof(int32_t)); + HDassert(rbuf); + + /* Fill up write buffer */ + for(i = 0; i < s; i++) + wbuf[i] = i + 1; + + /* ================================================================ */ + /* CASE 1: Prepending small block to large, fully dirty accumulator */ + /* ================================================================ */ + + /* Write data to the accumulator to fill it just under 1MB (max size), + * but not quite full. This will force the accumulator to, on subsequent + * writes, a) have to adjust since it's nearly full, and b) prevent + * an increase in size because it's already at it's maximum size */ + if(accum_write((1024 * 1024), (1024 * 1024) - 1, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a small (1KB) block that prepends to the front of the accumulator. */ + /* ==> Accumulator will need more buffer space */ + /* ==> Accumulator will try to resize, but see that it's getting too big */ + /* ==> Size of new block is less than half maximum size of accumulator */ + /* ==> New block is being prepended to accumulator */ + /* ==> Accumulator is dirty, it will be flushed. */ + /* ==> Dirty region overlaps region to eliminate from accumulator */ + if(accum_write((1024 * 1024) - 1024, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read back and verify first write */ + if(accum_read((1024 * 1024), (1024 * 1024) - 1, rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, (1024 * 1024) - 1) != 0) TEST_ERROR; + + /* Read back and verify second write */ + if(accum_read((1024 * 1024) - 1024, 1024, rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 1024) != 0) TEST_ERROR; + + /* Reset accumulator for next case */ + if(accum_reset() < 0) FAIL_STACK_ERROR; + + /* ================================================================ */ + /* Case 2: Prepending large block to large, fully dirty accumulator */ + /* ================================================================ */ + + /* Write data to the accumulator to fill it just under 1MB (max size), + * but not quite full. This will force the accumulator to, on subsequent + * writes, a) have to adjust since it's nearly full, and b) prevent + * an increase in size because it's already at it's maximum size */ + if(accum_write((1024 * 1024), (1024 * 1024) - 1, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a large (just under 1MB) block to the front of the accumulator. */ + /* ==> Accumulator will need more buffer space */ + /* ==> Accumulator will try to resize, but see that it's getting too big */ + /* ==> Size of new block is larger than half maximum size of accumulator */ + /* ==> New block is being prepended to accumulator */ + /* ==> Accumulator is dirty, it will be flushed. */ + /* ==> Dirty region overlaps region to eliminate from accumulator */ + if(accum_write(5, (1024 * 1024) - 5, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read back and verify both pieces of data */ + if(accum_read(1048576, 1048575, rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 1048576) != 0) TEST_ERROR; + + if(accum_read(5, 1048571, rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 1048571) != 0) TEST_ERROR; + + /* Reset accumulator for next case */ + if(accum_reset() < 0) FAIL_STACK_ERROR; + + /* ========================================================= */ + /* Case 3: Appending small block to large, clean accumulator */ + /* ========================================================= */ + + /* Write data to the accumulator to fill it just under 1MB (max size), + * but not quite full. This will force the accumulator to, on subsequent + * writes, a) have to adjust since it's nearly full, and b) prevent + * an increase in size because it's already at it's maximum size */ + if(accum_write(0, (1024 * 1024) - 1, wbuf) < 0) FAIL_STACK_ERROR; + + /* Flush the accumulator -- we want to test the case when + accumulator contains clean data */ + if(accum_flush() < 0) FAIL_STACK_ERROR + + /* Write a small (1KB) block to the end of the accumulator */ + /* ==> Accumulator will need more buffer space */ + /* ==> Accumulator will try to resize, but see that it's getting too big */ + /* ==> Size of new block is larger than half maximum size of accumulator */ + /* ==> New block being appended to accumulator */ + /* ==> Accumulator is NOT dirty */ + /* ==> Since we're appending, need to adjust location of accumulator */ + if(accum_write((1024 * 1024) - 1, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a piece of metadata outside current accumulator to force write + to disk */ + if(accum_write(0, 1, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read in the piece we wrote to disk above, and then verify that + the data is as expected */ + if(accum_read((1024 * 1024) - 1, 1024, rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 1024) != 0) TEST_ERROR; + + /* Reset accumulator for next case */ + if(accum_reset() < 0) FAIL_STACK_ERROR; + + /* ==================================================================== */ + /* Case 4: Appending small block to large, partially dirty accumulator, */ + /* with existing dirty region NOT aligning with the new block */ + /* ==================================================================== */ + + /* Write data to the accumulator to fill it just under 1MB (max size), + * but not quite full. This will force the accumulator to, on subsequent + * writes, a) have to adjust since it's nearly full, and b) prevent + * an increase in size because it's already at it's maximum size */ + if(accum_write(0, (1024 * 1024) - 5, wbuf) < 0) FAIL_STACK_ERROR; + + /* Flush the accumulator to clean it */ + if(accum_flush() < 0) FAIL_STACK_ERROR + + /* write to part of the accumulator so just the start of it is dirty */ + if(accum_write(0, 5, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a small (~340KB) piece of data to the other end of the accumulator */ + /* ==> Accumulator will need more buffer space */ + /* ==> Accumulator will try to resize, but see that it's getting too big */ + /* ==> Size of new block is less than than half maximum size of accumulator */ + /* ==> New block being appended to accumulator */ + /* ==> We can slide the dirty region down, to accomodate the request */ + /* ==> Max Buffer Size - (dirty offset + adjust size) >= 2 * size) */ + /* ==> Need to adjust location of accumulator while appending */ + /* ==> Accumulator will need to be reallocated */ + if(accum_write(1048571, 349523, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a piece of metadata outside current accumulator to force write + to disk */ + if(accum_write(1398900, 1, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read in the piece we wrote to disk above, and then verify that + the data is as expected */ + if(accum_read(1048571, 349523, rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 349523) != 0) TEST_ERROR; + + /* Reset accumulator for next case */ + if(accum_reset() < 0) FAIL_STACK_ERROR; + + /* ==================================================================== */ + /* Case 5: Appending small block to large, partially dirty accumulator, */ + /* with existing dirty region aligning with new block */ + /* ==================================================================== */ + + /* Write data to the accumulator to fill it just under max size (but not full) */ + if(accum_write(0, (1024 * 1024) - 5, wbuf) < 0) FAIL_STACK_ERROR; + + /* Flush the accumulator to clean it */ + if(accum_flush() < 0) FAIL_STACK_ERROR + + /* write to part of the accumulator so it's dirty, but not entirely dirty */ + /* (just the begging few bytes will be clean) */ + if(accum_write(10, (1024 * 1024) - 15, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a small piece of data to the dirty end of the accumulator */ + /* ==> Accumulator will need more buffer space */ + /* ==> Accumulator will try to resize, but see that it's getting too big */ + /* ==> Size of new block is less than than half maximum size of accumulator */ + /* ==> New block being appended to accumulator */ + /* ==> We can slide the dirty region down, to accomodate the request */ + /* ==> Max Buffer Size - (dirty offset + adjust size) < 2 * size) */ + /* ==> Need to adjust location of accumulator while appending */ + if(accum_write((1024 * 1024) - 5, 10, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a piece of metadata outside current accumulator to force write + to disk */ + if(accum_write(0, 1, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read in the piece we wrote to disk above, and then verify that + the data is as expected */ + if(accum_read((1024 * 1024) - 5, 10, rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 10) != 0) TEST_ERROR; + + /* Reset accumulator for next case */ + if(accum_reset() < 0) FAIL_STACK_ERROR; + + /* ================================================================= */ + /* Case 6: Appending small block to large, fully dirty accumulator */ + /* ================================================================= */ + + /* Write data to the accumulator to fill it just under 1MB (max size), + * but not quite full. This will force the accumulator to, on subsequent + * writes, a) have to adjust since it's nearly full, and b) prevent + * an increase in size because it's already at it's maximum size */ + if(accum_write(0, (1024 * 1024) - 5, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a small (~340KB) piece of data to the end of the accumulator */ + /* ==> Accumulator will need more buffer space */ + /* ==> Accumulator will try to resize, but see that it's getting too big */ + /* ==> Size of new block is less than than half maximum size of accumulator */ + /* ==> New block being appended to accumulator */ + /* ==> We cannot slide dirty region down, it's all dirty */ + /* ==> Dirty region overlaps region to eliminate from accumulator */ + /* ==> Need to adjust location of accumulator while appending */ + if(accum_write(1048571, 349523, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a piece of metadata outside current accumulator to force write + to disk */ + if(accum_write(1398900, 1, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read in the piece we wrote to disk above, and then verify that + the data is as expected */ + if(accum_read(1048571, 349523, rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 349523) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 0; + +error: + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 1; +} /* test_accum_adjust */ + + +/*------------------------------------------------------------------------- + * Function: test_read_after + * + * Purpose: This test will verify the case when metadata is read partly + * from the accumulator and partly from disk. The test will + * write a block of data at address 512, force the data to be + * written to disk, write new data partially overlapping the + * original block from below, then read data at address 512. + * The data read should be partly new and partly original. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Larry Knox + * October 8, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_read_after(void) +{ + int i = 0; + int s = 128; /* size of buffer */ + int32_t *wbuf, *rbuf; + + TESTING("reading data from both accumulator and disk"); + + /* Allocate buffers */ + wbuf = (int32_t *)HDmalloc((size_t)s * sizeof(int32_t)); + HDassert(wbuf); + rbuf = (int32_t *)HDcalloc((size_t)s, sizeof(int32_t)); + HDassert(rbuf); + + /* Fill up write buffer with 1s */ + for(i = 0; i < s; i++) + wbuf[i] = 1; + + /* Write data to the accumulator to fill it. */ + if(accum_write(512, 512, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write a piece of metadata outside current accumulator to force write + to disk */ + if(accum_write(0, 1, wbuf) < 0) FAIL_STACK_ERROR; + + /* Fill up write buffer with 2s */ + for(i = 0; i < s; i++) + wbuf[i] = 2; + + /* Write a block of 2s of the original size that will overlap the lower half + of the original block */ + if(accum_write(256, 512, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read 128 bytes at the original address, and then */ + if(accum_read(512, 512, rbuf) < 0) FAIL_STACK_ERROR; + + /* Set the second half of wbuf back to 1s */ + for(i = 64; i < s; i++) + wbuf[i] = 1; + + /* Read in the piece we wrote to disk above, and then verify that + the data is as expected */ + if(accum_read(512, 512, rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf, rbuf, 128) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 0; + +error: + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + + return 1; +} /* end test_read_after */ + + +/*------------------------------------------------------------------------- + * Function: test_big + * + * Purpose: This test exercises writing large pieces of metadata to the + * file. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * October 12, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_big(void) +{ + uint8_t *wbuf, *wbuf2, *rbuf, *zbuf; /* Buffers for reading & writing, etc */ + unsigned u; /* Local index variable */ + + /* Allocate space for the write & read buffers */ + wbuf = (uint8_t *)HDmalloc(BIG_BUF_SIZE); + HDassert(wbuf); + wbuf2 = (uint8_t *)HDmalloc(BIG_BUF_SIZE); + HDassert(wbuf2); + rbuf = (uint8_t *)HDcalloc(BIG_BUF_SIZE + 1536, 1); + HDassert(rbuf); + zbuf = (uint8_t *)HDcalloc(BIG_BUF_SIZE + 1536, 1); + HDassert(zbuf); + + /* Initialize write buffers */ + for(u = 0; u < BIG_BUF_SIZE; u++) { + wbuf[u] = (uint8_t)u; + wbuf2[u] = (uint8_t)(u + 1); + } /* end for */ + + TESTING("large metadata I/O operations"); + + /* Write large data segment to file */ + if(accum_write(0, BIG_BUF_SIZE, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read entire segment back from file */ + if(accum_read(0, BIG_BUF_SIZE, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(wbuf, rbuf, BIG_BUF_SIZE) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(0, BIG_BUF_SIZE, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section to middle of accumulator */ + if(accum_write(1024, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read entire segment back from file */ + /* (Read covers entire dirty region) */ + if(accum_read(0, BIG_BUF_SIZE, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(zbuf, rbuf, 1024) != 0) TEST_ERROR; + if(HDmemcmp(wbuf, rbuf + 1024, 1024) != 0) TEST_ERROR; + if(HDmemcmp(zbuf, rbuf + 2048, (BIG_BUF_SIZE - 2048)) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(1024, 1024, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section to overlap with end of "big" region */ + if(accum_write(BIG_BUF_SIZE - 512, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read entire segment back from file */ + /* (Read covers bottom half of dirty region) */ + if(accum_read(0, BIG_BUF_SIZE, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(zbuf, rbuf, (BIG_BUF_SIZE - 512)) != 0) TEST_ERROR; + if(HDmemcmp(wbuf, rbuf + (BIG_BUF_SIZE - 512), 512) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(BIG_BUF_SIZE - 512, 1024, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section to overlap with beginning of "big" region */ + if(accum_write(0, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read entire segment back from file */ + /* (Read covers bottom half of dirty region) */ + if(accum_read(512, BIG_BUF_SIZE, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(wbuf + 512, rbuf, 512) != 0) TEST_ERROR; + if(HDmemcmp(zbuf, rbuf + 512, (BIG_BUF_SIZE - 512)) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(0, 1024, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section to middle of accumulator */ + /* (With write buffer #1) */ + if(accum_write(1024, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write entire segment to from file */ + /* (With write buffer #2) */ + /* (Write covers entire dirty region) */ + if(accum_write(0, BIG_BUF_SIZE, wbuf2) < 0) FAIL_STACK_ERROR; + + /* Read entire segment back from file */ + if(accum_read(0, BIG_BUF_SIZE, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(wbuf2, rbuf, BIG_BUF_SIZE) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(0, BIG_BUF_SIZE, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section to overlap with end of "big" region */ + /* (With write buffer #1) */ + if(accum_write(BIG_BUF_SIZE - 512, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Write entire segment to from file */ + /* (With write buffer #2) */ + /* (Read covers bottom half of dirty region) */ + if(accum_write(0, BIG_BUF_SIZE, wbuf2) < 0) FAIL_STACK_ERROR; + + /* Read both segments back from file */ + if(accum_read(0, BIG_BUF_SIZE + 512, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(wbuf2, rbuf, BIG_BUF_SIZE) != 0) TEST_ERROR; + if(HDmemcmp(wbuf + 512, rbuf + BIG_BUF_SIZE, 512) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(0, BIG_BUF_SIZE + 512, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE + 512); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section to be past "big" region */ + /* (With write buffer #1) */ + if(accum_write(BIG_BUF_SIZE + 512, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read section before "big" region */ + /* (To enlarge accumulator, to it will intersect with big write) */ + if(accum_read(BIG_BUF_SIZE - 512, 1024, rbuf) < 0) FAIL_STACK_ERROR; + + /* Write entire segment to from file */ + /* (With write buffer #2) */ + /* (Doesn't overlap with small section) */ + if(accum_write(0, BIG_BUF_SIZE, wbuf2) < 0) FAIL_STACK_ERROR; + + /* Read both segments & gap back from file */ + if(accum_read(0, BIG_BUF_SIZE + 1024, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(wbuf2, rbuf, BIG_BUF_SIZE) != 0) TEST_ERROR; + if(HDmemcmp(zbuf, rbuf + BIG_BUF_SIZE, 512) != 0) TEST_ERROR; + if(HDmemcmp(wbuf, rbuf + BIG_BUF_SIZE + 512, 512) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(0, BIG_BUF_SIZE + 1536, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE + 1024); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section to be past "big" region */ + /* (With write buffer #1) */ + if(accum_write(BIG_BUF_SIZE + 512, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read section before "big" region */ + /* (To enlarge accumulator, so it will intersect with big write) */ + if(accum_read(BIG_BUF_SIZE - 512, 1024, rbuf) < 0) FAIL_STACK_ERROR; + if(accum_read(BIG_BUF_SIZE + 1536, 1024, rbuf) < 0) FAIL_STACK_ERROR; + + /* Write entire segment to from file */ + /* (With write buffer #2) */ + /* (Overwriting dirty region, but not invalidating entire accumulator) */ + if(accum_write(1536, BIG_BUF_SIZE, wbuf2) < 0) FAIL_STACK_ERROR; + + /* Read both segments & gap back from file */ + if(accum_read(0, BIG_BUF_SIZE + 1536, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(zbuf, rbuf, 1536) != 0) TEST_ERROR; + if(HDmemcmp(wbuf2, rbuf + 1536, BIG_BUF_SIZE) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(1536, BIG_BUF_SIZE, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE + 1536); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section before "big" region */ + /* (With write buffer #1) */ + if(accum_write(1024, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read section before "big" region */ + /* (To enlarge accumulator, so it will intersect with big write) */ + if(accum_read(0, 1024, rbuf) < 0) FAIL_STACK_ERROR; + + /* Write entire segment to from file */ + /* (With write buffer #2) */ + /* (Overwriting dirty region, but not invalidating entire accumulator) */ + if(accum_write(512, BIG_BUF_SIZE, wbuf2) < 0) FAIL_STACK_ERROR; + + /* Read both segments & gap back from file */ + if(accum_read(0, BIG_BUF_SIZE + 512, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(zbuf, rbuf, 512) != 0) TEST_ERROR; + if(HDmemcmp(wbuf2, rbuf + 512, BIG_BUF_SIZE) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(512, BIG_BUF_SIZE, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE + 512); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section before "big" region */ + /* (With write buffer #1) */ + if(accum_write(0, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read section before "big" region */ + /* (To enlarge accumulator, so it will intersect with big write) */ + if(accum_read(1024, 1024, rbuf) < 0) FAIL_STACK_ERROR; + + /* Write entire segment to from file */ + /* (With write buffer #2) */ + /* (Avoiding dirty region, and not invalidating entire accumulator) */ + if(accum_write(1536, BIG_BUF_SIZE, wbuf2) < 0) FAIL_STACK_ERROR; + + /* Read both segments & gap back from file */ + if(accum_read(0, BIG_BUF_SIZE + 1536, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(wbuf, rbuf, 1024) != 0) TEST_ERROR; + if(HDmemcmp(zbuf, rbuf + 1024, 512) != 0) TEST_ERROR; + if(HDmemcmp(wbuf2, rbuf + 1536, BIG_BUF_SIZE) != 0) TEST_ERROR; + + + /* Reset data in file back to zeros & reset the read buffer */ + if(accum_write(0, BIG_BUF_SIZE + 1536, zbuf) < 0) FAIL_STACK_ERROR; + HDmemset(rbuf, 0, BIG_BUF_SIZE + 1536); + if(accum_reset() < 0) FAIL_STACK_ERROR; + + + /* Write small section before "big" region */ + /* (With write buffer #1) */ + if(accum_write(0, 1024, wbuf) < 0) FAIL_STACK_ERROR; + + /* Read section before "big" region */ + /* (To enlarge accumulator, so it will intersect with big write) */ + if(accum_read(1024, 1024, rbuf) < 0) FAIL_STACK_ERROR; + + /* Write entire segment to from file */ + /* (With write buffer #2) */ + /* (Partially overwriting dirty region, and not invalidating entire accumulator) */ + if(accum_write(512, BIG_BUF_SIZE, wbuf2) < 0) FAIL_STACK_ERROR; + + /* Read both segments back from file */ + if(accum_read(0, BIG_BUF_SIZE + 512, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read */ + if(HDmemcmp(wbuf, rbuf, 512) != 0) TEST_ERROR; + if(HDmemcmp(wbuf2, rbuf + 512, BIG_BUF_SIZE) != 0) TEST_ERROR; + + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(wbuf); + HDfree(wbuf2); + HDfree(rbuf); + HDfree(zbuf); + + return 0; + +error: + HDfree(wbuf); + HDfree(wbuf2); + HDfree(rbuf); + HDfree(zbuf); + + return 1; +} /* end test_big() */ + + +/*------------------------------------------------------------------------- + * Function: test_random_write + * + * Purpose: This test writes random pieces of data to the file and + * then reads it all back. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * October 11, 2010 + * + *------------------------------------------------------------------------- + */ +unsigned +test_random_write(void) +{ + uint8_t *wbuf, *rbuf; /* Buffers for reading & writing */ + unsigned long seed = 0; /* Random # seed */ + size_t *off; /* Offset of buffer segments to write */ + size_t *len; /* Size of buffer segments to write */ + size_t cur_off; /* Current offset */ + size_t nsegments; /* Number of segments to write */ + size_t swap; /* Position to swap with */ + unsigned u; /* Local index variable */ + + /* Allocate space for the write & read buffers */ + wbuf = (uint8_t *)malloc(RANDOM_BUF_SIZE); + HDassert(wbuf); + rbuf = (uint8_t *)calloc(RANDOM_BUF_SIZE, 1); + HDassert(rbuf); + + /* Initialize write buffer */ + for(u = 0; u < RANDOM_BUF_SIZE; u++) + wbuf[u] = (uint8_t)u; + + TESTING("random writes to accumulator"); + + /* Choose random # seed */ + seed = (unsigned long)HDtime(NULL); +#ifdef QAK +/* seed = (unsigned long)1155438845; */ +HDfprintf(stderr, "Random # seed was: %lu\n", seed); +#endif /* QAK */ + HDsrandom(seed); + + /* Allocate space for the segment length buffer */ + off = (size_t *)malloc(MAX_RANDOM_SEGMENTS * sizeof(size_t)); + HDassert(off); + len = (size_t *)malloc(MAX_RANDOM_SEGMENTS * sizeof(size_t)); + HDassert(len); + + /* Randomly choose lengths of segments */ + cur_off = 0; + for(u = 0; u < MAX_RANDOM_SEGMENTS; ) { + size_t length = 0; /* Length of current segment */ + + /* Choose random length of segment, allowing for variance */ + do { + length += (size_t)(HDrandom() % RAND_SEG_LEN) + 1; + } while((HDrandom() & 256) >= 128); /* end while */ + + /* Check for going off end of buffer */ + if((cur_off + length) > RANDOM_BUF_SIZE) + length = RANDOM_BUF_SIZE - cur_off; + + /* Set offset & length of segment */ + off[u] = cur_off; + len[u] = length; + + /* Advance array offset */ + u++; + + /* Advance current offset */ + cur_off += length; + + /* If we've used up entire buffer before hitting limit of segments, get out */ + if(cur_off >= RANDOM_BUF_SIZE) + break; + } /* end for */ + nsegments = u; + + /* Increase length of last segment, if it doesn't reach end of buffer */ + if(nsegments < MAX_RANDOM_SEGMENTS) + len[nsegments - 1] = RANDOM_BUF_SIZE - off[nsegments - 1]; + + /* Shuffle order of segments, to randomize positions to write */ + for(u = 0; u < nsegments; u++) { + size_t tmp; /* Temporary holder for offset & length values */ + + /* Choose value within next few elements to to swap with */ + swap = ((size_t)HDrandom() % 8) + u; + if(swap >= nsegments) + swap = nsegments - 1; + + /* Swap values */ + tmp = off[u]; off[u] = off[swap]; off[swap] = tmp; + tmp = len[u]; len[u] = len[swap]; len[swap] = tmp; + } /* end for */ + + /* Write data segments to file */ + for(u = 0; u < nsegments; u++) { + if(accum_write(RANDOM_BASE_OFF + off[u], len[u], wbuf + off[u]) < 0) FAIL_STACK_ERROR; + + /* Verify individual reads */ + if(accum_read(RANDOM_BASE_OFF + off[u], len[u], rbuf) < 0) FAIL_STACK_ERROR; + if(HDmemcmp(wbuf + off[u], rbuf, len[u]) != 0) TEST_ERROR; + } /* end for */ + + /* Read entire region back from file */ + if(accum_read(RANDOM_BASE_OFF, RANDOM_BUF_SIZE, rbuf) < 0) FAIL_STACK_ERROR; + + /* Verify data read back in */ + if(HDmemcmp(wbuf, rbuf, RANDOM_BUF_SIZE) != 0) TEST_ERROR; + + if(accum_reset() < 0) FAIL_STACK_ERROR; + + PASSED(); + + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + HDfree(off); + HDfree(len); + + return 0; + +error: + /* Release memory */ + HDfree(wbuf); + HDfree(rbuf); + HDfree(off); + HDfree(len); + + HDfprintf(stderr, "Random # seed was: %lu\n", seed); + return 1; +} /* end test_random_write() */ + + +/*------------------------------------------------------------------------- + * Function: accum_printf + * + * Purpose: Debug function to print some stats about the accumulator + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Mike McGreevy + * October 7, 2010 + * + *------------------------------------------------------------------------- + */ +void +accum_printf(void) +{ + H5F_meta_accum_t * accum = &f->shared->accum; + + printf("\n"); + printf("Current contents of accumulator:\n"); + if (accum->alloc_size == 0) { + printf("=====================================================\n"); + printf(" No accumulator allocated.\n"); + printf("=====================================================\n"); + } else { + printf("=====================================================\n"); + printf(" accumulator allocated size == %lu\n", (unsigned long)accum->alloc_size); + printf(" accumulated data size == %lu\n", (unsigned long)accum->size); + printf(" accumulator dirty? == %d\n", accum->dirty); + printf("=====================================================\n"); + printf(" start of accumulated data, loc = %llu\n", accum->loc); + if (accum->dirty) printf(" start of dirty region, loc = %llu\n", accum->loc + accum->dirty_off); + if (accum->dirty) printf(" end of dirty region, loc = %llu\n", accum->loc + accum->dirty_off + accum->dirty_len); + printf(" end of accumulated data, loc = %llu\n", accum->loc + accum->size); + printf(" end of accumulator allocation, loc = %llu\n", accum->loc + accum->alloc_size); + printf("=====================================================\n"); + } + printf("\n\n"); +} /* accum_printf() */ + diff --git a/test/err_compat.c b/test/err_compat.c index be86a2d..c08e259 100644 --- a/test/err_compat.c +++ b/test/err_compat.c @@ -42,20 +42,187 @@ int ipoints2[DIM0][DIM1], icheck2[DIM0][DIM1]; #define DSET_NAME "a_dataset" #define FAKE_ID -1 -herr_t custom_print_cb(int n, H5E_error1_t *err_desc, void* client_data); +herr_t custom_print_cb1(int n, H5E_error1_t *err_desc, void* client_data); +herr_t custom_print_cb2(int n, H5E_error2_t *err_desc, void* client_data); /*------------------------------------------------------------------------- - * Function: test_error + * Function: user_print1 * - * Purpose: Test error API functions + * Purpose: This function is a user-defined old-style printing function. + * This is just a convenience function for H5Ewalk1() with a + * function that prints error messages. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 4 October 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +user_print1(FILE *stream) +{ + /* Customized way to print errors */ + fprintf(stderr, "\n********* Print error stack in customized way *********\n"); + if(H5Ewalk1(H5E_WALK_UPWARD, (H5E_walk1_t)custom_print_cb1, stream) < 0) + TEST_ERROR; + + return 0; + + error: + return -1; + +} + + +/*------------------------------------------------------------------------- + * Function: user_print2 + * + * Purpose: This function is a user-defined new-style printing function. + * This is just a convenience function for H5Ewalk2() with a + * function that prints error messages. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 4 October 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +user_print2(hid_t err_stack, FILE *stream) +{ + /* Customized way to print errors */ + fprintf(stderr, "\n********* Print error stack in customized way *********\n"); + if(H5Ewalk2(err_stack, H5E_WALK_UPWARD, (H5E_walk2_t)custom_print_cb2, stream) < 0) + TEST_ERROR; + + return 0; + + error: + return -1; + +} + + +/*------------------------------------------------------------------------- + * Function: custom_print_cb1 + * + * Purpose: Callback function to print error stack in customized way + * for H5Ewalk1. * * Return: Success: 0 * * Failure: -1 * * Programmer: Raymond Lu - * July 10, 2003 + * 4 October 2010 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +custom_print_cb1(int n, H5E_error1_t *err_desc, void* client_data) +{ + FILE *stream = (FILE *)client_data; + char *maj = NULL; + char *min = NULL; + const int indent = 4; + + if(NULL == (min = H5Eget_minor(err_desc->min_num))) + TEST_ERROR; + + if(NULL == (maj = H5Eget_major(err_desc->maj_num))) + TEST_ERROR; + + fprintf(stream, "%*serror #%03d: %s in %s(): line %u\n", + indent, "", n, err_desc->file_name, + err_desc->func_name, err_desc->line); + + fprintf(stream, "%*smajor: %s\n", indent * 2, "", maj); + fprintf(stream, "%*sminor: %s\n", indent * 2, "", min); + + HDfree(maj); + HDfree(min); + + return 0; + +error: + if(maj) + HDfree(maj); + if(min) + HDfree(min); + + return -1; +} + + +/*------------------------------------------------------------------------- + * Function: custom_print_cb2 + * + * Purpose: Callback function to print error stack in customized way + * for H5Ewalk1. + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Raymond Lu + * 4 October 2010 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +custom_print_cb2(int n, H5E_error2_t *err_desc, void* client_data) +{ + FILE *stream = (FILE *)client_data; + char *maj = NULL; + char *min = NULL; + const int indent = 4; + + if(NULL == (min = H5Eget_minor(err_desc->min_num))) + TEST_ERROR; + + if(NULL == (maj = H5Eget_major(err_desc->maj_num))) + TEST_ERROR; + + fprintf(stream, "%*serror #%03d: %s in %s(): line %u\n", + indent, "", n, err_desc->file_name, + err_desc->func_name, err_desc->line); + + fprintf(stream, "%*smajor: %s\n", indent * 2, "", maj); + fprintf(stream, "%*sminor: %s\n", indent * 2, "", min); + + HDfree(maj); + HDfree(min); + + return 0; + +error: + if(maj) + HDfree(maj); + if(min) + HDfree(min); + + return -1; +} + + +/*------------------------------------------------------------------------- + * Function: test_error1 + * + * Purpose: Test the backward compatibility of H5Eset/get_auto. + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Raymond Lu + * 17 September 2010 * * * Modifications: @@ -63,13 +230,143 @@ herr_t custom_print_cb(int n, H5E_error1_t *err_desc, void* client_data); *------------------------------------------------------------------------- */ static herr_t -test_error(hid_t file) +test_error1(void) { hid_t dataset, space; hsize_t dims[2]; - const char *FUNC_test_error="test_error"; - H5E_auto1_t old_func; + H5E_auto1_t old_func1; + H5E_auto2_t old_func2; void *old_data; + herr_t ret; + + TESTING("error API H5Eset/get_auto"); + fprintf(stderr, "\n"); + + /* Create the data space */ + dims[0] = DIM0; + dims[1] = DIM1; + if ((space = H5Screate_simple(2, dims, NULL))<0) TEST_ERROR; + + /* Use H5Eget_auto2 to query the default printing function. The library + *should indicate H5Eprint2 as the default. */ + if (H5Eget_auto2(H5E_DEFAULT, &old_func2, &old_data)<0) + TEST_ERROR; + if (old_data != NULL) + TEST_ERROR; + if (!old_func2 || (H5E_auto2_t)H5Eprint2 != old_func2) + TEST_ERROR; + + /* This function sets the default printing function to be H5Eprint2. */ + if(H5Eset_auto2(H5E_DEFAULT, old_func2, old_data)<0) + TEST_ERROR; + + /* Try the printing function. Dataset creation should fail because the file + * doesn't exist. */ + dataset = H5Dcreate2(FAKE_ID, DSET_NAME, H5T_STD_I32BE, space, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + if(dataset >= 0) + TEST_ERROR; + + /* This call should work. It simply returns H5Eprint1. */ + if((ret = H5Eget_auto1(&old_func1, &old_data))<0) + TEST_ERROR; + if (old_data != NULL) + TEST_ERROR; + if (!old_func1 || (H5E_auto1_t)H5Eprint1 != old_func1) + TEST_ERROR; + + /* This function changes the old-style printing function to be user_print1. */ + if(H5Eset_auto1((H5E_auto1_t)user_print1, stderr)<0) + TEST_ERROR; + + /* Try the printing function. Dataset creation should fail because the file + * doesn't exist. */ + dataset = H5Dcreate2(FAKE_ID, DSET_NAME, H5T_STD_I32BE, space, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + if(dataset >= 0) + TEST_ERROR; + + /* This call should fail because the test mixes H5Eget_auto2 with H5Eset_auto1. + * Once the H5Eset_auto1 is called with a user-defined printing function, + * a call to H5Eget_auto2 will fail. But keep in mind the printing function is + * user_print1. */ + if((ret = H5Eget_auto2(H5E_DEFAULT, &old_func2, &old_data))>=0) + TEST_ERROR; + + /* This function changes the new-style printing function to be user_print2. */ + if(H5Eset_auto2(H5E_DEFAULT, (H5E_auto2_t)user_print2, stderr)<0) + TEST_ERROR; + + /* Try the printing function. Dataset creation should fail because the file + * doesn't exist. */ + dataset = H5Dcreate2(FAKE_ID, DSET_NAME, H5T_STD_I32BE, space, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + if(dataset >= 0) + TEST_ERROR; + + /* This function changes the new-style printing function back to the default H5Eprint2. */ + if(H5Eset_auto2(H5E_DEFAULT, (H5E_auto2_t)H5Eprint2, NULL)<0) + TEST_ERROR; + + /* This call should work because the H5Eset_auto2 above restored the default printing + * function H5Eprint2. It simply returns user_print1. */ + if((ret = H5Eget_auto1(&old_func1, &old_data))<0) + TEST_ERROR; + if (old_data != NULL) + TEST_ERROR; + if (!old_func1 || (H5E_auto1_t)user_print1 != old_func1) + TEST_ERROR; + + /* This function changes the new-style printing function back to the default H5Eprint1. */ + if(H5Eset_auto1((H5E_auto1_t)H5Eprint1, NULL)<0) + TEST_ERROR; + + /* This call should work because the H5Eset_auto1 above restored the default printing + * function H5Eprint1. It simply returns H5Eprint2. */ + if((ret = H5Eget_auto2(H5E_DEFAULT, &old_func2, &old_data))<0) + TEST_ERROR; + if (old_data != NULL) + TEST_ERROR; + if (!old_func2 || (H5E_auto2_t)H5Eprint2 != old_func2) + TEST_ERROR; + + /* Try the printing function. Dataset creation should fail because the file + * doesn't exist. */ + dataset = H5Dcreate2(FAKE_ID, DSET_NAME, H5T_STD_I32BE, space, H5P_DEFAULT, + H5P_DEFAULT, H5P_DEFAULT); + if(dataset >= 0) + TEST_ERROR; + + return 0; + + error: + return -1; +} + + +/*------------------------------------------------------------------------- + * Function: test_error2 + * + * Purpose: Test error API functions, mainly on H5Epush1. + * + * Return: Success: 0 + * + * Failure: -1 + * + * Programmer: Raymond Lu + * July 10, 2003 + * + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +test_error2(hid_t file) +{ + hid_t dataset, space; + hsize_t dims[2]; + const char *FUNC_test_error="test_error2"; TESTING("error API based on data I/O"); fprintf(stderr, "\n"); @@ -92,22 +389,12 @@ test_error(hid_t file) goto error; } - /* Test enabling and disabling default printing */ - if (H5Eget_auto1(&old_func, &old_data)<0) - TEST_ERROR; - if (old_data != NULL) - TEST_ERROR; - if (!old_func) - TEST_ERROR; -#ifdef H5_USE_16_API - if (old_func != (H5E_auto1_t)H5Eprint1) - TEST_ERROR; -#else /* H5_USE_16_API */ - if (old_func != (H5E_auto1_t)H5Eprint2) - TEST_ERROR; -#endif /* H5_USE_16_API */ - - if(H5Eset_auto1(NULL, NULL)<0) + /* Disable the library's default printing function */ +#ifdef H5_USE_16_API_DEFAULT + if(H5Eset_auto(NULL, NULL)<0) +#else + if(H5Eset_auto(H5E_DEFAULT, NULL, NULL)<0) +#endif TEST_ERROR; /* Make H5Dwrite fail, verify default print is disabled */ @@ -117,9 +404,6 @@ test_error(hid_t file) goto error; } - if(H5Eset_auto1(old_func, old_data)<0) - TEST_ERROR; - /* In case program comes to this point, close dataset */ if(H5Dclose(dataset)<0) TEST_ERROR; @@ -157,7 +441,7 @@ dump_error(void) /* Customized way to print errors */ fprintf(stderr, "\n********* Print error stack in customized way *********\n"); - if(H5Ewalk1(H5E_WALK_UPWARD, custom_print_cb, stderr) < 0) + if(H5Ewalk1(H5E_WALK_UPWARD, custom_print_cb1, stderr) < 0) TEST_ERROR; return 0; @@ -166,57 +450,6 @@ dump_error(void) return -1; } -/*------------------------------------------------------------------------- - * Function: custom_print_cb - * - * Purpose: Callback function to print error stack in customized way. - * - * Return: Success: 0 - * - * Failure: -1 - * - * Programmer: Raymond Lu - * July 17, 2003 - * - * - * Modifications: - * - *------------------------------------------------------------------------- - */ -herr_t -custom_print_cb(int n, H5E_error1_t *err_desc, void* client_data) -{ - FILE *stream = (FILE *)client_data; - char *maj = NULL; - char *min = NULL; - const int indent = 4; - - if(NULL == (min = H5Eget_minor(err_desc->min_num))) - TEST_ERROR; - - if(NULL == (maj = H5Eget_major(err_desc->maj_num))) - TEST_ERROR; - - fprintf(stream, "%*serror #%03d: %s in %s(): line %u\n", - indent, "", n, err_desc->file_name, - err_desc->func_name, err_desc->line); - - fprintf(stream, "%*smajor: %s\n", indent * 2, "", maj); - fprintf(stream, "%*sminor: %s\n", indent * 2, "", min); - - HDfree(maj); - HDfree(min); - - return 0; - -error: - if(maj) - HDfree(maj); - if(min) - HDfree(min); - - return -1; -} /*------------------------------------------------------------------------- @@ -258,7 +491,9 @@ main(void) H5Eclear1(); /* Test error API */ - if(test_error(file) < 0) { + if(test_error1() < 0) TEST_ERROR ; + + if(test_error2(file) < 0) { H5Epush1(__FILE__, FUNC_main, __LINE__, H5E_ERROR, H5E_BADMESG, "Error test failed"); H5Eprint1(stderr); @@ -275,4 +510,3 @@ main(void) return 1; } #endif /* H5_NO_DEPRECATED_SYMBOLS */ - diff --git a/test/ohdr.c b/test/ohdr.c index dad06cf..8494d51 100644 --- a/test/ohdr.c +++ b/test/ohdr.c @@ -80,10 +80,10 @@ test_cont(char *filename, hid_t fapl) HDmemset(&oh_locA, 0, sizeof(oh_locA)); HDmemset(&oh_locB, 0, sizeof(oh_locB)); - if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)H5O_MIN_SIZE, H5P_GROUP_CREATE_DEFAULT, &oh_locA/*out*/) < 0) + if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)H5O_MIN_SIZE, (size_t)0, H5P_GROUP_CREATE_DEFAULT, &oh_locA/*out*/) < 0) FAIL_STACK_ERROR - if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)H5O_MIN_SIZE, H5P_GROUP_CREATE_DEFAULT, &oh_locB/*out*/) < 0) + if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)H5O_MIN_SIZE, (size_t)0, H5P_GROUP_CREATE_DEFAULT, &oh_locB/*out*/) < 0) FAIL_STACK_ERROR time_new = 11111111; @@ -107,6 +107,10 @@ test_cont(char *filename, hid_t fapl) if(H5O_msg_create(&oh_locA, H5O_NAME_ID, 0, 0, &short_name, H5P_DATASET_XFER_DEFAULT) < 0) FAIL_STACK_ERROR + if(1 != H5O_link(&oh_locA, 1, H5P_DATASET_XFER_DEFAULT)) + FAIL_STACK_ERROR + if(1 != H5O_link(&oh_locB, 1, H5P_DATASET_XFER_DEFAULT)) + FAIL_STACK_ERROR if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT) < 0) FAIL_STACK_ERROR if(H5O_expunge_chunks_test(&oh_locA, H5P_DATASET_XFER_DEFAULT) < 0) @@ -148,6 +152,149 @@ error: return -1; } /* test_cont() */ +/* + * Verify that object headers are held in the cache until they are linked + * to a location in the graph, or assigned an ID. This is done by + * creating an object header, then forcing it out of the cache by creating + * local heaps until the object header is evicted from the cache, then + * modifying the object header. The refcount on the object header is + * checked as verifying that the object header has remained in the cache. + */ +static herr_t +test_ohdr_cache(char *filename, hid_t fapl) +{ + hid_t file = -1; /* File ID */ + hid_t my_fapl; /* FAPL ID */ + hid_t my_dxpl; /* DXPL ID */ + H5AC_cache_config_t mdc_config; /* Metadata cache configuration info */ + H5F_t *f = NULL; /* File handle */ + H5HL_t *lheap, *lheap2, *lheap3; /* Pointer to local heaps */ + haddr_t lheap_addr, lheap_addr2, lheap_addr3; /* Local heap addresses */ + H5O_loc_t oh_loc; /* Object header location */ + time_t time_new; /* Time value for modification time message */ + unsigned rc; /* Refcount for object */ + + TESTING("object header creation in cache"); + + /* Make a copy of the FAPL */ + if((my_fapl = H5Pcopy(fapl)) < 0) + FAIL_STACK_ERROR + + /* Tweak down the size of the metadata cache to only 64K */ + mdc_config.version = H5AC__CURR_CACHE_CONFIG_VERSION; + if(H5Pget_mdc_config(my_fapl, &mdc_config) < 0) + FAIL_STACK_ERROR + mdc_config.set_initial_size = TRUE; + mdc_config.initial_size = 32 * 1024; + mdc_config.max_size = 64 * 1024; + mdc_config.min_size = 8 * 1024; + if(H5Pset_mdc_config(my_fapl, &mdc_config) < 0) + FAIL_STACK_ERROR + + /* Make a copy of the default DXPL */ + if((my_dxpl = H5Pcopy(H5P_DATASET_XFER_DEFAULT)) < 0) + FAIL_STACK_ERROR + + /* Create the file to operate on */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, my_fapl)) < 0) + FAIL_STACK_ERROR + if(H5Pclose(my_fapl) < 0) + FAIL_STACK_ERROR + if(NULL == (f = (H5F_t *)H5I_object(file))) + FAIL_STACK_ERROR + if(H5AC_ignore_tags(f) < 0) + FAIL_STACK_ERROR + + /* Create object (local heap) that occupies most of cache */ + if(H5HL_create(f, my_dxpl, (31 * 1024), &lheap_addr) < 0) + FAIL_STACK_ERROR + + /* Protect local heap (which actually pins it in the cache) */ + if(NULL == (lheap = H5HL_protect(f, my_dxpl, lheap_addr, H5AC_READ))) + FAIL_STACK_ERROR + + /* Create an object header */ + HDmemset(&oh_loc, 0, sizeof(oh_loc)); + if(H5O_create(f, my_dxpl, (size_t)2048, (size_t)1, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0) + FAIL_STACK_ERROR + + /* Query object header information */ + rc = 0; + if(H5O_get_rc(&oh_loc, my_dxpl, &rc) < 0) + FAIL_STACK_ERROR + if(0 != rc) + TEST_ERROR + + /* Create object (local heap) that occupies most of cache */ + if(H5HL_create(f, my_dxpl, (31 * 1024), &lheap_addr2) < 0) + FAIL_STACK_ERROR + + /* Protect local heap (which actually pins it in the cache) */ + if(NULL == (lheap2 = H5HL_protect(f, my_dxpl, lheap_addr2, H5AC_READ))) + FAIL_STACK_ERROR + + /* Unprotect local heap (which actually unpins it from the cache) */ + if(H5HL_unprotect(lheap2) < 0) + FAIL_STACK_ERROR + + /* Create object header message in new object header */ + time_new = 11111111; + if(H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, my_dxpl) < 0) + FAIL_STACK_ERROR + + /* Create object (local heap) that occupies most of cache */ + if(H5HL_create(f, my_dxpl, (31 * 1024), &lheap_addr3) < 0) + FAIL_STACK_ERROR + + /* Protect local heap (which actually pins it in the cache) */ + if(NULL == (lheap3 = H5HL_protect(f, my_dxpl, lheap_addr3, H5AC_READ))) + FAIL_STACK_ERROR + + /* Unprotect local heap (which actually unpins it from the cache) */ + if(H5HL_unprotect(lheap3) < 0) + FAIL_STACK_ERROR + + /* Query object header information */ + /* (Note that this is somewhat of a weak test, since it doesn't actually + * verify that the object header was evicted from the cache, but it's + * very difficult to verify when an entry is evicted from the cache in + * a non-invasive way -QAK) + */ + rc = 0; + if(H5O_get_rc(&oh_loc, my_dxpl, &rc) < 0) + FAIL_STACK_ERROR + if(0 != rc) + TEST_ERROR + + /* Decrement reference count o object header */ + if(H5O_dec_rc_by_loc(&oh_loc, my_dxpl) < 0) + FAIL_STACK_ERROR + + /* Close object header created */ + if(H5O_close(&oh_loc) < 0) + FAIL_STACK_ERROR + + /* Unprotect local heap (which actually unpins it from the cache) */ + if(H5HL_unprotect(lheap) < 0) + FAIL_STACK_ERROR + + if(H5Pclose(my_dxpl) < 0) + FAIL_STACK_ERROR + if(H5Fclose(file) < 0) + FAIL_STACK_ERROR + + PASSED(); + + return 0; + +error: + H5E_BEGIN_TRY { + H5Fclose(file); + } H5E_END_TRY; + + return -1; +} /* test_ohdr_cache() */ + /*------------------------------------------------------------------------- * Function: main @@ -216,7 +363,7 @@ main(void) */ TESTING("object header creation"); HDmemset(&oh_loc, 0, sizeof(oh_loc)); - if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0) + if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, (size_t)0, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0) FAIL_STACK_ERROR PASSED(); @@ -226,6 +373,8 @@ main(void) time_new = 11111111; if(H5O_msg_create(&oh_loc, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0) FAIL_STACK_ERROR + if(1 != H5O_link(&oh_loc, 1, H5P_DATASET_XFER_DEFAULT)) + FAIL_STACK_ERROR if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT) < 0) FAIL_STACK_ERROR if(H5AC_expunge_entry(f, H5P_DATASET_XFER_DEFAULT, H5AC_OHDR, oh_loc.addr, H5AC__NO_FLAGS_SET) < 0) @@ -378,12 +527,16 @@ main(void) */ TESTING("locking messages"); HDmemset(&oh_loc, 0, sizeof(oh_loc)); - if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0) + if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, (size_t)0, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0) + FAIL_STACK_ERROR + if(1 != H5O_link(&oh_loc, 1, H5P_DATASET_XFER_DEFAULT)) FAIL_STACK_ERROR /* Create second object header, to guarantee that first object header uses multiple chunks */ HDmemset(&oh_loc2, 0, sizeof(oh_loc2)); - if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, H5P_GROUP_CREATE_DEFAULT, &oh_loc2/*out*/) < 0) + if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, (size_t)0, H5P_GROUP_CREATE_DEFAULT, &oh_loc2/*out*/) < 0) + FAIL_STACK_ERROR + if(1 != H5O_link(&oh_loc2, 1, H5P_DATASET_XFER_DEFAULT)) FAIL_STACK_ERROR /* Fill object header with messages, creating multiple chunks */ @@ -452,12 +605,16 @@ main(void) /* Open first object header */ HDmemset(&oh_loc, 0, sizeof(oh_loc)); - if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0) + if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, (size_t)0, H5P_GROUP_CREATE_DEFAULT, &oh_loc/*out*/) < 0) + FAIL_STACK_ERROR + if(1 != H5O_link(&oh_loc, 1, H5P_DATASET_XFER_DEFAULT)) FAIL_STACK_ERROR /* Create second object header, to guarantee that first object header uses multiple chunks */ HDmemset(&oh_loc2, 0, sizeof(oh_loc2)); - if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, H5P_GROUP_CREATE_DEFAULT, &oh_loc2/*out*/) < 0) + if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)64, (size_t)0, H5P_GROUP_CREATE_DEFAULT, &oh_loc2/*out*/) < 0) + FAIL_STACK_ERROR + if(1 != H5O_link(&oh_loc2, 1, H5P_DATASET_XFER_DEFAULT)) FAIL_STACK_ERROR /* Add message to move to object header */ @@ -632,6 +789,10 @@ main(void) /* Close the file we created */ if(H5Fclose(file) < 0) TEST_ERROR + + /* Test object header creation metadata cache issues */ + if(test_ohdr_cache(filename, fapl) < 0) + TEST_ERROR } /* end for */ puts("All object header tests passed."); diff --git a/test/testerror.sh.in b/test/testerror.sh.in index 7f9657a..440be4f 100644 --- a/test/testerror.sh.in +++ b/test/testerror.sh.in @@ -71,6 +71,8 @@ TEST() { -e 's/line [0-9]*/line (number)/' \ -e 's/v[1-9]*\.[0-9]*\./version (number)\./' \ -e 's/[1-9]*\.[0-9]*\.[0-9]*[^)]*/version (number)/' \ + -e 's/H5Eget_auto[1-2]*/H5Eget_auto(1 or 2)/' \ + -e 's/H5Eset_auto[1-2]*/H5Eset_auto(1 or 2)/' \ $actual_err > $actual_ext cat $actual_ext >> $actual diff --git a/test/testfiles/err_compat_1 b/test/testfiles/err_compat_1 index 032e7bc..e2b37ab 100644 --- a/test/testfiles/err_compat_1 +++ b/test/testfiles/err_compat_1 @@ -1,7 +1,7 @@ ############################# Expected output for err_compat ############################# -Testing error API based on data I/O All error API tests passed. +Testing error API H5Eset/get_auto Testing error API based on data I/O All error API tests passed. This program tests the Error API compatible with HDF5 version (number). There're supposed to be some error messages ********* Print error stack in HDF5 default way ********* HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): @@ -15,10 +15,46 @@ HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): minor: Bad value HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): + #000: (file name) line (number) in H5Dcreate2(): not a location ID + major: Invalid arguments to routine + minor: Inappropriate type + #001: (file name) line (number) in H5G_loc(): invalid object ID + major: Invalid arguments to routine + minor: Bad value + +********* Print error stack in customized way ********* + error #000: (file name) in H5G_loc(): line (number) + major: Invalid arguments to routine + minor: Bad value + error #001: (file name) in H5Dcreate2(): line (number) + major: Invalid arguments to routine + minor: Inappropriate type + +********* Print error stack in customized way ********* + error #000: (file name) in H5Eget_auto(1 or 2)(): line (number) + major: Error API + minor: Can't get value + +********* Print error stack in customized way ********* + error #000: (file name) in H5G_loc(): line (number) + major: Invalid arguments to routine + minor: Bad value + error #001: (file name) in H5Dcreate2(): line (number) + major: Invalid arguments to routine + minor: Inappropriate type +HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): + #000: (file name) line (number) in H5Dcreate2(): not a location ID + major: Invalid arguments to routine + minor: Inappropriate type + #001: (file name) line (number) in H5G_loc(): invalid object ID + major: Invalid arguments to routine + minor: Bad value + +HDF5-DIAG: Error detected in HDF5 (version (number)) thread (IDs): #000: (file name) line (number) in main(): Error test failed major: Error API minor: Unrecognized message - #001: (file name) line (number) in test_error(): H5Dwrite shouldn't succeed + #001: (file name) line (number) in test_error2(): H5Dwrite shouldn't succeed major: Error API minor: Write failed #002: (file name) line (number) in H5Dwrite(): not a dataset diff --git a/testpar/t_mdset.c b/testpar/t_mdset.c index 51f06d7..80e2abf 100644 --- a/testpar/t_mdset.c +++ b/testpar/t_mdset.c @@ -1671,6 +1671,518 @@ void io_mode_confusion(void) #undef N +/* + * At present, the object header code maintains an image of its on disk + * representation, which is updates as necessary instead of generating on + * request. + * + * Prior to the fix that this test in designed to verify, the image of the + * on disk representation was only updated on flush -- not when the object + * header was marked clean. + * + * This worked perfectly well as long as all writes of a given object + * header were written from a single process. However, with the implementation + * of round robin metadata data writes in parallel HDF5, this is no longer + * the case -- it is possible for a given object header to be flushed from + * several different processes, with the object header simply being marked + * clean in all other processes on each flush. This resulted in NULL or + * out of data object header information being written to disk. + * + * To repair this, I modified the object header code to update its + * on disk image both on flush on when marked clean. + * + * This test is directed at verifying that the fix performs as expected. + * + * The test functions by creating a HDF5 file with several small datasets, + * and then flushing the file. This should result of at least one of + * the associated object headers being flushed by a process other than + * process 0. + * + * Then for each data set, add an attribute and flush the file again. + * + * Close the file and re-open it. + * + * Open the each of the data sets in turn. If all opens are successful, + * the test passes. Otherwise the test fails. + * + * Note that this test will probably become irrelevent shortly, when we + * land the journaling modifications on the trunk -- at which point all + * cache clients will have to construct on disk images on demand. + * + * JRM -- 10/13/10 + * + * Changes: None. + */ + +#define NUM_DATA_SETS 4 +#define LOCAL_DATA_SIZE 4 +#define LARGE_ATTR_SIZE 256 + +void rr_obj_hdr_flush_confusion(void) +{ + const char * dataset_name[NUM_DATA_SETS] = + { + "dataset_0", + "dataset_1", + "dataset_2", + "dataset_3" + }; + const char * att_name[NUM_DATA_SETS] = + { + "attribute_0", + "attribute_1", + "attribute_2", + "attribute_3" + }; + const char * lg_att_name[NUM_DATA_SETS] = + { + "large_attribute_0", + "large_attribute_1", + "large_attribute_2", + "large_attribute_3" + }; + int i; + int j; + hid_t file_id = -1; + hid_t fapl_id = -1; + hid_t dxpl_id = -1; + hid_t att_id[NUM_DATA_SETS]; + hid_t att_space[NUM_DATA_SETS]; + hid_t lg_att_id[NUM_DATA_SETS]; + hid_t lg_att_space[NUM_DATA_SETS]; + hid_t disk_space[NUM_DATA_SETS]; + hid_t mem_space[NUM_DATA_SETS]; + hid_t dataset[NUM_DATA_SETS]; + hsize_t att_size[1]; + hsize_t lg_att_size[1]; + hsize_t disk_count[1]; + hsize_t disk_size[1]; + hsize_t disk_start[1]; + hsize_t mem_count[1]; + hsize_t mem_size[1]; + hsize_t mem_start[1]; + herr_t err; + double data[LOCAL_DATA_SIZE]; + double att[LOCAL_DATA_SIZE]; + double lg_att[LARGE_ATTR_SIZE]; + + /* MPI variables */ + int mpi_size; + int mpi_rank; + + /* test bed related variables */ + const char * fcn_name = "rr_obj_hdr_flush_confusion"; + const hbool_t verbose = FALSE; + const H5Ptest_param_t * pt; + char * filename; + + /* + * setup test bed related variables: + */ + + pt = GetTestParameters(); + filename = pt->name; + + MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank); + MPI_Comm_size(MPI_COMM_WORLD, &mpi_size); + + /* + * Set up file access property list with parallel I/O access + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: Setting up property list.\n", + mpi_rank, fcn_name); + + fapl_id = H5Pcreate(H5P_FILE_ACCESS); + VRFY((fapl_id != -1), "H5Pcreate(H5P_FILE_ACCESS) failed"); + + err = H5Pset_fapl_mpio(fapl_id, MPI_COMM_WORLD, MPI_INFO_NULL); + VRFY((err >= 0 ), "H5Pset_fapl_mpio() failed"); + + + /* + * Create a new file collectively and release property list identifier. + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: Creating new file \"%s\".\n", + mpi_rank, fcn_name, filename); + + file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl_id); + VRFY((file_id >= 0 ), "H5Fcreate() failed"); + + err = H5Pclose(fapl_id); + VRFY((err >= 0 ), "H5Pclose(fapl_id) failed"); + + + /* + * create the data sets. + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: Creating the datasets.\n", + mpi_rank, fcn_name); + + disk_size[0] = (hsize_t)(LOCAL_DATA_SIZE * mpi_size); + mem_size[0] = (hsize_t)(LOCAL_DATA_SIZE); + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + disk_space[i] = H5Screate_simple(1, disk_size, NULL); + VRFY((disk_space[i] >= 0), "H5Screate_simple(1) failed.\n"); + + dataset[i] = H5Dcreate(file_id, dataset_name[i], H5T_NATIVE_DOUBLE, + disk_space[i], H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT); + + VRFY((dataset[i] >= 0), "H5Dcreate(1) failed.\n"); + } + + + /* + * setup data transfer property list + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: Setting up dxpl.\n", mpi_rank, fcn_name); + + dxpl_id = H5Pcreate(H5P_DATASET_XFER); + VRFY((dxpl_id != -1), "H5Pcreate(H5P_DATASET_XFER) failed.\n"); + + err = H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE); + VRFY((err >= 0), + "H5Pset_dxpl_mpio(dxpl_id, H5FD_MPIO_COLLECTIVE) failed.\n"); + + /* + * write data to the data sets + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: Writing datasets.\n", mpi_rank, fcn_name); + + disk_count[0] = (hsize_t)(LOCAL_DATA_SIZE); + disk_start[0] = (hsize_t)(LOCAL_DATA_SIZE * mpi_rank); + mem_count[0] = (hsize_t)(LOCAL_DATA_SIZE); + mem_start[0] = (hsize_t)(0); + + for ( j = 0; j < LOCAL_DATA_SIZE; j++ ) { + + data[j] = (double)(mpi_rank + 1); + } + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + err = H5Sselect_hyperslab(disk_space[i], H5S_SELECT_SET, disk_start, + NULL, disk_count, NULL); + VRFY((err >= 0), "H5Sselect_hyperslab(1) failed.\n"); + + mem_space[i] = H5Screate_simple(1, mem_size, NULL); + VRFY((mem_space[i] >= 0), "H5Screate_simple(2) failed.\n"); + + err = H5Sselect_hyperslab(mem_space[i], H5S_SELECT_SET, + mem_start, NULL, mem_count, NULL); + VRFY((err >= 0), "H5Sselect_hyperslab(2) failed.\n"); + + err = H5Dwrite(dataset[i], H5T_NATIVE_DOUBLE, mem_space[i], + disk_space[i], dxpl_id, data); + VRFY((err >= 0), "H5Dwrite(1) failed.\n"); + + for ( j = 0; j < LOCAL_DATA_SIZE; j++ ) data[j] *= 10.0; + } + + /* + * close the data spaces + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: closing dataspaces.\n", mpi_rank, fcn_name); + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + err = H5Sclose(disk_space[i]); + VRFY((err >= 0), "H5Sclose(disk_space[i]) failed.\n"); + + err = H5Sclose(mem_space[i]); + VRFY((err >= 0), "H5Sclose(mem_space[i]) failed.\n"); + } + + /* + * flush the metadata cache + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: flushing metadata cache.\n", + mpi_rank, fcn_name); + err = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((err >= 0), "H5Fflush(1) failed.\n"); + + + /* + * write attributes to each dataset + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: writing attributes.\n", mpi_rank, fcn_name); + + att_size[0] = (hsize_t)(LOCAL_DATA_SIZE); + + for ( j = 0; j < LOCAL_DATA_SIZE; j++ ) { + + att[j] = (double)(j + 1); + } + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + att_space[i] = H5Screate_simple(1, att_size, NULL); + VRFY((att_space[i] >= 0), "H5Screate_simple(3) failed.\n"); + + att_id[i] = H5Acreate(dataset[i], att_name[i], H5T_NATIVE_DOUBLE, + att_space[i], H5P_DEFAULT, H5P_DEFAULT); + VRFY((att_id[i] >= 0), "H5Acreate(1) failed.\n"); + + + err = H5Awrite(att_id[i], H5T_NATIVE_DOUBLE, att); + VRFY((err >= 0), "H5Awrite(1) failed.\n"); + + for ( j = 0; j < LOCAL_DATA_SIZE; j++ ) { + + att[j] /= 10.0; + } + } + + /* + * close attribute IDs and spaces + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: closing attr ids and spaces .\n", + mpi_rank, fcn_name); + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + err = H5Sclose(att_space[i]); + VRFY((err >= 0), "H5Sclose(att_space[i]) failed.\n"); + + err = H5Aclose(att_id[i]); + VRFY((err >= 0), "H5Aclose(att_id[i]) failed.\n"); + } + + /* + * flush the metadata cache again + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: flushing metadata cache.\n", + mpi_rank, fcn_name); + err = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((err >= 0), "H5Fflush(2) failed.\n"); + + + /* + * write large attributes to each dataset + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: writing large attributes.\n", + mpi_rank, fcn_name); + + lg_att_size[0] = (hsize_t)(LARGE_ATTR_SIZE); + + for ( j = 0; j < LARGE_ATTR_SIZE; j++ ) { + + lg_att[j] = (double)(j + 1); + } + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + lg_att_space[i] = H5Screate_simple(1, lg_att_size, NULL); + VRFY((lg_att_space[i] >= 0), "H5Screate_simple(4) failed.\n"); + + lg_att_id[i] = H5Acreate(dataset[i], lg_att_name[i], H5T_NATIVE_DOUBLE, + lg_att_space[i], H5P_DEFAULT, H5P_DEFAULT); + VRFY((lg_att_id[i] >= 0), "H5Acreate(2) failed.\n"); + + + err = H5Awrite(lg_att_id[i], H5T_NATIVE_DOUBLE, lg_att); + VRFY((err >= 0), "H5Awrite(2) failed.\n"); + + for ( j = 0; j < LARGE_ATTR_SIZE; j++ ) { + + lg_att[j] /= 10.0; + } + } + + /* + * flush the metadata cache yet again to clean the object headers. + * + * This is an attempt to crate a situation where we have dirty + * object header continuation chunks, but clean opject headers + * to verify a speculative bug fix -- it doesn't seem to work, + * but I will leave the code in anyway, as the object header + * code is going to change a lot in the near future. + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: flushing metadata cache.\n", + mpi_rank, fcn_name); + err = H5Fflush(file_id, H5F_SCOPE_GLOBAL); + VRFY((err >= 0), "H5Fflush(3) failed.\n"); + + + /* + * write different large attributes to each dataset + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: writing different large attributes.\n", + mpi_rank, fcn_name); + + for ( j = 0; j < LARGE_ATTR_SIZE; j++ ) { + + lg_att[j] = (double)(j + 2); + } + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + err = H5Awrite(lg_att_id[i], H5T_NATIVE_DOUBLE, lg_att); + VRFY((err >= 0), "H5Awrite(2) failed.\n"); + + for ( j = 0; j < LARGE_ATTR_SIZE; j++ ) { + + lg_att[j] /= 10.0; + } + } + + + /* + * close large attribute IDs and spaces + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: closing large attr ids and spaces .\n", + mpi_rank, fcn_name); + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + err = H5Sclose(lg_att_space[i]); + VRFY((err >= 0), "H5Sclose(lg_att_space[i]) failed.\n"); + + err = H5Aclose(lg_att_id[i]); + VRFY((err >= 0), "H5Aclose(lg_att_id[i]) failed.\n"); + } + + + /* + * close the data sets + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: closing datasets .\n", mpi_rank, fcn_name); + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + err = H5Dclose(dataset[i]); + VRFY((err >= 0), "H5Dclose(dataset[i])1 failed.\n"); + } + + /* + * close the data transfer property list. + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: closing dxpl .\n", mpi_rank, fcn_name); + + err = H5Pclose(dxpl_id); + VRFY((err >= 0), "H5Pclose(dxpl_id) failed.\n"); + + + /* + * Close file. + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: closing file.\n", mpi_rank, fcn_name); + + err = H5Fclose(file_id); + VRFY((err >= 0 ), "H5Fclose(1) failed"); + + /* + * Must now open file and attempt to read data sets -- do this on process + * zero only. Test passes if we are able to do this, and fails otherwise. + */ + + if ( mpi_rank == 0 ) { + + /* + * Re-open the file + */ + if(verbose) + HDfprintf(stdout, "%0d:%s: re-opening file.\n", mpi_rank, fcn_name); + file_id = H5Fopen(filename, H5F_ACC_RDONLY, H5P_DEFAULT); + VRFY((file_id >= 0 ), "H5Fopen() failed"); + + /* + * Attempt to open the data sets + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: opening datasets.\n", + mpi_rank, fcn_name); + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + dataset[i] = -1; + } + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + dataset[i] = H5Dopen2(file_id, dataset_name[i], H5P_DEFAULT); + + if ( dataset[i] < 0 ) { + + nerrors++; + } + } + + /* + * Close the data sets + */ + + if(verbose ) + HDfprintf(stdout, "%0d:%s: closing datasets again.\n", + mpi_rank, fcn_name); + + for ( i = 0; i < NUM_DATA_SETS; i++ ) { + + if ( dataset[i] >= 0 ) { + + err = H5Dclose(dataset[i]); + VRFY((err >= 0), "H5Dclose(dataset[i])1 failed.\n"); + } + } + + /* + * Close the file + */ + if(verbose) + HDfprintf(stdout, "%0d:%s: closing file again.\n", + mpi_rank, fcn_name); + err = H5Fclose(file_id); + VRFY((err >= 0 ), "H5Fclose(1) failed"); + + } + + if(verbose ) + HDfprintf(stdout, "%0d:%s: Done.\n", mpi_rank, fcn_name); + + return; + +} /* rr_obj_hdr_flush_confusion() */ + +#undef NUM_DATA_SETS +#undef LOCAL_DATA_SIZE +#undef LARGE_ATTR_SIZE + /*============================================================================= * End of t_mdset.c *===========================================================================*/ diff --git a/testpar/testphdf5.c b/testpar/testphdf5.c index 0864b11..44415e4 100644 --- a/testpar/testphdf5.c +++ b/testpar/testphdf5.c @@ -317,6 +317,7 @@ int main(int argc, char **argv) H5Ptest_param_t ndsets_params, ngroups_params; H5Ptest_param_t collngroups_params; H5Ptest_param_t io_mode_confusion_params; + H5Ptest_param_t rr_obj_flush_confusion_params; /* Un-buffer the stdout and stderr */ setbuf(stderr, NULL); @@ -472,6 +473,12 @@ int main(int argc, char **argv) "I/O mode confusion test -- hangs quickly on failure", &io_mode_confusion_params); + rr_obj_flush_confusion_params.name = PARATESTFILE; + rr_obj_flush_confusion_params.count = 0; /* value not used */ + AddTest("rrobjflushconf", rr_obj_hdr_flush_confusion, NULL, + "round robin object header flush confusion test", + &rr_obj_flush_confusion_params); + AddTest("tldsc", lower_dim_size_comp_test, NULL, "test lower dim size comp in span tree to mpi derived type", diff --git a/testpar/testphdf5.h b/testpar/testphdf5.h index ba46e4d..609db05 100644 --- a/testpar/testphdf5.h +++ b/testpar/testphdf5.h @@ -237,6 +237,7 @@ void coll_irregular_simple_chunk_write(void); void coll_irregular_complex_chunk_read(void); void coll_irregular_complex_chunk_write(void); void io_mode_confusion(void); +void rr_obj_hdr_flush_confusion(void); void lower_dim_size_comp_test(void); void link_chunk_collective_io_test(void); void contig_hyperslab_dr_pio_test(void); diff --git a/tools/h5copy/CMakeLists.txt b/tools/h5copy/CMakeLists.txt index 17dd7a5..ccb1466 100644 --- a/tools/h5copy/CMakeLists.txt +++ b/tools/h5copy/CMakeLists.txt @@ -137,15 +137,12 @@ IF (BUILD_TESTING) COMMAND ${CMAKE_COMMAND} -E remove ./testfiles/${HDF_FILE1}.out.h5 - ./testfiles/${HDF_FILE1}.out.ls ./testfiles/${HDF_FILE1}.out.out ./testfiles/${HDF_FILE1}.out.out.err ./testfiles/${HDF_FILE2}.out.h5 - ./testfiles/${HDF_FILE2}.out.ls ./testfiles/${HDF_FILE2}.out.out ./testfiles/${HDF_FILE2}.out.out.err ./testfiles/${HDF_EXT_SRC_FILE}.out.h5 - ./testfiles/${HDF_EXT_SRC_FILE}.out.ls ./testfiles/${HDF_EXT_SRC_FILE}.out.out ./testfiles/${HDF_EXT_SRC_FILE}.out.out.err ) diff --git a/tools/h5copy/h5copy.c b/tools/h5copy/h5copy.c index eb16754..3760d6f 100644 --- a/tools/h5copy/h5copy.c +++ b/tools/h5copy/h5copy.c @@ -13,7 +13,7 @@ * access to either file, you may request a copy from help@hdfgroup.org. * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ - +#include "H5private.h" #include "h5tools.h" #include "h5tools_utils.h" #include @@ -54,9 +54,8 @@ static struct long_options l_opts[] = { static void leave(int ret) { - h5tools_close(); - - exit(ret); + h5tools_close(); + exit(ret); } @@ -76,7 +75,7 @@ leave(int ret) static void usage (void) { - fprintf(stdout, "\ + fprintf(stdout, "\ usage: h5copy [OPTIONS] [OBJECTS...]\n\ OBJECTS\n\ -i, --input input file name\n\ @@ -144,45 +143,45 @@ usage: h5copy [OPTIONS] [OBJECTS...]\n\ static int parse_flag(const char* str_flag, unsigned *flag) { - unsigned fla=0; - - if (strcmp(str_flag,"shallow")==0) - { - fla = H5O_COPY_SHALLOW_HIERARCHY_FLAG; - } - else if (strcmp(str_flag,"soft")==0) - { - fla = H5O_COPY_EXPAND_SOFT_LINK_FLAG; - } - else if (strcmp(str_flag,"ext")==0) - { - fla = H5O_COPY_EXPAND_EXT_LINK_FLAG; - } - else if (strcmp(str_flag,"ref")==0) - { - fla = H5O_COPY_EXPAND_REFERENCE_FLAG; - } - else if (strcmp(str_flag,"noattr")==0) - { - fla = H5O_COPY_WITHOUT_ATTR_FLAG; - } - else if (strcmp(str_flag,"allflags")==0) - { - fla = H5O_COPY_ALL; - } - else if (strcmp(str_flag,"nullmsg")==0) - { - fla = H5O_COPY_PRESERVE_NULL_FLAG; - } - else - { - error_msg("Error in input flag\n"); - return -1; - } - - *flag = (*flag) | fla; - - return 0; + unsigned fla=0; + + if (HDstrcmp(str_flag,"shallow")==0) + { + fla = H5O_COPY_SHALLOW_HIERARCHY_FLAG; + } + else if (HDstrcmp(str_flag,"soft")==0) + { + fla = H5O_COPY_EXPAND_SOFT_LINK_FLAG; + } + else if (HDstrcmp(str_flag,"ext")==0) + { + fla = H5O_COPY_EXPAND_EXT_LINK_FLAG; + } + else if (HDstrcmp(str_flag,"ref")==0) + { + fla = H5O_COPY_EXPAND_REFERENCE_FLAG; + } + else if (HDstrcmp(str_flag,"noattr")==0) + { + fla = H5O_COPY_WITHOUT_ATTR_FLAG; + } + else if (HDstrcmp(str_flag,"allflags")==0) + { + fla = H5O_COPY_ALL; + } + else if (HDstrcmp(str_flag,"nullmsg")==0) + { + fla = H5O_COPY_PRESERVE_NULL_FLAG; + } + else + { + error_msg("Error in input flag\n"); + return -1; + } + + *flag = (*flag) | fla; + + return 0; } /*------------------------------------------------------------------------- @@ -200,141 +199,147 @@ static int parse_flag(const char* str_flag, unsigned *flag) int main (int argc, const char *argv[]) { - hid_t fid_src=-1; - hid_t fid_dst=-1; - char *fname_src=NULL; - char *fname_dst=NULL; - char *oname_src=NULL; - char *oname_dst=NULL; - unsigned flag=0; - unsigned verbose=0; - unsigned parents=0; - hid_t ocpl_id = (-1); /* Object copy property list */ - hid_t lcpl_id = (-1); /* Link creation property list */ - char str_flag[20]; - int opt; - int li_ret; - h5tool_link_info_t linkinfo; - - h5tools_setprogname(PROGRAMNAME); - h5tools_setstatus(EXIT_SUCCESS); -/* initialize h5tools lib */ - h5tools_init(); - - /* Check for no command line parameters */ - if(argc == 1) { - usage(); - leave(EXIT_FAILURE); - } /* end if */ - - /* parse command line options */ - while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) - { - switch ((char)opt) - { - case 'd': - oname_dst = strdup(opt_arg); - break; - - case 'f': - /* validate flag */ - if (parse_flag(opt_arg,&flag)<0) - { - usage(); - leave(EXIT_FAILURE); - } - strcpy(str_flag,opt_arg); - break; - - case 'h': - usage(); - leave(EXIT_SUCCESS); - break; - - case 'i': - fname_src = strdup(opt_arg); - break; - - case 'o': - fname_dst = strdup(opt_arg); - break; - - case 'p': - parents = 1; - break; - - case 's': - oname_src = strdup(opt_arg); - break; - - case 'V': - print_version(h5tools_getprogname()); - leave(EXIT_SUCCESS); - break; - - case 'v': - verbose = 1; - break; - - default: - usage(); - leave(EXIT_FAILURE); - } - } - -/*------------------------------------------------------------------------- - * check for missing file/object names - *-------------------------------------------------------------------------*/ - - if (fname_src==NULL) - { - error_msg("Input file name missing\n"); - usage(); - leave(EXIT_FAILURE); - } - - if (fname_dst==NULL) - { - error_msg("Output file name missing\n"); - usage(); - leave(EXIT_FAILURE); - } - - if (oname_src==NULL) - { - error_msg("Source object name missing\n"); - usage(); - leave(EXIT_FAILURE); - } - - if (oname_dst==NULL) - { - error_msg("Destination object name missing\n"); - usage(); - leave(EXIT_FAILURE); - } - - -/*------------------------------------------------------------------------- - * open input file - *-------------------------------------------------------------------------*/ + hid_t fid_src=-1; + hid_t fid_dst=-1; + char *fname_src=NULL; + char *fname_dst=NULL; + char *oname_src=NULL; + char *oname_dst=NULL; + unsigned flag=0; + unsigned verbose=0; + unsigned parents=0; + hid_t ocpl_id = (-1); /* Object copy property list */ + hid_t lcpl_id = (-1); /* Link creation property list */ + char str_flag[20]; + int opt; + int li_ret; + h5tool_link_info_t linkinfo; + int i, len; + char *str_ptr=NULL; + + h5tools_setprogname(PROGRAMNAME); + h5tools_setstatus(EXIT_SUCCESS); + /* initialize h5tools lib */ + h5tools_init(); + + /* init linkinfo struct */ + HDmemset(&linkinfo, 0, sizeof(h5tool_link_info_t)); + + /* Check for no command line parameters */ + if(argc == 1) + { + usage(); + leave(EXIT_FAILURE); + } /* end if */ - fid_src = h5tools_fopen(fname_src, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, 0); + /* parse command line options */ + while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) + { + switch ((char)opt) + { + case 'd': + oname_dst = HDstrdup(opt_arg); + break; + + case 'f': + /* validate flag */ + if (parse_flag(opt_arg,&flag)<0) + { + usage(); + leave(EXIT_FAILURE); + } + HDstrcpy(str_flag,opt_arg); + break; + + case 'h': + usage(); + leave(EXIT_SUCCESS); + break; + + case 'i': + fname_src = HDstrdup(opt_arg); + break; + + case 'o': + fname_dst = HDstrdup(opt_arg); + break; + + case 'p': + parents = 1; + break; + + case 's': + oname_src = HDstrdup(opt_arg); + break; + + case 'V': + print_version(h5tools_getprogname()); + leave(EXIT_SUCCESS); + break; + + case 'v': + verbose = 1; + break; + + default: + usage(); + leave(EXIT_FAILURE); + } + } /* end of while */ /*------------------------------------------------------------------------- - * test for error in opening input file + * check for missing file/object names *-------------------------------------------------------------------------*/ - if (fid_src==-1) - { - error_msg("Could not open input file <%s>...Exiting\n", fname_src); - if (fname_src) - free(fname_src); - leave(EXIT_FAILURE); - } -/*------------------------------------------------------------------------- - * open output file - *-------------------------------------------------------------------------*/ + if (fname_src==NULL) + { + error_msg("Input file name missing\n"); + usage(); + leave(EXIT_FAILURE); + } + + if (fname_dst==NULL) + { + error_msg("Output file name missing\n"); + usage(); + leave(EXIT_FAILURE); + } + + if (oname_src==NULL) + { + error_msg("Source object name missing\n"); + usage(); + leave(EXIT_FAILURE); + } + + if (oname_dst==NULL) + { + error_msg("Destination object name missing\n"); + usage(); + leave(EXIT_FAILURE); + } + + + /*------------------------------------------------------------------------- + * open input file + *-------------------------------------------------------------------------*/ + + fid_src = h5tools_fopen(fname_src, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, 0); + + /*------------------------------------------------------------------------- + * test for error in opening input file + *-------------------------------------------------------------------------*/ + if (fid_src==-1) + { + error_msg("Could not open input file <%s>...Exiting\n", fname_src); + if (fname_src) + HDfree(fname_src); + leave(EXIT_FAILURE); + } + + /*------------------------------------------------------------------------- + * open output file + *-------------------------------------------------------------------------*/ /* Attempt to open an existing HDF5 file first */ fid_dst = h5tools_fopen(fname_dst, H5F_ACC_RDWR, H5P_DEFAULT, NULL, NULL, 0); @@ -344,49 +349,46 @@ main (int argc, const char *argv[]) if(fid_dst < 0) fid_dst = H5Fcreate(fname_dst, H5F_ACC_EXCL, H5P_DEFAULT, H5P_DEFAULT); -/*------------------------------------------------------------------------- - * test for error in opening output file - *-------------------------------------------------------------------------*/ - if (fid_dst==-1) - { - error_msg("Could not open output file <%s>...Exiting\n", fname_dst); - if (fname_src) - free(fname_src); - if (fname_dst) - free(fname_dst); - leave(EXIT_FAILURE); - } - -/*------------------------------------------------------------------------- - * print some info - *-------------------------------------------------------------------------*/ - - if (verbose) - { - printf("Copying file <%s> and object <%s> to file <%s> and object <%s>\n", - fname_src, - oname_src, - fname_dst, - oname_dst); - if (flag) - printf("Using %s flag\n", str_flag); - } - - -/*------------------------------------------------------------------------- - * create property lists for copy - *-------------------------------------------------------------------------*/ - - /* create property to pass copy options */ - if ( (ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0) - goto error; + /*------------------------------------------------------------------------- + * test for error in opening output file + *-------------------------------------------------------------------------*/ + if (fid_dst==-1) + { + error_msg("Could not open output file <%s>...Exiting\n", fname_dst); + if (fname_src) + HDfree(fname_src); + if (fname_dst) + HDfree(fname_dst); + leave(EXIT_FAILURE); + } + + /*------------------------------------------------------------------------- + * print some info + *-------------------------------------------------------------------------*/ + + if (verbose) + { + printf("Copying file <%s> and object <%s> to file <%s> and object <%s>\n", + fname_src, oname_src, fname_dst, oname_dst); + if (flag) + printf("Using %s flag\n", str_flag); + } + + + /*------------------------------------------------------------------------- + * create property lists for copy + *-------------------------------------------------------------------------*/ + + /* create property to pass copy options */ + if ( (ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0) + goto error; - /* set options for object copy */ - if (flag) - { - if ( H5Pset_copy_object(ocpl_id, flag) < 0) - goto error; - } + /* set options for object copy */ + if (flag) + { + if ( H5Pset_copy_object(ocpl_id, flag) < 0) + goto error; + } /* Create link creation property list */ if((lcpl_id = H5Pcreate(H5P_LINK_CREATE)) < 0) { @@ -406,88 +408,107 @@ main (int argc, const char *argv[]) if(verbose) printf("%s: Creating parent groups\n", h5tools_getprogname()); } /* end if */ - -/*------------------------------------------------------------------------- - * do the copy - *-------------------------------------------------------------------------*/ - /* init linkinfo struct */ - memset(&linkinfo, 0, sizeof(h5tool_link_info_t)); + else /* error, if parent groups doesn't already exist in destination file */ + { + len = HDstrlen(oname_dst); + /* check if all the parents groups exist. skip root group */ + for (i = 1; i < len; i++) + { + if ('/'==oname_dst[i]) + { + str_ptr = (char*)HDcalloc((size_t)i+1, sizeof(char)); + HDstrncpy (str_ptr, oname_dst, (size_t)i); + str_ptr[i]='\0'; + if (H5Lexists(fid_dst, str_ptr, H5P_DEFAULT) <= 0) + { + error_msg("group <%s> doesn't exist. Use -p to create parent groups.\n", str_ptr); + HDfree(str_ptr); + goto error; + } + HDfree(str_ptr); + } + } + } + + /*------------------------------------------------------------------------- + * do the copy + *-------------------------------------------------------------------------*/ - if(verbose) - linkinfo.opt.msg_mode = 1; + if(verbose) + linkinfo.opt.msg_mode = 1; - li_ret = H5tools_get_symlink_info(fid_src, oname_src, &linkinfo, 1); - if (li_ret == 0) /* dangling link */ - { - if(H5Lcopy(fid_src, oname_src, - fid_dst, oname_dst, - H5P_DEFAULT, H5P_DEFAULT) < 0) - goto error; - } - else /* valid link */ - { - if (H5Ocopy(fid_src, /* Source file or group identifier */ - oname_src, /* Name of the source object to be copied */ - fid_dst, /* Destination file or group identifier */ - oname_dst, /* Name of the destination object */ - ocpl_id, /* Object copy property list */ - lcpl_id)<0) /* Link creation property list */ - goto error; - } - - /* free link info path */ - if (linkinfo.trg_path) - free(linkinfo.trg_path); - - /* close propertis */ - if(H5Pclose(ocpl_id)<0) - goto error; - if(H5Pclose(lcpl_id)<0) - goto error; - - /* close files */ - if (H5Fclose(fid_src)<0) - goto error; - if (H5Fclose(fid_dst)<0) - goto error; - - if (fname_src) - free(fname_src); - if (fname_dst) - free(fname_dst); - if (oname_dst) - free(oname_dst); - if (oname_src) - free(oname_src); - - h5tools_close(); - - return EXIT_SUCCESS; + li_ret = H5tools_get_symlink_info(fid_src, oname_src, &linkinfo, 1); + if (li_ret == 0) /* dangling link */ + { + if(H5Lcopy(fid_src, oname_src, + fid_dst, oname_dst, + H5P_DEFAULT, H5P_DEFAULT) < 0) + goto error; + } + else /* valid link */ + { + if (H5Ocopy(fid_src, /* Source file or group identifier */ + oname_src, /* Name of the source object to be copied */ + fid_dst, /* Destination file or group identifier */ + oname_dst, /* Name of the destination object */ + ocpl_id, /* Object copy property list */ + lcpl_id)<0) /* Link creation property list */ + goto error; + } + + /* free link info path */ + if (linkinfo.trg_path) + HDfree(linkinfo.trg_path); + + /* close propertis */ + if(H5Pclose(ocpl_id)<0) + goto error; + if(H5Pclose(lcpl_id)<0) + goto error; + + /* close files */ + if (H5Fclose(fid_src)<0) + goto error; + if (H5Fclose(fid_dst)<0) + goto error; + + if (fname_src) + HDfree(fname_src); + if (fname_dst) + HDfree(fname_dst); + if (oname_dst) + HDfree(oname_dst); + if (oname_src) + HDfree(oname_src); + + h5tools_close(); + + return EXIT_SUCCESS; error: - printf("Error in copy...Exiting\n"); + printf("Error in copy...Exiting\n"); - /* free link info path */ - if (linkinfo.trg_path) - free(linkinfo.trg_path); + /* free link info path */ + if (linkinfo.trg_path) + HDfree(linkinfo.trg_path); H5E_BEGIN_TRY { - H5Pclose(ocpl_id); - H5Pclose(lcpl_id); - H5Fclose(fid_src); - H5Fclose(fid_dst); - } H5E_END_TRY; - if (fname_src) - free(fname_src); - if (fname_dst) - free(fname_dst); - if (oname_dst) - free(oname_dst); - if (oname_src) - free(oname_src); - - h5tools_close(); - - return EXIT_FAILURE; + H5Pclose(ocpl_id); + H5Pclose(lcpl_id); + H5Fclose(fid_src); + H5Fclose(fid_dst); + } H5E_END_TRY; + if (fname_src) + HDfree(fname_src); + if (fname_dst) + HDfree(fname_dst); + if (oname_dst) + HDfree(oname_dst); + if (oname_src) + HDfree(oname_src); + + h5tools_close(); + + return EXIT_FAILURE; } diff --git a/tools/h5copy/testfiles/h5copy_misc1.out b/tools/h5copy/testfiles/h5copy_misc1.out new file mode 100644 index 0000000..370b1a5 --- /dev/null +++ b/tools/h5copy/testfiles/h5copy_misc1.out @@ -0,0 +1,3 @@ +Copying file and object to file <./testfiles/h5copytst.out.h5> and object +Error in copy...Exiting +h5copy error: group doesn't exist. Use -p to create parent groups. diff --git a/tools/h5copy/testh5copy.sh b/tools/h5copy/testh5copy.sh index 5ddb83e..1ce6a31 100644 --- a/tools/h5copy/testh5copy.sh +++ b/tools/h5copy/testh5copy.sh @@ -91,6 +91,15 @@ VERIFY_H5LS() echo "Verifying h5ls file structure $* $SPACES" | cut -c1-70 | tr -d '\012' } +# Print a line-line message left justified in a field of 70 characters +# beginning with the word "Verifying". +# +VERIFY_OUTPUT() +{ + SPACES=" " + echo "Verifying output files $* $SPACES" | cut -c1-70 | tr -d '\012' +} + # Run a test and print PASS or *FAIL*. If h5copy can complete # with exit status 0, consider it pass. If a test fails then increment # the `nerrors' global variable. @@ -143,44 +152,74 @@ TOOLTEST() } +# Compare the two text files +# PASS if same +# FAIL if different, and show the diff +# +# Assumed arguments: +# $1 is text file1 (expected output) +# $2 is text file2 (actual output) +CMP_OUTPUT() +{ + expect=$1 + actual=$2 + + VERIFY_OUTPUT $@ + if [ ! -f $expect ]; then + # Create the expect file if it doesn't yet exist. + echo " CREATED" + cp $actual $expect + elif $CMP $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected output differs from actual output" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/ /' + fi +} + TOOLTEST_FAIL() { - runh5diff=yes - if [ "$1" = -i ]; then + expectout="$INDIR/$1" + actualout="$OUTDIR/$1.out" + actualerr="$OUTDIR/$1.err" + shift + if [ "$1" = -i ]; then inputfile=$2 - else - runh5diff=no - fi - if [ "$3" = -o ]; then + fi + if [ "$3" = -o ]; then outputfile=$4 - else - runh5diff=no - fi - + fi + TESTING $H5COPY $5 $6 $7 $8 $9 ( - echo "#############################" - echo " output for '$H5COPY $@'" - echo "#############################" + #echo "#############################" + #echo " output for '$H5COPY $@'" + #echo "#############################" $RUNSERIAL $H5COPY_BIN $@ - ) > output.out + ) > $actualout 2> $actualerr RET=$? if [ $RET != 0 ]; then + echo " PASSED" + # Verifying output text from h5copy + if [ "$expectout" != "SKIP" ]; then + # combine stderr to stdout to compare the output at once. + # We may seperate stdout and stderr later. + cat $actualerr >> $actualout + CMP_OUTPUT $expectout $actualout + fi + else echo "*FAILED*" echo "failed result is:" - cat output.out + cat $actualout nerrors="`expr $nerrors + 1`" - else - echo " PASSED" - - # Clean up output file - if test -z "$HDF5_NOCLEANUP"; then - rm -f output.out - fi fi - if [ $runh5diff != no ]; then - H5DIFFTEST_FAIL $inputfile $outputfile $7 $9 + + # Clean up output file + if test -z "$HDF5_NOCLEANUP"; then + rm -f $actualout $actualerr fi } @@ -389,6 +428,28 @@ COPY_EXT_LINKS() fi } +# Test misc. +# +# Assumed arguments: +# +TEST_MISC() +{ + TESTFILE="$HDF_FILE1" + FILEOUT="$OUTDIR/`basename $HDF_FILE1 .h5`.out.h5" + + # Remove any output file left over from previous test run + rm -f $FILEOUT + + echo "Test copying object into group which doesn't exist, without -p" + TOOLTEST_FAIL h5copy_misc1.out -v -i $TESTFILE -o $FILEOUT -s /simple -d /g1/g2/simple + + # Remove output file created, if the "no cleanup" environment variable is + # not defined + if test -z "$HDF5_NOCLEANUP"; then + rm -f $FILEOUT + fi +} + ############################################################################## ### T H E T E S T S ### ############################################################################## @@ -396,6 +457,7 @@ COPY_EXT_LINKS() COPY_OBJECTS COPY_REFERENCES COPY_EXT_LINKS +TEST_MISC # Add newline for nicer formatting echo " " diff --git a/tools/h5stat/testfiles/h5stat_tsohm.ddl b/tools/h5stat/testfiles/h5stat_tsohm.ddl index c7ed5f4..6ee8aa8 100644 --- a/tools/h5stat/testfiles/h5stat_tsohm.ddl +++ b/tools/h5stat/testfiles/h5stat_tsohm.ddl @@ -31,7 +31,7 @@ File space information for file metadata (in bytes): Shared Messages: Header: 38 B-tree/List: 550 - Heap: 1316 + Heap: 1279 Free-space managers: Header: 0 Amount of free space: 0 @@ -71,7 +71,7 @@ Dataset datatype information: # of unique datatypes used by datasets: 1 Dataset datatype #0: Count (total/named) = (3/0) - Size (desc./elmt) = (14/4) + Size (desc./elmt) = (14/8) Total dataset datatype count: 3 Small # of attributes: Total # of objects with small # of attributes: 0 @@ -85,8 +85,8 @@ Free-space section bins: Total # of sections: 0 File space management strategy: H5F_FILE_SPACE_ALL Summary of file space information: - File metadata: 3887 bytes + File metadata: 3850 bytes Raw data: 0 bytes Amount/Percent of tracked free space: 0 bytes/0.0% Unaccounted space: 0 bytes -Total space: 3887 bytes +Total space: 3850 bytes diff --git a/tools/h5stat/testfiles/h5stat_tsohm.h5 b/tools/h5stat/testfiles/h5stat_tsohm.h5 index 193c34f..e6c89f6 100644 Binary files a/tools/h5stat/testfiles/h5stat_tsohm.h5 and b/tools/h5stat/testfiles/h5stat_tsohm.h5 differ diff --git a/tools/h5stat/testh5stat.sh.in b/tools/h5stat/testh5stat.sh.in index 2a09f9f..c6c377e 100644 --- a/tools/h5stat/testh5stat.sh.in +++ b/tools/h5stat/testh5stat.sh.in @@ -127,7 +127,7 @@ TOOLTEST h5stat_filters-UD.ddl -D h5stat_filters.h5 TOOLTEST h5stat_filters-UT.ddl -T h5stat_filters.h5 # # h5stat_tsohm.h5 is a copy of ../../../test/tsohm.h5 generated by tsohm.c -# as of release 1.8.0-alpha4 +# as of release 1.8.7-snap0 (on a 64-bit machine) TOOLTEST h5stat_tsohm.ddl h5stat_tsohm.h5 # h5stat_newgrat.h5 is generated by h5stat_gentest.c TOOLTEST h5stat_newgrat.ddl h5stat_newgrat.h5 diff --git a/tools/testfiles/file_space.h5 b/tools/testfiles/file_space.h5 index de5837a..425d0c2 100644 Binary files a/tools/testfiles/file_space.h5 and b/tools/testfiles/file_space.h5 differ diff --git a/vms/src/h5pubconf.h b/vms/src/h5pubconf.h index 5a5c8c4..42c5561 100644 --- a/vms/src/h5pubconf.h +++ b/vms/src/h5pubconf.h @@ -489,13 +489,13 @@ #define H5_PACKAGE_NAME "HDF5" /* Define to the full name and version of this package. */ -#define H5_PACKAGE_STRING "HDF5 1.9.76-FA_a5" +#define H5_PACKAGE_STRING "HDF5 1.9.77-FA_a5" /* Define to the one symbol short name of this package. */ #define H5_PACKAGE_TARNAME "hdf5" /* Define to the version of this package. */ -#define H5_PACKAGE_VERSION "1.9.76-FA_a5" +#define H5_PACKAGE_VERSION "1.9.77-FA_a5" /* Width for printf() for type `long long' or `__int64', use `ll' */ #define H5_PRINTF_LL_WIDTH "ll" @@ -651,7 +651,7 @@ /* #undef H5_USING_MEMCHECKER */ /* Version number of package */ -#define H5_VERSION "1.9.76-FA_a5" +#define H5_VERSION "1.9.77-FA_a5" /* Define if vsnprintf() returns the correct value for formatted strings that don't fit into size allowed */ diff --git a/windows/src/H5pubconf.h b/windows/src/H5pubconf.h index 2cc3348..df25005 100755 --- a/windows/src/H5pubconf.h +++ b/windows/src/H5pubconf.h @@ -478,13 +478,13 @@ #define H5_PACKAGE_NAME "HDF5" /* Define to the full name and version of this package. */ -#define H5_PACKAGE_STRING "HDF5 1.9.76-FA_a5" +#define H5_PACKAGE_STRING "HDF5 1.9.77-FA_a5" /* Define to the one symbol short name of this package. */ #define H5_PACKAGE_TARNAME "hdf5" /* Define to the version of this package. */ -#define H5_PACKAGE_VERSION "1.9.76-FA_a5" +#define H5_PACKAGE_VERSION "1.9.77-FA_a5" /* Width for printf() for type `long long' or `__int64', use `ll' */ #define H5_PRINTF_LL_WIDTH "I64" @@ -641,7 +641,7 @@ /* #undef H5_USING_MEMCHECKER */ /* Version number of package */ -#define H5_VERSION "1.9.76-FA_a5" +#define H5_VERSION "1.9.77-FA_a5" /* Define if vsnprintf() returns the correct value for formatted strings that don't fit into size allowed */ diff --git a/windows/tools/h5copy/testh5copy.bat b/windows/tools/h5copy/testh5copy.bat index 26f1b2c..495081c 100644 --- a/windows/tools/h5copy/testh5copy.bat +++ b/windows/tools/h5copy/testh5copy.bat @@ -84,7 +84,7 @@ rem for %%a in (%*) do ( if %%a neq PASSED ( if %%a neq *FAILED* ( - set verify_msg=!test_msg! %%~nxa + set verify_msg=!verify_msg! %%~nxa ) ) ) set verify_msg=%verify_msg% @@ -100,7 +100,7 @@ rem for %%a in (%*) do ( if %%a neq PASSED ( if %%a neq *FAILED* ( - set verifyh5ls_msg=!test_msg! %%~nxa + set verifyh5ls_msg=!verifyh5ls_msg! %%~nxa ) ) ) set verifyh5ls_msg=%verifyh5ls_msg% @@ -330,17 +330,17 @@ rem call :tooltest -i %testfile% -o %fileout% -v -s /grp_dsets/simple -d /grp_dsets/simple_group echo.Test copying ^& renaming group - call :tooltest_fail -i %testfile% -o %fileout% -v -s grp_dsets -d grp_rename + call :tooltest -i %testfile% -o %fileout% -v -s grp_dsets -d grp_rename echo.Test copying 'full' group hierarchy into group in destination file - call :tooltest_fail -i %testfile% -o %fileout% -v -s grp_dsets -d /grp_rename/grp_dsets + call :tooltest -i %testfile% -o %fileout% -v -s grp_dsets -d /grp_rename/grp_dsets echo.Test copying objects into group hier. that doesn't exist yet in destination file call :tooltest -i %testfile% -o %fileout% -vp -s simple -d /A/B1/simple call :tooltest -i %testfile% -o %fileout% -vp -s simple -d /A/B2/simple2 call :tooltest -i %testfile% -o %fileout% -vp -s /grp_dsets/simple -d /C/D/simple - call :tooltest_fail -i %testfile% -o %fileout% -vp -s /grp_dsets -d /E/F/grp_dsets - call :tooltest_fail -i %testfile% -o %fileout% -vp -s /grp_nested -d /G/H/grp_nested + call :tooltest -i %testfile% -o %fileout% -vp -s /grp_dsets -d /E/F/grp_dsets + call :tooltest -i %testfile% -o %fileout% -vp -s /grp_nested -d /G/H/grp_nested rem Verify that the file created above is correct call :h5lstest %fileout% -- cgit v0.12