diff options
90 files changed, 6942 insertions, 5307 deletions
diff --git a/CMakeInstallation.cmake b/CMakeInstallation.cmake index 74bf85f..c5db4ea 100644 --- a/CMakeInstallation.cmake +++ b/CMakeInstallation.cmake @@ -79,7 +79,7 @@ configure_package_config_file ( #----------------------------------------------------------------------------- if (NOT HDF5_EXTERNALLY_CONFIGURED) configure_file ( - ${HDF_RESOURCES_DIR}/FindHDF5.cmake.in + ${HDF_RESOURCES_DIR}/FindHDF5.cmake.in ${HDF5_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/FindHDF5${HDF_PACKAGE_EXT}.cmake @ONLY ) install ( @@ -134,7 +134,7 @@ else (H5_WORDS_BIGENDIAN) set (BYTESEX little-endian) endif (H5_WORDS_BIGENDIAN) configure_file ( - ${HDF_RESOURCES_DIR}/libhdf5.settings.cmake.in + ${HDF_RESOURCES_DIR}/libhdf5.settings.cmake.in ${HDF5_BINARY_DIR}/libhdf5.settings @ONLY ) install ( @@ -167,7 +167,7 @@ install ( option (HDF5_PACK_EXAMPLES "Package the HDF5 Library Examples Compressed File" OFF) if (HDF5_PACK_EXAMPLES) configure_file ( - ${HDF_RESOURCES_DIR}/HDF518_Examples.cmake.in + ${HDF_RESOURCES_DIR}/HDF518_Examples.cmake.in ${HDF5_BINARY_DIR}/HDF518_Examples.cmake @ONLY ) install ( @@ -182,7 +182,7 @@ if (HDF5_PACK_EXAMPLES) install ( DIRECTORY ${HDF5_BINARY_DIR}/HDF5Examples-0.1.1-Source DESTINATION ${HDF5_INSTALL_DATA_DIR} - USE_SOURCE_PERMISSIONS + USE_SOURCE_PERMISSIONS COMPONENT hdfdocuments ) install ( @@ -298,13 +298,13 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES) set (CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_VENDOR}/${CPACK_PACKAGE_NAME}/${CPACK_PACKAGE_VERSION}") set (CPACK_PACKAGE_ICON "${HDF_RESOURCES_EXT_DIR}/hdf.bmp") - set (CPACK_GENERATOR "TGZ") + set (CPACK_GENERATOR "TGZ") if (WIN32) - set (CPACK_GENERATOR "ZIP") + set (CPACK_GENERATOR "ZIP") - if (NSIS_EXECUTABLE) - list (APPEND CPACK_GENERATOR "NSIS") - endif (NSIS_EXECUTABLE) + if (NSIS_EXECUTABLE) + list (APPEND CPACK_GENERATOR "NSIS") + endif (NSIS_EXECUTABLE) # Installers for 32- vs. 64-bit CMake: # - Root install directory (displayed to end user at installer-run time) # - "NSIS package/display name" (text used in the installer GUI) @@ -327,10 +327,10 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES) set (CPACK_PACKAGE_INSTALL_DIRECTORY "${CPACK_PACKAGE_VENDOR}\\\\${CPACK_PACKAGE_NAME}\\\\${CPACK_PACKAGE_VERSION}") set (CPACK_NSIS_CONTACT "${HDF5_PACKAGE_BUGREPORT}") set (CPACK_NSIS_MODIFY_PATH ON) - - if (WIX_EXECUTABLE) - list (APPEND CPACK_GENERATOR "WIX") - endif (WIX_EXECUTABLE) + + if (WIX_EXECUTABLE) + list (APPEND CPACK_GENERATOR "WIX") + endif (WIX_EXECUTABLE) #WiX variables set (CPACK_WIX_UNINSTALL "1") # .. variable:: CPACK_WIX_LICENSE_RTF @@ -375,30 +375,24 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES) set(CPACK_WIX_PATCH_FILE "${HDF_RESOURCES_DIR}/patch.xml") endif (BUILD_SHARED_LIBS) elseif (APPLE) - list (APPEND CPACK_GENERATOR "DragNDrop") + list (APPEND CPACK_GENERATOR "DragNDrop") set (CPACK_COMPONENTS_ALL_IN_ONE_PACKAGE ON) set (CPACK_PACKAGING_INSTALL_PREFIX "/${CPACK_PACKAGE_INSTALL_DIRECTORY}") set (CPACK_PACKAGE_ICON "${HDF_RESOURCES_EXT_DIR}/hdf.icns") - option (HDF5_PACK_MACOSX_BUNDLE "Package the HDF5 Library in a Bundle" OFF) - if (HDF5_PACK_MACOSX_BUNDLE) - list (APPEND CPACK_GENERATOR "Bundle") + option (HDF5_PACK_MACOSX_FRAMEWORK "Package the HDF5 Library in a Frameworks" OFF) + if (HDF5_PACK_MACOSX_FRAMEWORK AND HDF5_BUILD_FRAMEWORKS) set (CPACK_BUNDLE_NAME "${HDF5_PACKAGE_STRING}") set (CPACK_BUNDLE_LOCATION "/") # make sure CMAKE_INSTALL_PREFIX ends in / set (CMAKE_INSTALL_PREFIX "/${CPACK_BUNDLE_NAME}.framework/Versions/${CPACK_PACKAGE_VERSION}/${CPACK_PACKAGE_NAME}/") set (CPACK_BUNDLE_ICON "${HDF_RESOURCES_EXT_DIR}/hdf.icns") set (CPACK_BUNDLE_PLIST "${HDF5_BINARY_DIR}/CMakeFiles/Info.plist") - set (CPACK_APPLE_GUI_INFO_STRING "HDF5 (Hierarchical Data Format 5) Software Library and Utilities") - set (CPACK_APPLE_GUI_COPYRIGHT "Copyright © 2006-2015 by The HDF Group. All rights reserved.") set (CPACK_SHORT_VERSION_STRING "${CPACK_PACKAGE_VERSION}") - set (CPACK_APPLE_GUI_BUNDLE_NAME "${HDF5_PACKAGE_STRING}") - set (CPACK_APPLE_GUI_VERSION_STRING "${CPACK_PACKAGE_VERSION_STRING}") - set (CPACK_APPLE_GUI_SHORT_VERSION_STRING "${CPACK_PACKAGE_VERSION}") #----------------------------------------------------------------------------- # Configure the Info.plist file for the install bundle #----------------------------------------------------------------------------- configure_file ( - ${HDF_RESOURCES_EXT_DIR}/CPack.Info.plist.in + ${HDF_RESOURCES_DIR}/CPack.Info.plist.in ${HDF5_BINARY_DIR}/CMakeFiles/Info.plist @ONLY ) configure_file ( @@ -411,19 +405,18 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES) ) install ( FILES ${HDF5_BINARY_DIR}/CMakeFiles/PkgInfo - ${HDF5_BINARY_DIR}/CMakeFiles/version.plist DESTINATION .. ) - endif (HDF5_PACK_MACOSX_BUNDLE) + endif (HDF5_PACK_MACOSX_FRAMEWORK AND HDF5_BUILD_FRAMEWORKS) else (WIN32) - list (APPEND CPACK_GENERATOR "STGZ") + list (APPEND CPACK_GENERATOR "STGZ") set (CPACK_PACKAGING_INSTALL_PREFIX "/${CPACK_PACKAGE_INSTALL_DIRECTORY}") set (CPACK_COMPONENTS_ALL_IN_ONE_PACKAGE ON) set (CPACK_DEBIAN_PACKAGE_SECTION "Libraries") set (CPACK_DEBIAN_PACKAGE_MAINTAINER "${HDF5_PACKAGE_BUGREPORT}") -# list (APPEND CPACK_GENERATOR "RPM") +# list (APPEND CPACK_GENERATOR "RPM") set (CPACK_RPM_PACKAGE_RELEASE "1") set (CPACK_RPM_COMPONENT_INSTALL ON) set (CPACK_RPM_PACKAGE_RELOCATABLE ON) @@ -431,7 +424,7 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES) set (CPACK_RPM_PACKAGE_GROUP "Development/Libraries") set (CPACK_RPM_PACKAGE_URL "${HDF5_PACKAGE_URL}") set (CPACK_RPM_PACKAGE_SUMMARY "HDF5 is a unique technology suite that makes possible the management of extremely large and complex data collections.") - set (CPACK_RPM_PACKAGE_DESCRIPTION + set (CPACK_RPM_PACKAGE_DESCRIPTION "The HDF5 technology suite includes: * A versatile data model that can represent very complex data objects and a wide variety of metadata. @@ -447,14 +440,14 @@ if (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES) The HDF5 data model, file format, API, library, and tools are open and distributed without charge. " ) - + #----------------------------------------------------------------------------- # Configure the spec file for the install RPM #----------------------------------------------------------------------------- # configure_file ("${HDF5_RESOURCES_DIR}/hdf5.spec.in" "${CMAKE_CURRENT_BINARY_DIR}/${HDF5_PACKAGE_NAME}.spec" @ONLY IMMEDIATE) # set (CPACK_RPM_USER_BINARY_SPECFILE "${CMAKE_CURRENT_BINARY_DIR}/${HDF5_PACKAGE_NAME}.spec") endif (WIN32) - + # By default, do not warn when built on machines using only VS Express: if (NOT DEFINED CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_NO_WARNINGS) set (CMAKE_INSTALL_SYSTEM_RUNTIME_LIBS_NO_WARNINGS ON) @@ -462,7 +455,7 @@ The HDF5 data model, file format, API, library, and tools are open and distribut include (InstallRequiredSystemLibraries) set (CPACK_INSTALL_CMAKE_PROJECTS "${HDF5_BINARY_DIR};HDF5;ALL;/") - + if (HDF5_PACKAGE_EXTLIBS) if (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN" OR HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ") if (ZLIB_FOUND AND ZLIB_USE_EXTERNAL) @@ -485,7 +478,7 @@ The HDF5 data model, file format, API, library, and tools are open and distribut endif (SZIP_FOUND AND SZIP_USE_EXTERNAL) endif (HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "SVN" OR HDF5_ALLOW_EXTERNAL_SUPPORT MATCHES "TGZ") endif (HDF5_PACKAGE_EXTLIBS) - + include (CPack) cpack_add_install_type(Full DISPLAY_NAME "Everything") @@ -511,121 +504,121 @@ The HDF5 data model, file format, API, library, and tools are open and distribut #--------------------------------------------------------------------------- # Now list the cpack commands #--------------------------------------------------------------------------- - CPACK_ADD_COMPONENT (libraries + cpack_add_component (libraries DISPLAY_NAME "HDF5 Libraries" GROUP Runtime INSTALL_TYPES Full Developer User ) - CPACK_ADD_COMPONENT (headers - DISPLAY_NAME "HDF5 Headers" + cpack_add_component (headers + DISPLAY_NAME "HDF5 Headers" DEPENDS libraries GROUP Development INSTALL_TYPES Full Developer ) - CPACK_ADD_COMPONENT (hdfdocuments + cpack_add_component (hdfdocuments DISPLAY_NAME "HDF5 Documents" GROUP Documents INSTALL_TYPES Full Developer ) - CPACK_ADD_COMPONENT (configinstall - DISPLAY_NAME "HDF5 CMake files" + cpack_add_component (configinstall + DISPLAY_NAME "HDF5 CMake files" DEPENDS libraries GROUP Development INSTALL_TYPES Full Developer User ) - + if (HDF5_BUILD_FORTRAN) - CPACK_ADD_COMPONENT (fortlibraries - DISPLAY_NAME "HDF5 Fortran Libraries" + cpack_add_component (fortlibraries + DISPLAY_NAME "HDF5 Fortran Libraries" DEPENDS libraries GROUP Runtime INSTALL_TYPES Full Developer User ) - CPACK_ADD_COMPONENT (fortheaders - DISPLAY_NAME "HDF5 Fortran Headers" + cpack_add_component (fortheaders + DISPLAY_NAME "HDF5 Fortran Headers" DEPENDS fortlibraries GROUP Development INSTALL_TYPES Full Developer ) endif (HDF5_BUILD_FORTRAN) - + if (HDF5_BUILD_CPP_LIB) - CPACK_ADD_COMPONENT (cpplibraries - DISPLAY_NAME "HDF5 C++ Libraries" + cpack_add_component (cpplibraries + DISPLAY_NAME "HDF5 C++ Libraries" DEPENDS libraries GROUP Runtime INSTALL_TYPES Full Developer User ) - CPACK_ADD_COMPONENT (cppheaders - DISPLAY_NAME "HDF5 C++ Headers" + cpack_add_component (cppheaders + DISPLAY_NAME "HDF5 C++ Headers" DEPENDS cpplibraries GROUP Development INSTALL_TYPES Full Developer ) endif (HDF5_BUILD_CPP_LIB) - + if (HDF5_BUILD_TOOLS) - CPACK_ADD_COMPONENT (toolsapplications - DISPLAY_NAME "HDF5 Tools Applications" + cpack_add_component (toolsapplications + DISPLAY_NAME "HDF5 Tools Applications" DEPENDS toolslibraries GROUP Applications INSTALL_TYPES Full Developer User ) - CPACK_ADD_COMPONENT (toolslibraries - DISPLAY_NAME "HDF5 Tools Libraries" + cpack_add_component (toolslibraries + DISPLAY_NAME "HDF5 Tools Libraries" DEPENDS libraries GROUP Runtime INSTALL_TYPES Full Developer User ) - CPACK_ADD_COMPONENT (toolsheaders - DISPLAY_NAME "HDF5 Tools Headers" + cpack_add_component (toolsheaders + DISPLAY_NAME "HDF5 Tools Headers" DEPENDS toolslibraries GROUP Development INSTALL_TYPES Full Developer ) endif (HDF5_BUILD_TOOLS) - + if (HDF5_BUILD_HL_LIB) - CPACK_ADD_COMPONENT (hllibraries - DISPLAY_NAME "HDF5 HL Libraries" + cpack_add_component (hllibraries + DISPLAY_NAME "HDF5 HL Libraries" DEPENDS libraries GROUP Runtime INSTALL_TYPES Full Developer User ) - CPACK_ADD_COMPONENT (hlheaders - DISPLAY_NAME "HDF5 HL Headers" + cpack_add_component (hlheaders + DISPLAY_NAME "HDF5 HL Headers" DEPENDS hllibraries GROUP Development INSTALL_TYPES Full Developer ) - CPACK_ADD_COMPONENT (hltoolsapplications - DISPLAY_NAME "HDF5 HL Tools Applications" + cpack_add_component (hltoolsapplications + DISPLAY_NAME "HDF5 HL Tools Applications" DEPENDS hllibraries GROUP Applications INSTALL_TYPES Full Developer User ) if (HDF5_BUILD_CPP_LIB) - CPACK_ADD_COMPONENT (hlcpplibraries - DISPLAY_NAME "HDF5 HL C++ Libraries" + cpack_add_component (hlcpplibraries + DISPLAY_NAME "HDF5 HL C++ Libraries" DEPENDS hllibraries GROUP Runtime INSTALL_TYPES Full Developer User ) - CPACK_ADD_COMPONENT (hlcppheaders - DISPLAY_NAME "HDF5 HL C++ Headers" + cpack_add_component (hlcppheaders + DISPLAY_NAME "HDF5 HL C++ Headers" DEPENDS hlcpplibraries GROUP Development INSTALL_TYPES Full Developer ) endif (HDF5_BUILD_CPP_LIB) if (HDF5_BUILD_FORTRAN) - CPACK_ADD_COMPONENT (hlfortlibraries - DISPLAY_NAME "HDF5 HL Fortran Libraries" + cpack_add_component (hlfortlibraries + DISPLAY_NAME "HDF5 HL Fortran Libraries" DEPENDS fortlibraries GROUP Runtime INSTALL_TYPES Full Developer User ) endif (HDF5_BUILD_FORTRAN) endif (HDF5_BUILD_HL_LIB) - + endif (NOT HDF5_EXTERNALLY_CONFIGURED AND NOT HDF5_NO_PACKAGES) diff --git a/CMakeLists.txt b/CMakeLists.txt index 33b4bca..52a883e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -170,6 +170,9 @@ if (NOT HDF5_INSTALL_BIN_DIR) set (HDF5_INSTALL_BIN_DIR bin) endif (NOT HDF5_INSTALL_BIN_DIR) if (NOT HDF5_INSTALL_LIB_DIR) + if (APPLE) + set (HDF5_INSTALL_FMWK_DIR ${CMAKE_INSTALL_FRAMEWORK_PREFIX}) + endif (APPLE) set (HDF5_INSTALL_LIB_DIR lib) endif (NOT HDF5_INSTALL_LIB_DIR) if (NOT HDF5_INSTALL_INCLUDE_DIR) @@ -177,6 +180,15 @@ if (NOT HDF5_INSTALL_INCLUDE_DIR) endif (NOT HDF5_INSTALL_INCLUDE_DIR) if (NOT HDF5_INSTALL_DATA_DIR) if (NOT WIN32) + if (APPLE) + option (HDF5_BUILD_FRAMEWORKS "Create a Mac OSX Framework" OFF) + if (HDF5_BUILD_FRAMEWORKS) + set (HDF5_INSTALL_EXTRA_DIR ../SharedSupport) + else (HDF5_BUILD_FRAMEWORKS) + set (HDF5_INSTALL_EXTRA_DIR share) + endif (HDF5_BUILD_FRAMEWORKS) + set (HDF5_INSTALL_FWRK_DIR ${CMAKE_INSTALL_FRAMEWORK_PREFIX}) + endif (APPLE) set (HDF5_INSTALL_DATA_DIR share) set (HDF5_INSTALL_CMAKE_DIR share/cmake) else (NOT WIN32) @@ -288,6 +300,14 @@ include (${HDF_RESOURCES_DIR}/ConfigureChecks.cmake) set (CMAKE_INCLUDE_CURRENT_DIR_IN_INTERFACE ON) #----------------------------------------------------------------------------- +# Mac OS X Options +#----------------------------------------------------------------------------- +option (HDF5_BUILD_FRAMEWORKS "TRUE to build as frameworks libraries, FALSE to build according to BUILD_SHARED_LIBS" TRUE) +if (HDF5_BUILD_FRAMEWORKS AND NOT BUILD_SHARED_LIBS) + set (BUILD_SHARED_LIBS ON CACHE BOOL "Build Shared Libraries") +endif (HDF5_BUILD_FRAMEWORKS AND NOT BUILD_SHARED_LIBS) + +#----------------------------------------------------------------------------- # Option to Build Shared/Static libs, default is static #----------------------------------------------------------------------------- option (BUILD_SHARED_LIBS "Build Shared Libraries" OFF) diff --git a/CTestConfig.cmake b/CTestConfig.cmake index 7bbce4f..7497b08 100644 --- a/CTestConfig.cmake +++ b/CTestConfig.cmake @@ -9,10 +9,10 @@ set (CTEST_NIGHTLY_START_TIME "18:00:00 CST") set (CTEST_DROP_METHOD "http") if (CDASH_LOCAL) - set (CTEST_DROP_SITE "72.36.68.252") + set (CTEST_DROP_SITE "10.10.10.82") set (CTEST_DROP_LOCATION "/submit.php?project=HDF5Trunk") else (CDASH_LOCAL) - set (CTEST_DROP_SITE "cdash.hdfgroup.uiuc.edu") + set (CTEST_DROP_SITE "cdash.hdfgroup.org") set (CTEST_DROP_LOCATION "/submit.php?project=HDF5+Trunk") endif (CDASH_LOCAL) set (CTEST_DROP_SITE_CDASH TRUE) @@ -42,6 +42,7 @@ ./configure.ac ./bin/COPYING +./bin/bbrelease _DO_NOT_DISTRIBUTE_ ./bin/buildhdf5 ./bin/checkapi _DO_NOT_DISTRIBUTE_ ./bin/checkposix _DO_NOT_DISTRIBUTE_ @@ -566,6 +567,7 @@ ./src/H5Aprivate.h ./src/H5Apublic.h ./src/H5AC.c +./src/H5ACmpio.c ./src/H5ACpkg.h ./src/H5ACprivate.h ./src/H5ACpublic.h @@ -586,6 +588,7 @@ ./src/H5B2stat.c ./src/H5B2test.c ./src/H5C.c +./src/H5Cmpio.c ./src/H5Cpkg.h ./src/H5Cprivate.h ./src/H5Cpublic.h @@ -2392,6 +2395,7 @@ ./config/cmake/cacheinit.cmake ./config/cmake/ConversionTests.c ./config/cmake/ConfigureChecks.cmake +./config/cmake/CPack.Info.plist.in ./config/cmake/CTestCustom.cmake ./config/cmake/FindHDF5.cmake.in ./config/cmake/H5cxx_config.h.in @@ -2410,7 +2414,6 @@ ./config/cmake_ext_mod/CheckTypeSize.cmake ./config/cmake_ext_mod/ConfigureChecks.cmake -./config/cmake_ext_mod/CPack.Info.plist.in ./config/cmake_ext_mod/CTestCustom.cmake ./config/cmake_ext_mod/FindMPI.cmake ./config/cmake_ext_mod/FindSZIP.cmake @@ -1,4 +1,4 @@ -HDF5 version 1.9.222 currently under development +HDF5 version 1.9.227 currently under development Please refer to the release_docs/INSTALL file for installation instructions. ------------------------------------------------------------------------------ diff --git a/bin/bbrelease b/bin/bbrelease new file mode 100755 index 0000000..fea498a --- /dev/null +++ b/bin/bbrelease @@ -0,0 +1,394 @@ +#!/bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the files COPYING and Copyright.html. COPYING can be found at the root +# of the source code distribution tree; Copyright.html can be found at the +# root level of an installed copy of the electronic HDF5 document set and +# is linked from the top-level documents page. It can also be found at +# http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have +# access to either file, you may request a copy from help@hdfgroup.org. +# + +# Make a release of hdf5. +# +# Programmer: Robb Matzke +# Creation date: on or before 1998-01-29. +# +# Modifications +# Robb Matzke, 1999-07-16 +# The SunOS 5.6 sed *must* have slashes as delimiters. I changed things like +# `sed s+/CVS++' to `sed 's/\/CVS//' +# +# Albert Cheng, 1999-10-26 +# Moved the MANIFEST checking to a separate command file so that +# it can be invoked individually. +# +# Albert Cheng, 2004-08-14 +# Added the --private option. +# +# James Laird, 2005-09-07 +# Added the md5 method. + +# Function definitions +# +# Print Usage page +USAGE() +{ +cat << EOF +Usage: $0 -d <dir> [--docver BRANCHNAME] [-h] [--nocheck] [--private] <methods> ... + -d DIR The name of the directory where the releas(es) should be + placed. + --docver BRANCHNAME This is added for 1.8 and beyond to get the correct + version of documentation files from the hdf5docs + repository. BRANCHNAME for v1.8 should be hdf5_1_8. + -h print the help page. + --nocheck Ignore errors in MANIFEST file. + --private Make a private release with today's date in version information. + --revision Make a private release with the code revision number in version information. + +This must be run at the top level of the source directory. +The other command-line options are the names of the programs to use +for compressing the resulting tar archive (if none are given then +"tar" is assumed): + + tar -- use tar and don't do any compressing. + gzip -- use gzip with "-9" and append ".gz" to the output name. + bzip2 -- use bzip2 with "-9" and append ".bz2" to the output name. + zip -- convert all text files to DOS style and form a zip file for Windows use. + doc -- produce the latest doc tree in addition to the archive. + +An md5 checksum is produced for each archive created and stored in the md5 file. + +Examples: + + $ bin/release -d /tmp + /tmp/hdf5-1.8.13-RELEASE.txt + /tmp/hdf5-1.8.13.md5 + /tmp/hdf5-1.8.13.tar + + $ bin/release -d /tmp gzip + /tmp/hdf5-1.8.13-RELEASE.txt + /tmp/hdf5-1.8.13.md5 + /tmp/hdf5-1.8.13.tar.gz + + $ bin/release -d /tmp tar gzip zip + /tmp/hdf5-1.8.13-RELEASE.txt + /tmp/hdf5-1.8.13.md5 + /tmp/hdf5-1.8.13.tar + /tmp/hdf5-1.8.13.tar.gz + /tmp/hdf5-1.8.13.tar.zip + +EOF + +} + +# Function name: tar2zip +# Convert the release tarball to a Windows zipball. +# +# Programmer: Albert Cheng +# Creation date: 2014-04-23 +# +# Modifications +# +# Steps: +# 1. untar the tarball in a temporay directory; +# Note: do this in a temporary directory to avoid changing +# the original source directory which maybe around. +# 2. convert all its text files to DOS (LF-CR) style; +# 3. form a zip file which is usable by Windows users. +# +# Parameters: +# $1 version +# $2 release tarball +# $3 output zipball file name +# +# Returns 0 if successful; 1 otherwise +# +tar2zip() +{ + if [ $# -ne 3 ]; then + echo "usage: tar2zip <tarfilename> <zipfilename>" + return 1 + fi + ztmpdir=/tmp/tmpdir$$ + mkdir -p $ztmpdir + version=$1 + tarfile=$2 + zipfile=$3 + + # step 1: untar tarball in ztmpdir + (cd $ztmpdir; tar xf -) < $tarfile + # sanity check + if [ ! -d $ztmpdir/$version ]; then + echo "untar did not create $ztmpdir/$version source dir" + # cleanup + rm -rf $ztmpdir + return 1 + fi + # step 2: convert text files + # There maybe a simpler way to do this. + # options used in unix2dos: + # -k Keep the date stamp + # -q quiet mode + # grep redirect output to /dev/null because -q or -s are not portable. + find $ztmpdir/$version | \ + while read inf; do \ + if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ + unix2dos -q -k $inf; \ + fi\ + done + # step 3: make zipball + # -9 maximum compression + # -y Store symbolic links as such in the zip archive + # -r recursive + # -q quiet + (cd $ztmpdir; zip -9 -y -r -q $version.zip $version) + mv $ztmpdir/$version.zip $zipfile + + # cleanup + rm -rf $ztmpdir +} + +# This command must be run at the top level of the hdf5 source directory. +# Verify this requirement. +if [ ! \( -f configure -a -f bin/release \) ]; then + echo "$0 must be run at the top level of the hdf5 source directory" + exit 1 +fi + +# Defaults +DEST=releases +VERS=`perl bin/h5vers` +VERS_OLD= +test "$VERS" || exit 1 +verbose=yes +check=yes +release_date=`date +%F` +today=`date +%Y%m%d` +pmode='no' +revmode='no' +tmpdir="../#release_tmp.$$" # tmp work directory +DOC_URL=http://svn.hdfgroup.uiuc.edu/hdf5doc/trunk +CPPLUS_RM_NAME=cpplus_RM + +# Restore previous Version information +RESTORE_VERSION() +{ + if [ X-${VERS_OLD} != X- ]; then + echo restoring version information back to $VERS_OLD + rm -f config/lt_vers.am + cp $tmpdir/lt_vers.am config/lt_vers.am + bin/h5vers -s $VERS_OLD + VERS_OLD= + fi +} + + +# Command-line arguments +while [ -n "$1" ]; do + arg=$1 + shift + case "$arg" in + -d) + DEST=$1 + shift + ;; + --nocheck) + check=no + ;; + -h) + USAGE + exit 0 + ;; + --private) + pmode=yes + ;; + --revision) + revmode=yes + ;; + --docver) + DOCVERSION=$1 + shift + ;; + -*) + echo "Unknown switch: $arg" 1>&2 + USAGE + exit 1 + ;; + *) + methods="$methods $arg" + ;; + esac +done + +# Default method is tar +if [ "X$methods" = "X" ]; then + methods="tar" +fi + +# Create the temporay work directory. +if mkdir $tmpdir; then + echo "temporary work directory for release. "\ + "Can be deleted after release completes." > $tmpdir/README +else + echo "Failed to mkdir tmpdir($tmpdir)" + exit 1 +fi + +# setup restoration in case of abort. +trap RESTORE_VERSION 0 + +if [ X$pmode = Xyes ]; then + VERS_OLD=$VERS + # Copy old version of config/lt_vers.am, since it's hard to + # "undo" changes to it. + cp config/lt_vers.am $tmpdir + # Set version information to m.n.r-of$today. + # (h5vers does not correctly handle just m.n.r-$today.) + VERS=`echo $VERS | sed -e s/-.*//`-of$today + echo Private release of $VERS + bin/h5vers -s $VERS +fi + +if [ X$revmode = Xyes ]; then + VERS_OLD=$VERS + # Copy old version of config/lt_vers.am, since it's hard to + # "undo" changes to it. + cp config/lt_vers.am $tmpdir + branch=`svn info | grep URL: | awk -F/ '{print $NF}'` + revision=`svn info | grep "Last Changed Rev:" | awk '{print $4}'` + # Set version information to m.n.r-r$revision. + # (h5vers does not correctly handle just m.n.r-$today.) + VERS=`echo $VERS | sed -e s/-.*//`-r$revision + echo Private release of $VERS + bin/h5vers -s $VERS + HDF5_VERS=hdf5-$branch-r$revision + # use a generic directory name for revision releases + HDF5_IN_VERS=hdfsrc +else + # Store hdf5-$VERS ("hdf5-1.7.51", e.g.) to a variable to avoid typos + HDF5_VERS=hdf5-$VERS + # directory name matches tar file name for non-revision releases + HDF5_IN_VERS=$HDF5_VERS +fi + +test "$verbose" && echo "Releasing $HDF5_VERS to $DEST" 1>&2 +if [ ! -d $DEST ]; then + echo " Destination directory $DEST does not exist" 1>&2 + exit 1 +fi + +# Check the validity of the MANIFEST file. +bin/chkmanifest || fail=yes +if [ "X$fail" = "Xyes" ]; then + if [ $check = yes ]; then + exit 1 + else + echo "Continuing anyway..." + fi +fi + +# Create a manifest that contains only files for distribution. +MANIFEST=$tmpdir/H5_MANIFEST +grep '^\.' MANIFEST | grep -v _DO_NOT_DISTRIBUTE_ >$MANIFEST + +# Prepare the source tree for a release. +#ln -s `pwd` $tmpdir/$HDF5_VERS || exit 1 +ln -s `pwd` $tmpdir/$HDF5_IN_VERS || exit 1 +# Save a backup copy of Makefile if exists. +test -f Makefile && mv Makefile $tmpdir/Makefile.x +cp -p Makefile.dist Makefile + +# Update README.txt and release_docs/RELEASE.txt with release information in +# line 1. +for f in README.txt release_docs/RELEASE.txt; do + echo "HDF5 version $VERS released on $release_date" >$f.x + sed -e 1d $f >>$f.x + mv $f.x $f + # Make sure new files are of the right access mode + chmod 644 $f +done + +# trunk is different than branches. +if [ "${DOCVERSION}" ]; then + DOC_URL=http://svn.hdfgroup.uiuc.edu/hdf5doc/branches/${DOCVERSION} +fi + +# Create the tar file +test "$verbose" && echo " Running tar..." 1>&2 +( \ + cd $tmpdir; \ + tar cf $HDF5_VERS.tar $HDF5_IN_VERS/Makefile \ + `sed 's/^\.\//'$HDF5_IN_VERS'\//' $MANIFEST` || exit 1 \ +) + +# Compress +MD5file=$HDF5_VERS.md5 +cp /dev/null $DEST/$MD5file +for comp in $methods; do + case $comp in + tar) + cp -p $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.tar + (cd $DEST; md5sum $HDF5_VERS.tar >> $MD5file) + ;; + gzip) + test "$verbose" && echo " Running gzip..." 1>&2 + gzip -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.gz + (cd $DEST; md5sum $HDF5_VERS.tar.gz >> $MD5file) + ;; + bzip2) + test "$verbose" && echo " Running bzip2..." 1>&2 + bzip2 -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.bz2 + (cd $DEST; md5sum $HDF5_VERS.tar.bz2 >> $MD5file) + ;; + zip) + test "$verbose" && echo " Creating zip ball..." 1>&2 + tar2zip $HDF5_IN_VERS $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.zip 1>&2 + (cd $DEST; md5sum $HDF5_VERS.zip >> $MD5file) + ;; + doc) + if [ "${DOCVERSION}" = "" ]; then + DOCVERSION=trunk + fi + test "$verbose" && echo " Creating docs..." 1>&2 + # Check out docs from svn repo + (cd $tmpdir; svn co $DOC_URL > /dev/null) || exit 1 + # Create doxygen C++ RM + (cd c++/src && doxygen cpp_doc_config > /dev/null ) || exit 1 + # Replace version of C++ RM with just-created version + rm -rf $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME + mv c++/src/$CPPLUS_RM_NAME $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME + # Compress the docs and move them to the release area + mv $tmpdir/$DOCVERSION $tmpdir/${HDF5_VERS}_docs + (cd $tmpdir && tar cf ${HDF5_VERS}_docs.tar ${HDF5_VERS}_docs) + mv $tmpdir/${HDF5_VERS}_docs.tar $DEST + ;; + *) + echo "***Error*** Unknown method $comp" + exit 1 + ;; + esac +done + +# Copy the RELEASE.txt to the release area. +cp release_docs/RELEASE.txt $DEST/$HDF5_VERS-RELEASE.txt + +# Remove distributed Makefile and restore previous Makefile if existed. +rm -f Makefile +test -f $tmpdir/Makefile.x && mv $tmpdir/Makefile.x Makefile + +# Restore OLD version information, then no need for trap. +if [ X$pmode = Xyes ] || [ X$revmode = Xyes ]; then + RESTORE_VERSION + trap 0 +fi + +# Remove temporary things +rm -rf $tmpdir + +exit 0 diff --git a/c++/src/CMakeLists.txt b/c++/src/CMakeLists.txt index 2f3a0c4..44b912c 100644 --- a/c++/src/CMakeLists.txt +++ b/c++/src/CMakeLists.txt @@ -94,7 +94,7 @@ set_target_properties (${HDF5_CPP_LIB_TARGET} PROPERTIES ) #----------------------------------------------------------------------------- -# Add file(s) to CMake Install +# Add file(s) to CMake Install #----------------------------------------------------------------------------- install ( FILES @@ -112,7 +112,7 @@ if (HDF5_EXPORTED_TARGETS) if (BUILD_SHARED_LIBS) INSTALL_TARGET_PDB (${HDF5_CPP_LIB_TARGET} ${HDF5_INSTALL_BIN_DIR} cpplibraries) endif (BUILD_SHARED_LIBS) - + install ( TARGETS ${HDF5_CPP_LIB_TARGET} @@ -121,5 +121,6 @@ if (HDF5_EXPORTED_TARGETS) LIBRARY DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT cpplibraries ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT cpplibraries RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT cpplibraries + FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT cpplibraries ) endif (HDF5_EXPORTED_TARGETS) diff --git a/c++/src/Makefile.in b/c++/src/Makefile.in index 9df8eff..5dff737 100644 --- a/c++/src/Makefile.in +++ b/c++/src/Makefile.in @@ -670,7 +670,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog *.clog2 # After making changes, run bin/reconfigure to update other configure related # files like Makefile.in. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 212 +LT_VERS_REVISION = 217 LT_VERS_AGE = 0 # This is our main target diff --git a/c++/src/cpp_doc_config b/c++/src/cpp_doc_config index a4aa9af..53d9a7d 100644 --- a/c++/src/cpp_doc_config +++ b/c++/src/cpp_doc_config @@ -38,7 +38,7 @@ PROJECT_NAME = "HDF5 C++ API" # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = HDF5 version 1.9.222 currently under development +PROJECT_NUMBER = HDF5 version 1.9.227 currently under development # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/config/cmake/CPack.Info.plist.in b/config/cmake/CPack.Info.plist.in new file mode 100644 index 0000000..08d371b --- /dev/null +++ b/config/cmake/CPack.Info.plist.in @@ -0,0 +1,26 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> +<plist version="1.0"> +<dict> + <key>CFBundleDevelopmentRegion</key> + <string>English</string> + <key>CFBundleExecutable</key> + <string>@CPACK_PACKAGE_FILE_NAME@</string> + <key>CFBundleIconFile</key> + <string>@CPACK_BUNDLE_ICON@</string> + <key>CFBundleIdentifier</key> + <string>org.@CPACK_PACKAGE_VENDOR@.@CPACK_PACKAGE_NAME@@CPACK_MODULE_VERSION_SUFFIX@</string> + <key>CFBundleInfoDictionaryVersion</key> + <string>6.0</string> + <key>CFBundlePackageType</key> + <string>FMWK</string> + <key>CFBundleSignature</key> + <string>????</string> + <key>CFBundleVersion</key> + <string>@CPACK_PACKAGE_VERSIO@</string> + <key>CFBundleShortVersionString</key> + <string>@CPACK_SHORT_VERSION_STRING@</string> + <key>CSResourcesFileMapped</key> + <true/> +</dict> +</plist> diff --git a/config/cmake/ConfigureChecks.cmake b/config/cmake/ConfigureChecks.cmake index e223553..09f1caf 100644 --- a/config/cmake/ConfigureChecks.cmake +++ b/config/cmake/ConfigureChecks.cmake @@ -33,6 +33,18 @@ endif (HDF5_METADATA_TRACE_FILE) MARK_AS_ADVANCED (HDF5_METADATA_TRACE_FILE) # ---------------------------------------------------------------------- +# Decide whether the data accuracy has higher priority during data +# conversions. If not, some hard conversions will still be prefered even +# though the data may be wrong (for example, some compilers don't +# support denormalized floating values) to maximize speed. +# +option (HDF5_WANT_DATA_ACCURACY "IF data accuracy is guaranteed during data conversions" ON) +if (HDF5_WANT_DATA_ACCURACY) + set (H5_WANT_DATA_ACCURACY 1) +endif (HDF5_WANT_DATA_ACCURACY) +MARK_AS_ADVANCED (HDF5_WANT_DATA_ACCURACY) + +# ---------------------------------------------------------------------- # Decide whether the presence of user's exception handling functions is # checked and data conversion exceptions are returned. This is mainly # for the speed optimization of hard conversions. Soft conversions can @@ -220,6 +232,24 @@ H5ConversionTests (H5_LDOUBLE_TO_LONG_SPECIAL "Checking IF your system converts # H5ConversionTests (H5_LONG_TO_LDOUBLE_SPECIAL "Checking IF your system can convert (unsigned) long to long double values with special algorithm") # ---------------------------------------------------------------------- +# Set the flag to indicate that the machine can accurately convert +# 'long double' to '(unsigned) long long' values. (This flag should be set for +# all machines, except for Mac OS 10.4 and SGI IRIX64 6.5. When the bit sequence +# of long double is 0x4351ccf385ebc8a0bfcc2a3c..., the values of (unsigned)long long +# start to go wrong on these two machines. Adjusting it higher to +# 0x4351ccf385ebc8a0dfcc... or 0x4351ccf385ebc8a0ffcc... will make the converted +# values wildly wrong. This test detects this wrong behavior and disable the test. +# +H5ConversionTests (H5_LDOUBLE_TO_LLONG_ACCURATE "Checking IF correctly converting long double to (unsigned) long long values") +# ---------------------------------------------------------------------- +# Set the flag to indicate that the machine can accurately convert +# '(unsigned) long long' to 'long double' values. (This flag should be set for +# all machines, except for Mac OS 10.4, when the bit sequences are 003fff..., +# 007fff..., 00ffff..., 01ffff..., ..., 7fffff..., the converted values are twice +# as big as they should be. +# +H5ConversionTests (H5_LLONG_TO_LDOUBLE_CORRECT "Checking IF correctly converting (unsigned) long long to long double values") +# ---------------------------------------------------------------------- # Check if pointer alignments are enforced # H5ConversionTests (H5_NO_ALIGNMENT_RESTRICTIONS "Checking IF alignment restrictions are strictly enforced") diff --git a/config/cmake/ConversionTests.c b/config/cmake/ConversionTests.c index cd2e8bb..d964bf8 100644 --- a/config/cmake/ConversionTests.c +++ b/config/cmake/ConversionTests.c @@ -116,6 +116,88 @@ done: #endif +#ifdef H5_LDOUBLE_TO_LLONG_ACCURATE_TEST +int main(void) +{ + long double ld = 20041683600089727.779961L; + long long ll; + unsigned long long ull; + unsigned char s[16]; + int ret = 0; + + if(sizeof(long double) == 16) { + /*make sure the long double type is the same as the failing type + *which has 16 bytes in size and 11 bits of exponent. If it is, + *the bit sequence should be like below. It's not + *a decent way to check but this info isn't available. */ + memcpy(s, &ld, 16); + if(s[0]==0x43 && s[1]==0x51 && s[2]==0xcc && s[3]==0xf3 && + s[4]==0x85 && s[5]==0xeb && s[6]==0xc8 && s[7]==0xa0 && + s[8]==0xbf && s[9]==0xcc && s[10]==0x2a && s[11]==0x3c) { + + /*slightly adjust the bit sequence (s[8]=0xdf). The converted + *values will go wild on Mac OS 10.4 and IRIX64 6.5.*/ + s[0]=0x43; s[1]=0x51; s[2]=0xcc; s[3]=0xf3; + s[4]=0x85; s[5]=0xeb; s[6]=0xc8; s[7]=0xa0; + s[8]=0xdf; s[9]=0xcc; s[10]=0x2a; s[11]=0x3c; + s[12]=0x3d; s[13]=0x85; s[14]=0x56; s[15]=0x20; + + memcpy(&ld, s, 16); + ll = (long long)ld; + ull = (unsigned long long)ld; + + if(ll != 20041683600089728 || ull != 20041683600089728) + ret = 1; + } + } +done: + exit(ret); +} +#endif + +#ifdef H5_LLONG_TO_LDOUBLE_CORRECT_TEST +int main(void) +{ + long double ld; + long long ll; + unsigned long long ull; + unsigned char s[16]; + int flag=0, ret=0; + + /*Determine if long double has 16 byte in size, 11 bit exponent, and + *the bias is 0x3ff */ + if(sizeof(long double) == 16) { + ld = 1.0L; + memcpy(s, &ld, 16); + if(s[0]==0x3f && s[1]==0xf0 && s[2]==0x00 && s[3]==0x00 && + s[4]==0x00 && s[5]==0x00 && s[6]==0x00 && s[7]==0x00) + flag = 1; + } + + if(flag==1 && sizeof(long long)==8) { + ll = 0x01ffffffffffffffLL; + ld = (long double)ll; + memcpy(s, &ld, 16); + /*Check if the bit sequence is as supposed to be*/ + if(s[0]!=0x43 || s[1]!=0x7f || s[2]!=0xff || s[3]!=0xff || + s[4]!=0xff || s[5]!=0xff || s[6]!=0xff || s[7]!=0xff || + s[8]!=0xf0 || s[9]!=0x00 || s[10]!=0x00 || s[11]!=0x00) + ret = 1; + } + if(flag==1 && sizeof(unsigned long long)==8) { + ull = 0x01ffffffffffffffULL; + ld = (long double)ull; + memcpy(s, &ld, 16); + if(s[0]!=0x43 || s[1]!=0x7f || s[2]!=0xff || s[3]!=0xff || + s[4]!=0xff || s[5]!=0xff || s[6]!=0xff || s[7]!=0xff || + s[8]!=0xf0 || s[9]!=0x00 || s[10]!=0x00 || s[11]!=0x00) + ret = 1; + } +done: + exit(ret); +} +#endif + #ifdef H5_NO_ALIGNMENT_RESTRICTIONS_TEST #include <stdlib.h> diff --git a/config/cmake/H5pubconf.h.in b/config/cmake/H5pubconf.h.in index fcbdbd5..31bcc08 100644 --- a/config/cmake/H5pubconf.h.in +++ b/config/cmake/H5pubconf.h.in @@ -398,10 +398,18 @@ /* Define if HDF5's high-level library headers should be included in hdf5.h */ #cmakedefine H5_INCLUDE_HL @H5_INCLUDE_HL@ +/* Define if your system can convert long double to (unsigned) long long + values correctly. */ +#cmakedefine H5_LDOUBLE_TO_LLONG_ACCURATE @H5_LDOUBLE_TO_LLONG_ACCURATE@ + /* Define if your system converts long double to (unsigned) long values with special algorithm. */ #cmakedefine H5_LDOUBLE_TO_LONG_SPECIAL @H5_LDOUBLE_TO_LONG_SPECIAL@ +/* Define if your system can convert (unsigned) long long to long double + values correctly. */ +#cmakedefine H5_LLONG_TO_LDOUBLE_CORRECT @H5_LLONG_TO_LDOUBLE_CORRECT@ + /* Define if your system can convert (unsigned) long to long double values with special algorithm. */ #cmakedefine H5_LONG_TO_LDOUBLE_SPECIAL @H5_LONG_TO_LDOUBLE_SPECIAL@ @@ -606,6 +614,9 @@ /* Version number of package */ #define H5_VERSION "@HDF5_PACKAGE_VERSION_STRING@" +/* Data accuracy is prefered to speed during data conversions */ +#cmakedefine H5_WANT_DATA_ACCURACY @H5_WANT_DATA_ACCURACY@ + /* Check exception handling functions during data conversions */ #cmakedefine H5_WANT_DCONV_EXCEPTION @H5_WANT_DCONV_EXCEPTION@ diff --git a/config/cmake/HDF5Macros.cmake b/config/cmake/HDF5Macros.cmake index ee9ed84..976b910 100644 --- a/config/cmake/HDF5Macros.cmake +++ b/config/cmake/HDF5Macros.cmake @@ -26,6 +26,18 @@ macro (H5_SET_LIB_OPTIONS libtarget libname libtype) BUILD_WITH_INSTALL_RPATH ${HDF5_BUILD_WITH_INSTALL_NAME} ) endif (HDF5_BUILD_WITH_INSTALL_NAME) + if (HDF5_BUILD_FRAMEWORKS) + if (${libtype} MATCHES "SHARED") + # adapt target to build frameworks instead of dylibs + set_target_properties(${libtarget} PROPERTIES + XCODE_ATTRIBUTE_INSTALL_PATH "@rpath" + FRAMEWORK TRUE + FRAMEWORK_VERSION ${HDF5_PACKAGE_VERSION_MAJOR} + MACOSX_FRAMEWORK_IDENTIFIER org.hdfgroup.${libtarget} + MACOSX_FRAMEWORK_SHORT_VERSION_STRING ${HDF5_PACKAGE_VERSION_MAJOR} + MACOSX_FRAMEWORK_BUNDLE_VERSION ${HDF5_PACKAGE_VERSION_MAJOR}) + endif (${libtype} MATCHES "SHARED") + endif (HDF5_BUILD_FRAMEWORKS) endif (APPLE) endmacro (H5_SET_LIB_OPTIONS) diff --git a/config/cmake/HDFCompilerFlags.cmake b/config/cmake/HDFCompilerFlags.cmake index 56d17bf..c637c73 100644 --- a/config/cmake/HDFCompilerFlags.cmake +++ b/config/cmake/HDFCompilerFlags.cmake @@ -5,7 +5,7 @@ if (CMAKE_COMPILER_IS_GNUCC) if (CMAKE_BUILD_TYPE MATCHES Debug) set (CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS} -std=c99") if (NOT CMAKE_C_COMPILER_VERSION VERSION_LESS 5.0) - set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -ftrapv -fno-common") + set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Og -ftrapv -fno-common") endif () else (CMAKE_BUILD_TYPE MATCHES Debug) set (CMAKE_C_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_C_FLAGS} -std=c99") @@ -18,7 +18,7 @@ if (CMAKE_COMPILER_IS_GNUCXX) if (CMAKE_BUILD_TYPE MATCHES Debug) set (CMAKE_CXX_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_CXX_FLAGS} -std=c99") if (NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 5.0) - set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -ftrapv -fno-common") + set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Og -ftrapv -fno-common") endif () else (CMAKE_BUILD_TYPE MATCHES Debug) set (CMAKE_CXX_FLAGS "${CMAKE_ANSI_CFLAGS} ${CMAKE_CXX_FLAGS} -std=c99") diff --git a/config/cmake/cacheinit.cmake b/config/cmake/cacheinit.cmake index 87d4fcc..f869949 100644 --- a/config/cmake/cacheinit.cmake +++ b/config/cmake/cacheinit.cmake @@ -8,6 +8,8 @@ set (BUILD_SHARED_LIBS ON CACHE BOOL "Build Shared Libraries" FORCE) set (BUILD_TESTING ON CACHE BOOL "Build HDF5 Unit Testing" FORCE) +set (CMAKE_INSTALL_FRAMEWORK_PREFIX "Library/Frameworks" CACHE STRING "Frameworks installation directory" FORCE) + set (HDF_PACKAGE_EXT "" CACHE STRING "Name of HDF package extension" FORCE) set (HDF5_BUILD_FORTRAN ON CACHE BOOL "Build FORTRAN support" FORCE) diff --git a/config/lt_vers.am b/config/lt_vers.am index f9e6f77..ac029c0 100644 --- a/config/lt_vers.am +++ b/config/lt_vers.am @@ -19,7 +19,7 @@ # After making changes, run bin/reconfigure to update other configure related # files like Makefile.in. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 212 +LT_VERS_REVISION = 217 LT_VERS_AGE = 0 ## If the API changes *at all*, increment LT_VERS_INTERFACE and @@ -1,7 +1,7 @@ #! /bin/sh # From configure.ac Id: configure.ac 22697 2012-08-19 14:35:47Z hdftest . # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for HDF5 1.9.222. +# Generated by GNU Autoconf 2.69 for HDF5 1.9.227. # # Report bugs to <help@hdfgroup.org>. # @@ -591,8 +591,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='HDF5' PACKAGE_TARNAME='hdf5' -PACKAGE_VERSION='1.9.222' -PACKAGE_STRING='HDF5 1.9.222' +PACKAGE_VERSION='1.9.227' +PACKAGE_STRING='HDF5 1.9.227' PACKAGE_BUGREPORT='help@hdfgroup.org' PACKAGE_URL='' @@ -911,6 +911,7 @@ with_mpe enable_direct_vfd with_default_plugindir enable_dconv_exception +enable_dconv_accuracy enable_build_all enable_deprecated_symbols with_default_api_version @@ -1472,7 +1473,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures HDF5 1.9.222 to adapt to many kinds of systems. +\`configure' configures HDF5 1.9.227 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1542,7 +1543,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of HDF5 1.9.222:";; + short | recursive ) echo "Configuration of HDF5 1.9.227:";; esac cat <<\_ACEOF @@ -1608,6 +1609,8 @@ Optional Features: --enable-dconv-exception if exception handling functions is checked during data conversions [default=yes] + --enable-dconv-accuracy if data accuracy is guaranteed during data + conversions [default=yes] --enable-build-all Build helper programs that only developers should need [default=no] --enable-deprecated-symbols @@ -1731,7 +1734,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -HDF5 configure 1.9.222 +HDF5 configure 1.9.227 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2668,7 +2671,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by HDF5 $as_me 1.9.222, which was +It was created by HDF5 $as_me 1.9.227, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -3539,7 +3542,7 @@ fi # Define the identity of the package. PACKAGE='hdf5' - VERSION='1.9.222' + VERSION='1.9.227' cat >>confdefs.h <<_ACEOF @@ -26241,48 +26244,60 @@ $as_echo "#define const /**/" >>confdefs.h fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for inline" >&5 -$as_echo_n "checking for inline... " >&6; } -if ${ac_cv_c_inline+:} false; then : - $as_echo_n "(cached) " >&6 -else - ac_cv_c_inline=no -for ac_kw in inline __inline__ __inline; do - cat confdefs.h - <<_ACEOF >conftest.$ac_ext -/* end confdefs.h. */ -#ifndef __cplusplus -typedef int foo_t; -static $ac_kw foo_t static_foo () {return 0; } -$ac_kw foo_t foo () {return 0; } -#endif +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if the compiler understands __inline__" >&5 +$as_echo_n "checking if the compiler understands __inline__... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +static __inline__ void f(void){return;}; _ACEOF if ac_fn_c_try_compile "$LINENO"; then : - ac_cv_c_inline=$ac_kw + +$as_echo "#define HAVE___INLINE__ 1" >>confdefs.h + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } + fi rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext - test "$ac_cv_c_inline" != no && break -done + +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if the compiler understands __inline" >&5 +$as_echo_n "checking if the compiler understands __inline... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +static __inline void f(void){return;}; +_ACEOF +if ac_fn_c_try_compile "$LINENO"; then : + +$as_echo "#define HAVE___INLINE 1" >>confdefs.h + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } fi -{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_c_inline" >&5 -$as_echo "$ac_cv_c_inline" >&6; } +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext -case $ac_cv_c_inline in - inline | yes) ;; - *) - case $ac_cv_c_inline in - no) ac_val=;; - *) ac_val=$ac_cv_c_inline;; - esac - cat >>confdefs.h <<_ACEOF -#ifndef __cplusplus -#define inline $ac_val -#endif +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if the compiler understands inline" >&5 +$as_echo_n "checking if the compiler understands inline... " >&6; } +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ +static inline void f(void){return;}; _ACEOF - ;; -esac +if ac_fn_c_try_compile "$LINENO"; then : + +$as_echo "#define HAVE_INLINE 1" >>confdefs.h + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi +rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext { $as_echo "$as_me:${as_lineno-$LINENO}: checking for __attribute__ extension" >&5 $as_echo_n "checking for __attribute__ extension... " >&6; } @@ -27465,6 +27480,33 @@ $as_echo "no" >&6; } fi ## ---------------------------------------------------------------------- +## Decide whether the data accuracy has higher priority during data +## conversions. If not, some hard conversions will still be prefered even +## though the data may be wrong (for example, some compilers don't +## support denormalized floating values) to maximize speed. +## +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking whether data accuracy is guaranteed during data conversions" >&5 +$as_echo_n "checking whether data accuracy is guaranteed during data conversions... " >&6; } +# Check whether --enable-dconv-accuracy was given. +if test "${enable_dconv_accuracy+set}" = set; then : + enableval=$enable_dconv_accuracy; DATA_ACCURACY=$enableval +else + DATA_ACCURACY=yes +fi + + +if test "$DATA_ACCURACY" = "yes"; then + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } + +$as_echo "#define WANT_DATA_ACCURACY 1" >>confdefs.h + +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + +## ---------------------------------------------------------------------- ## Set the flag to indicate that the machine has window style pathname, ## that is, "drive-letter:\" (e.g. "C:") or "drive-letter:/" (e.g. "C:/"). ## (This flag should be _unset_ for all machines, except for Windows, where @@ -27728,6 +27770,190 @@ $as_echo "no" >&6; } fi ## ---------------------------------------------------------------------- +## Set the flag to indicate that the machine can accurately convert +## 'long double' to '(unsigned) long long' values. (This flag should +## be set for all machines, except for Mac OS 10.4, SGI IRIX64 6.5 and +## Powerpc Linux using XL compilers. +## When the bit sequence of long double is 0x4351ccf385ebc8a0bfcc2a3c..., +## the values of (unsigned)long long start to go wrong on these +## two machines. Adjusting it higher to 0x4351ccf385ebc8a0dfcc... or +## 0x4351ccf385ebc8a0ffcc... will make the converted values wildly wrong. +## This test detects this wrong behavior and disable the test. +## +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if correctly converting long double to (unsigned) long long values" >&5 +$as_echo_n "checking if correctly converting long double to (unsigned) long long values... " >&6; } + +if test ${ac_cv_sizeof_long_double} = 0; then + hdf5_cv_ldouble_to_llong_accurate=${hdf5_cv_ldouble_to_llong_accurate=no} +else + if ${hdf5_cv_ldouble_to_llong_accurate+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "$cross_compiling" = yes; then : + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot run test program while cross compiling +See \`config.log' for more details" "$LINENO" 5; } +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + int main(void) + { + long double ld = 20041683600089727.779961L; + long long ll; + unsigned long long ull; + unsigned char s[16]; + int ret = 0; + + if(sizeof(long double) == 16) { + /*make sure the long double type is the same as the failing type + *which has 16 bytes in size and 11 bits of exponent. If it is, + *the bit sequence should be like below. It's not + *a decent way to check but this info isn't available. */ + memcpy(s, &ld, 16); + if(s[0]==0x43 && s[1]==0x51 && s[2]==0xcc && s[3]==0xf3 && + s[4]==0x85 && s[5]==0xeb && s[6]==0xc8 && s[7]==0xa0 && + s[8]==0xbf && s[9]==0xcc && s[10]==0x2a && s[11]==0x3c) { + + /*slightly adjust the bit sequence (s[8]=0xdf). The converted + *values will go wild on Mac OS 10.4 and IRIX64 6.5.*/ + s[0]=0x43; s[1]=0x51; s[2]=0xcc; s[3]=0xf3; + s[4]=0x85; s[5]=0xeb; s[6]=0xc8; s[7]=0xa0; + s[8]=0xdf; s[9]=0xcc; s[10]=0x2a; s[11]=0x3c; + s[12]=0x3d; s[13]=0x85; s[14]=0x56; s[15]=0x20; + + memcpy(&ld, s, 16); + ll = (long long)ld; + ull = (unsigned long long)ld; + + if(ll != 20041683600089728 || ull != 20041683600089728) + ret = 1; + } + } + done: + exit(ret); + } + +_ACEOF +if ac_fn_c_try_run "$LINENO"; then : + hdf5_cv_ldouble_to_llong_accurate=yes +else + hdf5_cv_ldouble_to_llong_accurate=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + +fi + +fi + +if test ${hdf5_cv_ldouble_to_llong_accurate} = "yes"; then + +$as_echo "#define LDOUBLE_TO_LLONG_ACCURATE 1" >>confdefs.h + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + + +## ---------------------------------------------------------------------- +## Set the flag to indicate that the machine can accurately convert +## '(unsigned) long long' to 'long double' values. (This flag should be +## set for all machines, except for Mac OS 10.4 and Powerpc Linux using +## XL compilers. +## When the bit sequences are 003fff..., 007fff..., 00ffff..., 01ffff..., +## ..., 7fffff..., the converted values are twice as big as they should be. +## +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking if correctly converting (unsigned) long long to long double values" >&5 +$as_echo_n "checking if correctly converting (unsigned) long long to long double values... " >&6; } + +if test ${ac_cv_sizeof_long_double} = 0; then + hdf5_cv_llong_to_ldouble_correct=${hdf5_cv_llong_to_ldouble_correct=no} +else + if ${hdf5_cv_llong_to_ldouble_correct+:} false; then : + $as_echo_n "(cached) " >&6 +else + if test "$cross_compiling" = yes; then : + { { $as_echo "$as_me:${as_lineno-$LINENO}: error: in \`$ac_pwd':" >&5 +$as_echo "$as_me: error: in \`$ac_pwd':" >&2;} +as_fn_error $? "cannot run test program while cross compiling +See \`config.log' for more details" "$LINENO" 5; } +else + cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + + int main(void) + { + long double ld; + long long ll; + unsigned long long ull; + unsigned char s[16]; + int flag=0, ret=0; + + /*Determine if long double has 16 byte in size, 11 bit exponent, and + *the bias is 0x3ff */ + if(sizeof(long double) == 16) { + ld = 1.0L; + memcpy(s, &ld, 16); + if(s[0]==0x3f && s[1]==0xf0 && s[2]==0x00 && s[3]==0x00 && + s[4]==0x00 && s[5]==0x00 && s[6]==0x00 && s[7]==0x00) + flag = 1; + } + + if(flag==1 && sizeof(long long)==8) { + ll = 0x01ffffffffffffffLL; + ld = (long double)ll; + memcpy(s, &ld, 16); + /*Check if the bit sequence is as supposed to be*/ + if(s[0]!=0x43 || s[1]!=0x7f || s[2]!=0xff || s[3]!=0xff || + s[4]!=0xff || s[5]!=0xff || s[6]!=0xff || s[7]!=0xff || + s[8]!=0xf0 || s[9]!=0x00 || s[10]!=0x00 || s[11]!=0x00) + ret = 1; + } + if(flag==1 && sizeof(unsigned long long)==8) { + ull = 0x01ffffffffffffffULL; + ld = (long double)ull; + memcpy(s, &ld, 16); + if(s[0]!=0x43 || s[1]!=0x7f || s[2]!=0xff || s[3]!=0xff || + s[4]!=0xff || s[5]!=0xff || s[6]!=0xff || s[7]!=0xff || + s[8]!=0xf0 || s[9]!=0x00 || s[10]!=0x00 || s[11]!=0x00) + ret = 1; + } + done: + exit(ret); + } + +_ACEOF +if ac_fn_c_try_run "$LINENO"; then : + hdf5_cv_llong_to_ldouble_correct=yes +else + hdf5_cv_llong_to_ldouble_correct=no +fi +rm -f core *.core core.conftest.* gmon.out bb.out conftest$ac_exeext \ + conftest.$ac_objext conftest.beam conftest.$ac_ext +fi + +fi + +fi + +if test ${hdf5_cv_llong_to_ldouble_correct} = "yes"; then + +$as_echo "#define LLONG_TO_LDOUBLE_CORRECT 1" >>confdefs.h + + { $as_echo "$as_me:${as_lineno-$LINENO}: result: yes" >&5 +$as_echo "yes" >&6; } +else + { $as_echo "$as_me:${as_lineno-$LINENO}: result: no" >&5 +$as_echo "no" >&6; } +fi + +## ---------------------------------------------------------------------- ## Set some variables for general configuration information to be saved ## and installed with the libraries (used to generate libhdf5.settings). ## @@ -28577,7 +28803,7 @@ Usage: $0 [OPTIONS] Report bugs to <bug-libtool@gnu.org>." lt_cl_version="\ -HDF5 config.lt 1.9.222 +HDF5 config.lt 1.9.227 configured by $0, generated by GNU Autoconf 2.69. Copyright (C) 2011 Free Software Foundation, Inc. @@ -30693,7 +30919,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by HDF5 $as_me 1.9.222, which was +This file was extended by HDF5 $as_me 1.9.227, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -30759,7 +30985,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -HDF5 config.status 1.9.222 +HDF5 config.status 1.9.227 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" @@ -33134,16 +33360,6 @@ if test "X$HDF_CXX" = "Xyes"; then chmod 755 c++/src/h5c++ fi -## We don't want inline defined for C++ compilers -## Don't worry about the C++ ifdef wrappers in the H5pubconf file, since -## 'H5_inline' isn't a C++ keyword. -cat >> src/H5config.h <<EOF - -#if defined(__cplusplus) && defined(inline) -#undef inline -#endif -EOF - ## HDF5 configure code created by autotools with gcc 4.9.2 is adding problematic ## linker flags: -l with no library name; -l <libname>, specifically gfortran or m. ## This sed script corrects "-l <libname>" first and then "-l " with no library name. diff --git a/configure.ac b/configure.ac index f9776dc..1e67f39 100644 --- a/configure.ac +++ b/configure.ac @@ -26,7 +26,7 @@ AC_PREREQ([2.69]) ## NOTE: Do not forget to change the version number here when we do a ## release!!! ## -AC_INIT([HDF5], [1.9.222], [help@hdfgroup.org]) +AC_INIT([HDF5], [1.9.227], [help@hdfgroup.org]) AC_CONFIG_SRCDIR([src/H5.c]) AC_CONFIG_HEADER([src/H5config.h]) @@ -1691,7 +1691,27 @@ AC_CHECK_FUNCS([tmpfile asprintf vasprintf vsnprintf waitpid]) ## Check compiler characteristics ## AC_C_CONST -AC_C_INLINE + +AC_MSG_CHECKING([if the compiler understands __inline__]) +AC_COMPILE_IFELSE( + [AC_LANG_SOURCE([[static __inline__ void f(void){return;};]])], + [AC_DEFINE([HAVE___INLINE__], [1], [Define if the compiler understands __inline__]) AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no])] +) + +AC_MSG_CHECKING([if the compiler understands __inline]) +AC_COMPILE_IFELSE( + [AC_LANG_SOURCE([[static __inline void f(void){return;};]])], + [AC_DEFINE([HAVE___INLINE], [1], [Define if the compiler understands __inline]) AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no])] +) + +AC_MSG_CHECKING([if the compiler understands inline]) +AC_COMPILE_IFELSE( + [AC_LANG_SOURCE([[static inline void f(void){return;};]])], + [AC_DEFINE([HAVE_INLINE], [1], [Define if the compiler understands inline]) AC_MSG_RESULT([yes])], + [AC_MSG_RESULT([no])] +) AC_MSG_CHECKING([for __attribute__ extension]) AC_TRY_COMPILE(,[int __attribute__((unused)) x], @@ -2249,6 +2269,27 @@ else fi ## ---------------------------------------------------------------------- +## Decide whether the data accuracy has higher priority during data +## conversions. If not, some hard conversions will still be prefered even +## though the data may be wrong (for example, some compilers don't +## support denormalized floating values) to maximize speed. +## +AC_MSG_CHECKING([whether data accuracy is guaranteed during data conversions]) +AC_ARG_ENABLE([dconv-accuracy], + [AS_HELP_STRING([--enable-dconv-accuracy], + [if data accuracy is guaranteed during + data conversions [default=yes]])], + [DATA_ACCURACY=$enableval], [DATA_ACCURACY=yes]) + +if test "$DATA_ACCURACY" = "yes"; then + AC_MSG_RESULT([yes]) + AC_DEFINE([WANT_DATA_ACCURACY], [1], + [Data accuracy is prefered to speed during data conversions]) +else + AC_MSG_RESULT([no]) +fi + +## ---------------------------------------------------------------------- ## Set the flag to indicate that the machine has window style pathname, ## that is, "drive-letter:\" (e.g. "C:") or "drive-letter:/" (e.g. "C:/"). ## (This flag should be _unset_ for all machines, except for Windows, where @@ -2428,6 +2469,138 @@ else fi ## ---------------------------------------------------------------------- +## Set the flag to indicate that the machine can accurately convert +## 'long double' to '(unsigned) long long' values. (This flag should +## be set for all machines, except for Mac OS 10.4, SGI IRIX64 6.5 and +## Powerpc Linux using XL compilers. +## When the bit sequence of long double is 0x4351ccf385ebc8a0bfcc2a3c..., +## the values of (unsigned)long long start to go wrong on these +## two machines. Adjusting it higher to 0x4351ccf385ebc8a0dfcc... or +## 0x4351ccf385ebc8a0ffcc... will make the converted values wildly wrong. +## This test detects this wrong behavior and disable the test. +## +AC_MSG_CHECKING([if correctly converting long double to (unsigned) long long values]) + +if test ${ac_cv_sizeof_long_double} = 0; then + hdf5_cv_ldouble_to_llong_accurate=${hdf5_cv_ldouble_to_llong_accurate=no} +else + AC_CACHE_VAL([hdf5_cv_ldouble_to_llong_accurate], + [AC_TRY_RUN([ + int main(void) + { + long double ld = 20041683600089727.779961L; + long long ll; + unsigned long long ull; + unsigned char s[16]; + int ret = 0; + + if(sizeof(long double) == 16) { + /*make sure the long double type is the same as the failing type + *which has 16 bytes in size and 11 bits of exponent. If it is, + *the bit sequence should be like below. It's not + *a decent way to check but this info isn't available. */ + memcpy(s, &ld, 16); + if(s[0]==0x43 && s[1]==0x51 && s[2]==0xcc && s[3]==0xf3 && + s[4]==0x85 && s[5]==0xeb && s[6]==0xc8 && s[7]==0xa0 && + s[8]==0xbf && s[9]==0xcc && s[10]==0x2a && s[11]==0x3c) { + + /*slightly adjust the bit sequence (s[8]=0xdf). The converted + *values will go wild on Mac OS 10.4 and IRIX64 6.5.*/ + s[0]=0x43; s[1]=0x51; s[2]=0xcc; s[3]=0xf3; + s[4]=0x85; s[5]=0xeb; s[6]=0xc8; s[7]=0xa0; + s[8]=0xdf; s[9]=0xcc; s[10]=0x2a; s[11]=0x3c; + s[12]=0x3d; s[13]=0x85; s[14]=0x56; s[15]=0x20; + + memcpy(&ld, s, 16); + ll = (long long)ld; + ull = (unsigned long long)ld; + + if(ll != 20041683600089728 || ull != 20041683600089728) + ret = 1; + } + } + done: + exit(ret); + } + ], [hdf5_cv_ldouble_to_llong_accurate=yes], [hdf5_cv_ldouble_to_llong_accurate=no],)]) +fi + +if test ${hdf5_cv_ldouble_to_llong_accurate} = "yes"; then + AC_DEFINE([LDOUBLE_TO_LLONG_ACCURATE], [1], + [Define if your system can convert long double to (unsigned) long long values correctly.]) + AC_MSG_RESULT([yes]) +else + AC_MSG_RESULT([no]) +fi + + +## ---------------------------------------------------------------------- +## Set the flag to indicate that the machine can accurately convert +## '(unsigned) long long' to 'long double' values. (This flag should be +## set for all machines, except for Mac OS 10.4 and Powerpc Linux using +## XL compilers. +## When the bit sequences are 003fff..., 007fff..., 00ffff..., 01ffff..., +## ..., 7fffff..., the converted values are twice as big as they should be. +## +AC_MSG_CHECKING([if correctly converting (unsigned) long long to long double values]) + +if test ${ac_cv_sizeof_long_double} = 0; then + hdf5_cv_llong_to_ldouble_correct=${hdf5_cv_llong_to_ldouble_correct=no} +else + AC_CACHE_VAL([hdf5_cv_llong_to_ldouble_correct], + [AC_TRY_RUN([ + int main(void) + { + long double ld; + long long ll; + unsigned long long ull; + unsigned char s[16]; + int flag=0, ret=0; + + /*Determine if long double has 16 byte in size, 11 bit exponent, and + *the bias is 0x3ff */ + if(sizeof(long double) == 16) { + ld = 1.0L; + memcpy(s, &ld, 16); + if(s[0]==0x3f && s[1]==0xf0 && s[2]==0x00 && s[3]==0x00 && + s[4]==0x00 && s[5]==0x00 && s[6]==0x00 && s[7]==0x00) + flag = 1; + } + + if(flag==1 && sizeof(long long)==8) { + ll = 0x01ffffffffffffffLL; + ld = (long double)ll; + memcpy(s, &ld, 16); + /*Check if the bit sequence is as supposed to be*/ + if(s[0]!=0x43 || s[1]!=0x7f || s[2]!=0xff || s[3]!=0xff || + s[4]!=0xff || s[5]!=0xff || s[6]!=0xff || s[7]!=0xff || + s[8]!=0xf0 || s[9]!=0x00 || s[10]!=0x00 || s[11]!=0x00) + ret = 1; + } + if(flag==1 && sizeof(unsigned long long)==8) { + ull = 0x01ffffffffffffffULL; + ld = (long double)ull; + memcpy(s, &ld, 16); + if(s[0]!=0x43 || s[1]!=0x7f || s[2]!=0xff || s[3]!=0xff || + s[4]!=0xff || s[5]!=0xff || s[6]!=0xff || s[7]!=0xff || + s[8]!=0xf0 || s[9]!=0x00 || s[10]!=0x00 || s[11]!=0x00) + ret = 1; + } + done: + exit(ret); + } + ], [hdf5_cv_llong_to_ldouble_correct=yes], [hdf5_cv_llong_to_ldouble_correct=no],)]) +fi + +if test ${hdf5_cv_llong_to_ldouble_correct} = "yes"; then + AC_DEFINE([LLONG_TO_LDOUBLE_CORRECT], [1], + [Define if your system can convert (unsigned) long long to long double values correctly.]) + AC_MSG_RESULT([yes]) +else + AC_MSG_RESULT([no]) +fi + +## ---------------------------------------------------------------------- ## Set some variables for general configuration information to be saved ## and installed with the libraries (used to generate libhdf5.settings). ## @@ -2904,16 +3077,6 @@ if test "X$HDF_CXX" = "Xyes"; then chmod 755 c++/src/h5c++ fi -## We don't want inline defined for C++ compilers -## Don't worry about the C++ ifdef wrappers in the H5pubconf file, since -## 'H5_inline' isn't a C++ keyword. -cat >> src/H5config.h <<EOF - -#if defined(__cplusplus) && defined(inline) -#undef inline -#endif -EOF - ## HDF5 configure code created by autotools with gcc 4.9.2 is adding problematic ## linker flags: -l with no library name; -l <libname>, specifically gfortran or m. ## This sed script corrects "-l <libname>" first and then "-l " with no library name. diff --git a/fortran/src/CMakeLists.txt b/fortran/src/CMakeLists.txt index c015072..0469aac 100644 --- a/fortran/src/CMakeLists.txt +++ b/fortran/src/CMakeLists.txt @@ -268,15 +268,15 @@ set_target_properties (${HDF5_F90_LIB_TARGET} PROPERTIES # Add file(s) to CMake Install #----------------------------------------------------------------------------- install ( - FILES - ${HDF5_F90_SRC_SOURCE_DIR}/H5f90.h - ${HDF5_F90_SRC_SOURCE_DIR}/H5f90i.h - ${HDF5_F90_SRC_SOURCE_DIR}/H5f90proto.h + FILES + ${HDF5_F90_SRC_SOURCE_DIR}/H5f90.h + ${HDF5_F90_SRC_SOURCE_DIR}/H5f90i.h + ${HDF5_F90_SRC_SOURCE_DIR}/H5f90proto.h ${HDF5_F90_BINARY_DIR}/H5f90i_gen.h ${HDF5_F90_BINARY_DIR}/H5fortran_types.f90 - DESTINATION + DESTINATION ${HDF5_INSTALL_INCLUDE_DIR} - COMPONENT + COMPONENT fortheaders ) @@ -343,5 +343,6 @@ if (HDF5_EXPORTED_TARGETS) LIBRARY DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT fortlibraries ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT fortlibraries RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT fortlibraries + FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT fortlibraries ) endif (HDF5_EXPORTED_TARGETS) diff --git a/fortran/src/Makefile.in b/fortran/src/Makefile.in index 2c36b8c..aba85ec 100644 --- a/fortran/src/Makefile.in +++ b/fortran/src/Makefile.in @@ -724,7 +724,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog *.clog2 # After making changes, run bin/reconfigure to update other configure related # files like Makefile.in. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 212 +LT_VERS_REVISION = 217 LT_VERS_AGE = 0 AM_FCLIBS = $(LIBHDF5) diff --git a/hl/c++/src/CMakeLists.txt b/hl/c++/src/CMakeLists.txt index 640cb2d..b511260 100644 --- a/hl/c++/src/CMakeLists.txt +++ b/hl/c++/src/CMakeLists.txt @@ -25,7 +25,7 @@ set_target_properties (${HDF5_HL_CPP_LIB_TARGET} PROPERTIES ) #----------------------------------------------------------------------------- -# Add file(s) to CMake Install +# Add file(s) to CMake Install #----------------------------------------------------------------------------- install ( FILES @@ -43,7 +43,7 @@ if (HDF5_EXPORTED_TARGETS) if (BUILD_SHARED_LIBS) INSTALL_TARGET_PDB (${HDF5_HL_CPP_LIB_TARGET} ${HDF5_INSTALL_BIN_DIR} hlcpplibraries) endif (BUILD_SHARED_LIBS) - + install ( TARGETS ${HDF5_HL_CPP_LIB_TARGET} @@ -52,5 +52,6 @@ if (HDF5_EXPORTED_TARGETS) LIBRARY DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT hlcpplibraries ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT hlcpplibraries RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT hlcpplibraries + FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT hlcpplibraries ) endif (HDF5_EXPORTED_TARGETS) diff --git a/hl/c++/src/Makefile.in b/hl/c++/src/Makefile.in index 6b6bc67..742b35c 100644 --- a/hl/c++/src/Makefile.in +++ b/hl/c++/src/Makefile.in @@ -662,7 +662,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog *.clog2 # After making changes, run bin/reconfigure to update other configure related # files like Makefile.in. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 212 +LT_VERS_REVISION = 217 LT_VERS_AGE = 0 # This is our main target diff --git a/hl/fortran/src/CMakeLists.txt b/hl/fortran/src/CMakeLists.txt index 6e6e44a..e5ffb78 100644 --- a/hl/fortran/src/CMakeLists.txt +++ b/hl/fortran/src/CMakeLists.txt @@ -50,8 +50,8 @@ add_library (${HDF5_HL_F90_LIB_TARGET} ${LIB_TYPE} ${HDF5_HL_F90_F_SRCS}) set (SHARED_LINK_FLAGS " ") if (BUILD_SHARED_LIBS) if (WIN32) - set_property (TARGET ${HDF5_HL_F90_LIB_TARGET} - APPEND PROPERTY COMPILE_DEFINITIONS + set_property (TARGET ${HDF5_HL_F90_LIB_TARGET} + APPEND PROPERTY COMPILE_DEFINITIONS BUILD_HDF5_HL_DLL ) if (MSVC) @@ -60,8 +60,8 @@ if (BUILD_SHARED_LIBS) endif (WIN32) endif (BUILD_SHARED_LIBS) if (WIN32) - set_property (TARGET ${HDF5_HL_F90_LIB_TARGET} - APPEND PROPERTY COMPILE_DEFINITIONS + set_property (TARGET ${HDF5_HL_F90_LIB_TARGET} + APPEND PROPERTY COMPILE_DEFINITIONS HDF5F90_WINDOWS ) endif (WIN32) @@ -110,7 +110,7 @@ if (HDF5_EXPORTED_TARGETS) INSTALL_TARGET_PDB (${HDF5_HL_F90_C_LIB_TARGET} ${HDF5_INSTALL_BIN_DIR} hlfortlibraries) #INSTALL_TARGET_PDB (${HDF5_HL_F90_LIB_TARGET} ${HDF5_INSTALL_BIN_DIR} hlfortlibraries) endif (BUILD_SHARED_LIBS) - + install ( TARGETS ${HDF5_HL_F90_C_LIB_TARGET} @@ -120,5 +120,6 @@ if (HDF5_EXPORTED_TARGETS) LIBRARY DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT hlfortlibraries ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT hlfortlibraries RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT hlfortlibraries + FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT hlfortlibraries ) endif (HDF5_EXPORTED_TARGETS) diff --git a/hl/fortran/src/Makefile.in b/hl/fortran/src/Makefile.in index 8820964..029e5a1 100644 --- a/hl/fortran/src/Makefile.in +++ b/hl/fortran/src/Makefile.in @@ -680,7 +680,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog *.clog2 # After making changes, run bin/reconfigure to update other configure related # files like Makefile.in. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 212 +LT_VERS_REVISION = 217 LT_VERS_AGE = 0 # Our main target, the high-level fortran library diff --git a/hl/src/CMakeLists.txt b/hl/src/CMakeLists.txt index 3c24e40..47029d7 100644 --- a/hl/src/CMakeLists.txt +++ b/hl/src/CMakeLists.txt @@ -44,7 +44,7 @@ set_target_properties (${HDF5_HL_LIB_TARGET} PROPERTIES set_global_variable (HDF5_LIBRARIES_TO_EXPORT "${HDF5_LIBRARIES_TO_EXPORT};${HDF5_HL_LIB_TARGET}") #----------------------------------------------------------------------------- -# Add file(s) to CMake Install +# Add file(s) to CMake Install #----------------------------------------------------------------------------- install ( FILES @@ -62,7 +62,7 @@ if (HDF5_EXPORTED_TARGETS) if (BUILD_SHARED_LIBS) INSTALL_TARGET_PDB (${HDF5_HL_LIB_TARGET} ${HDF5_INSTALL_BIN_DIR} hllibraries) endif (BUILD_SHARED_LIBS) - + install ( TARGETS ${HDF5_HL_LIB_TARGET} @@ -71,5 +71,6 @@ if (HDF5_EXPORTED_TARGETS) LIBRARY DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT hllibraries ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT hllibraries RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT hllibraries + FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT hllibraries ) endif (HDF5_EXPORTED_TARGETS) diff --git a/hl/src/Makefile.in b/hl/src/Makefile.in index 314b11d..d5b0b1e 100644 --- a/hl/src/Makefile.in +++ b/hl/src/Makefile.in @@ -661,7 +661,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog *.clog2 # After making changes, run bin/reconfigure to update other configure related # files like Makefile.in. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 212 +LT_VERS_REVISION = 217 LT_VERS_AGE = 0 # This library is our main target. diff --git a/hl/tools/CMakeLists.txt b/hl/tools/CMakeLists.txt index f58a513..c9017f3 100644 --- a/hl/tools/CMakeLists.txt +++ b/hl/tools/CMakeLists.txt @@ -46,7 +46,7 @@ if (BUILD_TESTING) TARGET_C_PROPERTIES (hl_h52gifgentest STATIC " " " ") target_link_libraries (hl_h52gifgentest ${HDF5_HL_LIB_TARGET} ${HDF5_LIB_TARGET}) set_target_properties (hl_h52gifgentest PROPERTIES FOLDER generator/tools/hl) - + # add_test (NAME hl_h52gifgentest COMMAND $<TARGET_FILE:hl_h52gifgentest>) endif (HDF5_BUILD_GENERATORS AND NOT BUILD_SHARED_LIBS) @@ -60,8 +60,5 @@ install ( TARGETS gif2h5 h52gif - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - hltoolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT hltoolsapplications ) diff --git a/release_docs/INSTALL_CMake.txt b/release_docs/INSTALL_CMake.txt index 7f67eb2..7a98bef 100644 --- a/release_docs/INSTALL_CMake.txt +++ b/release_docs/INSTALL_CMake.txt @@ -591,6 +591,7 @@ HDF5_STRICT_FORMAT_CHECKS "Whether to perform strict file format checks" HDF5_TEST_VFD "Execute tests with different VFDs" OFF HDF5_USE_16_API_DEFAULT "Use the HDF5 1.6.x API by default" OFF HDF5_USE_FOLDERS "Enable folder grouping of projects in IDEs." OFF +HDF5_WANT_DATA_ACCURACY "IF data accuracy is guaranteed during data conversions" ON HDF5_WANT_DCONV_EXCEPTION "exception handling functions is checked during data conversions" ON HDF5_ENABLE_THREADSAFE "Enable Threadsafety" OFF if (APPLE) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 18b800e..d034cd3 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1,4 +1,4 @@ -HDF5 version 1.9.222 currently under development +HDF5 version 1.9.227 currently under development ================================================================================ diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index e5f7e2f..11b37a5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -44,6 +44,7 @@ IDE_GENERATED_PROPERTIES ("H5A" "${H5A_HDRS}" "${H5A_SRCS}" ) set (H5AC_SRCS ${HDF5_SRC_DIR}/H5AC.c + ${HDF5_SRC_DIR}/H5ACmpio.c ) set (H5AC_HDRS @@ -82,6 +83,7 @@ IDE_GENERATED_PROPERTIES ("H5B2" "${H5B2_HDRS}" "${H5B2_SRCS}" ) set (H5C_SRCS ${HDF5_SRC_DIR}/H5C.c + ${HDF5_SRC_DIR}/H5Cmpio.c ) set (H5C_HDRS ${HDF5_SRC_DIR}/H5Cpkg.h @@ -896,7 +898,7 @@ if (HDF5_EXPORTED_TARGETS) if (BUILD_SHARED_LIBS) INSTALL_TARGET_PDB (${HDF5_LIB_TARGET} ${HDF5_INSTALL_BIN_DIR} libraries) endif (BUILD_SHARED_LIBS) - + install ( TARGETS ${HDF5_LIB_TARGET} @@ -905,5 +907,6 @@ if (HDF5_EXPORTED_TARGETS) LIBRARY DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT libraries ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT libraries RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT libraries + FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT libraries ) endif (HDF5_EXPORTED_TARGETS) @@ -145,6 +145,9 @@ H5_init_library(void) if(MPI_SUCCESS != (mpi_code = MPI_Comm_set_attr(MPI_COMM_SELF, key_val, NULL))) HMPI_GOTO_ERROR(FAIL, "MPI_Comm_set_attr failed", mpi_code) + + if(MPI_SUCCESS != (mpi_code = MPI_Comm_free_keyval(&key_val))) + HMPI_GOTO_ERROR(FAIL, "MPI_Comm_free_keyval failed", mpi_code) } } #endif /*H5_HAVE_PARALLEL*/ @@ -32,7 +32,6 @@ /****************/ #define H5AC_PACKAGE /*suppress error about including H5ACpkg */ -#define H5C_PACKAGE /*suppress error about including H5Cpkg */ #define H5F_PACKAGE /*suppress error about including H5Fpkg */ /* Interface initialization */ @@ -43,14 +42,11 @@ /***********/ #include "H5private.h" /* Generic Functions */ #include "H5ACpkg.h" /* Metadata cache */ -#include "H5Cpkg.h" /* Cache */ -#include "H5Dprivate.h" /* Dataset functions */ +#include "H5Cprivate.h" /* Cache */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* Files */ #include "H5FDprivate.h" /* File drivers */ -#include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ -#include "H5MMprivate.h" /* Memory management */ #include "H5Pprivate.h" /* Property lists */ @@ -63,38 +59,6 @@ /* Local Typedefs */ /******************/ -#ifdef H5_HAVE_PARALLEL -/**************************************************************************** - * - * structure H5AC_slist_entry_t - * - * The dirty entry list maintained via the d_slist_ptr field of H5AC_aux_t - * and the cleaned entry list maintained via the c_slist_ptr field of - * H5AC_aux_t are just lists of the file offsets of the dirty/cleaned - * entries. Unfortunately, the slist code makes us define a dynamically - * allocated structure to store these offsets in. This structure serves - * that purpose. Its fields are as follows: - * - * addr: file offset of a metadata entry. Entries are added to this - * list (if they aren't there already) when they are marked - * dirty in an unprotect, inserted, or moved. They are - * removed when they appear in a clean entries broadcast. - * - ****************************************************************************/ -typedef struct H5AC_slist_entry_t -{ - haddr_t addr; -} H5AC_slist_entry_t; - -/* User data for address list building callbacks */ -typedef struct H5AC_addr_list_ud_t -{ - H5AC_aux_t * aux_ptr; /* 'Auxiliary' parallel cache info */ - haddr_t * addr_buf_ptr; /* Array to store addresses */ - int i; /* Counter for position in array */ -} H5AC_addr_list_ud_t; -#endif /* H5_HAVE_PARALLEL */ - /********************/ /* Local Prototypes */ @@ -105,39 +69,6 @@ static herr_t H5AC__check_if_write_permitted(const H5F_t *f, static herr_t H5AC__ext_config_2_int_config(H5AC_cache_config_t *ext_conf_ptr, H5C_auto_size_ctl_t *int_conf_ptr); -#ifdef H5_HAVE_PARALLEL -static herr_t H5AC__broadcast_candidate_list(H5AC_t *cache_ptr, - int *num_entries_ptr, haddr_t **haddr_buf_ptr_ptr); -static herr_t H5AC__broadcast_clean_list(H5AC_t *cache_ptr); -static herr_t H5AC__construct_candidate_list(H5AC_t *cache_ptr, - H5AC_aux_t *aux_ptr, int sync_point_op); -static herr_t H5AC__copy_candidate_list_to_buffer(const H5AC_t *cache_ptr, - int *num_entries_ptr, haddr_t **haddr_buf_ptr_ptr); -static herr_t H5AC__flush_entries(H5F_t *f, hid_t dxpl_id); -static herr_t H5AC__log_deleted_entry(const H5AC_info_t *entry_ptr); -static herr_t H5AC__log_dirtied_entry(const H5AC_info_t *entry_ptr); -static herr_t H5AC__log_flushed_entry(H5C_t *cache_ptr, haddr_t addr, - hbool_t was_dirty, unsigned flags); -static herr_t H5AC__log_inserted_entry(const H5AC_info_t *entry_ptr); -static herr_t H5AC__log_moved_entry(const H5F_t *f, haddr_t old_addr, - haddr_t new_addr); -static herr_t H5AC__propagate_and_apply_candidate_list(H5F_t *f, hid_t dxpl_id); -static herr_t H5AC__propagate_flushed_and_still_clean_entries_list(H5F_t *f, - hid_t dxpl_id); -static herr_t H5AC__receive_haddr_list(MPI_Comm mpi_comm, int *num_entries_ptr, - haddr_t **haddr_buf_ptr_ptr); -static herr_t H5AC__receive_candidate_list(const H5AC_t *cache_ptr, - int *num_entries_ptr, haddr_t **haddr_buf_ptr_ptr); -static herr_t H5AC__receive_and_apply_clean_list(H5F_t *f, hid_t dxpl_id); -static herr_t H5AC__tidy_cache_0_lists(H5AC_t *cache_ptr, int num_candidates, - haddr_t *candidates_list_ptr); -static herr_t H5AC__rsp__dist_md_write__flush(H5F_t *f, hid_t dxpl_id); -static herr_t H5AC__rsp__dist_md_write__flush_to_min_clean(H5F_t *f, hid_t dxpl_id); -static herr_t H5AC__rsp__p0_only__flush(H5F_t *f, hid_t dxpl_id); -static herr_t H5AC__rsp__p0_only__flush_to_min_clean(H5F_t *f, hid_t dxpl_id); -static herr_t H5AC__run_sync_point(H5F_t *f, hid_t dxpl_id, int sync_point_op); -#endif /* H5_HAVE_PARALLEL */ - /*********************/ /* Package Variables */ @@ -165,14 +96,6 @@ hid_t H5AC_ind_write_coll_read_dxpl_id=(-1); /* Local Variables */ /*******************/ -#ifdef H5_HAVE_PARALLEL -/* Declare a free list to manage the H5AC_aux_t struct */ -H5FL_DEFINE_STATIC(H5AC_aux_t); - -/* Declare a free list to manage the H5AC_slist_entry_t struct */ -H5FL_DEFINE_STATIC(H5AC_slist_entry_t); -#endif /* H5_HAVE_PARALLEL */ - static const char *H5AC_entry_type_names[H5AC_NTYPES] = { "B-tree nodes", @@ -620,7 +543,7 @@ H5AC_dest(H5F_t *f, hid_t dxpl_id) if(H5C_clear_coll_entries(f->shared->cache, 0) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed.") - aux_ptr = (struct H5AC_aux_t *)(f->shared->cache->aux_ptr); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(f->shared->cache); if(aux_ptr) /* Sanity check */ HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); @@ -912,7 +835,7 @@ H5AC_insert_entry(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t add { H5AC_aux_t *aux_ptr; - if(NULL != (aux_ptr = (H5AC_aux_t *)f->shared->cache->aux_ptr)) { + if(NULL != (aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(f->shared->cache))) { /* Log the new entry */ if(H5AC__log_inserted_entry((H5AC_info_t *)thing) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTINS, FAIL, "H5AC__log_inserted_entry() failed") @@ -976,12 +899,11 @@ H5AC_mark_entry_dirty(void *thing) { H5AC_info_t *entry_ptr = (H5AC_info_t *)thing; H5C_t *cache_ptr = entry_ptr->cache_ptr; + H5AC_aux_t *aux_ptr; - HDassert(cache_ptr); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); if((!entry_ptr->is_dirty) && (!entry_ptr->is_protected) && - (entry_ptr->is_pinned) && (NULL != cache_ptr->aux_ptr)) + (entry_ptr->is_pinned) && (NULL != aux_ptr)) if(H5AC__log_dirtied_entry(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "can't log dirtied entry") } @@ -1048,7 +970,7 @@ H5AC_move_entry(H5F_t *f, const H5AC_class_t *type, haddr_t old_addr, haddr_t ne #ifdef H5_HAVE_PARALLEL /* Log moving the entry */ - if(NULL != (aux_ptr = (H5AC_aux_t *)f->shared->cache->aux_ptr)) + if(NULL != (aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(f->shared->cache))) if(H5AC__log_moved_entry(f, old_addr, new_addr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "can't log moved entry") #endif /* H5_HAVE_PARALLEL */ @@ -1308,11 +1230,10 @@ H5AC_resize_entry(void *thing, size_t new_size) { H5AC_info_t * entry_ptr = (H5AC_info_t *)thing; H5C_t *cache_ptr = entry_ptr->cache_ptr; + H5AC_aux_t *aux_ptr; - HDassert(cache_ptr); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - - if((!entry_ptr->is_dirty) && (NULL != cache_ptr->aux_ptr)) + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + if((!entry_ptr->is_dirty) && (NULL != aux_ptr)) if(H5AC__log_dirtied_entry(entry_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTMARKDIRTY, FAIL, "can't log dirtied entry") } @@ -1520,7 +1441,7 @@ H5AC_unprotect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, } /* end if */ #ifdef H5_HAVE_PARALLEL - if(NULL != (aux_ptr = (H5AC_aux_t *)f->shared->cache->aux_ptr)) { + if(NULL != (aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(f->shared->cache))) { if(dirtied && ((H5AC_info_t *)thing)->is_dirty == FALSE) if(H5AC__log_dirtied_entry((H5AC_info_t *)thing) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "can't log dirtied entry") @@ -1531,7 +1452,7 @@ H5AC_unprotect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, } /* end if */ #endif /* H5_HAVE_PARALLEL */ - if(H5C_unprotect(f, dxpl_id, type, addr, thing, flags) < 0) + if(H5C_unprotect(f, dxpl_id, addr, thing, flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "H5C_unprotect() failed.") #ifdef H5_HAVE_PARALLEL @@ -1550,78 +1471,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5AC_unprotect() */ - -/*------------------------------------------------------------------------- - * Function: HA5C_set_sync_point_done_callback - * - * Purpose: Set the value of the sync_point_done callback. This - * callback is used by the parallel test code to verify - * that the expected writes and only the expected writes - * take place during a sync point. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 5/9/10 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -herr_t -H5AC_set_sync_point_done_callback(H5C_t * cache_ptr, - void (* sync_point_done)(int num_writes, haddr_t * written_entries_tbl)) -{ - H5AC_aux_t * aux_ptr; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - /* Sanity checks */ - HDassert(cache_ptr && (cache_ptr->magic == H5C__H5C_T_MAGIC)); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - - aux_ptr->sync_point_done = sync_point_done; - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5AC_set_sync_point_done_callback() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: HA5C_set_write_done_callback - * - * Purpose: Set the value of the write_done callback. This callback - * is used to improve performance of the parallel test bed - * for the cache. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 5/11/06 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -herr_t -H5AC_set_write_done_callback(H5C_t * cache_ptr, void (* write_done)(void)) -{ - H5AC_aux_t * aux_ptr; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - /* Sanity checks */ - HDassert(cache_ptr && (cache_ptr->magic == H5C__H5C_T_MAGIC)); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert( aux_ptr != NULL ); - HDassert( aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC ); - - aux_ptr->write_done = write_done; - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5AC_set_write_done_callback() */ -#endif /* H5_HAVE_PARALLEL */ - #ifndef NDEBUG /* debugging functions */ /*------------------------------------------------------------------------- @@ -1710,14 +1559,18 @@ H5AC_get_cache_auto_resize_config(const H5AC_t *cache_ptr, FUNC_ENTER_NOAPI(FAIL) /* Check args */ - if((cache_ptr == NULL) || + if((cache_ptr == NULL) || (config_ptr == NULL) || + (config_ptr->version != H5AC__CURR_CACHE_CONFIG_VERSION)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr or config_ptr on entry.") #ifdef H5_HAVE_PARALLEL - ((cache_ptr->aux_ptr != NULL) && - (((H5AC_aux_t *)(cache_ptr->aux_ptr))->magic != H5AC__H5AC_AUX_T_MAGIC)) - || +{ + H5AC_aux_t *aux_ptr; + + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + if((aux_ptr != NULL) && (aux_ptr->magic != H5AC__H5AC_AUX_T_MAGIC)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad aux_ptr on entry.") +} #endif /* H5_HAVE_PARALLEL */ - (config_ptr == NULL) || (config_ptr->version != H5AC__CURR_CACHE_CONFIG_VERSION)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Bad cache_ptr or config_ptr on entry.") /* Retrieve the configuration */ if(H5C_get_cache_auto_resize_config((const H5C_t *)cache_ptr, &internal_config) < 0) @@ -1757,9 +1610,12 @@ H5AC_get_cache_auto_resize_config(const H5AC_t *cache_ptr, config_ptr->apply_empty_reserve = internal_config.apply_empty_reserve; config_ptr->empty_reserve = internal_config.empty_reserve; #ifdef H5_HAVE_PARALLEL - if(cache_ptr->aux_ptr != NULL) { - config_ptr->dirty_bytes_threshold = ((H5AC_aux_t *)(cache_ptr->aux_ptr))->dirty_bytes_threshold; - config_ptr->metadata_write_strategy = ((H5AC_aux_t *)(cache_ptr->aux_ptr))->metadata_write_strategy; +{ + H5AC_aux_t *aux_ptr; + + if(NULL != (aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr))) { + config_ptr->dirty_bytes_threshold = aux_ptr->dirty_bytes_threshold; + config_ptr->metadata_write_strategy = aux_ptr->metadata_write_strategy; } /* end if */ else { #endif /* H5_HAVE_PARALLEL */ @@ -1767,6 +1623,7 @@ H5AC_get_cache_auto_resize_config(const H5AC_t *cache_ptr, config_ptr->metadata_write_strategy = H5AC__DEFAULT_METADATA_WRITE_STRATEGY; #ifdef H5_HAVE_PARALLEL } /* end else */ +} #endif /* H5_HAVE_PARALLEL */ done: @@ -1892,13 +1749,17 @@ H5AC_set_cache_auto_resize_config(H5AC_t *cache_ptr, H5AC_cache_config_t *config trace_config = *config_ptr; #endif /* H5AC__TRACE_FILE_ENABLED */ - if((cache_ptr == NULL) + if(cache_ptr == NULL) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "bad cache_ptr on entry.") #ifdef H5_HAVE_PARALLEL - || ((cache_ptr->aux_ptr != NULL) && - (((H5AC_aux_t *)(cache_ptr->aux_ptr))->magic != H5AC__H5AC_AUX_T_MAGIC)) +{ + H5AC_aux_t *aux_ptr; + + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + if((aux_ptr != NULL) && (aux_ptr->magic != H5AC__H5AC_AUX_T_MAGIC)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "bad aux_ptr on entry.") +} #endif /* H5_HAVE_PARALLEL */ - ) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "bad cache_ptr on entry.") /* Validate external configuration */ if(H5AC_validate_config(config_ptr) != SUCCEED) @@ -1933,12 +1794,16 @@ H5AC_set_cache_auto_resize_config(H5AC_t *cache_ptr, H5AC_cache_config_t *config HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_set_evictions_enabled() failed.") #ifdef H5_HAVE_PARALLEL +{ + H5AC_aux_t *aux_ptr; + /* Set parallel configuration values */ /* (Which are only held in the H5AC layer -QAK) */ - if(cache_ptr->aux_ptr != NULL) { - ((H5AC_aux_t *)(cache_ptr->aux_ptr))->dirty_bytes_threshold = config_ptr->dirty_bytes_threshold; - ((H5AC_aux_t *)(cache_ptr->aux_ptr))->metadata_write_strategy = config_ptr->metadata_write_strategy; + if(NULL != (aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr))) { + aux_ptr->dirty_bytes_threshold = config_ptr->dirty_bytes_threshold; + aux_ptr->metadata_write_strategy = config_ptr->metadata_write_strategy; } /* end if */ +} #endif /* H5_HAVE_PARALLEL */ done: @@ -2129,9 +1994,6 @@ H5AC_open_trace_file(H5AC_t *cache_ptr, const char *trace_file_name) { char file_name[H5AC__MAX_TRACE_FILE_NAME_LEN + H5C__PREFIX_LEN + 2]; FILE * file_ptr = NULL; -#ifdef H5_HAVE_PARALLEL - H5AC_aux_t * aux_ptr = NULL; -#endif /* H5_HAVE_PARALLEL */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -2149,8 +2011,11 @@ H5AC_open_trace_file(H5AC_t *cache_ptr, const char *trace_file_name) HGOTO_ERROR(H5E_CACHE, H5E_FILEOPEN, FAIL, "trace file already open.") #ifdef H5_HAVE_PARALLEL - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - if(cache_ptr->aux_ptr == NULL) +{ + H5AC_aux_t * aux_ptr; + + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + if(aux_ptr == NULL) sprintf(file_name, "%s", trace_file_name); else { if(aux_ptr->magic != H5AC__H5AC_AUX_T_MAGIC) @@ -2161,6 +2026,7 @@ H5AC_open_trace_file(H5AC_t *cache_ptr, const char *trace_file_name) if(HDstrlen(file_name) > H5AC__MAX_TRACE_FILE_NAME_LEN + H5C__PREFIX_LEN + 1) HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "cooked trace file name too long.") +} #else /* H5_HAVE_PARALLEL */ HDsnprintf(file_name, (size_t)(H5AC__MAX_TRACE_FILE_NAME_LEN + H5C__PREFIX_LEN + 1), "%s", trace_file_name); @@ -2179,64 +2045,6 @@ done: } /* H5AC_open_trace_file() */ -/*------------------------------------------------------------------------- - * Function: H5AC_add_candidate() - * - * Purpose: Add the supplied metadata entry address to the candidate - * list. Verify that each entry added does not appear in - * the list prior to its insertion. - * - * This function is intended for used in constructing list - * of entried to be flushed during sync points. It shouldn't - * be called anywhere else. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 3/17/10 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -herr_t -H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr) -{ - H5AC_aux_t * aux_ptr; - H5AC_slist_entry_t * slist_entry_ptr = NULL; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - /* Sanity checks */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); - HDassert(aux_ptr->candidate_slist_ptr != NULL); - - /* Construct an entry for the supplied address, and insert - * it into the candidate slist. - */ - if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate candidate slist entry") - slist_entry_ptr->addr = addr; - - if(H5SL_insert(aux_ptr->candidate_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into dirty entry slist") - -done: - /* Clean up on error */ - if(ret_value < 0) - if(slist_entry_ptr) - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC_add_candidate() */ -#endif /* H5_HAVE_PARALLEL */ - - /*************************************************************************/ /*************************** Debugging Functions: ************************/ /*************************************************************************/ @@ -2348,236 +2156,6 @@ done: /**************************** Private Functions: *************************/ /*************************************************************************/ -/*------------------------------------------------------------------------- - * - * Function: H5AC__broadcast_candidate_list() - * - * Purpose: Broadcast the contents of the process 0 candidate entry - * slist. In passing, also remove all entries from said - * list. As the application of this will be handled by - * the same functions on all processes, construct and - * return a copy of the list in the same format as that - * received by the other processes. Note that if this - * copy is returned in *haddr_buf_ptr_ptr, the caller - * must free it. - * - * This function must only be called by the process with - * MPI_rank 0. - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: John Mainzer, 7/1/05 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__broadcast_candidate_list(H5AC_t *cache_ptr, int *num_entries_ptr, - haddr_t **haddr_buf_ptr_ptr) -{ - H5AC_aux_t * aux_ptr = NULL; - haddr_t * haddr_buf_ptr = NULL; - int mpi_result; - int num_entries; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->mpi_rank == 0); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); - HDassert(aux_ptr->candidate_slist_ptr != NULL); - HDassert(num_entries_ptr != NULL); - HDassert(*num_entries_ptr == 0); - HDassert(haddr_buf_ptr_ptr != NULL); - HDassert(*haddr_buf_ptr_ptr == NULL); - - /* First broadcast the number of entries in the list so that the - * receivers can set up buffers to receive them. If there aren't - * any, we are done. - */ - num_entries = (int)H5SL_count(aux_ptr->candidate_slist_ptr); - if(MPI_SUCCESS != (mpi_result = MPI_Bcast(&num_entries, 1, MPI_INT, 0, aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) - - if(num_entries > 0) { - size_t buf_size = 0; - int chk_num_entries = 0; - - /* convert the candidate list into the format we - * are used to receiving from process 0, and also load it - * into a buffer for transmission. - */ - if(H5AC__copy_candidate_list_to_buffer(cache_ptr, &chk_num_entries, &haddr_buf_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate buffer.") - HDassert(chk_num_entries == num_entries); - HDassert(haddr_buf_ptr != NULL); - - /* Now broadcast the list of candidate entries */ - buf_size = sizeof(haddr_t) * (size_t)num_entries; - if(MPI_SUCCESS != (mpi_result = MPI_Bcast((void *)haddr_buf_ptr, (int)buf_size, MPI_BYTE, 0, aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) - } /* end if */ - - /* Pass the number of entries and the buffer pointer - * back to the caller. Do this so that we can use the same code - * to apply the candidate list to all the processes. - */ - *num_entries_ptr = num_entries; - *haddr_buf_ptr_ptr = haddr_buf_ptr; - -done: - if(ret_value < 0) - if(haddr_buf_ptr) - haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__broadcast_candidate_list() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__broadcast_clean_list_cb() - * - * Purpose: Skip list callback for building array of addresses for - * broadcasting the clean list. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: Quincey Koziol, 6/12/15 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__broadcast_clean_list_cb(void *_item, void H5_ATTR_UNUSED *_key, - void *_udata) -{ - H5AC_slist_entry_t * slist_entry_ptr = (H5AC_slist_entry_t *)_item; /* Address of item */ - H5AC_addr_list_ud_t * udata = (H5AC_addr_list_ud_t *)_udata; /* Context for callback */ - haddr_t addr; - - FUNC_ENTER_STATIC_NOERR - - /* Sanity checks */ - HDassert(slist_entry_ptr); - HDassert(udata); - - /* Store the entry's address in the buffer */ - addr = slist_entry_ptr->addr; - udata->addr_buf_ptr[udata->i] = addr; - udata->i++; - - /* now release the entry */ - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - - /* and also remove the matching entry from the dirtied list - * if it exists. - */ - if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(udata->aux_ptr->d_slist_ptr, (void *)(&addr)))) - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5AC__broadcast_clean_list_cb() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__broadcast_clean_list() - * - * Purpose: Broadcast the contents of the process 0 cleaned entry - * slist. In passing, also remove all entries from said - * list, and also remove any matching entries from the dirtied - * slist. - * - * This function must only be called by the process with - * MPI_rank 0. - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: John Mainzer, 7/1/05 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__broadcast_clean_list(H5AC_t * cache_ptr) -{ - haddr_t * addr_buf_ptr = NULL; - H5AC_aux_t * aux_ptr; - int mpi_result; - int num_entries = 0; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)cache_ptr->aux_ptr; - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->mpi_rank == 0); - HDassert(aux_ptr->c_slist_ptr != NULL); - - /* First broadcast the number of entries in the list so that the - * receives can set up a buffer to receive them. If there aren't - * any, we are done. - */ - num_entries = (int)H5SL_count(aux_ptr->c_slist_ptr); - if(MPI_SUCCESS != (mpi_result = MPI_Bcast(&num_entries, 1, MPI_INT, 0, aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) - - if(num_entries > 0) { - H5AC_addr_list_ud_t udata; - size_t buf_size; - - /* allocate a buffer to store the list of entry base addresses in */ - buf_size = sizeof(haddr_t) * (size_t)num_entries; - if(NULL == (addr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for addr buffer") - - /* Set up user data for callback */ - udata.aux_ptr = aux_ptr; - udata.addr_buf_ptr = addr_buf_ptr; - udata.i = 0; - - /* Free all the clean list entries, building the address list in the callback */ - /* (Callback also removes the matching entries from the dirtied list) */ - if(H5SL_free(aux_ptr->c_slist_ptr, H5AC__broadcast_clean_list_cb, &udata) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "Can't build address list for clean entries") - - /* Now broadcast the list of cleaned entries */ - if(MPI_SUCCESS != (mpi_result = MPI_Bcast((void *)addr_buf_ptr, (int)buf_size, MPI_BYTE, 0, aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) - } /* end if */ - - /* if it is defined, call the sync point done callback. Note - * that this callback is defined purely for testing purposes, - * and should be undefined under normal operating circumstances. - */ - if(aux_ptr->sync_point_done) - (aux_ptr->sync_point_done)(num_entries, addr_buf_ptr); - -done: - if(addr_buf_ptr) - addr_buf_ptr = (haddr_t *)H5MM_xfree((void *)addr_buf_ptr); - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__broadcast_clean_list() */ -#endif /* H5_HAVE_PARALLEL */ - /*------------------------------------------------------------------------- * @@ -2617,7 +2195,7 @@ H5_ATTR_UNUSED HDassert(f != NULL); HDassert(f->shared != NULL); HDassert(f->shared->cache != NULL); - aux_ptr = (H5AC_aux_t *)(f->shared->cache->aux_ptr); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(f->shared->cache); if(aux_ptr != NULL) { HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); @@ -2635,202 +2213,6 @@ H5_ATTR_UNUSED /*------------------------------------------------------------------------- - * Function: H5AC__construct_candidate_list() - * - * Purpose: In the parallel case when the metadata_write_strategy is - * H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, process 0 uses - * this function to construct the list of cache entries to - * be flushed. This list is then propagated to the other - * caches, and then flushed in a distributed fashion. - * - * The sync_point_op parameter is used to determine the extent - * of the flush. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 3/17/10 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__construct_candidate_list(H5AC_t *cache_ptr, H5AC_aux_t *aux_ptr, - int sync_point_op) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); - HDassert((sync_point_op == H5AC_SYNC_POINT_OP__FLUSH_CACHE) || (aux_ptr->mpi_rank == 0)); - HDassert(aux_ptr->d_slist_ptr != NULL); - HDassert(aux_ptr->c_slist_ptr != NULL); - HDassert(H5SL_count(aux_ptr->c_slist_ptr) == 0); - HDassert(aux_ptr->candidate_slist_ptr != NULL); - HDassert(H5SL_count(aux_ptr->candidate_slist_ptr) == 0); - HDassert((sync_point_op == H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN) || (sync_point_op == H5AC_SYNC_POINT_OP__FLUSH_CACHE)); - - switch(sync_point_op) { - case H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN: - if(H5C_construct_candidate_list__min_clean((H5C_t *)cache_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_construct_candidate_list__min_clean() failed.") - break; - - case H5AC_SYNC_POINT_OP__FLUSH_CACHE: - if(H5C_construct_candidate_list__clean_cache((H5C_t *)cache_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_construct_candidate_list__clean_cache() failed.") - break; - - default: - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unknown sync point operation.") - break; - } /* end switch */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__construct_candidate_list() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__copy_candidate_list_to_buffer_cb - * - * Purpose: Skip list callback for building array of addresses for - * broadcasting the candidate list. - * - * Return: Return SUCCEED on success, and FAIL on failure. - * - * Programmer: Quincey Koziol, 6/12/15 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__copy_candidate_list_to_buffer_cb(void *_item, void H5_ATTR_UNUSED *_key, - void *_udata) -{ - H5AC_slist_entry_t * slist_entry_ptr = (H5AC_slist_entry_t *)_item; /* Address of item */ - H5AC_addr_list_ud_t * udata = (H5AC_addr_list_ud_t *)_udata; /* Context for callback */ - - FUNC_ENTER_STATIC_NOERR - - /* Sanity checks */ - HDassert(slist_entry_ptr); - HDassert(udata); - - /* Store the entry's address in the buffer */ - udata->addr_buf_ptr[udata->i] = slist_entry_ptr->addr; - udata->i++; - - /* now release the entry */ - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5AC__copy_candidate_list_to_buffer_cb() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__copy_candidate_list_to_buffer - * - * Purpose: Allocate buffer(s) and copy the contents of the candidate - * entry slist into it (them). In passing, remove all - * entries from the candidate slist. Note that the - * candidate slist must not be empty. - * - * If MPI_Offset_buf_ptr_ptr is not NULL, allocate a buffer - * of MPI_Offset, copy the contents of the candidate - * entry list into it with the appropriate conversions, - * and return the base address of the buffer in - * *MPI_Offset_buf_ptr. Note that this is the buffer - * used by process 0 to transmit the list of entries to - * be flushed to all other processes (in this file group). - * - * Similarly, allocate a buffer of haddr_t, load the contents - * of the candidate list into this buffer, and return its - * base address in *haddr_buf_ptr_ptr. Note that this - * latter buffer is constructed unconditionally. - * - * In passing, also remove all entries from the candidate - * entry slist. - * - * Return: Return SUCCEED on success, and FAIL on failure. - * - * Programmer: John Mainzer, 4/19/10 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__copy_candidate_list_to_buffer(const H5AC_t *cache_ptr, int *num_entries_ptr, - haddr_t **haddr_buf_ptr_ptr) -{ - H5AC_aux_t * aux_ptr = NULL; - H5AC_addr_list_ud_t udata; - haddr_t * haddr_buf_ptr = NULL; - size_t buf_size; - int num_entries = 0; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); - HDassert(aux_ptr->candidate_slist_ptr != NULL); - HDassert(H5SL_count(aux_ptr->candidate_slist_ptr) > 0); - HDassert(num_entries_ptr != NULL); - HDassert(*num_entries_ptr == 0); - HDassert(haddr_buf_ptr_ptr != NULL); - HDassert(*haddr_buf_ptr_ptr == NULL); - - num_entries = (int)H5SL_count(aux_ptr->candidate_slist_ptr); - - /* allocate a buffer(s) to store the list of candidate entry - * base addresses in - */ - buf_size = sizeof(haddr_t) * (size_t)num_entries; - if(NULL == (haddr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for haddr buffer") - - /* Set up user data for callback */ - udata.aux_ptr = aux_ptr; - udata.addr_buf_ptr = haddr_buf_ptr; - udata.i = 0; - - /* Free all the candidate list entries, building the address list in the callback */ - if(H5SL_free(aux_ptr->candidate_slist_ptr, H5AC__copy_candidate_list_to_buffer_cb, &udata) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "Can't build address list for candidate entries") - - /* Pass the number of entries and the buffer pointer - * back to the caller. - */ - *num_entries_ptr = num_entries; - *haddr_buf_ptr_ptr = haddr_buf_ptr; - -done: - if(ret_value < 0) - if(haddr_buf_ptr) - haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__copy_candidate_list_to_buffer() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- * Function: H5AC__ext_config_2_int_config() * * Purpose: Utility function to translate an instance of @@ -2896,1638 +2278,6 @@ done: } /* H5AC__ext_config_2_int_config() */ -/*------------------------------------------------------------------------- - * - * Function: H5AC__log_deleted_entry() - * - * Purpose: Log an entry which has been deleted. - * - * Only called for mpi_rank 0. We must make sure that the entry - * doesn't appear in the cleaned or dirty entry lists. - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: John Mainzer, 6/29/05 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__log_deleted_entry(const H5AC_info_t *entry_ptr) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - H5AC_slist_entry_t * slist_entry_ptr = NULL; - haddr_t addr; - - FUNC_ENTER_STATIC_NOERR - - /* Sanity checks */ - HDassert(entry_ptr); - addr = entry_ptr->addr; - cache_ptr = entry_ptr->cache_ptr; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->mpi_rank == 0); - HDassert(aux_ptr->d_slist_ptr != NULL); - HDassert(aux_ptr->c_slist_ptr != NULL); - - /* if the entry appears in the dirtied entry slist, remove it. */ - if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->d_slist_ptr, (void *)(&addr)))) - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - - /* if the entry appears in the cleaned entry slist, remove it. */ - if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)(&addr)))) - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5AC__log_deleted_entry() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__log_dirtied_entry() - * - * Purpose: Update the dirty_bytes count for a newly dirtied entry. - * - * If mpi_rank isn't 0, this simply means adding the size - * of the entries to the dirty_bytes count. - * - * If mpi_rank is 0, we must first check to see if the entry - * appears in the dirty entries slist. If it is, do nothing. - * If it isn't, add the size to th dirty_bytes count, add the - * entry to the dirty entries slist, and remove it from the - * cleaned list (if it is present there). - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: John Mainzer, 6/29/05 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__log_dirtied_entry(const H5AC_info_t *entry_ptr) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(entry_ptr); - HDassert(entry_ptr->is_dirty == FALSE); - cache_ptr = entry_ptr->cache_ptr; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - - if(aux_ptr->mpi_rank == 0) { - H5AC_slist_entry_t *slist_entry_ptr; - haddr_t addr = entry_ptr->addr; - - /* Sanity checks */ - HDassert(aux_ptr->d_slist_ptr != NULL); - HDassert(aux_ptr->c_slist_ptr != NULL); - - if(NULL == H5SL_search(aux_ptr->d_slist_ptr, (void *)(&addr))) { - /* insert the address of the entry in the dirty entry list, and - * add its size to the dirty_bytes count. - */ - if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate dirty slist entry .") - slist_entry_ptr->addr = addr; - - if(H5SL_insert(aux_ptr->d_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into dirty entry slist.") - - aux_ptr->dirty_bytes += entry_ptr->size; -#if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->unprotect_dirty_bytes += entry_ptr->size; - aux_ptr->unprotect_dirty_bytes_updates += 1; -#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ - } /* end if */ - - /* the entry is dirty. If it exists on the cleaned entries list, - * remove it. - */ - if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)(&addr)))) - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - } /* end if */ - else { - aux_ptr->dirty_bytes += entry_ptr->size; -#if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->unprotect_dirty_bytes += entry_size; - aux_ptr->unprotect_dirty_bytes_updates += 1; -#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ - } /* end else */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__log_dirtied_entry() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__log_flushed_entry() - * - * Purpose: Update the clean entry slist for the flush of an entry -- - * specifically, if the entry has been cleared, remove it - * from both the cleaned and dirtied lists if it is present. - * Otherwise, if the entry was dirty, insert the indicated - * entry address in the clean slist if it isn't there already. - * - * This function is only used in PHDF5, and should only - * be called for the process with mpi rank 0. - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: John Mainzer, 6/29/05 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__log_flushed_entry(H5C_t *cache_ptr, haddr_t addr, hbool_t was_dirty, - unsigned flags) -{ - hbool_t cleared; - H5AC_aux_t * aux_ptr; - H5AC_slist_entry_t * slist_entry_ptr = NULL; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity check */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->mpi_rank == 0); - HDassert(aux_ptr->c_slist_ptr != NULL); - - /* Set local flags */ - cleared = ((flags & H5C__FLUSH_CLEAR_ONLY_FLAG) != 0); - - if(cleared) { - /* If the entry has been cleared, must remove it from both the - * cleaned list and the dirtied list. - */ - if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)(&addr)))) - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->d_slist_ptr, (void *)(&addr)))) - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - } /* end if */ - else if(was_dirty) { - if(NULL == H5SL_search(aux_ptr->c_slist_ptr, (void *)(&addr))) { - /* insert the address of the entry in the clean entry list. */ - if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate clean slist entry .") - slist_entry_ptr->addr = addr; - - if(H5SL_insert(aux_ptr->c_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into clean entry slist.") - } /* end if */ - } /* end else-if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__log_flushed_entry() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__log_inserted_entry() - * - * Purpose: Update the dirty_bytes count for a newly inserted entry. - * - * If mpi_rank isnt 0, this simply means adding the size - * of the entry to the dirty_bytes count. - * - * If mpi_rank is 0, we must also add the entry to the - * dirty entries slist. - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: John Mainzer, 6/30/05 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__log_inserted_entry(const H5AC_info_t *entry_ptr) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(entry_ptr); - cache_ptr = entry_ptr->cache_ptr; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - - if(aux_ptr->mpi_rank == 0) { - H5AC_slist_entry_t *slist_entry_ptr; - - HDassert(aux_ptr->d_slist_ptr != NULL); - HDassert(aux_ptr->c_slist_ptr != NULL); - - /* Entry to insert should not be in dirty list currently */ - if(NULL != H5SL_search(aux_ptr->d_slist_ptr, (const void *)(&entry_ptr->addr))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Inserted entry already in dirty slist.") - - /* insert the address of the entry in the dirty entry list, and - * add its size to the dirty_bytes count. - */ - if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate dirty slist entry .") - slist_entry_ptr->addr = entry_ptr->addr; - if(H5SL_insert(aux_ptr->d_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into dirty entry slist.") - - /* Entry to insert should not be in clean list either */ - if(NULL != H5SL_search(aux_ptr->c_slist_ptr, (const void *)(&entry_ptr->addr))) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Inserted entry in clean slist.") - } /* end if */ - - aux_ptr->dirty_bytes += entry_ptr->size; - -#if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->insert_dirty_bytes += size; - aux_ptr->insert_dirty_bytes_updates += 1; -#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__log_inserted_entry() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__log_moved_entry() - * - * Purpose: Update the dirty_bytes count for a moved entry. - * - * WARNING - * - * At present, the way that the move call is used ensures - * that the moved entry is present in all caches by - * moving in a collective operation and immediately after - * unprotecting the target entry. - * - * This function uses this invariant, and will cause arcane - * failures if it is not met. If maintaining this invariant - * becomes impossible, we will have to rework this function - * extensively, and likely include a bit of IPC for - * synchronization. A better option might be to subsume - * move in the unprotect operation. - * - * Given that the target entry is in all caches, the function - * proceeds as follows: - * - * For processes with mpi rank other 0, it simply checks to - * see if the entry was dirty prior to the move, and adds - * the entries size to the dirty bytes count. - * - * In the process with mpi rank 0, the function first checks - * to see if the entry was dirty prior to the move. If it - * was, and if the entry doesn't appear in the dirtied list - * under its old address, it adds the entry's size to the - * dirty bytes count. - * - * The rank 0 process then removes any references to the - * entry under its old address from the cleands and dirtied - * lists, and inserts an entry in the dirtied list under the - * new address. - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: John Mainzer, 6/30/05 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__log_moved_entry(const H5F_t *f, haddr_t old_addr, haddr_t new_addr) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - hbool_t entry_in_cache; - hbool_t entry_dirty; - size_t entry_size; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(f); - HDassert(f->shared); - cache_ptr = (H5AC_t *)f->shared->cache; - HDassert(cache_ptr); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - - /* get entry status, size, etc here */ - if(H5C_get_entry_status(f, old_addr, &entry_size, &entry_in_cache, - &entry_dirty, NULL, NULL, NULL, NULL) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't get entry status.") - if(!entry_in_cache) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry not in cache.") - - if(aux_ptr->mpi_rank == 0) { - H5AC_slist_entry_t * slist_entry_ptr; - - HDassert(aux_ptr->d_slist_ptr != NULL); - HDassert(aux_ptr->c_slist_ptr != NULL); - - /* if the entry appears in the cleaned entry slist, under its old - * address, remove it. - */ - if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)(&old_addr)))) - slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); - - /* if the entry appears in the dirtied entry slist under its old - * address, remove it, but don't free it. Set addr to new_addr. - */ - if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->d_slist_ptr, (void *)(&old_addr)))) - slist_entry_ptr->addr = new_addr; - else { - /* otherwise, allocate a new entry that is ready - * for insertion, and increment dirty_bytes. - * - * Note that the fact that the entry wasn't in the dirtied - * list under its old address implies that it must have - * been clean to start with. - */ - HDassert(!entry_dirty); - if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate dirty slist entry .") - slist_entry_ptr->addr = new_addr; - - aux_ptr->dirty_bytes += entry_size; - -#if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->move_dirty_bytes += entry_size; - aux_ptr->move_dirty_bytes_updates += 1; -#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ - } /* end else */ - - /* insert / reinsert the entry in the dirty slist */ - if(H5SL_insert(aux_ptr->d_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into dirty entry slist.") - } /* end if */ - else if(!entry_dirty) { - aux_ptr->dirty_bytes += entry_size; - -#if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->move_dirty_bytes += entry_size; - aux_ptr->move_dirty_bytes_updates += 1; -#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ - } /* end else-if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__log_moved_entry() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__propagate_and_apply_candidate_list - * - * Purpose: Prior to the addition of support for multiple metadata - * write strategies, in PHDF5, only the metadata cache with - * mpi rank 0 was allowed to write to file. All other - * metadata caches on processes with rank greater than 0 - * were required to retain dirty entries until they were - * notified that the entry was clean. - * - * This constraint is relaxed with the distributed - * metadata write strategy, in which a list of candidate - * metadata cache entries is constructed by the process 0 - * cache and then distributed to the caches of all the other - * processes. Once the listed is distributed, many (if not - * all) processes writing writing a unique subset of the - * entries, and marking the remainder clean. The subsets - * are chosen so that each entry in the list of candidates - * is written by exactly one cache, and all entries are - * marked as being clean in all caches. - * - * While the list of candidate cache entries is prepared - * elsewhere, this function is the main routine for distributing - * and applying the list. It must be run simultaniously on - * all processes that have the relevant file open. To ensure - * proper synchronization, there is a barrier at the beginning - * of this function. - * - * At present, this function is called under one of two - * circumstances: - * - * 1) Dirty byte creation exceeds some user specified value. - * - * While metadata reads may occur independently, all - * operations writing metadata must be collective. Thus - * all metadata caches see the same sequence of operations, - * and therefore the same dirty data creation. - * - * This fact is used to synchronize the caches for purposes - * of propagating the list of candidate entries, by simply - * calling this function from all caches whenever some user - * specified threshold on dirty data is exceeded. (the - * process 0 cache creates the candidate list just before - * calling this function). - * - * 2) Under direct user control -- this operation must be - * collective. - * - * The operations to be managed by this function are as - * follows: - * - * All processes: - * - * 1) Participate in an opening barrier. - * - * For the process with mpi rank 0: - * - * 1) Load the contents of the candidate list - * (candidate_slist_ptr) into a buffer, and broadcast that - * buffer to all the other caches. Clear the candidate - * list in passing. - * - * If there is a positive number of candidates, proceed with - * the following: - * - * 2) Apply the candidate entry list. - * - * 3) Particpate in a closing barrier. - * - * 4) Remove from the dirty list (d_slist_ptr) and from the - * flushed and still clean entries list (c_slist_ptr), - * all addresses that appeared in the candidate list, as - * these entries are now clean. - * - * - * For all processes with mpi rank greater than 0: - * - * 1) Receive the candidate entry list broadcast - * - * If there is a positive number of candidates, proceed with - * the following: - * - * 2) Apply the candidate entry list. - * - * 3) Particpate in a closing barrier. - * - * Return: Success: non-negative - * - * Failure: negative - * - * Programmer: John Mainzer - * 3/17/10 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__propagate_and_apply_candidate_list(H5F_t *f, hid_t dxpl_id) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - haddr_t * candidates_list_ptr = NULL; - int mpi_result; - int num_candidates = 0; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(f != NULL); - cache_ptr = f->shared->cache; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); - - /* to prevent "messages from the future" we must synchronize all - * processes before we write any entries. - */ - if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) - - if(aux_ptr->mpi_rank == 0) { - if(H5AC__broadcast_candidate_list(cache_ptr, &num_candidates, &candidates_list_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't broadcast candidate slist.") - - HDassert(H5SL_count(aux_ptr->candidate_slist_ptr) == 0); - } /* end if */ - else { - if(H5AC__receive_candidate_list(cache_ptr, &num_candidates, &candidates_list_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't receive candidate broadcast.") - } /* end else */ - - if(num_candidates > 0) { - herr_t result; - - /* all processes apply the candidate list. - * H5C_apply_candidate_list() handles the details of - * distributing the writes across the processes. - */ - - /* Enable writes during this operation */ - aux_ptr->write_permitted = TRUE; - - /* Apply the candidate list */ - result = H5C_apply_candidate_list(f, dxpl_id, cache_ptr, num_candidates, - candidates_list_ptr, aux_ptr->mpi_rank, aux_ptr->mpi_size); - - /* Disable writes again */ - aux_ptr->write_permitted = FALSE; - - /* Check for error on the write operation */ - if(result < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't apply candidate list.") - - /* this code exists primarily for the test bed -- it allows us to - * enforce posix semantics on the server that pretends to be a - * file system in our parallel tests. - */ - if(aux_ptr->write_done) - (aux_ptr->write_done)(); - - /* to prevent "messages from the past" we must synchronize all - * processes again before we go on. - */ - if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) - - /* if this is process zero, tidy up the dirtied, - * and flushed and still clean lists. - */ - if(aux_ptr->mpi_rank == 0) - if(H5AC__tidy_cache_0_lists(cache_ptr, num_candidates, candidates_list_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't tidy up process 0 lists.") - } /* end if */ - - /* if it is defined, call the sync point done callback. Note - * that this callback is defined purely for testing purposes, - * and should be undefined under normal operating circumstances. - */ - if(aux_ptr->sync_point_done) - (aux_ptr->sync_point_done)(num_candidates, candidates_list_ptr); - -done: - if(candidates_list_ptr) - candidates_list_ptr = (haddr_t *)H5MM_xfree((void *)candidates_list_ptr); - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__propagate_and_apply_candidate_list() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__propagate_flushed_and_still_clean_entries_list - * - * Purpose: In PHDF5, if the process 0 only metadata write strategy - * is selected, only the metadata cache with mpi rank 0 is - * allowed to write to file. All other metadata caches on - * processes with rank greater than 0 must retain dirty - * entries until they are notified that the entry is now - * clean. - * - * This function is the main routine for handling this - * notification proceedure. It must be called - * simultaniously on all processes that have the relevant - * file open. To this end, it is called only during a - * sync point, with a barrier prior to the call. - * - * Note that any metadata entry writes by process 0 will - * occur after the barrier and just before this call. - * - * Typicaly, calls to this function will be triggered in - * one of two ways: - * - * 1) Dirty byte creation exceeds some user specified value. - * - * While metadata reads may occur independently, all - * operations writing metadata must be collective. Thus - * all metadata caches see the same sequence of operations, - * and therefore the same dirty data creation. - * - * This fact is used to synchronize the caches for purposes - * of propagating the list of flushed and still clean - * entries, by simply calling this function from all - * caches whenever some user specified threshold on dirty - * data is exceeded. - * - * 2) Under direct user control -- this operation must be - * collective. - * - * The operations to be managed by this function are as - * follows: - * - * For the process with mpi rank 0: - * - * 1) Load the contents of the flushed and still clean entries - * list (c_slist_ptr) into a buffer, and broadcast that - * buffer to all the other caches. - * - * 2) Clear the flushed and still clean entries list - * (c_slist_ptr). - * - * - * For all processes with mpi rank greater than 0: - * - * 1) Receive the flushed and still clean entries list broadcast - * - * 2) Mark the specified entries as clean. - * - * - * For all processes: - * - * 1) Reset the dirtied bytes count to 0. - * - * Return: Success: non-negative - * - * Failure: negative - * - * Programmer: John Mainzer - * July 5, 2005 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__propagate_flushed_and_still_clean_entries_list(H5F_t *f, hid_t dxpl_id) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(f != NULL); - cache_ptr = f->shared->cache; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY); - - if(aux_ptr->mpi_rank == 0) { - if(H5AC__broadcast_clean_list(cache_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't broadcast clean slist.") - HDassert(H5SL_count(aux_ptr->c_slist_ptr) == 0); - } /* end if */ - else { - if(H5AC__receive_and_apply_clean_list(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't receive and/or process clean slist broadcast.") - } /* end else */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__propagate_flushed_and_still_clean_entries_list() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC_receive_haddr_list() - * - * Purpose: Receive the list of entry addresses from process 0, - * and return it in a buffer pointed to by *haddr_buf_ptr_ptr. - * Note that the caller must free this buffer if it is - * returned. - * - * This function must only be called by the process with - * MPI_rank greater than 0. - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: Quincey Koziol, 6/11/2015 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__receive_haddr_list(MPI_Comm mpi_comm, int *num_entries_ptr, - haddr_t **haddr_buf_ptr_ptr) -{ - haddr_t * haddr_buf_ptr = NULL; - int mpi_result; - int num_entries; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(num_entries_ptr != NULL); - HDassert(*num_entries_ptr == 0); - HDassert(haddr_buf_ptr_ptr != NULL); - HDassert(*haddr_buf_ptr_ptr == NULL); - - /* First receive the number of entries in the list so that we - * can set up a buffer to receive them. If there aren't - * any, we are done. - */ - if(MPI_SUCCESS != (mpi_result = MPI_Bcast(&num_entries, 1, MPI_INT, 0, mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) - - if(num_entries > 0) { - size_t buf_size; - - /* allocate buffers to store the list of entry base addresses in */ - buf_size = sizeof(haddr_t) * (size_t)num_entries; - if(NULL == (haddr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for haddr buffer") - - /* Now receive the list of candidate entries */ - if(MPI_SUCCESS != (mpi_result = MPI_Bcast((void *)haddr_buf_ptr, (int)buf_size, MPI_BYTE, 0, mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) - } /* end if */ - - /* finally, pass the number of entries and the buffer pointer - * back to the caller. - */ - *num_entries_ptr = num_entries; - *haddr_buf_ptr_ptr = haddr_buf_ptr; - -done: - if(ret_value < 0) - if(haddr_buf_ptr) - haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC_receive_haddr_list() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__receive_and_apply_clean_list() - * - * Purpose: Receive the list of cleaned entries from process 0, - * and mark the specified entries as clean. - * - * This function must only be called by the process with - * MPI_rank greater than 0. - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: John Mainzer, 7/4/05 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__receive_and_apply_clean_list(H5F_t *f, hid_t dxpl_id) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - haddr_t * haddr_buf_ptr = NULL; - int num_entries = 0; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity check */ - HDassert(f != NULL); - cache_ptr = f->shared->cache; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->mpi_rank != 0); - - /* Retrieve the clean list from process 0 */ - if(H5AC__receive_haddr_list(aux_ptr->mpi_comm, &num_entries, &haddr_buf_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't receive clean list") - - if(num_entries > 0) - /* mark the indicated entries as clean */ - if(H5C_mark_entries_as_clean(f, dxpl_id, (int32_t)num_entries, haddr_buf_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't mark entries clean.") - - /* if it is defined, call the sync point done callback. Note - * that this callback is defined purely for testing purposes, - * and should be undefined under normal operating circumstances. - */ - if(aux_ptr->sync_point_done) - (aux_ptr->sync_point_done)(num_entries, haddr_buf_ptr); - -done: - if(haddr_buf_ptr) - haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__receive_and_apply_clean_list() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * - * Function: H5AC__receive_candidate_list() - * - * Purpose: Receive the list of candidate entries from process 0, - * and return it in a buffer pointed to by *haddr_buf_ptr_ptr. - * Note that the caller must free this buffer if it is - * returned. - * - * This function must only be called by the process with - * MPI_rank greater than 0. - * - * Return SUCCEED on success, and FAIL on failure. - * - * Return: Non-negative on success/Negative on failure. - * - * Programmer: John Mainzer, 3/17/10 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__receive_candidate_list(const H5AC_t *cache_ptr, int *num_entries_ptr, - haddr_t **haddr_buf_ptr_ptr) -{ - H5AC_aux_t * aux_ptr; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->mpi_rank != 0); - HDassert(aux_ptr-> metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); - HDassert(num_entries_ptr != NULL); - HDassert(*num_entries_ptr == 0); - HDassert(haddr_buf_ptr_ptr != NULL); - HDassert(*haddr_buf_ptr_ptr == NULL); - - /* Retrieve the candidate list from process 0 */ - if(H5AC__receive_haddr_list(aux_ptr->mpi_comm, num_entries_ptr, haddr_buf_ptr_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't receive clean list") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__receive_candidate_list() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__rsp__dist_md_write__flush - * - * Purpose: Routine for handling the details of running a sync point - * that is triggered by a flush -- which in turn must have been - * triggered by either a flush API call or a file close -- - * when the distributed metadata write strategy is selected. - * - * Upon entry, each process generates it own candidate list, - * being a sorted list of all dirty metadata entries currently - * in the metadata cache. Note that this list must be idendical - * across all processes, as all processes see the same stream - * of dirty metadata coming in, and use the same lists of - * candidate entries at each sync point. (At first glance, this - * argument sounds circular, but think of it in the sense of - * a recursive proof). - * - * If this this list is empty, we are done, and the function - * returns - * - * Otherwise, after the sorted list dirty metadata entries is - * constructed, each process uses the same algorithm to assign - * each entry on the candidate list to exactly one process for - * flushing. - * - * At this point, all processes participate in a barrier to - * avoid messages from the past/future bugs. - * - * Each process then flushes the entries assigned to it, and - * marks all other entries on the candidate list as clean. - * - * Finally, all processes participate in a second barrier to - * avoid messages from the past/future bugs. - * - * At the end of this process, process 0 and only process 0 - * must tidy up its lists of dirtied and cleaned entries. - * These lists are not used in the distributed metadata write - * strategy, but they must be maintained should we shift - * to a strategy that uses them. - * - * Return: Success: non-negative - * - * Failure: negative - * - * Programmer: John Mainzer - * April 28, 2010 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__rsp__dist_md_write__flush(H5F_t *f, hid_t dxpl_id) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - haddr_t * haddr_buf_ptr = NULL; - int mpi_result; - int num_entries = 0; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(f != NULL); - cache_ptr = f->shared->cache; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); - - /* first construct the candidate list -- initially, this will be in the - * form of a skip list. We will convert it later. - */ - if(H5C_construct_candidate_list__clean_cache(cache_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate list.") - - if(H5SL_count(aux_ptr->candidate_slist_ptr) > 0) { - herr_t result; - - /* convert the candidate list into the format we - * are used to receiving from process 0. - */ - if(H5AC__copy_candidate_list_to_buffer(cache_ptr, &num_entries, &haddr_buf_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate buffer.") - - /* initial sync point barrier */ - if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) - - /* Enable writes during this operation */ - aux_ptr->write_permitted = TRUE; - - /* Apply the candidate list */ - result = H5C_apply_candidate_list(f, dxpl_id, cache_ptr, num_entries, - haddr_buf_ptr, aux_ptr->mpi_rank, aux_ptr->mpi_size); - - /* Disable writes again */ - aux_ptr->write_permitted = FALSE; - - /* Check for error on the write operation */ - if(result < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't apply candidate list.") - - /* this code exists primarily for the test bed -- it allows us to - * enforce posix semantics on the server that pretends to be a - * file system in our parallel tests. - */ - if(aux_ptr->write_done) - (aux_ptr->write_done)(); - - /* final sync point barrier */ - if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) - - /* if this is process zero, tidy up the dirtied, - * and flushed and still clean lists. - */ - if(aux_ptr->mpi_rank == 0) - if(H5AC__tidy_cache_0_lists(cache_ptr, num_entries, haddr_buf_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't tidy up process 0 lists.") - } /* end if */ - - /* if it is defined, call the sync point done callback. Note - * that this callback is defined purely for testing purposes, - * and should be undefined under normal operating circumstances. - */ - if(aux_ptr->sync_point_done) - (aux_ptr->sync_point_done)(num_entries, haddr_buf_ptr); - -done: - if(haddr_buf_ptr) - haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__rsp__dist_md_write__flush() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__rsp__dist_md_write__flush_to_min_clean - * - * Purpose: Routine for handling the details of running a sync point - * triggered by the accumulation of dirty metadata (as - * opposed to a flush call to the API) when the distributed - * metadata write strategy is selected. - * - * After invocation and initial sanity checking this function - * first checks to see if evictions are enabled -- if they - * are not, the function does nothing and returns. - * - * Otherwise, process zero constructs a list of entries to - * be flushed in order to bring the process zero cache back - * within its min clean requirement. Note that this list - * (the candidate list) may be empty. - * - * Then, all processes participate in a barrier. - * - * After the barrier, process 0 broadcasts the number of - * entries in the candidate list prepared above, and all - * other processes receive this number. - * - * If this number is zero, we are done, and the function - * returns without further action. - * - * Otherwise, process 0 broadcasts the sorted list of - * candidate entries, and all other processes receive it. - * - * Then, each process uses the same algorithm to assign - * each entry on the candidate list to exactly one process - * for flushing. - * - * Each process then flushes the entries assigned to it, and - * marks all other entries on the candidate list as clean. - * - * Finally, all processes participate in a second barrier to - * avoid messages from the past/future bugs. - * - * At the end of this process, process 0 and only process 0 - * must tidy up its lists of dirtied and cleaned entries. - * These lists are not used in the distributed metadata write - * strategy, but they must be maintained should we shift - * to a strategy that uses them. - * - * Return: Success: non-negative - * - * Failure: negative - * - * Programmer: John Mainzer - * April 28, 2010 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__rsp__dist_md_write__flush_to_min_clean(H5F_t *f, hid_t dxpl_id) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - hbool_t evictions_enabled; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(f != NULL); - cache_ptr = f->shared->cache; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); - - /* Query if evictions are allowed */ - if(H5C_get_evictions_enabled((const H5C_t *)cache_ptr, &evictions_enabled) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_get_evictions_enabled() failed.") - - if(evictions_enabled) { - /* construct candidate list -- process 0 only */ - if(aux_ptr->mpi_rank == 0) - if(H5AC__construct_candidate_list(cache_ptr, aux_ptr, H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate list.") - - /* propagate and apply candidate list -- all processes */ - if(H5AC__propagate_and_apply_candidate_list(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't propagate and apply candidate list.") - } /* evictions enabled */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__rsp__dist_md_write__flush_to_min_clean() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__rsp__p0_only__flush - * - * Purpose: Routine for handling the details of running a sync point - * that is triggered a flush -- which in turn must have been - * triggered by either a flush API call or a file close -- - * when the process 0 only metadata write strategy is selected. - * - * First, all processes participate in a barrier. - * - * Then process zero flushes all dirty entries, and broadcasts - * they number of clean entries (if any) to all the other - * caches. - * - * If this number is zero, we are done. - * - * Otherwise, process 0 broadcasts the list of cleaned - * entries, and all other processes which are part of this - * file group receive it, and mark the listed entries as - * clean in their caches. - * - * Since all processes have the same set of dirty - * entries at the beginning of the sync point, and all - * entries that will be written are written before - * process zero broadcasts the number of cleaned entries, - * there is no need for a closing barrier. - * - * Return: Success: non-negative - * - * Failure: negative - * - * Programmer: John Mainzer - * April 28, 2010 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__rsp__p0_only__flush(H5F_t *f, hid_t dxpl_id) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - int mpi_result; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(f != NULL); - cache_ptr = f->shared->cache; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY); - - /* to prevent "messages from the future" we must - * synchronize all processes before we start the flush. - * Hence the following barrier. - */ - if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) - - /* Flush data to disk, from rank 0 process */ - if(aux_ptr->mpi_rank == 0) { - herr_t result; - - /* Enable writes during this operation */ - aux_ptr->write_permitted = TRUE; - - /* Flush the cache */ - result = H5C_flush_cache(f, dxpl_id, H5AC__NO_FLAGS_SET); - - /* Disable writes again */ - aux_ptr->write_permitted = FALSE; - - /* Check for error on the write operation */ - if(result < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush.") - - /* this code exists primarily for the test bed -- it allows us to - * enforce posix semantics on the server that pretends to be a - * file system in our parallel tests. - */ - if(aux_ptr->write_done) - (aux_ptr->write_done)(); - } /* end if */ - - /* Propagate cleaned entries to other ranks. */ - if(H5AC__propagate_flushed_and_still_clean_entries_list(f, H5AC_dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't propagate clean entries list.") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__rsp__p0_only__flush() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__rsp__p0_only__flush_to_min_clean - * - * Purpose: Routine for handling the details of running a sync point - * triggered by the accumulation of dirty metadata (as - * opposed to a flush call to the API) when the process 0 - * only metadata write strategy is selected. - * - * After invocation and initial sanity checking this function - * first checks to see if evictions are enabled -- if they - * are not, the function does nothing and returns. - * - * Otherwise, all processes participate in a barrier. - * - * After the barrier, if this is process 0, the function - * causes the cache to flush sufficient entries to get the - * cache back within its minimum clean fraction, and broadcast - * the number of entries which have been flushed since - * the last sync point, and are still clean. - * - * If this number is zero, we are done. - * - * Otherwise, process 0 broadcasts the list of cleaned - * entries, and all other processes which are part of this - * file group receive it, and mark the listed entries as - * clean in their caches. - * - * Since all processes have the same set of dirty - * entries at the beginning of the sync point, and all - * entries that will be written are written before - * process zero broadcasts the number of cleaned entries, - * there is no need for a closing barrier. - * - * Return: Success: non-negative - * - * Failure: negative - * - * Programmer: John Mainzer - * April 28, 2010 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__rsp__p0_only__flush_to_min_clean(H5F_t *f, hid_t dxpl_id) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - hbool_t evictions_enabled; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(f != NULL); - cache_ptr = f->shared->cache; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY); - - /* Query if evictions are allowed */ - if(H5C_get_evictions_enabled((const H5C_t *)cache_ptr, &evictions_enabled) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_get_evictions_enabled() failed.") - - /* Flush if evictions are allowed -- following call - * will cause process 0 to flush to min clean size, - * and then propagate the newly clean entries to the - * other processes. - * - * Otherwise, do nothing. - */ - if(evictions_enabled) { - int mpi_result; - - /* to prevent "messages from the future" we must synchronize all - * processes before we start the flush. This synchronization may - * already be done -- hence the do_barrier parameter. - */ - if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) - HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) - - if(0 == aux_ptr->mpi_rank) { - herr_t result; - - /* here, process 0 flushes as many entries as necessary to - * comply with the currently specified min clean size. - * Note that it is quite possible that no entries will be - * flushed. - */ - - /* Enable writes during this operation */ - aux_ptr->write_permitted = TRUE; - - /* Flush the cache */ - result = H5C_flush_to_min_clean(f, dxpl_id); - - /* Disable writes again */ - aux_ptr->write_permitted = FALSE; - - /* Check for error on the write operation */ - if(result < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_flush_to_min_clean() failed.") - - /* this call exists primarily for the test code -- it is used - * to enforce POSIX semantics on the process used to simulate - * reads and writes in t_cache.c. - */ - if(aux_ptr->write_done) - (aux_ptr->write_done)(); - } /* end if */ - - if(H5AC__propagate_flushed_and_still_clean_entries_list(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't propagate clean entries list.") - } /* end if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__rsp__p0_only__flush_to_min_clean() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__run_sync_point - * - * Purpose: Top level routine for managing a sync point between all - * meta data caches in the parallel case. Since all caches - * see the same sequence of dirty metadata, we simply count - * bytes of dirty metadata, and run a sync point whenever the - * number of dirty bytes of metadata seen since the last - * sync point exceeds a threshold that is common across all - * processes. We also run sync points in response to - * HDF5 API calls triggering either a flush or a file close. - * - * In earlier versions of PHDF5, only the metadata cache with - * mpi rank 0 was allowed to write to file. All other - * metadata caches on processes with rank greater than 0 were - * required to retain dirty entries until they were notified - * that the entry is was clean. - * - * This function was created to make it easier for us to - * experiment with other options, as it is a single point - * for the execution of sync points. - * - * Return: Success: non-negative - * - * Failure: negative - * - * Programmer: John Mainzer - * March 11, 2010 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__run_sync_point(H5F_t *f, hid_t dxpl_id, int sync_point_op) -{ - H5AC_t * cache_ptr; - H5AC_aux_t * aux_ptr; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(f != NULL); - cache_ptr = f->shared->cache; - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert((sync_point_op == H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN) || - (sync_point_op == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED)); - -#if H5AC_DEBUG_DIRTY_BYTES_CREATION -HDfprintf(stdout, "%d:H5AC_propagate...:%u: (u/uu/i/iu/r/ru) = %zu/%u/%zu/%u/%zu/%u\n", - aux_ptr->mpi_rank, - aux_ptr->dirty_bytes_propagations, - aux_ptr->unprotect_dirty_bytes, - aux_ptr->unprotect_dirty_bytes_updates, - aux_ptr->insert_dirty_bytes, - aux_ptr->insert_dirty_bytes_updates, - aux_ptr->rename_dirty_bytes, - aux_ptr->rename_dirty_bytes_updates); -#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ - - /* clear collective access flag on half of the entries in the - cache and mark them as independent in case they need to be - evicted later. All ranks are guranteed to mark the same entires - since we don't modify the order of the collectively accessed - entries except through collective access. */ - if(H5C_clear_coll_entries(cache_ptr, TRUE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed.") - - switch(aux_ptr->metadata_write_strategy) { - case H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY: - switch(sync_point_op) { - case H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN: - if(H5AC__rsp__p0_only__flush_to_min_clean(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5AC__rsp__p0_only__flush_to_min_clean() failed.") - break; - - case H5AC_SYNC_POINT_OP__FLUSH_CACHE: - if(H5AC__rsp__p0_only__flush(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5AC__rsp__p0_only__flush() failed.") - break; - - default: - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unknown flush op"); - break; - } /* end switch */ - break; - - case H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED: - switch(sync_point_op) { - case H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN: - if(H5AC__rsp__dist_md_write__flush_to_min_clean(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5AC__rsp__dist_md_write__flush_to_min_clean() failed.") - break; - - case H5AC_SYNC_POINT_OP__FLUSH_CACHE: - if(H5AC__rsp__dist_md_write__flush(f, dxpl_id) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5AC__rsp__dist_md_write__flush() failed.") - break; - - default: - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unknown flush op"); - break; - } /* end switch */ - break; - - default: - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown metadata write strategy.") - break; - } /* end switch */ - - /* reset the dirty bytes count */ - aux_ptr->dirty_bytes = 0; - -#if H5AC_DEBUG_DIRTY_BYTES_CREATION - aux_ptr->dirty_bytes_propagations += 1; - aux_ptr->unprotect_dirty_bytes = 0; - aux_ptr->unprotect_dirty_bytes_updates = 0; - aux_ptr->insert_dirty_bytes = 0; - aux_ptr->insert_dirty_bytes_updates = 0; - aux_ptr->rename_dirty_bytes = 0; - aux_ptr->rename_dirty_bytes_updates = 0; -#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__run_sync_point() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__tidy_cache_0_lists() - * - * Purpose: In the distributed metadata write strategy, not all dirty - * entries are written by process 0 -- thus we must tidy - * up the dirtied, and flushed and still clean lists - * maintained by process zero after each sync point. - * - * This procedure exists to tend to this issue. - * - * At this point, all entries that process 0 cleared should - * have been removed from both the dirty and flushed and - * still clean lists, and entries that process 0 has flushed - * should have been removed from the dirtied list and added - * to the flushed and still clean list. - * - * However, since the distributed metadata write strategy - * doesn't make use of these lists, the objective is simply - * to maintain these lists in consistent state that allows - * them to be used should the metadata write strategy change - * to one that uses these lists. - * - * Thus for our purposes, all we need to do is remove from - * the dirtied and flushed and still clean lists all - * references to entries that appear in the candidate list. - * - * Return: Success: non-negative - * - * Failure: negative - * - * Programmer: John Mainzer - * 4/20/10 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__tidy_cache_0_lists(H5AC_t *cache_ptr, int num_candidates, - haddr_t *candidates_list_ptr) -{ - H5AC_aux_t * aux_ptr; - int i; - - FUNC_ENTER_STATIC_NOERR - - /* Sanity checks */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); - aux_ptr = (H5AC_aux_t *)(cache_ptr->aux_ptr); - HDassert(aux_ptr != NULL); - HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); - HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); - HDassert(aux_ptr->mpi_rank == 0); - HDassert(num_candidates > 0); - HDassert(candidates_list_ptr != NULL); - - /* clean up dirtied and flushed and still clean lists by removing - * all entries on the candidate list. Cleared entries should - * have been removed from both the dirty and cleaned lists at - * this point, flushed entries should have been added to the - * cleaned list. However, for this metadata write strategy, - * we just want to remove all references to the candidate entries. - */ - for(i = 0; i < num_candidates; i++) { - H5AC_slist_entry_t * d_slist_entry_ptr; - H5AC_slist_entry_t * c_slist_entry_ptr; - haddr_t addr; - - addr = candidates_list_ptr[i]; - - /* addr may be either on the dirtied list, or on the flushed - * and still clean list. Remove it. - */ - if(NULL != (d_slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->d_slist_ptr, (void *)&addr))) - d_slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, d_slist_entry_ptr); - if(NULL != (c_slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)&addr))) - c_slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, c_slist_entry_ptr); - } /* end for */ - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5AC__tidy_cache_0_lists() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5AC__flush_entries - * - * Purpose: Flush the metadata cache associated with the specified file, - * only writing from rank 0, but propagating the cleaned entries - * to all ranks. - * - * Return: Non-negative on success/Negative on failure if there was a - * request to flush all items and something was protected. - * - * Programmer: Quincey Koziol - * koziol@hdfgroup.org - * Aug 22 2009 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -static herr_t -H5AC__flush_entries(H5F_t *f, hid_t dxpl_id) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_STATIC - - /* Sanity checks */ - HDassert(f); - HDassert(f->shared->cache); - - /* Check if we have >1 ranks */ - if(f->shared->cache->aux_ptr) - if(H5AC__run_sync_point(f, dxpl_id, H5AC_SYNC_POINT_OP__FLUSH_CACHE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't run sync point.") - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5AC__flush_entries() */ -#endif /* H5_HAVE_PARALLEL */ - - /*------------------------------------------------------------------------------ * Function: H5AC_ignore_tags() * diff --git a/src/H5ACmpio.c b/src/H5ACmpio.c new file mode 100644 index 0000000..78a2485 --- /dev/null +++ b/src/H5ACmpio.c @@ -0,0 +1,2282 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5ACmpio.c + * Jun 20 2015 + * Quincey Koziol <koziol@hdfgroup.org> + * + * Purpose: Functions in this file implement support for parallel + * I/O cache functionality + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#define H5AC_PACKAGE /*suppress error about including H5ACpkg */ +#define H5F_PACKAGE /*suppress error about including H5Fpkg */ + +/* Interface initialization */ +#define H5_INTERFACE_INIT_FUNC H5AC__init_mpio_interface + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5ACpkg.h" /* Metadata cache */ +#include "H5Cprivate.h" /* Cache */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* Files */ +#include "H5MMprivate.h" /* Memory management */ + +#ifdef H5_HAVE_PARALLEL + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + +/**************************************************************************** + * + * structure H5AC_slist_entry_t + * + * The dirty entry list maintained via the d_slist_ptr field of H5AC_aux_t + * and the cleaned entry list maintained via the c_slist_ptr field of + * H5AC_aux_t are just lists of the file offsets of the dirty/cleaned + * entries. Unfortunately, the slist code makes us define a dynamically + * allocated structure to store these offsets in. This structure serves + * that purpose. Its fields are as follows: + * + * addr: file offset of a metadata entry. Entries are added to this + * list (if they aren't there already) when they are marked + * dirty in an unprotect, inserted, or moved. They are + * removed when they appear in a clean entries broadcast. + * + ****************************************************************************/ +typedef struct H5AC_slist_entry_t +{ + haddr_t addr; +} H5AC_slist_entry_t; + +/* User data for address list building callbacks */ +typedef struct H5AC_addr_list_ud_t +{ + H5AC_aux_t * aux_ptr; /* 'Auxiliary' parallel cache info */ + haddr_t * addr_buf_ptr; /* Array to store addresses */ + int i; /* Counter for position in array */ +} H5AC_addr_list_ud_t; + + +/********************/ +/* Local Prototypes */ +/********************/ + +static herr_t H5AC__broadcast_candidate_list(H5AC_t *cache_ptr, + int *num_entries_ptr, haddr_t **haddr_buf_ptr_ptr); +static herr_t H5AC__broadcast_clean_list(H5AC_t *cache_ptr); +static herr_t H5AC__construct_candidate_list(H5AC_t *cache_ptr, + H5AC_aux_t *aux_ptr, int sync_point_op); +static herr_t H5AC__copy_candidate_list_to_buffer(const H5AC_t *cache_ptr, + int *num_entries_ptr, haddr_t **haddr_buf_ptr_ptr); +static herr_t H5AC__propagate_and_apply_candidate_list(H5F_t *f, hid_t dxpl_id); +static herr_t H5AC__propagate_flushed_and_still_clean_entries_list(H5F_t *f, + hid_t dxpl_id); +static herr_t H5AC__receive_haddr_list(MPI_Comm mpi_comm, int *num_entries_ptr, + haddr_t **haddr_buf_ptr_ptr); +static herr_t H5AC__receive_candidate_list(const H5AC_t *cache_ptr, + int *num_entries_ptr, haddr_t **haddr_buf_ptr_ptr); +static herr_t H5AC__receive_and_apply_clean_list(H5F_t *f, hid_t dxpl_id); +static herr_t H5AC__tidy_cache_0_lists(H5AC_t *cache_ptr, int num_candidates, + haddr_t *candidates_list_ptr); +static herr_t H5AC__rsp__dist_md_write__flush(H5F_t *f, hid_t dxpl_id); +static herr_t H5AC__rsp__dist_md_write__flush_to_min_clean(H5F_t *f, hid_t dxpl_id); +static herr_t H5AC__rsp__p0_only__flush(H5F_t *f, hid_t dxpl_id); +static herr_t H5AC__rsp__p0_only__flush_to_min_clean(H5F_t *f, hid_t dxpl_id); + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Declare a free list to manage the H5AC_aux_t struct */ +H5FL_DEFINE(H5AC_aux_t); + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Declare a free list to manage the H5AC_slist_entry_t struct */ +H5FL_DEFINE_STATIC(H5AC_slist_entry_t); + + +/*------------------------------------------------------------------------- + * Function: H5AC__init_mpio_interface + * + * Purpose: Initialize interface-specific information + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * 6/20/15 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__init_mpio_interface(void) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Funnel all work to H5AC_init() */ + if(H5AC_init() < 0) + HGOTO_ERROR(H5E_FUNC, H5E_CANTINIT, FAIL, "interface initialization failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__init_mpio_interface() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__set_sync_point_done_callback + * + * Purpose: Set the value of the sync_point_done callback. This + * callback is used by the parallel test code to verify + * that the expected writes and only the expected writes + * take place during a sync point. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 5/9/10 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__set_sync_point_done_callback(H5C_t * cache_ptr, + void (* sync_point_done)(int num_writes, haddr_t * written_entries_tbl)) +{ + H5AC_aux_t * aux_ptr; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + HDassert(cache_ptr); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + + aux_ptr->sync_point_done = sync_point_done; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5AC__set_sync_point_done_callback() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__set_write_done_callback + * + * Purpose: Set the value of the write_done callback. This callback + * is used to improve performance of the parallel test bed + * for the cache. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 5/11/06 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__set_write_done_callback(H5C_t * cache_ptr, void (* write_done)(void)) +{ + H5AC_aux_t * aux_ptr; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + HDassert(cache_ptr); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + + aux_ptr->write_done = write_done; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5AC__set_write_done_callback() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC_add_candidate() + * + * Purpose: Add the supplied metadata entry address to the candidate + * list. Verify that each entry added does not appear in + * the list prior to its insertion. + * + * This function is intended for used in constructing list + * of entried to be flushed during sync points. It shouldn't + * be called anywhere else. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 3/17/10 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC_add_candidate(H5AC_t * cache_ptr, haddr_t addr) +{ + H5AC_aux_t * aux_ptr; + H5AC_slist_entry_t * slist_entry_ptr = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity checks */ + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + HDassert(aux_ptr->candidate_slist_ptr != NULL); + + /* Construct an entry for the supplied address, and insert + * it into the candidate slist. + */ + if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate candidate slist entry") + slist_entry_ptr->addr = addr; + + if(H5SL_insert(aux_ptr->candidate_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into dirty entry slist") + +done: + /* Clean up on error */ + if(ret_value < 0) + if(slist_entry_ptr) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_add_candidate() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__broadcast_candidate_list() + * + * Purpose: Broadcast the contents of the process 0 candidate entry + * slist. In passing, also remove all entries from said + * list. As the application of this will be handled by + * the same functions on all processes, construct and + * return a copy of the list in the same format as that + * received by the other processes. Note that if this + * copy is returned in *haddr_buf_ptr_ptr, the caller + * must free it. + * + * This function must only be called by the process with + * MPI_rank 0. + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 7/1/05 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__broadcast_candidate_list(H5AC_t *cache_ptr, int *num_entries_ptr, + haddr_t **haddr_buf_ptr_ptr) +{ + H5AC_aux_t * aux_ptr = NULL; + haddr_t * haddr_buf_ptr = NULL; + int mpi_result; + int num_entries; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->mpi_rank == 0); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + HDassert(aux_ptr->candidate_slist_ptr != NULL); + HDassert(num_entries_ptr != NULL); + HDassert(*num_entries_ptr == 0); + HDassert(haddr_buf_ptr_ptr != NULL); + HDassert(*haddr_buf_ptr_ptr == NULL); + + /* First broadcast the number of entries in the list so that the + * receivers can set up buffers to receive them. If there aren't + * any, we are done. + */ + num_entries = (int)H5SL_count(aux_ptr->candidate_slist_ptr); + if(MPI_SUCCESS != (mpi_result = MPI_Bcast(&num_entries, 1, MPI_INT, 0, aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) + + if(num_entries > 0) { + size_t buf_size = 0; + int chk_num_entries = 0; + + /* convert the candidate list into the format we + * are used to receiving from process 0, and also load it + * into a buffer for transmission. + */ + if(H5AC__copy_candidate_list_to_buffer(cache_ptr, &chk_num_entries, &haddr_buf_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate buffer.") + HDassert(chk_num_entries == num_entries); + HDassert(haddr_buf_ptr != NULL); + + /* Now broadcast the list of candidate entries */ + buf_size = sizeof(haddr_t) * (size_t)num_entries; + if(MPI_SUCCESS != (mpi_result = MPI_Bcast((void *)haddr_buf_ptr, (int)buf_size, MPI_BYTE, 0, aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) + } /* end if */ + + /* Pass the number of entries and the buffer pointer + * back to the caller. Do this so that we can use the same code + * to apply the candidate list to all the processes. + */ + *num_entries_ptr = num_entries; + *haddr_buf_ptr_ptr = haddr_buf_ptr; + +done: + if(ret_value < 0) + if(haddr_buf_ptr) + haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__broadcast_candidate_list() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__broadcast_clean_list_cb() + * + * Purpose: Skip list callback for building array of addresses for + * broadcasting the clean list. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: Quincey Koziol, 6/12/15 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__broadcast_clean_list_cb(void *_item, void H5_ATTR_UNUSED *_key, + void *_udata) +{ + H5AC_slist_entry_t * slist_entry_ptr = (H5AC_slist_entry_t *)_item; /* Address of item */ + H5AC_addr_list_ud_t * udata = (H5AC_addr_list_ud_t *)_udata; /* Context for callback */ + haddr_t addr; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checks */ + HDassert(slist_entry_ptr); + HDassert(udata); + + /* Store the entry's address in the buffer */ + addr = slist_entry_ptr->addr; + udata->addr_buf_ptr[udata->i] = addr; + udata->i++; + + /* now release the entry */ + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + + /* and also remove the matching entry from the dirtied list + * if it exists. + */ + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(udata->aux_ptr->d_slist_ptr, (void *)(&addr)))) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5AC__broadcast_clean_list_cb() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__broadcast_clean_list() + * + * Purpose: Broadcast the contents of the process 0 cleaned entry + * slist. In passing, also remove all entries from said + * list, and also remove any matching entries from the dirtied + * slist. + * + * This function must only be called by the process with + * MPI_rank 0. + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 7/1/05 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__broadcast_clean_list(H5AC_t * cache_ptr) +{ + haddr_t * addr_buf_ptr = NULL; + H5AC_aux_t * aux_ptr; + int mpi_result; + int num_entries = 0; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->mpi_rank == 0); + HDassert(aux_ptr->c_slist_ptr != NULL); + + /* First broadcast the number of entries in the list so that the + * receives can set up a buffer to receive them. If there aren't + * any, we are done. + */ + num_entries = (int)H5SL_count(aux_ptr->c_slist_ptr); + if(MPI_SUCCESS != (mpi_result = MPI_Bcast(&num_entries, 1, MPI_INT, 0, aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) + + if(num_entries > 0) { + H5AC_addr_list_ud_t udata; + size_t buf_size; + + /* allocate a buffer to store the list of entry base addresses in */ + buf_size = sizeof(haddr_t) * (size_t)num_entries; + if(NULL == (addr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for addr buffer") + + /* Set up user data for callback */ + udata.aux_ptr = aux_ptr; + udata.addr_buf_ptr = addr_buf_ptr; + udata.i = 0; + + /* Free all the clean list entries, building the address list in the callback */ + /* (Callback also removes the matching entries from the dirtied list) */ + if(H5SL_free(aux_ptr->c_slist_ptr, H5AC__broadcast_clean_list_cb, &udata) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "Can't build address list for clean entries") + + /* Now broadcast the list of cleaned entries */ + if(MPI_SUCCESS != (mpi_result = MPI_Bcast((void *)addr_buf_ptr, (int)buf_size, MPI_BYTE, 0, aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) + } /* end if */ + + /* if it is defined, call the sync point done callback. Note + * that this callback is defined purely for testing purposes, + * and should be undefined under normal operating circumstances. + */ + if(aux_ptr->sync_point_done) + (aux_ptr->sync_point_done)(num_entries, addr_buf_ptr); + +done: + if(addr_buf_ptr) + addr_buf_ptr = (haddr_t *)H5MM_xfree((void *)addr_buf_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__broadcast_clean_list() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__construct_candidate_list() + * + * Purpose: In the parallel case when the metadata_write_strategy is + * H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED, process 0 uses + * this function to construct the list of cache entries to + * be flushed. This list is then propagated to the other + * caches, and then flushed in a distributed fashion. + * + * The sync_point_op parameter is used to determine the extent + * of the flush. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 3/17/10 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__construct_candidate_list(H5AC_t *cache_ptr, H5AC_aux_t *aux_ptr, + int sync_point_op) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(cache_ptr != NULL); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + HDassert((sync_point_op == H5AC_SYNC_POINT_OP__FLUSH_CACHE) || (aux_ptr->mpi_rank == 0)); + HDassert(aux_ptr->d_slist_ptr != NULL); + HDassert(aux_ptr->c_slist_ptr != NULL); + HDassert(H5SL_count(aux_ptr->c_slist_ptr) == 0); + HDassert(aux_ptr->candidate_slist_ptr != NULL); + HDassert(H5SL_count(aux_ptr->candidate_slist_ptr) == 0); + HDassert((sync_point_op == H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN) || (sync_point_op == H5AC_SYNC_POINT_OP__FLUSH_CACHE)); + + switch(sync_point_op) { + case H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN: + if(H5C_construct_candidate_list__min_clean((H5C_t *)cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_construct_candidate_list__min_clean() failed.") + break; + + case H5AC_SYNC_POINT_OP__FLUSH_CACHE: + if(H5C_construct_candidate_list__clean_cache((H5C_t *)cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_construct_candidate_list__clean_cache() failed.") + break; + + default: + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unknown sync point operation.") + break; + } /* end switch */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__construct_candidate_list() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__copy_candidate_list_to_buffer_cb + * + * Purpose: Skip list callback for building array of addresses for + * broadcasting the candidate list. + * + * Return: Return SUCCEED on success, and FAIL on failure. + * + * Programmer: Quincey Koziol, 6/12/15 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__copy_candidate_list_to_buffer_cb(void *_item, void H5_ATTR_UNUSED *_key, + void *_udata) +{ + H5AC_slist_entry_t * slist_entry_ptr = (H5AC_slist_entry_t *)_item; /* Address of item */ + H5AC_addr_list_ud_t * udata = (H5AC_addr_list_ud_t *)_udata; /* Context for callback */ + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checks */ + HDassert(slist_entry_ptr); + HDassert(udata); + + /* Store the entry's address in the buffer */ + udata->addr_buf_ptr[udata->i] = slist_entry_ptr->addr; + udata->i++; + + /* now release the entry */ + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5AC__copy_candidate_list_to_buffer_cb() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__copy_candidate_list_to_buffer + * + * Purpose: Allocate buffer(s) and copy the contents of the candidate + * entry slist into it (them). In passing, remove all + * entries from the candidate slist. Note that the + * candidate slist must not be empty. + * + * If MPI_Offset_buf_ptr_ptr is not NULL, allocate a buffer + * of MPI_Offset, copy the contents of the candidate + * entry list into it with the appropriate conversions, + * and return the base address of the buffer in + * *MPI_Offset_buf_ptr. Note that this is the buffer + * used by process 0 to transmit the list of entries to + * be flushed to all other processes (in this file group). + * + * Similarly, allocate a buffer of haddr_t, load the contents + * of the candidate list into this buffer, and return its + * base address in *haddr_buf_ptr_ptr. Note that this + * latter buffer is constructed unconditionally. + * + * In passing, also remove all entries from the candidate + * entry slist. + * + * Return: Return SUCCEED on success, and FAIL on failure. + * + * Programmer: John Mainzer, 4/19/10 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__copy_candidate_list_to_buffer(const H5AC_t *cache_ptr, int *num_entries_ptr, + haddr_t **haddr_buf_ptr_ptr) +{ + H5AC_aux_t * aux_ptr = NULL; + H5AC_addr_list_ud_t udata; + haddr_t * haddr_buf_ptr = NULL; + size_t buf_size; + int num_entries = 0; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + HDassert(aux_ptr->candidate_slist_ptr != NULL); + HDassert(H5SL_count(aux_ptr->candidate_slist_ptr) > 0); + HDassert(num_entries_ptr != NULL); + HDassert(*num_entries_ptr == 0); + HDassert(haddr_buf_ptr_ptr != NULL); + HDassert(*haddr_buf_ptr_ptr == NULL); + + num_entries = (int)H5SL_count(aux_ptr->candidate_slist_ptr); + + /* allocate a buffer(s) to store the list of candidate entry + * base addresses in + */ + buf_size = sizeof(haddr_t) * (size_t)num_entries; + if(NULL == (haddr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for haddr buffer") + + /* Set up user data for callback */ + udata.aux_ptr = aux_ptr; + udata.addr_buf_ptr = haddr_buf_ptr; + udata.i = 0; + + /* Free all the candidate list entries, building the address list in the callback */ + if(H5SL_free(aux_ptr->candidate_slist_ptr, H5AC__copy_candidate_list_to_buffer_cb, &udata) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "Can't build address list for candidate entries") + + /* Pass the number of entries and the buffer pointer + * back to the caller. + */ + *num_entries_ptr = num_entries; + *haddr_buf_ptr_ptr = haddr_buf_ptr; + +done: + if(ret_value < 0) + if(haddr_buf_ptr) + haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__copy_candidate_list_to_buffer() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__log_deleted_entry() + * + * Purpose: Log an entry which has been deleted. + * + * Only called for mpi_rank 0. We must make sure that the entry + * doesn't appear in the cleaned or dirty entry lists. + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 6/29/05 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__log_deleted_entry(const H5AC_info_t *entry_ptr) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + H5AC_slist_entry_t * slist_entry_ptr = NULL; + haddr_t addr; + + FUNC_ENTER_PACKAGE_NOERR + + /* Sanity checks */ + HDassert(entry_ptr); + addr = entry_ptr->addr; + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->mpi_rank == 0); + HDassert(aux_ptr->d_slist_ptr != NULL); + HDassert(aux_ptr->c_slist_ptr != NULL); + + /* if the entry appears in the dirtied entry slist, remove it. */ + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->d_slist_ptr, (void *)(&addr)))) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + + /* if the entry appears in the cleaned entry slist, remove it. */ + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)(&addr)))) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5AC__log_deleted_entry() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__log_dirtied_entry() + * + * Purpose: Update the dirty_bytes count for a newly dirtied entry. + * + * If mpi_rank isn't 0, this simply means adding the size + * of the entries to the dirty_bytes count. + * + * If mpi_rank is 0, we must first check to see if the entry + * appears in the dirty entries slist. If it is, do nothing. + * If it isn't, add the size to th dirty_bytes count, add the + * entry to the dirty entries slist, and remove it from the + * cleaned list (if it is present there). + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 6/29/05 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__log_dirtied_entry(const H5AC_info_t *entry_ptr) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(entry_ptr); + HDassert(entry_ptr->is_dirty == FALSE); + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + + if(aux_ptr->mpi_rank == 0) { + H5AC_slist_entry_t *slist_entry_ptr; + haddr_t addr = entry_ptr->addr; + + /* Sanity checks */ + HDassert(aux_ptr->d_slist_ptr != NULL); + HDassert(aux_ptr->c_slist_ptr != NULL); + + if(NULL == H5SL_search(aux_ptr->d_slist_ptr, (void *)(&addr))) { + /* insert the address of the entry in the dirty entry list, and + * add its size to the dirty_bytes count. + */ + if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate dirty slist entry .") + slist_entry_ptr->addr = addr; + + if(H5SL_insert(aux_ptr->d_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into dirty entry slist.") + + aux_ptr->dirty_bytes += entry_ptr->size; +#if H5AC_DEBUG_DIRTY_BYTES_CREATION + aux_ptr->unprotect_dirty_bytes += entry_ptr->size; + aux_ptr->unprotect_dirty_bytes_updates += 1; +#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ + } /* end if */ + + /* the entry is dirty. If it exists on the cleaned entries list, + * remove it. + */ + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)(&addr)))) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + } /* end if */ + else { + aux_ptr->dirty_bytes += entry_ptr->size; +#if H5AC_DEBUG_DIRTY_BYTES_CREATION + aux_ptr->unprotect_dirty_bytes += entry_size; + aux_ptr->unprotect_dirty_bytes_updates += 1; +#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__log_dirtied_entry() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__log_flushed_entry() + * + * Purpose: Update the clean entry slist for the flush of an entry -- + * specifically, if the entry has been cleared, remove it + * from both the cleaned and dirtied lists if it is present. + * Otherwise, if the entry was dirty, insert the indicated + * entry address in the clean slist if it isn't there already. + * + * This function is only used in PHDF5, and should only + * be called for the process with mpi rank 0. + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 6/29/05 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__log_flushed_entry(H5C_t *cache_ptr, haddr_t addr, hbool_t was_dirty, + unsigned flags) +{ + hbool_t cleared; + H5AC_aux_t * aux_ptr; + H5AC_slist_entry_t * slist_entry_ptr = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity check */ + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->mpi_rank == 0); + HDassert(aux_ptr->c_slist_ptr != NULL); + + /* Set local flags */ + cleared = ((flags & H5C__FLUSH_CLEAR_ONLY_FLAG) != 0); + + if(cleared) { + /* If the entry has been cleared, must remove it from both the + * cleaned list and the dirtied list. + */ + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)(&addr)))) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->d_slist_ptr, (void *)(&addr)))) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + } /* end if */ + else if(was_dirty) { + if(NULL == H5SL_search(aux_ptr->c_slist_ptr, (void *)(&addr))) { + /* insert the address of the entry in the clean entry list. */ + if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate clean slist entry .") + slist_entry_ptr->addr = addr; + + if(H5SL_insert(aux_ptr->c_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into clean entry slist.") + } /* end if */ + } /* end else-if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__log_flushed_entry() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__log_inserted_entry() + * + * Purpose: Update the dirty_bytes count for a newly inserted entry. + * + * If mpi_rank isnt 0, this simply means adding the size + * of the entry to the dirty_bytes count. + * + * If mpi_rank is 0, we must also add the entry to the + * dirty entries slist. + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 6/30/05 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__log_inserted_entry(const H5AC_info_t *entry_ptr) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(entry_ptr); + cache_ptr = entry_ptr->cache_ptr; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + + if(aux_ptr->mpi_rank == 0) { + H5AC_slist_entry_t *slist_entry_ptr; + + HDassert(aux_ptr->d_slist_ptr != NULL); + HDassert(aux_ptr->c_slist_ptr != NULL); + + /* Entry to insert should not be in dirty list currently */ + if(NULL != H5SL_search(aux_ptr->d_slist_ptr, (const void *)(&entry_ptr->addr))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Inserted entry already in dirty slist.") + + /* insert the address of the entry in the dirty entry list, and + * add its size to the dirty_bytes count. + */ + if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate dirty slist entry .") + slist_entry_ptr->addr = entry_ptr->addr; + if(H5SL_insert(aux_ptr->d_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into dirty entry slist.") + + /* Entry to insert should not be in clean list either */ + if(NULL != H5SL_search(aux_ptr->c_slist_ptr, (const void *)(&entry_ptr->addr))) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Inserted entry in clean slist.") + } /* end if */ + + aux_ptr->dirty_bytes += entry_ptr->size; + +#if H5AC_DEBUG_DIRTY_BYTES_CREATION + aux_ptr->insert_dirty_bytes += size; + aux_ptr->insert_dirty_bytes_updates += 1; +#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__log_inserted_entry() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__log_moved_entry() + * + * Purpose: Update the dirty_bytes count for a moved entry. + * + * WARNING + * + * At present, the way that the move call is used ensures + * that the moved entry is present in all caches by + * moving in a collective operation and immediately after + * unprotecting the target entry. + * + * This function uses this invariant, and will cause arcane + * failures if it is not met. If maintaining this invariant + * becomes impossible, we will have to rework this function + * extensively, and likely include a bit of IPC for + * synchronization. A better option might be to subsume + * move in the unprotect operation. + * + * Given that the target entry is in all caches, the function + * proceeds as follows: + * + * For processes with mpi rank other 0, it simply checks to + * see if the entry was dirty prior to the move, and adds + * the entries size to the dirty bytes count. + * + * In the process with mpi rank 0, the function first checks + * to see if the entry was dirty prior to the move. If it + * was, and if the entry doesn't appear in the dirtied list + * under its old address, it adds the entry's size to the + * dirty bytes count. + * + * The rank 0 process then removes any references to the + * entry under its old address from the cleands and dirtied + * lists, and inserts an entry in the dirtied list under the + * new address. + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 6/30/05 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__log_moved_entry(const H5F_t *f, haddr_t old_addr, haddr_t new_addr) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + hbool_t entry_in_cache; + hbool_t entry_dirty; + size_t entry_size; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared); + cache_ptr = (H5AC_t *)f->shared->cache; + HDassert(cache_ptr); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + + /* get entry status, size, etc here */ + if(H5C_get_entry_status(f, old_addr, &entry_size, &entry_in_cache, + &entry_dirty, NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't get entry status.") + if(!entry_in_cache) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry not in cache.") + + if(aux_ptr->mpi_rank == 0) { + H5AC_slist_entry_t * slist_entry_ptr; + + HDassert(aux_ptr->d_slist_ptr != NULL); + HDassert(aux_ptr->c_slist_ptr != NULL); + + /* if the entry appears in the cleaned entry slist, under its old + * address, remove it. + */ + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)(&old_addr)))) + slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, slist_entry_ptr); + + /* if the entry appears in the dirtied entry slist under its old + * address, remove it, but don't free it. Set addr to new_addr. + */ + if(NULL != (slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->d_slist_ptr, (void *)(&old_addr)))) + slist_entry_ptr->addr = new_addr; + else { + /* otherwise, allocate a new entry that is ready + * for insertion, and increment dirty_bytes. + * + * Note that the fact that the entry wasn't in the dirtied + * list under its old address implies that it must have + * been clean to start with. + */ + HDassert(!entry_dirty); + if(NULL == (slist_entry_ptr = H5FL_MALLOC(H5AC_slist_entry_t))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "Can't allocate dirty slist entry .") + slist_entry_ptr->addr = new_addr; + + aux_ptr->dirty_bytes += entry_size; + +#if H5AC_DEBUG_DIRTY_BYTES_CREATION + aux_ptr->move_dirty_bytes += entry_size; + aux_ptr->move_dirty_bytes_updates += 1; +#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ + } /* end else */ + + /* insert / reinsert the entry in the dirty slist */ + if(H5SL_insert(aux_ptr->d_slist_ptr, slist_entry_ptr, &(slist_entry_ptr->addr)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTINSERT, FAIL, "can't insert entry into dirty entry slist.") + } /* end if */ + else if(!entry_dirty) { + aux_ptr->dirty_bytes += entry_size; + +#if H5AC_DEBUG_DIRTY_BYTES_CREATION + aux_ptr->move_dirty_bytes += entry_size; + aux_ptr->move_dirty_bytes_updates += 1; +#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ + } /* end else-if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__log_moved_entry() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__propagate_and_apply_candidate_list + * + * Purpose: Prior to the addition of support for multiple metadata + * write strategies, in PHDF5, only the metadata cache with + * mpi rank 0 was allowed to write to file. All other + * metadata caches on processes with rank greater than 0 + * were required to retain dirty entries until they were + * notified that the entry was clean. + * + * This constraint is relaxed with the distributed + * metadata write strategy, in which a list of candidate + * metadata cache entries is constructed by the process 0 + * cache and then distributed to the caches of all the other + * processes. Once the listed is distributed, many (if not + * all) processes writing writing a unique subset of the + * entries, and marking the remainder clean. The subsets + * are chosen so that each entry in the list of candidates + * is written by exactly one cache, and all entries are + * marked as being clean in all caches. + * + * While the list of candidate cache entries is prepared + * elsewhere, this function is the main routine for distributing + * and applying the list. It must be run simultaniously on + * all processes that have the relevant file open. To ensure + * proper synchronization, there is a barrier at the beginning + * of this function. + * + * At present, this function is called under one of two + * circumstances: + * + * 1) Dirty byte creation exceeds some user specified value. + * + * While metadata reads may occur independently, all + * operations writing metadata must be collective. Thus + * all metadata caches see the same sequence of operations, + * and therefore the same dirty data creation. + * + * This fact is used to synchronize the caches for purposes + * of propagating the list of candidate entries, by simply + * calling this function from all caches whenever some user + * specified threshold on dirty data is exceeded. (the + * process 0 cache creates the candidate list just before + * calling this function). + * + * 2) Under direct user control -- this operation must be + * collective. + * + * The operations to be managed by this function are as + * follows: + * + * All processes: + * + * 1) Participate in an opening barrier. + * + * For the process with mpi rank 0: + * + * 1) Load the contents of the candidate list + * (candidate_slist_ptr) into a buffer, and broadcast that + * buffer to all the other caches. Clear the candidate + * list in passing. + * + * If there is a positive number of candidates, proceed with + * the following: + * + * 2) Apply the candidate entry list. + * + * 3) Particpate in a closing barrier. + * + * 4) Remove from the dirty list (d_slist_ptr) and from the + * flushed and still clean entries list (c_slist_ptr), + * all addresses that appeared in the candidate list, as + * these entries are now clean. + * + * + * For all processes with mpi rank greater than 0: + * + * 1) Receive the candidate entry list broadcast + * + * If there is a positive number of candidates, proceed with + * the following: + * + * 2) Apply the candidate entry list. + * + * 3) Particpate in a closing barrier. + * + * Return: Success: non-negative + * + * Failure: negative + * + * Programmer: John Mainzer + * 3/17/10 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__propagate_and_apply_candidate_list(H5F_t *f, hid_t dxpl_id) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + haddr_t * candidates_list_ptr = NULL; + int mpi_result; + int num_candidates = 0; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(f != NULL); + cache_ptr = f->shared->cache; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + + /* to prevent "messages from the future" we must synchronize all + * processes before we write any entries. + */ + if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) + + if(aux_ptr->mpi_rank == 0) { + if(H5AC__broadcast_candidate_list(cache_ptr, &num_candidates, &candidates_list_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't broadcast candidate slist.") + + HDassert(H5SL_count(aux_ptr->candidate_slist_ptr) == 0); + } /* end if */ + else { + if(H5AC__receive_candidate_list(cache_ptr, &num_candidates, &candidates_list_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't receive candidate broadcast.") + } /* end else */ + + if(num_candidates > 0) { + herr_t result; + + /* all processes apply the candidate list. + * H5C_apply_candidate_list() handles the details of + * distributing the writes across the processes. + */ + + /* Enable writes during this operation */ + aux_ptr->write_permitted = TRUE; + + /* Apply the candidate list */ + result = H5C_apply_candidate_list(f, dxpl_id, cache_ptr, num_candidates, + candidates_list_ptr, aux_ptr->mpi_rank, aux_ptr->mpi_size); + + /* Disable writes again */ + aux_ptr->write_permitted = FALSE; + + /* Check for error on the write operation */ + if(result < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't apply candidate list.") + + /* this code exists primarily for the test bed -- it allows us to + * enforce posix semantics on the server that pretends to be a + * file system in our parallel tests. + */ + if(aux_ptr->write_done) + (aux_ptr->write_done)(); + + /* to prevent "messages from the past" we must synchronize all + * processes again before we go on. + */ + if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) + + /* if this is process zero, tidy up the dirtied, + * and flushed and still clean lists. + */ + if(aux_ptr->mpi_rank == 0) + if(H5AC__tidy_cache_0_lists(cache_ptr, num_candidates, candidates_list_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't tidy up process 0 lists.") + } /* end if */ + + /* if it is defined, call the sync point done callback. Note + * that this callback is defined purely for testing purposes, + * and should be undefined under normal operating circumstances. + */ + if(aux_ptr->sync_point_done) + (aux_ptr->sync_point_done)(num_candidates, candidates_list_ptr); + +done: + if(candidates_list_ptr) + candidates_list_ptr = (haddr_t *)H5MM_xfree((void *)candidates_list_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__propagate_and_apply_candidate_list() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__propagate_flushed_and_still_clean_entries_list + * + * Purpose: In PHDF5, if the process 0 only metadata write strategy + * is selected, only the metadata cache with mpi rank 0 is + * allowed to write to file. All other metadata caches on + * processes with rank greater than 0 must retain dirty + * entries until they are notified that the entry is now + * clean. + * + * This function is the main routine for handling this + * notification proceedure. It must be called + * simultaniously on all processes that have the relevant + * file open. To this end, it is called only during a + * sync point, with a barrier prior to the call. + * + * Note that any metadata entry writes by process 0 will + * occur after the barrier and just before this call. + * + * Typicaly, calls to this function will be triggered in + * one of two ways: + * + * 1) Dirty byte creation exceeds some user specified value. + * + * While metadata reads may occur independently, all + * operations writing metadata must be collective. Thus + * all metadata caches see the same sequence of operations, + * and therefore the same dirty data creation. + * + * This fact is used to synchronize the caches for purposes + * of propagating the list of flushed and still clean + * entries, by simply calling this function from all + * caches whenever some user specified threshold on dirty + * data is exceeded. + * + * 2) Under direct user control -- this operation must be + * collective. + * + * The operations to be managed by this function are as + * follows: + * + * For the process with mpi rank 0: + * + * 1) Load the contents of the flushed and still clean entries + * list (c_slist_ptr) into a buffer, and broadcast that + * buffer to all the other caches. + * + * 2) Clear the flushed and still clean entries list + * (c_slist_ptr). + * + * + * For all processes with mpi rank greater than 0: + * + * 1) Receive the flushed and still clean entries list broadcast + * + * 2) Mark the specified entries as clean. + * + * + * For all processes: + * + * 1) Reset the dirtied bytes count to 0. + * + * Return: Success: non-negative + * + * Failure: negative + * + * Programmer: John Mainzer + * July 5, 2005 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__propagate_flushed_and_still_clean_entries_list(H5F_t *f, hid_t dxpl_id) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(f != NULL); + cache_ptr = f->shared->cache; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY); + + if(aux_ptr->mpi_rank == 0) { + if(H5AC__broadcast_clean_list(cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't broadcast clean slist.") + HDassert(H5SL_count(aux_ptr->c_slist_ptr) == 0); + } /* end if */ + else { + if(H5AC__receive_and_apply_clean_list(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't receive and/or process clean slist broadcast.") + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__propagate_flushed_and_still_clean_entries_list() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC_receive_haddr_list() + * + * Purpose: Receive the list of entry addresses from process 0, + * and return it in a buffer pointed to by *haddr_buf_ptr_ptr. + * Note that the caller must free this buffer if it is + * returned. + * + * This function must only be called by the process with + * MPI_rank greater than 0. + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: Quincey Koziol, 6/11/2015 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__receive_haddr_list(MPI_Comm mpi_comm, int *num_entries_ptr, + haddr_t **haddr_buf_ptr_ptr) +{ + haddr_t * haddr_buf_ptr = NULL; + int mpi_result; + int num_entries; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(num_entries_ptr != NULL); + HDassert(*num_entries_ptr == 0); + HDassert(haddr_buf_ptr_ptr != NULL); + HDassert(*haddr_buf_ptr_ptr == NULL); + + /* First receive the number of entries in the list so that we + * can set up a buffer to receive them. If there aren't + * any, we are done. + */ + if(MPI_SUCCESS != (mpi_result = MPI_Bcast(&num_entries, 1, MPI_INT, 0, mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) + + if(num_entries > 0) { + size_t buf_size; + + /* allocate buffers to store the list of entry base addresses in */ + buf_size = sizeof(haddr_t) * (size_t)num_entries; + if(NULL == (haddr_buf_ptr = (haddr_t *)H5MM_malloc(buf_size))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for haddr buffer") + + /* Now receive the list of candidate entries */ + if(MPI_SUCCESS != (mpi_result = MPI_Bcast((void *)haddr_buf_ptr, (int)buf_size, MPI_BYTE, 0, mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Bcast failed", mpi_result) + } /* end if */ + + /* finally, pass the number of entries and the buffer pointer + * back to the caller. + */ + *num_entries_ptr = num_entries; + *haddr_buf_ptr_ptr = haddr_buf_ptr; + +done: + if(ret_value < 0) + if(haddr_buf_ptr) + haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC_receive_haddr_list() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__receive_and_apply_clean_list() + * + * Purpose: Receive the list of cleaned entries from process 0, + * and mark the specified entries as clean. + * + * This function must only be called by the process with + * MPI_rank greater than 0. + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 7/4/05 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__receive_and_apply_clean_list(H5F_t *f, hid_t dxpl_id) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + haddr_t * haddr_buf_ptr = NULL; + int num_entries = 0; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(f != NULL); + cache_ptr = f->shared->cache; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->mpi_rank != 0); + + /* Retrieve the clean list from process 0 */ + if(H5AC__receive_haddr_list(aux_ptr->mpi_comm, &num_entries, &haddr_buf_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't receive clean list") + + if(num_entries > 0) + /* mark the indicated entries as clean */ + if(H5C_mark_entries_as_clean(f, dxpl_id, (int32_t)num_entries, haddr_buf_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't mark entries clean.") + + /* if it is defined, call the sync point done callback. Note + * that this callback is defined purely for testing purposes, + * and should be undefined under normal operating circumstances. + */ + if(aux_ptr->sync_point_done) + (aux_ptr->sync_point_done)(num_entries, haddr_buf_ptr); + +done: + if(haddr_buf_ptr) + haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__receive_and_apply_clean_list() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5AC__receive_candidate_list() + * + * Purpose: Receive the list of candidate entries from process 0, + * and return it in a buffer pointed to by *haddr_buf_ptr_ptr. + * Note that the caller must free this buffer if it is + * returned. + * + * This function must only be called by the process with + * MPI_rank greater than 0. + * + * Return SUCCEED on success, and FAIL on failure. + * + * Return: Non-negative on success/Negative on failure. + * + * Programmer: John Mainzer, 3/17/10 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__receive_candidate_list(const H5AC_t *cache_ptr, int *num_entries_ptr, + haddr_t **haddr_buf_ptr_ptr) +{ + H5AC_aux_t * aux_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->mpi_rank != 0); + HDassert(aux_ptr-> metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + HDassert(num_entries_ptr != NULL); + HDassert(*num_entries_ptr == 0); + HDassert(haddr_buf_ptr_ptr != NULL); + HDassert(*haddr_buf_ptr_ptr == NULL); + + /* Retrieve the candidate list from process 0 */ + if(H5AC__receive_haddr_list(aux_ptr->mpi_comm, num_entries_ptr, haddr_buf_ptr_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't receive clean list") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__receive_candidate_list() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__rsp__dist_md_write__flush + * + * Purpose: Routine for handling the details of running a sync point + * that is triggered by a flush -- which in turn must have been + * triggered by either a flush API call or a file close -- + * when the distributed metadata write strategy is selected. + * + * Upon entry, each process generates it own candidate list, + * being a sorted list of all dirty metadata entries currently + * in the metadata cache. Note that this list must be idendical + * across all processes, as all processes see the same stream + * of dirty metadata coming in, and use the same lists of + * candidate entries at each sync point. (At first glance, this + * argument sounds circular, but think of it in the sense of + * a recursive proof). + * + * If this this list is empty, we are done, and the function + * returns + * + * Otherwise, after the sorted list dirty metadata entries is + * constructed, each process uses the same algorithm to assign + * each entry on the candidate list to exactly one process for + * flushing. + * + * At this point, all processes participate in a barrier to + * avoid messages from the past/future bugs. + * + * Each process then flushes the entries assigned to it, and + * marks all other entries on the candidate list as clean. + * + * Finally, all processes participate in a second barrier to + * avoid messages from the past/future bugs. + * + * At the end of this process, process 0 and only process 0 + * must tidy up its lists of dirtied and cleaned entries. + * These lists are not used in the distributed metadata write + * strategy, but they must be maintained should we shift + * to a strategy that uses them. + * + * Return: Success: non-negative + * + * Failure: negative + * + * Programmer: John Mainzer + * April 28, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__rsp__dist_md_write__flush(H5F_t *f, hid_t dxpl_id) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + haddr_t * haddr_buf_ptr = NULL; + int mpi_result; + int num_entries = 0; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(f != NULL); + cache_ptr = f->shared->cache; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + + /* first construct the candidate list -- initially, this will be in the + * form of a skip list. We will convert it later. + */ + if(H5C_construct_candidate_list__clean_cache(cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate list.") + + if(H5SL_count(aux_ptr->candidate_slist_ptr) > 0) { + herr_t result; + + /* convert the candidate list into the format we + * are used to receiving from process 0. + */ + if(H5AC__copy_candidate_list_to_buffer(cache_ptr, &num_entries, &haddr_buf_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate buffer.") + + /* initial sync point barrier */ + if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) + + /* Enable writes during this operation */ + aux_ptr->write_permitted = TRUE; + + /* Apply the candidate list */ + result = H5C_apply_candidate_list(f, dxpl_id, cache_ptr, num_entries, + haddr_buf_ptr, aux_ptr->mpi_rank, aux_ptr->mpi_size); + + /* Disable writes again */ + aux_ptr->write_permitted = FALSE; + + /* Check for error on the write operation */ + if(result < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't apply candidate list.") + + /* this code exists primarily for the test bed -- it allows us to + * enforce posix semantics on the server that pretends to be a + * file system in our parallel tests. + */ + if(aux_ptr->write_done) + (aux_ptr->write_done)(); + + /* final sync point barrier */ + if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) + + /* if this is process zero, tidy up the dirtied, + * and flushed and still clean lists. + */ + if(aux_ptr->mpi_rank == 0) + if(H5AC__tidy_cache_0_lists(cache_ptr, num_entries, haddr_buf_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't tidy up process 0 lists.") + } /* end if */ + + /* if it is defined, call the sync point done callback. Note + * that this callback is defined purely for testing purposes, + * and should be undefined under normal operating circumstances. + */ + if(aux_ptr->sync_point_done) + (aux_ptr->sync_point_done)(num_entries, haddr_buf_ptr); + +done: + if(haddr_buf_ptr) + haddr_buf_ptr = (haddr_t *)H5MM_xfree((void *)haddr_buf_ptr); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__rsp__dist_md_write__flush() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__rsp__dist_md_write__flush_to_min_clean + * + * Purpose: Routine for handling the details of running a sync point + * triggered by the accumulation of dirty metadata (as + * opposed to a flush call to the API) when the distributed + * metadata write strategy is selected. + * + * After invocation and initial sanity checking this function + * first checks to see if evictions are enabled -- if they + * are not, the function does nothing and returns. + * + * Otherwise, process zero constructs a list of entries to + * be flushed in order to bring the process zero cache back + * within its min clean requirement. Note that this list + * (the candidate list) may be empty. + * + * Then, all processes participate in a barrier. + * + * After the barrier, process 0 broadcasts the number of + * entries in the candidate list prepared above, and all + * other processes receive this number. + * + * If this number is zero, we are done, and the function + * returns without further action. + * + * Otherwise, process 0 broadcasts the sorted list of + * candidate entries, and all other processes receive it. + * + * Then, each process uses the same algorithm to assign + * each entry on the candidate list to exactly one process + * for flushing. + * + * Each process then flushes the entries assigned to it, and + * marks all other entries on the candidate list as clean. + * + * Finally, all processes participate in a second barrier to + * avoid messages from the past/future bugs. + * + * At the end of this process, process 0 and only process 0 + * must tidy up its lists of dirtied and cleaned entries. + * These lists are not used in the distributed metadata write + * strategy, but they must be maintained should we shift + * to a strategy that uses them. + * + * Return: Success: non-negative + * + * Failure: negative + * + * Programmer: John Mainzer + * April 28, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__rsp__dist_md_write__flush_to_min_clean(H5F_t *f, hid_t dxpl_id) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + hbool_t evictions_enabled; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(f != NULL); + cache_ptr = f->shared->cache; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + + /* Query if evictions are allowed */ + if(H5C_get_evictions_enabled((const H5C_t *)cache_ptr, &evictions_enabled) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_get_evictions_enabled() failed.") + + if(evictions_enabled) { + /* construct candidate list -- process 0 only */ + if(aux_ptr->mpi_rank == 0) + if(H5AC__construct_candidate_list(cache_ptr, aux_ptr, H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't construct candidate list.") + + /* propagate and apply candidate list -- all processes */ + if(H5AC__propagate_and_apply_candidate_list(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't propagate and apply candidate list.") + } /* evictions enabled */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__rsp__dist_md_write__flush_to_min_clean() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__rsp__p0_only__flush + * + * Purpose: Routine for handling the details of running a sync point + * that is triggered a flush -- which in turn must have been + * triggered by either a flush API call or a file close -- + * when the process 0 only metadata write strategy is selected. + * + * First, all processes participate in a barrier. + * + * Then process zero flushes all dirty entries, and broadcasts + * they number of clean entries (if any) to all the other + * caches. + * + * If this number is zero, we are done. + * + * Otherwise, process 0 broadcasts the list of cleaned + * entries, and all other processes which are part of this + * file group receive it, and mark the listed entries as + * clean in their caches. + * + * Since all processes have the same set of dirty + * entries at the beginning of the sync point, and all + * entries that will be written are written before + * process zero broadcasts the number of cleaned entries, + * there is no need for a closing barrier. + * + * Return: Success: non-negative + * + * Failure: negative + * + * Programmer: John Mainzer + * April 28, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__rsp__p0_only__flush(H5F_t *f, hid_t dxpl_id) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + int mpi_result; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(f != NULL); + cache_ptr = f->shared->cache; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY); + + /* to prevent "messages from the future" we must + * synchronize all processes before we start the flush. + * Hence the following barrier. + */ + if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) + + /* Flush data to disk, from rank 0 process */ + if(aux_ptr->mpi_rank == 0) { + herr_t result; + + /* Enable writes during this operation */ + aux_ptr->write_permitted = TRUE; + + /* Flush the cache */ + result = H5C_flush_cache(f, dxpl_id, H5AC__NO_FLAGS_SET); + + /* Disable writes again */ + aux_ptr->write_permitted = FALSE; + + /* Check for error on the write operation */ + if(result < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush.") + + /* this code exists primarily for the test bed -- it allows us to + * enforce posix semantics on the server that pretends to be a + * file system in our parallel tests. + */ + if(aux_ptr->write_done) + (aux_ptr->write_done)(); + } /* end if */ + + /* Propagate cleaned entries to other ranks. */ + if(H5AC__propagate_flushed_and_still_clean_entries_list(f, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't propagate clean entries list.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__rsp__p0_only__flush() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__rsp__p0_only__flush_to_min_clean + * + * Purpose: Routine for handling the details of running a sync point + * triggered by the accumulation of dirty metadata (as + * opposed to a flush call to the API) when the process 0 + * only metadata write strategy is selected. + * + * After invocation and initial sanity checking this function + * first checks to see if evictions are enabled -- if they + * are not, the function does nothing and returns. + * + * Otherwise, all processes participate in a barrier. + * + * After the barrier, if this is process 0, the function + * causes the cache to flush sufficient entries to get the + * cache back within its minimum clean fraction, and broadcast + * the number of entries which have been flushed since + * the last sync point, and are still clean. + * + * If this number is zero, we are done. + * + * Otherwise, process 0 broadcasts the list of cleaned + * entries, and all other processes which are part of this + * file group receive it, and mark the listed entries as + * clean in their caches. + * + * Since all processes have the same set of dirty + * entries at the beginning of the sync point, and all + * entries that will be written are written before + * process zero broadcasts the number of cleaned entries, + * there is no need for a closing barrier. + * + * Return: Success: non-negative + * + * Failure: negative + * + * Programmer: John Mainzer + * April 28, 2010 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__rsp__p0_only__flush_to_min_clean(H5F_t *f, hid_t dxpl_id) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + hbool_t evictions_enabled; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(f != NULL); + cache_ptr = f->shared->cache; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY); + + /* Query if evictions are allowed */ + if(H5C_get_evictions_enabled((const H5C_t *)cache_ptr, &evictions_enabled) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_get_evictions_enabled() failed.") + + /* Flush if evictions are allowed -- following call + * will cause process 0 to flush to min clean size, + * and then propagate the newly clean entries to the + * other processes. + * + * Otherwise, do nothing. + */ + if(evictions_enabled) { + int mpi_result; + + /* to prevent "messages from the future" we must synchronize all + * processes before we start the flush. This synchronization may + * already be done -- hence the do_barrier parameter. + */ + if(MPI_SUCCESS != (mpi_result = MPI_Barrier(aux_ptr->mpi_comm))) + HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_result) + + if(0 == aux_ptr->mpi_rank) { + herr_t result; + + /* here, process 0 flushes as many entries as necessary to + * comply with the currently specified min clean size. + * Note that it is quite possible that no entries will be + * flushed. + */ + + /* Enable writes during this operation */ + aux_ptr->write_permitted = TRUE; + + /* Flush the cache */ + result = H5C_flush_to_min_clean(f, dxpl_id); + + /* Disable writes again */ + aux_ptr->write_permitted = FALSE; + + /* Check for error on the write operation */ + if(result < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5C_flush_to_min_clean() failed.") + + /* this call exists primarily for the test code -- it is used + * to enforce POSIX semantics on the process used to simulate + * reads and writes in t_cache.c. + */ + if(aux_ptr->write_done) + (aux_ptr->write_done)(); + } /* end if */ + + if(H5AC__propagate_flushed_and_still_clean_entries_list(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't propagate clean entries list.") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__rsp__p0_only__flush_to_min_clean() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__run_sync_point + * + * Purpose: Top level routine for managing a sync point between all + * meta data caches in the parallel case. Since all caches + * see the same sequence of dirty metadata, we simply count + * bytes of dirty metadata, and run a sync point whenever the + * number of dirty bytes of metadata seen since the last + * sync point exceeds a threshold that is common across all + * processes. We also run sync points in response to + * HDF5 API calls triggering either a flush or a file close. + * + * In earlier versions of PHDF5, only the metadata cache with + * mpi rank 0 was allowed to write to file. All other + * metadata caches on processes with rank greater than 0 were + * required to retain dirty entries until they were notified + * that the entry is was clean. + * + * This function was created to make it easier for us to + * experiment with other options, as it is a single point + * for the execution of sync points. + * + * Return: Success: non-negative + * + * Failure: negative + * + * Programmer: John Mainzer + * March 11, 2010 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__run_sync_point(H5F_t *f, hid_t dxpl_id, int sync_point_op) +{ + H5AC_t * cache_ptr; + H5AC_aux_t * aux_ptr; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(f != NULL); + cache_ptr = f->shared->cache; + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert((sync_point_op == H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN) || + (sync_point_op == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED)); + +#if H5AC_DEBUG_DIRTY_BYTES_CREATION +HDfprintf(stdout, "%d:H5AC_propagate...:%u: (u/uu/i/iu/r/ru) = %zu/%u/%zu/%u/%zu/%u\n", + aux_ptr->mpi_rank, + aux_ptr->dirty_bytes_propagations, + aux_ptr->unprotect_dirty_bytes, + aux_ptr->unprotect_dirty_bytes_updates, + aux_ptr->insert_dirty_bytes, + aux_ptr->insert_dirty_bytes_updates, + aux_ptr->rename_dirty_bytes, + aux_ptr->rename_dirty_bytes_updates); +#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ + + /* clear collective access flag on half of the entries in the + cache and mark them as independent in case they need to be + evicted later. All ranks are guranteed to mark the same entires + since we don't modify the order of the collectively accessed + entries except through collective access. */ + if(H5C_clear_coll_entries(cache_ptr, TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed.") + + switch(aux_ptr->metadata_write_strategy) { + case H5AC_METADATA_WRITE_STRATEGY__PROCESS_0_ONLY: + switch(sync_point_op) { + case H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN: + if(H5AC__rsp__p0_only__flush_to_min_clean(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5AC__rsp__p0_only__flush_to_min_clean() failed.") + break; + + case H5AC_SYNC_POINT_OP__FLUSH_CACHE: + if(H5AC__rsp__p0_only__flush(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5AC__rsp__p0_only__flush() failed.") + break; + + default: + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unknown flush op"); + break; + } /* end switch */ + break; + + case H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED: + switch(sync_point_op) { + case H5AC_SYNC_POINT_OP__FLUSH_TO_MIN_CLEAN: + if(H5AC__rsp__dist_md_write__flush_to_min_clean(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5AC__rsp__dist_md_write__flush_to_min_clean() failed.") + break; + + case H5AC_SYNC_POINT_OP__FLUSH_CACHE: + if(H5AC__rsp__dist_md_write__flush(f, dxpl_id) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5AC__rsp__dist_md_write__flush() failed.") + break; + + default: + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "unknown flush op"); + break; + } /* end switch */ + break; + + default: + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Unknown metadata write strategy.") + break; + } /* end switch */ + + /* reset the dirty bytes count */ + aux_ptr->dirty_bytes = 0; + +#if H5AC_DEBUG_DIRTY_BYTES_CREATION + aux_ptr->dirty_bytes_propagations += 1; + aux_ptr->unprotect_dirty_bytes = 0; + aux_ptr->unprotect_dirty_bytes_updates = 0; + aux_ptr->insert_dirty_bytes = 0; + aux_ptr->insert_dirty_bytes_updates = 0; + aux_ptr->rename_dirty_bytes = 0; + aux_ptr->rename_dirty_bytes_updates = 0; +#endif /* H5AC_DEBUG_DIRTY_BYTES_CREATION */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__run_sync_point() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__tidy_cache_0_lists() + * + * Purpose: In the distributed metadata write strategy, not all dirty + * entries are written by process 0 -- thus we must tidy + * up the dirtied, and flushed and still clean lists + * maintained by process zero after each sync point. + * + * This procedure exists to tend to this issue. + * + * At this point, all entries that process 0 cleared should + * have been removed from both the dirty and flushed and + * still clean lists, and entries that process 0 has flushed + * should have been removed from the dirtied list and added + * to the flushed and still clean list. + * + * However, since the distributed metadata write strategy + * doesn't make use of these lists, the objective is simply + * to maintain these lists in consistent state that allows + * them to be used should the metadata write strategy change + * to one that uses these lists. + * + * Thus for our purposes, all we need to do is remove from + * the dirtied and flushed and still clean lists all + * references to entries that appear in the candidate list. + * + * Return: Success: non-negative + * + * Failure: negative + * + * Programmer: John Mainzer + * 4/20/10 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5AC__tidy_cache_0_lists(H5AC_t *cache_ptr, int num_candidates, + haddr_t *candidates_list_ptr) +{ + H5AC_aux_t * aux_ptr; + int i; + + FUNC_ENTER_STATIC_NOERR + + /* Sanity checks */ + HDassert(cache_ptr != NULL); + aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(cache_ptr); + HDassert(aux_ptr != NULL); + HDassert(aux_ptr->magic == H5AC__H5AC_AUX_T_MAGIC); + HDassert(aux_ptr->metadata_write_strategy == H5AC_METADATA_WRITE_STRATEGY__DISTRIBUTED); + HDassert(aux_ptr->mpi_rank == 0); + HDassert(num_candidates > 0); + HDassert(candidates_list_ptr != NULL); + + /* clean up dirtied and flushed and still clean lists by removing + * all entries on the candidate list. Cleared entries should + * have been removed from both the dirty and cleaned lists at + * this point, flushed entries should have been added to the + * cleaned list. However, for this metadata write strategy, + * we just want to remove all references to the candidate entries. + */ + for(i = 0; i < num_candidates; i++) { + H5AC_slist_entry_t * d_slist_entry_ptr; + H5AC_slist_entry_t * c_slist_entry_ptr; + haddr_t addr; + + addr = candidates_list_ptr[i]; + + /* addr may be either on the dirtied list, or on the flushed + * and still clean list. Remove it. + */ + if(NULL != (d_slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->d_slist_ptr, (void *)&addr))) + d_slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, d_slist_entry_ptr); + if(NULL != (c_slist_entry_ptr = (H5AC_slist_entry_t *)H5SL_remove(aux_ptr->c_slist_ptr, (void *)&addr))) + c_slist_entry_ptr = H5FL_FREE(H5AC_slist_entry_t, c_slist_entry_ptr); + } /* end for */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5AC__tidy_cache_0_lists() */ + + +/*------------------------------------------------------------------------- + * Function: H5AC__flush_entries + * + * Purpose: Flush the metadata cache associated with the specified file, + * only writing from rank 0, but propagating the cleaned entries + * to all ranks. + * + * Return: Non-negative on success/Negative on failure if there was a + * request to flush all items and something was protected. + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Aug 22 2009 + * + *------------------------------------------------------------------------- + */ +herr_t +H5AC__flush_entries(H5F_t *f, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_PACKAGE + + /* Sanity checks */ + HDassert(f); + HDassert(f->shared->cache); + + /* Check if we have >1 ranks */ + if(H5C_get_aux_ptr(f->shared->cache)) + if(H5AC__run_sync_point(f, dxpl_id, H5AC_SYNC_POINT_OP__FLUSH_CACHE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't run sync point.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5AC__flush_entries() */ +#endif /* H5_HAVE_PARALLEL */ + diff --git a/src/H5ACpkg.h b/src/H5ACpkg.h index 28965d2..74bf079 100644 --- a/src/H5ACpkg.h +++ b/src/H5ACpkg.h @@ -41,8 +41,20 @@ /* Get needed headers */ #include "H5Cprivate.h" /* Cache */ +#include "H5FLprivate.h" /* Free Lists */ #include "H5SLprivate.h" /* Skip lists */ +/*****************************/ +/* Package Private Variables */ +/*****************************/ + +/* Declare extern the free list to manage the H5AC_aux_t struct */ +H5FL_EXTERN(H5AC_aux_t); + + +/**************************/ +/* Package Private Macros */ +/**************************/ #define H5AC_DEBUG_DIRTY_BYTES_CREATION 0 @@ -393,9 +405,18 @@ typedef struct H5AC_aux_t } H5AC_aux_t; /* struct H5AC_aux_t */ /* Package scoped functions */ -H5_DLL herr_t H5AC_set_sync_point_done_callback(H5C_t *cache_ptr, +H5_DLL herr_t H5AC__log_deleted_entry(const H5AC_info_t *entry_ptr); +H5_DLL herr_t H5AC__log_dirtied_entry(const H5AC_info_t *entry_ptr); +H5_DLL herr_t H5AC__log_flushed_entry(H5C_t *cache_ptr, haddr_t addr, + hbool_t was_dirty, unsigned flags); +H5_DLL herr_t H5AC__log_inserted_entry(const H5AC_info_t *entry_ptr); +H5_DLL herr_t H5AC__log_moved_entry(const H5F_t *f, haddr_t old_addr, + haddr_t new_addr); +H5_DLL herr_t H5AC__flush_entries(H5F_t *f, hid_t dxpl_id); +H5_DLL herr_t H5AC__run_sync_point(H5F_t *f, hid_t dxpl_id, int sync_point_op); +H5_DLL herr_t H5AC__set_sync_point_done_callback(H5C_t *cache_ptr, void (*sync_point_done)(int num_writes, haddr_t *written_entries_tbl)); -H5_DLL herr_t H5AC_set_write_done_callback(H5C_t * cache_ptr, +H5_DLL herr_t H5AC__set_write_done_callback(H5C_t * cache_ptr, void (* write_done)(void)); #endif /* H5_HAVE_PARALLEL */ diff --git a/src/H5B2cache.c b/src/H5B2cache.c index 64e304c..9e43ff8 100644 --- a/src/H5B2cache.c +++ b/src/H5B2cache.c @@ -393,7 +393,7 @@ H5B2__cache_hdr_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED le UINT32ENCODE(image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); FUNC_LEAVE_NOAPI(SUCCEED) } /* H5B2__cache_hdr_serialize() */ @@ -698,6 +698,11 @@ H5B2__cache_int_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED le /* Sanity check */ HDassert((size_t)(image - (uint8_t *)_image) <= len); +#ifdef H5_CLEAR_MEMORY + /* Clear rest of internal node */ + HDmemset(image, 0, len - (size_t)(image - (uint8_t *)_image)); +#endif /* H5_CLEAR_MEMORY */ + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5B2__cache_int_serialize() */ @@ -970,6 +975,11 @@ H5B2__cache_leaf_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED l /* Sanity check */ HDassert((size_t)(image - (uint8_t *)_image) <= len); +#ifdef H5_CLEAR_MEMORY + /* Clear rest of leaf node */ + HDmemset(image, 0, len - (size_t)(image - (uint8_t *)_image)); +#endif /* H5_CLEAR_MEMORY */ + done: FUNC_LEAVE_NOAPI(ret_value) } /* H5B2__cache_leaf_serialize() */ diff --git a/src/H5Bcache.c b/src/H5Bcache.c index dfa4dab..d071712 100644 --- a/src/H5Bcache.c +++ b/src/H5Bcache.c @@ -338,10 +338,16 @@ H5B__serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len, /* Encode the final key */ if(shared->type->encode(shared, image, native) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTENCODE, FAIL, "unable to encode B-tree key") + image += shared->sizeof_rkey; } /* end if */ /* Sanity check */ - HDassert((size_t)((const uint8_t *)image - (const uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) <= len); + +#ifdef H5_CLEAR_MEMORY + /* Clear rest of node */ + HDmemset(image, 0, len - (size_t)(image - (uint8_t *)_image)); +#endif /* H5_CLEAR_MEMORY */ done: FUNC_LEAVE_NOAPI(ret_value) @@ -41,7 +41,7 @@ * * Code Changes: * - * - Remove extra functionality in H5C_flush_single_entry()? + * - Remove extra functionality in H5C__flush_single_entry()? * * - Change protect/unprotect to lock/unlock. * @@ -70,19 +70,24 @@ * **************************************************************************/ +/****************/ +/* Module Setup */ +/****************/ + #define H5C_PACKAGE /*suppress error about including H5Cpkg */ #define H5F_PACKAGE /*suppress error about including H5Fpkg */ +/***********/ +/* Headers */ +/***********/ #include "H5private.h" /* Generic Functions */ #ifdef H5_HAVE_PARALLEL #include "H5ACprivate.h" /* Metadata cache */ #endif /* H5_HAVE_PARALLEL */ #include "H5Cpkg.h" /* Cache */ -#include "H5Dprivate.h" /* Dataset functions */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fpkg.h" /* Files */ -#include "H5FDprivate.h" /* File drivers */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ #include "H5MFprivate.h" /* File memory management */ @@ -91,9 +96,9 @@ #include "H5SLprivate.h" /* Skip lists */ -/* - * Private macros. - */ +/****************/ +/* Local Macros */ +/****************/ #if H5C_DO_MEMORY_SANITY_CHECKS #define H5C_IMAGE_EXTRA_SPACE 8 #define H5C_IMAGE_SANITY_VALUE "DeadBeef" @@ -101,30 +106,14 @@ #define H5C_IMAGE_EXTRA_SPACE 0 #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ -/* - * Private file-scope variables. - */ +/******************/ +/* Local Typedefs */ +/******************/ -/* Declare a free list to manage the H5C_t struct */ -H5FL_DEFINE_STATIC(H5C_t); - -#ifdef H5_HAVE_PARALLEL -typedef struct H5C_collective_write_t { - size_t length; - hbool_t free_buf; - void *buf; - haddr_t offset; -} H5C_collective_write_t; -static herr_t H5C_collective_write(H5F_t *f, - hid_t dxpl_id, - H5SL_t *collective_write_list); -static herr_t H5C_collective_write_free(void *_item, void *key, void *op_data); -#endif /* H5_HAVE_PARALLEL */ - -/* - * Private file-scope function declarations: - */ +/********************/ +/* Local Prototypes */ +/********************/ static herr_t H5C__auto_adjust_cache_size(H5F_t * f, hid_t dxpl_id, @@ -153,14 +142,6 @@ static herr_t H5C__flash_increase_cache_size(H5C_t * cache_ptr, size_t old_entry_size, size_t new_entry_size); -static herr_t H5C_flush_single_entry(const H5F_t * f, - hid_t dxpl_id, - haddr_t addr, - unsigned flags, - hbool_t del_entry_from_slist_on_destroy, - int64_t *entry_size_change_ptr, - H5SL_t * collective_write_list); - static herr_t H5C_flush_invalidate_cache(const H5F_t * f, hid_t dxpl_id, unsigned flags); @@ -192,8 +173,7 @@ static herr_t H5C_mark_tagged_entries(H5C_t * cache_ptr, haddr_t tag); static herr_t H5C_flush_marked_entries(H5F_t * f, - hid_t dxpl_id, - H5C_t * cache_ptr); + hid_t dxpl_id); static herr_t H5C__generate_image(H5F_t *f, H5C_t * cache_ptr, H5C_cache_entry_t *entry_ptr, hid_t dxpl_id, int64_t *entry_size_change_ptr); @@ -218,6 +198,24 @@ herr_t H5C_dump_cache(H5C_t * cache_ptr, const char * cache_name); herr_t H5C_dump_cache_skip_list(H5C_t * cache_ptr, char * calling_fcn); #endif /* debugging routines */ +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Declare a free list to manage the H5C_t struct */ +H5FL_DEFINE_STATIC(H5C_t); + + /**************************************************************************** * @@ -406,920 +404,6 @@ H5C__epoch_marker_fsf_size(const void H5_ATTR_UNUSED * thing, size_t H5_ATTR_UNU /*------------------------------------------------------------------------- - * Function: H5C_apply_candidate_list - * - * Purpose: Apply the supplied candidate list. - * - * We used to do this by simply having each process write - * every mpi_size-th entry in the candidate list, starting - * at index mpi_rank, and mark all the others clean. - * - * However, this can cause unnecessary contention in a file - * system by increasing the number of processes writing to - * adjacent locations in the HDF5 file. - * - * To attempt to minimize this, we now arange matters such - * that each process writes n adjacent entries in the - * candidate list, and marks all others clean. We must do - * this in such a fashion as to guarantee that each entry - * on the candidate list is written by exactly one process, - * and marked clean by all others. - * - * To do this, first construct a table mapping mpi_rank - * to the index of the first entry in the candidate list to - * be written by the process of that mpi_rank, and then use - * the table to control which entries are written and which - * are marked as clean as a function of the mpi_rank. - * - * Note that the table must be identical on all processes, as - * all see the same candidate list, mpi_size, and mpi_rank -- - * the inputs used to construct the table. - * - * We construct the table as follows. Let: - * - * n = num_candidates / mpi_size; - * - * m = num_candidates % mpi_size; - * - * Now allocate an array of integers of length mpi_size + 1, - * and call this array candidate_assignment_table. - * - * Conceptually, if the number of candidates is a multiple - * of the mpi_size, we simply pass through the candidate list - * and assign n entries to each process to flush, with the - * index of the first entry to flush in the location in - * the candidate_assignment_table indicated by the mpi_rank - * of the process. - * - * In the more common case in which the candidate list isn't - * isn't a multiple of the mpi_size, we pretend it is, and - * give num_candidates % mpi_size processes one extra entry - * each to make things work out. - * - * Once the table is constructed, we determine the first and - * last entry this process is to flush as follows: - * - * first_entry_to_flush = candidate_assignment_table[mpi_rank] - * - * last_entry_to_flush = - * candidate_assignment_table[mpi_rank + 1] - 1; - * - * With these values determined, we simply scan through the - * candidate list, marking all entries in the range - * [first_entry_to_flush, last_entry_to_flush] for flush, - * and all others to be cleaned. - * - * Finally, we scan the LRU from tail to head, flushing - * or marking clean the candidate entries as indicated. - * If necessary, we scan the pinned list as well. - * - * Note that this function will fail if any protected or - * clean entries appear on the candidate list. - * - * This function is used in managing sync points, and - * shouldn't be used elsewhere. - * - * Return: Success: SUCCEED - * - * Failure: FAIL - * - * Programmer: John Mainzer - * 3/17/10 - * - * Changes: Ported code to detect next entry status changes as the - * the result of a flush from the serial code in the scan of - * the LRU. Also added code to detect and adapt to the - * removal from the cache of the next entry in the scan of - * the LRU. - * - * Note that at present, all of these changes should not - * be required as the operations on entries as they are - * flushed that can cause these condiditions are not premitted - * in the parallel case. However, Quincey indicates that - * this may change, and thus has requested the modification. - * - * Note the assert(FALSE) in the if statement whose body - * restarts the scan of the LRU. As the body of the if - * statement should be unreachable, it should never be - * triggered until the constraints on the parallel case - * are relaxed. Please remove the assertion at that time. - * - * Also added warning on the Pinned Entry List scan, as it - * is potentially subject to the same issue. As there is - * no cognate of this scan in the serial code, I don't have - * a fix to port to it. - * - * JRM -- 4/10/19 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -#define H5C_APPLY_CANDIDATE_LIST__DEBUG 0 -herr_t -H5C_apply_candidate_list(H5F_t * f, - hid_t dxpl_id, - H5C_t * cache_ptr, - int num_candidates, - haddr_t * candidates_list_ptr, - int mpi_rank, - int mpi_size) -{ - hbool_t restart_scan; - hbool_t prev_is_dirty; - int i; - int m; - int n; - int first_entry_to_flush; - int last_entry_to_flush; - int entries_to_clear = 0; - int entries_to_flush = 0; - int entries_to_flush_or_clear_last = 0; - int entries_to_flush_collectively = 0; - int entries_cleared = 0; - int entries_flushed = 0; - int entries_delayed = 0; - int entries_flushed_or_cleared_last = 0; - int entries_flushed_collectively = 0; - int entries_examined = 0; - int initial_list_len; - int * candidate_assignment_table = NULL; - haddr_t addr; - H5C_cache_entry_t * clear_ptr = NULL; - H5C_cache_entry_t * next_ptr = NULL; - H5C_cache_entry_t * entry_ptr = NULL; - H5C_cache_entry_t * flush_ptr = NULL; - H5C_cache_entry_t * delayed_ptr = NULL; - H5SL_t * collective_write_list = NULL; -#if H5C_DO_SANITY_CHECKS - haddr_t last_addr; -#endif /* H5C_DO_SANITY_CHECKS */ -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - char tbl_buf[1024]; -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - HDassert( cache_ptr != NULL ); - HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); - HDassert( num_candidates > 0 ); - HDassert( num_candidates <= cache_ptr->slist_len ); - HDassert( candidates_list_ptr != NULL ); - HDassert( 0 <= mpi_rank ); - HDassert( mpi_rank < mpi_size ); - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: setting up candidate assignment table.\n", - FUNC, mpi_rank); - for ( i = 0; i < 1024; i++ ) tbl_buf[i] = '\0'; - sprintf(&(tbl_buf[0]), "candidate list = "); - for ( i = 0; i < num_candidates; i++ ) - { - sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), " 0x%llx", - (long long)(*(candidates_list_ptr + i))); - } - sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), "\n"); - HDfprintf(stdout, "%s", tbl_buf); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* Create skip list of entries for collective write */ - if(NULL == (collective_write_list = H5SL_create(H5SL_TYPE_HADDR, NULL))) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "can't create skip list for entries") - - n = num_candidates / mpi_size; - m = num_candidates % mpi_size; - HDassert(n >= 0); - - if(NULL == (candidate_assignment_table = (int *)H5MM_malloc(sizeof(int) * (size_t)(mpi_size + 1)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for candidate assignment table") - - candidate_assignment_table[0] = 0; - candidate_assignment_table[mpi_size] = num_candidates; - - if(m == 0) { /* mpi_size is an even divisor of num_candidates */ - HDassert(n > 0); - for(i = 1; i < mpi_size; i++) - candidate_assignment_table[i] = candidate_assignment_table[i - 1] + n; - } /* end if */ - else { - for(i = 1; i <= m; i++) - candidate_assignment_table[i] = candidate_assignment_table[i - 1] + n + 1; - - if(num_candidates < mpi_size) { - for(i = m + 1; i < mpi_size; i++) - candidate_assignment_table[i] = num_candidates; - } /* end if */ - else { - for(i = m + 1; i < mpi_size; i++) - candidate_assignment_table[i] = candidate_assignment_table[i - 1] + n; - } /* end else */ - } /* end else */ - HDassert((candidate_assignment_table[mpi_size - 1] + n) == num_candidates); - -#if H5C_DO_SANITY_CHECKS - /* verify that the candidate assignment table has the expected form */ - for ( i = 1; i < mpi_size - 1; i++ ) - { - int a, b; - - a = candidate_assignment_table[i] - candidate_assignment_table[i - 1]; - b = candidate_assignment_table[i + 1] - candidate_assignment_table[i]; - - HDassert( n + 1 >= a ); - HDassert( a >= b ); - HDassert( b >= n ); - } -#endif /* H5C_DO_SANITY_CHECKS */ - - first_entry_to_flush = candidate_assignment_table[mpi_rank]; - last_entry_to_flush = candidate_assignment_table[mpi_rank + 1] - 1; - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - for ( i = 0; i < 1024; i++ ) - tbl_buf[i] = '\0'; - sprintf(&(tbl_buf[0]), "candidate assignment table = "); - for(i = 0; i <= mpi_size; i++) - sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), " %d", candidate_assignment_table[i]); - sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), "\n"); - HDfprintf(stdout, "%s", tbl_buf); - - HDfprintf(stdout, "%s:%d: flush entries [%d, %d].\n", - FUNC, mpi_rank, first_entry_to_flush, last_entry_to_flush); - - HDfprintf(stdout, "%s:%d: marking entries.\n", FUNC, mpi_rank); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - for(i = 0; i < num_candidates; i++) { - addr = candidates_list_ptr[i]; - HDassert( H5F_addr_defined(addr) ); - -#if H5C_DO_SANITY_CHECKS - if ( i > 0 ) { - if ( last_addr == addr ) { - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Duplicate entry in cleaned list.\n") - } else if ( last_addr > addr ) { - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "candidate list not sorted.\n") - } - } - - last_addr = addr; -#endif /* H5C_DO_SANITY_CHECKS */ - - H5C__SEARCH_INDEX(cache_ptr, addr, entry_ptr, FAIL) - if(entry_ptr == NULL) { - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed candidate entry not in cache?!?!?.") - } else if(!entry_ptr->is_dirty) { - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed entry not dirty?!?!?.") - } else if ( entry_ptr->is_protected ) { - /* For now at least, we can't deal with protected entries. - * If we encounter one, scream and die. If it becomes an - * issue, we should be able to work around this. - */ - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed entry is protected?!?!?.") - } else { - /* determine whether the entry is to be cleared or flushed, - * and mark it accordingly. We will scan the protected and - * pinned list shortly, and clear or flush according to these - * markings. - */ - if((i >= first_entry_to_flush) && (i <= last_entry_to_flush)) { - entries_to_flush++; - entry_ptr->flush_immediately = TRUE; - } /* end if */ - else { - entries_to_clear++; - entry_ptr->clear_on_unprotect = TRUE; - } /* end else */ - - /* Entries marked as collectively accessed and are in the - candidate list to clear from the cache have to be - removed from the coll list. This is OK since the - candidate list is collective and uniform across all - ranks. */ - if(TRUE == entry_ptr->coll_access) { - entry_ptr->coll_access = FALSE; - H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) - } - } /* end else */ - } /* end for */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: num candidates/to clear/to flush = %d/%d/%d.\n", - FUNC, mpi_rank, (int)num_candidates, (int)entries_to_clear, - (int)entries_to_flush); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - - /* We have now marked all the entries on the candidate list for - * either flush or clear -- now scan the LRU and the pinned list - * for these entries and do the deed. - * - * Note that we are doing things in this round about manner so as - * to preserve the order of the LRU list to the best of our ability. - * If we don't do this, my experiments indicate that we will have a - * noticably poorer hit ratio as a result. - */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: scanning LRU list. len = %d.\n", FUNC, mpi_rank, - (int)(cache_ptr->LRU_list_len)); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* ===================================================================== * - * Now scan the LRU and PEL lists, flushing or clearing entries as - * needed. - * - * The flush_me_last and flush_me_collectively flags may dictate how or - * when some entries can be flushed, and should be addressed here. - * However, in their initial implementation, these flags only apply to the - * superblock, so there's only a relatively small change to this function - * to account for this one case where they come into play. If these flags - * are ever expanded upon, this function and the following flushing steps - * should be reworked to account for additional cases. - * ===================================================================== */ - - HDassert(entries_to_flush >= 0); - - restart_scan = FALSE; - entries_examined = 0; - initial_list_len = cache_ptr->LRU_list_len; - entry_ptr = cache_ptr->LRU_tail_ptr; - - /* Examine each entry in the LRU list */ - while ( ( entry_ptr != NULL ) - && - ( entries_examined <= (entries_to_flush + 1) * initial_list_len ) - && - ( (entries_cleared + entries_flushed) < num_candidates ) ) { - - if ( entry_ptr->prev != NULL ) - prev_is_dirty = entry_ptr->prev->is_dirty; - - /* If this process needs to clear this entry. */ - if(entry_ptr->clear_on_unprotect) { - - HDassert(entry_ptr->is_dirty); - - next_ptr = entry_ptr->next; - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->prev; - entries_cleared++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, - (long long)clear_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* No need to check for the next entry in the scan being - * removed from the cache, as this call to H5C_flush_single_entry() - * will not call either the pre_serialize or serialize callbacks. - */ - - if(H5C_flush_single_entry(f, - dxpl_id, - clear_ptr->addr, - H5C__FLUSH_CLEAR_ONLY_FLAG, - TRUE, - NULL, - NULL) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") - } /* end if */ - - /* Else, if this process needs to flush this entry. */ - else if (entry_ptr->flush_immediately) { - - HDassert(entry_ptr->is_dirty); - - next_ptr = entry_ptr->next; - entry_ptr->flush_immediately = FALSE; - flush_ptr = entry_ptr; - entry_ptr = entry_ptr->prev; - entries_flushed++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, - (long long)flush_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* reset entries_removed_counter and - * last_entry_removed_ptr prior to the call to - * H5C_flush_single_entry() so that we can spot - * unexpected removals of entries from the cache, - * and set the restart_scan flag if proceeding - * would be likely to cause us to scan an entry - * that is no longer in the cache. - * - * Note that as of this writing (April 2015) this - * case cannot occur in the parallel case. However - * Quincey is making noises about changing this, hence - * the insertion of this test. - * - * Note also that there is no test code to verify - * that this code actually works (although similar code - * in the serial version exists and is tested). - * - * Implementing a test will likely require implementing - * flush op like facilities in the parallel tests. At - * a guess this will not be terribly painful, but it - * will take a bit of time. - */ - cache_ptr->entries_removed_counter = 0; - cache_ptr->last_entry_removed_ptr = NULL; - - if(H5C_flush_single_entry(f, - dxpl_id, - flush_ptr->addr, - H5C__NO_FLAGS_SET, - TRUE, - NULL, - collective_write_list) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry.") - - if ( ( cache_ptr->entries_removed_counter > 1 ) || - ( cache_ptr->last_entry_removed_ptr == entry_ptr ) ) - - restart_scan = TRUE; - - } /* end else-if */ - - /* Otherwise, no action to be taken on this entry. Grab the next. */ - else { - entry_ptr = entry_ptr->prev; - - if ( entry_ptr != NULL ) - next_ptr = entry_ptr->next; - - } /* end else */ - - if ( ( entry_ptr != NULL ) - && - ( ( restart_scan ) - || - ( entry_ptr->is_dirty != prev_is_dirty ) - || - ( entry_ptr->next != next_ptr ) - || - ( entry_ptr->is_protected ) - || - ( entry_ptr->is_pinned ) - ) - ) { - - /* something has happened to the LRU -- start over - * from the tail. - * - * Recall that this code should be un-reachable at present, - * as all the operations by entries on flush that could cause - * it to be reachable are disallowed in the parallel case at - * present. Hence the following assertion which should be - * removed if the above changes. - */ - - HDassert( ! restart_scan ); - HDassert( entry_ptr->is_dirty == prev_is_dirty ); - HDassert( entry_ptr->next == next_ptr ); - HDassert( ! entry_ptr->is_protected ); - HDassert( ! entry_ptr->is_pinned ); - - HDassert(FALSE); /* see comment above */ - - restart_scan = FALSE; - entry_ptr = cache_ptr->LRU_tail_ptr; -/* - H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) -*/ - } - - entries_examined++; - } /* end while */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %d/%d/%d.\n", - FUNC, mpi_rank, entries_examined, - entries_cleared, entries_flushed); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* It is also possible that some of the cleared entries are on the - * pinned list. Must scan that also. - * - * WARNING: - * - * As we now allow unpinning, and removal of other entries as a side - * effect of flushing an entry, it is possible that the next entry - * in a PEL scan could either be no longer pinned, or no longer in - * the cache by the time we get to it. - * - * At present, this is not possible in this case, as we disallow such - * operations in the parallel version of the library. However, Quincey - * has been making noises about relaxing this. If and when he does, - * we have a potential problem here. - * - * The same issue exists in the serial cache, and there are tests - * to detect this problem when it occurs, and adjust to it. As seen - * above in the LRU scan, I have ported such tests to the parallel - * code where a close cognate exists in the serial code. - * - * I haven't done so here, as there are no PEL scans where the problem - * can occur in the serial code. Needless to say, this will have to - * be repaired if the constraints on pre_serialize and serialize - * callbacks are relaxed in the parallel version of the metadata cache. - * - * JRM -- 4/1/15 - */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n", - FUNC, mpi_rank, (int)(cache_ptr->pel_len)); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - entry_ptr = cache_ptr->pel_head_ptr; - while((entry_ptr != NULL) && - ((entries_cleared + entries_flushed + entries_delayed) - < num_candidates)) { - - /* If entry is marked for flush or for clear */ - if((entry_ptr->clear_on_unprotect||entry_ptr->flush_immediately)) { - - /* If this entry needs to be flushed last */ - if (entry_ptr->flush_me_last) { - - /* At this time, only the superblock supports being - flushed last. Conveniently, it also happens to be the only - entry that supports being flushed collectively, as well. Also - conveniently, it's always pinned, so we only need to check - for it while scanning the PEL here. Finally, it's never - included in a candidate list that excludes other dirty - entries in a cache, so we can handle this relatively simple - case here. - - For now, this function asserts this and saves the entry - to flush it after scanning the rest of the PEL list. - - If there are ever more entries that either need to be - flushed last and/or flushed collectively, this whole routine - will need to be reworked to handle all additional cases. As - it is the simple case of a single pinned entry needing - flushed last and collectively is just a minor addition to - this routine, but signficantly buffing up the usage of - flush_me_last or flush_me_collectively will require a more - intense rework of this function and potentially the function - of candidate lists as a whole. */ - - HDassert(entry_ptr->flush_me_collectively); - entries_to_flush_or_clear_last++; - entries_to_flush_collectively++; - HDassert(entries_to_flush_or_clear_last == 1); - HDassert(entries_to_flush_collectively == 1); - - /* Delay the entry. It will be flushed later. */ - delayed_ptr = entry_ptr; - entries_delayed++; - HDassert(entries_delayed == 1); - - } /* end if */ - - /* Else, this process needs to clear this entry. */ - else if (entry_ptr->clear_on_unprotect) { - HDassert(!entry_ptr->flush_immediately); - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_cleared++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, - (long long)clear_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - if(H5C_flush_single_entry(f, - dxpl_id, - clear_ptr->addr, - H5C__FLUSH_CLEAR_ONLY_FLAG, - TRUE, - NULL, - NULL) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") - } /* end else-if */ - - /* Else, if this process needs to independently flush this entry. */ - else if (entry_ptr->flush_immediately) { - entry_ptr->flush_immediately = FALSE; - flush_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_flushed++; - -#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) - HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, - (long long)flush_ptr->addr); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - if(H5C_flush_single_entry(f, - dxpl_id, - flush_ptr->addr, - H5C__NO_FLAGS_SET, - TRUE, - NULL, - collective_write_list) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") - } /* end else-if */ - } /* end if */ - - /* Otherwise, this entry is not marked for flush or clear. Grab the next. */ - else { - entry_ptr = entry_ptr->next; - } /* end else */ - - } /* end while */ - -#if H5C_APPLY_CANDIDATE_LIST__DEBUG - HDfprintf(stdout, - "%s:%d: pel entries examined/cleared/flushed = %d/%d/%d.\n", - FUNC, mpi_rank, entries_examined, - entries_cleared, entries_flushed); - HDfprintf(stdout, "%s:%d: done.\n", FUNC, mpi_rank); - - HDfsync(stdout); -#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ - - /* ====================================================================== * - * Now, handle all delayed entries. * - * * - * This can *only* be the superblock at this time, so it's relatively * - * easy to deal with. We're collectively flushing the entry saved from * - * above. This will need to be handled differently if there are ever more * - * than one entry needing this special treatment.) * - * ====================================================================== */ - - if (delayed_ptr) { - - if (delayed_ptr->clear_on_unprotect) { - if(H5C_flush_single_entry(f, - dxpl_id, - delayed_ptr->addr, - H5C__FLUSH_CLEAR_ONLY_FLAG, - TRUE, - NULL, - NULL) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, - "Can't flush entry collectively.") - entry_ptr->clear_on_unprotect = FALSE; - entries_cleared++; - } else if (delayed_ptr->flush_immediately) { - if(H5C_flush_single_entry(f, - dxpl_id, - delayed_ptr->addr, - H5C__NO_FLAGS_SET, - TRUE, - NULL, - collective_write_list) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, - "Can't flush entry collectively.") - entry_ptr->flush_immediately = FALSE; - entries_flushed++; - } /* end if */ - - entries_flushed_collectively++; - entries_flushed_or_cleared_last++; - } /* end if */ - - /* Write collective list */ - if(H5C_collective_write(f, - dxpl_id, - collective_write_list) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't write metadata collectively") - - /* ====================================================================== * - * Finished flushing everything. * - * ====================================================================== */ - - HDassert((entries_flushed == entries_to_flush)); - HDassert((entries_cleared == entries_to_clear)); - HDassert((entries_flushed_or_cleared_last == entries_to_flush_or_clear_last)); - HDassert((entries_flushed_collectively == entries_to_flush_collectively)); - - if((entries_flushed != entries_to_flush) || - (entries_cleared != entries_to_clear) || - (entries_flushed_or_cleared_last != entries_to_flush_or_clear_last) || - (entries_flushed_collectively != entries_to_flush_collectively)) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry count mismatch.") - -done: - if(candidate_assignment_table != NULL) - candidate_assignment_table = (int *)H5MM_xfree((void *)candidate_assignment_table); - - if(collective_write_list) - if(H5SL_destroy(collective_write_list, H5C_collective_write_free, NULL) < 0) - HDONE_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "failed to destroy skip list") - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_apply_candidate_list() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5C_construct_candidate_list__clean_cache - * - * Purpose: Construct the list of entries that should be flushed to - * clean all entries in the cache. - * - * This function is used in managing sync points, and - * shouldn't be used elsewhere. - * - * Return: Success: SUCCEED - * - * Failure: FAIL - * - * Programmer: John Mainzer - * 3/17/10 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -herr_t -H5C_construct_candidate_list__clean_cache(H5C_t * cache_ptr) -{ - size_t space_needed; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - HDassert( cache_ptr != NULL ); - HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); - - /* As a sanity check, set space needed to the size of the skip list. - * This should be the sum total of the sizes of all the dirty entries - * in the metadata cache. - */ - space_needed = cache_ptr->slist_size; - - /* Recall that while we shouldn't have any protected entries at this - * point, it is possible that some dirty entries may reside on the - * pinned list at this point. - */ - HDassert( cache_ptr->slist_size <= - (cache_ptr->dLRU_list_size + cache_ptr->pel_size) ); - HDassert( cache_ptr->slist_len <= - (cache_ptr->dLRU_list_len + cache_ptr->pel_len) ); - - if(space_needed > 0) { /* we have work to do */ - H5C_cache_entry_t *entry_ptr; - int nominated_entries_count = 0; - size_t nominated_entries_size = 0; - haddr_t nominated_addr; - - HDassert( cache_ptr->slist_len > 0 ); - - /* Scan the dirty LRU list from tail forward and nominate sufficient - * entries to free up the necessary space. - */ - entry_ptr = cache_ptr->dLRU_tail_ptr; - while((nominated_entries_size < space_needed) && - (nominated_entries_count < cache_ptr->slist_len) && - (entry_ptr != NULL)) { - HDassert( ! (entry_ptr->is_protected) ); - HDassert( ! (entry_ptr->is_read_only) ); - HDassert( entry_ptr->ro_ref_count == 0 ); - HDassert( entry_ptr->is_dirty ); - HDassert( entry_ptr->in_slist ); - - nominated_addr = entry_ptr->addr; - if(H5AC_add_candidate((H5AC_t *)cache_ptr, nominated_addr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_add_candidate() failed(1).") - - nominated_entries_size += entry_ptr->size; - nominated_entries_count++; - entry_ptr = entry_ptr->aux_prev; - } /* end while */ - HDassert( entry_ptr == NULL ); - - /* it is possible that there are some dirty entries on the - * protected entry list as well -- scan it too if necessary - */ - entry_ptr = cache_ptr->pel_head_ptr; - while((nominated_entries_size < space_needed) && - (nominated_entries_count < cache_ptr->slist_len) && - (entry_ptr != NULL)) { - if(entry_ptr->is_dirty) { - HDassert( ! (entry_ptr->is_protected) ); - HDassert( ! (entry_ptr->is_read_only) ); - HDassert( entry_ptr->ro_ref_count == 0 ); - HDassert( entry_ptr->is_dirty ); - HDassert( entry_ptr->in_slist ); - - nominated_addr = entry_ptr->addr; - if(H5AC_add_candidate((H5AC_t *)cache_ptr, nominated_addr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_add_candidate() failed(2).") - - nominated_entries_size += entry_ptr->size; - nominated_entries_count++; - } /* end if */ - - entry_ptr = entry_ptr->next; - } /* end while */ - - HDassert( nominated_entries_count == cache_ptr->slist_len ); - HDassert( nominated_entries_size == space_needed ); - } /* end if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_construct_candidate_list__clean_cache() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- - * Function: H5C_construct_candidate_list__min_clean - * - * Purpose: Construct the list of entries that should be flushed to - * get the cache back within its min clean constraints. - * - * This function is used in managing sync points, and - * shouldn't be used elsewhere. - * - * Return: Success: SUCCEED - * - * Failure: FAIL - * - * Programmer: John Mainzer - * 3/17/10 - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -herr_t -H5C_construct_candidate_list__min_clean(H5C_t * cache_ptr) -{ - size_t space_needed = 0; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - HDassert( cache_ptr != NULL ); - HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); - - /* compute the number of bytes (if any) that must be flushed to get the - * cache back within its min clean constraints. - */ - if(cache_ptr->max_cache_size > cache_ptr->index_size) { - if(((cache_ptr->max_cache_size - cache_ptr->index_size) + - cache_ptr->cLRU_list_size) >= cache_ptr->min_clean_size) - space_needed = 0; - else - space_needed = cache_ptr->min_clean_size - - ((cache_ptr->max_cache_size - cache_ptr->index_size) + - cache_ptr->cLRU_list_size); - } /* end if */ - else { - if(cache_ptr->min_clean_size <= cache_ptr->cLRU_list_size) - space_needed = 0; - else - space_needed = cache_ptr->min_clean_size - - cache_ptr->cLRU_list_size; - } /* end else */ - - if(space_needed > 0) { /* we have work to do */ - H5C_cache_entry_t *entry_ptr; - int nominated_entries_count = 0; - size_t nominated_entries_size = 0; - - HDassert( cache_ptr->slist_len > 0 ); - - /* Scan the dirty LRU list from tail forward and nominate sufficient - * entries to free up the necessary space. - */ - entry_ptr = cache_ptr->dLRU_tail_ptr; - while((nominated_entries_size < space_needed) && - (nominated_entries_count < cache_ptr->slist_len) && - (entry_ptr != NULL) && - (!entry_ptr->flush_me_last)) { - haddr_t nominated_addr; - - HDassert( ! (entry_ptr->is_protected) ); - HDassert( ! (entry_ptr->is_read_only) ); - HDassert( entry_ptr->ro_ref_count == 0 ); - HDassert( entry_ptr->is_dirty ); - HDassert( entry_ptr->in_slist ); - - nominated_addr = entry_ptr->addr; - if(H5AC_add_candidate((H5AC_t *)cache_ptr, nominated_addr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_add_candidate() failed.") - - nominated_entries_size += entry_ptr->size; - nominated_entries_count++; - entry_ptr = entry_ptr->aux_prev; - } /* end while */ - HDassert( nominated_entries_count <= cache_ptr->slist_len ); - HDassert( nominated_entries_size >= space_needed ); - } /* end if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_construct_candidate_list__min_clean() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- * Function: H5C_create * * Purpose: Allocate, initialize, and return the address of a new @@ -1883,7 +967,7 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, if(entry_ptr->is_pinned) HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "Target entry is pinned.") - /* If we get this far, call H5C_flush_single_entry() with the + /* If we get this far, call H5C__flush_single_entry() with the * H5C__FLUSH_INVALIDATE_FLAG and the H5C__FLUSH_CLEAR_ONLY_FLAG. * This will clear the entry, and then delete it from the cache. */ @@ -1895,8 +979,11 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, entry_was_dirty = entry_ptr->is_dirty; entry_size = entry_ptr->size; #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + + /* Delete the entry from the skip list on destroy */ + flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; - if(H5C_flush_single_entry(f, dxpl_id, entry_ptr->addr, flush_flags, TRUE, NULL, NULL) < 0) + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flush_flags, NULL, NULL) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTEXPUNGE, FAIL, "H5C_flush_single_entry() failed.") #if H5C_DO_SANITY_CHECKS @@ -1967,7 +1054,6 @@ herr_t H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags) { H5C_t * cache_ptr = f->shared->cache; - herr_t status; herr_t ret_value = SUCCEED; hbool_t destroy; hbool_t flushed_entries_last_pass; @@ -2024,17 +1110,8 @@ H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags) cache_ptr->flush_in_progress = TRUE; if ( destroy ) { - - status = H5C_flush_invalidate_cache(f, dxpl_id, flags); - - if ( status < 0 ) { - - /* This shouldn't happen -- if it does, we are toast so - * just scream and die. - */ - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "flush invalidate failed.") - } + if(H5C_flush_invalidate_cache(f, dxpl_id, flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "flush invalidate failed.") } else { /* When we are only flushing marked entries, the slist will usually * still contain entries when we have flushed everything we should. @@ -2047,7 +1124,7 @@ H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags) /* set the cache_ptr->slist_change_in_pre_serialize and * cache_ptr->slist_change_in_serialize to false. * - * These flags are set to TRUE by H5C_flush_single_entry if the + * These flags are set to TRUE by H5C__flush_single_entry if the * slist is modified by a pre_serialize or serialize call respectively. * H5C_flush_cache uses these flags to detect any modifications * to the slist that might corrupt the scan of the slist -- and @@ -2221,21 +1298,8 @@ H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags) flushed_entries_size += (int64_t)entry_ptr->size; entry_size_change = 0; #endif /* H5C_DO_SANITY_CHECKS */ - status = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - flags, - FALSE, - entry_size_change_ptr, - NULL); - if ( status < 0 ) { - - /* This shouldn't happen -- if it does, - * we are toast so just scream and die. - */ - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "dirty pinned entry flush failed.") - } /* end if */ + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flags, entry_size_change_ptr, NULL) < 0 ) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty pinned entry flush failed.") #if H5C_DO_SANITY_CHECKS /* it is possible that the entry size changed @@ -2282,21 +1346,8 @@ H5C_flush_cache(H5F_t *f, hid_t dxpl_id, unsigned flags) flushed_entries_size += (int64_t)entry_ptr->size; entry_size_change = 0; #endif /* H5C_DO_SANITY_CHECKS */ - status = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - flags, - FALSE, - entry_size_change_ptr, - NULL); - if ( status < 0 ) { - - /* This shouldn't happen -- if it does, - * we are toast so just scream and die. - */ - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "Can't flush entry.") - } + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flags, entry_size_change_ptr, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush entry.") #if H5C_DO_SANITY_CHECKS /* it is possible that the entry size changed @@ -2885,6 +1936,34 @@ done: /*------------------------------------------------------------------------- + * Function: H5C_get_aux_ptr + * + * Purpose: Get the aux_ptr field from the cache. + * + * This field will either be NULL (when accessing a file serially) + * or contains a pointer to the auxiliary info for parallel I/O. + * + * Return: NULL/non-NULL (can't fail) + * + * Programmer: Quincey Koziol + * 6/29/15 + * + *------------------------------------------------------------------------- + */ +void * +H5C_get_aux_ptr(const H5C_t *cache_ptr) +{ + FUNC_ENTER_NOAPI_NOERR + + /* Check arguments */ + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + + FUNC_LEAVE_NOAPI(cache_ptr->aux_ptr) +} /* H5C_get_aux_ptr() */ + + +/*------------------------------------------------------------------------- * Function: H5C_get_trace_file_ptr * * Purpose: Get the trace_file_ptr field from the cache. @@ -3279,13 +2358,13 @@ H5C_insert_entry(H5F_t * f, /* Make sure the size of the collective entries in the cache remain in check */ if(H5P_USER_TRUE == f->coll_md_read) { - if(cache_ptr->max_cache_size*0.8 < cache_ptr->coll_list_size) { + if(cache_ptr->max_cache_size*80 < cache_ptr->coll_list_size*100) { if(H5C_clear_coll_entries(cache_ptr, 1) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed.") } } else { - if(cache_ptr->max_cache_size*0.4 < cache_ptr->coll_list_size) { + if(cache_ptr->max_cache_size*40 < cache_ptr->coll_list_size*100) { if(H5C_clear_coll_entries(cache_ptr, 1) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed.") } @@ -3313,325 +2392,6 @@ done: /*------------------------------------------------------------------------- - * - * Function: H5C_mark_entries_as_clean - * - * Purpose: When the H5C code is used to implement the metadata caches - * in PHDF5, only the cache with MPI_rank 0 is allowed to - * actually write entries to disk -- all other caches must - * retain dirty entries until they are advised that the - * entries are clean. - * - * This function exists to allow the H5C code to receive these - * notifications. - * - * The function receives a list of entry base addresses - * which must refer to dirty entries in the cache. If any - * of the entries are either clean or don't exist, the - * function flags an error. - * - * The function scans the list of entries and flushes all - * those that are currently unprotected with the - * H5C__FLUSH_CLEAR_ONLY_FLAG. Those that are currently - * protected are flagged for clearing when they are - * unprotected. - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: John Mainzer - * 7/5/05 - * - * Changes: Tidied up code, removeing some old commented out - * code that had been left in pending success of the - * new version. - * - * Note that unlike H5C_apply_candidate_list(), - * H5C_mark_entries_as_clean() makes all its calls to - * H6C_flush_single_entry() with the - * H5C__FLUSH_CLEAR_ONLY_FLAG set. As a result, - * the pre_serialize() and serialize calls are not made. - * - * This then implies that (assuming such actions were - * permitted in the parallel case) no loads, dirties, - * resizes, or removals of other entries can occur as - * a side effect of the flush. Hence, there is no need - * for the checks for entry removal / status change - * that I ported to H5C_apply_candidate_list(). - * - * However, if (in addition to allowing such operations - * in the parallel case), we allow such operations outside - * of the pre_serialize / serialize routines, this may - * cease to be the case -- requiring a review of this - * function. - * - *------------------------------------------------------------------------- - */ -#ifdef H5_HAVE_PARALLEL -herr_t -H5C_mark_entries_as_clean(H5F_t * f, - hid_t dxpl_id, - int32_t ce_array_len, - haddr_t * ce_array_ptr) -{ - H5C_t * cache_ptr; - int entries_cleared; - int entries_examined; - int i; - int initial_list_len; - haddr_t addr; -#if H5C_DO_SANITY_CHECKS - int pinned_entries_marked = 0; - int protected_entries_marked = 0; - int other_entries_marked = 0; - haddr_t last_addr; -#endif /* H5C_DO_SANITY_CHECKS */ - H5C_cache_entry_t * clear_ptr = NULL; - H5C_cache_entry_t * entry_ptr = NULL; - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI(FAIL) - - HDassert( f ); - HDassert( f->shared ); - cache_ptr = f->shared->cache; - HDassert( cache_ptr ); - HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); - - HDassert( ce_array_len > 0 ); - HDassert( ce_array_ptr != NULL ); - -#if H5C_DO_EXTREME_SANITY_CHECKS - if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_lru_list(cache_ptr) < 0 ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "an extreme sanity check failed on entry.\n"); - } -#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ - - for ( i = 0; i < ce_array_len; i++ ) - { - addr = ce_array_ptr[i]; - -#if H5C_DO_SANITY_CHECKS - if ( i == 0 ) { - - last_addr = addr; - - } else { - - if ( last_addr == addr ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "Duplicate entry in cleaned list.\n"); - - } else if ( last_addr > addr ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "cleaned list not sorted.\n"); - } - } - -#if H5C_DO_EXTREME_SANITY_CHECKS - if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_lru_list(cache_ptr) < 0 ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "an extreme sanity check failed in for loop.\n"); - } -#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ -#endif /* H5C_DO_SANITY_CHECKS */ - - HDassert( H5F_addr_defined(addr) ); - - H5C__SEARCH_INDEX(cache_ptr, addr, entry_ptr, FAIL) - - if ( entry_ptr == NULL ) { -#if H5C_DO_SANITY_CHECKS - HDfprintf(stdout, - "H5C_mark_entries_as_clean: entry[%d] = %ld not in cache.\n", - (int)i, - (long)addr); -#endif /* H5C_DO_SANITY_CHECKS */ - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "Listed entry not in cache?!?!?.") - - } else if ( ! entry_ptr->is_dirty ) { - -#if H5C_DO_SANITY_CHECKS - HDfprintf(stdout, - "H5C_mark_entries_as_clean: entry %ld is not dirty!?!\n", - (long)addr); -#endif /* H5C_DO_SANITY_CHECKS */ - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "Listed entry not dirty?!?!?.") - - } else { - - /* Mark the entry to be cleared on unprotect. We will - * scan the LRU list shortly, and clear all those entries - * not currently protected. - */ - if(TRUE == entry_ptr->coll_access) { - entry_ptr->coll_access = FALSE; - H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) - } - - entry_ptr->clear_on_unprotect = TRUE; -#if H5C_DO_SANITY_CHECKS - if ( entry_ptr->is_protected ) { - - protected_entries_marked++; - - } else if ( entry_ptr->is_pinned ) { - - pinned_entries_marked++; - - } else { - - other_entries_marked++; - } -#endif /* H5C_DO_SANITY_CHECKS */ - } - } - - /* Scan through the LRU list from back to front, and flush the - * entries whose clear_on_unprotect flags are set. Observe that - * any protected entries will not be on the LRU, and therefore - * will not be flushed at this time. - * - * Note that unlike H5C_apply_candidate_list(), - * H5C_mark_entries_as_clean() makes all its calls to - * H6C_flush_single_entry() with the H5C__FLUSH_CLEAR_ONLY_FLAG - * set. As a result, the pre_serialize() and serialize calls are - * not made. - * - * This then implies that (assuming such actions were - * permitted in the parallel case) no loads, dirties, - * resizes, or removals of other entries can occur as - * a side effect of the flush. Hence, there is no need - * for the checks for entry removal / status change - * that I ported to H5C_apply_candidate_list(). - * - * However, if (in addition to allowing such operations - * in the parallel case), we allow such operations outside - * of the pre_serialize / serialize routines, this may - * cease to be the case -- requiring a review of this - * point. - * JRM -- 4/7/15 - */ - - entries_cleared = 0; - entries_examined = 0; - initial_list_len = cache_ptr->LRU_list_len; - entry_ptr = cache_ptr->LRU_tail_ptr; - - while ( ( entry_ptr != NULL ) && - ( entries_examined <= initial_list_len ) && - ( entries_cleared < ce_array_len ) ) - { - if ( entry_ptr->clear_on_unprotect ) { - - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->prev; - entries_cleared++; - - if ( H5C_flush_single_entry(f, - dxpl_id, - clear_ptr->addr, - H5C__FLUSH_CLEAR_ONLY_FLAG, - TRUE, - NULL, - NULL) < 0 ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") - } - } else { - - entry_ptr = entry_ptr->prev; - } - entries_examined++; - } - -#if H5C_DO_SANITY_CHECKS - HDassert( entries_cleared == other_entries_marked ); -#endif /* H5C_DO_SANITY_CHECKS */ - - /* It is also possible that some of the cleared entries are on the - * pinned list. Must scan that also. - */ - - entry_ptr = cache_ptr->pel_head_ptr; - - while ( entry_ptr != NULL ) - { - if ( entry_ptr->clear_on_unprotect ) { - - entry_ptr->clear_on_unprotect = FALSE; - clear_ptr = entry_ptr; - entry_ptr = entry_ptr->next; - entries_cleared++; - - if ( H5C_flush_single_entry(f, - dxpl_id, - clear_ptr->addr, - H5C__FLUSH_CLEAR_ONLY_FLAG, - TRUE, - NULL, - NULL) < 0 ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") - } - } else { - - entry_ptr = entry_ptr->next; - } - } - -#if H5C_DO_SANITY_CHECKS - HDassert( entries_cleared == pinned_entries_marked + other_entries_marked ); - HDassert( entries_cleared + protected_entries_marked == ce_array_len ); -#endif /* H5C_DO_SANITY_CHECKS */ - - HDassert( ( entries_cleared == ce_array_len ) || - ( (ce_array_len - entries_cleared) <= cache_ptr->pl_len ) ); - -#if H5C_DO_SANITY_CHECKS - i = 0; - entry_ptr = cache_ptr->pl_head_ptr; - while ( entry_ptr != NULL ) - { - if ( entry_ptr->clear_on_unprotect ) { - - i++; - } - entry_ptr = entry_ptr->next; - } - HDassert( (entries_cleared + i) == ce_array_len ); -#endif /* H5C_DO_SANITY_CHECKS */ - -done: - -#if H5C_DO_EXTREME_SANITY_CHECKS - if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) || - ( H5C_validate_lru_list(cache_ptr) < 0 ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "an extreme sanity check failed on exit.\n"); - } -#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ - - FUNC_LEAVE_NOAPI(ret_value) - -} /* H5C_mark_entries_as_clean() */ -#endif /* H5_HAVE_PARALLEL */ - - -/*------------------------------------------------------------------------- * Function: H5C_mark_entry_dirty * * Purpose: Mark a pinned or protected entry as dirty. The target entry @@ -4217,7 +2977,6 @@ H5C_protect(H5F_t * f, size_t empty_space; void * thing; H5C_cache_entry_t * entry_ptr; - haddr_t tag = HADDR_UNDEF; H5P_genplist_t * dxpl; /* dataset transfer property list */ void * ret_value; /* Return value */ @@ -4345,6 +3104,9 @@ H5C_protect(H5F_t * f, #endif /* H5_HAVE_PARALLEL */ #if H5C_DO_TAGGING_SANITY_CHECKS +{ + haddr_t tag = HADDR_UNDEF; + /* The entry is already in the cache, but make sure that the tag value being passed in via dxpl is still legal. This will ensure that had the entry NOT been in the cache, tagging was still set up correctly @@ -4363,6 +3125,7 @@ H5C_protect(H5F_t * f, HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "tag verification failed"); } /* end if */ +} #endif hit = TRUE; @@ -4685,14 +3448,14 @@ H5C_protect(H5F_t * f, if( TRUE == coll_access) { if(H5P_USER_TRUE == f->coll_md_read) { //fprintf(stderr, "COLL entries size = %zu, MAX = %zu\n", cache_ptr->coll_list_size, cache_ptr->max_cache_size); - if(cache_ptr->max_cache_size*0.8 < cache_ptr->coll_list_size) { + if(cache_ptr->max_cache_size*80 < cache_ptr->coll_list_size*100) { //fprintf(stderr, "COLL entries at 80.. CLEARING\n"); if(H5C_clear_coll_entries(cache_ptr, 1) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "H5C_clear_coll_entries() failed.") } } else { - if(cache_ptr->max_cache_size*0.4 < cache_ptr->coll_list_size) { + if(cache_ptr->max_cache_size*40 < cache_ptr->coll_list_size*100) { //fprintf(stderr, "COLL entries at 40.. CLEARING\n"); if(H5C_clear_coll_entries(cache_ptr, 1) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "H5C_clear_coll_entries() failed.") @@ -6134,18 +4897,6 @@ done: * argument must be the value returned by that call to * H5C_protect(). * - * The primary_dxpl_id and secondary_dxpl_id parameters - * specify the dxpl_ids used on the first write occasioned - * by the unprotect (primary_dxpl_id), and on all subsequent - * writes (secondary_dxpl_id). Since an uprotect cannot - * occasion a write at present, all this is moot for now. - * However, things change, and in any case, - * H5C_flush_single_entry() needs primary_dxpl_id and - * secondary_dxpl_id in its parameter list. - * - * The function can't cause a read either, so the dxpl_id - * parameters are moot in this case as well. - * * Return: Non-negative on success/Negative on failure * * If the deleted flag is TRUE, simply remove the target entry @@ -6162,7 +4913,6 @@ done: herr_t H5C_unprotect(H5F_t * f, hid_t dxpl_id, - const H5C_class_t * type, haddr_t addr, void * thing, unsigned int flags) @@ -6200,7 +4950,6 @@ H5C_unprotect(H5F_t * f, HDassert( cache_ptr ); HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); - HDassert( type ); HDassert( H5F_addr_defined(addr) ); HDassert( thing ); HDassert( ! ( pin_entry && unpin_entry ) ); @@ -6211,7 +4960,6 @@ H5C_unprotect(H5F_t * f, entry_ptr = (H5C_cache_entry_t *)thing; HDassert( entry_ptr->addr == addr ); - HDassert( entry_ptr->type == type ); /* also set the dirtied variable if the dirtied field is set in * the entry. @@ -6360,19 +5108,11 @@ H5C_unprotect(H5F_t * f, HDassert ( ! (entry_ptr->is_pinned ) ); /* verify that the target entry is in the cache. */ - H5C__SEARCH_INDEX(cache_ptr, addr, test_entry_ptr, FAIL) - - if ( test_entry_ptr == NULL ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ - "entry not in hash table?!?.") - } - else if ( test_entry_ptr != entry_ptr ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ - "hash table contains multiple entries for addr?!?.") - } + if(test_entry_ptr == NULL) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "entry not in hash table?!?.") + else if(test_entry_ptr != entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "hash table contains multiple entries for addr?!?.") /* Set the 'free file space' flag for the flush, if needed */ if(free_file_space) @@ -6382,16 +5122,11 @@ H5C_unprotect(H5F_t * f, if(take_ownership) flush_flags |= H5C__TAKE_OWNERSHIP_FLAG; - if ( H5C_flush_single_entry(f, - dxpl_id, - addr, - flush_flags, - TRUE, - NULL, - NULL) < 0 ) { + /* Delete the entry from the skip list on destroy */ + flush_flags |= H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG; - HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "Can't flush.") - } + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, flush_flags, NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "Can't flush entry") #if H5C_DO_SANITY_CHECKS if ( ( take_ownership ) && ( ! was_clean ) ) @@ -6410,30 +5145,14 @@ H5C_unprotect(H5F_t * f, else if ( clear_entry ) { /* verify that the target entry is in the cache. */ - H5C__SEARCH_INDEX(cache_ptr, addr, test_entry_ptr, FAIL) + if(test_entry_ptr == NULL) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "entry not in hash table?!?.") + else if(test_entry_ptr != entry_ptr) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "hash table contains multiple entries for addr?!?.") - if ( test_entry_ptr == NULL ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ - "entry not in hash table?!?.") - } - else if ( test_entry_ptr != entry_ptr ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, \ - "hash table contains multiple entries for addr?!?.") - } - - if ( H5C_flush_single_entry(f, - dxpl_id, - addr, - H5C__FLUSH_CLEAR_ONLY_FLAG, - TRUE, - NULL, - NULL) < 0 ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "Can't clear.") - } + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTUNPROTECT, FAIL, "Can't clear entry") } #endif /* H5_HAVE_PARALLEL */ } @@ -7698,7 +6417,6 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, hbool_t write_permitted) { H5C_t * cache_ptr = f->shared->cache; - herr_t result; size_t eviction_size_limit; size_t bytes_evicted = 0; hbool_t prev_is_dirty = FALSE; @@ -7755,7 +6473,7 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, /* reset entries_removed_counter and * last_entry_removed_ptr prior to the call to - * H5C_flush_single_entry() so that we can spot + * H5C__flush_single_entry() so that we can spot * unexpected removals of entries from the cache, * and set the restart_scan flag if proceeding * would be likely to cause us to scan an entry @@ -7764,13 +6482,8 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, cache_ptr->entries_removed_counter = 0; cache_ptr->last_entry_removed_ptr = NULL; - result = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - H5C__NO_FLAGS_SET, - FALSE, - NULL, - NULL); + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET, NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") if ( ( cache_ptr->entries_removed_counter > 1 ) || ( cache_ptr->last_entry_removed_ptr == prev_ptr ) ) @@ -7781,19 +6494,8 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, bytes_evicted += entry_ptr->size; - result = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - H5C__FLUSH_INVALIDATE_FLAG, - TRUE, - NULL, - NULL); - } - - if ( result < 0 ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unable to flush entry") + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0 ) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") } if ( prev_ptr != NULL ) { @@ -7872,20 +6574,8 @@ H5C__autoadjust__ageout__evict_aged_out_entries(H5F_t * f, prev_ptr = entry_ptr->prev; if ( ! (entry_ptr->is_dirty) ) { - - result = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - H5C__FLUSH_INVALIDATE_FLAG, - TRUE, - NULL, - NULL); - - if ( result < 0 ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unable to flush clean entry") - } + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush clean entry") } /* just skip the entry if it is dirty, as we can't do * anything with it now since we can't write. @@ -8380,7 +7070,7 @@ done: * was largely removed from the cache data structures before * the flush proper. However, re-entrant calls to the cache * in the parallel case required a re-factoring of the - * H5C_flush_single_entry() function to keep entries fully + * H5C__flush_single_entry() function to keep entries fully * in the cache until after the pre-serialize and serialize * calls. * JRM -- 12/25/14 @@ -8393,7 +7083,6 @@ H5C_flush_invalidate_cache(const H5F_t * f, unsigned flags) { H5C_t * cache_ptr = f->shared->cache; - herr_t status; hbool_t restart_slist_scan; int32_t protected_entries = 0; int32_t i; @@ -8429,16 +7118,9 @@ H5C_flush_invalidate_cache(const H5F_t * f, cooked_flags = flags & H5C__FLUSH_CLEAR_ONLY_FLAG; /* remove ageout markers if present */ - if ( cache_ptr->epoch_markers_active > 0 ) { - - status = H5C__autoadjust__ageout__remove_all_markers(cache_ptr); - - if ( status != SUCCEED ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "error removing all epoch markers.") - } - } + if(cache_ptr->epoch_markers_active > 0) + if(H5C__autoadjust__ageout__remove_all_markers(cache_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "error removing all epoch markers.") /* The flush proceedure here is a bit strange. * @@ -8493,7 +7175,7 @@ H5C_flush_invalidate_cache(const H5F_t * f, */ #if H5C_DO_SANITY_CHECKS - /* Depending on circumstances, H5C_flush_single_entry() will + /* Depending on circumstances, H5C__flush_single_entry() will * remove dirty entries from the slist as it flushes them. * Thus for sanity checks we must make note of the initial * slist length and size before we do any flushes. @@ -8526,7 +7208,7 @@ H5C_flush_invalidate_cache(const H5F_t * f, /* set the cache_ptr->slist_change_in_pre_serialize and * cache_ptr->slist_change_in_serialize to false. * - * These flags are set to TRUE by H5C_flush_single_entry if the + * These flags are set to TRUE by H5C__flush_single_entry if the * slist is modified by a pre_serialize or serialize call * respectively. * @@ -8635,7 +7317,7 @@ H5C_flush_invalidate_cache(const H5F_t * f, /* Test to see if we are can flush the entry now. * If we can, go ahead and flush, but don't tell - * H5C_flush_single_entry() to destroy the entry + * H5C__flush_single_entry() to destroy the entry * as pinned entries can't be evicted. */ if(entry_ptr->flush_dep_height == curr_flush_dep_height ) { @@ -8653,22 +7335,8 @@ H5C_flush_invalidate_cache(const H5F_t * f, entry_size_change = 0; #endif /* H5C_DO_SANITY_CHECKS */ - status = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - H5C__NO_FLAGS_SET, - FALSE, - entry_size_change_ptr, - NULL); - if ( status < 0 ) { - - /* This shouldn't happen -- if it does, we - * are toast so just scream and die. - */ - - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "dirty pinned entry flush failed.") - } /* end if */ + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET, entry_size_change_ptr, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty pinned entry flush failed.") #if H5C_DO_SANITY_CHECKS /* entry size may have changed during the flush. @@ -8716,22 +7384,8 @@ H5C_flush_invalidate_cache(const H5F_t * f, entry_size_change = 0; #endif /* H5C_DO_SANITY_CHECKS */ - status = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG), - TRUE, - entry_size_change_ptr, - NULL); - if ( status < 0 ) { - - /* This shouldn't happen -- if it does, we - * are toast so just scream and die. - */ - - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "dirty entry flush destroy failed.") - } /* end if */ + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG), entry_size_change_ptr, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "dirty entry flush destroy failed.") #if H5C_DO_SANITY_CHECKS /* entry size may have changed during the flush. @@ -8859,23 +7513,8 @@ H5C_flush_invalidate_cache(const H5F_t * f, entry_was_dirty = entry_ptr->is_dirty; - status = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG), - TRUE, - NULL, - NULL); - - if ( status < 0 ) { - - /* This shouldn't happen -- if it does, - * we are toast so just scream and die. - */ - - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "Entry flush destroy failed.") - } + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, (cooked_flags | H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG), NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Entry flush destroy failed.") if ( entry_was_dirty ) { @@ -9012,7 +7651,7 @@ done: /*------------------------------------------------------------------------- * - * Function: H5C_flush_single_entry + * Function: H5C__flush_single_entry * * Purpose: Flush or clear (and evict if requested) the cache entry * with the specified address and type. If the type is NULL, @@ -9071,37 +7710,32 @@ done: * *------------------------------------------------------------------------- */ -static herr_t -H5C_flush_single_entry(const H5F_t * f, - hid_t dxpl_id, - haddr_t addr, - unsigned flags, - hbool_t del_entry_from_slist_on_destroy, - int64_t * entry_size_change_ptr, - H5SL_t * collective_write_list) +herr_t +H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ptr, + unsigned flags, int64_t *entry_size_change_ptr, H5SL_t *collective_write_list) { - H5C_t * cache_ptr = f->shared->cache; + H5C_t * cache_ptr; /* Cache for file */ hbool_t destroy; /* external flag */ hbool_t clear_only; /* external flag */ hbool_t free_file_space; /* external flag */ hbool_t take_ownership; /* external flag */ + hbool_t del_from_slist_on_destroy; /* external flag */ hbool_t write_entry; /* internal flag */ hbool_t destroy_entry; /* internal flag */ hbool_t was_dirty; - herr_t status; haddr_t new_addr = HADDR_UNDEF; haddr_t old_addr = HADDR_UNDEF; size_t new_len = 0; size_t new_compressed_len = 0; - H5C_cache_entry_t * entry_ptr = NULL; herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT + FUNC_ENTER_PACKAGE - HDassert( f ); - HDassert( cache_ptr ); - HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); - HDassert( H5F_addr_defined(addr) ); + HDassert(f); + cache_ptr = f->shared->cache; + HDassert(cache_ptr); + HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(entry_ptr); /* If defined, initialize *entry_size_change_ptr to 0 */ if(entry_size_change_ptr != NULL) @@ -9112,6 +7746,7 @@ H5C_flush_single_entry(const H5F_t * f, clear_only = ((flags & H5C__FLUSH_CLEAR_ONLY_FLAG) != 0); free_file_space = ((flags & H5C__FREE_FILE_SPACE_FLAG) != 0); take_ownership = ((flags & H5C__TAKE_OWNERSHIP_FLAG) != 0); + del_from_slist_on_destroy = ((flags & H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG) != 0); /* Set the flag for destroying the entry, based on the 'take ownership' * and 'destroy' flags @@ -9121,111 +7756,83 @@ H5C_flush_single_entry(const H5F_t * f, else destroy_entry = destroy; - /* attempt to find the target entry in the hash table */ - H5C__SEARCH_INDEX(cache_ptr, addr, entry_ptr, FAIL) - HDassert(FALSE == entry_ptr->coll_access); /* we will write the entry to disk if it exists, is dirty, and if the * clear only flag is not set. */ - if ( ( entry_ptr != NULL ) && ( entry_ptr->is_dirty ) && ( ! clear_only ) ) + if(entry_ptr->is_dirty && !clear_only) write_entry = TRUE; else write_entry = FALSE; /* run initial sanity checks */ #if H5C_DO_SANITY_CHECKS - if ( entry_ptr != NULL ) { - - HDassert( ! ( ( destroy ) && ( entry_ptr->is_pinned ) ) ); + HDassert( ! ( destroy && entry_ptr->is_pinned ) ); - if ( entry_ptr->in_slist ) { - - HDassert(entry_ptr->is_dirty); - - if ( ( ( entry_ptr->flush_marker ) && ( ! entry_ptr->is_dirty ) ) || - ( entry_ptr->addr != addr ) ) { - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "entry in slist failed sanity checks.") - } - } else { - - HDassert(!entry_ptr->is_dirty); - HDassert(!entry_ptr->flush_marker); - HDassert(entry_ptr->addr == addr); + if(entry_ptr->in_slist) { + HDassert(entry_ptr->is_dirty); - if ( ( entry_ptr->is_dirty ) || - ( entry_ptr->flush_marker ) || - ( entry_ptr->addr != addr ) ) { + if((entry_ptr->flush_marker) && (!entry_ptr->is_dirty)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry in slist failed sanity checks.") + } else { + HDassert(!entry_ptr->is_dirty); + HDassert(!entry_ptr->flush_marker); - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "entry failed sanity checks.") - } - } + if((entry_ptr->is_dirty) || (entry_ptr->flush_marker)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry failed sanity checks.") } #endif /* H5C_DO_SANITY_CHECKS */ - if ( ( entry_ptr != NULL ) && ( entry_ptr->is_protected ) ) - { + if(entry_ptr->is_protected) { HDassert(!entry_ptr->is_protected); /* Attempt to flush a protected entry -- scream and die. */ - HGOTO_ERROR(H5E_CACHE, H5E_PROTECT, FAIL, \ - "Attempt to flush a protected entry.") - } + HGOTO_ERROR(H5E_CACHE, H5E_PROTECT, FAIL, "Attempt to flush a protected entry.") + } /* end if */ - /* if the entry exists, set entry_ptr->flush_in_progress = TRUE - * and set entry_ptr->flush_marker = FALSE + /* set entry_ptr->flush_in_progress = TRUE and set + * entry_ptr->flush_marker = FALSE * * in the parallel case, do some sanity checking in passing. */ - if ( entry_ptr != NULL ) - { - /* we have work to do */ - HDassert(entry_ptr->type); - - was_dirty = entry_ptr->is_dirty; /* needed later for logging */ + HDassert(entry_ptr->type); - /* We will set flush_in_progress back to FALSE at the end if the - * entry still exists at that point. - */ - entry_ptr->flush_in_progress = TRUE; + was_dirty = entry_ptr->is_dirty; /* needed later for logging */ - entry_ptr->flush_marker = FALSE; + /* We will set flush_in_progress back to FALSE at the end if the + * entry still exists at that point. + */ + entry_ptr->flush_in_progress = TRUE; + entry_ptr->flush_marker = FALSE; #ifdef H5_HAVE_PARALLEL #ifndef NDEBUG - /* If MPI based VFD is used, do special parallel I/O sanity checks. - * Note that we only do these sanity checks when the clear_only flag - * is not set, and the entry to be flushed is dirty. Don't bother - * otherwise as no file I/O can result. - */ - if(!clear_only && entry_ptr->is_dirty && - H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) { - H5P_genplist_t *dxpl; /* Dataset transfer property list */ - unsigned coll_meta; /* Collective metadata write flag */ + /* If MPI based VFD is used, do special parallel I/O sanity checks. + * Note that we only do these sanity checks when the clear_only flag + * is not set, and the entry to be flushed is dirty. Don't bother + * otherwise as no file I/O can result. + */ + if(!clear_only && entry_ptr->is_dirty && H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI)) { + H5P_genplist_t *dxpl; /* Dataset transfer property list */ + unsigned coll_meta; /* Collective metadata write flag */ - /* Get the dataset transfer property list */ - if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_CACHE, H5E_BADTYPE, FAIL, "not a dataset transfer property list") + /* Get the dataset transfer property list */ + if(NULL == (dxpl = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_CACHE, H5E_BADTYPE, FAIL, "not a dataset transfer property list") - /* Get the collective metadata write property */ - if(H5P_get(dxpl, H5AC_COLLECTIVE_META_WRITE_NAME, &coll_meta) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't retrieve xfer mode") + /* Get the collective metadata write property */ + if(H5P_get(dxpl, H5AC_COLLECTIVE_META_WRITE_NAME, &coll_meta) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "can't retrieve xfer mode") - /* Sanity check collective metadata write flag */ - HDassert(coll_meta); - } /* end if */ + /* Sanity check collective metadata write flag */ + HDassert(coll_meta); + } /* end if */ #endif /* NDEBUG */ #endif /* H5_HAVE_PARALLEL */ - } - - if ( ( entry_ptr != NULL ) && ( write_entry ) ) - { - /* serialize the entry if necessary, and then write it to disk. */ + /* serialize the entry if necessary, and then write it to disk. */ + if(write_entry) { unsigned serialize_flags = H5C__SERIALIZE_NO_FLAGS_SET; /* The entry is dirty, and we are doing either a flush, @@ -9241,47 +7848,31 @@ H5C_flush_single_entry(const H5F_t * f, * are about to delete the entry from the cache (i.e. on a * flush destroy). */ - HDassert(entry_ptr->is_dirty); #if H5C_DO_SANITY_CHECKS - if ( ( cache_ptr->check_write_permitted == NULL ) && - ( !(cache_ptr->write_permitted) ) ) - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "Write when writes are always forbidden!?!?!") + if(cache_ptr->check_write_permitted && !(cache_ptr->write_permitted)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Write when writes are always forbidden!?!?!") #endif /* H5C_DO_SANITY_CHECKS */ - if ( NULL == entry_ptr->image_ptr ) - { + if(NULL == entry_ptr->image_ptr) { size_t image_size; - if ( entry_ptr->compressed ) + if(entry_ptr->compressed) image_size = entry_ptr->compressed_size; else image_size = entry_ptr->size; - HDassert(image_size > 0); - entry_ptr->image_ptr = - H5MM_malloc(image_size + H5C_IMAGE_EXTRA_SPACE); - if ( NULL == entry_ptr->image_ptr) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, \ - "memory allocation failed for on disk image buffer") - } + if(NULL == (entry_ptr->image_ptr = H5MM_malloc(image_size + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") #if H5C_DO_MEMORY_SANITY_CHECKS - - HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + image_size, - H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); - + HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + image_size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ - } /* end if */ - if ( ! (entry_ptr->image_up_to_date) ) - { + if(!(entry_ptr->image_up_to_date)) { /* reset cache_ptr->slist_changed so we can detect slist * modifications in the pre_serialize call. */ @@ -9300,28 +7891,21 @@ H5C_flush_single_entry(const H5F_t * f, &new_addr, &new_len, &new_compressed_len, &serialize_flags) < 0 ) ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unable to pre-serialize entry") - } + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to pre-serialize entry") /* set cache_ptr->slist_change_in_pre_serialize if the * slist was modified. */ - if ( cache_ptr->slist_changed ) + if(cache_ptr->slist_changed) cache_ptr->slist_change_in_pre_serialize = TRUE; /* Check for any flags set in the pre-serialize callback */ - if ( serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET ) - { + if(serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) { /* Check for unexpected flags from serialize callback */ - if ( serialize_flags & ~(H5C__SERIALIZE_RESIZED_FLAG | - H5C__SERIALIZE_MOVED_FLAG | - H5C__SERIALIZE_COMPRESSED_FLAG)) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unknown serialize flag(s)") - } + if(serialize_flags & ~(H5C__SERIALIZE_RESIZED_FLAG | + H5C__SERIALIZE_MOVED_FLAG | + H5C__SERIALIZE_COMPRESSED_FLAG)) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unknown serialize flag(s)") #ifdef H5_HAVE_PARALLEL /* In the parallel case, resizes and moves in * the serialize operation can cause problems. @@ -9349,10 +7933,8 @@ H5C_flush_single_entry(const H5F_t * f, * If that ceases to be the case, further * tests will be necessary. */ - if ( cache_ptr->aux_ptr != NULL ) - - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "resize/move in serialize occured in parallel case.") + if(cache_ptr->aux_ptr != NULL) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "resize/move in serialize occured in parallel case.") #endif /* H5_HAVE_PARALLEL */ /* Resize the buffer if required */ @@ -9363,44 +7945,28 @@ H5C_flush_single_entry(const H5F_t * f, { size_t new_image_size; - if ( entry_ptr->compressed ) + if(entry_ptr->compressed) new_image_size = new_compressed_len; else new_image_size = new_len; - HDassert(new_image_size > 0); /* Release the current image */ - if ( entry_ptr->image_ptr ) - { + if(entry_ptr->image_ptr) entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); - } /* Allocate a new image buffer */ - entry_ptr->image_ptr = - H5MM_malloc(new_image_size + H5C_IMAGE_EXTRA_SPACE); - - if ( NULL == entry_ptr->image_ptr ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, \ - "memory allocation failed for on disk image buffer") - } - + if(NULL == (entry_ptr->image_ptr = H5MM_malloc(new_image_size + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") #if H5C_DO_MEMORY_SANITY_CHECKS - - HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + new_image_size, - H5C_IMAGE_SANITY_VALUE, - H5C_IMAGE_EXTRA_SPACE); - + HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + new_image_size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ - } /* end if */ /* If required, update the entry and the cache data structures * for a resize. */ - if ( serialize_flags & H5C__SERIALIZE_RESIZED_FLAG ) { - + if(serialize_flags & H5C__SERIALIZE_RESIZED_FLAG) { H5C__UPDATE_STATS_FOR_ENTRY_SIZE_CHANGE(cache_ptr, \ entry_ptr, new_len) @@ -9439,8 +8005,7 @@ H5C_flush_single_entry(const H5F_t * f, /* If required, udate the entry and the cache data structures * for a move */ - if(serialize_flags & H5C__SERIALIZE_MOVED_FLAG) - { + if(serialize_flags & H5C__SERIALIZE_MOVED_FLAG) { #if H5C_DO_SANITY_CHECKS int64_t saved_slist_len_increase; int64_t saved_slist_size_increase; @@ -9448,8 +8013,7 @@ H5C_flush_single_entry(const H5F_t * f, H5C__UPDATE_STATS_FOR_MOVE(cache_ptr, entry_ptr) - if ( entry_ptr->addr == old_addr ) - { + if(entry_ptr->addr == old_addr) { /* we must update cache data structures for the * change in address. */ @@ -9483,12 +8047,10 @@ H5C_flush_single_entry(const H5F_t * f, #endif /* H5C_DO_SANITY_CHECKS */ } else /* move is alread done for us -- just do sanity checks */ - { HDassert(entry_ptr->addr == new_addr); - } } /* end if */ - if ( serialize_flags & H5C__SERIALIZE_COMPRESSED_FLAG ) { + if(serialize_flags & H5C__SERIALIZE_COMPRESSED_FLAG) { /* just save the new compressed entry size in * entry_ptr->compressed_size. We don't need to * do more, as compressed size is only used for I/O. @@ -9502,7 +8064,7 @@ H5C_flush_single_entry(const H5F_t * f, { size_t image_len; - if ( entry_ptr->compressed ) + if(entry_ptr->compressed) image_len = entry_ptr->compressed_size; else image_len = entry_ptr->size; @@ -9513,27 +8075,19 @@ H5C_flush_single_entry(const H5F_t * f, cache_ptr->slist_changed = FALSE; - if ( entry_ptr->type->serialize(f, entry_ptr->image_ptr, - image_len, - (void *)entry_ptr) < 0 ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unable to serialize entry") - } + if(entry_ptr->type->serialize(f, entry_ptr->image_ptr, + image_len, (void *)entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to serialize entry") /* set cache_ptr->slist_change_in_serialize if the * slist was modified. */ - if ( cache_ptr->slist_changed ) + if(cache_ptr->slist_changed) cache_ptr->slist_change_in_pre_serialize = TRUE; #if H5C_DO_MEMORY_SANITY_CHECKS - - HDassert(0 == HDmemcmp(((uint8_t *)entry_ptr->image_ptr) + - image_len, - H5C_IMAGE_SANITY_VALUE, - H5C_IMAGE_EXTRA_SPACE)); - + HDassert(0 == HDmemcmp(((uint8_t *)entry_ptr->image_ptr) + image_len, + H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE)); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ entry_ptr->image_up_to_date = TRUE; @@ -9556,7 +8110,7 @@ H5C_flush_single_entry(const H5F_t * f, */ size_t image_size; - if ( entry_ptr->compressed ) + if(entry_ptr->compressed) image_size = entry_ptr->compressed_size; else image_size = entry_ptr->size; @@ -9580,29 +8134,18 @@ H5C_flush_single_entry(const H5F_t * f, } /* end if */ else #endif /* H5_HAVE_PARALLEL */ - if ( ( H5F_block_write(f, entry_ptr->type->mem_type, - entry_ptr->addr, - image_size, dxpl_id, - entry_ptr->image_ptr) < 0 ) ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "Can't write image to file.") - } + if(H5F_block_write(f, entry_ptr->type->mem_type, entry_ptr->addr, + image_size, dxpl_id, entry_ptr->image_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't write image to file.") } /* if the entry has a notify callback, notify it that we have * just flushed the entry. */ - - if ( ( entry_ptr->type->notify ) && - ( (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_FLUSH, - entry_ptr) < 0 ) ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, \ - "can't notify client of entry flush") - } - } /* if ( ( entry_ptr != NULL ) && ( write_entry ) ) */ - + if(entry_ptr->type->notify && + (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_AFTER_FLUSH, entry_ptr) < 0 ) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client of entry flush") + } /* if ( write_entry ) */ /* At this point, all pre-serialize and serialize calls have been * made if it was appropriate to make them. Similarly, the entry @@ -9612,294 +8155,214 @@ H5C_flush_single_entry(const H5F_t * f, * flush. */ - if ( entry_ptr != NULL ) - { - /* start by updating the statistics */ - - if ( clear_only ) { - - /* only log a clear if the entry was dirty */ - if ( was_dirty ) { - - H5C__UPDATE_STATS_FOR_CLEAR(cache_ptr, entry_ptr) - } - } else if ( write_entry ) { - - HDassert( was_dirty ); - - /* only log a flush if we actually wrote to disk */ - H5C__UPDATE_STATS_FOR_FLUSH(cache_ptr, entry_ptr) - - } - - if ( destroy ) - { - if ( take_ownership ) - { - HDassert(!destroy_entry); - } - else - { - HDassert(destroy_entry); - } - - H5C__UPDATE_STATS_FOR_EVICTION(cache_ptr, entry_ptr, \ - take_ownership) - } - - /* If the entry's type has a 'notify' callback and the entry is about - * to be removed from the cache, send a 'before eviction' notice while - * the entry is still fully integrated in the cache. - */ - if ( destroy ) { - - if ( ( entry_ptr->type->notify ) && - ( (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_BEFORE_EVICT, - entry_ptr) < 0 ) ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, \ - "can't notify client about entry to evict") - } + /* start by updating the statistics */ + if(clear_only) { + /* only log a clear if the entry was dirty */ + if(was_dirty) { + H5C__UPDATE_STATS_FOR_CLEAR(cache_ptr, entry_ptr) } /* end if */ + } else if(write_entry) { + HDassert(was_dirty); + /* only log a flush if we actually wrote to disk */ + H5C__UPDATE_STATS_FOR_FLUSH(cache_ptr, entry_ptr) + } - /* Update the cache internal data structures. */ - if ( destroy ) - { - /* Update the cache internal data structures as appropriate - * for a destroy. Specifically: - * - * 1) Delete it from the index - * - * 2) Delete it from the skip list if requested. - * - * 3) Update the replacement policy for eviction - * - * Finally, if the destroy_entry flag is set, discard the - * entry. - */ - - H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) - - if ( ( entry_ptr->in_slist ) && - ( del_entry_from_slist_on_destroy ) ) { - - H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr) - } - - H5C__UPDATE_RP_FOR_EVICTION(cache_ptr, entry_ptr, FAIL) - } + if(destroy) { + if(take_ownership) + HDassert(!destroy_entry); else - { - HDassert(clear_only || write_entry); - HDassert(entry_ptr->is_dirty); - HDassert(entry_ptr->in_slist); + HDassert(destroy_entry); - /* We are either doing a flush or a clear. - * - * A clear and a flush are the same from the point of - * view of the replacement policy and the slist. - * Hence no differentiation between them. - * - * JRM -- 7/7/07 - */ + H5C__UPDATE_STATS_FOR_EVICTION(cache_ptr, entry_ptr, take_ownership) + } + + /* If the entry's type has a 'notify' callback and the entry is about + * to be removed from the cache, send a 'before eviction' notice while + * the entry is still fully integrated in the cache. + */ + if(destroy) + if(entry_ptr->type->notify && (entry_ptr->type->notify)(H5C_NOTIFY_ACTION_BEFORE_EVICT, entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTNOTIFY, FAIL, "can't notify client about entry to evict") + + /* Update the cache internal data structures. */ + if(destroy) { + /* Update the cache internal data structures as appropriate + * for a destroy. Specifically: + * + * 1) Delete it from the index + * + * 2) Delete it from the skip list if requested. + * + * 3) Update the replacement policy for eviction + * + * Finally, if the destroy_entry flag is set, discard the + * entry. + */ - H5C__UPDATE_RP_FOR_FLUSH(cache_ptr, entry_ptr, FAIL) + H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr) + if(entry_ptr->in_slist && del_from_slist_on_destroy) H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr) + H5C__UPDATE_RP_FOR_EVICTION(cache_ptr, entry_ptr, FAIL) + } + else { + HDassert(clear_only || write_entry); + HDassert(entry_ptr->is_dirty); + HDassert(entry_ptr->in_slist); - /* mark the entry as clean and update the index for - * entry clean. Also, call the clear callback - * if defined. - */ - - entry_ptr->is_dirty = FALSE; + /* We are either doing a flush or a clear. + * + * A clear and a flush are the same from the point of + * view of the replacement policy and the slist. + * Hence no differentiation between them. + * + * JRM -- 7/7/07 + */ - H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN(cache_ptr, entry_ptr); + H5C__UPDATE_RP_FOR_FLUSH(cache_ptr, entry_ptr, FAIL) - if ( ( entry_ptr->type->clear != NULL ) && - ( (entry_ptr->type->clear)(f, (void *)entry_ptr, FALSE) ) ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unable to clear entry") - } - } + H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr) - /* reset the flush_in progress flag */ + /* mark the entry as clean and update the index for + * entry clean. Also, call the clear callback + * if defined. + */ + entry_ptr->is_dirty = FALSE; - entry_ptr->flush_in_progress = FALSE; + H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN(cache_ptr, entry_ptr); - } /* end if */ + if(entry_ptr->type->clear && (entry_ptr->type->clear)(f, (void *)entry_ptr, FALSE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear entry") + } + /* reset the flush_in progress flag */ + entry_ptr->flush_in_progress = FALSE; /* Internal cache data structures should now be up to date, and * consistant with the status of the entry. * * Now discard the entry if appropriate. */ - if ( entry_ptr != NULL ) - { - if ( destroy ) - { - /* start by freeing the buffer for the on disk image */ - if(entry_ptr->image_ptr != NULL) - entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); - - /* Check whether we should free the space in the file that - * the entry occupies - */ - if ( free_file_space ) - { + if(destroy) { + /* start by freeing the buffer for the on disk image */ + if(entry_ptr->image_ptr != NULL) + entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); - size_t fsf_size; + /* Check whether we should free the space in the file that + * the entry occupies + */ + if(free_file_space) { + size_t fsf_size; - /* Sanity checks */ - HDassert(H5F_addr_defined(entry_ptr->addr)); - HDassert(!H5F_IS_TMP_ADDR(f, entry_ptr->addr)); + /* Sanity checks */ + HDassert(H5F_addr_defined(entry_ptr->addr)); + HDassert(!H5F_IS_TMP_ADDR(f, entry_ptr->addr)); #ifndef NDEBUG { - hbool_t curr_compressed = FALSE; - size_t curr_len; - size_t curr_compressed_len = 0; - - /* Get the actual image size for the thing again */ - entry_ptr->type->image_len((void *)entry_ptr, &curr_len, &curr_compressed, &curr_compressed_len); - HDassert(curr_len == entry_ptr->size); - HDassert(curr_compressed == entry_ptr->compressed); - HDassert(curr_compressed_len == entry_ptr->compressed_size); + hbool_t curr_compressed = FALSE; + size_t curr_len; + size_t curr_compressed_len = 0; + + /* Get the actual image size for the thing again */ + entry_ptr->type->image_len((void *)entry_ptr, &curr_len, &curr_compressed, &curr_compressed_len); + HDassert(curr_len == entry_ptr->size); + HDassert(curr_compressed == entry_ptr->compressed); + HDassert(curr_compressed_len == entry_ptr->compressed_size); } #endif /* NDEBUG */ - /* if the file space free size callback is defined, use - * it to get the size of the block of file space to free. - * Otherwise use entry_ptr->compressed_size if - * entry_ptr->compressed == TRUE, and entry_ptr->size - * if entry_ptr->compressed == FALSE. - */ - if ( entry_ptr->type->fsf_size ) - { - if ( (entry_ptr->type->fsf_size)((void *)entry_ptr, - &fsf_size) < 0 ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, \ - "unable to get file space free size") - } - } - else if ( entry_ptr->compressed ) /* use compressed size */ - { - fsf_size = entry_ptr->compressed_size; - } - else /* no file space free size callback -- use entry size */ - { - fsf_size = entry_ptr->size; - } - - /* Release the space on disk */ - if ( H5MF_xfree(f, entry_ptr->type->mem_type, dxpl_id, - entry_ptr->addr, (hsize_t)fsf_size) < 0) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, \ - "unable to free file space for cache entry") - } - } /* end if ( free_file_space ) */ - - - /* Reset the pointer to the cache the entry is within. -QAK */ - entry_ptr->cache_ptr = NULL; - - /* increment entries_removed_counter and set - * last_entry_removed_ptr. As we are likely abuut to - * free the entry, recall that last_entry_removed_ptr - * must NEVER be dereferenced. - * - * Recall that these fields are maintained to allow functions - * that perform scans of lists of entries to detect the - * unexpected removal of entries (via expunge, eviction, - * or take ownership at present), so that they can re-start - * their scans if necessary. + /* if the file space free size callback is defined, use + * it to get the size of the block of file space to free. + * Otherwise use entry_ptr->compressed_size if + * entry_ptr->compressed == TRUE, and entry_ptr->size + * if entry_ptr->compressed == FALSE. */ - cache_ptr->last_entry_removed_ptr++; - cache_ptr->last_entry_removed_ptr = entry_ptr; - - /* Check for actually destroying the entry in memory */ - /* (As opposed to taking ownership of it) */ - if ( destroy_entry ) - { - /* if the entry is dirty and it has a clear callback, - * call this callback now. Since this callback exists, - * it follows tht the client maintains its own dirty bits, - * which must be cleared before the entry is freed to avoid - * sanity check failures. Also clear the dirty flag for - * the same reason. - */ - - if ( entry_ptr->is_dirty ) { - - entry_ptr->is_dirty = FALSE; - - if ( ( entry_ptr->type->clear != NULL ) && - ( (entry_ptr->type->clear)(f, (void *)entry_ptr, TRUE) ) ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unable to clear entry") - } - } - - /* we are about to discard the in core representation -- - * set the magic field to bad magic so we can detect a - * freed entry if we see one. - */ - entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; - - - /* verify that the image has been freed */ - HDassert( entry_ptr->image_ptr == NULL ); - - if ( entry_ptr->type->free_icr((void *)entry_ptr) < 0 ) - { - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "free_icr callback failed.") - } - } - else - { - HDassert(take_ownership); + if(entry_ptr->type->fsf_size) { + if((entry_ptr->type->fsf_size)((void *)entry_ptr, &fsf_size) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to get file space free size") + } /* end if */ + else if(entry_ptr->compressed) /* use compressed size */ + fsf_size = entry_ptr->compressed_size; + else /* no file space free size callback -- use entry size */ + fsf_size = entry_ptr->size; + + /* Release the space on disk */ + if(H5MF_xfree(f, entry_ptr->type->mem_type, dxpl_id, entry_ptr->addr, (hsize_t)fsf_size) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to free file space for cache entry") + } /* end if ( free_file_space ) */ + + /* Reset the pointer to the cache the entry is within. -QAK */ + entry_ptr->cache_ptr = NULL; + + /* increment entries_removed_counter and set + * last_entry_removed_ptr. As we are likely abuut to + * free the entry, recall that last_entry_removed_ptr + * must NEVER be dereferenced. + * + * Recall that these fields are maintained to allow functions + * that perform scans of lists of entries to detect the + * unexpected removal of entries (via expunge, eviction, + * or take ownership at present), so that they can re-start + * their scans if necessary. + */ + cache_ptr->last_entry_removed_ptr++; + cache_ptr->last_entry_removed_ptr = entry_ptr; + + /* Check for actually destroying the entry in memory */ + /* (As opposed to taking ownership of it) */ + if(destroy_entry) { + /* if the entry is dirty and it has a clear callback, + * call this callback now. Since this callback exists, + * it follows tht the client maintains its own dirty bits, + * which must be cleared before the entry is freed to avoid + * sanity check failures. Also clear the dirty flag for + * the same reason. + */ + if(entry_ptr->is_dirty) { + entry_ptr->is_dirty = FALSE; - /* client is taking ownership of the entry. - * set bad magic here too so the cache will choke - * unless the entry is re-inserted properly - */ - entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; + if(entry_ptr->type->clear && (entry_ptr->type->clear)(f, (void *)entry_ptr, TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear entry") } - } /* if ( destroy ) */ - } - if ( entry_ptr != NULL ) - { - if ( cache_ptr->log_flush ) { + /* we are about to discard the in core representation -- + * set the magic field to bad magic so we can detect a + * freed entry if we see one. + */ + entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; - status = (cache_ptr->log_flush)(cache_ptr, addr, was_dirty, flags); + /* verify that the image has been freed */ + HDassert(entry_ptr->image_ptr == NULL); - if ( status < 0 ) { + if(entry_ptr->type->free_icr((void *)entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "free_icr callback failed.") + } + else { + HDassert(take_ownership); - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "log_flush callback failed.") - } + /* client is taking ownership of the entry. + * set bad magic here too so the cache will choke + * unless the entry is re-inserted properly + */ + entry_ptr->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; } - } + } /* if (destroy) */ -done: + if(cache_ptr->log_flush) + if((cache_ptr->log_flush)(cache_ptr, entry_ptr->addr, was_dirty, flags) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "log_flush callback failed.") +done: HDassert( ( ret_value != SUCCEED ) || ( destroy_entry ) || - ( ( entry_ptr ) && ( ! entry_ptr->flush_in_progress ) ) ); + ( ! entry_ptr->flush_in_progress ) ); HDassert( ( ret_value != SUCCEED ) || ( destroy_entry ) || - ( take_ownership ) || - ( ( entry_ptr ) && ( ! entry_ptr->is_dirty ) ) ); + ( take_ownership ) || ( ! entry_ptr->is_dirty ) ); FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_flush_single_entry() */ +} /* H5C__flush_single_entry() */ /*------------------------------------------------------------------------- @@ -10430,7 +8893,7 @@ done: * attempts to load another entry. * * This error was exposed by a re-factor of the - * H5C_flush_single_entry() routine. However, it was + * H5C__flush_single_entry() routine. However, it was * a potential bug from the moment that entries were * allowed to load other entries on flush. * @@ -10464,7 +8927,6 @@ H5C_make_space_in_cache(H5F_t * f, hbool_t write_permitted) { H5C_t * cache_ptr = f->shared->cache; - herr_t result; #if H5C_COLLECT_CACHE_STATS int32_t clean_entries_skipped = 0; int32_t total_entries_scanned = 0; @@ -10552,7 +9014,7 @@ H5C_make_space_in_cache(H5F_t * f, /* reset entries_removed_counter and * last_entry_removed_ptr prior to the call to - * H5C_flush_single_entry() so that we can spot + * H5C__flush_single_entry() so that we can spot * unexpected removals of entries from the cache, * and set the restart_scan flag if proceeding * would be likely to cause us to scan an entry @@ -10566,13 +9028,9 @@ H5C_make_space_in_cache(H5F_t * f, H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) } - result = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - H5C__NO_FLAGS_SET, - FALSE, - NULL, - NULL); + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__NO_FLAGS_SET, NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") + if ( ( cache_ptr->entries_removed_counter > 1 ) || ( cache_ptr->last_entry_removed_ptr == prev_ptr ) ) @@ -10592,27 +9050,14 @@ H5C_make_space_in_cache(H5F_t * f, H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) } - result = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - H5C__FLUSH_INVALIDATE_FLAG, - TRUE, - NULL, - NULL); - - + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") } else { - - /* We have enough space so don't flush clean entry. - * Set result to SUCCEED to avoid triggering the error - * code below. - */ + /* We have enough space so don't flush clean entry. */ #if H5C_COLLECT_CACHE_STATS clean_entries_skipped++; #endif /* H5C_COLLECT_CACHE_STATS */ didnt_flush_entry = TRUE; - result = SUCCEED; - } #if H5C_COLLECT_CACHE_STATS @@ -10622,17 +9067,9 @@ H5C_make_space_in_cache(H5F_t * f, } else { /* Skip epoch markers and entries that are in the process - * of being flushed. Set result to SUCCEED to avoid - * triggering the error code below. + * of being flushed. */ didnt_flush_entry = TRUE; - result = SUCCEED; - } - - if ( result < 0 ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unable to flush entry") } if ( prev_ptr != NULL ) { @@ -10752,19 +9189,8 @@ H5C_make_space_in_cache(H5F_t * f, #ifdef H5_HAVE_PARALLEL if(!(entry_ptr->coll_access)) { #endif /* H5_HAVE_PARALLEL */ - result = H5C_flush_single_entry(f, - dxpl_id, - entry_ptr->addr, - H5C__FLUSH_INVALIDATE_FLAG, - TRUE, - NULL, - NULL); - - if ( result < 0 ) { - - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, \ - "unable to flush entry") - } + if(H5C__flush_single_entry(f, dxpl_id, entry_ptr, H5C__FLUSH_INVALIDATE_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to flush entry") #ifdef H5_HAVE_PARALLEL } #endif /* H5_HAVE_PARALLEL */ @@ -11565,7 +9991,7 @@ H5C_flush_tagged_entries(H5F_t * f, hid_t dxpl_id, H5C_t * cache_ptr, haddr_t ta HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't mark tagged entries") /* Flush all marked entries */ - if(H5C_flush_marked_entries(f, dxpl_id, cache_ptr) < 0) + if(H5C_flush_marked_entries(f, dxpl_id) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't flush marked entries") done: @@ -11638,7 +10064,7 @@ H5C_mark_tagged_entries(H5C_t * cache_ptr, haddr_t tag) *------------------------------------------------------------------------- */ static herr_t -H5C_flush_marked_entries(H5F_t * f, hid_t dxpl_id, H5C_t * cache_ptr) +H5C_flush_marked_entries(H5F_t * f, hid_t dxpl_id) { herr_t ret_value = SUCCEED; @@ -11646,8 +10072,7 @@ H5C_flush_marked_entries(H5F_t * f, hid_t dxpl_id, H5C_t * cache_ptr) /* Assertions */ HDassert(0); /* This function is not yet used. We shouldn't be in here yet. */ - HDassert(cache_ptr != NULL); - HDassert(cache_ptr->magic == H5C__H5C_T_MAGIC); + HDassert(f != NULL); /* Flush all marked entries */ if(H5C_flush_cache(f, dxpl_id, H5C__FLUSH_MARKED_ENTRIES_FLAG | H5C__FLUSH_IGNORE_PROTECTED_FLAG) < 0) @@ -11657,208 +10082,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5C_flush_marked_entries */ -#ifdef H5_HAVE_PARALLEL - -/*------------------------------------------------------------------------- - * - * Function: H5C_clear_coll_entries - * - * Purpose: Clear half or the entire list of collective entries and - * mark them as independent. - * - * Return: FAIL if error is detected, SUCCEED otherwise. - * - * Programmer: Mohamad Chaarawi - * April, 2015 - * - *------------------------------------------------------------------------- - */ -herr_t -H5C_clear_coll_entries(H5C_t * cache_ptr, hbool_t partial) -{ - int32_t list_len, coll_entries_cleared = 0; - H5C_cache_entry_t * entry_ptr = NULL; - H5C_cache_entry_t * prev_ptr; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - - entry_ptr = cache_ptr->coll_tail_ptr; - list_len = cache_ptr->coll_list_len; - - while(entry_ptr && (coll_entries_cleared < (partial ? list_len/2 : list_len))) { - prev_ptr = entry_ptr->coll_prev; - - HDassert(entry_ptr->coll_access); - - entry_ptr->coll_access = FALSE; - H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) - coll_entries_cleared ++; - - entry_ptr = prev_ptr; - } - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* H5C_clear_coll_entries */ - -static herr_t -H5C_collective_write(H5F_t *f, - hid_t dxpl_id, - H5SL_t *collective_write_list) -{ - H5P_genplist_t *plist = NULL; - H5FD_mpio_xfer_t xfer_mode = H5FD_MPIO_COLLECTIVE; - H5FD_mpio_xfer_t orig_xfer_mode = -1; - H5SL_node_t *node; - H5C_collective_write_t *item; - int count; - void *base_buf; - int *length_array = NULL; - MPI_Aint *buf_array = NULL; - MPI_Aint *offset_array = NULL; - MPI_Datatype btype; - MPI_Datatype ftype; - hbool_t btype_created = FALSE; - hbool_t ftype_created = FALSE; - int mpi_code; - int i; - herr_t ret_value = SUCCEED; - - FUNC_ENTER_NOAPI_NOINIT - - count = (int)H5SL_count(collective_write_list); - - if(NULL == (plist = (H5P_genplist_t *)H5I_object(dxpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list") - - if(H5P_get(plist, H5D_XFER_IO_XFER_MODE_NAME, &orig_xfer_mode) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O property") - - if(count > 0) { - if(H5P_set(plist, H5D_XFER_IO_XFER_MODE_NAME, &xfer_mode) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O property") - - /* Allocate arrays */ - if(NULL == (length_array = (int *)H5MM_malloc((size_t)count * sizeof(int)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for collective write table length array") - if(NULL == (buf_array = (MPI_Aint *)H5MM_malloc((size_t)count * sizeof(MPI_Aint)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for collective buf table length array") - if(NULL == (offset_array = (MPI_Aint *)H5MM_malloc((size_t)count * sizeof(MPI_Aint)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for collective offset table length array") - - /* Fill arrays */ - node = H5SL_first(collective_write_list); - HDassert(node); - if(NULL == (item = (H5C_collective_write_t *)H5SL_item(node))) - HGOTO_ERROR(H5E_CACHE, H5E_NOTFOUND, FAIL, "can't retrieve skip list item") - - length_array[0] = (int)item->length; - base_buf = item->buf; - buf_array[0] = (MPI_Aint)0; - offset_array[0] = (MPI_Aint)item->offset; - - node = H5SL_next(node); - i = 1; - while(node) { - if(NULL == (item = (H5C_collective_write_t *)H5SL_item(node))) - HGOTO_ERROR(H5E_CACHE, H5E_NOTFOUND, FAIL, "can't retrieve skip list item") - - length_array[i] = (int)item->length; - buf_array[i] = (MPI_Aint)item->buf - (MPI_Aint)base_buf; - offset_array[i] = (MPI_Aint)item->offset; - node = H5SL_next(node); - i++; - } /* end while */ - - /* Create memory mpi type */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed(count, length_array, buf_array, MPI_BYTE, &btype))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) - btype_created = TRUE; - if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&btype))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - - /* Create file mpi type */ - if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed(count, length_array, offset_array, MPI_BYTE, &ftype))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) - ftype_created = TRUE; - if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&ftype))) - HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) - - /* Pass buf type, file type to the file driver. */ - if(H5FD_mpi_setup_collective(dxpl_id, &btype, &ftype) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O properties") - - /* Write data */ - if(H5F_block_write(f, H5FD_MEM_DEFAULT, (haddr_t)0, (size_t)1, dxpl_id, base_buf) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to write entries collectively") - } /* end if */ - else { - MPI_Status mpi_stat; - MPI_File mpi_fh_p; - MPI_File mpi_fh; - - if(H5F_get_mpi_handle(f, (MPI_File **)&mpi_fh_p) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get mpi file handle") - mpi_fh = *(MPI_File*)mpi_fh_p; - - /* just to match up with the 1st MPI_File_set_view from H5FD_mpio_write() */ - if(MPI_SUCCESS != (mpi_code = MPI_File_set_view(mpi_fh, (MPI_Offset)0, MPI_BYTE, - MPI_BYTE, "native", MPI_INFO_NULL))) - HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) - - /* just to match up with MPI_File_write_at_all from H5FD_mpio_write() */ - HDmemset(&mpi_stat, 0, sizeof(MPI_Status)); - if(MPI_SUCCESS != (mpi_code = MPI_File_write_at_all(mpi_fh, (MPI_Offset)0, NULL, 0, - MPI_BYTE, &mpi_stat))) - HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at_all failed", mpi_code) - - /* just to match up with the 2nd MPI_File_set_view (reset) in H5FD_mpio_write() */ - if(MPI_SUCCESS != (mpi_code = MPI_File_set_view(mpi_fh, (MPI_Offset)0, MPI_BYTE, - MPI_BYTE, "native", MPI_INFO_NULL))) - HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) - } /* end else */ - -done: - /* Free arrays */ - length_array = (int *)H5MM_xfree(length_array); - buf_array = (MPI_Aint *)H5MM_xfree(buf_array); - offset_array = (MPI_Aint *)H5MM_xfree(offset_array); - - /* Free MPI Types */ - if(btype_created && MPI_SUCCESS != (mpi_code = MPI_Type_free(&btype))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - if(ftype_created && MPI_SUCCESS != (mpi_code = MPI_Type_free(&ftype))) - HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) - - /* Reset dxpl */ - if(orig_xfer_mode != H5FD_MPIO_COLLECTIVE) { - HDassert(plist); - if(H5P_set(plist, H5D_XFER_IO_XFER_MODE_NAME, &orig_xfer_mode) < 0) - HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O property") - } /* end if */ - - FUNC_LEAVE_NOAPI(ret_value); -} /* end H5C_collective_write() */ - -static herr_t -H5C_collective_write_free(void *_item, void H5_ATTR_UNUSED *key, void H5_ATTR_UNUSED *op_data) -{ - H5C_collective_write_t *item = (H5C_collective_write_t *)_item; - - FUNC_ENTER_NOAPI_NOINIT_NOERR - - HDassert(item); - - if(item->free_buf) - item->buf = H5MM_xfree(item->buf); - /*!FIXME change to use free list for items */ - H5MM_free(item); - - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5C_double_collective_table() */ -#endif /* H5_HAVE_PARALLEL */ - #if H5C_DO_TAGGING_SANITY_CHECKS /*------------------------------------------------------------------------- diff --git a/src/H5Cmpio.c b/src/H5Cmpio.c new file mode 100644 index 0000000..aa6b49a --- /dev/null +++ b/src/H5Cmpio.c @@ -0,0 +1,1470 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Cmpio.c + * June 20 2015 + * Quincey Koziol + * + * Purpose: Functions in this file implement support for parallel I/O for + * generic cache code. + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#define H5C_PACKAGE /*suppress error about including H5Cpkg */ +#define H5F_PACKAGE /*suppress error about including H5Fpkg */ + + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5ACprivate.h" /* Metadata cache */ +#include "H5Cpkg.h" /* Cache */ +#include "H5Dprivate.h" /* Datasets */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5Fpkg.h" /* Files */ +#include "H5FDprivate.h" /* File drivers */ +#include "H5Iprivate.h" /* IDs */ +#include "H5MMprivate.h" /* Memory management */ +#include "H5Pprivate.h" /* Property lists */ +#include "H5SLprivate.h" /* Skip lists */ + +#ifdef H5_HAVE_PARALLEL + +/****************/ +/* Local Macros */ +/****************/ +#define H5C_APPLY_CANDIDATE_LIST__DEBUG 0 + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + + + +/*------------------------------------------------------------------------- + * Function: H5C_apply_candidate_list + * + * Purpose: Apply the supplied candidate list. + * + * We used to do this by simply having each process write + * every mpi_size-th entry in the candidate list, starting + * at index mpi_rank, and mark all the others clean. + * + * However, this can cause unnecessary contention in a file + * system by increasing the number of processes writing to + * adjacent locations in the HDF5 file. + * + * To attempt to minimize this, we now arange matters such + * that each process writes n adjacent entries in the + * candidate list, and marks all others clean. We must do + * this in such a fashion as to guarantee that each entry + * on the candidate list is written by exactly one process, + * and marked clean by all others. + * + * To do this, first construct a table mapping mpi_rank + * to the index of the first entry in the candidate list to + * be written by the process of that mpi_rank, and then use + * the table to control which entries are written and which + * are marked as clean as a function of the mpi_rank. + * + * Note that the table must be identical on all processes, as + * all see the same candidate list, mpi_size, and mpi_rank -- + * the inputs used to construct the table. + * + * We construct the table as follows. Let: + * + * n = num_candidates / mpi_size; + * + * m = num_candidates % mpi_size; + * + * Now allocate an array of integers of length mpi_size + 1, + * and call this array candidate_assignment_table. + * + * Conceptually, if the number of candidates is a multiple + * of the mpi_size, we simply pass through the candidate list + * and assign n entries to each process to flush, with the + * index of the first entry to flush in the location in + * the candidate_assignment_table indicated by the mpi_rank + * of the process. + * + * In the more common case in which the candidate list isn't + * isn't a multiple of the mpi_size, we pretend it is, and + * give num_candidates % mpi_size processes one extra entry + * each to make things work out. + * + * Once the table is constructed, we determine the first and + * last entry this process is to flush as follows: + * + * first_entry_to_flush = candidate_assignment_table[mpi_rank] + * + * last_entry_to_flush = + * candidate_assignment_table[mpi_rank + 1] - 1; + * + * With these values determined, we simply scan through the + * candidate list, marking all entries in the range + * [first_entry_to_flush, last_entry_to_flush] for flush, + * and all others to be cleaned. + * + * Finally, we scan the LRU from tail to head, flushing + * or marking clean the candidate entries as indicated. + * If necessary, we scan the pinned list as well. + * + * Note that this function will fail if any protected or + * clean entries appear on the candidate list. + * + * This function is used in managing sync points, and + * shouldn't be used elsewhere. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: John Mainzer + * 3/17/10 + * + * Changes: Ported code to detect next entry status changes as the + * the result of a flush from the serial code in the scan of + * the LRU. Also added code to detect and adapt to the + * removal from the cache of the next entry in the scan of + * the LRU. + * + * Note that at present, all of these changes should not + * be required as the operations on entries as they are + * flushed that can cause these condiditions are not premitted + * in the parallel case. However, Quincey indicates that + * this may change, and thus has requested the modification. + * + * Note the assert(FALSE) in the if statement whose body + * restarts the scan of the LRU. As the body of the if + * statement should be unreachable, it should never be + * triggered until the constraints on the parallel case + * are relaxed. Please remove the assertion at that time. + * + * Also added warning on the Pinned Entry List scan, as it + * is potentially subject to the same issue. As there is + * no cognate of this scan in the serial code, I don't have + * a fix to port to it. + * + * JRM -- 4/10/19 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_apply_candidate_list(H5F_t * f, + hid_t dxpl_id, + H5C_t * cache_ptr, + int num_candidates, + haddr_t * candidates_list_ptr, + int mpi_rank, + int mpi_size) +{ + hbool_t restart_scan; + hbool_t prev_is_dirty; + int i; + int m; + int n; + int first_entry_to_flush; + int last_entry_to_flush; + int entries_to_clear = 0; + int entries_to_flush = 0; + int entries_to_flush_or_clear_last = 0; + int entries_to_flush_collectively = 0; + int entries_cleared = 0; + int entries_flushed = 0; + int entries_delayed = 0; + int entries_flushed_or_cleared_last = 0; + int entries_flushed_collectively = 0; + int entries_examined = 0; + int initial_list_len; + int * candidate_assignment_table = NULL; + haddr_t addr; + H5C_cache_entry_t * clear_ptr = NULL; + H5C_cache_entry_t * next_ptr = NULL; + H5C_cache_entry_t * entry_ptr = NULL; + H5C_cache_entry_t * flush_ptr = NULL; + H5C_cache_entry_t * delayed_ptr = NULL; + H5SL_t * collective_write_list = NULL; +#if H5C_DO_SANITY_CHECKS + haddr_t last_addr; +#endif /* H5C_DO_SANITY_CHECKS */ +#if H5C_APPLY_CANDIDATE_LIST__DEBUG + char tbl_buf[1024]; +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + HDassert( cache_ptr != NULL ); + HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); + HDassert( num_candidates > 0 ); + HDassert( num_candidates <= cache_ptr->slist_len ); + HDassert( candidates_list_ptr != NULL ); + HDassert( 0 <= mpi_rank ); + HDassert( mpi_rank < mpi_size ); + +#if H5C_APPLY_CANDIDATE_LIST__DEBUG + HDfprintf(stdout, "%s:%d: setting up candidate assignment table.\n", + FUNC, mpi_rank); + for ( i = 0; i < 1024; i++ ) tbl_buf[i] = '\0'; + sprintf(&(tbl_buf[0]), "candidate list = "); + for ( i = 0; i < num_candidates; i++ ) + { + sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), " 0x%llx", + (long long)(*(candidates_list_ptr + i))); + } + sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), "\n"); + HDfprintf(stdout, "%s", tbl_buf); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + /* Create skip list of entries for collective write */ + if(NULL == (collective_write_list = H5SL_create(H5SL_TYPE_HADDR, NULL))) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCREATE, FAIL, "can't create skip list for entries") + + n = num_candidates / mpi_size; + m = num_candidates % mpi_size; + HDassert(n >= 0); + + if(NULL == (candidate_assignment_table = (int *)H5MM_malloc(sizeof(int) * (size_t)(mpi_size + 1)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for candidate assignment table") + + candidate_assignment_table[0] = 0; + candidate_assignment_table[mpi_size] = num_candidates; + + if(m == 0) { /* mpi_size is an even divisor of num_candidates */ + HDassert(n > 0); + for(i = 1; i < mpi_size; i++) + candidate_assignment_table[i] = candidate_assignment_table[i - 1] + n; + } /* end if */ + else { + for(i = 1; i <= m; i++) + candidate_assignment_table[i] = candidate_assignment_table[i - 1] + n + 1; + + if(num_candidates < mpi_size) { + for(i = m + 1; i < mpi_size; i++) + candidate_assignment_table[i] = num_candidates; + } /* end if */ + else { + for(i = m + 1; i < mpi_size; i++) + candidate_assignment_table[i] = candidate_assignment_table[i - 1] + n; + } /* end else */ + } /* end else */ + HDassert((candidate_assignment_table[mpi_size - 1] + n) == num_candidates); + +#if H5C_DO_SANITY_CHECKS + /* verify that the candidate assignment table has the expected form */ + for ( i = 1; i < mpi_size - 1; i++ ) + { + int a, b; + + a = candidate_assignment_table[i] - candidate_assignment_table[i - 1]; + b = candidate_assignment_table[i + 1] - candidate_assignment_table[i]; + + HDassert( n + 1 >= a ); + HDassert( a >= b ); + HDassert( b >= n ); + } +#endif /* H5C_DO_SANITY_CHECKS */ + + first_entry_to_flush = candidate_assignment_table[mpi_rank]; + last_entry_to_flush = candidate_assignment_table[mpi_rank + 1] - 1; + +#if H5C_APPLY_CANDIDATE_LIST__DEBUG + for ( i = 0; i < 1024; i++ ) + tbl_buf[i] = '\0'; + sprintf(&(tbl_buf[0]), "candidate assignment table = "); + for(i = 0; i <= mpi_size; i++) + sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), " %d", candidate_assignment_table[i]); + sprintf(&(tbl_buf[HDstrlen(tbl_buf)]), "\n"); + HDfprintf(stdout, "%s", tbl_buf); + + HDfprintf(stdout, "%s:%d: flush entries [%d, %d].\n", + FUNC, mpi_rank, first_entry_to_flush, last_entry_to_flush); + + HDfprintf(stdout, "%s:%d: marking entries.\n", FUNC, mpi_rank); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + for(i = 0; i < num_candidates; i++) { + addr = candidates_list_ptr[i]; + HDassert( H5F_addr_defined(addr) ); + +#if H5C_DO_SANITY_CHECKS + if ( i > 0 ) { + if ( last_addr == addr ) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Duplicate entry in cleaned list.\n") + } else if ( last_addr > addr ) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "candidate list not sorted.\n") + } + } + + last_addr = addr; +#endif /* H5C_DO_SANITY_CHECKS */ + + H5C__SEARCH_INDEX(cache_ptr, addr, entry_ptr, FAIL) + if(entry_ptr == NULL) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed candidate entry not in cache?!?!?.") + } else if(!entry_ptr->is_dirty) { + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed entry not dirty?!?!?.") + } else if ( entry_ptr->is_protected ) { + /* For now at least, we can't deal with protected entries. + * If we encounter one, scream and die. If it becomes an + * issue, we should be able to work around this. + */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Listed entry is protected?!?!?.") + } else { + /* determine whether the entry is to be cleared or flushed, + * and mark it accordingly. We will scan the protected and + * pinned list shortly, and clear or flush according to these + * markings. + */ + if((i >= first_entry_to_flush) && (i <= last_entry_to_flush)) { + entries_to_flush++; + entry_ptr->flush_immediately = TRUE; + } /* end if */ + else { + entries_to_clear++; + entry_ptr->clear_on_unprotect = TRUE; + } /* end else */ + + /* Entries marked as collectively accessed and are in the + candidate list to clear from the cache have to be + removed from the coll list. This is OK since the + candidate list is collective and uniform across all + ranks. */ + if(TRUE == entry_ptr->coll_access) { + entry_ptr->coll_access = FALSE; + H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) + } + } /* end else */ + } /* end for */ + +#if H5C_APPLY_CANDIDATE_LIST__DEBUG + HDfprintf(stdout, "%s:%d: num candidates/to clear/to flush = %d/%d/%d.\n", + FUNC, mpi_rank, (int)num_candidates, (int)entries_to_clear, + (int)entries_to_flush); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + + /* We have now marked all the entries on the candidate list for + * either flush or clear -- now scan the LRU and the pinned list + * for these entries and do the deed. + * + * Note that we are doing things in this round about manner so as + * to preserve the order of the LRU list to the best of our ability. + * If we don't do this, my experiments indicate that we will have a + * noticably poorer hit ratio as a result. + */ + +#if H5C_APPLY_CANDIDATE_LIST__DEBUG + HDfprintf(stdout, "%s:%d: scanning LRU list. len = %d.\n", FUNC, mpi_rank, + (int)(cache_ptr->LRU_list_len)); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + /* ===================================================================== * + * Now scan the LRU and PEL lists, flushing or clearing entries as + * needed. + * + * The flush_me_last and flush_me_collectively flags may dictate how or + * when some entries can be flushed, and should be addressed here. + * However, in their initial implementation, these flags only apply to the + * superblock, so there's only a relatively small change to this function + * to account for this one case where they come into play. If these flags + * are ever expanded upon, this function and the following flushing steps + * should be reworked to account for additional cases. + * ===================================================================== */ + + HDassert(entries_to_flush >= 0); + + restart_scan = FALSE; + entries_examined = 0; + initial_list_len = cache_ptr->LRU_list_len; + entry_ptr = cache_ptr->LRU_tail_ptr; + + /* Examine each entry in the LRU list */ + while ( ( entry_ptr != NULL ) + && + ( entries_examined <= (entries_to_flush + 1) * initial_list_len ) + && + ( (entries_cleared + entries_flushed) < num_candidates ) ) { + + if ( entry_ptr->prev != NULL ) + prev_is_dirty = entry_ptr->prev->is_dirty; + + /* If this process needs to clear this entry. */ + if(entry_ptr->clear_on_unprotect) { + + HDassert(entry_ptr->is_dirty); + + next_ptr = entry_ptr->next; + entry_ptr->clear_on_unprotect = FALSE; + clear_ptr = entry_ptr; + entry_ptr = entry_ptr->prev; + entries_cleared++; + +#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) + HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, + (long long)clear_ptr->addr); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + /* No need to check for the next entry in the scan being + * removed from the cache, as this call to H5C__flush_single_entry() + * will not call either the pre_serialize or serialize callbacks. + */ + + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, + H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, + NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") + } /* end if */ + + /* Else, if this process needs to flush this entry. */ + else if (entry_ptr->flush_immediately) { + + HDassert(entry_ptr->is_dirty); + + next_ptr = entry_ptr->next; + entry_ptr->flush_immediately = FALSE; + flush_ptr = entry_ptr; + entry_ptr = entry_ptr->prev; + entries_flushed++; + +#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) + HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, + (long long)flush_ptr->addr); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + /* reset entries_removed_counter and + * last_entry_removed_ptr prior to the call to + * H5C__flush_single_entry() so that we can spot + * unexpected removals of entries from the cache, + * and set the restart_scan flag if proceeding + * would be likely to cause us to scan an entry + * that is no longer in the cache. + * + * Note that as of this writing (April 2015) this + * case cannot occur in the parallel case. However + * Quincey is making noises about changing this, hence + * the insertion of this test. + * + * Note also that there is no test code to verify + * that this code actually works (although similar code + * in the serial version exists and is tested). + * + * Implementing a test will likely require implementing + * flush op like facilities in the parallel tests. At + * a guess this will not be terribly painful, but it + * will take a bit of time. + */ + cache_ptr->entries_removed_counter = 0; + cache_ptr->last_entry_removed_ptr = NULL; + + if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, + NULL, collective_write_list) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry.") + + if ( ( cache_ptr->entries_removed_counter > 1 ) || + ( cache_ptr->last_entry_removed_ptr == entry_ptr ) ) + + restart_scan = TRUE; + + } /* end else-if */ + + /* Otherwise, no action to be taken on this entry. Grab the next. */ + else { + entry_ptr = entry_ptr->prev; + + if ( entry_ptr != NULL ) + next_ptr = entry_ptr->next; + + } /* end else */ + + if ( ( entry_ptr != NULL ) + && + ( ( restart_scan ) + || + ( entry_ptr->is_dirty != prev_is_dirty ) + || + ( entry_ptr->next != next_ptr ) + || + ( entry_ptr->is_protected ) + || + ( entry_ptr->is_pinned ) + ) + ) { + + /* something has happened to the LRU -- start over + * from the tail. + * + * Recall that this code should be un-reachable at present, + * as all the operations by entries on flush that could cause + * it to be reachable are disallowed in the parallel case at + * present. Hence the following assertion which should be + * removed if the above changes. + */ + + HDassert( ! restart_scan ); + HDassert( entry_ptr->is_dirty == prev_is_dirty ); + HDassert( entry_ptr->next == next_ptr ); + HDassert( ! entry_ptr->is_protected ); + HDassert( ! entry_ptr->is_pinned ); + + HDassert(FALSE); /* see comment above */ + + restart_scan = FALSE; + entry_ptr = cache_ptr->LRU_tail_ptr; +/* + H5C__UPDATE_STATS_FOR_LRU_SCAN_RESTART(cache_ptr) +*/ + } + + entries_examined++; + } /* end while */ + +#if H5C_APPLY_CANDIDATE_LIST__DEBUG + HDfprintf(stdout, "%s:%d: entries examined/cleared/flushed = %d/%d/%d.\n", + FUNC, mpi_rank, entries_examined, + entries_cleared, entries_flushed); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + /* It is also possible that some of the cleared entries are on the + * pinned list. Must scan that also. + * + * WARNING: + * + * As we now allow unpinning, and removal of other entries as a side + * effect of flushing an entry, it is possible that the next entry + * in a PEL scan could either be no longer pinned, or no longer in + * the cache by the time we get to it. + * + * At present, this is not possible in this case, as we disallow such + * operations in the parallel version of the library. However, Quincey + * has been making noises about relaxing this. If and when he does, + * we have a potential problem here. + * + * The same issue exists in the serial cache, and there are tests + * to detect this problem when it occurs, and adjust to it. As seen + * above in the LRU scan, I have ported such tests to the parallel + * code where a close cognate exists in the serial code. + * + * I haven't done so here, as there are no PEL scans where the problem + * can occur in the serial code. Needless to say, this will have to + * be repaired if the constraints on pre_serialize and serialize + * callbacks are relaxed in the parallel version of the metadata cache. + * + * JRM -- 4/1/15 + */ + +#if H5C_APPLY_CANDIDATE_LIST__DEBUG + HDfprintf(stdout, "%s:%d: scanning pinned entry list. len = %d\n", + FUNC, mpi_rank, (int)(cache_ptr->pel_len)); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + entry_ptr = cache_ptr->pel_head_ptr; + while((entry_ptr != NULL) && + ((entries_cleared + entries_flushed + entries_delayed) + < num_candidates)) { + + /* If entry is marked for flush or for clear */ + if((entry_ptr->clear_on_unprotect||entry_ptr->flush_immediately)) { + + /* If this entry needs to be flushed last */ + if (entry_ptr->flush_me_last) { + + /* At this time, only the superblock supports being + flushed last. Conveniently, it also happens to be the only + entry that supports being flushed collectively, as well. Also + conveniently, it's always pinned, so we only need to check + for it while scanning the PEL here. Finally, it's never + included in a candidate list that excludes other dirty + entries in a cache, so we can handle this relatively simple + case here. + + For now, this function asserts this and saves the entry + to flush it after scanning the rest of the PEL list. + + If there are ever more entries that either need to be + flushed last and/or flushed collectively, this whole routine + will need to be reworked to handle all additional cases. As + it is the simple case of a single pinned entry needing + flushed last and collectively is just a minor addition to + this routine, but signficantly buffing up the usage of + flush_me_last or flush_me_collectively will require a more + intense rework of this function and potentially the function + of candidate lists as a whole. */ + + HDassert(entry_ptr->flush_me_collectively); + entries_to_flush_or_clear_last++; + entries_to_flush_collectively++; + HDassert(entries_to_flush_or_clear_last == 1); + HDassert(entries_to_flush_collectively == 1); + + /* Delay the entry. It will be flushed later. */ + delayed_ptr = entry_ptr; + entries_delayed++; + HDassert(entries_delayed == 1); + + } /* end if */ + + /* Else, this process needs to clear this entry. */ + else if (entry_ptr->clear_on_unprotect) { + HDassert(!entry_ptr->flush_immediately); + entry_ptr->clear_on_unprotect = FALSE; + clear_ptr = entry_ptr; + entry_ptr = entry_ptr->next; + entries_cleared++; + +#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) + HDfprintf(stdout, "%s:%d: clearing 0x%llx.\n", FUNC, mpi_rank, + (long long)clear_ptr->addr); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, + H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, + NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") + } /* end else-if */ + + /* Else, if this process needs to independently flush this entry. */ + else if (entry_ptr->flush_immediately) { + entry_ptr->flush_immediately = FALSE; + flush_ptr = entry_ptr; + entry_ptr = entry_ptr->next; + entries_flushed++; + +#if ( H5C_APPLY_CANDIDATE_LIST__DEBUG > 1 ) + HDfprintf(stdout, "%s:%d: flushing 0x%llx.\n", FUNC, mpi_rank, + (long long)flush_ptr->addr); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + if(H5C__flush_single_entry(f, dxpl_id, flush_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, + NULL, collective_write_list) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") + } /* end else-if */ + } /* end if */ + + /* Otherwise, this entry is not marked for flush or clear. Grab the next. */ + else { + entry_ptr = entry_ptr->next; + } /* end else */ + + } /* end while */ + +#if H5C_APPLY_CANDIDATE_LIST__DEBUG + HDfprintf(stdout, + "%s:%d: pel entries examined/cleared/flushed = %d/%d/%d.\n", + FUNC, mpi_rank, entries_examined, + entries_cleared, entries_flushed); + HDfprintf(stdout, "%s:%d: done.\n", FUNC, mpi_rank); + + HDfsync(stdout); +#endif /* H5C_APPLY_CANDIDATE_LIST__DEBUG */ + + /* ====================================================================== * + * Now, handle all delayed entries. * + * * + * This can *only* be the superblock at this time, so it's relatively * + * easy to deal with. We're collectively flushing the entry saved from * + * above. This will need to be handled differently if there are ever more * + * than one entry needing this special treatment.) * + * ====================================================================== */ + + if (delayed_ptr) { + + if (delayed_ptr->clear_on_unprotect) { + if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, + NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry collectively.") + entry_ptr->clear_on_unprotect = FALSE; + entries_cleared++; + } else if (delayed_ptr->flush_immediately) { + if(H5C__flush_single_entry(f, dxpl_id, delayed_ptr, H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, + NULL, collective_write_list) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't flush entry collectively.") + entry_ptr->flush_immediately = FALSE; + entries_flushed++; + } /* end if */ + + + entries_flushed_collectively++; + entries_flushed_or_cleared_last++; + } /* end if */ + + /* Write collective list */ + if(H5C_collective_write(f, + dxpl_id, + collective_write_list) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't write metadata collectively") + + /* ====================================================================== * + * Finished flushing everything. * + * ====================================================================== */ + + HDassert((entries_flushed == entries_to_flush)); + HDassert((entries_cleared == entries_to_clear)); + HDassert((entries_flushed_or_cleared_last == entries_to_flush_or_clear_last)); + HDassert((entries_flushed_collectively == entries_to_flush_collectively)); + + if((entries_flushed != entries_to_flush) || + (entries_cleared != entries_to_clear) || + (entries_flushed_or_cleared_last != entries_to_flush_or_clear_last) || + (entries_flushed_collectively != entries_to_flush_collectively)) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "entry count mismatch.") + +done: + if(candidate_assignment_table != NULL) + candidate_assignment_table = (int *)H5MM_xfree((void *)candidate_assignment_table); + + if(collective_write_list) + if(H5SL_destroy(collective_write_list, H5C_collective_write_free, NULL) < 0) + HDONE_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "failed to destroy skip list") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_apply_candidate_list() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_construct_candidate_list__clean_cache + * + * Purpose: Construct the list of entries that should be flushed to + * clean all entries in the cache. + * + * This function is used in managing sync points, and + * shouldn't be used elsewhere. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: John Mainzer + * 3/17/10 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_construct_candidate_list__clean_cache(H5C_t * cache_ptr) +{ + size_t space_needed; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + HDassert( cache_ptr != NULL ); + HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); + + /* As a sanity check, set space needed to the size of the skip list. + * This should be the sum total of the sizes of all the dirty entries + * in the metadata cache. + */ + space_needed = cache_ptr->slist_size; + + /* Recall that while we shouldn't have any protected entries at this + * point, it is possible that some dirty entries may reside on the + * pinned list at this point. + */ + HDassert( cache_ptr->slist_size <= + (cache_ptr->dLRU_list_size + cache_ptr->pel_size) ); + HDassert( cache_ptr->slist_len <= + (cache_ptr->dLRU_list_len + cache_ptr->pel_len) ); + + if(space_needed > 0) { /* we have work to do */ + H5C_cache_entry_t *entry_ptr; + int nominated_entries_count = 0; + size_t nominated_entries_size = 0; + haddr_t nominated_addr; + + HDassert( cache_ptr->slist_len > 0 ); + + /* Scan the dirty LRU list from tail forward and nominate sufficient + * entries to free up the necessary space. + */ + entry_ptr = cache_ptr->dLRU_tail_ptr; + while((nominated_entries_size < space_needed) && + (nominated_entries_count < cache_ptr->slist_len) && + (entry_ptr != NULL)) { + HDassert( ! (entry_ptr->is_protected) ); + HDassert( ! (entry_ptr->is_read_only) ); + HDassert( entry_ptr->ro_ref_count == 0 ); + HDassert( entry_ptr->is_dirty ); + HDassert( entry_ptr->in_slist ); + + nominated_addr = entry_ptr->addr; + if(H5AC_add_candidate((H5AC_t *)cache_ptr, nominated_addr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_add_candidate() failed(1).") + + nominated_entries_size += entry_ptr->size; + nominated_entries_count++; + entry_ptr = entry_ptr->aux_prev; + } /* end while */ + HDassert( entry_ptr == NULL ); + + /* it is possible that there are some dirty entries on the + * protected entry list as well -- scan it too if necessary + */ + entry_ptr = cache_ptr->pel_head_ptr; + while((nominated_entries_size < space_needed) && + (nominated_entries_count < cache_ptr->slist_len) && + (entry_ptr != NULL)) { + if(entry_ptr->is_dirty) { + HDassert( ! (entry_ptr->is_protected) ); + HDassert( ! (entry_ptr->is_read_only) ); + HDassert( entry_ptr->ro_ref_count == 0 ); + HDassert( entry_ptr->is_dirty ); + HDassert( entry_ptr->in_slist ); + + nominated_addr = entry_ptr->addr; + if(H5AC_add_candidate((H5AC_t *)cache_ptr, nominated_addr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_add_candidate() failed(2).") + + nominated_entries_size += entry_ptr->size; + nominated_entries_count++; + } /* end if */ + + entry_ptr = entry_ptr->next; + } /* end while */ + + HDassert( nominated_entries_count == cache_ptr->slist_len ); + HDassert( nominated_entries_size == space_needed ); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_construct_candidate_list__clean_cache() */ + + +/*------------------------------------------------------------------------- + * Function: H5C_construct_candidate_list__min_clean + * + * Purpose: Construct the list of entries that should be flushed to + * get the cache back within its min clean constraints. + * + * This function is used in managing sync points, and + * shouldn't be used elsewhere. + * + * Return: Success: SUCCEED + * + * Failure: FAIL + * + * Programmer: John Mainzer + * 3/17/10 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_construct_candidate_list__min_clean(H5C_t * cache_ptr) +{ + size_t space_needed = 0; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + HDassert( cache_ptr != NULL ); + HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); + + /* compute the number of bytes (if any) that must be flushed to get the + * cache back within its min clean constraints. + */ + if(cache_ptr->max_cache_size > cache_ptr->index_size) { + if(((cache_ptr->max_cache_size - cache_ptr->index_size) + + cache_ptr->cLRU_list_size) >= cache_ptr->min_clean_size) + space_needed = 0; + else + space_needed = cache_ptr->min_clean_size - + ((cache_ptr->max_cache_size - cache_ptr->index_size) + + cache_ptr->cLRU_list_size); + } /* end if */ + else { + if(cache_ptr->min_clean_size <= cache_ptr->cLRU_list_size) + space_needed = 0; + else + space_needed = cache_ptr->min_clean_size - + cache_ptr->cLRU_list_size; + } /* end else */ + + if(space_needed > 0) { /* we have work to do */ + H5C_cache_entry_t *entry_ptr; + int nominated_entries_count = 0; + size_t nominated_entries_size = 0; + + HDassert( cache_ptr->slist_len > 0 ); + + /* Scan the dirty LRU list from tail forward and nominate sufficient + * entries to free up the necessary space. + */ + entry_ptr = cache_ptr->dLRU_tail_ptr; + while((nominated_entries_size < space_needed) && + (nominated_entries_count < cache_ptr->slist_len) && + (entry_ptr != NULL) && + (!entry_ptr->flush_me_last)) { + haddr_t nominated_addr; + + HDassert( ! (entry_ptr->is_protected) ); + HDassert( ! (entry_ptr->is_read_only) ); + HDassert( entry_ptr->ro_ref_count == 0 ); + HDassert( entry_ptr->is_dirty ); + HDassert( entry_ptr->in_slist ); + + nominated_addr = entry_ptr->addr; + if(H5AC_add_candidate((H5AC_t *)cache_ptr, nominated_addr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "H5AC_add_candidate() failed.") + + nominated_entries_size += entry_ptr->size; + nominated_entries_count++; + entry_ptr = entry_ptr->aux_prev; + } /* end while */ + HDassert( nominated_entries_count <= cache_ptr->slist_len ); + HDassert( nominated_entries_size >= space_needed ); + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_construct_candidate_list__min_clean() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5C_mark_entries_as_clean + * + * Purpose: When the H5C code is used to implement the metadata caches + * in PHDF5, only the cache with MPI_rank 0 is allowed to + * actually write entries to disk -- all other caches must + * retain dirty entries until they are advised that the + * entries are clean. + * + * This function exists to allow the H5C code to receive these + * notifications. + * + * The function receives a list of entry base addresses + * which must refer to dirty entries in the cache. If any + * of the entries are either clean or don't exist, the + * function flags an error. + * + * The function scans the list of entries and flushes all + * those that are currently unprotected with the + * H5C__FLUSH_CLEAR_ONLY_FLAG. Those that are currently + * protected are flagged for clearing when they are + * unprotected. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: John Mainzer + * 7/5/05 + * + * Changes: Tidied up code, removeing some old commented out + * code that had been left in pending success of the + * new version. + * + * Note that unlike H5C_apply_candidate_list(), + * H5C_mark_entries_as_clean() makes all its calls to + * H6C_flush_single_entry() with the + * H5C__FLUSH_CLEAR_ONLY_FLAG set. As a result, + * the pre_serialize() and serialize calls are not made. + * + * This then implies that (assuming such actions were + * permitted in the parallel case) no loads, dirties, + * resizes, or removals of other entries can occur as + * a side effect of the flush. Hence, there is no need + * for the checks for entry removal / status change + * that I ported to H5C_apply_candidate_list(). + * + * However, if (in addition to allowing such operations + * in the parallel case), we allow such operations outside + * of the pre_serialize / serialize routines, this may + * cease to be the case -- requiring a review of this + * function. + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_mark_entries_as_clean(H5F_t * f, + hid_t dxpl_id, + int32_t ce_array_len, + haddr_t * ce_array_ptr) +{ + H5C_t * cache_ptr; + int entries_cleared; + int entries_examined; + int i; + int initial_list_len; + haddr_t addr; +#if H5C_DO_SANITY_CHECKS + int pinned_entries_marked = 0; + int protected_entries_marked = 0; + int other_entries_marked = 0; + haddr_t last_addr; +#endif /* H5C_DO_SANITY_CHECKS */ + H5C_cache_entry_t * clear_ptr = NULL; + H5C_cache_entry_t * entry_ptr = NULL; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + HDassert( f ); + HDassert( f->shared ); + cache_ptr = f->shared->cache; + HDassert( cache_ptr ); + HDassert( cache_ptr->magic == H5C__H5C_T_MAGIC ); + + HDassert( ce_array_len > 0 ); + HDassert( ce_array_ptr != NULL ); + +#if H5C_DO_EXTREME_SANITY_CHECKS + if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) || + ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) || + ( H5C_validate_lru_list(cache_ptr) < 0 ) ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "an extreme sanity check failed on entry.\n"); + } +#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + + for ( i = 0; i < ce_array_len; i++ ) + { + addr = ce_array_ptr[i]; + +#if H5C_DO_SANITY_CHECKS + if ( i == 0 ) { + + last_addr = addr; + + } else { + + if ( last_addr == addr ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Duplicate entry in cleaned list.\n"); + + } else if ( last_addr > addr ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "cleaned list not sorted.\n"); + } + } + +#if H5C_DO_EXTREME_SANITY_CHECKS + if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) || + ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) || + ( H5C_validate_lru_list(cache_ptr) < 0 ) ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "an extreme sanity check failed in for loop.\n"); + } +#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ +#endif /* H5C_DO_SANITY_CHECKS */ + + HDassert( H5F_addr_defined(addr) ); + + H5C__SEARCH_INDEX(cache_ptr, addr, entry_ptr, FAIL) + + if ( entry_ptr == NULL ) { +#if H5C_DO_SANITY_CHECKS + HDfprintf(stdout, + "H5C_mark_entries_as_clean: entry[%d] = %ld not in cache.\n", + (int)i, + (long)addr); +#endif /* H5C_DO_SANITY_CHECKS */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Listed entry not in cache?!?!?.") + + } else if ( ! entry_ptr->is_dirty ) { + +#if H5C_DO_SANITY_CHECKS + HDfprintf(stdout, + "H5C_mark_entries_as_clean: entry %ld is not dirty!?!\n", + (long)addr); +#endif /* H5C_DO_SANITY_CHECKS */ + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "Listed entry not dirty?!?!?.") + + } else { + + /* Mark the entry to be cleared on unprotect. We will + * scan the LRU list shortly, and clear all those entries + * not currently protected. + */ + + /* Make sure first that we clear the collective flag from + it so it can be cleared */ + if(TRUE == entry_ptr->coll_access) { + entry_ptr->coll_access = FALSE; + H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) + } + + entry_ptr->clear_on_unprotect = TRUE; +#if H5C_DO_SANITY_CHECKS + if ( entry_ptr->is_protected ) { + + protected_entries_marked++; + + } else if ( entry_ptr->is_pinned ) { + + pinned_entries_marked++; + + } else { + + other_entries_marked++; + } +#endif /* H5C_DO_SANITY_CHECKS */ + } + } + + /* Scan through the LRU list from back to front, and flush the + * entries whose clear_on_unprotect flags are set. Observe that + * any protected entries will not be on the LRU, and therefore + * will not be flushed at this time. + * + * Note that unlike H5C_apply_candidate_list(), + * H5C_mark_entries_as_clean() makes all its calls to + * H6C_flush_single_entry() with the H5C__FLUSH_CLEAR_ONLY_FLAG + * set. As a result, the pre_serialize() and serialize calls are + * not made. + * + * This then implies that (assuming such actions were + * permitted in the parallel case) no loads, dirties, + * resizes, or removals of other entries can occur as + * a side effect of the flush. Hence, there is no need + * for the checks for entry removal / status change + * that I ported to H5C_apply_candidate_list(). + * + * However, if (in addition to allowing such operations + * in the parallel case), we allow such operations outside + * of the pre_serialize / serialize routines, this may + * cease to be the case -- requiring a review of this + * point. + * JRM -- 4/7/15 + */ + + entries_cleared = 0; + entries_examined = 0; + initial_list_len = cache_ptr->LRU_list_len; + entry_ptr = cache_ptr->LRU_tail_ptr; + + while ( ( entry_ptr != NULL ) && + ( entries_examined <= initial_list_len ) && + ( entries_cleared < ce_array_len ) ) + { + if ( entry_ptr->clear_on_unprotect ) { + + entry_ptr->clear_on_unprotect = FALSE; + clear_ptr = entry_ptr; + entry_ptr = entry_ptr->prev; + entries_cleared++; + + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, + H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, + NULL, NULL) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") + } else { + + entry_ptr = entry_ptr->prev; + } + entries_examined++; + } + +#if H5C_DO_SANITY_CHECKS + HDassert( entries_cleared == other_entries_marked ); +#endif /* H5C_DO_SANITY_CHECKS */ + + /* It is also possible that some of the cleared entries are on the + * pinned list. Must scan that also. + */ + + entry_ptr = cache_ptr->pel_head_ptr; + + while ( entry_ptr != NULL ) + { + if ( entry_ptr->clear_on_unprotect ) { + + entry_ptr->clear_on_unprotect = FALSE; + clear_ptr = entry_ptr; + entry_ptr = entry_ptr->next; + entries_cleared++; + + if(H5C__flush_single_entry(f, dxpl_id, clear_ptr, + H5C__FLUSH_CLEAR_ONLY_FLAG | H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG, + NULL, NULL) < 0 ) + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "Can't clear entry.") + } else { + + entry_ptr = entry_ptr->next; + } + } + +#if H5C_DO_SANITY_CHECKS + HDassert( entries_cleared == pinned_entries_marked + other_entries_marked ); + HDassert( entries_cleared + protected_entries_marked == ce_array_len ); +#endif /* H5C_DO_SANITY_CHECKS */ + + HDassert( ( entries_cleared == ce_array_len ) || + ( (ce_array_len - entries_cleared) <= cache_ptr->pl_len ) ); + +#if H5C_DO_SANITY_CHECKS + i = 0; + entry_ptr = cache_ptr->pl_head_ptr; + while ( entry_ptr != NULL ) + { + if ( entry_ptr->clear_on_unprotect ) { + + i++; + } + entry_ptr = entry_ptr->next; + } + HDassert( (entries_cleared + i) == ce_array_len ); +#endif /* H5C_DO_SANITY_CHECKS */ + +done: + +#if H5C_DO_EXTREME_SANITY_CHECKS + if ( ( H5C_validate_protected_entry_list(cache_ptr) < 0 ) || + ( H5C_validate_pinned_entry_list(cache_ptr) < 0 ) || + ( H5C_validate_lru_list(cache_ptr) < 0 ) ) { + + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ + "an extreme sanity check failed on exit.\n"); + } +#endif /* H5C_DO_EXTREME_SANITY_CHECKS */ + + FUNC_LEAVE_NOAPI(ret_value) + +} /* H5C_mark_entries_as_clean() */ + + +/*------------------------------------------------------------------------- + * + * Function: H5C_clear_coll_entries + * + * Purpose: Clear half or the entire list of collective entries and + * mark them as independent. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: Mohamad Chaarawi + * April, 2015 + * + *------------------------------------------------------------------------- + */ +herr_t +H5C_clear_coll_entries(H5C_t * cache_ptr, hbool_t partial) +{ + int32_t list_len, coll_entries_cleared = 0; + H5C_cache_entry_t * entry_ptr = NULL; + H5C_cache_entry_t * prev_ptr; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + entry_ptr = cache_ptr->coll_tail_ptr; + list_len = cache_ptr->coll_list_len; + + while(entry_ptr && (coll_entries_cleared < (partial ? list_len/2 : list_len))) { + prev_ptr = entry_ptr->coll_prev; + + HDassert(entry_ptr->coll_access); + + entry_ptr->coll_access = FALSE; + H5C__REMOVE_FROM_COLL_LIST(cache_ptr, entry_ptr, FAIL) + coll_entries_cleared ++; + + entry_ptr = prev_ptr; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C_clear_coll_entries */ + +herr_t +H5C_collective_write(H5F_t *f, + hid_t dxpl_id, + H5SL_t *collective_write_list) +{ + H5P_genplist_t *plist = NULL; + H5FD_mpio_xfer_t xfer_mode = H5FD_MPIO_COLLECTIVE; + H5FD_mpio_xfer_t orig_xfer_mode; + H5SL_node_t *node; + H5C_collective_write_t *item; + int count; + void *base_buf; + int *length_array = NULL; + MPI_Aint *buf_array = NULL; + MPI_Aint *offset_array = NULL; + MPI_Datatype btype; + MPI_Datatype ftype; + hbool_t btype_created = FALSE; + hbool_t ftype_created = FALSE; + int mpi_code; + int i; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + count = (int)H5SL_count(collective_write_list); + + if(NULL == (plist = (H5P_genplist_t *)H5I_object(dxpl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data transfer property list") + + if(H5P_get(plist, H5D_XFER_IO_XFER_MODE_NAME, &orig_xfer_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O property") + + if(count > 0) { + if(H5P_set(plist, H5D_XFER_IO_XFER_MODE_NAME, &xfer_mode) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O property") + + /* Allocate arrays */ + if(NULL == (length_array = (int *)H5MM_malloc((size_t)count * sizeof(int)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for collective write table length array") + if(NULL == (buf_array = (MPI_Aint *)H5MM_malloc((size_t)count * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for collective buf table length array") + if(NULL == (offset_array = (MPI_Aint *)H5MM_malloc((size_t)count * sizeof(MPI_Aint)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for collective offset table length array") + + /* Fill arrays */ + node = H5SL_first(collective_write_list); + HDassert(node); + if(NULL == (item = (H5C_collective_write_t *)H5SL_item(node))) + HGOTO_ERROR(H5E_CACHE, H5E_NOTFOUND, FAIL, "can't retrieve skip list item") + + length_array[0] = (int)item->length; + base_buf = item->buf; + buf_array[0] = (MPI_Aint)0; + offset_array[0] = (MPI_Aint)item->offset; + + node = H5SL_next(node); + i = 1; + while(node) { + if(NULL == (item = (H5C_collective_write_t *)H5SL_item(node))) + HGOTO_ERROR(H5E_CACHE, H5E_NOTFOUND, FAIL, "can't retrieve skip list item") + + length_array[i] = (int)item->length; + buf_array[i] = (MPI_Aint)item->buf - (MPI_Aint)base_buf; + offset_array[i] = (MPI_Aint)item->offset; + node = H5SL_next(node); + i++; + } /* end while */ + + /* Create memory mpi type */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed(count, length_array, buf_array, MPI_BYTE, &btype))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + btype_created = TRUE; + if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&btype))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + + /* Create file mpi type */ + if(MPI_SUCCESS != (mpi_code = MPI_Type_create_hindexed(count, length_array, offset_array, MPI_BYTE, &ftype))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_create_hindexed failed", mpi_code) + ftype_created = TRUE; + if(MPI_SUCCESS != (mpi_code = MPI_Type_commit(&ftype))) + HMPI_GOTO_ERROR(FAIL, "MPI_Type_commit failed", mpi_code) + + /* Pass buf type, file type to the file driver. */ + if(H5FD_mpi_setup_collective(dxpl_id, &btype, &ftype) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O properties") + + /* Write data */ + if(H5F_block_write(f, H5FD_MEM_DEFAULT, (haddr_t)0, (size_t)1, dxpl_id, base_buf) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to write entries collectively") + } /* end if */ + else { + MPI_Status mpi_stat; + MPI_File mpi_fh_p; + MPI_File mpi_fh; + + if(H5F_get_mpi_handle(f, (MPI_File **)&mpi_fh_p) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "can't get mpi file handle") + mpi_fh = *(MPI_File*)mpi_fh_p; + + /* just to match up with the 1st MPI_File_set_view from H5FD_mpio_write() */ + if(MPI_SUCCESS != (mpi_code = MPI_File_set_view(mpi_fh, (MPI_Offset)0, MPI_BYTE, + MPI_BYTE, "native", MPI_INFO_NULL))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) + + /* just to match up with MPI_File_write_at_all from H5FD_mpio_write() */ + HDmemset(&mpi_stat, 0, sizeof(MPI_Status)); + if(MPI_SUCCESS != (mpi_code = MPI_File_write_at_all(mpi_fh, (MPI_Offset)0, NULL, 0, + MPI_BYTE, &mpi_stat))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_write_at_all failed", mpi_code) + + /* just to match up with the 2nd MPI_File_set_view (reset) in H5FD_mpio_write() */ + if(MPI_SUCCESS != (mpi_code = MPI_File_set_view(mpi_fh, (MPI_Offset)0, MPI_BYTE, + MPI_BYTE, "native", MPI_INFO_NULL))) + HMPI_GOTO_ERROR(FAIL, "MPI_File_set_view failed", mpi_code) + } /* end else */ + +done: + /* Free arrays */ + length_array = (int *)H5MM_xfree(length_array); + buf_array = (MPI_Aint *)H5MM_xfree(buf_array); + offset_array = (MPI_Aint *)H5MM_xfree(offset_array); + + /* Free MPI Types */ + if(btype_created && MPI_SUCCESS != (mpi_code = MPI_Type_free(&btype))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + if(ftype_created && MPI_SUCCESS != (mpi_code = MPI_Type_free(&ftype))) + HMPI_DONE_ERROR(FAIL, "MPI_Type_free failed", mpi_code) + + /* Reset dxpl */ + if(orig_xfer_mode != H5FD_MPIO_COLLECTIVE) { + HDassert(plist); + if(H5P_set(plist, H5D_XFER_IO_XFER_MODE_NAME, &orig_xfer_mode) < 0) + HDONE_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set MPI-I/O property") + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5C_collective_write() */ + +herr_t +H5C_collective_write_free(void *_item, void H5_ATTR_UNUSED *key, void H5_ATTR_UNUSED *op_data) +{ + H5C_collective_write_t *item = (H5C_collective_write_t *)_item; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(item); + + if(item->free_buf) + item->buf = H5MM_xfree(item->buf); + /*!FIXME change to use free list for items */ + H5MM_free(item); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5C_collective_write_free() */ +#endif /* H5_HAVE_PARALLEL */ + diff --git a/src/H5Cpkg.h b/src/H5Cpkg.h index 3e15bca..22a964e 100644 --- a/src/H5Cpkg.h +++ b/src/H5Cpkg.h @@ -60,8 +60,6 @@ /* Cache configuration settings */ #define H5C__HASH_TABLE_LEN (64 * 1024) /* must be a power of 2 */ #define H5C__H5C_T_MAGIC 0x005CAC0E -#define H5C__MAX_NUM_TYPE_IDS 28 -#define H5C__PREFIX_LEN 32 /**************************************************************************** * @@ -595,7 +593,7 @@ if ( ( (entry_ptr) == NULL ) || \ #define H5C__UPDATE_STATS_FOR_CLEAR(cache_ptr, entry_ptr) \ { \ (((cache_ptr)->clears)[(entry_ptr)->type->id])++; \ - if ( (entry_ptr)->is_pinned ) \ + if((entry_ptr)->is_pinned) \ (((cache_ptr)->pinned_clears)[(entry_ptr)->type->id])++; \ ((entry_ptr)->clears)++; \ } @@ -709,9 +707,9 @@ if ( ( (entry_ptr) == NULL ) || \ #define H5C__UPDATE_STATS_FOR_CLEAR(cache_ptr, entry_ptr) \ { \ - if ( (entry_ptr)->is_pinned ) \ - (((cache_ptr)->pinned_clears)[(entry_ptr)->type->id])++; \ (((cache_ptr)->clears)[(entry_ptr)->type->id])++; \ + if((entry_ptr)->is_pinned) \ + (((cache_ptr)->pinned_clears)[(entry_ptr)->type->id])++; \ } #define H5C__UPDATE_STATS_FOR_FLUSH(cache_ptr, entry_ptr) \ @@ -2713,6 +2711,7 @@ if ( ( (cache_ptr)->index_size != \ #ifdef H5_HAVE_PARALLEL #if H5C_DO_SANITY_CHECKS + #define H5C__COLL_DLL_PRE_REMOVE_SC(entry_ptr, hd_ptr, tail_ptr, len, Size, fv) \ if ( ( (hd_ptr) == NULL ) || \ ( (tail_ptr) == NULL ) || \ @@ -4017,6 +4016,15 @@ struct H5C_t { #endif /* NDEBUG */ }; +#ifdef H5_HAVE_PARALLEL +typedef struct H5C_collective_write_t { + size_t length; + hbool_t free_buf; + void *buf; + haddr_t offset; +} H5C_collective_write_t; +#endif /* H5_HAVE_PARALLEL */ + /*****************************/ /* Package Private Variables */ /*****************************/ @@ -4025,7 +4033,12 @@ struct H5C_t { /******************************/ /* Package Private Prototypes */ /******************************/ - +H5_DLL herr_t H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, + H5C_cache_entry_t *entry_ptr, unsigned flags, int64_t *entry_size_change_ptr, H5SL_t *collective_write_list); +#ifdef H5_HAVE_PARALLEL +H5_DLL herr_t H5C_collective_write(H5F_t *f, hid_t dxpl_id, H5SL_t *collective_write_list); +H5_DLL herr_t H5C_collective_write_free(void *_item, void *key, void *op_data); +#endif /* H5_HAVE_PARALLEL */ #endif /* _H5Cpkg_H */ diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index 9e29a7f..2ad2c6a 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -40,30 +40,9 @@ /* Library Private Macros */ /**************************/ -#ifndef NDEBUG -#define H5C_DO_SANITY_CHECKS 1 -#define H5C_DO_SLIST_SANITY_CHECKS 0 -#define H5C_DO_TAGGING_SANITY_CHECKS 1 -#define H5C_DO_EXTREME_SANITY_CHECKS 0 -#else /* NDEBUG */ -/* With rare execptions, the following defines should be set - * to 0 if NDEBUG is defined - */ -#define H5C_DO_SANITY_CHECKS 0 -#define H5C_DO_SLIST_SANITY_CHECKS 0 -#define H5C_DO_TAGGING_SANITY_CHECKS 0 -#define H5C_DO_EXTREME_SANITY_CHECKS 0 -#endif /* NDEBUG */ - -/* Note: The memory sanity checks aren't going to work until I/O filters are - * changed to call a particular alloc/free routine for their buffers, - * because the H5AC__SERIALIZE_RESIZED_FLAG set by the fractal heap - * direct block serialize callback calls H5Z_pipeline(). When the I/O - * filters are changed, then we should implement "cache image alloc/free" - * routines that the fractal heap direct block (and global heap) serialize - * calls can use when resizing (and re-allocating) their image in the - * cache. -QAK */ -#define H5C_DO_MEMORY_SANITY_CHECKS 0 +/* Cache configuration settings */ +#define H5C__MAX_NUM_TYPE_IDS 28 +#define H5C__PREFIX_LEN 32 /* This sanity checking constant was picked out of the air. Increase * or decrease it if appropriate. Its purposes is to detect corrupt @@ -73,29 +52,6 @@ */ #define H5C_MAX_ENTRY_SIZE ((size_t)(32 * 1024 * 1024)) -/* H5C_COLLECT_CACHE_STATS controls overall collection of statistics - * on cache activity. In general, this #define should be set to 1 in - * debug mode, and 0 in production mode.. - */ - -#ifndef NDEBUG -#define H5C_COLLECT_CACHE_STATS 1 -#else /* NDEBUG */ -#define H5C_COLLECT_CACHE_STATS 0 -#endif /* NDEBUG */ - -/* H5C_COLLECT_CACHE_ENTRY_STATS controls collection of statistics - * in individual cache entries. - * - * H5C_COLLECT_CACHE_ENTRY_STATS should only be defined to true if - * H5C_COLLECT_CACHE_STATS is also defined to true. - */ -#if H5C_COLLECT_CACHE_STATS -#define H5C_COLLECT_CACHE_ENTRY_STATS 1 -#else -#define H5C_COLLECT_CACHE_ENTRY_STATS 0 -#endif /* H5C_COLLECT_CACHE_STATS */ - #ifdef H5_HAVE_PARALLEL /* we must maintain the clean and dirty LRU lists when we are compiled * with parallel support. @@ -236,6 +192,7 @@ * H5C__FLUSH_CLEAR_ONLY_FLAG * H5C__FLUSH_MARKED_ENTRIES_FLAG * H5C__TAKE_OWNERSHIP_FLAG + * H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG */ #define H5C__NO_FLAGS_SET 0x0000 #define H5C__SET_FLUSH_MARKER_FLAG 0x0001 @@ -252,6 +209,56 @@ #define H5C__TAKE_OWNERSHIP_FLAG 0x1000 #define H5C__FLUSH_LAST_FLAG 0x2000 #define H5C__FLUSH_COLLECTIVELY_FLAG 0x4000 +#define H5C__DEL_FROM_SLIST_ON_DESTROY_FLAG 0x8000 + +/* Debugging/sanity checking/statistics settings */ +#ifndef NDEBUG +#define H5C_DO_SANITY_CHECKS 1 +#define H5C_DO_SLIST_SANITY_CHECKS 0 +#define H5C_DO_TAGGING_SANITY_CHECKS 1 +#define H5C_DO_EXTREME_SANITY_CHECKS 0 +#else /* NDEBUG */ +/* With rare execptions, the following defines should be set + * to 0 if NDEBUG is defined + */ +#define H5C_DO_SANITY_CHECKS 0 +#define H5C_DO_SLIST_SANITY_CHECKS 0 +#define H5C_DO_TAGGING_SANITY_CHECKS 0 +#define H5C_DO_EXTREME_SANITY_CHECKS 0 +#endif /* NDEBUG */ + +/* Note: The memory sanity checks aren't going to work until I/O filters are + * changed to call a particular alloc/free routine for their buffers, + * because the H5AC__SERIALIZE_RESIZED_FLAG set by the fractal heap + * direct block serialize callback calls H5Z_pipeline(). When the I/O + * filters are changed, then we should implement "cache image alloc/free" + * routines that the fractal heap direct block (and global heap) serialize + * calls can use when resizing (and re-allocating) their image in the + * cache. -QAK */ +#define H5C_DO_MEMORY_SANITY_CHECKS 0 + +/* H5C_COLLECT_CACHE_STATS controls overall collection of statistics + * on cache activity. In general, this #define should be set to 1 in + * debug mode, and 0 in production mode.. + */ + +#ifndef NDEBUG +#define H5C_COLLECT_CACHE_STATS 1 +#else /* NDEBUG */ +#define H5C_COLLECT_CACHE_STATS 0 +#endif /* NDEBUG */ + +/* H5C_COLLECT_CACHE_ENTRY_STATS controls collection of statistics + * in individual cache entries. + * + * H5C_COLLECT_CACHE_ENTRY_STATS should only be defined to true if + * H5C_COLLECT_CACHE_STATS is also defined to true. + */ +#if H5C_COLLECT_CACHE_STATS +#define H5C_COLLECT_CACHE_ENTRY_STATS 1 +#else +#define H5C_COLLECT_CACHE_ENTRY_STATS 0 +#endif /* H5C_COLLECT_CACHE_STATS */ /****************************/ @@ -1883,12 +1890,11 @@ H5_DLL herr_t H5C_get_entry_status(const H5F_t *f, haddr_t addr, hbool_t *is_protected_ptr, hbool_t *is_pinned_ptr, hbool_t *is_flush_dep_parent_ptr, hbool_t *is_flush_dep_child_ptr); H5_DLL herr_t H5C_get_evictions_enabled(const H5C_t *cache_ptr, hbool_t *evictions_enabled_ptr); +H5_DLL void * H5C_get_aux_ptr(const H5C_t *cache_ptr); H5_DLL FILE *H5C_get_trace_file_ptr(const H5C_t *cache_ptr); H5_DLL FILE *H5C_get_trace_file_ptr_from_entry(const H5C_cache_entry_t *entry_ptr); H5_DLL herr_t H5C_insert_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, haddr_t addr, void *thing, unsigned int flags); -H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t *f, hid_t dxpl_id, int32_t ce_array_len, - haddr_t *ce_array_ptr); H5_DLL herr_t H5C_mark_entry_dirty(void *thing); H5_DLL herr_t H5C_move_entry(H5C_t *cache_ptr, const H5C_class_t *type, haddr_t old_addr, haddr_t new_addr); @@ -1909,8 +1915,8 @@ H5_DLL void H5C_stats__reset(H5C_t *cache_ptr); H5_DLL herr_t H5C_dump_cache(H5C_t *cache_ptr, const char *cache_name); H5_DLL herr_t H5C_unpin_entry(void *thing); H5_DLL herr_t H5C_destroy_flush_dependency(void *parent_thing, void *child_thing); -H5_DLL herr_t H5C_unprotect(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, - haddr_t addr, void *thing, unsigned int flags); +H5_DLL herr_t H5C_unprotect(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *thing, + unsigned int flags); H5_DLL herr_t H5C_validate_resize_config(H5C_auto_size_ctl_t *config_ptr, unsigned int tests); H5_DLL herr_t H5C_ignore_tags(H5C_t *cache_ptr); @@ -1923,6 +1929,8 @@ H5_DLL herr_t H5C_apply_candidate_list(H5F_t *f, hid_t dxpl_id, H5_DLL herr_t H5C_construct_candidate_list__clean_cache(H5C_t *cache_ptr); H5_DLL herr_t H5C_construct_candidate_list__min_clean(H5C_t *cache_ptr); H5_DLL herr_t H5C_clear_coll_entries(H5C_t * cache_ptr, hbool_t partial); +H5_DLL herr_t H5C_mark_entries_as_clean(H5F_t *f, hid_t dxpl_id, int32_t ce_array_len, + haddr_t *ce_array_ptr); #endif /* H5_HAVE_PARALLEL */ #ifndef NDEBUG /* debugging functions */ diff --git a/src/H5EAcache.c b/src/H5EAcache.c index 0d83fb3..8a05282 100644 --- a/src/H5EAcache.c +++ b/src/H5EAcache.c @@ -488,7 +488,7 @@ H5EA__cache_hdr_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED le UINT32ENCODE(image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); END_FUNC(STATIC) /* end H5EA__cache_hdr_serialize() */ @@ -785,7 +785,7 @@ H5EA__cache_iblock_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED UINT32ENCODE(image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); CATCH @@ -1141,7 +1141,7 @@ H5EA__cache_sblock_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED UINT32ENCODE(image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); END_FUNC(STATIC) /* end H5EA__cache_sblock_serialize() */ @@ -1494,7 +1494,7 @@ H5EA__cache_dblock_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED UINT32ENCODE(image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); CATCH @@ -1816,7 +1816,7 @@ H5EA__cache_dblk_page_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNU UINT32ENCODE(image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); CATCH diff --git a/src/H5Eprivate.h b/src/H5Eprivate.h index 2065b08..ba17ada 100644 --- a/src/H5Eprivate.h +++ b/src/H5Eprivate.h @@ -186,7 +186,7 @@ H5_DLL herr_t H5E_init(void); H5_DLL herr_t H5E_push_stack(H5E_t *estack, const char *file, const char *func, unsigned line, hid_t cls_id, hid_t maj_id, hid_t min_id, const char *desc); H5_DLL herr_t H5E_printf_stack(H5E_t *estack, const char *file, const char *func, - unsigned line, hid_t cls_id, hid_t maj_id, hid_t min_id, const char *fmt, ...)__attribute__((format (printf, 8, 9))); + unsigned line, hid_t cls_id, hid_t maj_id, hid_t min_id, const char *fmt, ...)H5_ATTR_FORMAT(printf, 8, 9); H5_DLL herr_t H5E_clear_stack(H5E_t *estack); H5_DLL herr_t H5E_dump_api_stack(hbool_t is_api); diff --git a/src/H5FAcache.c b/src/H5FAcache.c index 298eb56..b95da40 100644 --- a/src/H5FAcache.c +++ b/src/H5FAcache.c @@ -407,7 +407,7 @@ H5FA__cache_hdr_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED le UINT32ENCODE(image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); END_FUNC(STATIC) /* end H5FA__cache_hdr_serialize() */ @@ -704,7 +704,7 @@ H5FA__cache_dblock_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED UINT32ENCODE(image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); CATCH @@ -976,7 +976,7 @@ H5FA__cache_dblk_page_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNU UINT32ENCODE(image, metadata_chksum); /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); CATCH diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index cf14770..ac1f840 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -470,7 +470,7 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len, hbool_t * *image_len = (size_t)H5F_SUPERBLOCK_SIZE(sblock); /* Set *compressed_ptr to FALSE unconditionally */ - *compressed_ptr; + *compressed_ptr = FALSE; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5F__cache_superblock_image_len() */ @@ -716,7 +716,7 @@ H5F__cache_superblock_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNU } /* end else */ /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); done: FUNC_LEAVE_NOAPI(ret_value) @@ -976,7 +976,7 @@ H5F__cache_drvrinfo_serialize(const H5F_t *f, void *_image, size_t len, image += 8 + drvinfo->len; /* Sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= len); + HDassert((size_t)(image - (uint8_t *)_image) == len); done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Gcache.c b/src/H5Gcache.c index 8dfecb1..f765b41 100644 --- a/src/H5Gcache.c +++ b/src/H5Gcache.c @@ -316,8 +316,10 @@ H5G__cache_node_serialize(const H5F_t *f, void *_image, size_t len, if(H5G__ent_encode_vec(f, &image, sym->entry, sym->nsyms) < 0) HGOTO_ERROR(H5E_SYM, H5E_CANTENCODE, FAIL, "can't serialize") +#ifdef H5_CLEAR_MEMORY /* Clear rest of symbol table node */ - HDmemset(image, 0, sym->node_size - (size_t)(image - (uint8_t *)_image)); + HDmemset(image, 0, len - (size_t)(image - (uint8_t *)_image)); +#endif /* H5_CLEAR_MEMORY */ done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5HLcache.c b/src/H5HLcache.c index 17ded00..2054e34 100644 --- a/src/H5HLcache.c +++ b/src/H5HLcache.c @@ -560,7 +560,19 @@ H5HL__cache_prefix_serialize(const H5F_t *f, void *_image, size_t len, /* Copy the heap data block into the cache image */ HDmemcpy(image, heap->dblk_image, heap->dblk_size); + + /* Sanity check */ + HDassert((size_t)(image - (uint8_t *)_image) + heap->dblk_size == len); } /* end if */ + else { + /* Sanity check */ + HDassert((size_t)(image - (uint8_t *)_image) <= len); + +#ifdef H5_CLEAR_MEMORY + /* Clear rest of local heap */ + HDmemset(image, 0, len - (size_t)(image - (uint8_t *)_image)); +#endif /* H5_CLEAR_MEMORY */ + } /* end else */ FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HL__cache_prefix_serialize() */ @@ -99,6 +99,13 @@ typedef struct { hbool_t app_ref; /* Whether this is an appl. ref. call */ } H5I_iterate_ud_t; +/* User data for H5I__clear_type_cb */ +typedef struct { + H5I_id_type_t *type_ptr; /* Pointer to the type being cleard */ + hbool_t force; /* Whether to always remove the id */ + hbool_t app_ref; /* Whether this is an appl. ref. call */ +} H5I_clear_type_ud_t; + /*-------------------- Locally scoped variables -----------------------------*/ /* Array of pointers to atomic types */ @@ -122,6 +129,7 @@ H5FL_DEFINE_STATIC(H5I_id_type_t); H5FL_DEFINE_STATIC(H5I_class_t); /*--------------------- Local function prototypes ---------------------------*/ +static htri_t H5I__clear_type_cb(void *_id, void *key, void *udata); static int H5I__destroy_type(H5I_type_t type); static void *H5I__remove_verify(hid_t id, H5I_type_t id_type); static void *H5I__remove_common(H5I_id_type_t *type_ptr, hid_t id); @@ -528,80 +536,98 @@ done: herr_t H5I_clear_type(H5I_type_t type, hbool_t force, hbool_t app_ref) { - H5I_id_type_t *type_ptr; /* ptr to the atomic type */ - H5SL_node_t *curr_node; /* Current skip list node ptr */ - H5SL_node_t *next_node; /* Next skip list node ptr */ - int ret_value = SUCCEED; /* Return value */ + H5I_clear_type_ud_t udata; /* udata struct for callback */ + int ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) if(type <= H5I_BADID || type >= H5I_next_type) - HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid type number") + HGOTO_ERROR(H5E_ARGS, H5E_BADRANGE, FAIL, "invalid type number") - type_ptr = H5I_id_type_list_g[type]; - if(type_ptr == NULL || type_ptr->init_count <= 0) - HGOTO_ERROR(H5E_ATOM, H5E_BADGROUP, FAIL, "invalid type") + udata.type_ptr = H5I_id_type_list_g[type]; + if(udata.type_ptr == NULL || udata.type_ptr->init_count <= 0) + HGOTO_ERROR(H5E_ATOM, H5E_BADGROUP, FAIL, "invalid type") - /* - * Call free method for all objects in type regardless of their reference - * counts. Ignore the return value from from the free method and remove - * object from type regardless if FORCE is non-zero. - */ - for(curr_node = H5SL_first(type_ptr->ids); curr_node; curr_node = next_node) { - H5I_id_info_t *cur; /* Current ID being worked with */ - hbool_t delete_node; /* Flag to indicate node should be removed from linked list */ + /* Finish constructing udata */ + udata.force = force; + udata.app_ref = app_ref; - /* Get ID for this node */ - if(NULL == (cur = (H5I_id_info_t *)H5SL_item(curr_node))) - HGOTO_ERROR(H5E_ATOM, H5E_CANTGET, FAIL, "can't get ID info for node") + /* Attempt to free all ids in the type */ + if(H5SL_try_free_safe(udata.type_ptr->ids, H5I__clear_type_cb, &udata) < 0) + HGOTO_ERROR(H5E_ATOM, H5E_CANTDELETE, FAIL, "can't free ids in type") - /* - * Do nothing to the object if the reference count is larger than - * one and forcing is off. - */ - if(!force && (cur->count - (!app_ref * cur->app_count)) > 1) - delete_node = FALSE; - else { - /* Check for a 'free' function and call it, if it exists */ - /* (Casting away const OK -QAK) */ - if(type_ptr->cls->free_func && (type_ptr->cls->free_func)((void *)cur->obj_ptr) < 0) { - if(force) { +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5I_clear_type() */ + + +/*------------------------------------------------------------------------- + * Function: H5I__clear_type_cb + * + * Purpose: Attempts to free the specified ID , calling the free + * function for the object. + * + * Return: Success: Non-negative + * Failure: negative + * + * Programmer: Neil Fortner + * Friday, July 10, 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5I__clear_type_cb(void *_id, void H5_ATTR_UNUSED *key, void *_udata) +{ + H5I_id_info_t *id = (H5I_id_info_t *)_id; /* Current ID being worked with */ + H5I_clear_type_ud_t *udata = (H5I_clear_type_ud_t *)_udata; /* udata struct */ + htri_t ret_value = FALSE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + HDassert(id); + HDassert(udata); + HDassert(udata->type_ptr); + + /* + * Do nothing to the object if the reference count is larger than + * one and forcing is off. + */ + if(udata->force || (id->count - (!udata->app_ref * id->app_count)) <= 1) { + /* Check for a 'free' function and call it, if it exists */ + /* (Casting away const OK -QAK) */ + if(udata->type_ptr->cls->free_func && (udata->type_ptr->cls->free_func)((void *)id->obj_ptr) < 0) { + if(udata->force) { #ifdef H5I_DEBUG - if(H5DEBUG(I)) { - fprintf(H5DEBUG(I), "H5I: free type=%d obj=0x%08lx " - "failure ignored\n", (int)type, - (unsigned long)(cur->obj_ptr)); - } /* end if */ + if(H5DEBUG(I)) { + fprintf(H5DEBUG(I), "H5I: free type=%d obj=0x%08lx " + "failure ignored\n", + (int)udata->type_ptr->cls->type_id, + (unsigned long)(id->obj_ptr)); + } /* end if */ #endif /*H5I_DEBUG*/ - /* Indicate node should be removed from list */ - delete_node = TRUE; - } /* end if */ - else { - /* Indicate node should _NOT_ be remove from list */ - delete_node = FALSE; - } /* end else */ - } /* end if */ - else { /* Indicate node should be removed from list */ - delete_node = TRUE; - } /* end else */ + ret_value = TRUE; + } /* end if */ + } /* end if */ + else { + /* Indicate node should be removed from list */ + ret_value = TRUE; } /* end else */ - /* Get the next node in the list */ - next_node = H5SL_next(curr_node); + /* Remove ID if requested */ + if(ret_value) { + /* Free ID info */ + id = H5FL_FREE(H5I_id_info_t, id); - /* Check if we should delete this node or not */ - if(delete_node) { - /* Remove the node from the type */ - if(NULL == H5I__remove_common(type_ptr, cur->id)) - HGOTO_ERROR(H5E_ATOM, H5E_CANTDELETE, FAIL, "can't remove ID node") + /* Decrement the number of IDs in the type */ + udata->type_ptr->id_count--; } /* end if */ - } /* end for */ + } /* end if */ + -done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5I_clear_type() */ +} /* end H5I__clear_type_cb() */ /*------------------------------------------------------------------------- @@ -44,6 +44,9 @@ #include "H5Iprivate.h" /* IDs */ #include "H5Lprivate.h" /* Links */ #include "H5MFprivate.h" /* File memory management */ +#ifdef H5O_ENABLE_BOGUS +#include "H5MMprivate.h" /* Memory management */ +#endif /* H5O_ENABLE_BOGUS */ #include "H5Opkg.h" /* Object headers */ #include "H5SMprivate.h" /* Shared object header messages */ @@ -196,10 +199,10 @@ H5O_init_interface(void) FUNC_ENTER_NOAPI_NOINIT_NOERR /* H5O interface sanity checks */ - HDassert(H5O_MSG_TYPES == NELMTS(H5O_msg_class_g)); - HDassert(sizeof(H5O_fheap_id_t) == H5O_FHEAP_ID_LEN); + HDcompile_assert(H5O_MSG_TYPES == NELMTS(H5O_msg_class_g)); + HDcompile_assert(sizeof(H5O_fheap_id_t) == H5O_FHEAP_ID_LEN); - HDassert(H5O_UNKNOWN_ID < H5O_MSG_TYPES); + HDcompile_assert(H5O_UNKNOWN_ID < H5O_MSG_TYPES); FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5O_init_interface() */ diff --git a/src/H5Ocache.c b/src/H5Ocache.c index db1bd1f..203d0fc 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -1130,7 +1130,7 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image id = *chunk_image++; /* Check for unknown message ID getting encoded in file */ - if(id == H5O_UNKNOWN_ID) + if(id >= H5O_UNKNOWN_ID) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "'unknown' message ID encoded in file?!?") /* Message size */ @@ -1218,9 +1218,10 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image /* Set message to "unknown" class */ oh->mesg[mesgno].type = H5O_msg_class_g[H5O_UNKNOWN_ID]; - /* Check for "fail if unknown" message flag */ - if((udata->file_intent & H5F_ACC_RDWR) && - (flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE)) + /* Check for "fail if unknown" message flags */ + if(((udata->file_intent & H5F_ACC_RDWR) && + (flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE)) + || (flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS)) HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, FAIL, "unknown message with 'fail if unknown' flag found") /* Check for "mark if unknown" message flag, etc. */ else if((flags & H5O_MSG_FLAG_MARK_IF_UNKNOWN) && diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 92d0107..dbe74f7 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -75,7 +75,8 @@ typedef struct H5O_t H5O_t; #define H5O_MSG_FLAG_MARK_IF_UNKNOWN 0x10u #define H5O_MSG_FLAG_WAS_UNKNOWN 0x20u #define H5O_MSG_FLAG_SHAREABLE 0x40u -#define H5O_MSG_FLAG_BITS (H5O_MSG_FLAG_CONSTANT|H5O_MSG_FLAG_SHARED|H5O_MSG_FLAG_DONTSHARE|H5O_MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE|H5O_MSG_FLAG_MARK_IF_UNKNOWN|H5O_MSG_FLAG_WAS_UNKNOWN|H5O_MSG_FLAG_SHAREABLE) +#define H5O_MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS 0x80u +#define H5O_MSG_FLAG_BITS (H5O_MSG_FLAG_CONSTANT|H5O_MSG_FLAG_SHARED|H5O_MSG_FLAG_DONTSHARE|H5O_MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE|H5O_MSG_FLAG_MARK_IF_UNKNOWN|H5O_MSG_FLAG_WAS_UNKNOWN|H5O_MSG_FLAG_SHAREABLE|H5O_MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS) /* Flags for updating messages */ #define H5O_UPDATE_TIME 0x01u @@ -84,6 +85,9 @@ typedef struct H5O_t H5O_t; /* Hash value constants */ #define H5O_HASH_SIZE 32 +/* Enable reading/writing "bogus" messages */ +/* #define H5O_ENABLE_BOGUS */ + /* ========= Object Creation properties ============ */ #define H5O_CRT_ATTR_MAX_COMPACT_NAME "max compact attr" /* Max. # of attributes to store compactly */ #define H5O_CRT_ATTR_MIN_DENSE_NAME "min dense attr" /* Min. # of attributes to store densely */ @@ -196,7 +200,6 @@ typedef struct H5O_copy_t { #define H5O_UNKNOWN_ID 0x0018 /* Placeholder message ID for unknown message. */ /* (this should never exist in a file) */ - /* Shared object message types. * Shared objects can be committed, in which case the shared message contains * the location of the object header that holds the message, or shared in the @@ -439,9 +442,6 @@ typedef struct H5O_layout_t { H5O_storage_t storage; /* Information for storing dataset elements */ } H5O_layout_t; -/* Enable reading/writing "bogus" messages */ -/* #define H5O_ENABLE_BOGUS */ - #ifdef H5O_ENABLE_BOGUS /* * "Bogus" Message. diff --git a/src/H5Oshared.h b/src/H5Oshared.h index 3ec5709..2ac2a98 100644 --- a/src/H5Oshared.h +++ b/src/H5Oshared.h @@ -47,7 +47,7 @@ * *------------------------------------------------------------------------- */ -static H5_inline void * +static H5_INLINE void * H5O_SHARED_DECODE(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, unsigned mesg_flags, unsigned *ioflags, const uint8_t *p) { @@ -107,7 +107,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline herr_t +static H5_INLINE herr_t H5O_SHARED_ENCODE(H5F_t *f, hbool_t disable_shared, uint8_t *p, const void *_mesg) { const H5O_shared_t *sh_mesg = (const H5O_shared_t *)_mesg; /* Pointer to shared message portion of actual message */ @@ -162,7 +162,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline size_t +static H5_INLINE size_t H5O_SHARED_SIZE(const H5F_t *f, hbool_t disable_shared, const void *_mesg) { const H5O_shared_t *sh_mesg = (const H5O_shared_t *)_mesg; /* Pointer to shared message portion of actual message */ @@ -215,7 +215,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline herr_t +static H5_INLINE herr_t H5O_SHARED_DELETE(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, void *_mesg) { H5O_shared_t *sh_mesg = (H5O_shared_t *)_mesg; /* Pointer to shared message portion of actual message */ @@ -267,7 +267,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline herr_t +static H5_INLINE herr_t H5O_SHARED_LINK(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, void *_mesg) { H5O_shared_t *sh_mesg = (H5O_shared_t *)_mesg; /* Pointer to shared message portion of actual message */ @@ -318,7 +318,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline void * +static H5_INLINE void * H5O_SHARED_COPY_FILE(H5F_t *file_src, void *_native_src, H5F_t *file_dst, hbool_t *recompute_size, unsigned *mesg_flags, H5O_copy_t *cpy_info, void *udata, hid_t dxpl_id) @@ -382,7 +382,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline herr_t +static H5_INLINE herr_t H5O_SHARED_POST_COPY_FILE(const H5O_loc_t *oloc_src, const void *mesg_src, H5O_loc_t *oloc_dst, void *mesg_dst, unsigned *mesg_flags, hid_t dxpl_id, H5O_copy_t *cpy_info) @@ -453,7 +453,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline herr_t +static H5_INLINE herr_t H5O_SHARED_DEBUG(H5F_t *f, hid_t dxpl_id, const void *_mesg, FILE *stream, int indent, int fwidth) { @@ -74,11 +74,26 @@ /* Define the code template for searches for the "OP" in the H5SL_LOCATE macro */ #define H5SL_LOCATE_SEARCH_FOUND(SLIST, X, I) \ - HGOTO_DONE(X->item); +{ \ + HDassert(!X->removed); \ + HGOTO_DONE(X->item); \ +} /* end block */ + +/* Define the code template for deferred removals for the "OP" in the + * H5SL_LOCATE macro */ +#define H5SL_LOCATE_SEARCH_DEFER_REMOVE_FOUND(SLIST, X, I) \ +{ \ + HDassert(!X->removed); \ + X->removed = TRUE; \ + HGOTO_DONE(X->item); \ +} /* end block */ /* Define the code template for finds for the "OP" in the H5SL_LOCATE macro */ #define H5SL_LOCATE_FIND_FOUND(SLIST, X, I) \ - HGOTO_DONE(X); +{ \ + HDassert(!X->removed); \ + HGOTO_DONE(X); \ +} /* end block */ /* Define a code template for comparing scalar keys for the "CMP" in the H5SL_LOCATE macro */ @@ -133,8 +148,8 @@ #define H5SL_LOCATE_GENERIC_HASHINIT(KEY, HASHVAL) -/* Macro used to find node for operation */ -#define H5SL_LOCATE(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ +/* Macro used to find node for operation, if all keys are valid */ +#define H5SL_LOCATE_OPT(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ { \ int _i; /* Local index variable */ \ unsigned _count; /* Num nodes searched at this height */ \ @@ -150,15 +165,53 @@ } /* end for */ \ X = X->forward[0]; \ if(X != NULL && H5_GLUE3(H5SL_LOCATE_,CMP,_EQ)(SLIST, TYPE, X, KEY, HASHVAL) ) { \ - /* What to do when a node is found */ \ + /* What to do when a node is found */ \ H5_GLUE3(H5SL_LOCATE_,OP,_FOUND)(SLIST, X, _i) \ } /* end if */ \ } +/* Macro used to find node for operation, if there may be "removed" nodes in the + * list (whose keys cannot be read) */ +#define H5SL_LOCATE_SAFE(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ +{ \ + int _i; /* Local index variable */ \ + H5SL_node_t *_low = X; \ + H5SL_node_t *_high = NULL; \ + \ + H5_GLUE3(H5SL_LOCATE_,CMP,_HASHINIT)(KEY, HASHVAL) \ + for(_i = (int)SLIST->curr_level; _i >= 0; _i--) { \ + X = _low->forward[_i]; \ + while(X != _high) { \ + if(!X->removed) { \ + if(H5_GLUE3(H5SL_LOCATE_,CMP,_CMP)(SLIST, TYPE, X, KEY, HASHVAL)) \ + _low = X; \ + else \ + break; \ + } /* end if */ \ + X = X->forward[_i]; \ + } /* end while */ \ + _high = X; \ + if(X != NULL && H5_GLUE3(H5SL_LOCATE_,CMP,_EQ)(SLIST, TYPE, X, KEY, HASHVAL) ) { \ + /* What to do when a node is found */ \ + H5_GLUE3(H5SL_LOCATE_,OP,_FOUND)(SLIST, X, _i) \ + break; \ + } /* end if */ \ + } /* end for */ \ +} + +/* Macro used to find node for operation */ +#define H5SL_LOCATE(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ +{ \ + if((SLIST)->safe_iterating) \ + H5SL_LOCATE_SAFE(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ + else \ + H5SL_LOCATE_OPT(OP, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ +} + /* Macro used to grow a node by 1. Does not update pointers. LVL is the current * level of X. Does not update LVL but does update X->lvl. */ -#define H5SL_GROW(X, LVL) \ +#define H5SL_GROW(X, LVL, ERR) \ { \ /* Check if we need to increase allocation of forward pointers */ \ if(LVL + 1 >= 1u << X->log_nalloc) { \ @@ -176,8 +229,10 @@ HDassert(H5SL_fac_nused_g == H5SL_fac_nalloc_g); \ /* Double the size of the array of factory pointers */ \ H5SL_fac_nalloc_g *= 2; \ - H5SL_fac_g = (H5FL_fac_head_t **)H5MM_realloc((void *)H5SL_fac_g, \ - H5SL_fac_nalloc_g * sizeof(H5FL_fac_head_t *)); \ + if(NULL == (H5SL_fac_g = (H5FL_fac_head_t **)H5MM_realloc( \ + (void *)H5SL_fac_g, H5SL_fac_nalloc_g \ + * sizeof(H5FL_fac_head_t *)))) \ + HGOTO_ERROR(H5E_SLIST, H5E_CANTALLOC, ERR, "memory allocation failed") \ } /* end if */ \ \ /* Create the new factory */ \ @@ -187,7 +242,7 @@ \ /* Allocate space for new forward pointers */ \ if(NULL == (_tmp = (H5SL_node_t **)H5FL_FAC_MALLOC(H5SL_fac_g[X->log_nalloc]))) \ - HGOTO_ERROR(H5E_SLIST, H5E_NOSPACE, NULL, "memory allocation failed") \ + HGOTO_ERROR(H5E_SLIST, H5E_CANTALLOC, ERR, "memory allocation failed") \ HDmemcpy((void *)_tmp, (const void *)X->forward, (LVL + 1) * sizeof(H5SL_node_t *)); \ X->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[X->log_nalloc-1], (void *)X->forward); \ X->forward = _tmp; \ @@ -222,16 +277,16 @@ /* Macro used to grow the level of a node by 1, with appropriate changes to the * head node if necessary. PREV is the previous node of the height that X is to * grow to. */ -#define H5SL_PROMOTE(SLIST, X, PREV) \ +#define H5SL_PROMOTE(SLIST, X, PREV, ERR) \ { \ size_t _lvl = X->level; \ \ - H5SL_GROW(X, _lvl); \ + H5SL_GROW(X, _lvl, ERR); \ \ if(_lvl == (size_t) SLIST->curr_level) { \ HDassert(PREV == SLIST->header); \ /* Grow the head */ \ - H5SL_GROW(PREV, _lvl); \ + H5SL_GROW(PREV, _lvl, ERR) \ SLIST->curr_level++; \ X->forward[_lvl+1] = NULL; \ } else { \ @@ -298,7 +353,7 @@ /* Promote the middle node if necessary */ \ if(_count == 3) { \ HDassert(X == _last->forward[_i]->forward[_i]); \ - H5SL_PROMOTE(SLIST, X, _last) \ + H5SL_PROMOTE(SLIST, X, _last, NULL) \ } \ \ /* Prepare to drop down */ \ @@ -314,161 +369,167 @@ /* Macro used to remove node */ #define H5SL_REMOVE(CMP, SLIST, X, TYPE, KEY, HASHVAL) \ { \ - H5SL_node_t *_last = X; /* Lowest node in the current gap */ \ - H5SL_node_t *_llast = X; /* Lowest node in the previous gap */ \ - H5SL_node_t *_next = NULL; /* Highest node in the currect gap */ \ - H5SL_node_t *_drop = NULL; /* Low node of the gap to drop into */ \ - H5SL_node_t *_ldrop = NULL; /* Low node of gap before the one to drop into */ \ - H5SL_node_t *_head = SLIST->header; /* Head of the skip list */ \ - int _count; /* Number of nodes in the current gap */ \ - int _i = (int)SLIST->curr_level; \ + /* Check for deferred removal */ \ + if(SLIST->safe_iterating) \ + H5SL_LOCATE(SEARCH_DEFER_REMOVE, CMP, SLIST, X, TYPE, KEY, HASHVAL) \ + else { \ + H5SL_node_t *_last = X; /* Lowest node in the current gap */ \ + H5SL_node_t *_llast = X; /* Lowest node in the previous gap */ \ + H5SL_node_t *_next = NULL; /* Highest node in the currect gap */ \ + H5SL_node_t *_drop = NULL; /* Low node of the gap to drop into */ \ + H5SL_node_t *_ldrop = NULL; /* Low node of gap before the one to drop into */ \ + H5SL_node_t *_head = SLIST->header; /* Head of the skip list */ \ + int _count; /* Number of nodes in the current gap */ \ + int _i = (int)SLIST->curr_level; \ \ - if(_i < 0) \ - HGOTO_DONE(NULL); \ + if(_i < 0) \ + HGOTO_DONE(NULL); \ \ - H5_GLUE3(H5SL_LOCATE_,CMP,_HASHINIT)(KEY, HASHVAL) \ + H5_GLUE3(H5SL_LOCATE_,CMP,_HASHINIT)(KEY, HASHVAL) \ \ - /* Find the gap to drop in to at the highest level */ \ - while(X && (!X->key || H5_GLUE3(H5SL_LOCATE_,CMP,_CMP)(SLIST, TYPE, X, KEY, HASHVAL))) { \ - _llast = _last; \ - _last = X; \ - X = X->forward[_i]; \ - } \ - _next = X; \ + /* Find the gap to drop in to at the highest level */ \ + while(X && (!X->key || H5_GLUE3(H5SL_LOCATE_,CMP,_CMP)(SLIST, TYPE, X, KEY, HASHVAL))) { \ + _llast = _last; \ + _last = X; \ + X = X->forward[_i]; \ + } \ + _next = X; \ \ - /* Main loop */ \ - for(_i--; _i >= 0; _i--) { \ - /* Search for the node to drop into, also count the number of nodes */ \ - /* of height _i in this gap and keep track of of the node before */ \ - /* the one to drop into (_ldrop will become _llast, _drop will */ \ - /* become _last). */ \ - X = _ldrop = _last; \ - _drop = NULL; \ - for(_count = 0; ; _count++) { \ - /* Terminate if this is the last node in the gap */ \ - if(X->forward[_i] == _next) { \ - if(!_drop) \ - _drop = X; \ - break; \ - } /* end if */ \ + /* Main loop */ \ + for(_i--; _i >= 0; _i--) { \ + /* Search for the node to drop into, also count the number of */ \ + /* nodes of height _i in this gap and keep track of of the node */ \ + /* before the one to drop into (_ldrop will become _llast, */ \ + /* _drop will become _last). */ \ + X = _ldrop = _last; \ + _drop = NULL; \ + for(_count = 0; ; _count++) { \ + /* Terminate if this is the last node in the gap */ \ + if(X->forward[_i] == _next) { \ + if(!_drop) \ + _drop = X; \ + break; \ + } /* end if */ \ \ - /* If we have already found the node to drop into and there is */ \ - /* more than one node in this gap, we can stop searching */ \ - if(_drop) { \ - HDassert(_count >= 1); \ - _count = 2; \ - break; \ - } else { /* !_drop */ \ - /* Check if this node is the start of the next gap */ \ - if (!H5_GLUE3(H5SL_LOCATE_,CMP,_CMP)(SLIST, TYPE, X->forward[_i], KEY, HASHVAL)) { \ - _drop = X; \ - /* Again check if we can stop searching */ \ - if(_count) { \ - _count = 2; \ - break; \ + /* If we have already found the node to drop into and there */ \ + /* is more than one node in this gap, we can stop searching */ \ + if(_drop) { \ + HDassert(_count >= 1); \ + _count = 2; \ + break; \ + } else { /* !_drop */ \ + /* Check if this node is the start of the next gap */ \ + if (!H5_GLUE3(H5SL_LOCATE_,CMP,_CMP)(SLIST, TYPE, X->forward[_i], KEY, HASHVAL)) { \ + _drop = X; \ + /* Again check if we can stop searching */ \ + if(_count) { \ + _count = 2; \ + break; \ + } /* end if */ \ } /* end if */ \ - } /* end if */ \ - else \ - _ldrop = X; \ - } /* end else */ \ + else \ + _ldrop = X; \ + } /* end else */ \ \ - /* No need to check the last node in the gap if there are 3, as */ \ - /* there cannot be a fourth */ \ - if(_count == 2) { \ - if(!_drop) \ - _drop = X->forward[_i]; \ - break; \ - } /* end if */ \ - X = X->forward[_i]; \ - } /* end for */ \ - HDassert(_count >= 1 && _count <= 3); \ - HDassert(!_drop->forward[_i] || \ - !H5_GLUE3(H5SL_LOCATE_,CMP,_CMP)(SLIST, TYPE, _drop->forward[_i], KEY, HASHVAL)); \ + /* No need to check the last node in the gap if there are */ \ + /* 3, as there cannot be a fourth */ \ + if(_count == 2) { \ + if(!_drop) \ + _drop = X->forward[_i]; \ + break; \ + } /* end if */ \ + X = X->forward[_i]; \ + } /* end for */ \ + HDassert(_count >= 1 && _count <= 3); \ + HDassert(!_drop->forward[_i] || \ + !H5_GLUE3(H5SL_LOCATE_,CMP,_CMP)(SLIST, TYPE, _drop->forward[_i], KEY, HASHVAL)); \ \ - /* Check if we need to adjust node heights */ \ - if(_count == 1) { \ - /* Check if we are in the first gap */ \ - if(_llast == _last) { \ - /* We are in the first gap, count the number of nodes of */ \ - /* height _i in the next gap. We need only check one node */ \ - /* to see if we should promote the first node in the next */ \ - /* gap */ \ - _llast = _next->forward[_i+1]; \ + /* Check if we need to adjust node heights */ \ + if(_count == 1) { \ + /* Check if we are in the first gap */ \ + if(_llast == _last) { \ + /* We are in the first gap, count the number of nodes */ \ + /* of height _i in the next gap. We need only check */ \ + /* onenode to see if we should promote the first node */ \ + /* in the next gap */ \ + _llast = _next->forward[_i+1]; \ \ - /* Demote the separator node */ \ - H5SL_DEMOTE(_next, _last) \ + /* Demote the separator node */ \ + H5SL_DEMOTE(_next, _last) \ \ - /* If there are 2 or more nodes, promote the first */ \ - if(_next->forward[_i]->forward[_i] != _llast) { \ - X = _next->forward[_i]; \ - H5SL_PROMOTE(SLIST, X, _last) \ - } else if(!_head->forward[_i+1]) { \ - /* shrink the header */ \ - HDassert(_i == SLIST->curr_level - 1); \ - HDassert((size_t) SLIST->curr_level == _head->level); \ + /* If there are 2 or more nodes, promote the first */ \ + if(_next->forward[_i]->forward[_i] != _llast) { \ + X = _next->forward[_i]; \ + H5SL_PROMOTE(SLIST, X, _last, NULL) \ + } else if(!_head->forward[_i+1]) { \ + /* shrink the header */ \ + HDassert(_i == SLIST->curr_level - 1); \ + HDassert((size_t) SLIST->curr_level == _head->level); \ \ - H5SL_SHRINK(_head, (size_t) (_i+1)) \ - SLIST->curr_level--; \ - } /* end else */ \ - } else { \ - /* We are not in the first gap, count the number of nodes */ \ - /* of height _i in the previous gap. Note we "look ahead" */ \ - /* in this loop so X has the value of the last node in the */ \ - /* previous gap. */ \ - X = _llast->forward[_i]; \ - for(_count = 1; _count < 3 && X->forward[_i] != _last; _count++) \ - X = X->forward[_i]; \ - HDassert(X->forward[_i] == _last); \ + H5SL_SHRINK(_head, (size_t) (_i+1)) \ + SLIST->curr_level--; \ + } /* end else */ \ + } else { \ + /* We are not in the first gap, count the number of */ \ + /* nodes of height _i in the previous gap. Note we */ \ + /* "look ahead" in this loop so X has the value of the */ \ + /* last node in the previous gap. */ \ + X = _llast->forward[_i]; \ + for(_count = 1; _count < 3 && X->forward[_i] != _last; _count++) \ + X = X->forward[_i]; \ + HDassert(X->forward[_i] == _last); \ \ - /* Demote the separator node */ \ - H5SL_DEMOTE(_last, _llast) \ + /* Demote the separator node */ \ + H5SL_DEMOTE(_last, _llast) \ \ - /* If there are 2 or more nodes, promote the last */ \ - if(_count >= 2) \ - H5SL_PROMOTE(SLIST, X, _llast) \ - else if(!_head->forward[_i+1]) { \ - /* shrink the header */ \ - HDassert(_i == SLIST->curr_level - 1); \ - HDassert((size_t) SLIST->curr_level == _head->level); \ + /* If there are 2 or more nodes, promote the last */ \ + if(_count >= 2) \ + H5SL_PROMOTE(SLIST, X, _llast, NULL) \ + else if(!_head->forward[_i+1]) { \ + /* shrink the header */ \ + HDassert(_i == SLIST->curr_level - 1); \ + HDassert((size_t) SLIST->curr_level == _head->level); \ \ - H5SL_SHRINK(_head, (size_t) (_i+1)) \ - SLIST->curr_level--; \ + H5SL_SHRINK(_head, (size_t) (_i+1)) \ + SLIST->curr_level--; \ + } /* end else */ \ } /* end else */ \ - } /* end else */ \ - } /* end if */ \ + } /* end if */ \ \ - /* Prepare to drop down */ \ - _llast = _ldrop; \ - _last = _drop; \ - _next = _drop->forward[_i]; \ - } /* end for */ \ + /* Prepare to drop down */ \ + _llast = _ldrop; \ + _last = _drop; \ + _next = _drop->forward[_i]; \ + } /* end for */ \ \ - /* Check if we've found the node */ \ - if(_next && H5_GLUE3(H5SL_LOCATE_,CMP,_EQ)(SLIST, TYPE, _next, KEY, HASHVAL)) { \ - void *tmp = _next->item; \ - X = _next; \ + /* Check if we've found the node */ \ + if(_next && H5_GLUE3(H5SL_LOCATE_,CMP,_EQ)(SLIST, TYPE, _next, KEY, HASHVAL)) { \ + void *tmp = _next->item; \ + X = _next; \ \ - /* If the node has a height > 0, swap it with its (lower) neighbor */ \ - if(X->level) { \ - X = X->backward; \ - _next->key = X->key; \ - _next->item = X->item; \ - _next->hashval = X->hashval; \ - } /* end if */ \ - HDassert(!X->level); \ + /* If the node has a height > 0, swap it with its (lower) */ \ + /* neighbor */ \ + if(X->level) { \ + X = X->backward; \ + _next->key = X->key; \ + _next->item = X->item; \ + _next->hashval = X->hashval; \ + } /* end if */ \ + HDassert(!X->level); \ \ - /* Remove the node */ \ - X->backward->forward[0] = X->forward[0]; \ - if(SLIST->last == X) \ - SLIST->last = X->backward; \ - else \ - X->forward[0]->backward = X->backward; \ - SLIST->nobjs--; \ - X->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[0], X->forward); \ - X = H5FL_FREE(H5SL_node_t, X); \ + /* Remove the node */ \ + X->backward->forward[0] = X->forward[0]; \ + if(SLIST->last == X) \ + SLIST->last = X->backward; \ + else \ + X->forward[0]->backward = X->backward; \ + SLIST->nobjs--; \ + X->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[0], X->forward); \ + X = H5FL_FREE(H5SL_node_t, X); \ \ - HGOTO_DONE(tmp); \ - } /* end if */ \ + HGOTO_DONE(tmp); \ + } /* end if */ \ + } /* end else */ \ } @@ -490,6 +551,7 @@ struct H5SL_node_t { size_t level; /* The level of this node */ size_t log_nalloc; /* log2(Number of slots allocated in forward) */ uint32_t hashval; /* Hash value for key (only for strings, currently) */ + hbool_t removed; /* Whether the node is "removed" (actual removal deferred) */ struct H5SL_node_t **forward; /* Array of forward pointers from this node */ struct H5SL_node_t *backward; /* Backward pointer from this node */ }; @@ -505,6 +567,7 @@ struct H5SL_t { size_t nobjs; /* Number of active objects in skip list */ H5SL_node_t *header; /* Header for nodes in skip list */ H5SL_node_t *last; /* Pointer to last node in skip list */ + hbool_t safe_iterating; /* Whether a routine is "safely" iterating over the list and removals should be deferred */ }; /* Static functions */ @@ -599,6 +662,7 @@ H5SL_new_node(void *item, const void *key, uint32_t hashval) ret_value->item = item; ret_value->level = 0; ret_value->hashval = hashval; + ret_value->removed = FALSE; if(NULL == (ret_value->forward = (H5SL_node_t **)H5FL_FAC_MALLOC(H5SL_fac_g[0]))) { ret_value = H5FL_FREE(H5SL_node_t, ret_value); HGOTO_ERROR(H5E_SLIST, H5E_NOSPACE, NULL, "memory allocation failed") @@ -712,7 +776,7 @@ H5SL_insert_common(H5SL_t *slist, void *item, const void *key) if(x->forward[0]) x->forward[0]->backward = x; else { - HDassert(slist->last); + HDassert(slist->last == prev); slist->last = x; } @@ -766,14 +830,14 @@ H5SL_release_common(H5SL_t *slist, H5SL_operator_t op, void *op_data) /* (Pre-condition) */ /* Free skip list nodes */ - node=slist->header->forward[0]; - while(node!=NULL) { - next_node=node->forward[0]; + node = slist->header->forward[0]; + while(node) { + next_node = node->forward[0]; /* Call callback, if one is given */ - if(op!=NULL) + if(op) /* Casting away const OK -QAK */ - (void)(op)(node->item,(void *)node->key,op_data); + (void)(op)(node->item, (void *)node->key, op_data); node->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[node->log_nalloc], node->forward); node = H5FL_FREE(H5SL_node_t, node); @@ -782,18 +846,18 @@ H5SL_release_common(H5SL_t *slist, H5SL_operator_t op, void *op_data) /* Reset the header pointers */ slist->header->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[slist->header->log_nalloc], slist->header->forward); - if(NULL == (slist->header->forward = (H5SL_node_t **) H5FL_FAC_MALLOC(H5SL_fac_g[0]))) + if(NULL == (slist->header->forward = (H5SL_node_t **)H5FL_FAC_MALLOC(H5SL_fac_g[0]))) HGOTO_ERROR(H5E_SLIST, H5E_NOSPACE, FAIL, "memory allocation failed") slist->header->forward[0] = NULL; slist->header->log_nalloc = 0; slist->header->level = 0; /* Reset the last pointer */ - slist->last=slist->header; + slist->last = slist->header; /* Reset the dynamic internal fields */ - slist->curr_level=-1; - slist->nobjs=0; + slist->curr_level = -1; + slist->nobjs = 0; done: FUNC_LEAVE_NOAPI(ret_value) @@ -893,6 +957,7 @@ H5SL_create(H5SL_type_t type, H5SL_cmp_t cmp) /* Set the dynamic internal fields */ new_slist->curr_level = -1; new_slist->nobjs = 0; + new_slist->safe_iterating = FALSE; /* Allocate the header node */ if(NULL == (header = H5SL_new_node(NULL, NULL, (uint32_t)ULONG_MAX))) @@ -948,6 +1013,9 @@ H5SL_count(H5SL_t *slist) /* Check args */ HDassert(slist); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -987,6 +1055,9 @@ H5SL_insert(H5SL_t *slist, void *item, const void *key) HDassert(slist); HDassert(key); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -1034,6 +1105,9 @@ H5SL_add(H5SL_t *slist, void *item, const void *key) HDassert(slist); HDassert(key); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -1166,6 +1240,9 @@ H5SL_remove_first(H5SL_t *slist) /* Check args */ HDassert(slist); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -1211,7 +1288,7 @@ H5SL_remove_first(H5SL_t *slist) HDassert(tmp->forward[i]->forward[i]->forward[i] == next || tmp->forward[i]->forward[i]->forward[i]->forward[i] == next); tmp = tmp->forward[i]; - H5SL_PROMOTE(slist, tmp, head); + H5SL_PROMOTE(slist, tmp, head, NULL); /* In this case, since there is a node of height = i+1 here * now (tmp), we know the skip list must be valid and can * break */ @@ -1358,6 +1435,9 @@ H5SL_less(H5SL_t *slist, const void *key) HDassert(slist); HDassert(key); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -1464,6 +1544,9 @@ H5SL_greater(H5SL_t *slist, const void *key) HDassert(slist); HDassert(key); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -1848,6 +1931,9 @@ H5SL_first(H5SL_t *slist) /* Check args */ HDassert(slist); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -1882,6 +1968,9 @@ H5SL_next(H5SL_node_t *slist_node) /* Check args */ HDassert(slist_node); + /* Not currently supported */ + HDassert(!slist_node->removed); + /* Check internal consistency */ /* (Pre-condition) */ @@ -1916,6 +2005,9 @@ H5SL_prev(H5SL_node_t *slist_node) /* Check args */ HDassert(slist_node); + /* Not currently supported */ + HDassert(!slist_node->removed); + /* Check internal consistency */ /* (Pre-condition) */ @@ -1951,6 +2043,9 @@ H5SL_last(H5SL_t *slist) /* Check args */ HDassert(slist); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -1985,6 +2080,9 @@ H5SL_item(H5SL_node_t *slist_node) /* Check args */ HDassert(slist_node); + /* Not currently supported */ + HDassert(!slist_node->removed); + /* Check internal consistency */ /* (Pre-condition) */ @@ -2048,8 +2146,9 @@ H5SL_iterate(H5SL_t *slist, H5SL_operator_t op, void *op_data) /* Call the iterator callback */ /* Casting away const OK -QAK */ - if((ret_value = (op)(node->item, (void *)node->key, op_data)) != 0) - break; + if(!node->removed) + if((ret_value = (op)(node->item, (void *)node->key, op_data)) != 0) + break; /* Advance to next node */ node = next; @@ -2087,6 +2186,9 @@ H5SL_release(H5SL_t *slist) /* Check args */ HDassert(slist); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -2133,6 +2235,9 @@ H5SL_free(H5SL_t *slist, H5SL_operator_t op, void *op_data) /* Check args */ HDassert(slist); + /* Not currently supported */ + HDassert(!slist->safe_iterating); + /* Check internal consistency */ /* (Pre-condition) */ @@ -2145,6 +2250,185 @@ H5SL_free(H5SL_t *slist, H5SL_operator_t op, void *op_data) /*-------------------------------------------------------------------------- NAME + H5SL_try_free_safe + PURPOSE + Makes the supplied callback on all nodes in the skip list, freeing each + node that the callback returns TRUE for. + USAGE + herr_t PURPOSE(slist,op,opdata) + H5SL_t *slist; IN/OUT: Pointer to skip list to release nodes + H5SL_try_free_op_t op; IN: Callback function to try to free item & key + void *op_data; IN/OUT: Pointer to application data for callback + + RETURNS + Returns non-negative on success, negative on failure. + DESCRIPTION + Makes the supplied callback on all nodes in the skip list, freeing each + node that the callback returns TRUE for. The iteration is performed in + a safe manner, such that the callback can call H5SL_remove(), + H5SL_search(), H5SL_find(), and H5SL_iterate() on nodes in this + skiplist, except H5SL_remove() may not be call on *this* node. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + This function is written to be most efficient when most nodes are + removed from the skiplist, as it rebuilds the nodes afterwards. + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5SL_try_free_safe(H5SL_t *slist, H5SL_try_free_op_t op, void *op_data) +{ + H5SL_node_t *node, *next_node, *last_node; /* Pointers to skip list nodes */ + htri_t op_ret; + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI_NOINIT + + /* Check args */ + HDassert(slist); + HDassert(op); + + /* Not currently supported */ + HDassert(!slist->safe_iterating); + + /* Check internal consistency */ + /* (Pre-condition) */ + + /* Mark skip list as safe iterating, so nodes aren't freed out from under + * us */ + slist->safe_iterating = TRUE; + + /* Iterate over skip list nodes, making the callback for each and marking + * them as removed if requested by the callback */ + node = slist->header->forward[0]; + while(node) { + /* Check if the node was already removed */ + if(!node->removed) { + /* Call callback */ + /* Casting away const OK -NAF */ + if((op_ret = (op)(node->item , (void *)node->key, op_data)) < 0) + HGOTO_ERROR(H5E_SLIST, H5E_CALLBACK, FAIL, "callback operation failed") + + /* Check if op indicated that the node should be removed */ + if(op_ret) + /* Mark the node as removed */ + node->removed = TRUE; + } /* end if */ + + /* Advance node */ + node = node->forward[0]; + } /* end while */ + + /* Reset safe_iterating */ + slist->safe_iterating = FALSE; + + /* Iterate over nodes, freeing ones marked as removed */ + node = slist->header->forward[0]; + last_node = slist->header; + while(node) { + /* Save next node */ + next_node = node->forward[0]; + + /* Check if the node was marked as removed */ + if(node->removed) { + /* Remove the node */ + node->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[node->log_nalloc], node->forward); + node = H5FL_FREE(H5SL_node_t, node); + slist->nobjs--; + } /* end if */ + else { + /* Update backwards and forwards[0] pointers, and set the level to + * 0. Since the list is flattened we must rebuild the skiplist + * afterwards. */ + /* Set level to 0. Note there is no need to preserve + * node->forward[0] since it was cached above and will always be + * updated later. */ + if(node->level > 0) { + node->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[node->log_nalloc], (void *)node->forward); + if(NULL == (node->forward = (H5SL_node_t **)H5FL_FAC_MALLOC(H5SL_fac_g[0]))) + HGOTO_ERROR(H5E_SLIST, H5E_CANTALLOC, FAIL, "memory allocation failed") + node->log_nalloc = 0; + node->level = 0; + } /* end if */ + + /* Update pointers */ + last_node->forward[0] = node; + node->backward = last_node; + last_node = node; + } /* end else */ + + /* Advance node */ + node = next_node; + } /* end while */ + + /* Final pointer update */ + last_node->forward[0] = NULL; + slist->last = last_node; + + /* Demote skip list to level 0 */ + if(slist->curr_level > 0) { + HDassert(slist->header->level == (size_t)slist->curr_level); + + node = slist->header->forward[0]; + slist->header->forward = (H5SL_node_t **)H5FL_FAC_FREE(H5SL_fac_g[slist->header->log_nalloc], (void *)slist->header->forward); + if(NULL == (slist->header->forward = (H5SL_node_t **)H5FL_FAC_MALLOC(H5SL_fac_g[0]))) + HGOTO_ERROR(H5E_SLIST, H5E_CANTALLOC, FAIL, "memory allocation failed") + slist->header->forward[0] = node; + slist->header->log_nalloc = 0; + slist->header->level = 0; + } /* end if */ + + /* Check if there are any nodes left */ + if(slist->nobjs > 0) { + int i; + + HDassert(slist->header->forward[0]); + + /* Set skiplist level to 0 */ + slist->curr_level = 0; + + /* Rebuild the forward arrays */ + for(i = 0; slist->curr_level >= i; i++) { + HDassert(slist->curr_level == i); + + /* Promote every third node this level until we run out of nodes */ + node = last_node = slist->header; + while(1) { + /* Check second node in gap, if not present, no need to promote + * further this level. */ + HDassert(node->forward[i]); + node = node->forward[i]->forward[i]; + if(!node) + break; + + /* Check third and fourth node in gap, if either is not present, + * no need to promote further this level. */ + node = node->forward[i]; + if(!node || !node->forward[i]) + break; + + /* Promote the third node in the gap */ + H5SL_PROMOTE(slist, node, last_node, FAIL) + last_node = node; + } /* end while */ + } /* end for */ + } /* end if */ + else { + HDassert(!slist->header->forward[0]); + HDassert(slist->last == slist->header); + HDassert(slist->nobjs == 0); + + /* Reset the skiplist level */ + slist->curr_level = -1; + } /* end else */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5SL_try_free_safe() */ + + +/*-------------------------------------------------------------------------- + NAME H5SL_destroy PURPOSE Close a skip list, deallocating it and freeing all its nodes. diff --git a/src/H5SLprivate.h b/src/H5SLprivate.h index ce2f091..856099b 100644 --- a/src/H5SLprivate.h +++ b/src/H5SLprivate.h @@ -63,6 +63,10 @@ typedef int (*H5SL_cmp_t)(const void *key1, const void *key2); typedef herr_t (*H5SL_operator_t)(void *item, void *key, void *operator_data/*in,out*/); +/* Typedef for H5SL_try_free_safe operation callback */ +typedef htri_t (*H5SL_try_free_op_t)(void *item, void *key, + void *operator_data/*in,out*/); + /********************/ /* Private routines */ /********************/ @@ -86,6 +90,8 @@ H5_DLL void *H5SL_item(H5SL_node_t *slist_node); H5_DLL herr_t H5SL_iterate(H5SL_t *slist, H5SL_operator_t op, void *op_data); H5_DLL herr_t H5SL_release(H5SL_t *slist); H5_DLL herr_t H5SL_free(H5SL_t *slist, H5SL_operator_t op, void *op_data); +H5_DLL herr_t H5SL_try_free_safe(H5SL_t *slist, H5SL_try_free_op_t op, + void *op_data); H5_DLL herr_t H5SL_close(H5SL_t *slist); H5_DLL herr_t H5SL_destroy(H5SL_t *slist, H5SL_operator_t op, void *op_data); H5_DLL int H5SL_term_interface(void); diff --git a/src/H5SMcache.c b/src/H5SMcache.c index d8c0824..293088c 100644 --- a/src/H5SMcache.c +++ b/src/H5SMcache.c @@ -675,13 +675,13 @@ H5SM__cache_list_serialize(const H5F_t *f, void *_image, size_t len, computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (uint8_t *)_image), 0); UINT32ENCODE(image, computed_chksum); + /* sanity check */ + HDassert((size_t)(image - (uint8_t *)_image) <= list->header->list_size); + #ifdef H5_CLEAR_MEMORY HDmemset(image, 0, (list->header->list_size - (size_t)(image - (uint8_t *)_image))); #endif /* H5_CLEAR_MEMORY */ - /* sanity check */ - HDassert((size_t)(image - (uint8_t *)_image) <= list->header->list_size); - done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5SM__cache_list_serialize() */ @@ -1222,12 +1222,16 @@ H5T_init_interface(void) /* From long long to floats */ status |= H5T_register(H5T_PERS_HARD, "llong_flt", native_llong, native_float, H5T__conv_llong_float, H5AC_ind_dxpl_id, FALSE); status |= H5T_register(H5T_PERS_HARD, "llong_dbl", native_llong, native_double, H5T__conv_llong_double, H5AC_ind_dxpl_id, FALSE); +#ifdef H5T_CONV_INTERNAL_LLONG_LDOUBLE status |= H5T_register(H5T_PERS_HARD, "llong_ldbl", native_llong, native_ldouble, H5T__conv_llong_ldouble, H5AC_ind_dxpl_id, FALSE); +#endif /* H5T_CONV_INTERNAL_LLONG_LDOUBLE */ /* From unsigned long long to floats */ status |= H5T_register(H5T_PERS_HARD, "ullong_flt", native_ullong, native_float, H5T__conv_ullong_float, H5AC_ind_dxpl_id, FALSE); status |= H5T_register(H5T_PERS_HARD, "ullong_dbl", native_ullong, native_double, H5T__conv_ullong_double, H5AC_ind_dxpl_id, FALSE); +#ifdef H5T_CONV_INTERNAL_ULLONG_LDOUBLE status |= H5T_register(H5T_PERS_HARD, "ullong_ldbl", native_ullong, native_ldouble, H5T__conv_ullong_ldouble, H5AC_ind_dxpl_id, FALSE); +#endif /* H5T_CONV_INTERNAL_ULLONG_LDOUBLE */ /* From floats to char */ status |= H5T_register(H5T_PERS_HARD, "flt_schar", native_float, native_schar, H5T__conv_float_schar, H5AC_ind_dxpl_id, FALSE); @@ -1271,12 +1275,16 @@ H5T_init_interface(void) /* From floats to long long */ status |= H5T_register(H5T_PERS_HARD, "flt_llong", native_float, native_llong, H5T__conv_float_llong, H5AC_ind_dxpl_id, FALSE); status |= H5T_register(H5T_PERS_HARD, "dbl_llong", native_double, native_llong, H5T__conv_double_llong, H5AC_ind_dxpl_id, FALSE); +#ifdef H5T_CONV_INTERNAL_LDOUBLE_LLONG status |= H5T_register(H5T_PERS_HARD, "ldbl_llong", native_ldouble, native_llong, H5T__conv_ldouble_llong, H5AC_ind_dxpl_id, FALSE); +#endif /* H5T_CONV_INTERNAL_LDOUBLE_LLONG */ /* From floats to unsigned long long */ status |= H5T_register(H5T_PERS_HARD, "flt_ullong", native_float, native_ullong, H5T__conv_float_ullong, H5AC_ind_dxpl_id, FALSE); status |= H5T_register(H5T_PERS_HARD, "dbl_ullong", native_double, native_ullong, H5T__conv_double_ullong, H5AC_ind_dxpl_id, FALSE); +#if H5T_CONV_INTERNAL_LDOUBLE_ULLONG status |= H5T_register(H5T_PERS_HARD, "ldbl_ullong", native_ldouble, native_ullong, H5T__conv_ldouble_ullong, H5AC_ind_dxpl_id, FALSE); +#endif /* H5T_CONV_INTERNAL_LDOUBLE_ULLONG */ /* * The special no-op conversion is the fastest, so we list it last. The diff --git a/src/H5Tconv.c b/src/H5Tconv.c index 81c947d..b14f6e5 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -7936,6 +7936,7 @@ H5T__conv_llong_double (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, * *------------------------------------------------------------------------- */ +#if H5T_CONV_INTERNAL_LLONG_LDOUBLE herr_t H5T__conv_llong_ldouble (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, size_t buf_stride, @@ -7944,6 +7945,7 @@ H5T__conv_llong_ldouble (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, { H5T_CONV_xF(LLONG, LDOUBLE, long long, long double, -, -); } +#endif /* H5T_CONV_INTERNAL_LLONG_LDOUBLE */ /*------------------------------------------------------------------------- @@ -8011,6 +8013,7 @@ H5T__conv_ullong_double (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, * *------------------------------------------------------------------------- */ +#if H5T_CONV_INTERNAL_ULLONG_LDOUBLE herr_t H5T__conv_ullong_ldouble (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, size_t buf_stride, @@ -8019,6 +8022,7 @@ H5T__conv_ullong_ldouble (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, { H5T_CONV_xF(ULLONG, LDOUBLE, unsigned long long, long double, -, -); } +#endif /*H5T_CONV_INTERNAL_ULLONG_LDOUBLE*/ /*------------------------------------------------------------------------- @@ -8792,6 +8796,7 @@ H5_GCC_DIAG_ON(float-equal) * *------------------------------------------------------------------------- */ +#if H5T_CONV_INTERNAL_LDOUBLE_LLONG herr_t H5T__conv_ldouble_llong (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, size_t buf_stride, @@ -8802,6 +8807,7 @@ H5_GCC_DIAG_OFF(float-equal) H5T_CONV_Fx(LDOUBLE, LLONG, long double, long long, LLONG_MIN, LLONG_MAX); H5_GCC_DIAG_ON(float-equal) } +#endif /*H5T_CONV_INTERNAL_LDOUBLE_LLONG*/ /*------------------------------------------------------------------------- @@ -8819,6 +8825,7 @@ H5_GCC_DIAG_ON(float-equal) * *------------------------------------------------------------------------- */ +#if H5T_CONV_INTERNAL_LDOUBLE_ULLONG herr_t H5T__conv_ldouble_ullong (hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, size_t buf_stride, @@ -8829,6 +8836,7 @@ H5_GCC_DIAG_OFF(float-equal) H5T_CONV_Fx(LDOUBLE, ULLONG, long double, unsigned long long, 0, ULLONG_MAX); H5_GCC_DIAG_ON(float-equal) } +#endif /*H5T_CONV_INTERNAL_LDOUBLE_ULLONG*/ /*------------------------------------------------------------------------- diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h index 4697bfc..0b0cd61 100644 --- a/src/H5Tpkg.h +++ b/src/H5Tpkg.h @@ -111,6 +111,39 @@ /* (_not_ setting H5T_VISIT_SIMPLE and setting either H5T_VISIT_COMPLEX_FIRST or H5T_VISIT_COMPLEX_LAST will mean visiting all nodes _except_ "simple" "leafs" in the "tree" */ +/* Define an internal macro for converting long long to long double. Mac OS 10.4 gives some + * incorrect conversions. */ +#if (H5_WANT_DATA_ACCURACY && defined(H5_LLONG_TO_LDOUBLE_CORRECT)) || (!H5_WANT_DATA_ACCURACY) +#define H5T_CONV_INTERNAL_LLONG_LDOUBLE 1 +#endif + +/* Define an internal macro for converting unsigned long long to long double. SGI compilers give + * some incorect conversion. 64-bit Solaris does different rounding. Windows Visual Studio 6 does + * not support unsigned long long. For FreeBSD(sleipnir), the last 2 bytes of mantissa are lost when + * compiler tries to do the conversion. For Cygwin, compiler doesn't do rounding correctly. + * Mac OS 10.4 gives some incorrect result. */ +#if (H5_WANT_DATA_ACCURACY && defined(H5_LLONG_TO_LDOUBLE_CORRECT)) || (!H5_WANT_DATA_ACCURACY) +#define H5T_CONV_INTERNAL_ULLONG_LDOUBLE 1 +#endif + +/* Define an internal macro for converting long double to long long. SGI compilers give some incorrect + * conversions. Mac OS 10.4 gives incorrect conversions. HP-UX 11.00 compiler generates floating exception. + * The hard conversion on Windows .NET 2003 has a bug and gives wrong exception value. */ +#if (H5_WANT_DATA_ACCURACY && defined(H5_LDOUBLE_TO_LLONG_ACCURATE)) || \ + (!H5_WANT_DATA_ACCURACY) +#define H5T_CONV_INTERNAL_LDOUBLE_LLONG 1 +#endif + +/* Define an internal macro for converting long double to unsigned long long. SGI compilers give some + * incorrect conversions. Mac OS 10.4 gives incorrect conversions. HP-UX 11.00 compiler generates + * floating exception. */ +#if (H5_WANT_DATA_ACCURACY && defined(H5_LDOUBLE_TO_LLONG_ACCURATE)) || \ + (!H5_WANT_DATA_ACCURACY) +#define H5T_CONV_INTERNAL_LDOUBLE_ULLONG 1 +#else +#define H5T_CONV_INTERNAL_LDOUBLE_ULLONG 0 +#endif + /* Statistics about a conversion function */ struct H5T_stats_t { unsigned ncalls; /*num calls to conversion function */ diff --git a/src/H5VMprivate.h b/src/H5VMprivate.h index 33f2b28..bf4e3ca 100644 --- a/src/H5VMprivate.h +++ b/src/H5VMprivate.h @@ -125,7 +125,7 @@ H5_DLL ssize_t H5VM_memcpyvv(void *_dst, * *------------------------------------------------------------------------- */ -static H5_inline hsize_t H5_ATTR_UNUSED +static H5_INLINE hsize_t H5_ATTR_UNUSED H5VM_vector_reduce_product(unsigned n, const hsize_t *v) { hsize_t ret_value = 1; @@ -157,7 +157,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline htri_t H5_ATTR_UNUSED +static H5_INLINE htri_t H5_ATTR_UNUSED H5VM_vector_zerop_u(int n, const hsize_t *v) { htri_t ret_value=TRUE; /* Return value */ @@ -192,7 +192,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline htri_t H5_ATTR_UNUSED +static H5_INLINE htri_t H5_ATTR_UNUSED H5VM_vector_zerop_s(int n, const hssize_t *v) { htri_t ret_value=TRUE; /* Return value */ @@ -229,7 +229,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline int H5_ATTR_UNUSED +static H5_INLINE int H5_ATTR_UNUSED H5VM_vector_cmp_u (unsigned n, const hsize_t *v1, const hsize_t *v2) { int ret_value=0; /* Return value */ @@ -271,7 +271,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline int H5_ATTR_UNUSED +static H5_INLINE int H5_ATTR_UNUSED H5VM_vector_cmp_s (unsigned n, const hssize_t *v1, const hssize_t *v2) { int ret_value=0; /* Return value */ @@ -308,7 +308,7 @@ done: * *------------------------------------------------------------------------- */ -static H5_inline void H5_ATTR_UNUSED +static H5_INLINE void H5_ATTR_UNUSED H5VM_vector_inc(int n, hsize_t *v1, const hsize_t *v2) { while (n--) *v1++ += *v2++; @@ -355,7 +355,7 @@ static const unsigned char LogTable256[] = * *------------------------------------------------------------------------- */ -static H5_inline unsigned H5_ATTR_UNUSED +static H5_INLINE unsigned H5_ATTR_UNUSED H5VM_log2_gen(uint64_t n) { unsigned r; /* r will be log2(n) */ @@ -403,7 +403,7 @@ static const unsigned MultiplyDeBruijnBitPosition[32] = * *------------------------------------------------------------------------- */ -static H5_inline unsigned H5_ATTR_UNUSED +static H5_INLINE unsigned H5_ATTR_UNUSED H5VM_log2_of2(uint32_t n) { #ifndef NDEBUG @@ -424,7 +424,7 @@ H5VM_log2_of2(uint32_t n) * *------------------------------------------------------------------------- */ -static H5_inline hsize_t H5_ATTR_UNUSED +static H5_INLINE hsize_t H5_ATTR_UNUSED H5VM_power2up(hsize_t n) { hsize_t ret_value = 1; /* Return value */ @@ -449,7 +449,7 @@ H5VM_power2up(hsize_t n) * *------------------------------------------------------------------------- */ -static H5_inline unsigned H5_ATTR_UNUSED +static H5_INLINE unsigned H5_ATTR_UNUSED H5VM_limit_enc_size(uint64_t limit) { return (H5VM_log2_gen(limit) / 8) + 1; @@ -478,7 +478,7 @@ static const unsigned char H5VM_bit_clear_g[8] = {0x7F, 0xBF, 0xDF, 0xEF, 0xF7, * *------------------------------------------------------------------------- */ -static H5_inline hbool_t H5_ATTR_UNUSED +static H5_INLINE hbool_t H5_ATTR_UNUSED H5VM_bit_get(const unsigned char *buf, size_t offset) { /* Test the appropriate bit in the buffer */ @@ -505,7 +505,7 @@ H5VM_bit_get(const unsigned char *buf, size_t offset) * *------------------------------------------------------------------------- */ -static H5_inline void H5_ATTR_UNUSED +static H5_INLINE void H5_ATTR_UNUSED H5VM_bit_set(unsigned char *buf, size_t offset, hbool_t val) { /* Set/reset the appropriate bit in the buffer */ diff --git a/src/H5api_adpt.h b/src/H5api_adpt.h index 52167d0..74066f7 100644 --- a/src/H5api_adpt.h +++ b/src/H5api_adpt.h @@ -24,13 +24,16 @@ /* * Does the compiler support the __attribute__(()) syntax? It's no * big deal if we don't. + * + * Note that Solaris Studio supports attribute, but does not support the + * attributes we use. */ #ifdef __cplusplus # define H5_ATTR_FORMAT(X,Y,Z) /*void*/ # define H5_ATTR_UNUSED /*void*/ # define H5_ATTR_NORETURN /*void*/ #else /* __cplusplus */ -#ifdef H5_HAVE_ATTRIBUTE +#if defined(H5_HAVE_ATTRIBUTE) && !defined(__SUNPRO_C) # define H5_ATTR_FORMAT(X,Y,Z) __attribute__((format(X, Y, Z))) # define H5_ATTR_UNUSED __attribute__((unused)) # define H5_ATTR_NORETURN __attribute__((noreturn)) diff --git a/src/H5config.h.in b/src/H5config.h.in index c089cd3..c0f4466 100644 --- a/src/H5config.h.in +++ b/src/H5config.h.in @@ -133,6 +133,9 @@ /* Define to 1 if you have the `get_fpc_csr' function. */ #undef HAVE_GET_FPC_CSR +/* Define if the compiler understands inline */ +#undef HAVE_INLINE + /* Define if library will contain instrumentation to detect correct optimization operation */ #undef HAVE_INSTRUMENTED_LIBRARY @@ -356,13 +359,27 @@ /* Define to 1 if you have the `_scrsize' function. */ #undef HAVE__SCRSIZE +/* Define if the compiler understands __inline */ +#undef HAVE___INLINE + +/* Define if the compiler understands __inline__ */ +#undef HAVE___INLINE__ + /* Define if HDF5's high-level library headers should be included in hdf5.h */ #undef INCLUDE_HL +/* Define if your system can convert long double to (unsigned) long long + values correctly. */ +#undef LDOUBLE_TO_LLONG_ACCURATE + /* Define if your system converts long double to (unsigned) long values with special algorithm. */ #undef LDOUBLE_TO_LONG_SPECIAL +/* Define if your system can convert (unsigned) long long to long double + values correctly. */ +#undef LLONG_TO_LDOUBLE_CORRECT + /* Define if your system can convert (unsigned) long to long double values with special algorithm. */ #undef LONG_TO_LDOUBLE_SPECIAL @@ -545,6 +562,9 @@ /* Version number of package */ #undef VERSION +/* Data accuracy is prefered to speed during data conversions */ +#undef WANT_DATA_ACCURACY + /* Check exception handling functions during data conversions */ #undef WANT_DCONV_EXCEPTION @@ -569,12 +589,6 @@ /* Define to empty if `const' does not conform to ANSI C. */ #undef const -/* Define to `__inline__' or `__inline' if that's what the C compiler - calls it, or to nothing if 'inline' is not supported under any name. */ -#ifndef __cplusplus -#undef inline -#endif - /* Define to `long int' if <sys/types.h> does not define. */ #undef off_t diff --git a/src/H5private.h b/src/H5private.h index f7a4171..9efd145 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -183,11 +183,22 @@ #endif /*H5_HAVE_WIN32_API*/ -/* H5_inline */ -#ifndef H5_inline -#define H5_inline -#endif /* H5_inline */ - +/* Various ways that inline functions can be declared */ +#if defined(H5_HAVE___INLINE__) + /* GNU (alternative form) */ + #define H5_INLINE __inline__ +#elif defined(H5_HAVE___INLINE) + /* Visual Studio */ + #define H5_INLINE __inline +#elif defined(H5_HAVE_INLINE) + /* GNU, C++ + * Use "inline" as a last resort on the off-chance that there will + * be C++ problems. + */ + #define H5_INLINE inline +#else + #define H5_INLINE +#endif /* inline choices */ #ifndef F_OK # define F_OK 00 diff --git a/src/H5public.h b/src/H5public.h index e625367..1d99c11 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -94,10 +94,10 @@ extern "C" { /* Version numbers */ #define H5_VERS_MAJOR 1 /* For major interface/format changes */ #define H5_VERS_MINOR 9 /* For minor interface/format changes */ -#define H5_VERS_RELEASE 222 /* For tweaks, bug-fixes, or development */ +#define H5_VERS_RELEASE 227 /* For tweaks, bug-fixes, or development */ #define H5_VERS_SUBRELEASE "" /* For pre-releases like snap0 */ /* Empty string for real releases. */ -#define H5_VERS_INFO "HDF5 library version: 1.9.222" /* Full version string */ +#define H5_VERS_INFO "HDF5 library version: 1.9.227" /* Full version string */ #define H5check() H5check_version(H5_VERS_MAJOR,H5_VERS_MINOR, \ H5_VERS_RELEASE) diff --git a/src/Makefile.am b/src/Makefile.am index 2df095e..b6c4dad 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -42,9 +42,11 @@ DISTCLEANFILES=H5pubconf.h # library sources libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \ - H5AC.c H5B.c H5Bcache.c H5Bdbg.c \ + H5AC.c H5ACmpio.c \ + H5B.c H5Bcache.c H5Bdbg.c \ H5B2.c H5B2cache.c H5B2dbg.c H5B2hdr.c H5B2int.c H5B2stat.c H5B2test.c \ - H5C.c H5CS.c \ + H5C.c H5Cmpio.c \ + H5CS.c \ H5D.c H5Dbtree.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ H5Ddeprec.c H5Defl.c H5Dfill.c H5Dint.c \ H5Dio.c H5Dlayout.c \ diff --git a/src/Makefile.in b/src/Makefile.in index 6dcba84..8a60a77 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -150,28 +150,29 @@ LTLIBRARIES = $(lib_LTLIBRARIES) libhdf5_la_LIBADD = am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \ H5timer.lo H5trace.lo H5A.lo H5Abtree2.lo H5Adense.lo \ - H5Adeprec.lo H5Aint.lo H5Atest.lo H5AC.lo H5B.lo H5Bcache.lo \ - H5Bdbg.lo H5B2.lo H5B2cache.lo H5B2dbg.lo H5B2hdr.lo \ - H5B2int.lo H5B2stat.lo H5B2test.lo H5C.lo H5CS.lo H5D.lo \ - H5Dbtree.lo H5Dchunk.lo H5Dcompact.lo H5Dcontig.lo H5Ddbg.lo \ - H5Ddeprec.lo H5Defl.lo H5Dfill.lo H5Dint.lo H5Dio.lo \ - H5Dlayout.lo H5Dmpio.lo H5Doh.lo H5Dscatgath.lo H5Dselect.lo \ - H5Dtest.lo H5E.lo H5Edeprec.lo H5Eint.lo H5EA.lo H5EAcache.lo \ - H5EAdbg.lo H5EAdblkpage.lo H5EAdblock.lo H5EAhdr.lo \ - H5EAiblock.lo H5EAint.lo H5EAsblock.lo H5EAstat.lo H5EAtest.lo \ - H5F.lo H5Fint.lo H5Faccum.lo H5Fcwfs.lo H5Fdbg.lo H5Fdeprec.lo \ - H5Fefc.lo H5Ffake.lo H5Fio.lo H5Fmount.lo H5Fmpi.lo \ - H5Fquery.lo H5Fsfile.lo H5Fsuper.lo H5Fsuper_cache.lo \ - H5Ftest.lo H5FA.lo H5FAcache.lo H5FAdbg.lo H5FAdblock.lo \ - H5FAdblkpage.lo H5FAhdr.lo H5FAstat.lo H5FAtest.lo H5FD.lo \ - H5FDcore.lo H5FDdirect.lo H5FDfamily.lo H5FDint.lo H5FDlog.lo \ - H5FDmpi.lo H5FDmpio.lo H5FDmulti.lo H5FDsec2.lo H5FDspace.lo \ - H5FDstdio.lo H5FL.lo H5FO.lo H5FS.lo H5FScache.lo H5FSdbg.lo \ - H5FSsection.lo H5FSstat.lo H5FStest.lo H5G.lo H5Gbtree2.lo \ - H5Gcache.lo H5Gcompact.lo H5Gdense.lo H5Gdeprec.lo H5Gent.lo \ - H5Gint.lo H5Glink.lo H5Gloc.lo H5Gname.lo H5Gnode.lo H5Gobj.lo \ - H5Goh.lo H5Groot.lo H5Gstab.lo H5Gtest.lo H5Gtraverse.lo \ - H5HF.lo H5HFbtree2.lo H5HFcache.lo H5HFdbg.lo H5HFdblock.lo \ + H5Adeprec.lo H5Aint.lo H5Atest.lo H5AC.lo H5ACmpio.lo H5B.lo \ + H5Bcache.lo H5Bdbg.lo H5B2.lo H5B2cache.lo H5B2dbg.lo \ + H5B2hdr.lo H5B2int.lo H5B2stat.lo H5B2test.lo H5C.lo \ + H5Cmpio.lo H5CS.lo H5D.lo H5Dbtree.lo H5Dchunk.lo \ + H5Dcompact.lo H5Dcontig.lo H5Ddbg.lo H5Ddeprec.lo H5Defl.lo \ + H5Dfill.lo H5Dint.lo H5Dio.lo H5Dlayout.lo H5Dmpio.lo H5Doh.lo \ + H5Dscatgath.lo H5Dselect.lo H5Dtest.lo H5E.lo H5Edeprec.lo \ + H5Eint.lo H5EA.lo H5EAcache.lo H5EAdbg.lo H5EAdblkpage.lo \ + H5EAdblock.lo H5EAhdr.lo H5EAiblock.lo H5EAint.lo \ + H5EAsblock.lo H5EAstat.lo H5EAtest.lo H5F.lo H5Fint.lo \ + H5Faccum.lo H5Fcwfs.lo H5Fdbg.lo H5Fdeprec.lo H5Fefc.lo \ + H5Ffake.lo H5Fio.lo H5Fmount.lo H5Fmpi.lo H5Fquery.lo \ + H5Fsfile.lo H5Fsuper.lo H5Fsuper_cache.lo H5Ftest.lo H5FA.lo \ + H5FAcache.lo H5FAdbg.lo H5FAdblock.lo H5FAdblkpage.lo \ + H5FAhdr.lo H5FAstat.lo H5FAtest.lo H5FD.lo H5FDcore.lo \ + H5FDdirect.lo H5FDfamily.lo H5FDint.lo H5FDlog.lo H5FDmpi.lo \ + H5FDmpio.lo H5FDmulti.lo H5FDsec2.lo H5FDspace.lo H5FDstdio.lo \ + H5FL.lo H5FO.lo H5FS.lo H5FScache.lo H5FSdbg.lo H5FSsection.lo \ + H5FSstat.lo H5FStest.lo H5G.lo H5Gbtree2.lo H5Gcache.lo \ + H5Gcompact.lo H5Gdense.lo H5Gdeprec.lo H5Gent.lo H5Gint.lo \ + H5Glink.lo H5Gloc.lo H5Gname.lo H5Gnode.lo H5Gobj.lo H5Goh.lo \ + H5Groot.lo H5Gstab.lo H5Gtest.lo H5Gtraverse.lo H5HF.lo \ + H5HFbtree2.lo H5HFcache.lo H5HFdbg.lo H5HFdblock.lo \ H5HFdtable.lo H5HFhdr.lo H5HFhuge.lo H5HFiblock.lo H5HFiter.lo \ H5HFman.lo H5HFsection.lo H5HFspace.lo H5HFstat.lo H5HFtest.lo \ H5HFtiny.lo H5HG.lo H5HGcache.lo H5HGdbg.lo H5HGquery.lo \ @@ -723,7 +724,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog *.clog2 # After making changes, run bin/reconfigure to update other configure related # files like Makefile.in. LT_VERS_INTERFACE = 6 -LT_VERS_REVISION = 211 +LT_VERS_REVISION = 217 LT_VERS_AGE = 0 # Our main target, the HDF5 library @@ -740,9 +741,11 @@ DISTCLEANFILES = H5pubconf.h # library sources libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \ - H5AC.c H5B.c H5Bcache.c H5Bdbg.c \ + H5AC.c H5ACmpio.c \ + H5B.c H5Bcache.c H5Bdbg.c \ H5B2.c H5B2cache.c H5B2dbg.c H5B2hdr.c H5B2int.c H5B2stat.c H5B2test.c \ - H5C.c H5CS.c \ + H5C.c H5Cmpio.c \ + H5CS.c \ H5D.c H5Dbtree.c H5Dchunk.c H5Dcompact.c H5Dcontig.c H5Ddbg.c \ H5Ddeprec.c H5Defl.c H5Dfill.c H5Dint.c \ H5Dio.c H5Dlayout.c \ @@ -963,6 +966,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5A.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5AC.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5ACmpio.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Abtree2.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Adense.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Adeprec.Plo@am__quote@ @@ -980,6 +984,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Bdbg.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5C.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5CS.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Cmpio.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5D.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dbtree.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dchunk.Plo@am__quote@ diff --git a/test/cache.c b/test/cache.c index a98a995..38adbda 100644 --- a/test/cache.c +++ b/test/cache.c @@ -9499,7 +9499,7 @@ check_flush_cache__flush_op_test(H5F_t * file_ptr, * Modifications: * * Updated test for minor changes in the behaviour - * of H5C_flush_single_entry(). + * of H5C__flush_single_entry(). * * JRM -- 2/16/15 * @@ -10575,7 +10575,7 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) failure_mssg = "unexpected size/len in flush op eviction test 12."; } - /* modifications to the H5C_flush_single_entry() function have + /* modifications to the H5C__flush_single_entry() function have * changed the behavior of the cache slightly, causing * this test to fail. Comment out for now -- come back and * fix if all goes well. @@ -10664,7 +10664,7 @@ check_flush_cache__flush_op_eviction_test(H5F_t * file_ptr) failure_mssg = "unexpected size/len in flush op eviction test 13."; } - /* modifications to the H5C_flush_single_entry() function have + /* modifications to the H5C__flush_single_entry() function have * changed the behavior of the cache slightly, causing * this test to fail. Comment out for now -- come back and * fix if all goes well. @@ -14207,8 +14207,7 @@ check_resize_entry(void) } else { result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[LARGE_ENTRY_TYPE]), entry_ptr->addr, - (void *)entry_ptr, H5C__DIRTIED_FLAG); + entry_ptr->addr, (void *)entry_ptr, H5C__DIRTIED_FLAG); if ( result < 0 ) { @@ -14293,8 +14292,7 @@ check_resize_entry(void) } else { result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[LARGE_ENTRY_TYPE]), entry_ptr->addr, - (void *)entry_ptr, H5C__DIRTIED_FLAG); + entry_ptr->addr, (void *)entry_ptr, H5C__DIRTIED_FLAG); if ( result < 0 ) { @@ -14654,8 +14652,7 @@ check_resize_entry(void) } else { result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[LARGE_ENTRY_TYPE]), entry_ptr->addr, - (void *)entry_ptr, H5C__DIRTIED_FLAG); + entry_ptr->addr, (void *)entry_ptr, H5C__DIRTIED_FLAG); if ( result < 0 ) { @@ -14742,8 +14739,7 @@ check_resize_entry(void) } else { result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[LARGE_ENTRY_TYPE]), entry_ptr->addr, - (void *)entry_ptr, H5C__DIRTIED_FLAG); + entry_ptr->addr, (void *)entry_ptr, H5C__DIRTIED_FLAG); if ( result < 0 ) { @@ -16246,8 +16242,7 @@ check_double_pin_err(void) if ( pass ) { result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[0]), entry_ptr->addr, - (void *)entry_ptr, H5C__PIN_ENTRY_FLAG); + entry_ptr->addr, (void *)entry_ptr, H5C__PIN_ENTRY_FLAG); if ( result > 0 ) { @@ -16331,8 +16326,7 @@ check_double_unpin_err(void) if ( pass ) { result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[0]), entry_ptr->addr, - (void *)entry_ptr, H5C__UNPIN_ENTRY_FLAG); + entry_ptr->addr, (void *)entry_ptr, H5C__UNPIN_ENTRY_FLAG); if ( result > 0 ) { @@ -16612,8 +16606,7 @@ check_double_unprotect_err(void) if ( pass ) { result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[0]), entry_ptr->addr, - (void *)entry_ptr, H5C__NO_FLAGS_SET); + entry_ptr->addr, (void *)entry_ptr, H5C__NO_FLAGS_SET); if ( result > 0 ) { @@ -16989,8 +16982,7 @@ check_unprotect_ro_dirty_err(void) if ( pass ) { result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[0]), entry_ptr->addr, - (void *)entry_ptr, H5C__DIRTIED_FLAG); + entry_ptr->addr, (void *)entry_ptr, H5C__DIRTIED_FLAG); if ( result >= 0 ) { @@ -17033,8 +17025,7 @@ check_unprotect_ro_dirty_err(void) if ( pass ) { result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[0]), entry_ptr->addr, - (void *)entry_ptr, H5C__DIRTIED_FLAG); + entry_ptr->addr, (void *)entry_ptr, H5C__DIRTIED_FLAG); if ( result > 0 ) { @@ -34878,8 +34869,8 @@ check_stats__smoke_check_1(H5F_t * file_ptr) if ( ( cache_ptr->total_ht_insertions != 33 ) || ( cache_ptr->total_ht_deletions != 1 ) || - ( cache_ptr->successful_ht_searches != 65 ) || - ( cache_ptr->total_successful_ht_search_depth != 195 ) || + ( cache_ptr->successful_ht_searches != 32 ) || + ( cache_ptr->total_successful_ht_search_depth != 96 ) || ( cache_ptr->failed_ht_searches != 33 ) || ( cache_ptr->total_failed_ht_search_depth != 52 ) || ( cache_ptr->max_index_len != 32 ) || @@ -34989,8 +34980,8 @@ check_stats__smoke_check_1(H5F_t * file_ptr) if ( ( cache_ptr->total_ht_insertions != 33 ) || ( cache_ptr->total_ht_deletions != 33 ) || - ( cache_ptr->successful_ht_searches != 98 ) || - ( cache_ptr->total_successful_ht_search_depth != 198 ) || + ( cache_ptr->successful_ht_searches != 33 ) || + ( cache_ptr->total_successful_ht_search_depth != 99 ) || ( cache_ptr->failed_ht_searches != 33 ) || ( cache_ptr->total_failed_ht_search_depth != 52 ) || ( cache_ptr->max_index_len != 32 ) || diff --git a/test/cache_common.c b/test/cache_common.c index 8ccd7e3..f3da908 100644 --- a/test/cache_common.c +++ b/test/cache_common.c @@ -3959,7 +3959,7 @@ unprotect_entry(H5F_t * file_ptr, entry_ptr->is_dirty = TRUE; result = H5C_unprotect(file_ptr, H5P_DATASET_XFER_DEFAULT, - &(types[type]), entry_ptr->addr, (void *)entry_ptr, flags); + entry_ptr->addr, (void *)entry_ptr, flags); if ( ( result < 0 ) || ( ( entry_ptr->header.is_protected ) && diff --git a/test/dt_arith.c b/test/dt_arith.c index 8ededc7..b3f5cad 100644 --- a/test/dt_arith.c +++ b/test/dt_arith.c @@ -5032,8 +5032,32 @@ run_int_fp_conv(const char *name) #endif #endif /* H5_SIZEOF_LONG!=H5_SIZEOF_INT */ #if H5_SIZEOF_LONG_LONG!=H5_SIZEOF_LONG +#if H5_LLONG_TO_LDOUBLE_CORRECT nerrors += test_conv_int_fp(name, TEST_NORMAL, H5T_NATIVE_LLONG, H5T_NATIVE_LDOUBLE); +#else /* H5_LLONG_TO_LDOUBLE_CORRECT */ + { + char str[256]; /*hello string */ + + HDsnprintf(str, sizeof(str), "Testing %s %s -> %s conversions", + name, "long long", "long double"); + printf("%-70s", str); + SKIPPED(); + HDputs(" Test skipped due to compiler error in handling conversion."); + } +#endif /* H5_LLONG_TO_LDOUBLE_CORRECT */ +#if H5_LLONG_TO_LDOUBLE_CORRECT nerrors += test_conv_int_fp(name, TEST_NORMAL, H5T_NATIVE_ULLONG, H5T_NATIVE_LDOUBLE); +#else /* H5_LLONG_TO_LDOUBLE_CORRECT */ + { + char str[256]; /*hello string */ + + HDsnprintf(str, sizeof(str), "Testing %s %s -> %s conversions", + name, "unsigned long long", "long double"); + printf("%-70s", str); + SKIPPED(); + HDputs(" Test skipped due to compiler not handling conversion."); + } +#endif /* H5_LLONG_TO_LDOUBLE_CORRECT */ #endif #endif @@ -5134,8 +5158,40 @@ run_fp_int_conv(const char *name) #endif /*H5_SIZEOF_LONG!=H5_SIZEOF_INT && H5_SIZEOF_LONG_DOUBLE!=0 */ #if H5_SIZEOF_LONG_LONG!=H5_SIZEOF_LONG && H5_SIZEOF_LONG_DOUBLE!=0 +#ifdef H5_LDOUBLE_TO_LLONG_ACCURATE nerrors += test_conv_int_fp(name, test_values, H5T_NATIVE_LDOUBLE, H5T_NATIVE_LLONG); +#else /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ + { + char str[256]; /*string */ + + HDsnprintf(str, sizeof(str), "Testing %s %s -> %s conversions", + name, "long double", "long long"); + printf("%-70s", str); + SKIPPED(); +#if H5_SIZEOF_LONG_DOUBLE!=0 + HDputs(" Test skipped due to hardware conversion error."); +#else + HDputs(" Test skipped due to disabled long double."); +#endif + } +#endif /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ +#if defined(H5_LDOUBLE_TO_LLONG_ACCURATE) nerrors += test_conv_int_fp(name, test_values, H5T_NATIVE_LDOUBLE, H5T_NATIVE_ULLONG); +#else /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ + { + char str[256]; /*string */ + + HDsnprintf(str, sizeof(str), "Testing %s %s -> %s conversions", + name, "long double", "unsigned long long"); + printf("%-70s", str); + SKIPPED(); +#if H5_SIZEOF_LONG_DOUBLE!=0 + HDputs(" Test skipped due to hardware conversion error."); +#else + HDputs(" Test skipped due to disabled long double."); +#endif + } +#endif /*H5_LDOUBLE_TO_LLONG_ACCURATE*/ #endif #endif #ifndef H5_VMS diff --git a/test/gen_bogus.c b/test/gen_bogus.c index df1e81f..e9ce686 100644 --- a/test/gen_bogus.c +++ b/test/gen_bogus.c @@ -61,7 +61,7 @@ int main(void) if((did = H5Dcreate2(fid, "/Dataset1", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; if(H5Dclose(did) < 0) goto error; - /* Set "fail if unknown" message flag for bogus message */ + /* Set "fail if unknown and open for write" message flag for bogus message */ bogus_flags = H5O_MSG_FLAG_FAIL_IF_UNKNOWN_AND_OPEN_FOR_WRITE; if(H5Pset(dcpl, H5O_BOGUS_MSG_FLAGS_NAME, &bogus_flags) < 0) goto error; @@ -69,12 +69,20 @@ int main(void) if((did = H5Dcreate2(fid, "/Dataset2", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; if(H5Dclose(did) < 0) goto error; + /* Set "fail if unknown always" message flag for bogus message */ + bogus_flags = H5O_MSG_FLAG_FAIL_IF_UNKNOWN_ALWAYS; + if(H5Pset(dcpl, H5O_BOGUS_MSG_FLAGS_NAME, &bogus_flags) < 0) goto error; + + /* Create third dataset, with "fail if unknown always" message flag */ + if((did = H5Dcreate2(fid, "/Dataset3", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; + if(H5Dclose(did) < 0) goto error; + /* Set "mark if unknown" message flag for bogus message */ bogus_flags = H5O_MSG_FLAG_MARK_IF_UNKNOWN; if(H5Pset(dcpl, H5O_BOGUS_MSG_FLAGS_NAME, &bogus_flags) < 0) goto error; - /* Create second dataset, with "mark if unknown" message flag */ - if((did = H5Dcreate2(fid, "/Dataset3", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; + /* Create fourth dataset, with "mark if unknown" message flag */ + if((did = H5Dcreate2(fid, "/Dataset4", H5T_NATIVE_INT, sid, H5P_DEFAULT, dcpl, H5P_DEFAULT)) < 0) goto error; if(H5Dclose(did) < 0) goto error; /* Close dataset creation property list */ diff --git a/test/h5test.h b/test/h5test.h index fa0c11b..5b203c6 100644 --- a/test/h5test.h +++ b/test/h5test.h @@ -158,7 +158,7 @@ H5TEST_DLL void ParseTestVerbosity(char *argv); H5TEST_DLL int GetTestNumErrs(void); H5TEST_DLL void IncTestNumErrs(void); H5TEST_DLL const void *GetTestParameters(void); -H5TEST_DLL int TestErrPrintf(const char *format, ...) __attribute__ ((format (printf, 1, 2))); +H5TEST_DLL int TestErrPrintf(const char *format, ...) H5_ATTR_FORMAT(printf, 1, 2); H5TEST_DLL void SetTest(const char *testname, int action); H5TEST_DLL void TestAlarmOn(void); H5TEST_DLL void TestAlarmOff(void); diff --git a/test/ohdr.c b/test/ohdr.c index 1ba2b10..45f2238 100644 --- a/test/ohdr.c +++ b/test/ohdr.c @@ -665,7 +665,9 @@ main(void) PASSED(); - /* Test reading datasets with undefined object header messages */ + /* Test reading datasets with undefined object header messages + * and the various "fail/mark if unknown" object header message flags + */ HDputs("Accessing objects with unknown header messages:"); { hid_t file2; /* File ID for 'bogus' object file */ @@ -687,7 +689,7 @@ main(void) PASSED(); - TESTING("object with unknown header message & 'fail if unknown and open for write' flag set"); + TESTING("object in r/o file with unknown header message & 'fail if unknown and open for write' flag set"); /* Open the dataset with the unknown header message, and "fail if unknown and open for write" flag */ if((dset = H5Dopen2(file2, "/Dataset2", H5P_DEFAULT)) < 0) @@ -697,10 +699,23 @@ main(void) PASSED(); + TESTING("object in r/o file with unknown header message & 'fail if unknown always' flag set"); + + /* Attempt to open the dataset with the unknown header message, and "fail if unknown always" flag */ + H5E_BEGIN_TRY { + dset = H5Dopen2(file2, "/Dataset3", H5P_DEFAULT); + } H5E_END_TRY; + if(dset >= 0) { + H5Dclose(dset); + TEST_ERROR + } /* end if */ + + PASSED(); + TESTING("object with unknown header message & 'mark if unknown' flag set"); /* Copy object with "mark if unknown" flag on message into file that can be modified */ - if(H5Ocopy(file2, "/Dataset3", file, "/Dataset3", H5P_DEFAULT, H5P_DEFAULT) < 0) + if(H5Ocopy(file2, "/Dataset4", file, "/Dataset4", H5P_DEFAULT, H5P_DEFAULT) < 0) TEST_ERROR /* Close the file we created (to flush changes to file) */ @@ -712,7 +727,7 @@ main(void) TEST_ERROR /* Open the dataset with the "mark if unknown" message */ - if((dset = H5Dopen2(file, "/Dataset3", H5P_DEFAULT)) < 0) + if((dset = H5Dopen2(file, "/Dataset4", H5P_DEFAULT)) < 0) TEST_ERROR /* Check that the "unknown" message was _NOT_ marked */ @@ -732,7 +747,7 @@ main(void) TEST_ERROR /* Open the dataset with the "mark if unknown" message */ - if((dset = H5Dopen2(file, "/Dataset3", H5P_DEFAULT)) < 0) + if((dset = H5Dopen2(file, "/Dataset4", H5P_DEFAULT)) < 0) TEST_ERROR /* Create data space */ @@ -764,7 +779,7 @@ main(void) TEST_ERROR /* Re-open the dataset with the "mark if unknown" message */ - if((dset = H5Dopen2(file, "/Dataset3", H5P_DEFAULT)) < 0) + if((dset = H5Dopen2(file, "/Dataset4", H5P_DEFAULT)) < 0) TEST_ERROR /* Check that the "unknown" message was marked */ @@ -785,7 +800,7 @@ main(void) if((file2 = H5Fopen(testfile, H5F_ACC_RDWR, H5P_DEFAULT)) < 0) TEST_ERROR - TESTING("object with unknown header message & 'fail if unknown and open for write' flag set"); + TESTING("object in r/w file with unknown header message & 'fail if unknown and open for write' flag set"); /* Attempt to open the dataset with the unknown header message, and "fail if unknown and open for write" flag */ H5E_BEGIN_TRY { @@ -796,6 +811,19 @@ main(void) TEST_ERROR } /* end if */ + PASSED(); + + TESTING("object in r/w file with unknown header message & 'fail if unknown always' flag set"); + + /* Attempt to open the dataset with the unknown header message, and "fail if unknown always" flag */ + H5E_BEGIN_TRY { + dset = H5Dopen2(file2, "/Dataset3", H5P_DEFAULT); + } H5E_END_TRY; + if(dset >= 0) { + H5Dclose(dset); + TEST_ERROR + } /* end if */ + /* Close the file with the bogus objects */ if(H5Fclose(file2) < 0) TEST_ERROR diff --git a/test/tbogus.h5 b/test/tbogus.h5 Binary files differindex 906921c..875fbd8 100644 --- a/test/tbogus.h5 +++ b/test/tbogus.h5 @@ -533,12 +533,211 @@ out: return -1; } + /* Test removing ids in callback for H5Iclear_type */ + +/* There was a rare bug where, if an id free callback being called by + * H5I_clear_type() removed another id in that type, a segfault could occur. + * This test tests for that error (and freeing ids "out of order" within + * H5Iclear_type() in general). */ +/* Macro definitions */ +#define TEST_RCT_MAX_NOBJS 25 +#define TEST_RCT_MIN_NOBJS 5 +#define TEST_RCT_NITER 50 + +/* Structure to hold the list of objects */ +typedef struct { + struct test_rct_obj_t *list; /* List of objects */ + long nobjs; /* Number of objects in list */ + long nobjs_rem; /* Number of objects in list that have not been freed */ +} test_rct_list_t; + +/* Structure for an object */ +typedef struct test_rct_obj_t { + hid_t id; /* ID for this object */ + int nfrees; /* Number of times this object has been freed */ + hbool_t freeing; /* Whether we are currently freeing this object directly (through H5Idec_ref()) */ + test_rct_list_t *obj_list; /* List of all objects */ +} test_rct_obj_t; + +/* Free callback */ +static herr_t test_rct_free(void *_obj) { + test_rct_obj_t *obj = (test_rct_obj_t *)_obj; + long rem_idx, i; + herr_t ret; /* return value */ + + /* Mark this object as freed */ + obj->nfrees++; + obj->obj_list->nobjs_rem--; + + /* Check freeing and nobjs_rem */ + if(!obj->freeing && (obj->obj_list->nobjs_rem > 0)) { + /* Remove a random object from the list */ + rem_idx = HDrandom() % obj->obj_list->nobjs_rem; + + /* Scan the list, finding the rem_idx'th object that has not been + * freed */ + for(i = 0; i < obj->obj_list->nobjs; i++) + if(obj->obj_list->list[i].nfrees == 0) { + if(rem_idx == 0) + break; + else + rem_idx--; + } /* end if */ + if(i == obj->obj_list->nobjs) { + ERROR("invalid obj_list"); + goto out; + } /* end if */ + else { + /* Remove the object. Mark as "freeing" so its own callback does + * not free another object. */ + obj->obj_list->list[i].freeing = TRUE; + ret = H5Idec_ref(obj->obj_list->list[i].id); + CHECK(ret, FAIL, "H5Idec_ref"); + if(ret == FAIL) + goto out; + obj->obj_list->list[i].freeing = FALSE; + } /* end else */ + } /* end if */ + + /* Verify nobjs_rem is non-negative */ + if(obj->obj_list->nobjs_rem < 0) { + ERROR("invalid nobjs_rem"); + goto out; + } /* end if */ + + return 0; + +out: + return -1; +} /* end test_rct_free() */ + +/* Test function */ +static int test_remove_clear_type(void) +{ + H5I_type_t obj_type; + test_rct_list_t obj_list; + test_rct_obj_t list[TEST_RCT_MAX_NOBJS]; + long i, j; + long nobjs_found; + hsize_t nmembers; + herr_t ret; /* return value */ + + /* Register type */ + obj_type = H5Iregister_type((size_t)8, 0, test_rct_free); + CHECK(obj_type, H5I_BADID, "H5Iregister_type"); + if(obj_type == H5I_BADID) + goto out; + + /* Init obj_list.list */ + obj_list.list = list; + + for(i = 0; i < TEST_RCT_NITER; i++) { + /* Build object list */ + obj_list.nobjs = obj_list.nobjs_rem = TEST_RCT_MIN_NOBJS + (HDrandom() % (long)(TEST_RCT_MAX_NOBJS - TEST_RCT_MIN_NOBJS + 1)); + for(j = 0; j < obj_list.nobjs; j++) { + list[j].nfrees = 0; + list[j].freeing = FALSE; + list[j].obj_list = &obj_list; + list[j].id = H5Iregister(obj_type, &list[j]); + CHECK(list[j].id, FAIL, "H5Iregister"); + if(list[j].id == FAIL) + goto out; + if(HDrandom() % 2) { + ret = H5Iinc_ref(list[j].id); + CHECK(ret, FAIL, "H5Iinc_ref"); + if(ret == FAIL) + goto out; + } /* end if */ + } /* end for */ + + /* Clear the type */ + ret = H5Iclear_type(obj_type, FALSE); + CHECK(ret, FAIL, "H5Iclear_type"); + if(ret == FAIL) + goto out; + + /* Verify list */ + nobjs_found = 0; + for(j = 0; j < obj_list.nobjs; j++) { + if(list[j].nfrees == 0) + nobjs_found++; + else { + VERIFY(list[j].nfrees, (long)1, "list[j].nfrees"); + if(list[j].nfrees != (long)1) + goto out; + } /* end else */ + VERIFY(list[j].freeing, FALSE, "list[j].freeing"); + if(list[j].freeing != FALSE) + goto out; + } /* end for */ + + /* Verify number of objects */ + VERIFY(obj_list.nobjs_rem, nobjs_found, "obj_list.nobjs_rem"); + if(obj_list.nobjs_rem != nobjs_found) + goto out; + ret = H5Inmembers(obj_type, &nmembers); + CHECK(ret, FAIL, "H5Inmembers"); + if(ret == FAIL) + goto out; + VERIFY(nmembers, (size_t)nobjs_found, "H5Inmembers"); + if(nmembers != (size_t)nobjs_found) + goto out; + + /* Clear the type with force set to TRUE */ + ret = H5Iclear_type(obj_type, TRUE); + CHECK(ret, FAIL, "H5Iclear_type"); + if(ret == FAIL) + goto out; + + /* Verify list */ + for(j = 0; j < obj_list.nobjs; j++) { + VERIFY(list[j].nfrees, (long)1, "list[j].nfrees"); + if(list[j].nfrees != (long)1) + goto out; + VERIFY(list[j].freeing, FALSE, "list[j].freeing"); + if(list[j].freeing != FALSE) + goto out; + } /* end for */ + + /* Verify number of objects is 0 */ + VERIFY(obj_list.nobjs_rem, (long)0, "obj_list.nobjs_rem"); + if(obj_list.nobjs_rem != (long)0) + goto out; + ret = H5Inmembers(obj_type, &nmembers); + CHECK(ret, FAIL, "H5Inmembers"); + if(ret == FAIL) + goto out; + VERIFY(nmembers, (size_t)0, "H5Inmembers"); + if(nmembers != (size_t)0) + goto out; + } /* end for */ + + /* Destroy type */ + ret = H5Idestroy_type(obj_type); + CHECK(ret, FAIL, "H5Idestroy_type"); + if(ret == FAIL) + goto out; + + return 0; + +out: + /* Cleanup. For simplicity, just destroy the types and ignore errors. */ + H5E_BEGIN_TRY + H5Idestroy_type(obj_type); + H5E_END_TRY + return -1; +} /* end test_remove_clear_type() */ + void test_ids(void) { + /* Set the random # seed */ + HDsrandom((unsigned)HDtime(NULL)); + if (basic_id_test() < 0) TestErrPrintf("Basic ID test failed\n"); if (id_predefined_test() < 0) TestErrPrintf("Predefined ID type test failed\n"); if (test_is_valid() < 0) TestErrPrintf("H5Iis_valid test failed\n"); if (test_get_type() < 0) TestErrPrintf("H5Iget_type test failed\n"); if (test_id_type_list() < 0) TestErrPrintf("ID type list test failed\n"); + if (test_remove_clear_type() < 0) TestErrPrintf("ID remove during H5Iclear_type test failed\n"); } diff --git a/test/tskiplist.c b/test/tskiplist.c index 07e63fd..62c5f8e 100644 --- a/test/tskiplist.c +++ b/test/tskiplist.c @@ -1171,6 +1171,208 @@ test_skiplist_free(void) /**************************************************************** ** +** test_skiplist_try_free_safe(): Test H5SL (skip list) code. +** Tests 'try_free_safe' operation in skip lists. +** +****************************************************************/ +/* Macro definitions */ +#define TEST_TFS_MAX_NOBJS 100 +#define TEST_TFS_MIN_NOBJS 5 +#define TEST_TFS_NITER 50 + +/* Structure to hold the list of objects */ +typedef struct { + H5SL_t *slist; /* Skiplist holding the objects */ + struct test_tfs_obj_t *list; /* Linear list of objects */ + int nobjs; /* Number of objects in list */ + int nobjs_rem; /* Number of objects in list that have not been freed */ +} test_tfs_list_t; + +/* Structure for an object */ +typedef struct test_tfs_obj_t { + int idx; /* Index (key) for this object */ + int nfrees; /* Number of times this object has been freed */ +} test_tfs_obj_t; + +/* op_data struct for H5SL_iterate() */ +typedef struct test_tfs_it_ud_t { + test_tfs_list_t *obj_list; /* List of objects */ + int last_idx; /* Index of last object visited in iteration */ + int ncalls; /* Number of times this callback was called */ +} test_tfs_it_ud_t; + +/* iterate callback */ +static herr_t test_tfs_iter(void *_obj, void *key, void *_udata) { + test_tfs_obj_t *obj = (test_tfs_obj_t *)_obj; + test_tfs_it_ud_t *udata = (test_tfs_it_ud_t *)_udata; + + /* Check consistency */ + VERIFY((void *)&obj->idx, key, "obj->idx"); + VERIFY(obj, &udata->obj_list->list[obj->idx], "obj_list->list[obj->idx]"); + + /* Increment number of calls */ + udata->ncalls++; + + /* Verify we were given the correct object */ + do { + udata->last_idx++; + } while(udata->obj_list->list[udata->last_idx].nfrees != 0); + VERIFY(udata->last_idx, obj->idx, "H5SL_iterate"); + + return 0; +} /* end test_tfs_iter() */ + +/* try_free_safe callback */ +static htri_t test_tfs_free(void *_obj, void *key, void *_obj_list) { + test_tfs_obj_t *obj = (test_tfs_obj_t *)_obj; + test_tfs_list_t *obj_list = (test_tfs_list_t *)_obj_list; + test_tfs_it_ud_t iter_ud; + int nrems, rem_idx, i, j; + test_tfs_obj_t *obj_ret; + herr_t ret; /* return value */ + htri_t ret_value; + + /* Check consistency */ + VERIFY((void *)&obj->idx, key, "obj->idx"); + VERIFY(obj, &obj_list->list[obj->idx], "obj_list->list[obj->idx]"); + + /* Mark this object as freed (to make sure it isn't recursively freed, that + * is not something we support, we will undo this if we decide later not to + * free the object) */ + obj->nfrees++; + obj_list->nobjs_rem--; + + /* Decide how many objects to remove */ + nrems = (int)(HDrandom() % (long)3); + + /* Remove objects */ + for(i = 0; i < nrems; i++) + /* Check nobjs_rem */ + if(obj_list->nobjs_rem > 0) { + /* Remove a random object from the list */ + rem_idx = (int)(HDrandom() % (long)obj_list->nobjs_rem); + + /* Scan the list, finding the rem_idx'th object that has not been + * freed */ + for(j = 0; j < obj_list->nobjs; j++) + if(obj_list->list[j].nfrees == 0) { + if(rem_idx == 0) + break; + else + rem_idx--; + } /* end if */ + if(j == obj_list->nobjs) + ERROR("invalid obj_list"); + else { + /* Remove the object */ + obj_ret = (test_tfs_obj_t *)H5SL_remove(obj_list->slist, &j); + CHECK(obj_ret, NULL, "H5SL_remove"); + obj_ret->nfrees++; + obj_list->nobjs_rem--; + } /* end else */ + } /* end if */ + + /* Mark this object as not freed so we know to expect it in the iterate call + */ + obj->nfrees--; + obj_list->nobjs_rem++; + + /* Iterate over skip list (maybe) */ + if(HDrandom() % (long)5) { + iter_ud.obj_list = obj_list; + iter_ud.last_idx = -1; + iter_ud.ncalls = 0; + ret = H5SL_iterate(obj_list->slist, test_tfs_iter, &iter_ud); + CHECK(ret, FAIL, "H5SL_iterate"); + VERIFY(iter_ud.ncalls, obj_list->nobjs_rem, "H5SL_iterate"); + } /* end if */ + + /* Verify nobjs_rem is non-negative */ + if(obj_list->nobjs_rem < 0) + ERROR("invalid nobjs_rem"); + + /* Decide whether this object should be freed */ + if(HDrandom() % (long)2) { + /* Free the object */ + ret_value = TRUE; + obj->nfrees++; + obj_list->nobjs_rem--; + } /* end if */ + else + /* Do not free the object */ + ret_value = FALSE; + + return ret_value; +} /* end test_tfs_free() */ + +/* Test function */ +static void +test_skiplist_try_free_safe(void) +{ + test_tfs_list_t obj_list; + test_tfs_obj_t list[TEST_TFS_MAX_NOBJS]; + int i, j; + int nobjs_found; + hsize_t count; + herr_t ret; /* Generic return value */ + + /* Output message about test being performed */ + MESSAGE(7, ("Testing Skip List 'Try Free Safe' Operation\n")); + + /* Create a skip list */ + obj_list.slist = H5SL_create(H5SL_TYPE_INT, NULL); + CHECK(obj_list.slist, NULL, "H5SL_create"); + + /* Init obj_list.list */ + obj_list.list = list; + for(j = 0; j < TEST_TFS_MAX_NOBJS; j++) + list[j].idx = j; + + for(i = 0; i < TEST_TFS_NITER; i++) { + /* Build object list */ + obj_list.nobjs = obj_list.nobjs_rem = (int)(TEST_TFS_MIN_NOBJS + (HDrandom() % (long)(TEST_TFS_MAX_NOBJS - TEST_TFS_MIN_NOBJS + 1))); + for(j = 0; j < obj_list.nobjs; j++) { + list[j].nfrees = 0; + ret = H5SL_insert(obj_list.slist, &list[j], &list[j].idx); + CHECK(ret, FAIL, "H5SL_insert"); + } /* end for */ + + /* Call H5S_try_free_safe() - free most of the items in the skip list in + * a safe manner */ + ret = H5SL_try_free_safe(obj_list.slist, test_tfs_free, &obj_list); + CHECK(ret, FAIL, "H5SL_try_free_safe"); + + /* Verify list */ + nobjs_found = 0; + for(j = 0; j < obj_list.nobjs; j++) + if(list[j].nfrees == 0) + nobjs_found++; + else + VERIFY(list[j].nfrees, (long)1, "list[j].nfrees"); + + /* Verify number of objects */ + VERIFY(obj_list.nobjs_rem, nobjs_found, "obj_list.nobjs_rem"); + count = H5SL_count(obj_list.slist); + VERIFY(count, (size_t)nobjs_found, "H5SL_count"); + + /* Release the skip list, forcibly freeing all nodes (will not make + * callbacks) */ + ret = H5SL_release(obj_list.slist); + CHECK(ret, FAIL, "H5SL_release"); + + /* Verify number of objects is 0 */ + count = H5SL_count(obj_list.slist); + VERIFY(count, (size_t)0, "H5SL_count"); + } /* end for */ + + /* Close the skip list */ + ret = H5SL_close(obj_list.slist); + CHECK(ret, FAIL, "H5SL_close"); + +} /* end test_skiplist_try_free_safe() */ + +/**************************************************************** +** ** test_skiplist_less(): Test H5SL (skip list) code. ** Tests 'less' operation in skip lists. ** @@ -1577,6 +1779,7 @@ test_skiplist(void) test_skiplist_add(); /* Test 'add' operation */ test_skiplist_destroy(); /* Test 'destroy' operation */ test_skiplist_free(); /* Test 'free' operation */ + test_skiplist_try_free_safe(); /* Test 'try_free_safe' operation */ test_skiplist_less(); /* Test 'less' operation */ test_skiplist_greater(); /* Test 'greater' operation */ test_skiplist_below(); /* Test 'below' operation */ diff --git a/testpar/t_cache.c b/testpar/t_cache.c index 8bbc33c..8072215 100644 --- a/testpar/t_cache.c +++ b/testpar/t_cache.c @@ -4249,7 +4249,7 @@ setup_cache_for_test(hid_t * fid_ptr, if ( success ) { - if ( H5AC_set_write_done_callback(cache_ptr, do_sync) != SUCCEED ) { + if ( H5AC__set_write_done_callback(cache_ptr, do_sync) != SUCCEED ) { nerrors++; if ( verbose ) { @@ -4264,12 +4264,12 @@ setup_cache_for_test(hid_t * fid_ptr, if ( success ) { - if ( H5AC_set_sync_point_done_callback(cache_ptr, verify_writes) != SUCCEED ) { + if ( H5AC__set_sync_point_done_callback(cache_ptr, verify_writes) != SUCCEED ) { nerrors++; if ( verbose ) { HDfprintf(stdout, - "%d:%s: H5AC_set_sync_point_done_callback failed.\n", + "%d:%s: H5AC__set_sync_point_done_callback failed.\n", world_mpi_rank, FUNC); } } @@ -4997,7 +4997,7 @@ unlock_entry(H5F_t * file_ptr, nerrors++; if ( verbose ) { - HDfprintf(stdout, "%d:%s: error in H5C_unprotect().\n", + HDfprintf(stdout, "%d:%s: error in H5AC_unprotect().\n", world_mpi_rank, FUNC); } } else { diff --git a/testpar/t_file_image.c b/testpar/t_file_image.c index 544ba32..a2246b6 100644 --- a/testpar/t_file_image.c +++ b/testpar/t_file_image.c @@ -241,6 +241,9 @@ file_image_daisy_chain_test(void) vector_ok = FALSE; VRFY((vector_ok), "verified received vector."); + HDfree(vector_ptr); + vector_ptr = NULL; + /* 7) closes the core file and exit. */ err = H5Sclose(space_id); diff --git a/testpar/t_mpi.c b/testpar/t_mpi.c index 7bd2f58..873b952 100644 --- a/testpar/t_mpi.c +++ b/testpar/t_mpi.c @@ -281,7 +281,7 @@ test_mpio_gb_file(char *filename) printf("Skipped GB file range test " "because MPI_Offset cannot support it\n"); }else{ - buf = HDmalloc(MB); + buf = (char *)HDmalloc(MB); VRFY((buf!=NULL), "malloc succeed"); /* open a new file. Remove it first in case it exists. */ @@ -678,10 +678,9 @@ static int test_mpio_derived_dtype(char *filename) { int mpi_err_strlen; int mpi_err; int i; - int nerrors = 0; /* number of errors */ MPI_Datatype etype,filetype; MPI_Datatype adv_filetype,bas_filetype[2]; - MPI_Datatype etypenew, filetypenew; + MPI_Datatype filetypenew; MPI_Offset disp; MPI_Status Status; MPI_Aint adv_disp[2]; @@ -1100,7 +1099,7 @@ main(int argc, char **argv) * calls. By then, MPI calls may not work. */ if (H5dont_atexit() < 0){ - printf("Failed to turn off atexit processing. Continue.\n", mpi_rank); + printf("Failed to turn off atexit processing. Continue.\n"); }; H5open(); if (parse_options(argc, argv) != 0){ diff --git a/tools/h5copy/CMakeLists.txt b/tools/h5copy/CMakeLists.txt index 7291197..c18b1f9 100644 --- a/tools/h5copy/CMakeLists.txt +++ b/tools/h5copy/CMakeLists.txt @@ -42,12 +42,9 @@ endif (BUILD_TESTING) #----------------------------------------------------------------------------- #INSTALL_PROGRAM_PDB (h5copy ${HDF5_INSTALL_BIN_DIR} toolsapplications) - + install ( TARGETS h5copy - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) diff --git a/tools/h5diff/CMakeLists.txt b/tools/h5diff/CMakeLists.txt index 39095a5..0a5c813 100644 --- a/tools/h5diff/CMakeLists.txt +++ b/tools/h5diff/CMakeLists.txt @@ -41,12 +41,12 @@ if (BUILD_TESTING) TARGET_C_PROPERTIES (h5diffgentest STATIC " " " ") target_link_libraries (h5diffgentest ${HDF5_LIB_TARGET}) set_target_properties (h5diffgentest PROPERTIES FOLDER generator/tools) - + #add_test (NAME h5diffgentest COMMAND $<TARGET_FILE:h5diffgentest>) endif (HDF5_BUILD_GENERATORS AND NOT BUILD_SHARED_LIBS) include (CMakeTests.cmake) - + endif (BUILD_TESTING) ############################################################################## @@ -64,10 +64,7 @@ endif (BUILD_TESTING) install ( TARGETS h5diff - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) if (H5_HAVE_PARALLEL) @@ -77,9 +74,6 @@ if (H5_HAVE_PARALLEL) install ( TARGETS ph5diff - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) endif (H5_HAVE_PARALLEL) diff --git a/tools/h5dump/CMakeLists.txt b/tools/h5dump/CMakeLists.txt index 7658c04..9d5db3f 100644 --- a/tools/h5dump/CMakeLists.txt +++ b/tools/h5dump/CMakeLists.txt @@ -31,7 +31,7 @@ if (BUILD_TESTING) TARGET_C_PROPERTIES (h5dumpgentest STATIC " " " ") target_link_libraries (h5dumpgentest ${HDF5_LIB_TARGET} ${HDF5_TOOLS_LIB_TARGET}) set_target_properties (h5dumpgentest PROPERTIES FOLDER generator/tools) - + #add_test (NAME h5dumpgentest COMMAND $<TARGET_FILE:h5dumpgentest>) endif (HDF5_BUILD_GENERATORS AND NOT BUILD_SHARED_LIBS) @@ -57,8 +57,5 @@ endif (BUILD_TESTING) install ( TARGETS h5dump - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) diff --git a/tools/h5import/CMakeLists.txt b/tools/h5import/CMakeLists.txt index bf1515e..ada3b07 100644 --- a/tools/h5import/CMakeLists.txt +++ b/tools/h5import/CMakeLists.txt @@ -47,8 +47,5 @@ endif (BUILD_TESTING) install ( TARGETS h5import - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) diff --git a/tools/h5jam/CMakeLists.txt b/tools/h5jam/CMakeLists.txt index 4498132..ceb2eb9 100644 --- a/tools/h5jam/CMakeLists.txt +++ b/tools/h5jam/CMakeLists.txt @@ -50,7 +50,7 @@ if (BUILD_TESTING) TARGET_C_PROPERTIES (testhdf5 ${LIB_TYPE} " " " ") target_link_libraries (h5jamgentest ${HDF5_LIB_TARGET}) set_target_properties (h5jamgentest PROPERTIES FOLDER generator/tools) - + #add_test (NAME h5jamgentest COMMAND $<TARGET_FILE:h5jamgentest>) endif (HDF5_BUILD_GENERATORS AND NOT BUILD_SHARED_LIBS) @@ -73,8 +73,5 @@ endif (BUILD_TESTING) install ( TARGETS h5jam h5unjam - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) diff --git a/tools/h5ls/CMakeLists.txt b/tools/h5ls/CMakeLists.txt index 984b36b..116f735 100644 --- a/tools/h5ls/CMakeLists.txt +++ b/tools/h5ls/CMakeLists.txt @@ -40,8 +40,5 @@ endif (BUILD_TESTING) install ( TARGETS h5ls - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) diff --git a/tools/h5ls/h5ls.c b/tools/h5ls/h5ls.c index 28f7b8a..fe128fa 100644 --- a/tools/h5ls/h5ls.c +++ b/tools/h5ls/h5ls.c @@ -155,6 +155,8 @@ static struct dispatch_t { } static void print_type(h5tools_str_t *buffer, hid_t type, int ind); +static hbool_t print_int_type(h5tools_str_t *buffer, hid_t type, int ind); +static hbool_t print_float_type(h5tools_str_t *buffer, hid_t type, int ind); static herr_t visit_obj(hid_t file, const char *oname, iter_t *iter); @@ -294,7 +296,7 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces) if (escape_spaces) { if (buffer) h5tools_str_append(buffer, "\\ "); nprint += 2; - } + } else { if (buffer) h5tools_str_append(buffer, " "); nprint++; @@ -304,7 +306,7 @@ print_string(h5tools_str_t *buffer, const char *s, hbool_t escape_spaces) if (isprint((int)*s)) { if (buffer) h5tools_str_append(buffer, "%c", *s); nprint++; - } + } else { if (buffer) h5tools_str_append(buffer, "\\%03o", *((const unsigned char*)s)); nprint += 4; @@ -385,102 +387,106 @@ print_obj_name(h5tools_str_t *buffer, const iter_t *iter, const char *oname, *------------------------------------------------------------------------- */ static hbool_t -print_native_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) +print_native_type(h5tools_str_t *buffer, hid_t type, int ind) { - if (H5Tequal(type, H5T_NATIVE_SCHAR)==TRUE) { - h5tools_str_append(buffer, "native signed char"); - } else if (H5Tequal(type, H5T_NATIVE_UCHAR)==TRUE) { - h5tools_str_append(buffer, "native unsigned char"); - } else if (H5Tequal(type, H5T_NATIVE_INT)==TRUE) { - h5tools_str_append(buffer, "native int"); - } else if (H5Tequal(type, H5T_NATIVE_UINT)==TRUE) { - h5tools_str_append(buffer, "native unsigned int"); - } else if (H5Tequal(type, H5T_NATIVE_SHORT)==TRUE) { - h5tools_str_append(buffer, "native short"); - } else if (H5Tequal(type, H5T_NATIVE_USHORT)==TRUE) { - h5tools_str_append(buffer, "native unsigned short"); - } else if (H5Tequal(type, H5T_NATIVE_LONG)==TRUE) { - h5tools_str_append(buffer, "native long"); - } else if (H5Tequal(type, H5T_NATIVE_ULONG)==TRUE) { - h5tools_str_append(buffer, "native unsigned long"); - } else if (H5Tequal(type, H5T_NATIVE_LLONG)==TRUE) { - h5tools_str_append(buffer, "native long long"); - } else if (H5Tequal(type, H5T_NATIVE_ULLONG)==TRUE) { - h5tools_str_append(buffer, "native unsigned long long"); - } else if (H5Tequal(type, H5T_NATIVE_FLOAT)==TRUE) { - h5tools_str_append(buffer, "native float"); - } else if (H5Tequal(type, H5T_NATIVE_DOUBLE)==TRUE) { - h5tools_str_append(buffer, "native double"); + if(!simple_output_g) { + if (H5Tequal(type, H5T_NATIVE_SCHAR)==TRUE) { + h5tools_str_append(buffer, "native signed char"); + } else if (H5Tequal(type, H5T_NATIVE_UCHAR)==TRUE) { + h5tools_str_append(buffer, "native unsigned char"); + } else if (H5Tequal(type, H5T_NATIVE_INT)==TRUE) { + h5tools_str_append(buffer, "native int"); + } else if (H5Tequal(type, H5T_NATIVE_UINT)==TRUE) { + h5tools_str_append(buffer, "native unsigned int"); + } else if (H5Tequal(type, H5T_NATIVE_SHORT)==TRUE) { + h5tools_str_append(buffer, "native short"); + } else if (H5Tequal(type, H5T_NATIVE_USHORT)==TRUE) { + h5tools_str_append(buffer, "native unsigned short"); + } else if (H5Tequal(type, H5T_NATIVE_LONG)==TRUE) { + h5tools_str_append(buffer, "native long"); + } else if (H5Tequal(type, H5T_NATIVE_ULONG)==TRUE) { + h5tools_str_append(buffer, "native unsigned long"); + } else if (H5Tequal(type, H5T_NATIVE_LLONG)==TRUE) { + h5tools_str_append(buffer, "native long long"); + } else if (H5Tequal(type, H5T_NATIVE_ULLONG)==TRUE) { + h5tools_str_append(buffer, "native unsigned long long"); + } else if (H5Tequal(type, H5T_NATIVE_FLOAT)==TRUE) { + h5tools_str_append(buffer, "native float"); + } else if (H5Tequal(type, H5T_NATIVE_DOUBLE)==TRUE) { + h5tools_str_append(buffer, "native double"); #if H5_SIZEOF_LONG_DOUBLE !=0 - } else if (H5Tequal(type, H5T_NATIVE_LDOUBLE)==TRUE) { - h5tools_str_append(buffer, "native long double"); + } else if (H5Tequal(type, H5T_NATIVE_LDOUBLE)==TRUE) { + h5tools_str_append(buffer, "native long double"); #endif - } else if (H5Tequal(type, H5T_NATIVE_INT8)==TRUE) { - h5tools_str_append(buffer, "native int8_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT8)==TRUE) { - h5tools_str_append(buffer, "native uint8_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT16)==TRUE) { - h5tools_str_append(buffer, "native int16_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT16)==TRUE) { - h5tools_str_append(buffer, "native uint16_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT32)==TRUE) { - h5tools_str_append(buffer, "native int32_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT32)==TRUE) { - h5tools_str_append(buffer, "native uint32_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT64)==TRUE) { - h5tools_str_append(buffer, "native int64_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT64)==TRUE) { - h5tools_str_append(buffer, "native uint64_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST8)==TRUE) { - h5tools_str_append(buffer, "native int_least8_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST8)==TRUE) { - h5tools_str_append(buffer, "native uint_least8_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST16)==TRUE) { - h5tools_str_append(buffer, "native int_least16_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST16)==TRUE) { - h5tools_str_append(buffer, "native uint_least16_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST32)==TRUE) { - h5tools_str_append(buffer, "native int_least32_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST32)==TRUE) { - h5tools_str_append(buffer, "native uint_least32_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST64)==TRUE) { - h5tools_str_append(buffer, "native int_least64_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST64)==TRUE) { - h5tools_str_append(buffer, "native uint_least64_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_FAST8)==TRUE) { - h5tools_str_append(buffer, "native int_fast8_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST8)==TRUE) { - h5tools_str_append(buffer, "native uint_fast8_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_FAST16)==TRUE) { - h5tools_str_append(buffer, "native int_fast16_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST16)==TRUE) { - h5tools_str_append(buffer, "native uint_fast16_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_FAST32)==TRUE) { - h5tools_str_append(buffer, "native int_fast32_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST32)==TRUE) { - h5tools_str_append(buffer, "native uint_fast32_t"); - } else if (H5Tequal(type, H5T_NATIVE_INT_FAST64)==TRUE) { - h5tools_str_append(buffer, "native int_fast64_t"); - } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST64)==TRUE) { - h5tools_str_append(buffer, "native uint_fast64_t"); - } else if (H5Tequal(type, H5T_NATIVE_B8)==TRUE) { - h5tools_str_append(buffer, "native 8-bit field"); - } else if (H5Tequal(type, H5T_NATIVE_B16)==TRUE) { - h5tools_str_append(buffer, "native 16-bit field"); - } else if (H5Tequal(type, H5T_NATIVE_B32)==TRUE) { - h5tools_str_append(buffer, "native 32-bit field"); - } else if (H5Tequal(type, H5T_NATIVE_B64)==TRUE) { - h5tools_str_append(buffer, "native 64-bit field"); - } else if (H5Tequal(type, H5T_NATIVE_HSIZE)==TRUE) { - h5tools_str_append(buffer, "native hsize_t"); - } else if (H5Tequal(type, H5T_NATIVE_HSSIZE)==TRUE) { - h5tools_str_append(buffer, "native hssize_t"); - } else if (H5Tequal(type, H5T_NATIVE_HERR)==TRUE) { - h5tools_str_append(buffer, "native herr_t"); - } else if (H5Tequal(type, H5T_NATIVE_HBOOL)==TRUE) { - h5tools_str_append(buffer, "native hbool_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT8)==TRUE) { + h5tools_str_append(buffer, "native int8_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT8)==TRUE) { + h5tools_str_append(buffer, "native uint8_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT16)==TRUE) { + h5tools_str_append(buffer, "native int16_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT16)==TRUE) { + h5tools_str_append(buffer, "native uint16_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT32)==TRUE) { + h5tools_str_append(buffer, "native int32_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT32)==TRUE) { + h5tools_str_append(buffer, "native uint32_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT64)==TRUE) { + h5tools_str_append(buffer, "native int64_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT64)==TRUE) { + h5tools_str_append(buffer, "native uint64_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST8)==TRUE) { + h5tools_str_append(buffer, "native int_least8_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST8)==TRUE) { + h5tools_str_append(buffer, "native uint_least8_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST16)==TRUE) { + h5tools_str_append(buffer, "native int_least16_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST16)==TRUE) { + h5tools_str_append(buffer, "native uint_least16_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST32)==TRUE) { + h5tools_str_append(buffer, "native int_least32_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST32)==TRUE) { + h5tools_str_append(buffer, "native uint_least32_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT_LEAST64)==TRUE) { + h5tools_str_append(buffer, "native int_least64_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT_LEAST64)==TRUE) { + h5tools_str_append(buffer, "native uint_least64_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT_FAST8)==TRUE) { + h5tools_str_append(buffer, "native int_fast8_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST8)==TRUE) { + h5tools_str_append(buffer, "native uint_fast8_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT_FAST16)==TRUE) { + h5tools_str_append(buffer, "native int_fast16_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST16)==TRUE) { + h5tools_str_append(buffer, "native uint_fast16_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT_FAST32)==TRUE) { + h5tools_str_append(buffer, "native int_fast32_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST32)==TRUE) { + h5tools_str_append(buffer, "native uint_fast32_t"); + } else if (H5Tequal(type, H5T_NATIVE_INT_FAST64)==TRUE) { + h5tools_str_append(buffer, "native int_fast64_t"); + } else if (H5Tequal(type, H5T_NATIVE_UINT_FAST64)==TRUE) { + h5tools_str_append(buffer, "native uint_fast64_t"); + } else if (H5Tequal(type, H5T_NATIVE_B8)==TRUE) { + h5tools_str_append(buffer, "native 8-bit field"); + } else if (H5Tequal(type, H5T_NATIVE_B16)==TRUE) { + h5tools_str_append(buffer, "native 16-bit field"); + } else if (H5Tequal(type, H5T_NATIVE_B32)==TRUE) { + h5tools_str_append(buffer, "native 32-bit field"); + } else if (H5Tequal(type, H5T_NATIVE_B64)==TRUE) { + h5tools_str_append(buffer, "native 64-bit field"); + } else if (H5Tequal(type, H5T_NATIVE_HSIZE)==TRUE) { + h5tools_str_append(buffer, "native hsize_t"); + } else if (H5Tequal(type, H5T_NATIVE_HSSIZE)==TRUE) { + h5tools_str_append(buffer, "native hssize_t"); + } else if (H5Tequal(type, H5T_NATIVE_HERR)==TRUE) { + h5tools_str_append(buffer, "native herr_t"); + } else if (H5Tequal(type, H5T_NATIVE_HBOOL)==TRUE) { + h5tools_str_append(buffer, "native hbool_t"); + } else { + return print_int_type(buffer, type, ind); + } } else { - return FALSE; + return print_int_type(buffer, type, ind); } return TRUE; } @@ -503,22 +509,22 @@ print_native_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) *------------------------------------------------------------------------- */ static hbool_t -print_ieee_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) +print_ieee_type(h5tools_str_t *buffer, hid_t type, int ind) { if (H5Tequal(type, H5T_IEEE_F32BE)==TRUE) { h5tools_str_append(buffer, "IEEE 32-bit big-endian float"); - } + } else if (H5Tequal(type, H5T_IEEE_F32LE)==TRUE) { h5tools_str_append(buffer, "IEEE 32-bit little-endian float"); - } + } else if (H5Tequal(type, H5T_IEEE_F64BE)==TRUE) { h5tools_str_append(buffer, "IEEE 64-bit big-endian float"); - } + } else if (H5Tequal(type, H5T_IEEE_F64LE)==TRUE) { h5tools_str_append(buffer, "IEEE 64-bit little-endian float"); - } + } else { - return FALSE; + return print_float_type(buffer, type, ind); } return TRUE; } @@ -650,17 +656,17 @@ print_int_type(h5tools_str_t *buffer, hid_t type, int ind) order = H5Tget_order(type); if (H5T_ORDER_LE==order) { order_s = " little-endian"; - } + } else if (H5T_ORDER_BE==order) { order_s = " big-endian"; - } + } else if (H5T_ORDER_VAX==order) { order_s = " mixed-endian"; - } + } else { order_s = " unknown-byte-order"; } - } + } else { order_s = ""; } @@ -669,14 +675,14 @@ print_int_type(h5tools_str_t *buffer, hid_t type, int ind) if ((sign=H5Tget_sign(type))>=0) { if (H5T_SGN_NONE==sign) { sign_s = " unsigned"; - } + } else if (H5T_SGN_2==sign) { sign_s = ""; - } + } else { sign_s = " unknown-sign"; } - } + } else { sign_s = " unknown-sign"; } @@ -727,17 +733,17 @@ print_float_type(h5tools_str_t *buffer, hid_t type, int ind) order = H5Tget_order(type); if (H5T_ORDER_LE==order) { order_s = " little-endian"; - } + } else if (H5T_ORDER_BE==order) { order_s = " big-endian"; - } + } else if (H5T_ORDER_VAX==order) { order_s = " mixed-endian"; - } + } else { order_s = " unknown-byte-order"; } - } + } else { order_s = ""; } @@ -949,13 +955,13 @@ print_enum_type(h5tools_str_t *buffer, hid_t type, int ind) h5tools_str_append(buffer, "0x"); for(j = 0; j < dst_size; j++) h5tools_str_append(buffer, "%02x", value[i*dst_size+j]); - } + } else if(H5T_SGN_NONE == H5Tget_sign(native)) { /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" *strangely, unless use another pointer "copy".*/ copy = value + i * dst_size; h5tools_str_append(buffer, HSIZE_T_FORMAT, *((unsigned long long*)((void*)copy))); - } + } else { /*On SGI Altix(cobalt), wrong values were printed out with "value+i*dst_size" *strangely, unless use another pointer "copy".*/ @@ -1074,7 +1080,7 @@ print_string_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) if (H5Tis_variable_str(type)) { h5tools_str_append(buffer, "variable-length"); - } + } else { h5tools_str_append(buffer, "%lu-byte", (unsigned long)H5Tget_size(type)); } @@ -1108,10 +1114,10 @@ print_reference_type(h5tools_str_t *buffer, hid_t type, int H5_ATTR_UNUSED ind) if (H5Tequal(type, H5T_STD_REF_OBJ)==TRUE) { h5tools_str_append(buffer, "object reference"); - } + } else if (H5Tequal(type, H5T_STD_REF_DSETREG)==TRUE) { h5tools_str_append(buffer, "dataset region reference"); - } + } else { h5tools_str_append(buffer, "%lu-byte unknown reference", (unsigned long)H5Tget_size(type)); @@ -1220,7 +1226,7 @@ print_array_type(h5tools_str_t *buffer, hid_t type, int ind) h5tools_str_append(buffer, "]"); HDfree(dims); - } + } else h5tools_str_append(buffer, " [SCALAR]\n", rawoutstream); @@ -1324,10 +1330,8 @@ print_type(h5tools_str_t *buffer, hid_t type, int ind) } /* end if */ /* Print the type */ - if((!simple_output_g && print_native_type(buffer, type, ind)) || + if(print_native_type(buffer, type, ind) || print_ieee_type(buffer, type, ind) || - print_int_type(buffer, type, ind) || - print_float_type(buffer, type, ind) || print_cmpd_type(buffer, type, ind) || print_enum_type(buffer, type, ind) || print_string_type(buffer, type, ind) || @@ -1371,7 +1375,7 @@ dump_dataset_values(hid_t dset) h5tools_context_t ctx; /* print context */ h5tool_format_t outputformat; h5tool_format_t *info = &ls_dataformat; - + hid_t f_type = H5Dget_type(dset); size_t size = H5Tget_size(f_type); @@ -1402,7 +1406,7 @@ dump_dataset_values(hid_t dset) outputformat.elmt_suf1 = " "; outputformat.str_locale = ESCAPE_HTML; - } + } else { if (no_line_wrap_g) { outputformat.line_per_line = 1; @@ -1422,7 +1426,7 @@ dump_dataset_values(hid_t dset) outputformat.cmpd_pre = NULL; outputformat.cmpd_suf = NULL; outputformat.cmpd_sep = NULL; - + outputformat.vlen_sep = NULL; outputformat.vlen_pre = NULL; outputformat.vlen_suf = NULL; @@ -1439,7 +1443,7 @@ dump_dataset_values(hid_t dset) /* Print all data in hexadecimal format if the `-x' or `--hexdump' * command line switch was given. */ outputformat.raw = TRUE; - } + } else if (string_g && 1==size && H5T_INTEGER==H5Tget_class(f_type)) { /* Print 1-byte integer data as an ASCI character string instead of * integers if the `-s' or `--string' command-line option was given. */ @@ -1469,7 +1473,7 @@ dump_dataset_values(hid_t dset) H5Tclose(f_type); h5tools_str_close(&buffer); - + PRINTVALSTREAM(rawoutstream, "\n"); } @@ -1517,7 +1521,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain h5tools_str_reset(&buffer); h5tools_str_append(&buffer, " Attribute: "); - + print_string(&buffer, attr_name, TRUE); if((attr = H5Aopen(obj, attr_name, H5P_DEFAULT))) { @@ -1576,7 +1580,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain outputformat.line_cont = " "; outputformat.str_repeat = 8; - } + } else { h5tools_str_reset(&buffer); h5tools_str_append(&buffer, " Data:\n"); @@ -1607,7 +1611,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain outputformat.cmpd_pre = NULL; outputformat.cmpd_suf = NULL; outputformat.cmpd_sep = NULL; - + outputformat.vlen_sep = NULL; outputformat.vlen_pre = NULL; outputformat.vlen_suf = NULL; @@ -1651,7 +1655,7 @@ list_attr(hid_t obj, const char *attr_name, const H5A_info_t H5_ATTR_UNUSED *ain H5Sclose(space); H5Tclose(type); H5Aclose(attr); - } + } h5tools_str_close(&buffer); PRINTVALSTREAM(rawoutstream, "\n"); @@ -1708,7 +1712,7 @@ dataset_list1(hid_t dset) h5tools_str_append(&buffer, "%s"HSIZE_T_FORMAT, i?", ":"", cur_size[i]); if (max_size[i]==H5S_UNLIMITED) { h5tools_str_append(&buffer, "/%s", "Inf"); - } + } else if (max_size[i]!=cur_size[i] || verbose_g>0) { h5tools_str_append(&buffer, "/"HSIZE_T_FORMAT, max_size[i]); } @@ -1755,6 +1759,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) size_t cd_nelmts; /* filter client number of values */ size_t cd_num; /* filter client data counter */ char f_name[256]; /* filter/file name */ + char dset_name[256]; /* filter/file name */ char s[64]; /* temporary string buffer */ off_t f_offset; /* offset in external file */ hsize_t f_size; /* bytes used in external file */ @@ -1764,6 +1769,7 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) double utilization; /* percent utilization of storage */ H5T_class_t tclass; /* datatype class identifier */ int i; + H5D_layout_t stl; hsize_t curr_pos = 0; /* total data element position */ h5tools_str_t buffer; /* string into which to render */ h5tools_context_t ctx; /* print context */ @@ -1779,20 +1785,71 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) space = H5Dget_space(dset); type = H5Dget_type(dset); - /* Print information about chunked storage */ - if (H5D_CHUNKED==H5Pget_layout(dcpl)) { - hsize_t chsize[64]; /* chunk size in elements */ - - ndims = H5Pget_chunk(dcpl, (int)NELMTS(chsize), chsize/*out*/); - h5tools_str_append(&buffer, " %-10s {", "Chunks:"); - total = H5Tget_size(type); - for (i=0; i<ndims; i++) { - h5tools_str_append(&buffer, "%s"HSIZE_T_FORMAT, i?", ":"", chsize[i]); - total *= chsize[i]; - } - h5tools_str_append(&buffer, "} "HSIZE_T_FORMAT" bytes\n", total); + stl = H5Pget_layout(dcpl); + switch (stl) { + case H5D_CHUNKED: + { + hsize_t chsize[64]; /* chunk size in elements */ + + ndims = H5Pget_chunk(dcpl, (int)NELMTS(chsize), chsize/*out*/); + h5tools_str_append(&buffer, " %-10s {", "Chunks:"); + total = H5Tget_size(type); + for (i=0; i<ndims; i++) { + h5tools_str_append(&buffer, "%s"HSIZE_T_FORMAT, i?", ":"", chsize[i]); + total *= chsize[i]; + } + h5tools_str_append(&buffer, "} "HSIZE_T_FORMAT" bytes\n", total); + } + break; + case H5D_COMPACT: + break; + case H5D_CONTIGUOUS: + /* Print information about external storage */ + if((nf = H5Pget_external_count(dcpl)) > 0) { + for(i = 0, max_len = 0; i < nf; i++) { + if(H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, NULL, NULL) < 0) + continue; + n = print_string(NULL, f_name, TRUE); + max_len = MAX(max_len, n); + } /* end for */ + h5tools_str_append(&buffer, " %-10s %d external file%s\n", + "Extern:", nf, 1==nf?"":"s"); + h5tools_str_append(&buffer, " %4s %10s %10s %10s %s\n", + "ID", "DSet-Addr", "File-Addr", "Bytes", "File"); + h5tools_str_append(&buffer, " %4s %10s %10s %10s ", + "----", "----------", "----------", "----------"); + for (i=0; i<max_len; i++) h5tools_str_append(&buffer, "-"); + h5tools_str_append(&buffer, "\n"); + for (i=0, total=0; i<nf; i++) { + if (H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, &f_offset, &f_size)<0) { + h5tools_str_append(&buffer, + " #%03d %10"H5_PRINTF_LL_WIDTH"u %10s %10s ***ERROR*** %s\n", + i, total, "", "", + i+1<nf?"Following addresses are incorrect":""); + } + else if (H5S_UNLIMITED==f_size) { + h5tools_str_append(&buffer, " #%03d %10"H5_PRINTF_LL_WIDTH"u %10"H5_PRINTF_LL_WIDTH"u %10s ", + i, total, (hsize_t)f_offset, "INF"); + print_string(&buffer, f_name, TRUE); + } + else { + h5tools_str_append(&buffer, " #%03d %10"H5_PRINTF_LL_WIDTH"u %10"H5_PRINTF_LL_WIDTH"u %10"H5_PRINTF_LL_WIDTH"u ", + i, total, (hsize_t)f_offset, f_size); + print_string(&buffer, f_name, TRUE); + } + h5tools_str_append(&buffer, "\n"); + total += f_size; + } + h5tools_str_append(&buffer, " %4s %10s %10s %10s ", + "----", "----------", "----------", "----------"); + for (i=0; i<max_len; i++) + h5tools_str_append(&buffer, "-"); + h5tools_str_append(&buffer, "\n"); + } /* end if */ + break; + default: + break; } - /* Print total raw storage size */ total = (hsize_t)H5Sget_simple_extent_npoints(space) * H5Tget_size(type); used = H5Dget_storage_size(dset); @@ -1836,49 +1893,6 @@ dataset_list2(hid_t dset, const char H5_ATTR_UNUSED *name) h5tools_str_append(&buffer, "\n"); - /* Print information about external strorage */ - if((nf = H5Pget_external_count(dcpl)) > 0) { - for(i = 0, max_len = 0; i < nf; i++) { - if(H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, NULL, NULL) < 0) - continue; - n = print_string(NULL, f_name, TRUE); - max_len = MAX(max_len, n); - } /* end for */ - h5tools_str_append(&buffer, " %-10s %d external file%s\n", - "Extern:", nf, 1==nf?"":"s"); - h5tools_str_append(&buffer, " %4s %10s %10s %10s %s\n", - "ID", "DSet-Addr", "File-Addr", "Bytes", "File"); - h5tools_str_append(&buffer, " %4s %10s %10s %10s ", - "----", "----------", "----------", "----------"); - for (i=0; i<max_len; i++) h5tools_str_append(&buffer, "-"); - h5tools_str_append(&buffer, "\n"); - for (i=0, total=0; i<nf; i++) { - if (H5Pget_external(dcpl, (unsigned)i, sizeof(f_name), f_name, &f_offset, &f_size)<0) { - h5tools_str_append(&buffer, - " #%03d %10"H5_PRINTF_LL_WIDTH"u %10s %10s ***ERROR*** %s\n", - i, total, "", "", - i+1<nf?"Following addresses are incorrect":""); - } - else if (H5S_UNLIMITED==f_size) { - h5tools_str_append(&buffer, " #%03d %10"H5_PRINTF_LL_WIDTH"u %10"H5_PRINTF_LL_WIDTH"u %10s ", - i, total, (hsize_t)f_offset, "INF"); - print_string(&buffer, f_name, TRUE); - } - else { - h5tools_str_append(&buffer, " #%03d %10"H5_PRINTF_LL_WIDTH"u %10"H5_PRINTF_LL_WIDTH"u %10"H5_PRINTF_LL_WIDTH"u ", - i, total, (hsize_t)f_offset, f_size); - print_string(&buffer, f_name, TRUE); - } - h5tools_str_append(&buffer, "\n"); - total += f_size; - } - h5tools_str_append(&buffer, " %4s %10s %10s %10s ", - "----", "----------", "----------", "----------"); - for (i=0; i<max_len; i++) - h5tools_str_append(&buffer, "-"); - h5tools_str_append(&buffer, "\n"); - } /* end if */ - /* Print information about raw data filters */ if((nf = H5Pget_nfilters(dcpl)) > 0) { for(i = 0; i < nf; i++) { @@ -2185,7 +2199,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) h5tools_str_append(&buffer, " "); /* Check if we have already seen this softlink */ - if(symlink_is_visited(iter->symlink_list, linfo->type, NULL, buf)) + if(symlink_is_visited(iter->symlink_list, linfo->type, NULL, buf)) { h5tools_str_append(&buffer, "{Already Visited}\n"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); @@ -2194,7 +2208,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); /* Add this link to the list of seen softlinks */ - if(symlink_visit_add(iter->symlink_list, linfo->type, NULL, buf) < 0) + if(symlink_visit_add(iter->symlink_list, linfo->type, NULL, buf) < 0) goto done; /* Adjust user data to specify that we are operating on the @@ -2206,7 +2220,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) if(!recursive_g) grp_literal_g = TRUE; /* Recurse through the soft link */ - if(visit_obj(iter->fid, name, iter) < 0) + if(visit_obj(iter->fid, name, iter) < 0) { grp_literal_g = orig_grp_literal; goto done; @@ -2239,7 +2253,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) else if (no_dangling_link_g && ret == 0) iter->symlink_list->dangle_link = TRUE; - if(H5Lunpack_elink_val(buf, linfo->u.val_size, NULL, &filename, &path) < 0) + if(H5Lunpack_elink_val(buf, linfo->u.val_size, NULL, &filename, &path) < 0) goto done; h5tools_str_append(&buffer, "External Link {"); @@ -2260,7 +2274,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) h5tools_str_append(&buffer, " "); /* Check if we have already seen this elink */ - if(symlink_is_visited(iter->symlink_list, linfo->type, filename, path)) + if(symlink_is_visited(iter->symlink_list, linfo->type, filename, path)) { h5tools_str_append(&buffer, "{Already Visited}\n"); h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); @@ -2269,7 +2283,7 @@ list_lnk(const char *name, const H5L_info_t *linfo, void *_iter) h5tools_render_element(rawoutstream, info, &ctx, &buffer, &curr_pos, (size_t)info->line_ncols, (hsize_t)0, (hsize_t)0); /* Add this link to the list of seen elinks */ - if(symlink_visit_add(iter->symlink_list, linfo->type, filename, path) < 0) + if(symlink_visit_add(iter->symlink_list, linfo->type, filename, path) < 0) { goto done; } @@ -2319,7 +2333,7 @@ done: * * Purpose: Begins iteration on an object * - * Return: + * Return: * Success: 0 * Failure: -1 * @@ -2478,7 +2492,7 @@ get_width(void) * * Purpose: check if command line arguments are valid * - * Return: + * Return: * Success: TRUE (1) * Failure: FALSE (0) * @@ -2486,19 +2500,19 @@ get_width(void) * Jonathan Kim (06/15/2010) * *-------------------------------------------------------------------------*/ -static hbool_t +static hbool_t is_valid_args(void) { hbool_t ret = TRUE; - if(recursive_g && grp_literal_g) + if(recursive_g && grp_literal_g) { HDfprintf(rawerrorstream, "Error: 'recursive' option not compatible with 'group info' option!\n\n"); ret = FALSE; goto out; } - if(no_dangling_link_g && !follow_symlink_g) + if(no_dangling_link_g && !follow_symlink_g) { HDfprintf(rawerrorstream, "Error: --no-dangling-links must be used along with --follow-symlinks option!\n\n"); ret = FALSE; @@ -2876,7 +2890,7 @@ main(int argc, const char *argv[]) if(x) HDfree(oname); - for(u=0; u < symlink_list.nused; u++) + for(u=0; u < symlink_list.nused; u++) { if (symlink_list.objs[u].type == H5L_TYPE_EXTERNAL) HDfree(symlink_list.objs[u].file); @@ -2884,7 +2898,7 @@ main(int argc, const char *argv[]) HDfree(symlink_list.objs[u].path); } HDfree(symlink_list.objs); - + /* if no-dangling-links option specified and dangling link found */ if (no_dangling_link_g && iter.symlink_list->dangle_link) err_exit = 1; diff --git a/tools/h5repack/CMakeLists.txt b/tools/h5repack/CMakeLists.txt index 42d9f3d..3b31078 100644 --- a/tools/h5repack/CMakeLists.txt +++ b/tools/h5repack/CMakeLists.txt @@ -73,14 +73,14 @@ if (BUILD_TESTING) TARGET ${HDF5_TOOL_PLUGIN_LIB_TARGET} POST_BUILD COMMAND ${CMAKE_COMMAND} - ARGS -E copy_if_different + ARGS -E copy_if_different "$<TARGET_FILE:${HDF5_TOOL_PLUGIN_LIB_TARGET}>" "${CMAKE_BINARY_DIR}/plugins/$<TARGET_FILE_NAME:${HDF5_TOOL_PLUGIN_LIB_TARGET}>" ) endif (BUILD_SHARED_LIBS) include (CMakeTests.cmake) - + endif (BUILD_TESTING) ############################################################################## @@ -98,8 +98,5 @@ endif (BUILD_TESTING) install ( TARGETS h5repack - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) diff --git a/tools/h5stat/CMakeLists.txt b/tools/h5stat/CMakeLists.txt index 5230693..66c8c7e 100644 --- a/tools/h5stat/CMakeLists.txt +++ b/tools/h5stat/CMakeLists.txt @@ -27,7 +27,7 @@ if (BUILD_TESTING) TARGET_C_PROPERTIES (h5stat_gentest STATIC " " " ") target_link_libraries (h5stat_gentest ${HDF5_LIB_TARGET}) set_target_properties (h5stat_gentest PROPERTIES FOLDER generator/tools) - + #add_test (NAME h5stat_gentest COMMAND $<TARGET_FILE:h5stat_gentest>) endif (HDF5_BUILD_GENERATORS AND NOT BUILD_SHARED_LIBS) @@ -49,8 +49,5 @@ endif (BUILD_TESTING) install ( TARGETS h5stat - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) diff --git a/tools/lib/CMakeLists.txt b/tools/lib/CMakeLists.txt index dbd72cd..a103f3d 100644 --- a/tools/lib/CMakeLists.txt +++ b/tools/lib/CMakeLists.txt @@ -48,7 +48,7 @@ H5_SET_LIB_OPTIONS ( HDF5_TOOLS_LIB_NAME_DEBUG ) #set_target_properties (${HDF5_TOOLS_LIB_TARGET} PROPERTIES COMPILE_DEFINITIONS H5DIFF_DEBUG) -set_target_properties (${HDF5_TOOLS_LIB_TARGET} PROPERTIES +set_target_properties (${HDF5_TOOLS_LIB_TARGET} PROPERTIES FOLDER libraries/tools INTERFACE_INCLUDE_DIRECTORIES "$<INSTALL_INTERFACE:$<INSTALL_PREFIX>/include>" ) @@ -60,7 +60,7 @@ set_target_properties (${HDF5_TOOLS_LIB_TARGET} PROPERTIES ############################################################################## #----------------------------------------------------------------------------- -# Add file(s) to CMake Install +# Add file(s) to CMake Install #----------------------------------------------------------------------------- install ( FILES @@ -78,7 +78,7 @@ if (HDF5_EXPORTED_TARGETS) if (BUILD_SHARED_LIBS) INSTALL_TARGET_PDB (${HDF5_TOOLS_LIB_TARGET} ${HDF5_INSTALL_BIN_DIR} toolslibraries) endif (BUILD_SHARED_LIBS) - + install ( TARGETS ${HDF5_TOOLS_LIB_TARGET} @@ -87,5 +87,6 @@ if (HDF5_EXPORTED_TARGETS) LIBRARY DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT toolslibraries ARCHIVE DESTINATION ${HDF5_INSTALL_LIB_DIR} COMPONENT toolslibraries RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolslibraries + FRAMEWORK DESTINATION ${HDF5_INSTALL_FWRK_DIR} COMPONENT toolslibraries ) endif (HDF5_EXPORTED_TARGETS) diff --git a/tools/misc/CMakeLists.txt b/tools/misc/CMakeLists.txt index e0f94d0..0bab2e6 100644 --- a/tools/misc/CMakeLists.txt +++ b/tools/misc/CMakeLists.txt @@ -80,8 +80,5 @@ endif (BUILD_TESTING) install ( TARGETS h5debug h5repart h5mkgrp - RUNTIME DESTINATION - ${HDF5_INSTALL_BIN_DIR} - COMPONENT - toolsapplications + RUNTIME DESTINATION ${HDF5_INSTALL_BIN_DIR} COMPONENT toolsapplications ) diff --git a/tools/perform/chunk.c b/tools/perform/chunk.c index 625809a..74bcc46 100644 --- a/tools/perform/chunk.c +++ b/tools/perform/chunk.c @@ -33,8 +33,8 @@ # include <string.h> #endif - -#if !defined(H5_HAVE_ATTRIBUTE) || defined __cplusplus +/* Solaris Studio defines attribute, but for the attributes we need */ +#if !defined(H5_HAVE_ATTRIBUTE) || defined __cplusplus || defined(__SUNPRO_C) # undef __attribute__ # define __attribute__(X) /*void*/ # define H5_ATTR_UNUSED /*void*/ diff --git a/tools/perform/overhead.c b/tools/perform/overhead.c index 0e9166c..0288ffa 100644 --- a/tools/perform/overhead.c +++ b/tools/perform/overhead.c @@ -46,7 +46,8 @@ # include <io.h> #endif -#ifndef H5_HAVE_ATTRIBUTE +/* Solaris Studio defines attribute, but for the attributes we need */ +#if !defined(H5_HAVE_ATTRIBUTE) || defined __cplusplus || defined(__SUNPRO_C) # undef __attribute__ # define __attribute__(X) /*void*/ # define H5_ATTR_UNUSED /*void*/ |