From 71d8975b54310631bf8354e8a2e6751c03e4b2ba Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Mon, 30 Oct 2023 12:45:31 -0500 Subject: Update version to 1.14.4-1. (#3799) Clean 1.14.3 entries from RELEASE.txt. Add hdf5-1.14.3-RELEASE.txt contents to HISTORY-1_14.txt. --- README.md | 2 +- c++/src/cpp_doc_config | 2 +- config/cmake/scripts/HDF5config.cmake | 2 +- configure.ac | 2 +- java/src/hdf/hdf5lib/H5.java | 4 +- java/test/TestH5.java | 4 +- release_docs/HISTORY-1_14.txt | 703 +++++++++++++++++++++ release_docs/RELEASE.txt | 345 +--------- src/H5public.h | 4 +- .../h5repack_layout.h5-plugin_version_test.ddl | 14 +- 10 files changed, 733 insertions(+), 349 deletions(-) diff --git a/README.md b/README.md index 306ca8a..dd99efa 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -HDF5 version 1.14.3-1 currently under development +HDF5 version 1.14.4-1 currently under development ![HDF5 Logo](doxygen/img/HDF5.png) diff --git a/c++/src/cpp_doc_config b/c++/src/cpp_doc_config index bf544f4..c512872 100644 --- a/c++/src/cpp_doc_config +++ b/c++/src/cpp_doc_config @@ -38,7 +38,7 @@ PROJECT_NAME = # could be handy for archiving the generated documentation or if some version # control system is used. -PROJECT_NUMBER = "1.14.3-1, currently under development" +PROJECT_NUMBER = "1.14.4-1, currently under development" # Using the PROJECT_BRIEF tag one can provide an optional one line description # for a project that appears at the top of each page and should give viewer a diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index 36020ee..bbd0be4 100644 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -37,7 +37,7 @@ cmake_minimum_required (VERSION 3.18) # CTEST_SOURCE_NAME - source folder ############################################################################## -set (CTEST_SOURCE_VERSION "1.14.3") +set (CTEST_SOURCE_VERSION "1.14.4") set (CTEST_SOURCE_VERSEXT "-1") ############################################################################## diff --git a/configure.ac b/configure.ac index 57cebc5..4033953 100644 --- a/configure.ac +++ b/configure.ac @@ -22,7 +22,7 @@ AC_PREREQ([2.71]) ## NOTE: Do not forget to change the version number here when we do a ## release!!! ## -AC_INIT([HDF5], [1.14.3-1], [help@hdfgroup.org]) +AC_INIT([HDF5], [1.14.4-1], [help@hdfgroup.org]) AC_CONFIG_SRCDIR([src/H5.c]) AC_CONFIG_HEADERS([src/H5config.h]) diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 0fcdd83..60eca65 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -231,7 +231,7 @@ import org.slf4j.LoggerFactory; * which prints out the HDF5 error stack, as described in the HDF5 C API @ref H5Eprint(). This * may be used by Java exception handlers to print out the HDF5 error stack.
* - * @version HDF5 1.14.3
+ * @version HDF5 1.14.4
* See also: * @ref HDFARRAY hdf.hdf5lib.HDFArray
* @ref HDF5CONST hdf.hdf5lib.HDF5Constants
@@ -273,7 +273,7 @@ public class H5 implements java.io.Serializable { * * Make sure to update the versions number when a different library is used. */ - public final static int LIB_VERSION[] = {1, 14, 3}; + public final static int LIB_VERSION[] = {1, 14, 4}; /** * @ingroup JH5 diff --git a/java/test/TestH5.java b/java/test/TestH5.java index e178b66..fd1a926 100644 --- a/java/test/TestH5.java +++ b/java/test/TestH5.java @@ -313,7 +313,7 @@ public class TestH5 { @Test public void testH5get_libversion() { - int libversion[] = {1, 14, 3}; + int libversion[] = {1, 14, 4}; try { H5.H5get_libversion(libversion); @@ -354,7 +354,7 @@ public class TestH5 { @Test public void testH5check_version() { - int majnum = 1, minnum = 14, relnum = 3; + int majnum = 1, minnum = 14, relnum = 4; try { H5.H5check_version(majnum, minnum, relnum); diff --git a/release_docs/HISTORY-1_14.txt b/release_docs/HISTORY-1_14.txt index 9f60f99..af3cc32 100644 --- a/release_docs/HISTORY-1_14.txt +++ b/release_docs/HISTORY-1_14.txt @@ -3,12 +3,715 @@ HDF5 History This file contains development history of the HDF5 1.14 branch +04. Release Information for hdf5-1.14.3 03. Release Information for hdf5-1.14.2 02. Release Information for hdf5-1.14.1 01. Release Information for hdf5-1.14.0 [Search on the string '%%%%' for section breaks of each release.] +%%%%1.14.3%%%% + +HDF5 version 1.14.3 released on 2023-10-27 +================================================================================ + + +INTRODUCTION +============ + +This document describes the differences between this release and the previous +HDF5 release. It contains information on the platforms tested and known +problems in this release. For more details check the HISTORY*.txt files in the +HDF5 source. + +Note that documentation in the links below will be updated at the time of each +final release. + +Links to HDF5 documentation can be found on The HDF5 web page: + + https://portal.hdfgroup.org/display/HDF5/HDF5 + +The official HDF5 releases can be obtained from: + + https://www.hdfgroup.org/downloads/hdf5/ + +Changes from release to release and new features in the HDF5-1.14.x release series +can be found at: + + https://portal.hdfgroup.org/display/HDF5/Release+Specific+Information + +If you have any questions or comments, please send them to the HDF Help Desk: + + help@hdfgroup.org + + +CONTENTS +======== + +- New Features +- Support for new platforms and languages +- Bug Fixes since HDF5-1.14.2 +- Platforms Tested +- Known Problems +- CMake vs. Autotools installations + + +New Features +============ + + Configuration: + ------------- + - Improved support for Intel oneAPI + + * Separates the old 'classic' Intel compiler settings and warnings + from the oneAPI settings + * Uses `-check nouninit` in debug builds to avoid false positives + when building H5_buildiface with `-check all` + * Both Autotools and CMake + + - Added new options for CMake and Autotools to control the Doxygen + warnings as errors setting. + + * HDF5_ENABLE_DOXY_WARNINGS: ON/OFF (Default: ON) + * --enable-doxygen-errors: enable/disable (Default: enable) + + The default will fail to compile if the doxygen parsing generates warnings. + The option can be disabled for certain versions of doxygen with parsing + issues. i.e. 1.9.5, 1.9.8. + + Addresses GitHub issue #3398 + + - Added support for AOCC and classic Flang w/ the Autotools + + * Adds a config/clang-fflags options file to support Flang + * Corrects missing "-Wl," from linker options in the libtool wrappers + when using Flang, the MPI Fortran compiler wrappers, and building + the shared library. This would often result in unrecognized options + like -soname. + * Enable -nomp w/ Flang to avoid linking to the OpenMPI library. + + CMake can build the parallel, shared library w/ Fortran using AOCC + and Flang, so no changes were needed for that build system. + + Fixes GitHub issues #3439, #1588, #366, #280 + + - Converted the build of libaec and zlib to use FETCH_CONTENT with CMake. + + Using the CMake FetchContent module, the external filters can populate + content at configure time via any method supported by the ExternalProject + module. Whereas ExternalProject_Add() downloads at build time, the + FetchContent module makes content available immediately, allowing the + configure step to use the content in commands like add_subdirectory(), + include() or file() operations. + + Removed HDF options for using FETCH_CONTENT explicitly: + BUILD_SZIP_WITH_FETCHCONTENT:BOOL + BUILD_ZLIB_WITH_FETCHCONTENT:BOOL + + - Thread-safety + static library disabled on Windows w/ CMake + + The thread-safety feature requires hooks in DllMain(), which is only + present in the shared library. + + We previously just warned about this, but now any CMake configuration + that tries to build thread-safety and the static library will fail. + This cannot be overridden with ALLOW_UNSUPPORTED. + + Fixes GitHub issue #3613 + + - Autotools builds now build the szip filter by default when an appropriate + library is found + + Since libaec is prevalent and BSD-licensed for both encoding and + decoding, we build the szip filter by default now. + + Both autotools and CMake build systems will process the szip filter the same as + the zlib filter is processed. + + - Removed CMake cross-compiling variables + + * HDF5_USE_PREGEN + * HDF5_BATCH_H5DETECT + + These were used to work around H5detect and H5make_libsettings and + are no longer required. + + - Running H5make_libsettings is no longer required for cross-compiling + + The functionality of H5make_libsettings is now handled via template files, + so H5make_libsettings has been removed. + + - Running H5detect is no longer required for cross-compiling + + The functionality of H5detect is now exercised at library startup, + so H5detect has been removed. + + + Library: + -------- + - Added a simple cache to the read-only S3 (ros3) VFD + + The read-only S3 VFD now caches the first N bytes of a file stored + in S3 to avoid a lot of small I/O operations when opening files. + This cache is per-file and created when the file is opened. + + N is currently 16 MiB or the size of the file, whichever is smaller. + + Addresses GitHub issue #3381 + + - Added new API function H5Pget_actual_selection_io_mode() + + This function allows the user to determine if the library performed + selection I/O, vector I/O, or scalar (legacy) I/O during the last HDF5 + operation performed with the provided DXPL. + + + Parallel Library: + ----------------- + - Added optimized support for the parallel compression feature when + using the multi-dataset I/O API routines collectively + + Previously, calling H5Dwrite_multi/H5Dread_multi collectively in parallel + with a list containing one or more filtered datasets would cause HDF5 to + break out of the optimized multi-dataset I/O mode and instead perform I/O + by looping over each dataset in the I/O request. The library has now been + updated to perform I/O in a more optimized manner in this case by first + performing I/O on all the filtered datasets at once and then performing + I/O on all the unfiltered datasets at once. + + - Changed H5Pset_evict_on_close so that it can be called with a parallel + build of HDF5 + + Previously, H5Pset_evict_on_close would always fail when called from a + parallel build of HDF5, stating that the feature is not supported with + parallel HDF5. This failure would occur even if a parallel build of HDF5 + was used with a serial HDF5 application. H5Pset_evict_on_close can now + be called regardless of the library build type and the library will + instead fail during H5Fcreate/H5Fopen if the "evict on close" property + has been set to true and the file is being opened for parallel access + with more than 1 MPI process. + + + Fortran Library: + ---------------- + - Fixed an uninitialized error return value for hdferr + to return the error state of the h5aopen_by_idx_f API. + + - Added h5pget_vol_cap_flags_f and related Fortran VOL + capability definitions. + + - Fortran async APIs H5A, H5D, H5ES, H5G, H5F, H5L and H5O were added. + + - Added Fortran APIs: + h5pset_selection_io_f, h5pget_selection_io_f, + h5pget_actual_selection_io_mode_f, + h5pset_modify_write_buf_f, h5pget_modify_write_buf_f + + - Added Fortran APIs: + h5get_free_list_sizes_f, h5dwrite_chunk_f, h5dread_chunk_f, + h5fget_info_f, h5lvisit_f, h5lvisit_by_name_f, + h5pget_no_selection_io_cause_f, h5pget_mpio_no_collective_cause_f, + h5sselect_shape_same_f, h5sselect_intersect_block_f, + h5pget_file_space_page_size_f, h5pset_file_space_page_size_f, + h5pget_file_space_strategy_f, h5pset_file_space_strategy_f + + - Removed "-commons" linking option on Darwin, as COMMON and EQUIVALENCE + are no longer used in the Fortran source. + + Fixes GitHub issue #3571 + + C++ Library: + ------------ + - + + + Java Library: + ------------- + - + + + Tools: + ------ + - + + + High-Level APIs: + ---------------- + - Added Fortran HL API: h5doappend_f + + + C Packet Table API: + ------------------- + - + + + Internal header file: + --------------------- + - + + + Documentation: + -------------- + - + + +Support for new platforms, languages and compilers +================================================== + - + + +Bug Fixes since HDF5-1.14.2 release +=================================== + Library + ------- + - Fixed some issues with chunk index metadata not getting read + collectively when collective metadata reads are enabled + + When looking up dataset chunks during I/O, the parallel library + temporarily disables collective metadata reads since it's generally + unlikely that the application will read the same chunks from all + MPI ranks. Leaving collective metadata reads enabled during + chunk lookups can lead to hangs or other bad behavior depending + on the chunk indexing structure used for the dataset in question. + However, due to the way that dataset chunk index metadata was + previously loaded in a deferred manner, this could mean that + the metadata for the main chunk index structure or its + accompanying pieces of metadata (e.g., fixed array data blocks) + could end up being read independently if these chunk lookup + operations are the first chunk index-related operation that + occurs on a dataset. This behavior is generally observed when + opening a dataset for which the metadata isn't in the metadata + cache yet and then immediately performing I/O on that dataset. + This behavior is not generally observed when creating a dataset + and then performing I/O on it, as the relevant metadata will + usually be in the metadata cache as a side effect of creating + the chunk index structures during dataset creation. + + This issue has been fixed by adding callbacks to the different + chunk indexing structure classes that allow more explicit control + over when chunk index metadata gets loaded. When collective + metadata reads are enabled, the necessary index metadata will now + get loaded collectively by all MPI ranks at the start of dataset + I/O to ensure that the ranks don't unintentionally read this + metadata independently further on. These changes fix collective + loading of the main chunk index structure, as well as v2 B-tree + root nodes, extensible array index blocks and fixed array data + blocks. There are still pieces of metadata that cannot currently + be loaded collectively, however, such as extensible array data + blocks, data block pages and super blocks, as well as fixed array + data block pages. These pieces of metadata are not necessarily + read in by all MPI ranks since this depends on which chunks the + ranks have selected in the dataset. Therefore, reading of these + pieces of metadata remains an independent operation. + + - Fixed potential hangs in parallel library during collective I/O with + independent metadata writes + + When performing collective parallel writes to a dataset where metadata + writes are requested as (or left as the default setting of) independent, + hangs could potentially occur during metadata cache sync points. This + was due to incorrect management of the internal state tracking whether + an I/O operation should be collective or not, causing the library to + attempt collective writes of metadata when they were meant to be + independent writes. During the metadata cache sync points, if the number + of cache entries being flushed was a multiple of the number of MPI ranks + in the MPI communicator used to access the HDF5 file, an equal amount of + collective MPI I/O calls were made and the dataset write call would be + successful. However, when the number of cache entries being flushed was + NOT a multiple of the number of MPI ranks, the ranks with more entries + than others would get stuck in an MPI_File_set_view call, while other + ranks would get stuck in a post-write MPI_Barrier call. This issue has + been fixed by correctly switching to independent I/O temporarily when + writing metadata independently during collective dataset I/O. + + - Fixed a bug with the way the Subfiling VFD assigns I/O concentrators + + During a file open operation, the Subfiling VFD determines the topology + of the application and uses that to select a subset of MPI ranks that + I/O will be forwarded to, called I/O concentrators. The code for this + had previously assumed that the parallel job launcher application (e.g., + mpirun, srun, etc.) would distribute MPI ranks sequentially to a node's + processors until all processors on that node have been assigned before + going on to the next node. When the launcher application mapped MPI ranks + to nodes in a different fashion, such as round-robin, this could cause + the Subfiling VFD to incorrectly map MPI ranks as I/O concentrators, + leading to missing subfiles. + + - Fixed a file space allocation bug in the parallel library for chunked + datasets + + With the addition of support for incremental file space allocation for + chunked datasets with filters applied to them that are created/accessed + in parallel, a bug was introduced to the library's parallel file space + allocation code. This could cause file space to not be allocated correctly + for datasets without filters applied to them that are created with serial + file access and later opened with parallel file access. In turn, this could + cause parallel writes to those datasets to place incorrect data in the file. + + - Fixed an assertion failure in Parallel HDF5 when a file can't be created + due to an invalid library version bounds setting + + An assertion failure could occur in H5MF_settle_raw_data_fsm when a file + can't be created with Parallel HDF5 due to specifying the use of a paged, + persistent file free space manager + (H5Pset_file_space_strategy(..., H5F_FSPACE_STRATEGY_PAGE, 1, ...)) with + an invalid library version bounds combination + (H5Pset_libver_bounds(..., H5F_LIBVER_EARLIEST, H5F_LIBVER_V18)). This + has now been fixed. + + - Fixed an assertion in a previous fix for CVE-2016-4332 + + An assert could fail when processing corrupt files that have invalid + shared message flags (as in CVE-2016-4332). + + The assert statement in question has been replaced with pointer checks + that don't raise errors. Since the function is in cleanup code, we do + our best to close and free things, even when presented with partially + initialized structs. + + Fixes CVE-2016-4332 and HDFFV-9950 (confirmed via the cve_hdf5 repo) + + - Fixed performance regression with some compound type conversions + + In-place type conversion was introduced for most use cases in 1.14.2. + While being able to use the read buffer for type conversion potentially + improves performance by performing the entire I/O at once, it also + disables the optimized compound type conversion used when the destination + is a subset of the source. Disabled in-place type conversion when using + this optimized conversion and there is no benefit in terms of the I/O + size. + + - Reading a H5std_string (std::string) via a C++ DataSet previously + truncated the string at the first null byte as if reading a C string. + Fixed length datasets are now read into H5std_string as a fixed length + string of the appropriate size. Variable length datasets will still be + truncated at the first null byte. + + Fixes Github issue #3034 + + - Fixed write buffer overflow in H5O__alloc_chunk + + The overflow was found by OSS-Fuzz https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=58658 + + Java Library + ------------ + - + + + Configuration + ------------- + - Fixes the ordering of INCLUDES when building with CMake + + Include directories in the source or build tree should come before other + directories to prioritize headers in the sources over installed ones. + + Fixes GitHub #1027 + + - The accum test now passes on macOS 12+ (Monterey) w/ CMake + + Due to changes in the way macOS handles LD_LIBRARY_PATH, the accum test + started failing on macOS 12+ when building with CMake. CMake has been + updated to set DYLD_LIBRARY_PATH on macOS and the test now passes. + + Fixes GitHub #2994, #2261, and #1289 + + - Changed the default settings used by CMake for the GZIP filter + + The default for the option HDF5_ENABLE_Z_LIB_SUPPORT was OFF. Now the default is ON. + This was done to match the defaults used by the autotools configure.ac. + In addition, the CMake message level for not finding a suitable filter library was + changed from FATAL_ERROR (which would halt the build process) to WARNING (which + will print a message to stderr). Associated files and documentation were changed to match. + + In addition, the default settings in the config/cmake/cacheinit.cmake file were changed to + allow CMake to disable building the filters if the tgz file could not be found. The option + to allow CMake to download the file from the original Github location requires setting + the ZLIB_USE_LOCALCONTENT option to OFF for gzip. And setting the LIBAEC_USE_LOCALCONTENT + option to OFF for libaec (szip). + + Fixes GitHub issue #2926 + + + Tools + ----- + - Fixed an issue with unmatched MPI messages in ph5diff + + The "manager" MPI rank in ph5diff was unintentionally sending "program end" + messages to its workers twice, leading to an error from MPICH similar to the + following: + + Abort(810645519) on node 1 (rank 1 in comm 0): Fatal error in internal_Finalize: Other MPI error, error stack: + internal_Finalize(50)...........: MPI_Finalize failed + MPII_Finalize(394)..............: + MPIR_Comm_delete_internal(1224).: Communicator (handle=44000000) being freed has 1 unmatched message(s) + MPIR_Comm_release_always(1250)..: + MPIR_finalize_builtin_comms(154): + + + Performance + ------------- + - + + + Fortran API + ----------- + - + + + High-Level Library + ------------------ + - + + + Fortran High-Level APIs + ----------------------- + - + + + Documentation + ------------- + - + + + F90 APIs + -------- + - + + + C++ APIs + -------- + - + + + Testing + ------- + - Disabled running of MPI Atomicity tests for OpenMPI major versions < 5 + + Support for MPI atomicity operations is not implemented for major + versions of OpenMPI less than version 5. This would cause the MPI + atomicity tests for parallel HDF5 to sporadically fail when run + with OpenMPI. Testphdf5 now checks if OpenMPI is being used and will + skip running the atomicity tests if the major version of OpenMPI is + < 5. + + - Fixed Fortran 2003 test with gfortran-v13, optimization levels O2,O3 + + Fixes failing Fortran 2003 test with gfortran, optimization level O2,O3 + with -fdefault-real-16. Fixes GH #2928. + + +Platforms Tested +=================== + + Linux 5.19.0-1023-aws GNU gcc, gfortran, g++ + #24-Ubuntu SMP x86_64 GNU/Linux (Ubuntu 11.3.0-1ubuntu1~22.04) 11.3.0 + Ubuntu 22.04 Ubuntu clang version 14.0.0-1ubuntu1 + Intel(R) oneAPI DPC++/C++ Compiler 2023.1.0 + ifort (IFORT) 2021.9.0 20230302 + (cmake and autotools) + + Linux 5.16.14-200.fc35 GNU gcc (GCC) 11.2.1 20220127 (Red Hat 11.2.1-9) + #1 SMP x86_64 GNU/Linux GNU Fortran (GCC) 11.2.1 20220127 (Red Hat 11.2.1-9) + Fedora35 clang version 13.0.0 (Fedora 13.0.0-3.fc35) + (cmake and autotools) + + Linux 5.14.21-cray_shasta_c cray-mpich/8.1.27 + #1 SMP x86_64 GNU/Linux cce/15.0.0 + (frontier) gcc/12.2.0 + (cmake) + + Linux 5.11.0-34-generic GNU gcc (GCC) 9.4.0-1ubuntu1 + #36-Ubuntu SMP x86_64 GNU/Linux GNU Fortran (GCC) 9.4.0-1ubuntu1 + Ubuntu 20.04 Ubuntu clang version 10.0.0-4ubuntu1 + Intel(R) oneAPI DPC++/C++ Compiler 2023.1.0 + ifort (IFORT) 2021.9.0 20230302 + (cmake and autotools) + + Linux 4.14.0-115.35.1.1chaos aue/openmpi/4.1.4-arm-22.1.0.12 + #1 SMP aarch64 GNU/Linux Arm C/C++/Fortran Compiler version 22.1 + (stria) (based on LLVM 13.0.1) + (cmake) + + Linux 4.14.0-115.35.1.3chaos spectrum-mpi/rolling-release + #1 SMP ppc64le GNU/Linux clang 12.0.1 + (vortex) GCC 8.3.1 + XL 2021.09.22 + (cmake) + + Linux-4.14.0-115.21.2 spectrum-mpi/rolling-release + #1 SMP ppc64le GNU/Linux clang 12.0.1, 14.0.5 + (lassen) GCC 8.3.1 + XL 16.1.1.2, 2021.09.22, 2022.08.05 + (cmake) + + Linux-4.12.14-197.99-default cray-mpich/7.7.14 + #1 SMP x86_64 GNU/Linux cce 12.0.3 + (theta) GCC 11.2.0 + llvm 9.0 + Intel 19.1.2 + + Linux 3.10.0-1160.36.2.el7.ppc64 gcc (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + #1 SMP ppc64be GNU/Linux g++ (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + Power8 (echidna) GNU Fortran (GCC) 4.8.5 20150623 (Red Hat 4.8.5-39) + + Linux 3.10.0-1160.24.1.el7 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos7 Version 4.8.5 20150623 (Red Hat 4.8.5-4) + (jelly/kituo/moohan) Version 4.9.3, Version 7.2.0, Version 8.3.0, + Version 9.1.0, Version 10.2.0 + Intel(R) C (icc), C++ (icpc), Fortran (icc) + compilers: + Version 17.0.0.098 Build 20160721 + GNU C (gcc) and C++ (g++) 4.8.5 compilers + with NAG Fortran Compiler Release 7.1(Hanzomon) + Intel(R) C (icc) and C++ (icpc) 17.0.0.098 compilers + with NAG Fortran Compiler Release 7.1(Hanzomon) + MPICH 3.1.4 compiled with GCC 4.9.3 + MPICH 3.3 compiled with GCC 7.2.0 + OpenMPI 3.1.3 compiled with GCC 7.2.0 and 4.1.2 + compiled with GCC 9.1.0 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Versions 18.4.0 and 19.10-0 + NVIDIA nvc, nvfortran and nvc++ version 22.5-0 + (autotools and cmake) + + + Linux-3.10.0-1160.0.0.1chaos openmpi-4.1.2 + #1 SMP x86_64 GNU/Linux clang 6.0.0, 11.0.1 + (quartz) GCC 7.3.0, 8.1.0 + Intel 19.0.4, 2022.2, oneapi.2022.2 + + Linux-3.10.0-1160.90.1.1chaos openmpi/4.1 + #1 SMP x86_64 GNU/Linux GCC 7.2.0 + (skybridge) Intel/19.1 + (cmake) + + Linux-3.10.0-1160.90.1.1chaos openmpi/4.1 + #1 SMP x86_64 GNU/Linux GCC 7.2.0 + (attaway) Intel/19.1 + (cmake) + + Linux-3.10.0-1160.90.1.1chaos openmpi-intel/4.1 + #1 SMP x86_64 GNU/Linux Intel/19.1.2, 21.3.0 and 22.2.0 + (chama) (cmake) + + macOS Apple M1 11.6 Apple clang version 12.0.5 (clang-1205.0.22.11) + Darwin 20.6.0 arm64 gfortran GNU Fortran (Homebrew GCC 11.2.0) 11.1.0 + (macmini-m1) Intel icc/icpc/ifort version 2021.3.0 202106092021.3.0 20210609 + + macOS Big Sur 11.3.1 Apple clang version 12.0.5 (clang-1205.0.22.9) + Darwin 20.4.0 x86_64 gfortran GNU Fortran (Homebrew GCC 10.2.0_3) 10.2.0 + (bigsur-1) Intel icc/icpc/ifort version 2021.2.0 20210228 + + Mac OS X El Capitan 10.11.6 Apple clang version 7.3.0 from Xcode 7.3 + 64-bit gfortran GNU Fortran (GCC) 5.2.0 + (osx1011test) Intel icc/icpc/ifort version 16.0.2 + + Linux 2.6.32-573.22.1.el6 GNU C (gcc), Fortran (gfortran), C++ (g++) + #1 SMP x86_64 GNU/Linux compilers: + Centos6 Version 4.4.7 20120313 + (platypus) Version 4.9.3, 5.3.0, 6.2.0 + MPICH 3.1.4 compiled with GCC 4.9.3 + PGI C, Fortran, C++ for 64-bit target on + x86_64; + Version 19.10-0 + + Windows 10 x64 Visual Studio 2019 w/ clang 12.0.0 + with MSVC-like command-line (C/C++ only - cmake) + Visual Studio 2019 w/ Intel oneAPI 2023.2 C/C++ only - cmake) + Visual Studio 2022 w/ clang 16.0.5 + with MSVC-like command-line (C/C++ only - cmake) + Visual Studio 2022 w/ Intel oneAPI 2023.2 (C/C++ only - cmake) + Visual Studio 2019 w/ MSMPI 10.1 (C only - cmake) + + +Known Problems +============== + + Building HDF5 Fortran on Windows with Intel oneAPI 2023.2 currently fails for + this release with link errors. As a result, Windows binaries for this release + will not include Fortran. The problem will be addressed in HDF5 1.14.4. + + IEEE standard arithmetic enables software to raise exceptions such as overflow, + division by zero, and other illegal operations without interrupting or halting + the program flow. The HDF5 C library intentionally performs these exceptions. + Therefore, the "-ieee=full" nagfor switch is necessary when compiling a program + to avoid stopping on an exception. + + CMake files do not behave correctly with paths containing spaces. + Do not use spaces in paths because the required escaping for handling spaces + results in very complex and fragile build files. + ADB - 2019/05/07 + + At present, metadata cache images may not be generated by parallel + applications. Parallel applications can read files with metadata cache + images, but since this is a collective operation, a deadlock is possible + if one or more processes do not participate. + + CPP ptable test fails on both VS2017 and VS2019 with Intel compiler, JIRA + issue: HDFFV-10628. This test will pass with VS2015 with Intel compiler. + + The subsetting option in ph5diff currently will fail and should be avoided. + The subsetting option works correctly in serial h5diff. + + Several tests currently fail on certain platforms: + MPI_TEST-t_bigio fails with spectrum-mpi on ppc64le platforms. + + MPI_TEST-t_subfiling_vfd and MPI_TEST_EXAMPLES-ph5_subfiling fail with + cray-mpich on theta and with XL compilers on ppc64le platforms. + + MPI_TEST_testphdf5_tldsc fails with cray-mpich 7.7 on cori and theta. + + Known problems in previous releases can be found in the HISTORY*.txt files + in the HDF5 source. Please report any new problems found to + help@hdfgroup.org. + + +CMake vs. Autotools installations +================================= +While both build systems produce similar results, there are differences. +Each system produces the same set of folders on linux (only CMake works +on standard Windows); bin, include, lib and share. Autotools places the +COPYING and RELEASE.txt file in the root folder, CMake places them in +the share folder. + +The bin folder contains the tools and the build scripts. Additionally, CMake +creates dynamic versions of the tools with the suffix "-shared". Autotools +installs one set of tools depending on the "--enable-shared" configuration +option. + build scripts + ------------- + Autotools: h5c++, h5cc, h5fc + CMake: h5c++, h5cc, h5hlc++, h5hlcc + +The include folder holds the header files and the fortran mod files. CMake +places the fortran mod files into separate shared and static subfolders, +while Autotools places one set of mod files into the include folder. Because +CMake produces a tools library, the header files for tools will appear in +the include folder. + +The lib folder contains the library files, and CMake adds the pkgconfig +subfolder with the hdf5*.pc files used by the bin/build scripts created by +the CMake build. CMake separates the C interface code from the fortran code by +creating C-stub libraries for each Fortran library. In addition, only CMake +installs the tools library. The names of the szip libraries are different +between the build systems. + +The share folder will have the most differences because CMake builds include +a number of CMake specific files for support of CMake's find_package and support +for the HDF5 Examples CMake project. + +The issues with the gif tool are: + HDFFV-10592 CVE-2018-17433 + HDFFV-10593 CVE-2018-17436 + HDFFV-11048 CVE-2020-10809 +These CVE issues have not yet been addressed and are avoided by not building +the gif tool by default. Enable building the High-Level tools with these options: + autotools: --enable-hlgiftools + cmake: HDF5_BUILD_HL_GIF_TOOLS=ON + + %%%%1.14.2%%%% HDF5 version 1.14.2 released on 2023-08-11 diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index f7cb681..4878b34 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -1,4 +1,4 @@ -HDF5 version 1.14.3-1 currently under development +HDF5 version 1.14.4-1 currently under development ================================================================================ @@ -36,7 +36,7 @@ CONTENTS - New Features - Support for new platforms and languages -- Bug Fixes since HDF5-1.14.2 +- Bug Fixes since HDF5-1.14.3 - Platforms Tested - Known Problems - CMake vs. Autotools installations @@ -47,164 +47,23 @@ New Features Configuration: ------------- - - Improved support for Intel oneAPI - - * Separates the old 'classic' Intel compiler settings and warnings - from the oneAPI settings - * Uses `-check nouninit` in debug builds to avoid false positives - when building H5_buildiface with `-check all` - * Both Autotools and CMake - - - Added new options for CMake and Autotools to control the Doxygen - warnings as errors setting. - - * HDF5_ENABLE_DOXY_WARNINGS: ON/OFF (Default: ON) - * --enable-doxygen-errors: enable/disable (Default: enable) - - The default will fail compile if the doxygen parsing generates warnings. - The option can be disabled if certain versions of doxygen have parsing - issues. i.e. 1.9.5, 1.9.8. - - Addresses GitHub issue #3398 - - - Added support for AOCC and classic Flang w/ the Autotools - - * Adds a config/clang-fflags options file to support Flang - * Corrects missing "-Wl," from linker options in the libtool wrappers - when using Flang, the MPI Fortran compiler wrappers, and building - the shared library. This would often result in unrecognized options - like -soname. - * Enable -nomp w/ Flang to avoid linking to the OpenMPI library. - - CMake can build the parallel, shared library w/ Fortran using AOCC - and Flang, so no changes were needed for that build system. - - Fixes GitHub issues #3439, #1588, #366, #280 - - - Converted the build of libaec and zlib to use FETCH_CONTENT with CMake. - - Using the CMake FetchContent module, the external filters can populate - content at configure time via any method supported by the ExternalProject - module. Whereas ExternalProject_Add() downloads at build time, the - FetchContent module makes content available immediately, allowing the - configure step to use the content in commands like add_subdirectory(), - include() or file() operations. - - Removed HDF options for using FETCH_CONTENT explicitly: - BUILD_SZIP_WITH_FETCHCONTENT:BOOL - BUILD_ZLIB_WITH_FETCHCONTENT:BOOL - - - Thread-safety + static library disabled on Windows w/ CMake - - The thread-safety feature requires hooks in DllMain(), which is only - present in the shared library. - - We previously just warned about this, but now any CMake configuration - that tries to build thread-safety and the static library will fail. - This cannot be overridden with ALLOW_UNSUPPORTED. - - Fixes GitHub issue #3613 - - - Autotools builds now build the szip filter by default when an appropriate - library is found - - Since libaec is prevalent and BSD-licensed for both encoding and - decoding, we build the szip filter by default now. - - Both autotools and CMake build systems will process the szip filter the same as - the zlib filter is processed. - - - Removed CMake cross-compiling variables - - * HDF5_USE_PREGEN - * HDF5_BATCH_H5DETECT - - These were used to work around H5detect and H5make_libsettings and - are no longer required. - - - Running H5make_libsettings is no longer required for cross-compiling - - The functionality of H5make_libsettings is now handled via template files, - so H5make_libsettings has been removed. - - - Running H5detect is no longer required for cross-compiling - - The functionality of H5detect is now exercised at library startup, - so H5detect has been removed. + - Library: -------- - - Added a simple cache to the read-only S3 (ros3) VFD - - The read-only S3 VFD now caches the first N bytes of a file stored - in S3 to avoid a lot of small I/O operations when opening files. - This cache is per-file and created when the file is opened. - - N is currently 16 MiB or the size of the file, whichever is smaller. - - Addresses GitHub issue #3381 - - - Added new API function H5Pget_actual_selection_io_mode() - - This function allows the user to determine if the library performed - selection I/O, vector I/O, or scalar (legacy) I/O during the last HDF5 - operation performed with the provided DXPL. + - Parallel Library: ----------------- - - Added optimized support for the parallel compression feature when - using the multi-dataset I/O API routines collectively - - Previously, calling H5Dwrite_multi/H5Dread_multi collectively in parallel - with a list containing one or more filtered datasets would cause HDF5 to - break out of the optimized multi-dataset I/O mode and instead perform I/O - by looping over each dataset in the I/O request. The library has now been - updated to perform I/O in a more optimized manner in this case by first - performing I/O on all the filtered datasets at once and then performing - I/O on all the unfiltered datasets at once. - - - Changed H5Pset_evict_on_close so that it can be called with a parallel - build of HDF5 - - Previously, H5Pset_evict_on_close would always fail when called from a - parallel build of HDF5, stating that the feature is not supported with - parallel HDF5. This failure would occur even if a parallel build of HDF5 - was used with a serial HDF5 application. H5Pset_evict_on_close can now - be called regardless of the library build type and the library will - instead fail during H5Fcreate/H5Fopen if the "evict on close" property - has been set to true and the file is being opened for parallel access - with more than 1 MPI process. + - Fortran Library: ---------------- - - Fixed an uninitialized error return value for hdferr - to return the error state of the h5aopen_by_idx_f API. - - - Added h5pget_vol_cap_flags_f and related Fortran VOL - capability definitions. - - - Fortran async APIs H5A, H5D, H5ES, H5G, H5F, H5L and H5O were added. - - - Added Fortran APIs: - h5pset_selection_io_f, h5pget_selection_io_f, - h5pget_actual_selection_io_mode_f, - h5pset_modify_write_buf_f, h5pget_modify_write_buf_f - - - Added Fortran APIs: - h5get_free_list_sizes_f, h5dwrite_chunk_f, h5dread_chunk_f, - h5fget_info_f, h5lvisit_f, h5lvisit_by_name_f, - h5pget_no_selection_io_cause_f, h5pget_mpio_no_collective_cause_f, - h5sselect_shape_same_f, h5sselect_intersect_block_f, - h5pget_file_space_page_size_f, h5pset_file_space_page_size_f, - h5pget_file_space_strategy_f, h5pset_file_space_strategy_f - - - Removed "-commons" linking option on Darwin, as COMMON and EQUIVALENCE - are no longer used in the Fortran source. + - - Fixes GitHub issue #3571 C++ Library: ------------ @@ -223,7 +82,7 @@ New Features High-Level APIs: ---------------- - - Added Fortran HL API: h5doappend_f + - C Packet Table API: @@ -246,138 +105,12 @@ Support for new platforms, languages and compilers - -Bug Fixes since HDF5-1.14.2 release +Bug Fixes since HDF5-1.14.3 release =================================== Library ------- - - Fixed some issues with chunk index metadata not getting read - collectively when collective metadata reads are enabled - - When looking up dataset chunks during I/O, the parallel library - temporarily disables collective metadata reads since it's generally - unlikely that the application will read the same chunks from all - MPI ranks. Leaving collective metadata reads enabled during - chunk lookups can lead to hangs or other bad behavior depending - on the chunk indexing structure used for the dataset in question. - However, due to the way that dataset chunk index metadata was - previously loaded in a deferred manner, this could mean that - the metadata for the main chunk index structure or its - accompanying pieces of metadata (e.g., fixed array data blocks) - could end up being read independently if these chunk lookup - operations are the first chunk index-related operation that - occurs on a dataset. This behavior is generally observed when - opening a dataset for which the metadata isn't in the metadata - cache yet and then immediately performing I/O on that dataset. - This behavior is not generally observed when creating a dataset - and then performing I/O on it, as the relevant metadata will - usually be in the metadata cache as a side effect of creating - the chunk index structures during dataset creation. - - This issue has been fixed by adding callbacks to the different - chunk indexing structure classes that allow more explicit control - over when chunk index metadata gets loaded. When collective - metadata reads are enabled, the necessary index metadata will now - get loaded collectively by all MPI ranks at the start of dataset - I/O to ensure that the ranks don't unintentionally read this - metadata independently further on. These changes fix collective - loading of the main chunk index structure, as well as v2 B-tree - root nodes, extensible array index blocks and fixed array data - blocks. There are still pieces of metadata that cannot currently - be loaded collectively, however, such as extensible array data - blocks, data block pages and super blocks, as well as fixed array - data block pages. These pieces of metadata are not necessarily - read in by all MPI ranks since this depends on which chunks the - ranks have selected in the dataset. Therefore, reading of these - pieces of metadata remains an independent operation. - - - Fixed potential hangs in parallel library during collective I/O with - independent metadata writes - - When performing collective parallel writes to a dataset where metadata - writes are requested as (or left as the default setting of) independent, - hangs could potentially occur during metadata cache sync points. This - was due to incorrect management of the internal state tracking whether - an I/O operation should be collective or not, causing the library to - attempt collective writes of metadata when they were meant to be - independent writes. During the metadata cache sync points, if the number - of cache entries being flushed was a multiple of the number of MPI ranks - in the MPI communicator used to access the HDF5 file, an equal amount of - collective MPI I/O calls were made and the dataset write call would be - successful. However, when the number of cache entries being flushed was - NOT a multiple of the number of MPI ranks, the ranks with more entries - than others would get stuck in an MPI_File_set_view call, while other - ranks would get stuck in a post-write MPI_Barrier call. This issue has - been fixed by correctly switching to independent I/O temporarily when - writing metadata independently during collective dataset I/O. - - - Fixed a bug with the way the Subfiling VFD assigns I/O concentrators - - During a file open operation, the Subfiling VFD determines the topology - of the application and uses that to select a subset of MPI ranks that - I/O will be forwarded to, called I/O concentrators. The code for this - had previously assumed that the parallel job launcher application (e.g., - mpirun, srun, etc.) would distribute MPI ranks sequentially among a node - until all processors on that node have been assigned before going on to - the next node. When the launcher application mapped MPI ranks to nodes - in a different fashion, such as round-robin, this could cause the Subfiling - VFD to incorrectly map MPI ranks as I/O concentrators, leading to missing - subfiles. - - - Fixed a file space allocation bug in the parallel library for chunked - datasets - - With the addition of support for incremental file space allocation for - chunked datasets with filters applied to them that are created/accessed - in parallel, a bug was introduced to the library's parallel file space - allocation code. This could cause file space to not be allocated correctly - for datasets without filters applied to them that are created with serial - file access and later opened with parallel file access. In turn, this could - cause parallel writes to those datasets to place incorrect data in the file. - - - Fixed an assertion failure in Parallel HDF5 when a file can't be created - due to an invalid library version bounds setting - - An assertion failure could occur in H5MF_settle_raw_data_fsm when a file - can't be created with Parallel HDF5 due to specifying the use of a paged, - persistent file free space manager - (H5Pset_file_space_strategy(..., H5F_FSPACE_STRATEGY_PAGE, 1, ...)) with - an invalid library version bounds combination - (H5Pset_libver_bounds(..., H5F_LIBVER_EARLIEST, H5F_LIBVER_V18)). This - has now been fixed. - - - Fixed an assertion in a previous fix for CVE-2016-4332 - - An assert could fail when processing corrupt files that have invalid - shared message flags (as in CVE-2016-4332). - - The assert statement in question has been replaced with pointer checks - that don't raise errors. Since the function is in cleanup code, we do - our best to close and free things, even when presented with partially - initialized structs. - - Fixes CVE-2016-4332 and HDFFV-9950 (confirmed via the cve_hdf5 repo) - - - Fixed performance regression with some compound type conversions - - In-place type conversion was introduced for most use cases in 1.14.2. - While being able to use the read buffer for type conversion potentially - improves performance by performing the entire I/O at once, it also - disables the optimized compound type conversion used when the destination - is a subset of the source. Disabled in-place type conversion when using - this optimized conversion and there is no benefit in terms of the I/O - size. - - - Reading a H5std_string (std::string) via a C++ DataSet previously - truncated the string at the first null byte as if reading a C string. - Fixed length datasets are now read into H5std_string as a fixed length - string of the appropriate size. Variable length datasets will still be - truncated at the first null byte. - - Fixes Github issue #3034 - - - Fixed write buffer overflow in H5O__alloc_chunk - - The overflow was found by OSS-Fuzz https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=58658 + - + Java Library ------------ @@ -386,52 +119,12 @@ Bug Fixes since HDF5-1.14.2 release Configuration ------------- - - Fixes the ordering of INCLUDES when building with CMake - - Include directories in the source or build tree should come before other - directories to prioritize headers in the sources over installed ones. - - Fixes GitHub #1027 - - - The accum test now passes on macOS 12+ (Monterey) w/ CMake - - Due to changes in the way macOS handles LD_LIBRARY_PATH, the accum test - started failing on macOS 12+ when building with CMake. CMake has been - updated to set DYLD_LIBRARY_PATH on macOS and the test now passes. - - Fixes GitHub #2994, #2261, and #1289 - - - Changed the default settings used by CMake for the GZIP filter - - The default for the option HDF5_ENABLE_Z_LIB_SUPPORT was OFF. Now the default is ON. - This was done to match the defaults used by the autotools configure.ac. - In addition, the CMake message level for not finding a suitable filter library was - changed from FATAL_ERROR (which would halt the build process) to WARNING (which - will print a message to stderr). Associated files and documentation were changed to match. - - In addition, the default settings in the config/cmake/cacheinit.cmake file were changed to - allow CMake to disable building the filters if the tgz file could not be found. The option - to allow CMake to download the file from the original Github location requires setting - the ZLIB_USE_LOCALCONTENT option to OFF for gzip. And setting the LIBAEC_USE_LOCALCONTENT - option to OFF for libaec (szip). - - Fixes GitHub issue #2926 + - Tools ----- - - Fixed an issue with unmatched MPI messages in ph5diff - - The "manager" MPI rank in ph5diff was unintentionally sending "program end" - messages to its workers twice, leading to an error from MPICH similar to the - following: - - Abort(810645519) on node 1 (rank 1 in comm 0): Fatal error in internal_Finalize: Other MPI error, error stack: - internal_Finalize(50)...........: MPI_Finalize failed - MPII_Finalize(394)..............: - MPIR_Comm_delete_internal(1224).: Communicator (handle=44000000) being freed has 1 unmatched message(s) - MPIR_Comm_release_always(1250)..: - MPIR_finalize_builtin_comms(154): + - Performance @@ -471,19 +164,7 @@ Bug Fixes since HDF5-1.14.2 release Testing ------- - - Disabled running of MPI Atomicity tests for OpenMPI major versions < 5 - - Support for MPI atomicity operations is not implemented for major - versions of OpenMPI less than version 5. This would cause the MPI - atomicity tests for parallel HDF5 to sporadically fail when run - with OpenMPI. Testphdf5 now checks if OpenMPI is being used and will - skip running the atomicity tests if the major version of OpenMPI is - < 5. - - - Fixed Fortran 2003 test with gfortran-v13, optimization levels O2,O3 - - Fixes failing Fortran 2003 test with gfortran, optimization level O2,O3 - with -fdefault-real-16. Fixes GH #2928. + - Platforms Tested diff --git a/src/H5public.h b/src/H5public.h index 2d96d1e..a40ca00 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -83,7 +83,7 @@ /** * For tweaks, bug-fixes, or development */ -#define H5_VERS_RELEASE 3 +#define H5_VERS_RELEASE 4 /** * For pre-releases like \c snap0. Empty string for official releases. */ @@ -91,7 +91,7 @@ /** * Full version string */ -#define H5_VERS_INFO "HDF5 library version: 1.14.3-1" +#define H5_VERS_INFO "HDF5 library version: 1.14.4-1" #define H5check() H5check_version(H5_VERS_MAJOR, H5_VERS_MINOR, H5_VERS_RELEASE) diff --git a/tools/test/h5repack/expected/h5repack_layout.h5-plugin_version_test.ddl b/tools/test/h5repack/expected/h5repack_layout.h5-plugin_version_test.ddl index f42d333..b614140 100644 --- a/tools/test/h5repack/expected/h5repack_layout.h5-plugin_version_test.ddl +++ b/tools/test/h5repack/expected/h5repack_layout.h5-plugin_version_test.ddl @@ -11,7 +11,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 14 3 } + PARAMS { 9 1 14 4 } } } FILLVALUE { @@ -33,7 +33,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 14 3 } + PARAMS { 9 1 14 4 } } } FILLVALUE { @@ -55,7 +55,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 14 3 } + PARAMS { 9 1 14 4 } } } FILLVALUE { @@ -77,7 +77,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 14 3 } + PARAMS { 9 1 14 4 } } } FILLVALUE { @@ -99,7 +99,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 14 3 } + PARAMS { 9 1 14 4 } } } FILLVALUE { @@ -121,7 +121,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 14 3 } + PARAMS { 9 1 14 4 } } } FILLVALUE { @@ -143,7 +143,7 @@ GROUP "/" { USER_DEFINED_FILTER { FILTER_ID 260 COMMENT dynlib4 - PARAMS { 9 1 14 3 } + PARAMS { 9 1 14 4 } } } FILLVALUE { -- cgit v0.12