From 777d9fc570f739e37e535f9d4ff35b8d9499f0e4 Mon Sep 17 00:00:00 2001 From: Dana Robinson <43805+derobins@users.noreply.github.com> Date: Tue, 20 Dec 2022 16:50:01 -0800 Subject: 1.14.0: Brings the following changesets over from develop: (#2330) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Brings the following changesets over from develop: b9244a85d9f1cc5e9bbec61ca73c0cbd9c4cf249 Align arg types of H5D_chunk_iter_op_t with H5Dget_chunk_info (#2074) 70cf2c390bc2eef8e57f8fa023341011e2d02d9d Removed idioms and misc. text clean-up (#2320) 8102fa8c972bdc0d8fd8f3dae604e070893150d6 Only document Fortran functions (#2319) 784061b15e176b9919c19a220ce278a9f4cddf0e moved onion VFD to FAPL group (#2321) 6b6bcdead66f0456ac0528683faac6a8e48b6565 Hdffv 11052 (#2315) 10c693a04ff0c4a5219879d7f8900157dcbece66 Update hdf5_header.html 0cb58080875070db09b5ecae92482519d58872bc Hdffv 11052 (#2303) a1c81eda203addced514ef655f7a9079f3f0bb04 added doc. warning for H5Literate_async return value (#2295) 502b32b0f22a4bcf6333c85c256db34162c2764a Updated H5ES documenation (#2293) a9036005c3916e6fda0296026323f00d043300f8 Fix for HDFFV-11052: h5debug fails on a corrupted file (h5_nrefs_POC)… (#2291) * Brings the following changes over from develop: c1e44d32e616518e0626bdfe042bed0052846fef Fix doxygen warnings and remove javadocs (#2324) 149b8e9769887c5b23400b526dc10463f88f2c3e Disable hl tools by default (#2313) --- configure.ac | 17 +++++++++++ doxygen/Doxyfile.in | 12 ++++++-- doxygen/dox/ReferenceManual.dox | 13 +++------ doxygen/dox/ViewTools.dox | 8 ++--- doxygen/dox/cookbook/Accessibility.dox | 2 +- doxygen/dox/cookbook/Attributes.dox | 2 +- doxygen/dox/cookbook/Files.dox | 2 +- doxygen/dox/cookbook/Performance.dox | 2 +- doxygen/hdf5_header.html | 2 +- doxygen/hdf5doxy_layout.xml | 8 ++--- fortran/src/H5Pff.F90 | 25 ---------------- hl/tools/CMakeLists.txt | 10 ++++++- hl/tools/Makefile.am | 8 ++++- java/src/Makefile.am | 8 ++--- java/src/hdf/hdf5lib/CMakeLists.txt | 12 -------- release_docs/INSTALL_CMake.txt | 1 + release_docs/RELEASE.txt | 45 +++++++++++++++++++++++++--- src/H5Amodule.h | 2 +- src/H5D.c | 16 +++++----- src/H5Dchunk.c | 4 +-- src/H5Dpublic.h | 2 +- src/H5ESpublic.h | 10 +++---- src/H5FDonion.h | 8 ++--- src/H5Fint.c | 4 ++- src/H5Gmodule.h | 2 +- src/H5Lpublic.h | 9 ++++-- src/H5Tpublic.h | 2 +- src/H5VLconnector_passthru.h | 2 +- src/H5VLnative_file.c | 52 ++++++++++++++++++--------------- test/CMakeTests.cmake | 1 + test/chunk_info.c | 18 ++++++------ test/cve_2020_10812.h5 | Bin 0 -> 2565 bytes test/tmisc.c | 47 +++++++++++++++++++++++++++++ tools/src/misc/h5debug.c | 8 +++-- 34 files changed, 229 insertions(+), 135 deletions(-) create mode 100755 test/cve_2020_10812.h5 diff --git a/configure.ac b/configure.ac index 44b334c..e19feec 100644 --- a/configure.ac +++ b/configure.ac @@ -833,6 +833,7 @@ AC_LANG_POP(C++) ## library is disabled. AC_SUBST([HDF5_HL]) AC_SUBST([HDF5_HL_TOOLS]) +AC_SUBST([HDF5_HL_GIF_TOOLS]) ## The high-level library and high-level tools are enabled unless the build mode ## is clean. @@ -888,6 +889,21 @@ else AC_MSG_RESULT([no]) fi +AC_MSG_CHECKING([if the high-level GIF tools are enabled]) +AC_ARG_ENABLE([hltools], + [AS_HELP_STRING([--enable-hlgiftools], + [Enable the high-level GIF tools. + [default=no] + ])], + [HDF5_HL_GIF_TOOLS=$enableval]) + +if test "X${HDF5_GIF_HL}" = "Xyes" -a "X-$HDF5_HL_TOOLS" = "X-yes" -a "X-$HDF5_HL_GIF_TOOLS" = "X-yes"; then + AC_MSG_RESULT([yes]) + HL_GIF_TOOLS="tools" +else + AC_MSG_RESULT([no]) +fi + ## ---------------------------------------------------------------------- ## Enable new references for dimension scales @@ -3907,6 +3923,7 @@ AM_CONDITIONAL([BUILD_TESTS_CONDITIONAL], [test "X$HDF5_TESTS" = "Xyes"]) AM_CONDITIONAL([BUILD_TESTS_PARALLEL_CONDITIONAL], [test -n "$TESTPARALLEL"]) AM_CONDITIONAL([BUILD_TOOLS_CONDITIONAL], [test "X$HDF5_TOOLS" = "Xyes"]) AM_CONDITIONAL([BUILD_TOOLS_HL_CONDITIONAL], [test "X$HDF5_HL_TOOLS" = "Xyes"]) +AM_CONDITIONAL([BUILD_TOOLS_HL_GIF_CONDITIONAL], [test "X$HDF5_HL_GIF_TOOLS" = "Xyes"]) AM_CONDITIONAL([BUILD_DOXYGEN_CONDITIONAL], [test "X$HDF5_DOXYGEN" = "Xyes"]) ## ---------------------------------------------------------------------- diff --git a/doxygen/Doxyfile.in b/doxygen/Doxyfile.in index f45f746..d9b0fe9 100644 --- a/doxygen/Doxyfile.in +++ b/doxygen/Doxyfile.in @@ -854,7 +854,7 @@ INPUT_ENCODING = UTF-8 # *.vhdl, *.ucf, *.qsf and *.ice. FILE_PATTERNS = H5*public.h H5*module.h H5*develop.h H5FD*.h \ - H5VLconnector.h H5VLconnector_passthru.h H5VLnative.h \ + H5VLconnector.h H5VLconnector_passthru.h H5VLnative.h H5PLextern.h \ H5Zdevelop.h \ H5version.h \ H5*.java \ @@ -897,7 +897,15 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -EXCLUDE_PATTERNS = +EXCLUDE_PATTERNS = */fortran/test/* +EXCLUDE_PATTERNS += */fortran/testpar/* +EXCLUDE_PATTERNS += */fortran/examples/* +EXCLUDE_PATTERNS += */fortran/src/*.c +EXCLUDE_PATTERNS += */fortran/src/*.h +EXCLUDE_PATTERNS += */hl/fortran/examples/* +EXCLUDE_PATTERNS += */hl/fortran/test/* +EXCLUDE_PATTERNS += */hl/fortran/src/*.c +EXCLUDE_PATTERNS += */hl/fortran/src/*.h # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the diff --git a/doxygen/dox/ReferenceManual.dox b/doxygen/dox/ReferenceManual.dox index e79de67..b9bcd49 100644 --- a/doxygen/dox/ReferenceManual.dox +++ b/doxygen/dox/ReferenceManual.dox @@ -145,23 +145,19 @@ Functions with \ref ASYNC
- -Mind the gap - Follow these simple rules and stay out of trouble: -\li \Bold{Handle discipline:} The HDF5 C-API is rife with handles or +\li \Bold{Handle discipline:} The HDF5 API is rife with handles or identifiers, which you typically obtain by creating new HDF5 items, copying - items, or retrieving facets of items. \Emph{You acquire a handle, you own it!} - (Colin Powell) In other words, you are responsible for releasing the underlying + items, or retrieving facets of items. Consequently, \Bold{and most importantly}, you are + responsible for releasing the underlying resources via the matching \Code{H5*close()} call, or deal with the consequences of resource leakage. \li \Bold{Closed means closed:} Do not pass identifiers that were previously \Code{H5*close()}-d to other API functions! It will generate an error. \li \Bold{Dynamic memory allocation:} The API contains a few functions in which the HDF5 library dynamically allocates memory on the caller's behalf. The caller owns - this memory and eventually must free it by calling H5free_memory(). (\Bold{Not} - the `free` function \Emph{du jour}!) + this memory and eventually must free it by calling H5free_memory() and not language-explicit memory functions. \li \Bold{Be careful with that saw:} Do not modify the underlying collection when an iteration is in progress! \li \Bold{Use of locations:} Certain API functions, typically called \Code{H5***_by_name} @@ -169,7 +165,6 @@ Follow these simple rules and stay out of trouble: If the identifier fully specifies the object in question, pass \Code{'.'} (a dot) for the name! -Break a leg! diff --git a/doxygen/dox/ViewTools.dox b/doxygen/dox/ViewTools.dox index 0b685a0..2212d4b 100644 --- a/doxygen/dox/ViewTools.dox +++ b/doxygen/dox/ViewTools.dox @@ -829,6 +829,7 @@ by simply viewing the specified dataset with the -d option must be specified - -before -\par subsetting options (if not using the shorthand method). +Where, the -d option must be specified +before subsetting options (if not using the shorthand method). The -A 0 option suppresses the printing of attributes. diff --git a/doxygen/dox/cookbook/Accessibility.dox b/doxygen/dox/cookbook/Accessibility.dox index f100283..28009be 100644 --- a/doxygen/dox/cookbook/Accessibility.dox +++ b/doxygen/dox/cookbook/Accessibility.dox @@ -1,6 +1,6 @@ /** \page Accessibility -\section Accessibility +\section secAccessibility Accessibility \subsection CB_MaintainCompat Maintaining Compatibility with other HDF5 Library Versions diff --git a/doxygen/dox/cookbook/Attributes.dox b/doxygen/dox/cookbook/Attributes.dox index 68fd159..5914909 100644 --- a/doxygen/dox/cookbook/Attributes.dox +++ b/doxygen/dox/cookbook/Attributes.dox @@ -1,6 +1,6 @@ /** \page Attributes -\section Attributes +\section secAttributes Attributes \subsection CB_LargeAttributes Creating "Large" HDF5 Attributes diff --git a/doxygen/dox/cookbook/Files.dox b/doxygen/dox/cookbook/Files.dox index 169d638..4893771 100644 --- a/doxygen/dox/cookbook/Files.dox +++ b/doxygen/dox/cookbook/Files.dox @@ -1,6 +1,6 @@ /** \page Files -\section Files +\section secFiles Files \subsection CB_FreeSpace Tracking Free Space in HDF5 Files diff --git a/doxygen/dox/cookbook/Performance.dox b/doxygen/dox/cookbook/Performance.dox index 7ac3a18..5e945b2 100644 --- a/doxygen/dox/cookbook/Performance.dox +++ b/doxygen/dox/cookbook/Performance.dox @@ -1,6 +1,6 @@ /** \page Performance -\section Performance +\section secPerformance Performance \subsection CB_MDCPerf Assessing HDF5 Metadata Cache Performance diff --git a/doxygen/hdf5_header.html b/doxygen/hdf5_header.html index 23f41f9..36a3265 100644 --- a/doxygen/hdf5_header.html +++ b/doxygen/hdf5_header.html @@ -21,7 +21,7 @@ $mathjax -
Please, help us to better know about our user community by answering the following short survey: https://www.hdfgroup.org/website-survey/
+
Please, help us to better serve our user community by answering the following short survey: https://www.hdfgroup.org/website-survey/
diff --git a/doxygen/hdf5doxy_layout.xml b/doxygen/hdf5doxy_layout.xml index 588052b..d156f2c 100644 --- a/doxygen/hdf5doxy_layout.xml +++ b/doxygen/hdf5doxy_layout.xml @@ -4,13 +4,13 @@ - - - - + + + + diff --git a/fortran/src/H5Pff.F90 b/fortran/src/H5Pff.F90 index 9c30637..75d7365 100644 --- a/fortran/src/H5Pff.F90 +++ b/fortran/src/H5Pff.F90 @@ -4305,31 +4305,6 @@ SUBROUTINE h5pset_attr_phase_change_f(ocpl_id, max_compact, min_dense, hdferr) CHARACTER(LEN=*), INTENT(IN) :: name INTEGER , INTENT(IN) :: value INTEGER , INTENT(OUT) :: hdferr - END SUBROUTINE h5pset - -!> -!! \ingroup FH5P -!! -!! \brief Sets a property list value. -!! -!! \attention \fortran_obsolete -!! -!! \param prp_id Property list identifier to modify. -!! \param name Name of property to modify. -!! \param value Property value, supported types are: -!! \li INTEGER -!! \li REAL -!! \li DOUBLE PRECISION -!! \li CHARACTER(LEN=*) -!! \param hdferr \fortran_error -!! -!! See C API: @ref H5Pset() -!! - SUBROUTINE h5pset_f(prp_id, name, value, hdferr) - INTEGER(HID_T), INTENT(IN) :: prp_id - CHARACTER(LEN=*), INTENT(IN) :: name - INTEGER, INTENT(IN) :: value - INTEGER, INTENT(OUT) :: hdferr END SUBROUTINE h5pset_f !> !! \ingroup FH5P diff --git a/hl/tools/CMakeLists.txt b/hl/tools/CMakeLists.txt index a209fbb..7df2b79 100644 --- a/hl/tools/CMakeLists.txt +++ b/hl/tools/CMakeLists.txt @@ -1,6 +1,14 @@ cmake_minimum_required (VERSION 3.18) project (HDF5_HL_TOOLS C) -add_subdirectory (gif2h5) + #----------------------------------------------------------------------------- + #-- Option to build the High level GIF Tools + #----------------------------------------------------------------------------- + if (EXISTS "${HDF5_HL_SOURCE_DIR}/gif2h5" AND IS_DIRECTORY "${HDF5_HL_SOURCE_DIR}/gif2h5") + option (HDF5_BUILD_HL_GIF_TOOLS "Build HDF5 HL GIF Tools" OFF) + if (HDF5_BUILD_HL_GIF_TOOLS) + add_subdirectory (gif2h5) + endif () + endif () add_subdirectory (h5watch) diff --git a/hl/tools/Makefile.am b/hl/tools/Makefile.am index 6687f40..f3fe0d3 100644 --- a/hl/tools/Makefile.am +++ b/hl/tools/Makefile.am @@ -18,7 +18,13 @@ include $(top_srcdir)/config/commence.am +if BUILD_TOOLS_HL_GIF_CONDITIONAL + TOOLS_GIF_DIR = gif2h5 +else + TOOLS_GIF_DIR = +endif + # All subdirectories -SUBDIRS=gif2h5 h5watch +SUBDIRS=$(TOOLS_DIR) h5watch include $(top_srcdir)/config/conclude.am diff --git a/java/src/Makefile.am b/java/src/Makefile.am index 0076932..36fca3a 100644 --- a/java/src/Makefile.am +++ b/java/src/Makefile.am @@ -111,26 +111,22 @@ hdf5_java_JAVA = \ ${pkgpath}/HDFArray.java \ ${pkgpath}/HDFNativeData.java -$(jarfile): classhdf5_java.stamp classes docs +$(jarfile): classhdf5_java.stamp classes $(JAR) cvf $@ -C $(JAVAROOT)/ $(pkgpath) hdf5_java_DATA = $(jarfile) -.PHONY: docs classes +.PHONY: classes WINDOWTITLE = 'HDF5 Java' DOCTITLE = '

HDF5 Java Wrapper

' SRCDIR = '$(pkgpath)' -docs: - $(JAVADOC) -sourcepath $(srcdir) -d javadoc -Xdoclint:none -use -splitIndex -windowtitle $(WINDOWTITLE) -doctitle $(DOCTITLE) -J-Xmx180m -verbose -overview $(top_srcdir)/java/src/hdf/overview.html -classpath $(CLASSPATH_ENV) hdf.hdf5lib - CLEANFILES = classhdf5_java.stamp $(jarfile) $(JAVAROOT)/$(pkgpath)/callbacks/*.class $(JAVAROOT)/$(pkgpath)/exceptions/*.class $(JAVAROOT)/$(pkgpath)/structs/*.class $(JAVAROOT)/$(pkgpath)/*.class clean: rm -rf $(JAVAROOT)/* rm -f $(jarfile) - rm -rf javadoc rm -f classhdf5_java.stamp diff --git a/java/src/hdf/hdf5lib/CMakeLists.txt b/java/src/hdf/hdf5lib/CMakeLists.txt index 4e340f1..41cf4e9 100644 --- a/java/src/hdf/hdf5lib/CMakeLists.txt +++ b/java/src/hdf/hdf5lib/CMakeLists.txt @@ -134,15 +134,3 @@ set_target_properties (${HDF5_JAVA_HDF5_LIB_TARGET} PROPERTIES FOLDER libraries/ if (HDF5_ENABLE_FORMATTERS) clang_format (HDF5_JAVA_SRC_FORMAT ${HDF5_JAVA_HDF_HDF5_CALLBACKS_SOURCES} ${HDF5_JAVA_HDF_HDF5_EXCEPTIONS_SOURCES} ${HDF5_JAVA_HDF_HDF5_STRUCTS_SOURCES} ${HDF5_JAVA_HDF_HDF5_SOURCES}) endif () - -create_javadoc(hdf5_java_doc - FILES ${HDF5_JAVADOC_HDF_HDF5_CALLBACKS_SOURCES} ${HDF5_JAVADOC_HDF_HDF5_EXCEPTIONS_SOURCES} ${HDF5_JAVADOC_HDF_HDF5_STRUCTS_SOURCES} ${HDF5_JAVADOC_HDF_HDF5_SOURCES} - OVERVIEW ${HDF5_JAVA_HDF5_SRC_DIR}/overview.html - CLASSPATH ${CMAKE_JAVA_INCLUDE_PATH} - WINDOWTITLE "HDF5 Java" - DOCTITLE "

HDF5 Java Wrapper

" - INSTALLPATH ${HDF5_INSTALL_DOC_DIR} - AUTHOR TRUE - USE TRUE - VERSION TRUE -) diff --git a/release_docs/INSTALL_CMake.txt b/release_docs/INSTALL_CMake.txt index 94b14d9..b17a7e5 100644 --- a/release_docs/INSTALL_CMake.txt +++ b/release_docs/INSTALL_CMake.txt @@ -741,6 +741,7 @@ HDF5_BUILD_JAVA "Build JAVA support" OFF HDF5_BUILD_HL_LIB "Build HIGH Level HDF5 Library" ON HDF5_BUILD_TOOLS "Build HDF5 Tools" ON HDF5_BUILD_HL_TOOLS "Build HIGH Level HDF5 Tools" ON +HDF5_BUILD_HL_GIF_TOOLS "Build HIGH Level HDF5 GIF Tools" OFF ---------------- HDF5 Folder Build Options --------------------- Defaults relative to $ diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 90309fd..a69d97d 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -171,6 +171,25 @@ Bug Fixes since HDF5-1.13.3 release =================================== Library ------- + - Seg fault on file close + + h5debug fails at file close with core dump on a file that has an + illegal file size in its cache image. In H5F_dest(), the library + performs all the closing operations for the file and keeps track of + the error encountered when reading the file cache image. + At the end of the routine, it frees the file's file structure and + returns error. Due to the error return, the file object is not removed + from the ID node table. This eventually causes assertion failure in + H5VL__native_file_close() when the library finally exits and tries to + access that file object in the table for closing. + + The closing routine, H5F_dest(), will not free the file structure if + there is error, keeping a valid file structure in the ID node table. + It will be freed later in H5VL__native_file_close() when the + library exits and terminates the file package. + + (VC - 2022/12/14, HDFFV-11052, CVE-2020-10812) + - Fix CVE-2018-13867 / GHSA-j8jr-chrh-qfrf Validate location (offset) of the accumulated metadata when comparing. @@ -282,6 +301,24 @@ Bug Fixes since HDF5-1.13.3 release Configuration ------------- + - Remove Javadoc generation + + The use of doxygen now supersedes the requirement to build javadocs. We do not + have the resources to continue to support two documentation methods and have + chosen doxygen as our standard. + + (ADB - 2022/12/19) + + - Change the default for building the high-level tools + + The gif2hdf5 and hdf2gif high-level tools are deprecated and will be removed + in a future release. The default build setting for them have been changed from enabled + to disabled. A user can enable the build of these tools if needed. + autotools: --enable-hlgiftools + cmake: HDF5_BUILD_HL_GIF_TOOLS=ON + + (ADB - 2022/12/16) + - Change the settings of the *pc files to use the correct format The pkg-config files generated by CMake uses incorrect syntax for the 'Requires' @@ -564,7 +601,7 @@ The issues with the gif tool are: HDFFV-10592 CVE-2018-17433 HDFFV-10593 CVE-2018-17436 HDFFV-11048 CVE-2020-10809 -These CVE issues have not yet been addressed and can be avoided by not building -the gif tool. Disable building the High-Level tools with these options: - autotools: --disable-hltools - cmake: HDF5_BUILD_HL_TOOLS=OFF +These CVE issues have not yet been addressed and are avoided by not building +the gif tool by default. Enable building the High-Level tools with these options: + autotools: --enable-hltools + cmake: HDF5_BUILD_HL_TOOLS=ON diff --git a/src/H5Amodule.h b/src/H5Amodule.h index 7f88a22..4823d0d 100644 --- a/src/H5Amodule.h +++ b/src/H5Amodule.h @@ -363,7 +363,7 @@ * will be ignored by HDF5. * * The use of ASCII or UTF-8 characters is determined by the character encoding property. See - * #H5Pset_char_encoding in the \ref RM. + * #H5Pset_char_encoding in the \ref RM. * *

No Special I/O or Storage

* diff --git a/src/H5D.c b/src/H5D.c index d921356..006f4a9 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -2515,18 +2515,18 @@ done: * * typedef int (*H5D_chunk_iter_op_t)( * const hsize_t *offset, - * uint32_t filter_mask, + * unsigned filter_mask, * haddr_t addr, - * uint32_t nbytes, + * hsize_t size, * void *op_data); * * H5D_chunk_iter_op_t parameters: - * hsize_t *offset; IN/OUT: Array of starting logical coordinates of chunk. - * uint32_t filter_mask; IN: Filter mask of chunk. - * haddr_t addr; IN: Offset in file of chunk data. - * uint32_t nbytes; IN: Size in number of bytes of chunk data in file. - * void *op_data; IN/OUT: Pointer to any user-defined data - * associated with the operation. + * hsize_t *offset; IN/OUT: Logical position of the chunk’s first element in units of dataset + * elements + * unsigned filter_mask; IN: Bitmask indicating the filters used when the chunk was written haddr_t + * addr; IN: Chunk address in the file + * hsize_t; IN: Chunk size in bytes, 0 if the chunk does not exist + * void *op_data; IN/OUT: Pointer to any user-defined data associated with the operation. * * The return values from an operator are: * Zero (H5_ITER_CONT) causes the iterator to continue, returning zero when all diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 40f8359..830560d 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -8278,8 +8278,8 @@ H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata) FUNC_ENTER_PACKAGE_NOERR /* Check for callback failure and pass along return value */ - if ((ret_value = (data->op)(offset, chunk_rec->filter_mask, chunk_rec->chunk_addr, chunk_rec->nbytes, - data->op_data)) < 0) + if ((ret_value = (data->op)(offset, (unsigned)chunk_rec->filter_mask, chunk_rec->chunk_addr, + (hsize_t)chunk_rec->nbytes, data->op_data)) < 0) HERROR(H5E_DATASET, H5E_CANTNEXT, "iteration operator failed"); FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index ac76bc8..f7d208d 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -238,7 +238,7 @@ typedef herr_t (*H5D_gather_func_t)(const void *dst_buf, size_t dst_buf_bytes_us * \li A negative (#H5_ITER_ERROR) causes the iterator to immediately * return that value, indicating failure. */ -typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t size, +typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *op_data); //! diff --git a/src/H5ESpublic.h b/src/H5ESpublic.h index 3d46e82..ecfd08f 100644 --- a/src/H5ESpublic.h +++ b/src/H5ESpublic.h @@ -189,7 +189,7 @@ H5_DLL herr_t H5ESwait(hid_t es_id, uint64_t timeout, size_t *num_in_progress, h * \param[out] err_occurred Status indicating if error is present in the event set * \returns \herr_t * - * \details H5ESget_count() attempts to cancel operations in an event set specified + * \details H5EScancel() attempts to cancel operations in an event set specified * by \p es_id. H5ES_NONE is a valid value for \p es_id, but functions as a no-op. * * \since 1.13.0 @@ -217,14 +217,14 @@ H5_DLL herr_t H5ESget_count(hid_t es_id, size_t *count); /** * \ingroup H5ES * - * \brief Retrieves the next operation counter to be assigned in an event set + * \brief Retrieves the accumulative operation counter for an event set * * \es_id - * \param[out] counter The next counter value to be assigned to an event + * \param[out] counter The accumulative counter value for an event set * \returns \herr_t * - * \details H5ESget_op_counter() retrieves the \p counter that will be assigned - * to the next operation inserted into the event set \p es_id. + * \details H5ESget_op_counter() retrieves the current accumulative count of + * event set operations since the event set creation of \p es_id. * * \note This is designed for wrapper libraries mainly, to use as a mechanism * for matching operations inserted into the event set with possible diff --git a/src/H5FDonion.h b/src/H5FDonion.h index 63c2d77..0e605d0 100644 --- a/src/H5FDonion.h +++ b/src/H5FDonion.h @@ -142,11 +142,11 @@ H5_DLL hid_t H5FD_onion_init(void); /** * -------------------------------------------------------------------------- - * \ingroup H5P + * \ingroup FAPL * * \brief get the onion info from the file access property list * - * \param[in] fapl_id The ID of the file access property list + * \fapl_id * \param[out] fa_out The pointer to the structure H5FD_onion_fapl_info_t * * \return \herr_t @@ -159,11 +159,11 @@ H5_DLL herr_t H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out); /** * -------------------------------------------------------------------------- - * \ingroup H5P + * \ingroup FAPL * * \brief set the onion info for the file access property list * - * \param[in] fapl_id The ID of the file access property list + * \fapl_id * \param[in] fa The pointer to the structure H5FD_onion_fapl_info_t * * \return \herr_t diff --git a/src/H5Fint.c b/src/H5Fint.c index 2c1b4b2..7ad35fc 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -1615,7 +1615,9 @@ H5F__dest(H5F_t *f, hbool_t flush) if (H5FO_top_dest(f) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "problems closing file") f->shared = NULL; - f = H5FL_FREE(H5F_t, f); + + if (ret_value >= 0) + f = H5FL_FREE(H5F_t, f); FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__dest() */ diff --git a/src/H5Gmodule.h b/src/H5Gmodule.h index 4b58ee6..d4738f6 100644 --- a/src/H5Gmodule.h +++ b/src/H5Gmodule.h @@ -923,7 +923,7 @@ * containing thousands to millions of members. Links are stored in * a fractal heap and indexed with an improved B-tree. * \li The new implementation also enables the use of link names consisting of - * non-ASCII character sets (see H5Pset_char_encoding()) and is + * non-ASCII character sets (see #H5Pset_char_encoding) and is * required for all link types other than hard or soft links, e.g., * external and user-defined links (see the \ref H5L APIs). * diff --git a/src/H5Lpublic.h b/src/H5Lpublic.h index 6f6c638..d2d9e9d 100644 --- a/src/H5Lpublic.h +++ b/src/H5Lpublic.h @@ -729,7 +729,7 @@ H5_DLL herr_t H5Lexists_async(hid_t loc_id, const char *name, hbool_t *exists, h * \p cset specifies the character set in which the link name is * encoded. Valid values include the following: * \csets - * This value is set with H5Pset_char_encoding(). + * This value is set with #H5Pset_char_encoding. * * \c token is the location that a hard link points to, and * \c val_size is the size of a soft link or user defined link value. @@ -923,6 +923,11 @@ H5_DLL herr_t H5Literate2(hid_t grp_id, H5_index_t idx_type, H5_iter_order_t ord /** * -------------------------------------------------------------------------- * \ingroup ASYNC + * + * \warning The returned value of the callback routine op will not be set + * in the return value for H5Literate_async(), so the \p herr_t value + * should not be used for determining the return state of the callback routine. + * * \async_variant_of{H5Literate} */ #ifndef H5_DOXYGEN @@ -1520,7 +1525,7 @@ typedef herr_t (*H5L_iterate1_t)(hid_t group, const char *name, const H5L_info1_ * \c cset specifies the character set in which the link name is * encoded. Valid values include the following: * \csets - * This value is set with H5Pset_char_encoding(). + * This value is set with #H5Pset_char_encoding. * * \c address and \c val_size are returned for hard and symbolic * links, respectively. Symbolic links include soft and external links diff --git a/src/H5Tpublic.h b/src/H5Tpublic.h index dcaffd8..16172a8 100644 --- a/src/H5Tpublic.h +++ b/src/H5Tpublic.h @@ -1035,7 +1035,7 @@ H5_DLLVAR hid_t H5T_NATIVE_UINT_FAST64_g; * When creating a fixed-length string datatype, \p size will * be the length of the string in bytes. The length of the * string in characters will depend on i the encoding used; see - * H5Pset_char_encoding(). + * #H5Pset_char_encoding. * * ENUMs created with this function have a signed native integer * base datatype. Use H5Tenum_create() if a different integer base diff --git a/src/H5VLconnector_passthru.h b/src/H5VLconnector_passthru.h index 68dd33a..d10bac4 100644 --- a/src/H5VLconnector_passthru.h +++ b/src/H5VLconnector_passthru.h @@ -71,7 +71,7 @@ H5_DLL herr_t H5VLcmp_connector_cls(int *cmp, hid_t connector_id1, hid_t connect * - #H5I_MAP * - #H5I_ATTR * - * \return \hid_t + * \return \hid_t{VOL connector} * * \note This routine is mainly targeted toward wrapping objects for * iteration routine callbacks (i.e. the callbacks from H5Aiterate*, diff --git a/src/H5VLnative_file.c b/src/H5VLnative_file.c index 907a12d..f2f0ea7 100644 --- a/src/H5VLnative_file.c +++ b/src/H5VLnative_file.c @@ -753,29 +753,35 @@ H5VL__native_file_close(void *file, hid_t H5_ATTR_UNUSED dxpl_id, void H5_ATTR_U FUNC_ENTER_PACKAGE /* This routine should only be called when a file ID's ref count drops to zero */ - HDassert(H5F_ID_EXISTS(f)); - - /* Flush file if this is the last reference to this id and we have write - * intent, unless it will be flushed by the "shared" file being closed. - * This is only necessary to replicate previous behaviour, and could be - * disabled by an option/property to improve performance. - */ - if ((H5F_NREFS(f) > 1) && (H5F_INTENT(f) & H5F_ACC_RDWR)) { - /* Get the file ID corresponding to the H5F_t struct */ - if (H5I_find_id(f, H5I_FILE, &file_id) < 0 || H5I_INVALID_HID == file_id) - HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "invalid ID") - - /* Get the number of references outstanding for this file ID */ - if ((nref = H5I_get_ref(file_id, FALSE)) < 0) - HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "can't get ID ref count") - if (nref == 1) - if (H5F__flush(f) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cache") - } /* end if */ - - /* Close the file */ - if (H5F__close(f) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "can't close file") + HDassert(f->shared == NULL || H5F_ID_EXISTS(f)); + + if (f->shared == NULL) + f = H5FL_FREE(H5F_t, f); + + else { + + /* Flush file if this is the last reference to this id and we have write + * intent, unless it will be flushed by the "shared" file being closed. + * This is only necessary to replicate previous behaviour, and could be + * disabled by an option/property to improve performance. + */ + if ((H5F_NREFS(f) > 1) && (H5F_INTENT(f) & H5F_ACC_RDWR)) { + /* Get the file ID corresponding to the H5F_t struct */ + if (H5I_find_id(f, H5I_FILE, &file_id) < 0 || H5I_INVALID_HID == file_id) + HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "invalid ID") + + /* Get the number of references outstanding for this file ID */ + if ((nref = H5I_get_ref(file_id, FALSE)) < 0) + HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "can't get ID ref count") + if (nref == 1) + if (H5F__flush(f) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cache") + } /* end if */ + + /* Close the file */ + if (H5F__close(f) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "can't close file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index 74f63f4..5086194 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -126,6 +126,7 @@ set (HDF5_REFERENCE_TEST_FILES btree_idx_1_6.h5 btree_idx_1_8.h5 corrupt_stab_msg.h5 + cve_2020_10812.h5 deflate.h5 family_v16-000000.h5 family_v16-000001.h5 diff --git a/test/chunk_info.c b/test/chunk_info.c index e38752f..5651b26 100644 --- a/test/chunk_info.c +++ b/test/chunk_info.c @@ -1508,9 +1508,9 @@ error: typedef struct chunk_iter_info_t { hsize_t offset[2]; - uint32_t filter_mask; + unsigned filter_mask; haddr_t addr; - uint32_t nbytes; + hsize_t size; } chunk_iter_info_t; typedef struct chunk_iter_udata_t { @@ -1519,7 +1519,7 @@ typedef struct chunk_iter_udata_t { } chunk_iter_udata_t; static int -iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbytes, void *op_data) +iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *op_data) { chunk_iter_udata_t *cidata = (chunk_iter_udata_t *)op_data; int idx = cidata->last_index + 1; @@ -1528,7 +1528,7 @@ iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbyt cidata->chunk_info[idx].offset[1] = offset[1]; cidata->chunk_info[idx].filter_mask = filter_mask; cidata->chunk_info[idx].addr = addr; - cidata->chunk_info[idx].nbytes = nbytes; + cidata->chunk_info[idx].size = size; cidata->last_index++; @@ -1536,8 +1536,8 @@ iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbyt } static int -iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, - haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) +iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, unsigned H5_ATTR_UNUSED filter_mask, + haddr_t H5_ATTR_UNUSED addr, hsize_t H5_ATTR_UNUSED size, void *op_data) { chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; *chunk_info += 1; @@ -1545,8 +1545,8 @@ iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filte } static int -iter_cb_fail(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, - haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) +iter_cb_fail(const hsize_t H5_ATTR_UNUSED *offset, unsigned H5_ATTR_UNUSED filter_mask, + haddr_t H5_ATTR_UNUSED addr, hsize_t H5_ATTR_UNUSED size, void *op_data) { chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; *chunk_info += 1; @@ -1718,7 +1718,7 @@ test_basic_query(hid_t fapl) FAIL_PUTS_ERROR("offset[1] mismatch"); if (chunk_infos[0].filter_mask != 0) FAIL_PUTS_ERROR("filter mask mismatch"); - if (chunk_infos[0].nbytes != 96) + if (chunk_infos[0].size != 96) FAIL_PUTS_ERROR("size mismatch"); if (chunk_infos[1].offset[0] != CHUNK_NX) diff --git a/test/cve_2020_10812.h5 b/test/cve_2020_10812.h5 new file mode 100755 index 0000000..a20369d Binary files /dev/null and b/test/cve_2020_10812.h5 differ diff --git a/test/tmisc.c b/test/tmisc.c index 5cad368..5fb44d4 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -330,6 +330,11 @@ typedef struct { #define MISC35_SPACE_DIM3 13 #define MISC35_NPOINTS 10 +/* Definitions for misc. test #37 */ +/* The test file is formerly named h5_nrefs_POC. + See https://nvd.nist.gov/vuln/detail/CVE-2020-10812 */ +#define CVE_2020_10812_FILENAME "cve_2020_10812.h5" + /**************************************************************** ** ** test_misc1(): test unlinking a dataset from a group and immediately @@ -6046,6 +6051,47 @@ test_misc36(void) /**************************************************************** ** +** test_misc37(): +** Test for seg fault issue when closing the provided test file +** which has an illegal file size in its cache image. +** See HDFFV-11052/CVE-2020-10812 for details. +** +****************************************************************/ +static void +test_misc37(void) +{ + const char *testfile = H5_get_srcdir_filename(CVE_2020_10812_FILENAME); + hbool_t driver_is_default_compatible; + hid_t fid; + herr_t ret; + + /* Output message about test being performed */ + MESSAGE(5, ("Fix for HDFFV-11052/CVE-2020-10812")); + + ret = h5_driver_is_default_vfd_compatible(H5P_DEFAULT, &driver_is_default_compatible); + CHECK(ret, FAIL, "h5_driver_is_default_vfd_compatible"); + + if (!driver_is_default_compatible) { + HDprintf("-- SKIPPED --\n"); + return; + } + + fid = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fopen"); + + /* This should fail due to the illegal file size. + It should fail gracefully and not seg fault */ + H5E_BEGIN_TRY + { + ret = H5Fclose(fid); + } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Fclose"); + +} /* end test_misc37() */ + +/**************************************************************** +** ** test_misc(): Main misc. test routine. ** ****************************************************************/ @@ -6111,6 +6157,7 @@ test_misc(void) test_misc34(); /* Test behavior of 0 and NULL in H5MM API calls */ test_misc35(); /* Test behavior of free-list & allocation statistics API calls */ test_misc36(); /* Exercise H5atclose and H5is_library_terminating */ + test_misc37(); /* Test for seg fault failure at file close */ } /* test_misc() */ diff --git a/tools/src/misc/h5debug.c b/tools/src/misc/h5debug.c index 05d37b8..b15ae09 100644 --- a/tools/src/misc/h5debug.c +++ b/tools/src/misc/h5debug.c @@ -815,8 +815,12 @@ main(int argc, char *argv[]) done: if (fapl > 0) H5Pclose(fapl); - if (fid > 0) - H5Fclose(fid); + if (fid > 0) { + if (H5Fclose(fid) < 0) { + HDfprintf(stderr, "Error in closing file!\n"); + exit_value = 1; + } + } /* Pop API context */ if (api_ctx_pushed) -- cgit v0.12