diff options
| -rw-r--r-- | doxygen/Doxyfile.in | 10 | ||||
| -rw-r--r-- | doxygen/dox/ReferenceManual.dox | 13 | ||||
| -rw-r--r-- | doxygen/hdf5_header.html | 2 | ||||
| -rw-r--r-- | release_docs/RELEASE.txt | 19 | ||||
| -rw-r--r-- | src/H5D.c | 16 | ||||
| -rw-r--r-- | src/H5Dchunk.c | 4 | ||||
| -rw-r--r-- | src/H5Dpublic.h | 2 | ||||
| -rw-r--r-- | src/H5ESpublic.h | 10 | ||||
| -rw-r--r-- | src/H5FDonion.h | 8 | ||||
| -rw-r--r-- | src/H5Fint.c | 4 | ||||
| -rw-r--r-- | src/H5Lpublic.h | 5 | ||||
| -rw-r--r-- | src/H5VLnative_file.c | 52 | ||||
| -rw-r--r-- | test/CMakeTests.cmake | 1 | ||||
| -rw-r--r-- | test/chunk_info.c | 18 | ||||
| -rwxr-xr-x | test/cve_2020_10812.h5 | bin | 0 -> 2565 bytes | |||
| -rw-r--r-- | test/tmisc.c | 47 | ||||
| -rw-r--r-- | tools/src/misc/h5debug.c | 8 |
17 files changed, 153 insertions, 66 deletions
diff --git a/doxygen/Doxyfile.in b/doxygen/Doxyfile.in index f45f746..28c7327 100644 --- a/doxygen/Doxyfile.in +++ b/doxygen/Doxyfile.in @@ -897,7 +897,15 @@ EXCLUDE_SYMLINKS = NO # Note that the wildcards are matched against the file with absolute path, so to # exclude all test directories for example use the pattern */test/* -EXCLUDE_PATTERNS = +EXCLUDE_PATTERNS = */fortran/test/* +EXCLUDE_PATTERNS += */fortran/testpar/* +EXCLUDE_PATTERNS += */fortran/examples/* +EXCLUDE_PATTERNS += */fortran/src/*.c +EXCLUDE_PATTERNS += */fortran/src/*.h +EXCLUDE_PATTERNS += */hl/fortran/examples/* +EXCLUDE_PATTERNS += */hl/fortran/test/* +EXCLUDE_PATTERNS += */hl/fortran/src/*.c +EXCLUDE_PATTERNS += */hl/fortran/src/*.h # The EXCLUDE_SYMBOLS tag can be used to specify one or more symbol names # (namespaces, classes, functions, etc.) that should be excluded from the diff --git a/doxygen/dox/ReferenceManual.dox b/doxygen/dox/ReferenceManual.dox index e79de67..b9bcd49 100644 --- a/doxygen/dox/ReferenceManual.dox +++ b/doxygen/dox/ReferenceManual.dox @@ -145,23 +145,19 @@ Functions with \ref ASYNC<br /> </tr> </table> -</td></tr> -<tr><th>Mind the gap</th></tr> -<tr><td> Follow these simple rules and stay out of trouble: -\li \Bold{Handle discipline:} The HDF5 C-API is rife with handles or +\li \Bold{Handle discipline:} The HDF5 API is rife with handles or identifiers, which you typically obtain by creating new HDF5 items, copying - items, or retrieving facets of items. \Emph{You acquire a handle, you own it!} - (Colin Powell) In other words, you are responsible for releasing the underlying + items, or retrieving facets of items. Consequently, \Bold{and most importantly}, you are + responsible for releasing the underlying resources via the matching \Code{H5*close()} call, or deal with the consequences of resource leakage. \li \Bold{Closed means closed:} Do not pass identifiers that were previously \Code{H5*close()}-d to other API functions! It will generate an error. \li \Bold{Dynamic memory allocation:} The API contains a few functions in which the HDF5 library dynamically allocates memory on the caller's behalf. The caller owns - this memory and eventually must free it by calling H5free_memory(). (\Bold{Not} - the `free` function \Emph{du jour}!) + this memory and eventually must free it by calling H5free_memory() and not language-explicit memory functions. \li \Bold{Be careful with that saw:} Do not modify the underlying collection when an iteration is in progress! \li \Bold{Use of locations:} Certain API functions, typically called \Code{H5***_by_name} @@ -169,7 +165,6 @@ Follow these simple rules and stay out of trouble: If the identifier fully specifies the object in question, pass \Code{'.'} (a dot) for the name! -Break a leg! </td> </tr> </table> diff --git a/doxygen/hdf5_header.html b/doxygen/hdf5_header.html index 23f41f9..36a3265 100644 --- a/doxygen/hdf5_header.html +++ b/doxygen/hdf5_header.html @@ -21,7 +21,7 @@ $mathjax </head> <body> -<div style="background:#FFDDDD;font-size:120%;text-align:center;margin:0;padding:5px">Please, help us to better know about our user community by answering the following short survey: <a href="https://www.hdfgroup.org/website-survey/">https://www.hdfgroup.org/website-survey/</a></div> +<div style="background:#FFDDDD;font-size:120%;text-align:center;margin:0;padding:5px">Please, help us to better serve our user community by answering the following short survey: <a href="https://www.hdfgroup.org/website-survey/">https://www.hdfgroup.org/website-survey/</a></div> <div id="top"><!-- do not remove this div, it is closed by doxygen! --> diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 930bc42..1bccc70 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -171,6 +171,25 @@ Bug Fixes since HDF5-1.13.3 release =================================== Library ------- + - Seg fault on file close + + h5debug fails at file close with core dump on a file that has an + illegal file size in its cache image. In H5F_dest(), the library + performs all the closing operations for the file and keeps track of + the error encountered when reading the file cache image. + At the end of the routine, it frees the file's file structure and + returns error. Due to the error return, the file object is not removed + from the ID node table. This eventually causes assertion failure in + H5VL__native_file_close() when the library finally exits and tries to + access that file object in the table for closing. + + The closing routine, H5F_dest(), will not free the file structure if + there is error, keeping a valid file structure in the ID node table. + It will be freed later in H5VL__native_file_close() when the + library exits and terminates the file package. + + (VC - 2022/12/14, HDFFV-11052, CVE-2020-10812) + - Fix CVE-2018-13867 / GHSA-j8jr-chrh-qfrf Validate location (offset) of the accumulated metadata when comparing. @@ -2515,18 +2515,18 @@ done: * * typedef int (*H5D_chunk_iter_op_t)( * const hsize_t *offset, - * uint32_t filter_mask, + * unsigned filter_mask, * haddr_t addr, - * uint32_t nbytes, + * hsize_t size, * void *op_data); * * H5D_chunk_iter_op_t parameters: - * hsize_t *offset; IN/OUT: Array of starting logical coordinates of chunk. - * uint32_t filter_mask; IN: Filter mask of chunk. - * haddr_t addr; IN: Offset in file of chunk data. - * uint32_t nbytes; IN: Size in number of bytes of chunk data in file. - * void *op_data; IN/OUT: Pointer to any user-defined data - * associated with the operation. + * hsize_t *offset; IN/OUT: Logical position of the chunk’s first element in units of dataset + * elements + * unsigned filter_mask; IN: Bitmask indicating the filters used when the chunk was written haddr_t + * addr; IN: Chunk address in the file + * hsize_t; IN: Chunk size in bytes, 0 if the chunk does not exist + * void *op_data; IN/OUT: Pointer to any user-defined data associated with the operation. * * The return values from an operator are: * Zero (H5_ITER_CONT) causes the iterator to continue, returning zero when all diff --git a/src/H5Dchunk.c b/src/H5Dchunk.c index 40f8359..830560d 100644 --- a/src/H5Dchunk.c +++ b/src/H5Dchunk.c @@ -8278,8 +8278,8 @@ H5D__chunk_iter_cb(const H5D_chunk_rec_t *chunk_rec, void *udata) FUNC_ENTER_PACKAGE_NOERR /* Check for callback failure and pass along return value */ - if ((ret_value = (data->op)(offset, chunk_rec->filter_mask, chunk_rec->chunk_addr, chunk_rec->nbytes, - data->op_data)) < 0) + if ((ret_value = (data->op)(offset, (unsigned)chunk_rec->filter_mask, chunk_rec->chunk_addr, + (hsize_t)chunk_rec->nbytes, data->op_data)) < 0) HERROR(H5E_DATASET, H5E_CANTNEXT, "iteration operator failed"); FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Dpublic.h b/src/H5Dpublic.h index ac76bc8..f7d208d 100644 --- a/src/H5Dpublic.h +++ b/src/H5Dpublic.h @@ -238,7 +238,7 @@ typedef herr_t (*H5D_gather_func_t)(const void *dst_buf, size_t dst_buf_bytes_us * \li A negative (#H5_ITER_ERROR) causes the iterator to immediately * return that value, indicating failure. */ -typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t size, +typedef int (*H5D_chunk_iter_op_t)(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *op_data); //! <!-- [H5D_chunk_iter_op_t_snip] --> diff --git a/src/H5ESpublic.h b/src/H5ESpublic.h index 3d46e82..ecfd08f 100644 --- a/src/H5ESpublic.h +++ b/src/H5ESpublic.h @@ -189,7 +189,7 @@ H5_DLL herr_t H5ESwait(hid_t es_id, uint64_t timeout, size_t *num_in_progress, h * \param[out] err_occurred Status indicating if error is present in the event set * \returns \herr_t * - * \details H5ESget_count() attempts to cancel operations in an event set specified + * \details H5EScancel() attempts to cancel operations in an event set specified * by \p es_id. H5ES_NONE is a valid value for \p es_id, but functions as a no-op. * * \since 1.13.0 @@ -217,14 +217,14 @@ H5_DLL herr_t H5ESget_count(hid_t es_id, size_t *count); /** * \ingroup H5ES * - * \brief Retrieves the next operation counter to be assigned in an event set + * \brief Retrieves the accumulative operation counter for an event set * * \es_id - * \param[out] counter The next counter value to be assigned to an event + * \param[out] counter The accumulative counter value for an event set * \returns \herr_t * - * \details H5ESget_op_counter() retrieves the \p counter that will be assigned - * to the next operation inserted into the event set \p es_id. + * \details H5ESget_op_counter() retrieves the current accumulative count of + * event set operations since the event set creation of \p es_id. * * \note This is designed for wrapper libraries mainly, to use as a mechanism * for matching operations inserted into the event set with possible diff --git a/src/H5FDonion.h b/src/H5FDonion.h index 63c2d77..0e605d0 100644 --- a/src/H5FDonion.h +++ b/src/H5FDonion.h @@ -142,11 +142,11 @@ H5_DLL hid_t H5FD_onion_init(void); /** * -------------------------------------------------------------------------- - * \ingroup H5P + * \ingroup FAPL * * \brief get the onion info from the file access property list * - * \param[in] fapl_id The ID of the file access property list + * \fapl_id * \param[out] fa_out The pointer to the structure H5FD_onion_fapl_info_t * * \return \herr_t @@ -159,11 +159,11 @@ H5_DLL herr_t H5Pget_fapl_onion(hid_t fapl_id, H5FD_onion_fapl_info_t *fa_out); /** * -------------------------------------------------------------------------- - * \ingroup H5P + * \ingroup FAPL * * \brief set the onion info for the file access property list * - * \param[in] fapl_id The ID of the file access property list + * \fapl_id * \param[in] fa The pointer to the structure H5FD_onion_fapl_info_t * * \return \herr_t diff --git a/src/H5Fint.c b/src/H5Fint.c index 2c1b4b2..7ad35fc 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -1615,7 +1615,9 @@ H5F__dest(H5F_t *f, hbool_t flush) if (H5FO_top_dest(f) < 0) HDONE_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "problems closing file") f->shared = NULL; - f = H5FL_FREE(H5F_t, f); + + if (ret_value >= 0) + f = H5FL_FREE(H5F_t, f); FUNC_LEAVE_NOAPI(ret_value) } /* end H5F__dest() */ diff --git a/src/H5Lpublic.h b/src/H5Lpublic.h index 6f6c638..60e83f2 100644 --- a/src/H5Lpublic.h +++ b/src/H5Lpublic.h @@ -923,6 +923,11 @@ H5_DLL herr_t H5Literate2(hid_t grp_id, H5_index_t idx_type, H5_iter_order_t ord /** * -------------------------------------------------------------------------- * \ingroup ASYNC + * + * \warning The returned value of the callback routine op will not be set + * in the return value for H5Literate_async(), so the \p herr_t value + * should not be used for determining the return state of the callback routine. + * * \async_variant_of{H5Literate} */ #ifndef H5_DOXYGEN diff --git a/src/H5VLnative_file.c b/src/H5VLnative_file.c index 907a12d..f2f0ea7 100644 --- a/src/H5VLnative_file.c +++ b/src/H5VLnative_file.c @@ -753,29 +753,35 @@ H5VL__native_file_close(void *file, hid_t H5_ATTR_UNUSED dxpl_id, void H5_ATTR_U FUNC_ENTER_PACKAGE /* This routine should only be called when a file ID's ref count drops to zero */ - HDassert(H5F_ID_EXISTS(f)); - - /* Flush file if this is the last reference to this id and we have write - * intent, unless it will be flushed by the "shared" file being closed. - * This is only necessary to replicate previous behaviour, and could be - * disabled by an option/property to improve performance. - */ - if ((H5F_NREFS(f) > 1) && (H5F_INTENT(f) & H5F_ACC_RDWR)) { - /* Get the file ID corresponding to the H5F_t struct */ - if (H5I_find_id(f, H5I_FILE, &file_id) < 0 || H5I_INVALID_HID == file_id) - HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "invalid ID") - - /* Get the number of references outstanding for this file ID */ - if ((nref = H5I_get_ref(file_id, FALSE)) < 0) - HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "can't get ID ref count") - if (nref == 1) - if (H5F__flush(f) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cache") - } /* end if */ - - /* Close the file */ - if (H5F__close(f) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "can't close file") + HDassert(f->shared == NULL || H5F_ID_EXISTS(f)); + + if (f->shared == NULL) + f = H5FL_FREE(H5F_t, f); + + else { + + /* Flush file if this is the last reference to this id and we have write + * intent, unless it will be flushed by the "shared" file being closed. + * This is only necessary to replicate previous behaviour, and could be + * disabled by an option/property to improve performance. + */ + if ((H5F_NREFS(f) > 1) && (H5F_INTENT(f) & H5F_ACC_RDWR)) { + /* Get the file ID corresponding to the H5F_t struct */ + if (H5I_find_id(f, H5I_FILE, &file_id) < 0 || H5I_INVALID_HID == file_id) + HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "invalid ID") + + /* Get the number of references outstanding for this file ID */ + if ((nref = H5I_get_ref(file_id, FALSE)) < 0) + HGOTO_ERROR(H5E_ID, H5E_CANTGET, FAIL, "can't get ID ref count") + if (nref == 1) + if (H5F__flush(f) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTFLUSH, FAIL, "unable to flush cache") + } /* end if */ + + /* Close the file */ + if (H5F__close(f) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDEC, FAIL, "can't close file") + } done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/test/CMakeTests.cmake b/test/CMakeTests.cmake index 74f63f4..5086194 100644 --- a/test/CMakeTests.cmake +++ b/test/CMakeTests.cmake @@ -126,6 +126,7 @@ set (HDF5_REFERENCE_TEST_FILES btree_idx_1_6.h5 btree_idx_1_8.h5 corrupt_stab_msg.h5 + cve_2020_10812.h5 deflate.h5 family_v16-000000.h5 family_v16-000001.h5 diff --git a/test/chunk_info.c b/test/chunk_info.c index e38752f..5651b26 100644 --- a/test/chunk_info.c +++ b/test/chunk_info.c @@ -1508,9 +1508,9 @@ error: typedef struct chunk_iter_info_t { hsize_t offset[2]; - uint32_t filter_mask; + unsigned filter_mask; haddr_t addr; - uint32_t nbytes; + hsize_t size; } chunk_iter_info_t; typedef struct chunk_iter_udata_t { @@ -1519,7 +1519,7 @@ typedef struct chunk_iter_udata_t { } chunk_iter_udata_t; static int -iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbytes, void *op_data) +iter_cb(const hsize_t *offset, unsigned filter_mask, haddr_t addr, hsize_t size, void *op_data) { chunk_iter_udata_t *cidata = (chunk_iter_udata_t *)op_data; int idx = cidata->last_index + 1; @@ -1528,7 +1528,7 @@ iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbyt cidata->chunk_info[idx].offset[1] = offset[1]; cidata->chunk_info[idx].filter_mask = filter_mask; cidata->chunk_info[idx].addr = addr; - cidata->chunk_info[idx].nbytes = nbytes; + cidata->chunk_info[idx].size = size; cidata->last_index++; @@ -1536,8 +1536,8 @@ iter_cb(const hsize_t *offset, uint32_t filter_mask, haddr_t addr, uint32_t nbyt } static int -iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, - haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) +iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, unsigned H5_ATTR_UNUSED filter_mask, + haddr_t H5_ATTR_UNUSED addr, hsize_t H5_ATTR_UNUSED size, void *op_data) { chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; *chunk_info += 1; @@ -1545,8 +1545,8 @@ iter_cb_stop(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filte } static int -iter_cb_fail(const hsize_t H5_ATTR_UNUSED *offset, uint32_t H5_ATTR_UNUSED filter_mask, - haddr_t H5_ATTR_UNUSED addr, uint32_t H5_ATTR_UNUSED nbytes, void *op_data) +iter_cb_fail(const hsize_t H5_ATTR_UNUSED *offset, unsigned H5_ATTR_UNUSED filter_mask, + haddr_t H5_ATTR_UNUSED addr, hsize_t H5_ATTR_UNUSED size, void *op_data) { chunk_iter_info_t **chunk_info = (chunk_iter_info_t **)op_data; *chunk_info += 1; @@ -1718,7 +1718,7 @@ test_basic_query(hid_t fapl) FAIL_PUTS_ERROR("offset[1] mismatch"); if (chunk_infos[0].filter_mask != 0) FAIL_PUTS_ERROR("filter mask mismatch"); - if (chunk_infos[0].nbytes != 96) + if (chunk_infos[0].size != 96) FAIL_PUTS_ERROR("size mismatch"); if (chunk_infos[1].offset[0] != CHUNK_NX) diff --git a/test/cve_2020_10812.h5 b/test/cve_2020_10812.h5 Binary files differnew file mode 100755 index 0000000..a20369d --- /dev/null +++ b/test/cve_2020_10812.h5 diff --git a/test/tmisc.c b/test/tmisc.c index 5cad368..5fb44d4 100644 --- a/test/tmisc.c +++ b/test/tmisc.c @@ -330,6 +330,11 @@ typedef struct { #define MISC35_SPACE_DIM3 13 #define MISC35_NPOINTS 10 +/* Definitions for misc. test #37 */ +/* The test file is formerly named h5_nrefs_POC. + See https://nvd.nist.gov/vuln/detail/CVE-2020-10812 */ +#define CVE_2020_10812_FILENAME "cve_2020_10812.h5" + /**************************************************************** ** ** test_misc1(): test unlinking a dataset from a group and immediately @@ -6046,6 +6051,47 @@ test_misc36(void) /**************************************************************** ** +** test_misc37(): +** Test for seg fault issue when closing the provided test file +** which has an illegal file size in its cache image. +** See HDFFV-11052/CVE-2020-10812 for details. +** +****************************************************************/ +static void +test_misc37(void) +{ + const char *testfile = H5_get_srcdir_filename(CVE_2020_10812_FILENAME); + hbool_t driver_is_default_compatible; + hid_t fid; + herr_t ret; + + /* Output message about test being performed */ + MESSAGE(5, ("Fix for HDFFV-11052/CVE-2020-10812")); + + ret = h5_driver_is_default_vfd_compatible(H5P_DEFAULT, &driver_is_default_compatible); + CHECK(ret, FAIL, "h5_driver_is_default_vfd_compatible"); + + if (!driver_is_default_compatible) { + HDprintf("-- SKIPPED --\n"); + return; + } + + fid = H5Fopen(testfile, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(fid, FAIL, "H5Fopen"); + + /* This should fail due to the illegal file size. + It should fail gracefully and not seg fault */ + H5E_BEGIN_TRY + { + ret = H5Fclose(fid); + } + H5E_END_TRY; + VERIFY(ret, FAIL, "H5Fclose"); + +} /* end test_misc37() */ + +/**************************************************************** +** ** test_misc(): Main misc. test routine. ** ****************************************************************/ @@ -6111,6 +6157,7 @@ test_misc(void) test_misc34(); /* Test behavior of 0 and NULL in H5MM API calls */ test_misc35(); /* Test behavior of free-list & allocation statistics API calls */ test_misc36(); /* Exercise H5atclose and H5is_library_terminating */ + test_misc37(); /* Test for seg fault failure at file close */ } /* test_misc() */ diff --git a/tools/src/misc/h5debug.c b/tools/src/misc/h5debug.c index 05d37b8..b15ae09 100644 --- a/tools/src/misc/h5debug.c +++ b/tools/src/misc/h5debug.c @@ -815,8 +815,12 @@ main(int argc, char *argv[]) done: if (fapl > 0) H5Pclose(fapl); - if (fid > 0) - H5Fclose(fid); + if (fid > 0) { + if (H5Fclose(fid) < 0) { + HDfprintf(stderr, "Error in closing file!\n"); + exit_value = 1; + } + } /* Pop API context */ if (api_ctx_pushed) |