Merge pull request #1748 in HDFFV/hdf5 from ~DEROBINS/hdf5_der:links_iterate_zero_bug to develop

* commit '0a75da70a61d41a0bf20a5c617f3eb678ed0b103':
  Yanked qsort assert
  Switched to HGOTO_DONE() in the links code.
  Updated the HDqsort() macro to ensure we don't pass NULL buffers to qsort(3) in the future.
  Fixed a bug in the links code where iterating over an empty group would pass a NULL pointer to qsort(3), which is undefined behavior.

2 files changed, 26 insertions, 5 deletions

diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index 0bd31a0..986b1e8 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -385,6 +385,18 @@ Bug Fixes since HDF5-1.10.3 release
 
       (JTH - 2018/08/25, HDFFV-10501)
 
+    - When iterating over an old-style group (i.e., when not using the latest
+      file format) of size 0, a NULL pointer representing the empty links
+      table would be sent to qsort(3) for sorting, which is undefined behavior.
+
+      Iterating over an empty group is explicitly tested in the links test.
+      This has not caused any failures to date and was flagged by gcc's
+      -fsanitize=undefined.
+
+      The library no longer attempts to sort an empty array.
+
+      (DER - 2019/06/18, HDFFV-10829)
+
     Java Library:
     ----------------
     - JNI native library dependencies
diff --git a/src/H5Glink.c b/src/H5Glink.c
index 82a2dcf..04ccbc5 100644
--- a/src/H5Glink.c
+++ b/src/H5Glink.c
@@ -402,15 +402,14 @@ done:
 
 
 /*-------------------------------------------------------------------------
- * Function:	H5G__link_sort_table
+ * Function:    H5G__link_sort_table
  *
  * Purpose:     Sort table containing a list of links for a group
  *
- * Return:	Success:        Non-negative
- *		Failure:	Negative
+ * Return:      SUCCEED/FAIL
  *
- * Programmer:	Quincey Koziol
- *	        Nov 20, 2006
+ * Programmer:  Quincey Koziol
+ *              Nov 20, 2006
  *
  *-------------------------------------------------------------------------
  */
@@ -418,11 +417,20 @@ herr_t
 H5G__link_sort_table(H5G_link_table_t *ltable, H5_index_t idx_type,
     H5_iter_order_t order)
 {
+    herr_t  ret_value = SUCCEED;
+
     FUNC_ENTER_PACKAGE_NOERR
 
     /* Sanity check */
     HDassert(ltable);
 
+    /* Can't sort when empty since the links table will be NULL */
+    if(0 == ltable->nlinks)
+        HGOTO_DONE(ret_value);
+
+    /* This should never be NULL if the number of links is non-zero */
+    HDassert(ltable->lnks);
+
     /* Pick appropriate sorting routine */
     if(idx_type == H5_INDEX_NAME) {
         if(order == H5_ITER_INC)
@@ -442,6 +450,7 @@ H5G__link_sort_table(H5G_link_table_t *ltable, H5_index_t idx_type,
             HDassert(order == H5_ITER_NATIVE);
     } /* end else */
 
+done:
     FUNC_LEAVE_NOAPI(SUCCEED)
 } /* end H5G__link_sort_table() */