Partial merge issue #642 develop branch PRs to Hdf5 1 10 (#718)

* Revert addition of & to 2 parameters in DSetCreatPropList::setVirtual to
maintain binary compatibility.

* Fix H5Eget_auto2/H5Eauto_is_v2 to not clear error stack (#625)

* Removes gratuitous (double)x.yF casts (#632)

* Committing clang-format changes

* Removes gratuitous (double)x.yF casts

* Committing clang-format changes

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

* Cleans up a const warning left over from previous constification (#633)

* Committing clang-format changes

* Adds consts to a few global variables

* Cleans up a const warning left over from previous constification

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

* Purges UFAIL from the library (#637)

* Committing clang-format changes

* Purges UFAIL from the library

* H5HL_insert change requested in PR

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

* Bmr dev hdffv 11223 (#640)

* Fixed HDFFV-11223 (CVE-2018-14460)

Description
    - Added checks against buffer size to prevent segfault, in case of data
      corruption, for sdim->size and sdim->max.
    - Renamed data files in an existing test to shorten their length
      as agreed with other developers previously.
Platforms tested:
    Linux/64 (jelly)

* Committing clang-format changes

* Updated for test files

* Updated for HDFFV-11223

Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>

* Committing clang-format changes

* Restore "error:" in line 2666.

* Revert "Fix H5Eget_auto2/H5Eauto_is_v2 to not clear error stack (#625)"

This reverts commit 426b50484841118cf633fd6147302a63a30fd746.

Co-authored-by: jhendersonHDF <jhenderson@hdfgroup.org>
Co-authored-by: Dana Robinson <43805+derobins@users.noreply.github.com>
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: bmribler <39579120+bmribler@users.noreply.github.com>

1 files changed, 32 insertions, 27 deletions

diff --git a/src/H5SM.c b/src/H5SM.c
index ad2d9ef..a72a4dd 100644
--- a/src/H5SM.c
+++ b/src/H5SM.c
@@ -1220,16 +1220,16 @@ static herr_t
 H5SM__write_mesg(H5F_t *f, H5O_t *open_oh, H5SM_index_header_t *header, hbool_t defer, unsigned type_id,
                  void *mesg, unsigned *cache_flags_ptr)
 {
-    H5SM_list_t *        list = NULL;          /* List index */
-    H5SM_mesg_key_t      key;                  /* Key used to search the index */
-    H5SM_list_cache_ud_t cache_udata;          /* User-data for metadata cache callback */
-    H5O_shared_t         shared;               /* Shared H5O message */
-    hbool_t              found = FALSE;        /* Was the message in the index? */
-    H5HF_t *             fheap = NULL;         /* Fractal heap handle */
-    H5B2_t *             bt2   = NULL;         /* v2 B-tree handle for index */
-    size_t               buf_size;             /* Size of the encoded message */
-    void *               encoding_buf = NULL;  /* Buffer for encoded message */
-    size_t               empty_pos    = UFAIL; /* Empty entry in list */
+    H5SM_list_t *        list = NULL;             /* List index */
+    H5SM_mesg_key_t      key;                     /* Key used to search the index */
+    H5SM_list_cache_ud_t cache_udata;             /* User-data for metadata cache callback */
+    H5O_shared_t         shared;                  /* Shared H5O message */
+    hbool_t              found = FALSE;           /* Was the message in the index? */
+    H5HF_t *             fheap = NULL;            /* Fractal heap handle */
+    H5B2_t *             bt2   = NULL;            /* v2 B-tree handle for index */
+    size_t               buf_size;                /* Size of the encoded message */
+    void *               encoding_buf = NULL;     /* Buffer for encoded message */
+    size_t               empty_pos    = SIZE_MAX; /* Empty entry in list */
     herr_t               ret_value    = SUCCEED;
 
     FUNC_ENTER_STATIC_TAG(H5AC__SOHM_TAG)
@@ -1283,11 +1283,11 @@ H5SM__write_mesg(H5F_t *f, H5O_t *open_oh, H5SM_index_header_t *header, hbool_t
             HGOTO_ERROR(H5E_SOHM, H5E_CANTINSERT, FAIL, "unable to search for message in list")
 
         if (defer) {
-            if (list_pos != UFAIL)
+            if (list_pos != SIZE_MAX)
                 found = TRUE;
         } /* end if */
         else {
-            if (list_pos != UFAIL) {
+            if (list_pos != SIZE_MAX) {
                 /* If the message was previously shared in an object header, share
                  * it in the heap now.
                  */
@@ -1442,13 +1442,13 @@ H5SM__write_mesg(H5F_t *f, H5O_t *open_oh, H5SM_index_header_t *header, hbool_t
             /* Insert the new message into the SOHM index */
             if (header->index_type == H5SM_LIST) {
                 /* Index is a list.  Find an empty spot if we haven't already */
-                if (empty_pos == UFAIL) {
+                if (empty_pos == SIZE_MAX) {
                     size_t pos;
 
                     if (H5SM__find_in_list(list, NULL, &empty_pos, &pos) < 0)
                         HGOTO_ERROR(H5E_SOHM, H5E_CANTINSERT, FAIL, "unable to search for message in list")
 
-                    if (pos == UFAIL || empty_pos == UFAIL)
+                    if (pos == SIZE_MAX || empty_pos == SIZE_MAX)
                         HGOTO_ERROR(H5E_SOHM, H5E_CANTINSERT, FAIL, "unable to find empty entry in list")
                 }
                 /* Insert message into list */
@@ -1608,9 +1608,14 @@ done:
  *
  *              If EMPTY_POS is NULL, don't store anything in it.
  *
- * Return:      Message's position in the list on success
- *              UFAIL if message couldn't be found
- *              empty_pos set to position of empty message or UFAIL.
+ * Return:      Success:    SUCCEED
+ *                          pos = position (if found)
+ *                          pos = SIZE_MAX (if not found)
+ *                          empty_pos = indeterminate (if found)
+ *                          empty_pos = 1st empty position (if not found)
+ *
+ *              Failure:    FAIL
+ *                          pos & empty_pos indeterminate
  *
  * Programmer:  James Laird
  *              Tuesday, May 2, 2006
@@ -1631,7 +1636,7 @@ H5SM__find_in_list(const H5SM_list_t *list, const H5SM_mesg_key_t *key, size_t *
 
     /* Initialize empty_pos to an invalid value */
     if (empty_pos)
-        *empty_pos = UFAIL;
+        *empty_pos = SIZE_MAX;
 
     /* Find the first (only) message equal to the key passed in.
      * Also record the first empty position we find.
@@ -1654,11 +1659,11 @@ H5SM__find_in_list(const H5SM_list_t *list, const H5SM_mesg_key_t *key, size_t *
 
             /* Found earlier position possible, don't check any more */
             empty_pos = NULL;
-        } /* end if */
-    }     /* end for */
+        }
+    }
 
     /* If we reached this point, we didn't find the message */
-    *pos = UFAIL;
+    *pos = SIZE_MAX;
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -1826,7 +1831,7 @@ H5SM__delete_from_index(H5F_t *f, H5O_t *open_oh, H5SM_index_header_t *header, c
         /* Find the message in the list */
         if (H5SM__find_in_list(list, &key, NULL, &list_pos) < 0)
             HGOTO_ERROR(H5E_SOHM, H5E_NOTFOUND, FAIL, "unable to search for message in list")
-        if (list_pos == UFAIL)
+        if (list_pos == SIZE_MAX)
             HGOTO_ERROR(H5E_SOHM, H5E_NOTFOUND, FAIL, "message not in index")
 
         if (list->messages[list_pos].location == H5SM_IN_HEAP)
@@ -2208,7 +2213,7 @@ H5SM_get_refcount(H5F_t *f, unsigned type_id, const H5O_shared_t *sh_mesg, hsize
         /* Find the message in the list */
         if (H5SM__find_in_list(list, &key, NULL, &list_pos) < 0)
             HGOTO_ERROR(H5E_SOHM, H5E_NOTFOUND, FAIL, "unable to search for message in list")
-        if (list_pos == UFAIL)
+        if (list_pos == SIZE_MAX)
             HGOTO_ERROR(H5E_SOHM, H5E_NOTFOUND, FAIL, "message not in index")
 
         /* Copy the message */
@@ -2505,7 +2510,7 @@ H5SM_list_free(H5SM_list_t *list)
  *
  * Purpose:     Print debugging information for the master table.
  *
- *              If table_vers and num_indexes are not UFAIL, they are used
+ *              If table_vers and num_indexes are not UINT_MAX, they are used
  *              instead of the values in the superblock.
  *
  * Return:      Non-negative on success/Negative on failure
@@ -2532,14 +2537,14 @@ H5SM_table_debug(H5F_t *f, haddr_t table_addr, FILE *stream, int indent, int fwi
     HDassert(indent >= 0);
     HDassert(fwidth >= 0);
 
-    /* If table_vers and num_indexes are UFAIL, replace them with values from
+    /* If table_vers and num_indexes are UINT_MAX, replace them with values from
      * userblock
      */
-    if (table_vers == UFAIL)
+    if (table_vers == UINT_MAX)
         table_vers = H5F_SOHM_VERS(f);
     else if (table_vers != H5F_SOHM_VERS(f))
         HDfprintf(stream, "*** SOHM TABLE VERSION DOESN'T MATCH VERSION IN SUPERBLOCK!\n");
-    if (num_indexes == UFAIL)
+    if (num_indexes == UINT_MAX)
         num_indexes = H5F_SOHM_NINDEXES(f);
     else if (num_indexes != H5F_SOHM_NINDEXES(f))
         HDfprintf(stream, "*** NUMBER OF SOHM INDEXES DOESN'T MATCH VALUE IN SUPERBLOCK!\n");