Fix for HDFFV-10355 (CVE-2017-17506).

1 files changed, 10 insertions, 6 deletions

diff --git a/src/H5SM.c b/src/H5SM.c
index d5ede7e..8e28529 100644
--- a/src/H5SM.c
+++ b/src/H5SM.c
@@ -70,7 +70,7 @@ static herr_t H5SM_write_mesg(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
 static herr_t H5SM_decr_ref(void *record, void *op_data, hbool_t *changed);
 static herr_t H5SM_delete_from_index(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
     H5SM_index_header_t *header, const H5O_shared_t * mesg,
-    unsigned *cache_flags, void ** /*out*/ encoded_mesg);
+    unsigned *cache_flags, size_t * /*out*/ mesg_size, void ** /*out*/ encoded_mesg);
 static herr_t H5SM_type_to_flag(unsigned type_id, unsigned *type_flag);
 static herr_t H5SM_read_iter_op(H5O_t *oh, H5O_mesg_t *mesg, unsigned sequence,
     unsigned *oh_modified, void *_udata);
@@ -1549,6 +1549,7 @@ H5SM_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, H5O_shared_t *sh_mesg)
     unsigned              cache_flags = H5AC__NO_FLAGS_SET;
     H5SM_table_cache_ud_t cache_udata;      /* User-data for callback */
     ssize_t               index_num;
+    size_t                mesg_size = 0;
     void                 *mesg_buf = NULL;
     void                 *native_mesg = NULL;
     unsigned              type_id;              /* Message type ID to operate on */
@@ -1578,8 +1579,8 @@ H5SM_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, H5O_shared_t *sh_mesg)
      * zero and any file space it uses needs to be freed.  mesg_buf holds the
      * serialized form of the message.
      */
-    if(H5SM_delete_from_index(f, dxpl_id, open_oh, &(table->indexes[index_num]), sh_mesg, &cache_flags, &mesg_buf) < 0)
-	HGOTO_ERROR(H5E_SOHM, H5E_CANTDELETE, FAIL, "unable to delete mesage from SOHM index")
+    if(H5SM_delete_from_index(f, dxpl_id, open_oh, &(table->indexes[index_num]), sh_mesg, &cache_flags, &mesg_size, &mesg_buf) < 0)
+        HGOTO_ERROR(H5E_SOHM, H5E_CANTDELETE, FAIL, "unable to delete mesage from SOHM index")
 
     /* Release the master SOHM table */
     if(H5AC_unprotect(f, dxpl_id, H5AC_SOHM_TABLE, H5F_SOHM_ADDR(f), table, cache_flags) < 0)
@@ -1591,7 +1592,7 @@ H5SM_delete(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh, H5O_shared_t *sh_mesg)
      * master table needs to be unprotected when we do this.
      */
     if(mesg_buf) {
-        if(NULL == (native_mesg = H5O_msg_decode(f, dxpl_id, open_oh, type_id, (const unsigned char *)mesg_buf)))
+        if(NULL == (native_mesg = H5O_msg_decode(f, dxpl_id, open_oh, type_id, mesg_size, (const unsigned char *)mesg_buf)))
             HGOTO_ERROR(H5E_SOHM, H5E_CANTDECODE, FAIL, "can't decode shared message.")
 
         if(H5O_msg_delete(f, dxpl_id, open_oh, type_id, native_mesg) < 0)
@@ -1778,7 +1779,7 @@ H5SM_decr_ref(void *record, void *op_data, hbool_t *changed)
 static herr_t
 H5SM_delete_from_index(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
     H5SM_index_header_t *header, const H5O_shared_t *mesg,
-    unsigned *cache_flags, void ** /*out*/ encoded_mesg)
+    unsigned *cache_flags, size_t * /*out*/ mesg_size, void ** /*out*/ encoded_mesg)
 {
     H5SM_list_t     *list = NULL;
     H5SM_mesg_key_t key;
@@ -1910,6 +1911,7 @@ H5SM_delete_from_index(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
 
         /* Return the message's encoding so anything it references can be freed */
         *encoded_mesg = encoding_buf;
+        *mesg_size = buf_size;
 
         /* If there are no messages left in the index, delete it */
         if(header->num_messages == 0) {
@@ -1951,8 +1953,10 @@ done:
     /* Free the message encoding, if we're not returning it in encoded_mesg
      * or if there's been an error.
      */
-    if(encoding_buf && (NULL == *encoded_mesg || ret_value < 0))
+    if(encoding_buf && (NULL == *encoded_mesg || ret_value < 0)) {
         encoding_buf = H5MM_xfree(encoding_buf);
+        *mesg_size = 0;
+    }
 
     FUNC_LEAVE_NOAPI_TAG(ret_value, FAIL)
 } /* end H5SM_delete_from_index() */