[svn-r29150] fix for Jira issue 9670 - HDF5 segfaults on corrupted file.

Change compare callback in Btree2 class to correctly account for errors.

tested on bb-8.

1 files changed, 17 insertions, 8 deletions

diff --git a/src/H5B2test.c b/src/H5B2test.c
index e0c96fb..24163e0 100644
--- a/src/H5B2test.c
+++ b/src/H5B2test.c
@@ -65,7 +65,7 @@ typedef struct H5B2_test_ctx_t {
 static void *H5B2__test_crt_context(void *udata);
 static herr_t H5B2__test_dst_context(void *ctx);
 static herr_t H5B2__test_store(void *nrecord, const void *udata);
-static herr_t H5B2__test_compare(const void *rec1, const void *rec2);
+static herr_t H5B2__test_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5B2__test_encode(uint8_t *raw, const void *nrecord, void *ctx);
 static herr_t H5B2__test_decode(const uint8_t *raw, void *nrecord, void *ctx);
 static herr_t H5B2__test_debug(FILE *stream, int indent, int fwidth,
@@ -73,7 +73,7 @@ static herr_t H5B2__test_debug(FILE *stream, int indent, int fwidth,
 
 /* v2 B-tree driver callbacks for 'test2' B-trees */
 static herr_t H5B2__test2_store(void *nrecord, const void *udata);
-static herr_t H5B2__test2_compare(const void *rec1, const void *rec2);
+static herr_t H5B2__test2_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5B2__test2_encode(uint8_t *raw, const void *nrecord, void *ctx);
 static herr_t H5B2__test2_decode(const uint8_t *raw, void *nrecord, void *ctx);
 static herr_t H5B2__test2_debug(FILE *stream, int indent, int fwidth,
@@ -236,11 +236,13 @@ H5B2__test_store(void *nrecord, const void *udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5B2__test_compare(const void *rec1, const void *rec2)
+H5B2__test_compare(const void *rec1, const void *rec2, int *result)
 {
     FUNC_ENTER_STATIC_NOERR
 
-    FUNC_LEAVE_NOAPI((herr_t)(*(const hssize_t *)rec1 - *(const hssize_t *)rec2))
+    *result = (int)(*(const hssize_t *)rec1 - *(const hssize_t *)rec2);
+
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5B2__test_compare() */
 
 
@@ -369,11 +371,13 @@ H5B2__test2_store(void *nrecord, const void *udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5B2__test2_compare(const void *rec1, const void *rec2)
+H5B2__test2_compare(const void *rec1, const void *rec2, int *result)
 {
     FUNC_ENTER_STATIC_NOERR
 
-    FUNC_LEAVE_NOAPI((herr_t)(((const H5B2_test_rec_t *)rec1)->key - ((const H5B2_test_rec_t *)rec2)->key))
+    *result = (int)(((const H5B2_test_rec_t *)rec1)->key - ((const H5B2_test_rec_t *)rec2)->key);
+
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5B2__test2_compare() */
 
 
@@ -551,7 +555,10 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata,
             HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node")
 
         /* Locate node pointer for child */
-        cmp = H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx);
+        if(H5B2__locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, 
+                               udata, &idx, &cmp) < 0)
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+
         if(cmp > 0)
             idx++;
 
@@ -591,7 +598,9 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata,
             HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node")
 
         /* Locate record */
-        cmp = H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx);
+        if(H5B2__locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, 
+                               udata, &idx, &cmp) < 0)
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
 
         /* Unlock current node */
         if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr.addr, leaf, H5AC__NO_FLAGS_SET) < 0)