[svn-r29150] fix for Jira issue 9670 - HDF5 segfaults on corrupted file.

Change compare callback in Btree2 class to correctly account for errors.

tested on bb-8.

1 files changed, 22 insertions, 23 deletions

diff --git a/src/H5Gbtree2.c b/src/H5Gbtree2.c
index 1a99b12..ff7e200 100644
--- a/src/H5Gbtree2.c
+++ b/src/H5Gbtree2.c
@@ -78,7 +78,7 @@ typedef struct H5G_fh_ud_cmp_t {
 
 /* v2 B-tree driver callbacks for 'creation order' index */
 static herr_t H5G_dense_btree2_corder_store(void *native, const void *udata);
-static herr_t H5G_dense_btree2_corder_compare(const void *rec1, const void *rec2);
+static herr_t H5G_dense_btree2_corder_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5G_dense_btree2_corder_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5G_dense_btree2_corder_decode(const uint8_t *raw, void *native,
@@ -88,7 +88,7 @@ static herr_t H5G_dense_btree2_corder_debug(FILE *stream, int indent, int fwidth
 
 /* v2 B-tree driver callbacks for 'name' index */
 static herr_t H5G_dense_btree2_name_store(void *native, const void *udata);
-static herr_t H5G_dense_btree2_name_compare(const void *rec1, const void *rec2);
+static herr_t H5G__dense_btree2_name_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5G_dense_btree2_name_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5G_dense_btree2_name_decode(const uint8_t *raw, void *native,
@@ -111,7 +111,7 @@ const H5B2_class_t H5G_BT2_NAME[1]={{   /* B-tree class information */
     NULL,                               /* Create client callback context */
     NULL,                               /* Destroy client callback context */
     H5G_dense_btree2_name_store,        /* Record storage callback */
-    H5G_dense_btree2_name_compare,      /* Record comparison callback */
+    H5G__dense_btree2_name_compare,      /* Record comparison callback */
     H5G_dense_btree2_name_encode,       /* Record encoding callback */
     H5G_dense_btree2_name_decode,       /* Record decoding callback */
     H5G_dense_btree2_name_debug         /* Record debugging callback */
@@ -216,7 +216,7 @@ H5G_dense_btree2_name_store(void *_nrecord, const void *_udata)
 
 
 /*-------------------------------------------------------------------------
- * Function:	H5G_dense_btree2_name_compare
+ * Function:	H5G__dense_btree2_name_compare
  *
  * Purpose:	Compare two native information records, according to some key
  *
@@ -230,13 +230,13 @@ H5G_dense_btree2_name_store(void *_nrecord, const void *_udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5G_dense_btree2_name_compare(const void *_bt2_udata, const void *_bt2_rec)
+H5G__dense_btree2_name_compare(const void *_bt2_udata, const void *_bt2_rec, int *result)
 {
     const H5G_bt2_ud_common_t *bt2_udata = (const H5G_bt2_ud_common_t *)_bt2_udata;
     const H5G_dense_bt2_name_rec_t *bt2_rec = (const H5G_dense_bt2_name_rec_t *)_bt2_rec;
-    herr_t ret_value = FAIL;    /* Return value */
+    herr_t ret_value = SUCCEED;    /* Return value */
 
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
+    FUNC_ENTER_STATIC
 
     /* Sanity check */
     HDassert(bt2_udata);
@@ -246,20 +246,19 @@ H5G_dense_btree2_name_compare(const void *_bt2_udata, const void *_bt2_rec)
 {
 unsigned u;
 
-HDfprintf(stderr, "%s: bt2_udata = {'%s', %x}\n", "H5G_dense_btree2_name_compare", bt2_udata->name, (unsigned)bt2_udata->name_hash);
-HDfprintf(stderr, "%s: bt2_rec = {%x, ", "H5G_dense_btree2_name_compare", (unsigned)bt2_rec->hash);
+HDfprintf(stderr, "%s: bt2_udata = {'%s', %x}\n", "H5G__dense_btree2_name_compare", bt2_udata->name, (unsigned)bt2_udata->name_hash);
+HDfprintf(stderr, "%s: bt2_rec = {%x, ", "H5G__dense_btree2_name_compare", (unsigned)bt2_rec->hash);
 for(u = 0; u < H5G_DENSE_FHEAP_ID_LEN; u++)
     HDfprintf(stderr, "%02x%s", bt2_rec->id[u], (u < (H5G_DENSE_FHEAP_ID_LEN - 1) ? " " : "}\n"));
 }
 #endif /* QAK */
     /* Check hash value */
     if(bt2_udata->name_hash < bt2_rec->hash)
-        ret_value = (-1);
+        *result = (-1);
     else if(bt2_udata->name_hash > bt2_rec->hash)
-        ret_value = 1;
+        *result = 1;
     else {
         H5G_fh_ud_cmp_t fh_udata;       /* User data for fractal heap 'op' callback */
-        herr_t status;                  /* Status from fractal heap 'op' routine */
 
         /* Sanity check */
         HDassert(bt2_udata->name_hash == bt2_rec->hash);
@@ -276,16 +275,17 @@ for(u = 0; u < H5G_DENSE_FHEAP_ID_LEN; u++)
         fh_udata.cmp = 0;
 
         /* Check if the user's link and the B-tree's link have the same name */
-        status = H5HF_op(bt2_udata->fheap, bt2_udata->dxpl_id, bt2_rec->id,
-                H5G_dense_fh_name_cmp, &fh_udata);
-        HDassert(status >= 0);
+        if(H5HF_op(bt2_udata->fheap, bt2_udata->dxpl_id, bt2_rec->id,
+                   H5G_dense_fh_name_cmp, &fh_udata) < 0)
+            HGOTO_ERROR(H5E_HEAP, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
 
         /* Callback will set comparison value */
-        ret_value = fh_udata.cmp;
+        *result = fh_udata.cmp;
     } /* end else */
 
+done:
     FUNC_LEAVE_NOAPI(ret_value)
-} /* H5G_dense_btree2_name_compare() */
+} /* H5G__dense_btree2_name_compare() */
 
 
 /*-------------------------------------------------------------------------
@@ -419,11 +419,10 @@ H5G_dense_btree2_corder_store(void *_nrecord, const void *_udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5G_dense_btree2_corder_compare(const void *_bt2_udata, const void *_bt2_rec)
+H5G_dense_btree2_corder_compare(const void *_bt2_udata, const void *_bt2_rec, int *result)
 {
     const H5G_bt2_ud_common_t *bt2_udata = (const H5G_bt2_ud_common_t *)_bt2_udata;
     const H5G_dense_bt2_corder_rec_t *bt2_rec = (const H5G_dense_bt2_corder_rec_t *)_bt2_rec;
-    herr_t ret_value = FAIL;    /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
@@ -443,13 +442,13 @@ for(u = 0; u < H5G_DENSE_FHEAP_ID_LEN; u++)
 #endif /* QAK */
     /* Check creation order value */
     if(bt2_udata->corder < bt2_rec->corder)
-        ret_value = -1;
+        *result = -1;
     else if(bt2_udata->corder > bt2_rec->corder)
-        ret_value = 1;
+        *result = 1;
     else
-        ret_value = 0;
+        *result = 0;
 
-    FUNC_LEAVE_NOAPI(ret_value)
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5G_dense_btree2_corder_compare() */