[svn-r29156] merge 29150 from trunk.

fix for Jira issue 9670 - HDF5 segfaults on corrupted file.
Change compare callback in Btree2 class to correctly account for errors.

tested on bb-8.

11 files changed, 202 insertions, 151 deletions

diff --git a/src/H5Abtree2.c b/src/H5Abtree2.c
index 8958e25..483c329 100644
--- a/src/H5Abtree2.c
+++ b/src/H5Abtree2.c
@@ -80,7 +80,7 @@ typedef struct H5A_fh_ud_cmp_t {
 
 /* v2 B-tree driver callbacks for 'creation order' index */
 static herr_t H5A_dense_btree2_corder_store(void *native, const void *udata);
-static herr_t H5A_dense_btree2_corder_compare(const void *rec1, const void *rec2);
+static herr_t H5A_dense_btree2_corder_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5A_dense_btree2_corder_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5A_dense_btree2_corder_decode(const uint8_t *raw, void *native,
@@ -90,7 +90,7 @@ static herr_t H5A_dense_btree2_corder_debug(FILE *stream, const H5F_t *f, hid_t
 
 /* v2 B-tree driver callbacks for 'name' index */
 static herr_t H5A_dense_btree2_name_store(void *native, const void *udata);
-static herr_t H5A_dense_btree2_name_compare(const void *rec1, const void *rec2);
+static herr_t H5A_dense_btree2_name_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5A_dense_btree2_name_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5A_dense_btree2_name_decode(const uint8_t *raw, void *native,
@@ -249,13 +249,13 @@ H5A_dense_btree2_name_store(void *_nrecord, const void *_udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5A_dense_btree2_name_compare(const void *_bt2_udata, const void *_bt2_rec)
+H5A_dense_btree2_name_compare(const void *_bt2_udata, const void *_bt2_rec, int *result)
 {
     const H5A_bt2_ud_common_t *bt2_udata = (const H5A_bt2_ud_common_t *)_bt2_udata;
     const H5A_dense_bt2_name_rec_t *bt2_rec = (const H5A_dense_bt2_name_rec_t *)_bt2_rec;
-    herr_t ret_value;           /* Return value */
+    herr_t ret_value = SUCCEED;            /* Return value */
 
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
+    FUNC_ENTER_NOAPI_NOINIT
 
     /* Sanity check */
     HDassert(bt2_udata);
@@ -263,13 +263,12 @@ H5A_dense_btree2_name_compare(const void *_bt2_udata, const void *_bt2_rec)
 
     /* Check hash value */
     if(bt2_udata->name_hash < bt2_rec->hash)
-        ret_value = (-1);
+        *result = (-1);
     else if(bt2_udata->name_hash > bt2_rec->hash)
-        ret_value = 1;
+        *result = 1;
     else {
         H5A_fh_ud_cmp_t fh_udata;       /* User data for fractal heap 'op' callback */
         H5HF_t *fheap;                  /* Fractal heap handle to use for finding object */
-        herr_t status;                  /* Status from fractal heap 'op' routine */
 
         /* Sanity check */
         HDassert(bt2_udata->name_hash == bt2_rec->hash);
@@ -294,13 +293,14 @@ H5A_dense_btree2_name_compare(const void *_bt2_udata, const void *_bt2_rec)
         HDassert(fheap);
 
         /* Check if the user's attribute and the B-tree's attribute have the same name */
-        status = H5HF_op(fheap, bt2_udata->dxpl_id, &bt2_rec->id, H5A_dense_fh_name_cmp, &fh_udata);
-        HDassert(status >= 0);
+        if(H5HF_op(fheap, bt2_udata->dxpl_id, &bt2_rec->id, H5A_dense_fh_name_cmp, &fh_udata) < 0)
+            HGOTO_ERROR(H5E_HEAP, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
 
         /* Callback will set comparison value */
-        ret_value = fh_udata.cmp;
+        *result = fh_udata.cmp;
     } /* end else */
 
+done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* H5A_dense_btree2_name_compare() */
 
@@ -441,11 +441,10 @@ H5A_dense_btree2_corder_store(void *_nrecord, const void *_udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5A_dense_btree2_corder_compare(const void *_bt2_udata, const void *_bt2_rec)
+H5A_dense_btree2_corder_compare(const void *_bt2_udata, const void *_bt2_rec, int *result)
 {
     const H5A_bt2_ud_common_t *bt2_udata = (const H5A_bt2_ud_common_t *)_bt2_udata;
     const H5A_dense_bt2_corder_rec_t *bt2_rec = (const H5A_dense_bt2_corder_rec_t *)_bt2_rec;
-    herr_t ret_value;           /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
@@ -455,13 +454,13 @@ H5A_dense_btree2_corder_compare(const void *_bt2_udata, const void *_bt2_rec)
 
     /* Check creation order value */
     if(bt2_udata->corder < bt2_rec->corder)
-        ret_value = -1;
+        *result = -1;
     else if(bt2_udata->corder > bt2_rec->corder)
-        ret_value = 1;
+        *result = 1;
     else
-        ret_value = 0;
+        *result = 0;
 
-    FUNC_LEAVE_NOAPI(ret_value)
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5A_dense_btree2_corder_compare() */
 
 
diff --git a/src/H5B2.c b/src/H5B2.c
index 7539a24..1a3223d 100644
--- a/src/H5B2.c
+++ b/src/H5B2.c
@@ -450,8 +450,10 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op,
      *  find candidates or avoid further searching.
      */
     if(hdr->min_native_rec != NULL) {
-	if((cmp = (hdr->cls->compare)(udata, hdr->min_native_rec)) < 0)
-	    HGOTO_DONE(FALSE) 	/* Less than the least record--not found */ 
+        if((hdr->cls->compare)(udata, hdr->min_native_rec, &cmp) < 0)
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+        if(cmp < 0)
+            HGOTO_DONE(FALSE) 	/* Less than the least record--not found */ 
 	else if(cmp == 0) { /* Record is found */
 	    if(op && (op)(hdr->min_native_rec, op_data) < 0)
                 HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "'found' callback failed for B-tree find operation")
@@ -459,8 +461,10 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op,
 	} /* end if */
     } /* end if */
     if(hdr->max_native_rec != NULL) {
-	if((cmp = (hdr->cls->compare)(udata, hdr->max_native_rec)) > 0)
-	    HGOTO_DONE(FALSE) 	/* Greater than the greatest record--not found */
+	if((hdr->cls->compare)(udata, hdr->max_native_rec, &cmp) < 0)
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+        if(cmp > 0)
+            HGOTO_DONE(FALSE) 	/* Less than the least record--not found */ 
 	else if(cmp == 0) { /* Record is found */
 	    if(op && (op)(hdr->max_native_rec, op_data) < 0)
                 HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "'found' callback failed for B-tree find operation")
@@ -483,7 +487,13 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op,
             HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node")
 
         /* Locate node pointer for child */
-        cmp = H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx);
+        if(H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, 
+                               udata, &idx, &cmp) < 0) {
+            /* Unlock current node before failing */
+            H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, H5AC__NO_FLAGS_SET);
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+        }
+
         if(cmp > 0)
             idx++;
 
@@ -546,7 +556,12 @@ H5B2_find(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_found_t op,
             HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node")
 
         /* Locate record */
-        cmp = H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx);
+        if(H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, 
+                               udata, &idx, &cmp) < 0) {
+            /* unlock current node before failing */
+            H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr.addr, leaf, H5AC__NO_FLAGS_SET);
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+        }
 
         if(cmp != 0) {
             /* Unlock leaf node */
@@ -1074,7 +1089,13 @@ H5B2_modify(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_modify_t op,
             HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node")
 
         /* Locate node pointer for child */
-        cmp = H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx);
+        if(H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, 
+                               udata, &idx, &cmp) < 0) {
+            /* Unlock current node */
+            H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_INT, curr_node_ptr.addr, internal, H5AC__NO_FLAGS_SET);
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+        }
+
         if(cmp > 0)
             idx++;
 
@@ -1146,7 +1167,11 @@ H5B2_modify(H5B2_t *bt2, hid_t dxpl_id, void *udata, H5B2_modify_t op,
             HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node")
 
         /* Locate record */
-        cmp = H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx);
+        if(H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, 
+                               udata, &idx, &cmp) < 0) {
+            H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr.addr, leaf, H5AC__NO_FLAGS_SET);
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+        }
 
         if(cmp != 0) {
             /* Unlock leaf node */
diff --git a/src/H5B2int.c b/src/H5B2int.c
index 01403eb..2a5fc38 100644
--- a/src/H5B2int.c
+++ b/src/H5B2int.c
@@ -132,20 +132,24 @@ H5FL_SEQ_EXTERN(H5B2_node_info_t);
  *
  *-------------------------------------------------------------------------
  */
-int
+herr_t
 H5B2_locate_record(const H5B2_class_t *type, unsigned nrec, size_t *rec_off,
-    const uint8_t *native, const void *udata, unsigned *idx)
+                    const uint8_t *native, const void *udata, unsigned *idx, int *cmp)
 {
     unsigned	lo = 0, hi;     /* Low & high index values */
     unsigned    my_idx = 0;     /* Final index value */
-    int         cmp = -1;       /* Key comparison value */
+    herr_t      ret_value = SUCCEED;
+
+    FUNC_ENTER_NOAPI_NOINIT
 
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
+    *cmp = -1;
 
     hi = nrec;
-    while(lo < hi && cmp) {
+    while(lo < hi && *cmp) {
 	my_idx = (lo + hi) / 2;
-	if((cmp = (type->compare)(udata, native + rec_off[my_idx])) < 0)
+	if((type->compare)(udata, native + rec_off[my_idx], cmp) < 0)
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+	if(*cmp < 0)
 	    hi = my_idx;
 	else
 	    lo = my_idx + 1;
@@ -153,7 +157,8 @@ H5B2_locate_record(const H5B2_class_t *type, unsigned nrec, size_t *rec_off,
 
     *idx = my_idx;
 
-    FUNC_LEAVE_NOAPI(cmp)
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5B2_locate_record */
 
 
@@ -1555,7 +1560,9 @@ H5B2_insert_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr,
         idx = 0;
     else {
         /* Find correct location to insert this record */
-        if((cmp = H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx)) == 0)
+        if(H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0)
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+        if(cmp == 0)
             HGOTO_ERROR(H5E_BTREE, H5E_EXISTS, FAIL, "record is already in B-tree")
         if(cmp > 0)
             idx++;
@@ -1649,7 +1656,10 @@ H5B2_insert_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, unsigned depth,
         size_t      split_nrec; /* Number of records to split node at */
 
         /* Locate node pointer for child */
-        if((cmp = H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx)) == 0)
+        if(H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, 
+                               udata, &idx, &cmp) < 0)
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+        if(cmp == 0)
             HGOTO_ERROR(H5E_BTREE, H5E_EXISTS, FAIL, "record is already in B-tree")
         if(cmp > 0)
             idx++;
@@ -1704,8 +1714,11 @@ H5B2_insert_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, unsigned depth,
             } /* end else */
 
             /* Locate node pointer for child (after split/redistribute) */
-/* Actually, this can be easily updated (for 2-node redistrib.) and shouldn't require re-searching */
-            if((cmp = H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx)) == 0)
+            /* Actually, this can be easily updated (for 2-node redistrib.) and shouldn't require re-searching */
+            if(H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, 
+                                   udata, &idx, &cmp) < 0)
+                HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+            if(cmp == 0)
                 HGOTO_ERROR(H5E_BTREE, H5E_EXISTS, FAIL, "record is already in B-tree")
             if(cmp > 0)
                 idx++;
@@ -2116,6 +2129,7 @@ H5B2_remove_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr,
     haddr_t     leaf_addr = HADDR_UNDEF;  /* Leaf address on disk */
     unsigned    leaf_flags = H5AC__NO_FLAGS_SET; /* Flags for unprotecting leaf node */
     unsigned    idx;                    /* Location of record which matches key */
+    int         cmp;                    /* Comparison value of records */
     herr_t	ret_value = SUCCEED;    /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT
@@ -2135,7 +2149,9 @@ H5B2_remove_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_ptr,
     HDassert(leaf->nrec == curr_node_ptr->node_nrec);
 
     /* Find correct location to remove this record */
-    if(H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx) != 0)
+    if(H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0)
+        HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+    if(cmp != 0)
         HGOTO_ERROR(H5E_BTREE, H5E_NOTFOUND, FAIL, "record is not in B-tree")
 
     /* Check for invalidating the min/max record for the tree */
@@ -2283,7 +2299,9 @@ H5B2_remove_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, hbool_t *depth_decreased,
         if(swap_loc)
             idx = 0;
         else {
-            cmp = H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx);
+            if(H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, 
+                                   udata, &idx, &cmp) < 0)
+                HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
             if(cmp >= 0)
                 idx++;
         } /* end else */
@@ -2345,7 +2363,8 @@ H5B2_remove_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, hbool_t *depth_decreased,
                 idx = 0;
             else {
 /* Actually, this can be easily updated (for 2-node redistrib.) and shouldn't require re-searching */
-                cmp = H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx);
+                if(H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx, &cmp) < 0)
+                    HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
                 if(cmp >= 0)
                     idx++;
             } /* end else */
@@ -2833,7 +2852,8 @@ H5B2_neighbor_leaf(H5B2_hdr_t *hdr, hid_t dxpl_id, H5B2_node_ptr_t *curr_node_pt
         HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node")
 
     /* Locate node pointer for child */
-    cmp = H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx);
+    if(H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx, &cmp) < 0)
+        HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
     if(cmp > 0)
         idx++;
     else
@@ -2920,7 +2940,9 @@ H5B2_neighbor_internal(H5B2_hdr_t *hdr, hid_t dxpl_id, unsigned depth,
         HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree internal node")
 
     /* Locate node pointer for child */
-    cmp = H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx);
+    if(H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, 
+                           udata, &idx, &cmp) < 0)
+        HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
     if(cmp > 0)
         idx++;
 
diff --git a/src/H5B2pkg.h b/src/H5B2pkg.h
index 3ea9534..e4a5d1c 100644
--- a/src/H5B2pkg.h
+++ b/src/H5B2pkg.h
@@ -326,7 +326,7 @@ H5_DLL herr_t H5B2_node_size(H5B2_hdr_t *hdr, hid_t dxpl_id,
 
 /* Routines for locating records */
 H5_DLL int H5B2_locate_record(const H5B2_class_t *type, unsigned nrec,
-    size_t *rec_off, const uint8_t *native, const void *udata, unsigned *idx);
+    size_t *rec_off, const uint8_t *native, const void *udata, unsigned *idx, int *result);
 H5_DLL herr_t H5B2_neighbor_internal(H5B2_hdr_t *hdr, hid_t dxpl_id,
     unsigned depth, H5B2_node_ptr_t *curr_node_ptr, void *neighbor_loc,
     H5B2_compare_t comp, void *udata, H5B2_found_t op, void *op_data);
diff --git a/src/H5B2private.h b/src/H5B2private.h
index 08d3ce1..fd9bf0e 100644
--- a/src/H5B2private.h
+++ b/src/H5B2private.h
@@ -89,7 +89,7 @@ struct H5B2_class_t {
     void *(*crt_context)(void *udata);          /* Create context for other client callbacks */
     herr_t (*dst_context)(void *ctx);           /* Destroy client callback context */
     herr_t (*store)(void *nrecord, const void *udata);              	/* Store application record in native record table */
-    herr_t (*compare)(const void *rec1, const void *rec2); 		/* Compare two native records */
+    herr_t (*compare)(const void *rec1, const void *rec2, int *result); /* Compare two native records */
     herr_t (*encode)(uint8_t *raw, const void *record, void *ctx);  	/* Encode record from native form to disk storage form */
     herr_t (*decode)(const uint8_t *raw, void *record, void *ctx);  	/* Decode record from disk storage form to native form */
     herr_t (*debug)(FILE *stream, const H5F_t *f, hid_t dxpl_id,    	/* Print a record for debugging */
diff --git a/src/H5B2test.c b/src/H5B2test.c
index 2537cbc..dde0e71 100644
--- a/src/H5B2test.c
+++ b/src/H5B2test.c
@@ -64,7 +64,7 @@ typedef struct H5B2_test_ctx_t {
 static void *H5B2_test_crt_context(void *udata);
 static herr_t H5B2_test_dst_context(void *ctx);
 static herr_t H5B2_test_store(void *nrecord, const void *udata);
-static herr_t H5B2_test_compare(const void *rec1, const void *rec2);
+static herr_t H5B2_test_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5B2_test_encode(uint8_t *raw, const void *nrecord, void *ctx);
 static herr_t H5B2_test_decode(const uint8_t *raw, void *nrecord, void *ctx);
 static herr_t H5B2_test_debug(FILE *stream, const H5F_t *f, hid_t dxpl_id,
@@ -215,11 +215,13 @@ H5B2_test_store(void *nrecord, const void *udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5B2_test_compare(const void *rec1, const void *rec2)
+H5B2_test_compare(const void *rec1, const void *rec2, int *result)
 {
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
-    FUNC_LEAVE_NOAPI((herr_t)(*(const hssize_t *)rec1 - *(const hssize_t *)rec2))
+    *result = (int)(*(const hssize_t *)rec1 - *(const hssize_t *)rec2);
+
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5B2_test_compare() */
 
 
@@ -434,7 +436,10 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata,
             HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node")
 
         /* Locate node pointer for child */
-        cmp = H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, udata, &idx);
+        if(H5B2_locate_record(hdr->cls, internal->nrec, hdr->nat_off, internal->int_native, 
+                               udata, &idx, &cmp) < 0)
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
+
         if(cmp > 0)
             idx++;
 
@@ -474,7 +479,9 @@ H5B2_get_node_info_test(H5B2_t *bt2, hid_t dxpl_id, void *udata,
             HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to protect B-tree leaf node")
 
         /* Locate record */
-        cmp = H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, udata, &idx);
+        if(H5B2_locate_record(hdr->cls, leaf->nrec, hdr->nat_off, leaf->leaf_native, 
+                               udata, &idx, &cmp) < 0)
+            HGOTO_ERROR(H5E_BTREE, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
 
         /* Unlock current node */
         if(H5AC_unprotect(hdr->f, dxpl_id, H5AC_BT2_LEAF, curr_node_ptr.addr, leaf, H5AC__NO_FLAGS_SET) < 0)
diff --git a/src/H5Gbtree2.c b/src/H5Gbtree2.c
index 8d37940..1323ea0 100644
--- a/src/H5Gbtree2.c
+++ b/src/H5Gbtree2.c
@@ -78,7 +78,7 @@ typedef struct H5G_fh_ud_cmp_t {
 
 /* v2 B-tree driver callbacks for 'creation order' index */
 static herr_t H5G_dense_btree2_corder_store(void *native, const void *udata);
-static herr_t H5G_dense_btree2_corder_compare(const void *rec1, const void *rec2);
+static herr_t H5G_dense_btree2_corder_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5G_dense_btree2_corder_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5G_dense_btree2_corder_decode(const uint8_t *raw, void *native,
@@ -88,7 +88,7 @@ static herr_t H5G_dense_btree2_corder_debug(FILE *stream, const H5F_t *f, hid_t
 
 /* v2 B-tree driver callbacks for 'name' index */
 static herr_t H5G_dense_btree2_name_store(void *native, const void *udata);
-static herr_t H5G_dense_btree2_name_compare(const void *rec1, const void *rec2);
+static herr_t H5G_dense_btree2_name_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5G_dense_btree2_name_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5G_dense_btree2_name_decode(const uint8_t *raw, void *native,
@@ -234,36 +234,25 @@ H5G_dense_btree2_name_store(void *_nrecord, const void *_udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5G_dense_btree2_name_compare(const void *_bt2_udata, const void *_bt2_rec)
+H5G_dense_btree2_name_compare(const void *_bt2_udata, const void *_bt2_rec, int *result)
 {
     const H5G_bt2_ud_common_t *bt2_udata = (const H5G_bt2_ud_common_t *)_bt2_udata;
     const H5G_dense_bt2_name_rec_t *bt2_rec = (const H5G_dense_bt2_name_rec_t *)_bt2_rec;
-    herr_t ret_value;           /* Return value */
+    herr_t ret_value = SUCCEED;    /* Return value */
 
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
+    FUNC_ENTER_NOAPI_NOINIT
 
     /* Sanity check */
     HDassert(bt2_udata);
     HDassert(bt2_rec);
 
-#ifdef QAK
-{
-unsigned u;
-
-HDfprintf(stderr, "%s: bt2_udata = {'%s', %x}\n", "H5G_dense_btree2_name_compare", bt2_udata->name, (unsigned)bt2_udata->name_hash);
-HDfprintf(stderr, "%s: bt2_rec = {%x, ", "H5G_dense_btree2_name_compare", (unsigned)bt2_rec->hash);
-for(u = 0; u < H5G_DENSE_FHEAP_ID_LEN; u++)
-    HDfprintf(stderr, "%02x%s", bt2_rec->id[u], (u < (H5G_DENSE_FHEAP_ID_LEN - 1) ? " " : "}\n"));
-}
-#endif /* QAK */
     /* Check hash value */
     if(bt2_udata->name_hash < bt2_rec->hash)
-        ret_value = (-1);
+        *result = (-1);
     else if(bt2_udata->name_hash > bt2_rec->hash)
-        ret_value = 1;
+        *result = 1;
     else {
         H5G_fh_ud_cmp_t fh_udata;       /* User data for fractal heap 'op' callback */
-        herr_t status;                  /* Status from fractal heap 'op' routine */
 
         /* Sanity check */
         HDassert(bt2_udata->name_hash == bt2_rec->hash);
@@ -280,14 +269,15 @@ for(u = 0; u < H5G_DENSE_FHEAP_ID_LEN; u++)
         fh_udata.cmp = 0;
 
         /* Check if the user's link and the B-tree's link have the same name */
-        status = H5HF_op(bt2_udata->fheap, bt2_udata->dxpl_id, bt2_rec->id,
-                H5G_dense_fh_name_cmp, &fh_udata);
-        HDassert(status >= 0);
+        if(H5HF_op(bt2_udata->fheap, bt2_udata->dxpl_id, bt2_rec->id,
+                   H5G_dense_fh_name_cmp, &fh_udata) < 0)
+            HGOTO_ERROR(H5E_HEAP, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
 
         /* Callback will set comparison value */
-        ret_value = fh_udata.cmp;
+        *result = fh_udata.cmp;
     } /* end else */
 
+done:
     FUNC_LEAVE_NOAPI(ret_value)
 } /* H5G_dense_btree2_name_compare() */
 
@@ -424,11 +414,10 @@ H5G_dense_btree2_corder_store(void *_nrecord, const void *_udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5G_dense_btree2_corder_compare(const void *_bt2_udata, const void *_bt2_rec)
+H5G_dense_btree2_corder_compare(const void *_bt2_udata, const void *_bt2_rec, int *result)
 {
     const H5G_bt2_ud_common_t *bt2_udata = (const H5G_bt2_ud_common_t *)_bt2_udata;
     const H5G_dense_bt2_corder_rec_t *bt2_rec = (const H5G_dense_bt2_corder_rec_t *)_bt2_rec;
-    herr_t ret_value;           /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
@@ -448,13 +437,13 @@ for(u = 0; u < H5G_DENSE_FHEAP_ID_LEN; u++)
 #endif /* QAK */
     /* Check creation order value */
     if(bt2_udata->corder < bt2_rec->corder)
-        ret_value = -1;
+        *result = -1;
     else if(bt2_udata->corder > bt2_rec->corder)
-        ret_value = 1;
+        *result = 1;
     else
-        ret_value = 0;
+        *result = 0;
 
-    FUNC_LEAVE_NOAPI(ret_value)
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5G_dense_btree2_corder_compare() */
 
 
diff --git a/src/H5HFbtree2.c b/src/H5HFbtree2.c
index 0229ebe..4c22575 100644
--- a/src/H5HFbtree2.c
+++ b/src/H5HFbtree2.c
@@ -73,7 +73,7 @@ static herr_t H5HF_huge_bt2_dst_context(void *ctx);
 static void *H5HF_huge_bt2_crt_dbg_context(H5F_t *f, hid_t dxpl_id, haddr_t addr);
 
 static herr_t H5HF_huge_bt2_indir_store(void *native, const void *udata);
-static herr_t H5HF_huge_bt2_indir_compare(const void *rec1, const void *rec2);
+static herr_t H5HF_huge_bt2_indir_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5HF_huge_bt2_indir_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5HF_huge_bt2_indir_decode(const uint8_t *raw, void *native,
@@ -82,7 +82,7 @@ static herr_t H5HF_huge_bt2_indir_debug(FILE *stream, const H5F_t *f, hid_t dxpl
     int indent, int fwidth, const void *record, const void *_udata);
 
 static herr_t H5HF_huge_bt2_filt_indir_store(void *native, const void *udata);
-static herr_t H5HF_huge_bt2_filt_indir_compare(const void *rec1, const void *rec2);
+static herr_t H5HF_huge_bt2_filt_indir_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5HF_huge_bt2_filt_indir_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5HF_huge_bt2_filt_indir_decode(const uint8_t *raw, void *native,
@@ -91,7 +91,7 @@ static herr_t H5HF_huge_bt2_filt_indir_debug(FILE *stream, const H5F_t *f, hid_t
     int indent, int fwidth, const void *record, const void *_udata);
 
 static herr_t H5HF_huge_bt2_dir_store(void *native, const void *udata);
-static herr_t H5HF_huge_bt2_dir_compare(const void *rec1, const void *rec2);
+static herr_t H5HF_huge_bt2_dir_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5HF_huge_bt2_dir_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5HF_huge_bt2_dir_decode(const uint8_t *raw, void *native,
@@ -100,7 +100,7 @@ static herr_t H5HF_huge_bt2_dir_debug(FILE *stream, const H5F_t *f, hid_t dxpl_i
     int indent, int fwidth, const void *record, const void *_udata);
 
 static herr_t H5HF_huge_bt2_filt_dir_store(void *native, const void *udata);
-static herr_t H5HF_huge_bt2_filt_dir_compare(const void *rec1, const void *rec2);
+static herr_t H5HF_huge_bt2_filt_dir_compare(const void *rec1, const void *rec2, int *result);
 static herr_t H5HF_huge_bt2_filt_dir_encode(uint8_t *raw, const void *native,
     void *ctx);
 static herr_t H5HF_huge_bt2_filt_dir_decode(const uint8_t *raw, void *native,
@@ -408,20 +408,14 @@ H5HF_huge_bt2_indir_store(void *nrecord, const void *udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5HF_huge_bt2_indir_compare(const void *_rec1, const void *_rec2)
+H5HF_huge_bt2_indir_compare(const void *_rec1, const void *_rec2, int *result)
 {
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
-#ifdef QAK
-{
-const H5HF_huge_bt2_indir_rec_t *rec1 = (const H5HF_huge_bt2_indir_rec_t *)_rec1;
-const H5HF_huge_bt2_indir_rec_t *rec2 = (const H5HF_huge_bt2_indir_rec_t *)_rec2;
+    *result = (int)(((const H5HF_huge_bt2_indir_rec_t *)_rec1)->id - 
+                    ((const H5HF_huge_bt2_indir_rec_t *)_rec2)->id);
 
-HDfprintf(stderr, "%s: rec1 = {%a, %Hu, %Hu}\n", "H5HF_huge_bt2_indir_compare", rec1->addr, rec1->len, rec1->id);
-HDfprintf(stderr, "%s: rec2 = {%a, %Hu, %Hu}\n", "H5HF_huge_bt2_indir_compare", rec2->addr, rec2->len, rec2->id);
-}
-#endif /* QAK */
-    FUNC_LEAVE_NOAPI((herr_t)(((const H5HF_huge_bt2_indir_rec_t *)_rec1)->id - ((const H5HF_huge_bt2_indir_rec_t *)_rec2)->id))
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5HF_huge_bt2_indir_compare() */
 
 
@@ -626,20 +620,14 @@ H5HF_huge_bt2_filt_indir_store(void *nrecord, const void *udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5HF_huge_bt2_filt_indir_compare(const void *_rec1, const void *_rec2)
+H5HF_huge_bt2_filt_indir_compare(const void *_rec1, const void *_rec2, int *result)
 {
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
-#ifdef QAK
-{
-const H5HF_huge_bt2_filt_indir_rec_t *rec1 = (const H5HF_huge_bt2_filt_indir_rec_t *)_rec1;
-const H5HF_huge_bt2_filt_indir_rec_t *rec2 = (const H5HF_huge_bt2_filt_indir_rec_t *)_rec2;
+    *result = (int)(((const H5HF_huge_bt2_filt_indir_rec_t *)_rec1)->id - 
+                    ((const H5HF_huge_bt2_filt_indir_rec_t *)_rec2)->id);
 
-HDfprintf(stderr, "%s: rec1 = {%a, %Hu, %x, %Hu, %Hu}\n", "H5HF_huge_bt2_filt_indir_compare", rec1->addr, rec1->len, rec1->filter_mask, rec1->obj_size, rec1->id);
-HDfprintf(stderr, "%s: rec2 = {%a, %Hu, %x, %Hu, %Hu}\n", "H5HF_huge_bt2_filt_indir_compare", rec2->addr, rec2->len, rec2->filter_mask, rec2->obj_size, rec2->id);
-}
-#endif /* QAK */
-    FUNC_LEAVE_NOAPI((herr_t)(((const H5HF_huge_bt2_filt_indir_rec_t *)_rec1)->id - ((const H5HF_huge_bt2_filt_indir_rec_t *)_rec2)->id))
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5HF_huge_bt2_filt_indir_compare() */
 
 
@@ -815,11 +803,10 @@ H5HF_huge_bt2_dir_store(void *nrecord, const void *udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5HF_huge_bt2_dir_compare(const void *_rec1, const void *_rec2)
+H5HF_huge_bt2_dir_compare(const void *_rec1, const void *_rec2, int *result)
 {
     const H5HF_huge_bt2_dir_rec_t *rec1 = (const H5HF_huge_bt2_dir_rec_t *)_rec1;
     const H5HF_huge_bt2_dir_rec_t *rec2 = (const H5HF_huge_bt2_dir_rec_t *)_rec2;
-    herr_t ret_value;           /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
@@ -828,17 +815,17 @@ HDfprintf(stderr, "%s: rec1 = {%a, %Hu}\n", "H5HF_huge_bt2_dir_compare", rec1->a
 HDfprintf(stderr, "%s: rec2 = {%a, %Hu}\n", "H5HF_huge_bt2_dir_compare", rec2->addr, rec2->len);
 #endif /* QAK */
     if(rec1->addr < rec2->addr)
-        ret_value = -1;
+        *result = -1;
     else if(rec1->addr > rec2->addr)
-        ret_value = 1;
+        *result = 1;
     else if(rec1->len < rec2->len)
-        ret_value = -1;
+        *result = -1;
     else if(rec1->len > rec2->len)
-        ret_value = 1;
+        *result = 1;
     else
-        ret_value = 0;
+        *result = 0;
 
-    FUNC_LEAVE_NOAPI(ret_value)
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5HF_huge_bt2_dir_compare() */
 
 
@@ -1040,11 +1027,10 @@ H5HF_huge_bt2_filt_dir_store(void *nrecord, const void *udata)
  *-------------------------------------------------------------------------
  */
 static herr_t
-H5HF_huge_bt2_filt_dir_compare(const void *_rec1, const void *_rec2)
+H5HF_huge_bt2_filt_dir_compare(const void *_rec1, const void *_rec2, int *result)
 {
     const H5HF_huge_bt2_filt_dir_rec_t *rec1 = (const H5HF_huge_bt2_filt_dir_rec_t *)_rec1;
     const H5HF_huge_bt2_filt_dir_rec_t *rec2 = (const H5HF_huge_bt2_filt_dir_rec_t *)_rec2;
-    herr_t ret_value;           /* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT_NOERR
 
@@ -1053,17 +1039,17 @@ HDfprintf(stderr, "%s: rec1 = {%a, %Hu, %x, %Hu}\n", "H5HF_huge_bt2_filt_dir_com
 HDfprintf(stderr, "%s: rec2 = {%a, %Hu, %x, %Hu}\n", "H5HF_huge_bt2_filt_dir_compare", rec2->addr, rec2->len, rec2->filter_mask, rec2->obj_size);
 #endif /* QAK */
     if(rec1->addr < rec2->addr)
-        ret_value = -1;
+        *result = -1;
     else if(rec1->addr > rec2->addr)
-        ret_value = 1;
+        *result = 1;
     else if(rec1->len < rec2->len)
-        ret_value = -1;
+        *result = -1;
     else if(rec1->len > rec2->len)
-        ret_value = 1;
+        *result = 1;
     else
-        ret_value = 0;
+        *result = 0;
 
-    FUNC_LEAVE_NOAPI(ret_value)
+    FUNC_LEAVE_NOAPI(SUCCEED)
 } /* H5HF_huge_bt2_filt_dir_compare() */
 
 
diff --git a/src/H5SM.c b/src/H5SM.c
index f8c47c1..861853d 100644
--- a/src/H5SM.c
+++ b/src/H5SM.c
@@ -59,7 +59,8 @@ static herr_t H5SM_create_index(H5F_t *f, H5SM_index_header_t *header,
 static herr_t H5SM_delete_index(H5F_t *f, H5SM_index_header_t *header,
                                 hid_t dxpl_id, hbool_t delete_heap);
 static haddr_t H5SM_create_list(H5F_t *f, H5SM_index_header_t *header, hid_t dxpl_id);
-static size_t H5SM_find_in_list(const H5SM_list_t *list, const H5SM_mesg_key_t *key, size_t *empty_pos);
+static herr_t H5SM_find_in_list(const H5SM_list_t *list, const H5SM_mesg_key_t *key, 
+                                size_t *empty_pos, size_t *list_pos);
 static herr_t H5SM_convert_list_to_btree(H5F_t * f, H5SM_index_header_t * header,
                 H5SM_list_t **_list, H5HF_t *fheap, H5O_t *open_oh, hid_t dxpl_id);
 static herr_t H5SM_convert_btree_to_list(H5F_t * f, H5SM_index_header_t * header, hid_t dxpl_id);
@@ -1284,7 +1285,9 @@ H5SM_write_mesg(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
          * Also record the first empty list position we find in case we need it
          * later.
          */
-        list_pos = H5SM_find_in_list(list, &key, &empty_pos);
+        if(H5SM_find_in_list(list, &key, &empty_pos, &list_pos) < 0)
+	    HGOTO_ERROR(H5E_SOHM, H5E_CANTINSERT, FAIL, "unable to search for message in list")
+
         if(defer) {
             if(list_pos != UFAIL)
                 found = TRUE;
@@ -1433,10 +1436,15 @@ H5SM_write_mesg(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
             /* Insert the new message into the SOHM index */
             if(header->index_type == H5SM_LIST) {
                 /* Index is a list.  Find an empty spot if we haven't already */
-                if(empty_pos == UFAIL)
-                    if((H5SM_find_in_list(list, NULL, &empty_pos) == UFAIL) || empty_pos == UFAIL)
-                        HGOTO_ERROR(H5E_SOHM, H5E_CANTINSERT, FAIL, "unable to find empty entry in list")
+                if(empty_pos == UFAIL) {
+                    size_t pos;
+
+                    if(H5SM_find_in_list(list, NULL, &empty_pos, &pos) < 0)
+                        HGOTO_ERROR(H5E_SOHM, H5E_CANTINSERT, FAIL, "unable to search for message in list")
 
+                    if(pos == UFAIL || empty_pos == UFAIL)
+                        HGOTO_ERROR(H5E_SOHM, H5E_CANTINSERT, FAIL, "unable to find empty entry in list")
+                }
                 /* Insert message into list */
                 HDassert(list->messages[empty_pos].location == H5SM_NO_LOC);
                 HDassert(key.message.location != H5SM_NO_LOC);
@@ -1599,13 +1607,13 @@ done:
  *
  *-------------------------------------------------------------------------
  */
-static size_t
-H5SM_find_in_list(const H5SM_list_t *list, const H5SM_mesg_key_t *key, size_t *empty_pos)
+static herr_t
+H5SM_find_in_list(const H5SM_list_t *list, const H5SM_mesg_key_t *key, size_t *empty_pos, size_t *pos)
 {
-    size_t               x;
-    size_t               ret_value;
+    size_t x;
+    herr_t ret_value = SUCCEED;       /* Return value */
 
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
+    FUNC_ENTER_NOAPI_NOINIT
 
     HDassert(list);
     /* Both key and empty_pos can be NULL, but not both! */
@@ -1619,9 +1627,17 @@ H5SM_find_in_list(const H5SM_list_t *list, const H5SM_mesg_key_t *key, size_t *e
      * Also record the first empty position we find.
      */
     for(x = 0; x < list->header->list_max; x++) {
-        if((list->messages[x].location != H5SM_NO_LOC) &&
-                (0 == H5SM_message_compare(key, &(list->messages[x]))))
-            HGOTO_DONE(x)
+        if(list->messages[x].location != H5SM_NO_LOC) {
+            int cmp;
+
+            if(H5SM_message_compare(key, &(list->messages[x]), &cmp) < 0)
+                HGOTO_ERROR(H5E_SOHM, H5E_CANTCOMPARE, FAIL, "can't compare message records")
+
+            if(0 == cmp) {
+                *pos = x;
+                HGOTO_DONE(SUCCEED)
+            }
+        }
         else if(empty_pos && list->messages[x].location == H5SM_NO_LOC) {
             /* Note position */
             *empty_pos = x;
@@ -1632,7 +1648,7 @@ H5SM_find_in_list(const H5SM_list_t *list, const H5SM_mesg_key_t *key, size_t *e
     } /* end for */
 
     /* If we reached this point, we didn't find the message */
-    ret_value = UFAIL;
+    *pos = UFAIL;
 
 done:
     FUNC_LEAVE_NOAPI(ret_value)
@@ -1803,7 +1819,9 @@ H5SM_delete_from_index(H5F_t *f, hid_t dxpl_id, H5O_t *open_oh,
 	    HGOTO_ERROR(H5E_SOHM, H5E_CANTPROTECT, FAIL, "unable to load SOHM index")
 
         /* Find the message in the list */
-        if((list_pos = H5SM_find_in_list(list, &key, NULL)) == UFAIL)
+        if(H5SM_find_in_list(list, &key, NULL, &list_pos) < 0)
+            HGOTO_ERROR(H5E_SOHM, H5E_NOTFOUND, FAIL, "unable to search for message in list")
+        if(list_pos == UFAIL)
 	    HGOTO_ERROR(H5E_SOHM, H5E_NOTFOUND, FAIL, "message not in index")
 
         if(list->messages[list_pos].location == H5SM_IN_HEAP)
@@ -2176,7 +2194,9 @@ H5SM_get_refcount(H5F_t *f, hid_t dxpl_id, unsigned type_id,
 	    HGOTO_ERROR(H5E_SOHM, H5E_CANTPROTECT, FAIL, "unable to load SOHM index")
 
         /* Find the message in the list */
-        if((list_pos = H5SM_find_in_list(list, &key, NULL)) == UFAIL)
+        if(H5SM_find_in_list(list, &key, NULL, &list_pos) < 0)
+            HGOTO_ERROR(H5E_SOHM, H5E_NOTFOUND, FAIL, "unable to search for message in list")
+        if(list_pos == UFAIL)
 	    HGOTO_ERROR(H5E_SOHM, H5E_NOTFOUND, FAIL, "message not in index")
 
         /* Copy the message */
diff --git a/src/H5SMmessage.c b/src/H5SMmessage.c
index 6965fd7..e11722d 100644
--- a/src/H5SMmessage.c
+++ b/src/H5SMmessage.c
@@ -190,13 +190,13 @@ done:
  *-------------------------------------------------------------------------
  */
 herr_t
-H5SM_message_compare(const void *rec1, const void *rec2)
+H5SM_message_compare(const void *rec1, const void *rec2, int *result)
 {
     const H5SM_mesg_key_t *key = (const H5SM_mesg_key_t *) rec1;
     const H5SM_sohm_t *mesg = (const H5SM_sohm_t *) rec2;
-    herr_t ret_value = 0;
+    herr_t ret_value = SUCCEED;
 
-    FUNC_ENTER_NOAPI_NOINIT_NOERR
+    FUNC_ENTER_NOAPI_NOINIT
 
     /* If the key has an fheap ID, we're looking for a message that's
      * already in the index; if the fheap ID matches, we've found the message
@@ -205,28 +205,31 @@ H5SM_message_compare(const void *rec1, const void *rec2)
      * message in the index, we've found the message.
      */
     if(mesg->location == H5SM_IN_HEAP && key->message.location == H5SM_IN_HEAP) {
-        if(key->message.u.heap_loc.fheap_id.val == mesg->u.heap_loc.fheap_id.val)
-            HGOTO_DONE(0);
+        if(key->message.u.heap_loc.fheap_id.val == mesg->u.heap_loc.fheap_id.val) {
+            *result = 0;
+            HGOTO_DONE(SUCCEED);
+        }
     } /* end if */
     else if(mesg->location == H5SM_IN_OH && key->message.location == H5SM_IN_OH) {
         if(key->message.u.mesg_loc.oh_addr == mesg->u.mesg_loc.oh_addr &&
-                key->message.u.mesg_loc.index == mesg->u.mesg_loc.index &&
-                key->message.msg_type_id == mesg->msg_type_id)
-            HGOTO_DONE(0);
+           key->message.u.mesg_loc.index == mesg->u.mesg_loc.index &&
+           key->message.msg_type_id == mesg->msg_type_id) {
+            *result = 0;
+            HGOTO_DONE(SUCCEED);
+        }
     } /* end if */
 
     /* Compare hash values */
     if(key->message.hash > mesg->hash)
-        ret_value = 1;
+        *result = 1;
     else if(key->message.hash < mesg->hash)
-        ret_value = -1;
+        *result = -1;
     /* If the hash values match, make sure the messages are really the same */
     else {
         /* Hash values match; compare the encoded message with the one in
          * the index.
          */
         H5SM_compare_udata_t udata;
-        herr_t status;
 
         HDassert(key->message.hash == mesg->hash);
         HDassert(key->encoding_size > 0 && key->encoding);
@@ -239,8 +242,8 @@ H5SM_message_compare(const void *rec1, const void *rec2)
          */
         if(mesg->location == H5SM_IN_HEAP) {
             /* Call heap op routine with comparison callback */
-            status = H5HF_op(key->fheap, key->dxpl_id, &(mesg->u.heap_loc.fheap_id), H5SM_compare_cb, &udata);
-            HDassert(status >= 0);
+            if(H5HF_op(key->fheap, key->dxpl_id, &(mesg->u.heap_loc.fheap_id), H5SM_compare_cb, &udata) < 0)
+                HGOTO_ERROR(H5E_HEAP, H5E_CANTCOMPARE, FAIL, "can't compare btree2 records")
         } /* end if */
         else {
             H5O_loc_t oloc;             /* Object owning the message */
@@ -251,8 +254,8 @@ H5SM_message_compare(const void *rec1, const void *rec2)
             HDassert(mesg->location == H5SM_IN_OH);
 
             /* Reset the object location */
-            status = H5O_loc_reset(&oloc);
-            HDassert(status >= 0);
+            if(H5O_loc_reset(&oloc) < 0)
+                HGOTO_ERROR(H5E_SYM, H5E_CANTRESET, FAIL, "unable to initialize target location")
 
             /* Set up object location */
             oloc.file = key->file;
@@ -264,11 +267,11 @@ H5SM_message_compare(const void *rec1, const void *rec2)
             /* Locate the right message and compare with it */
             op.op_type = H5O_MESG_OP_LIB;
             op.u.lib_op = H5SM_compare_iter_op;
-            status = H5O_msg_iterate(&oloc, mesg->msg_type_id, &op, &udata, key->dxpl_id);
-            HDassert(status >= 0);
+            if(H5O_msg_iterate(&oloc, mesg->msg_type_id, &op, &udata, key->dxpl_id) < 0)
+                HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, FAIL, "error iterating over links")
         } /* end else */
 
-        ret_value = udata.ret;
+        *result = udata.ret;
     } /* end if */
 
 done:
diff --git a/src/H5SMpkg.h b/src/H5SMpkg.h
index 68dc484..dd31ce6 100644
--- a/src/H5SMpkg.h
+++ b/src/H5SMpkg.h
@@ -268,7 +268,7 @@ H5_DLLVAR const H5B2_class_t H5SM_INDEX[1];
 H5_DLL ssize_t H5SM_get_index(const H5SM_master_table_t *table, unsigned type_id);
 
 /* Encode and decode routines, used for B-tree and cache encoding/decoding */
-H5_DLL herr_t H5SM_message_compare(const void *rec1, const void *rec2);
+H5_DLL herr_t H5SM_message_compare(const void *rec1, const void *rec2, int *result);
 H5_DLL herr_t H5SM_message_encode(uint8_t *raw, const void *native, void *ctx);
 H5_DLL herr_t H5SM_message_decode(const uint8_t *raw, void *native, void *ctx);