[svn-r26556] Merge the fix for HDFFV-9173 from trunk to 1_8:

H5Pset_istore_k() will validate the "ik" value to not exceed the max v1 btree entries (2 bytes)
The same check for H5Pset_sym_k()  "ik" value.
h5committested.

3 files changed, 20 insertions, 0 deletions

diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h
index 80266fd..d8014c1 100644
--- a/src/H5Fprivate.h
+++ b/src/H5Fprivate.h
@@ -428,6 +428,8 @@
                                                     if it is changed, the code
                                                     must compensate. -QAK
                                                  */
+#define HDF5_BTREE_IK_MAX_ENTRIES       65536 	/* 2^16 - 2 bytes for storing entries (children) */
+						/* See format specification on version 1 B-trees */
 
 /* Macros to define signatures of all objects in the file */
 
diff --git a/src/H5Pfcpl.c b/src/H5Pfcpl.c
index d114650..a054471 100644
--- a/src/H5Pfcpl.c
+++ b/src/H5Pfcpl.c
@@ -496,6 +496,9 @@ H5Pset_sym_k(hid_t plist_id, unsigned ik, unsigned lk)
 
     /* Set values */
     if (ik > 0) {
+	if((ik * 2) >= HDF5_BTREE_IK_MAX_ENTRIES)
+	    HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "istore IK value exceeds maximum B-tree entries");
+
         if(H5P_get(plist, H5F_CRT_BTREE_RANK_NAME, btree_k) < 0)
             HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get rank for btree interanl nodes");
         btree_k[H5B_SNODE_ID] = ik;
@@ -593,6 +596,9 @@ H5Pset_istore_k(hid_t plist_id, unsigned ik)
     if (ik == 0)
         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "istore IK value must be positive");
 
+    if((ik * 2) >= HDF5_BTREE_IK_MAX_ENTRIES)
+        HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "istore IK value exceeds maximum B-tree entries");
+
     /* Get the plist structure */
     if(NULL == (plist = H5P_object_verify(plist_id,H5P_FILE_CREATE)))
         HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID");
diff --git a/test/tmisc.c b/test/tmisc.c
index e4322a2..45e85cc 100644
--- a/test/tmisc.c
+++ b/test/tmisc.c
@@ -1865,9 +1865,21 @@ test_misc11(void)
     ret = H5Pset_sizes(fcpl, (size_t)MISC11_SIZEOF_OFF, (size_t)MISC11_SIZEOF_LEN);
     CHECK(ret, FAIL, "H5Pset_sizes");
 
+    /* This should fail as (32770*2) will exceed ^16 - 2 bytes for storing btree entries */
+    H5E_BEGIN_TRY {
+	ret=H5Pset_sym_k(fcpl, 32770, 0);
+    } H5E_END_TRY;
+    VERIFY(ret, FAIL, "H5Pset_sym_k");
+
     ret=H5Pset_sym_k(fcpl,MISC11_SYM_IK,MISC11_SYM_LK);
     CHECK(ret, FAIL, "H5Pset_sym_k");
 
+    /* This should fail as (32770*2) will exceed ^16 - 2 bytes for storing btree entries */
+    H5E_BEGIN_TRY {
+	ret=H5Pset_istore_k(fcpl, 32770);
+    } H5E_END_TRY;
+    VERIFY(ret, FAIL, "H5Pset_istore_k");
+
     ret=H5Pset_istore_k(fcpl,MISC11_ISTORE_IK);
     CHECK(ret, FAIL, "H5Pset_istore_k");