[svn-r17760] Purpose: Fix problem with H5TB API

Description:
The H5TB API makes some improper assumptions about the order of compound
datatype members.  Namely, it assumes that members remain in the order in which
they were inserted.  Unfortunately, this assumption is inherent in the design of
the interface.  The library has been patched so that this assumption holds in
situations relevant to H5TB.

Tested: jam, linew, amani (h5committest)

4 files changed, 85 insertions, 163 deletions

diff --git a/src/H5Odtype.c b/src/H5Odtype.c
index a9812690..fc71d65 100644
--- a/src/H5Odtype.c
+++ b/src/H5Odtype.c
@@ -269,9 +269,9 @@ H5O_dtype_decode_helper(H5F_t *f, unsigned *ioflags/*in,out*/, const uint8_t **p
                  */
                 dt->shared->u.compnd.nmembs = flags & 0xffff;
                 HDassert(dt->shared->u.compnd.nmembs > 0);
-                dt->shared->u.compnd.packed = TRUE; /* Start off packed */
                 dt->shared->u.compnd.nalloc = dt->shared->u.compnd.nmembs;
                 dt->shared->u.compnd.memb = (H5T_cmemb_t *)H5MM_calloc(dt->shared->u.compnd.nalloc * sizeof(H5T_cmemb_t));
+                dt->shared->u.compnd.memb_size = 0;
                 if(NULL == dt->shared->u.compnd.memb)
                     HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed")
                 for(i = 0; i < dt->shared->u.compnd.nmembs; i++) {
@@ -385,6 +385,7 @@ H5O_dtype_decode_helper(H5F_t *f, unsigned *ioflags/*in,out*/, const uint8_t **p
 
                     /* Member size */
                     dt->shared->u.compnd.memb[i].size = temp_type->shared->size;
+                    dt->shared->u.compnd.memb_size += temp_type->shared->size;
 
                     /* Set the field datatype (finally :-) */
                     dt->shared->u.compnd.memb[i].type = temp_type;
@@ -400,31 +401,11 @@ H5O_dtype_decode_helper(H5F_t *f, unsigned *ioflags/*in,out*/, const uint8_t **p
 
                     /* Update the maximum member position covered */
                     max_memb_pos = MAX(max_memb_pos, (dt->shared->u.compnd.memb[i].offset + dt->shared->u.compnd.memb[i].size));
-
-                    /* Check if the datatype stayed packed */
-                    if(dt->shared->u.compnd.packed) {
-                        /* Check if the member type is packed */
-                        if(H5T_is_packed(temp_type) > 0) {
-                            if(i == 0) {
-                                /* If the is the first member, the datatype is not packed
-                                 * if the first member isn't at offset 0
-                                 */
-                                if(dt->shared->u.compnd.memb[i].offset > 0)
-                                    dt->shared->u.compnd.packed = FALSE;
-                            } /* end if */
-                            else {
-                                /* If the is not the first member, the datatype is not
-                                 * packed if the new member isn't adjoining the previous member
-                                 */
-                                if(dt->shared->u.compnd.memb[i].offset != (dt->shared->u.compnd.memb[i - 1].offset + dt->shared->u.compnd.memb[i - 1].size))
-                                    dt->shared->u.compnd.packed = FALSE;
-                            } /* end else */
-                        } /* end if */
-                        else
-                            dt->shared->u.compnd.packed = FALSE;
-                    } /* end if */
                 } /* end for */
 
+                /* Check if the compound type is packed */
+                H5T_update_packed(dt);
+
                 /* Upgrade the compound if requested */
                 if(version < upgrade_to) {
                     version = upgrade_to;
diff --git a/src/H5T.c b/src/H5T.c
index 1e30f77..4041cd5 100644
--- a/src/H5T.c
+++ b/src/H5T.c
@@ -2976,8 +2976,8 @@ H5T_create(H5T_class_t type, size_t size)
             dt->shared->type = type;
 
             if(type==H5T_COMPOUND) {
-                dt->shared->u.compnd.packed=TRUE;       /* Start out packed */
-                dt->shared->u.compnd.sorted=H5T_SORT_VALUE; /* Start out sorted by value */
+                dt->shared->u.compnd.packed=FALSE;       /* Start out unpacked */
+                dt->shared->u.compnd.memb_size=0;
             } /* end if */
             else if(type==H5T_OPAQUE)
                 /* Initialize the tag in case it's not set later.  A null tag will
@@ -3678,7 +3678,13 @@ H5T_set_size(H5T_t *dt, size_t size)
 
                     if(size<(max_offset+max_size))
                         HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "size shrinking will cut off last member ");
+
+                    /* Compound must not have been packed previously */
+                    /* We will check if resizing changed the packed state of
+                     * this type at the end of this function */
+                    HDassert(!dt->shared->u.compnd.packed);
                 }
+
                 break;
 
             case H5T_STRING:
@@ -3754,6 +3760,10 @@ H5T_set_size(H5T_t *dt, size_t size)
                 dt->shared->u.atomic.prec = prec;
             }
         } /* end if */
+
+        /* Check if the new compound type is packed */
+        if(dt->shared->type == H5T_COMPOUND)
+            H5T_update_packed(dt);
     }
 
 done:
diff --git a/src/H5Tcompound.c b/src/H5Tcompound.c
index 02d6bd1..db7a701b 100644
--- a/src/H5Tcompound.c
+++ b/src/H5Tcompound.c
@@ -45,11 +45,6 @@
 /******************/
 /* Local Typedefs */
 /******************/
-/* "Key" (+ user data) for bsearch callback */
-typedef struct{
-    size_t              offset;     /* Offset of member to be added */
-    const H5T_cmemb_t   *max_under; /* Member with maximum offset seen that is not above "offset" */
-} H5T_insert_compar_t;
 
 
 /********************/
@@ -421,48 +416,6 @@ done:
 
 
 /*-------------------------------------------------------------------------
- * Function:	H5T_insert_compar
- *
- * Purpose:	Callback function for bsearch called from H5T_insert.
- *              Reports whether obj has a lower of higher offset than
- *              that stored in key.  Also keeps track of the highest
- *              offset seen that is not higher than that in key.
- *
- * Return:	-1 if key < obj
- *              0 if key == obj
- *              1 if key > obj
- *
- * Programmer:	Neil Fortner
- *		Wednesday, January  7, 1998
- *
- * Modifications:
- *
- *-------------------------------------------------------------------------
- */
-static int
-H5T_insert_compar(const void *_key, const void *_obj)
-{
-    H5T_insert_compar_t *key = *(H5T_insert_compar_t * const *)_key;    /* User data */
-    const H5T_cmemb_t   *memb = (const H5T_cmemb_t *)_obj;              /* Compound member being examined */
-    int                 ret_value;                                      /* Return value */
-
-    FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5T_insert_compar)
-
-    if(key->offset > memb->offset) {
-        if(key->max_under == NULL || memb->offset > key->max_under->offset)
-            key->max_under = memb;
-        ret_value = 1;
-    } /* end if */
-    else if(key->offset < memb->offset)
-        ret_value = -1;
-    else
-        ret_value = 0;  /* Should not happen */
-
-    FUNC_LEAVE_NOAPI(ret_value)
-} /* end H5T_insert_compar() */
-
-
-/*-------------------------------------------------------------------------
  * Function:	H5T_insert
  *
  * Purpose:	Adds a new MEMBER to the compound datatype PARENT.  The new
@@ -482,8 +435,6 @@ H5T_insert(H5T_t *parent, const char *name, size_t offset, const H5T_t *member)
 {
     unsigned	idx;                        /* Index of member to insert */
     size_t	total_size;
-    H5T_insert_compar_t key;                /* Key for bsearch compare function */
-    H5T_insert_compar_t *keyptr = &key;     /* Pointer to key */
     unsigned	i;                          /* Local index variable */
     herr_t      ret_value = SUCCEED;        /* Return value */
 
@@ -500,39 +451,20 @@ H5T_insert(H5T_t *parent, const char *name, size_t offset, const H5T_t *member)
 	if(!HDstrcmp(parent->shared->u.compnd.memb[i].name, name))
 	    HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINSERT, FAIL, "member name is not unique")
 
+    /* Does the new member overlap any existing member ? */
     total_size = member->shared->size;
+    for(i = 0; i < parent->shared->u.compnd.nmembs; i++)
+        if((offset <= parent->shared->u.compnd.memb[i].offset &&
+                 (offset + total_size) > parent->shared->u.compnd.memb[i].offset) ||
+                (parent->shared->u.compnd.memb[i].offset <= offset &&
+                 (parent->shared->u.compnd.memb[i].offset +
+                 parent->shared->u.compnd.memb[i].size) > offset))
+            HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINSERT, FAIL, "member overlaps with another member")
 
     /* Does the new member overlap the end of the compound type? */
     if((offset + total_size) > parent->shared->size)
         HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINSERT, FAIL, "member extends past end of compound type")
 
-    if(parent->shared->u.compnd.sorted != H5T_SORT_VALUE)
-        if(H5T_sort_value(parent, NULL) < 0)
-            HGOTO_ERROR(H5E_INTERNAL, H5E_CANTCOMPARE, FAIL, "value sort failed")
-
-    /* Find the position to insert the new member */
-    if(parent->shared->u.compnd.nmembs == 0)
-        idx = 0;
-    else {
-        /* Key value (including user data) for compar callback */
-        key.offset = offset;
-        key.max_under = NULL;
-
-        /* Do a binary search on the offsets of the (now sorted) members.  We do
-         * not expect to find an exact match (if we do it is an error), rely on
-         * the user data in the key to keep track of the closest member below
-         * the new member. */
-        if(NULL != HDbsearch(&keyptr, parent->shared->u.compnd.memb, parent->shared->u.compnd.nmembs,
-                sizeof(parent->shared->u.compnd.memb[0]), H5T_insert_compar))
-            HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINSERT, FAIL, "member overlaps with another member")
-        idx = (key.max_under == NULL) ? 0 : (unsigned) (key.max_under - parent->shared->u.compnd.memb + 1);
-    } /* end else */
-
-    /* Does the new member overlap any existing member ? */
-    if((idx < parent->shared->u.compnd.nmembs && (offset + total_size) > parent->shared->u.compnd.memb[idx].offset) ||
-            (idx && (parent->shared->u.compnd.memb[idx-1].offset + parent->shared->u.compnd.memb[idx-1].size) > offset))
-        HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINSERT, FAIL, "member overlaps with another member")
-
     /* Increase member array if necessary */
     if(parent->shared->u.compnd.nmembs >= parent->shared->u.compnd.nalloc) {
         unsigned na = MAX(1, parent->shared->u.compnd.nalloc * 2);
@@ -544,68 +476,23 @@ H5T_insert(H5T_t *parent, const char *name, size_t offset, const H5T_t *member)
         parent->shared->u.compnd.memb = x;
     } /* end if */
 
-    /* Determine if the compound datatype stays packed */
-    if(parent->shared->u.compnd.packed) {
-        /* Check if the member type is packed */
-        if(H5T_is_packed(member) > 0) {
-            if(idx == 0) {
-                /* If the is the first member, the datatype is not packed
-                 * if the first member isn't at offset 0
-                 */
-                if(offset > 0)
-                    parent->shared->u.compnd.packed = FALSE;
-            } /* end if */
-            else {
-                /* If the is not the first member, the datatype is not
-                 * packed if the new member isn't adjoining the previous member
-                 */
-                if(offset != (parent->shared->u.compnd.memb[idx - 1].offset + parent->shared->u.compnd.memb[idx - 1].size))
-                    parent->shared->u.compnd.packed = FALSE;
-            } /* end else */
-        } /* end if */
-        else
-            parent->shared->u.compnd.packed = FALSE;
-    } /* end if */
-    else
-        /* Check if inserting this member causes the parent to become packed */
-        /* First check if it completely closes a gap */
-        /* No need to check if it's being appended to the end */
-        if(idx != parent->shared->u.compnd.nmembs
-                && (offset + total_size) == parent->shared->u.compnd.memb[idx].offset
-                && (idx == 0 ? offset == 0  : (parent->shared->u.compnd.memb[idx-1].offset
-                + parent->shared->u.compnd.memb[idx-1].size) == offset)
-                && H5T_is_packed(member) > 0) {
-
-            /* Start out packed */
-            parent->shared->u.compnd.packed = TRUE;
-
-            /* Check if the entire type is now packed */
-            if((idx != 0 && parent->shared->u.compnd.memb[0].offset != 0)
-                    || !H5T_is_packed(parent->shared->u.compnd.memb[0].type))
-                parent->shared->u.compnd.packed = FALSE;
-            else
-                for(i = 1; i < parent->shared->u.compnd.nmembs; i++)
-                    if((i != idx && parent->shared->u.compnd.memb[i].offset
-                            != (parent->shared->u.compnd.memb[i - 1].offset
-                            + parent->shared->u.compnd.memb[i - 1].size))
-                            || !H5T_is_packed(parent->shared->u.compnd.memb[i].type)) {
-                        parent->shared->u.compnd.packed = FALSE;
-                        break;
-                    } /* end if */
-        } /* end if */
-
-    /* Reshape the memb array to accomodate the new member */
-    if(idx != parent->shared->u.compnd.nmembs)
-        HDmemmove(&parent->shared->u.compnd.memb[idx+1], &parent->shared->u.compnd.memb[idx],
-                (parent->shared->u.compnd.nmembs - idx) * sizeof(parent->shared->u.compnd.memb[0]));
-
-    /* Add member to member array */
+    /* Add member to end of member array */
+    idx = parent->shared->u.compnd.nmembs;
     parent->shared->u.compnd.memb[idx].name = H5MM_xstrdup(name);
     parent->shared->u.compnd.memb[idx].offset = offset;
     parent->shared->u.compnd.memb[idx].size = total_size;
     parent->shared->u.compnd.memb[idx].type = H5T_copy(member, H5T_COPY_ALL);
 
+    parent->shared->u.compnd.sorted = H5T_SORT_NONE;
     parent->shared->u.compnd.nmembs++;
+    parent->shared->u.compnd.memb_size+=total_size;
+
+    /* It should not be possible to get this far if the type is already packed
+     * - the new member would overlap something */
+    HDassert(!(parent->shared->u.compnd.packed));
+
+    /* Determine if the compound datatype becomes packed */
+    H5T_update_packed(parent);
 
     /* Set the "force conversion" flag if the field's datatype indicates */
     if(member->shared->force_conv == TRUE)
@@ -730,13 +617,55 @@ H5T_is_packed(const H5T_t *dt)
 
     /* If this is a compound datatype, check if it is packed */
     if(dt->shared->type == H5T_COMPOUND) {
-        H5T_compnd_t *compnd = &(dt->shared->u.compnd); /* Convenience pointer to compound info */
-        ret_value = (htri_t)(compnd->packed && compnd->nmembs > 0
-                && compnd->memb[compnd->nmembs - 1].offset
-                + compnd->memb[compnd->nmembs - 1].size
-                == dt->shared->size);
+        ret_value = (htri_t)(dt->shared->u.compnd.packed);
     } /* end if */
 
     FUNC_LEAVE_NOAPI(ret_value)
 } /* end H5T_is_packed() */
 
+
+/*-------------------------------------------------------------------------
+ * Function:	H5T_update_packed
+ *
+ * Purpose:	Checks whether a datatype which is compound became packed
+ *              after recent changes.  This function does not assume that
+ *              the status of the "packed" field is correct, and sets
+ *              this field to the correct value.
+ *
+ * Return:	void
+ *
+ * Programmer:	Neil Fortner
+ *		Monday, October 19, 2009
+ *
+ * Modifications:
+ *
+ *-------------------------------------------------------------------------
+ */
+void
+H5T_update_packed(const H5T_t *dt)
+{
+    unsigned    i;                      /* Index */
+
+    FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5T_update_packed)
+
+    HDassert(dt);
+    HDassert(dt->shared->type == H5T_COMPOUND);
+
+    /* First check if all space is used in the "top level" type */
+    if(dt->shared->size == dt->shared->u.compnd.memb_size) {
+        /* Set the packed flag to TRUE */
+        dt->shared->u.compnd.packed = TRUE;
+
+        /* Now check if all members are packed */
+        for(i = 0; i < dt->shared->u.compnd.nmembs; i++)
+            if(!H5T_is_packed(dt->shared->u.compnd.memb[i].type)) {
+                dt->shared->u.compnd.packed = FALSE;
+                break;
+            } /* end if */
+    } /* end if */
+    else
+        dt->shared->u.compnd.packed = FALSE;
+
+    FUNC_LEAVE_NOAPI_VOID
+} /* end H5T_update_packed() */
+
diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h
index 18bf7cf..45b0e7f 100644
--- a/src/H5Tpkg.h
+++ b/src/H5Tpkg.h
@@ -307,6 +307,7 @@ typedef struct H5T_compnd_t {
     H5T_sort_t	sorted;		/*how are members sorted?	     */
     hbool_t     packed;		/*are members packed together?       */
     H5T_cmemb_t	*memb;		/*array of struct members	     */
+    size_t      memb_size;	/*total of all member sizes          */
 } H5T_compnd_t;
 
 /* An enumeration datatype */
@@ -1388,6 +1389,7 @@ H5_DLL herr_t H5T_insert(H5T_t *parent, const char *name, size_t offset,
         const H5T_t *member);
 H5_DLL size_t H5T_get_member_size(const H5T_t *dt, unsigned membno);
 H5_DLL htri_t H5T_is_packed(const H5T_t *dt);
+H5_DLL void H5T_update_packed(const H5T_t *dt);
 H5_DLL H5T_subset_info_t *H5T_conv_struct_subset(const H5T_cdata_t *cdata);
 
 /* Enumerated type functions */