[svn-r16436] Checkin for moving messages forward into continuation message:

	src/H5Oalloc.c, test/ohdr.c, toshm.c
Fixed couple of problems in src/H5Odbg.c.

4 files changed, 251 insertions, 10 deletions

diff --git a/src/H5Oalloc.c b/src/H5Oalloc.c
index 7c6cf9d..6d5d736 100644
--- a/src/H5Oalloc.c
+++ b/src/H5Oalloc.c
@@ -68,7 +68,8 @@ static htri_t H5O_alloc_extend_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh,
     unsigned chunkno, size_t size, unsigned * msg_idx);
 static unsigned H5O_alloc_new_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh,
     size_t size);
-static htri_t H5O_move_msgs_forward(H5O_t *oh);
+static htri_t H5O_move_cont(H5F_t *f, H5O_t *oh, unsigned cont_u, hid_t dxpl_id);
+static htri_t H5O_move_msgs_forward(H5F_t *f, H5O_t *oh, hid_t dxpl_id);
 static htri_t H5O_merge_null(H5F_t *f, H5O_t *oh, hid_t dxpl_id);
 static htri_t H5O_remove_empty_chunks(H5F_t *f, H5O_t *oh, hid_t dxpl_id);
 static herr_t H5O_alloc_shrink_chunk(H5F_t *f, H5O_t *oh, hid_t dxpl_id, unsigned chunkno);
@@ -1091,6 +1092,118 @@ done:
 
 
 /*-------------------------------------------------------------------------
+ * Function:    H5O_move_cont
+ *
+ * Purpose:     Check and move message(s) forward into a continuation message
+ *
+ * Return:      Success:        non-negative (TRUE/FALSE)
+ *              Failure:        negative
+ *
+ * Programmer:  Vailin Choi
+ *		Feb. 2009
+ *
+ *-------------------------------------------------------------------------
+ */
+static htri_t
+H5O_move_cont(H5F_t *f, H5O_t *oh, unsigned cont_u, hid_t dxpl_id)
+{
+    unsigned 	v;		/* local index variable */
+    H5O_mesg_t 	*cont_msg;	/* pointer to the continuation message */
+    H5O_mesg_t 	*nonnull_msg;	/* pointer to the current message to operate on */
+    H5O_mesg_t 	*null_msg;	/* pointer to the current message to operate on */
+    size_t     	total_size=0;	/* total size of nonnull messages in the chunk pointed to by cont message */
+    size_t     	move_size=0;	/* size of the message to be moved */
+    uint8_t    	*move_start, *move_end;	/* pointers to area of messages to move */
+    size_t  	gap_size;		/* size of gap produced */
+    unsigned   	deleted_chunkno;       	/* Chunk # to delete */
+    htri_t 	ret_value = FALSE;      /* Return value */
+
+    FUNC_ENTER_NOAPI_NOINIT(H5O_move_cont)
+
+    /* Check arguments. */
+    HDassert(f);
+    HDassert(oh);
+ 
+    cont_msg = &oh->mesg[cont_u];
+    H5O_LOAD_NATIVE(f, dxpl_id, 0, oh, cont_msg, FAIL)
+    deleted_chunkno = ((H5O_cont_t *)(cont_msg->native))->chunkno;
+
+    /* proceed further only if continuation message is pointing to the last chunk */
+    if(deleted_chunkno != (oh->nchunks - 1))
+	HGOTO_DONE(FALSE)
+
+    /* find size of all nonnull messages in the chunk pointed to by the continuation message */
+    for(v = 0, nonnull_msg = &oh->mesg[0]; v < oh->nmesgs; v++, nonnull_msg++)
+	if(nonnull_msg->chunkno == deleted_chunkno && nonnull_msg->type->id != H5O_NULL_ID) {
+	    HDassert(nonnull_msg->type->id != H5O_CONT_ID);
+	    total_size += nonnull_msg->raw_size + H5O_SIZEOF_MSGHDR_OH(oh);
+	}
+
+    /* check if messages can fit into the continuation message */
+    if(total_size && total_size <= (cont_msg->raw_size + H5O_SIZEOF_MSGHDR_OH(oh))) {
+
+	/* convert continuation message into a null message */
+	if(H5O_release_mesg(f, dxpl_id, oh, cont_msg, TRUE) < 0)
+	    HGOTO_ERROR(H5E_OHDR, H5E_CANTDELETE, FAIL, "unable to convert into null message")
+
+	move_start = cont_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh);
+	move_end = cont_msg->raw + cont_msg->raw_size;
+
+	/* move message(s) forward into continuation message */
+	for(v = 0, nonnull_msg = &oh->mesg[0]; v < oh->nmesgs; v++, nonnull_msg++)
+	    if(nonnull_msg->chunkno == deleted_chunkno && nonnull_msg->type->id != H5O_NULL_ID) {
+		move_size = nonnull_msg->raw_size + H5O_SIZEOF_MSGHDR_OH(oh);
+		HDmemcpy(move_start, nonnull_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh), move_size);
+		nonnull_msg->raw = move_start + H5O_SIZEOF_MSGHDR_OH(oh);
+		nonnull_msg->chunkno = cont_msg->chunkno;
+		nonnull_msg->dirty = TRUE;
+		move_start += move_size;
+	    }
+
+	HDassert(move_start <= move_end);
+
+	/* check if there is space remaining in the continuation message */
+	/* the remaining space can be gap or a null message */
+	gap_size = move_end - move_start;
+	if(gap_size >= (size_t)H5O_SIZEOF_MSGHDR_OH(oh)) {
+	    cont_msg->raw_size = gap_size - H5O_SIZEOF_MSGHDR_OH(oh);
+	    cont_msg->raw = move_start + H5O_SIZEOF_MSGHDR_OH(oh);
+	    cont_msg->dirty = TRUE;
+	} else {
+	    if(gap_size && (H5O_add_gap(oh, cont_msg->chunkno, cont_u, move_start, gap_size) < 0))
+		HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "can't insert gap in chunk")
+	    /* Release any information/memory for continuation message */
+	    H5O_msg_free_mesg(cont_msg);
+	    if(cont_u < (oh->nmesgs - 1))
+		HDmemmove(&oh->mesg[cont_u], &oh->mesg[cont_u + 1], ((oh->nmesgs - 1) - cont_u) * sizeof(H5O_mesg_t));
+	    oh->nmesgs--;
+	}
+
+	/* remove all null messages in deleted chunk from list of messages */
+	/*	 Note: unsigned v wrapping around at the end */
+	for (v = oh->nmesgs - 1, null_msg = &oh->mesg[v]; v < oh->nmesgs; v--, null_msg--)
+	    if(null_msg->type->id == H5O_NULL_ID && null_msg->chunkno == deleted_chunkno) {
+
+                /* Release any information/memory for message */
+                H5O_msg_free_mesg(null_msg);
+
+		if(v < (oh->nmesgs - 1))
+		    HDmemmove(&oh->mesg[v], &oh->mesg[v + 1], ((oh->nmesgs - 1) - v) * sizeof(H5O_mesg_t));
+		oh->nmesgs--;
+	    } /* end if */
+
+	(void)H5FL_BLK_FREE(chunk_image, oh->chunk[deleted_chunkno].image);
+
+	oh->nchunks--;
+	ret_value = TRUE;
+    }  /* end if */
+
+done:
+    FUNC_LEAVE_NOAPI(ret_value)
+} /* H5O_move_cont() */
+
+
+/*-------------------------------------------------------------------------
  *
  * Function:    H5O_move_msgs_forward
  *
@@ -1101,11 +1214,14 @@ done:
  * Programmer:	Quincey Koziol
  *		koziol@ncsa.uiuc.edu
  *		Oct 17 2005
+ * Modifications:
+ *   Feb. 2009: Vailin Choi
+ *      Add changes to move messages forward into "continuation" message
  *
  *-------------------------------------------------------------------------
  */
 static htri_t
-H5O_move_msgs_forward(H5O_t *oh)
+H5O_move_msgs_forward(H5F_t *f, H5O_t *oh, hid_t dxpl_id)
 {
     hbool_t packed_msg;                 /* Flag to indicate that messages were packed */
     hbool_t did_packing = FALSE;        /* Whether any messages were packed */
@@ -1182,7 +1298,17 @@ H5O_move_msgs_forward(H5O_t *oh)
             } /* end if */
             else {
                 H5O_mesg_t *null_msg;       /* Pointer to current message to operate on */
-                unsigned	v;              /* Local index variable */
+                unsigned   v;              /* Local index variable */
+		htri_t	   status;
+
+                if(H5O_CONT_ID == curr_msg->type->id) {
+		    if((status = H5O_move_cont(f, oh, u, dxpl_id)) < 0)
+			HGOTO_ERROR(H5E_OHDR, H5E_CANTDELETE, FAIL, "Error in moving messages into cont message")
+		    else if(status > 0) { /* message(s) got moved into "continuation" message */
+                        packed_msg = TRUE;
+			break;
+		    }
+		}
 
                 /* Loop over messages again, looking for large enough null message in earlier chunk */
                 for(v = 0, null_msg = &oh->mesg[0]; v < oh->nmesgs; v++, null_msg++) {
@@ -1583,6 +1709,11 @@ done:
  *		koziol@ncsa.uiuc.edu
  *		Oct  4 2005
  *
+ * Modifications:
+ *   Feb. 2009: Vailin Choi
+ *      Add 2 more parameters to H5O_move_msgs_forward() for moving
+ *	messages forward into "continuation" message
+ *
  *-------------------------------------------------------------------------
  */
 herr_t
@@ -1603,7 +1734,7 @@ H5O_condense_header(H5F_t *f, H5O_t *oh, hid_t dxpl_id)
         rescan_header = FALSE;
 
         /* Scan for messages that can be moved earlier in chunks */
-        result = H5O_move_msgs_forward(oh);
+        result = H5O_move_msgs_forward(f, oh, dxpl_id);
         if(result < 0)
             HGOTO_ERROR(H5E_OHDR, H5E_CANTPACK, FAIL, "can't move header messages forward")
         if(result > 0)
diff --git a/src/H5Odbg.c b/src/H5Odbg.c
index 8a8b783..cd041e0 100644
--- a/src/H5Odbg.c
+++ b/src/H5Odbg.c
@@ -270,12 +270,16 @@ done:
  *		matzke@llnl.gov
  *		Aug  6 1997
  *
+ * Modifications:
+ *   Feb. 2009: Vailin Choi
+ *	Fixed bug in the accumulation of chunk_total
+ *	Used the appropriate flag when printing creation order tracked/indexed
  *-------------------------------------------------------------------------
  */
 herr_t
 H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, int indent, int fwidth)
 {
-    size_t	mesg_total = 0, chunk_total = 0;
+    size_t	mesg_total = 0, chunk_total = 0, gap_total = 0;
     unsigned	*sequence;
     unsigned	i;              /* Local index variable */
     herr_t	ret_value = SUCCEED;
@@ -311,10 +315,10 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i
         /* Display object's status flags */
 	HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth,
 		   "Attribute creation order tracked:",
-		   (oh->flags & H5P_CRT_ORDER_TRACKED) ? "Yes" : "No");
+		   (oh->flags & H5O_HDR_ATTR_CRT_ORDER_TRACKED) ? "Yes" : "No");
 	HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth,
 		   "Attribute creation order indexed:",
-		   (oh->flags & H5P_CRT_ORDER_INDEXED) ? "Yes" : "No");
+		   (oh->flags & H5O_HDR_ATTR_CRT_ORDER_INDEXED) ? "Yes" : "No");
 	HDfprintf(stream, "%*s%-*s %s\n", indent, "", fwidth,
 		   "Attribute storage phase change values:",
 		   (oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) ? "Non-default" : "Default");
@@ -392,6 +396,7 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i
 
         /* Accumulate chunk's size to total */
 	chunk_total += chunk_size;
+	gap_total += oh->chunk[i].gap;
 
 	HDfprintf(stream, "%*s%-*s %Zu\n", indent + 3, "", MAX(0, fwidth - 3),
 		  "Size in bytes:",
@@ -412,6 +417,10 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i
         /* Accumulate message's size to total */
 	mesg_total += H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[i].raw_size;
 
+	/* For version 2 object header, add size of "OCHK" for continuation chunk */
+	if (oh->mesg[i].type->id == H5O_CONT_ID)
+	    mesg_total += H5O_SIZEOF_CHKHDR_OH(oh);
+
 	HDfprintf(stream, "%*sMessage %d...\n", indent, "", i);
 
 	/* check for bad message id */
@@ -501,7 +510,7 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i
     } /* end for */
     sequence = (unsigned *)H5MM_xfree(sequence);
 
-    if(mesg_total != chunk_total)
+    if((mesg_total + gap_total) != chunk_total)
 	HDfprintf(stream, "*** TOTAL SIZE DOES NOT MATCH ALLOCATED SIZE!\n");
 
 done:
diff --git a/test/ohdr.c b/test/ohdr.c
index 2d4f057..905551d 100644
--- a/test/ohdr.c
+++ b/test/ohdr.c
@@ -43,6 +43,101 @@ const char *FILENAME[] = {
  */
 #define FILE_BOGUS "tbogus.h5"
 
+/*
+ *  Verify that messages are moved forward into a "continuation message":
+ *	Create an object header with several continuation chunks
+ *	Remove a message in the last chunk
+ *	The remaining message(s) in the last chunk should be moved forward into the continuation message
+ *	The process will repeat when the continuation message is big enough to hold all the 
+ *		messages in the last chunk.
+ *	Result: the number of chunks should be reduced
+ */
+static herr_t
+test_cont(char *filename, hid_t fapl)
+{
+    hid_t	file=-1;
+    H5F_t	*f = NULL;
+    H5O_info_t  oinfo;               
+    H5O_loc_t	oh_locA, oh_locB;
+    time_t	time_new;
+    char	*short_name = "T";
+    char	*long_name = "This is the message";
+    size_t	nchunks;
+
+    TESTING("object header continuation block");
+
+    /* Create the file to operate on */
+    if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR
+    if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR
+
+    HDmemset(&oh_locA, 0, sizeof(oh_locA));
+    HDmemset(&oh_locB, 0, sizeof(oh_locB));
+
+    if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)H5O_MIN_SIZE, H5P_GROUP_CREATE_DEFAULT, &oh_locA/*out*/) < 0)
+            FAIL_STACK_ERROR
+
+    if(H5O_create(f, H5P_DATASET_XFER_DEFAULT, (size_t)H5O_MIN_SIZE, H5P_GROUP_CREATE_DEFAULT, &oh_locB/*out*/) < 0)
+            FAIL_STACK_ERROR
+
+    time_new = 11111111;
+
+    if(H5O_msg_create(&oh_locA, H5O_NAME_ID, 0, 0, &long_name, H5P_DATASET_XFER_DEFAULT) < 0)
+	FAIL_STACK_ERROR
+
+    if(H5O_msg_create(&oh_locB, H5O_MTIME_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
+	FAIL_STACK_ERROR
+    if(H5O_msg_create(&oh_locB, H5O_MTIME_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
+	FAIL_STACK_ERROR
+    if(H5O_msg_create(&oh_locB, H5O_MTIME_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
+	FAIL_STACK_ERROR
+
+    if(H5O_msg_create(&oh_locA, H5O_MTIME_NEW_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
+	FAIL_STACK_ERROR
+
+    if(H5O_msg_create(&oh_locB, H5O_MTIME_ID, 0, 0, &time_new, H5P_DATASET_XFER_DEFAULT) < 0)
+            FAIL_STACK_ERROR
+
+    if(H5O_msg_create(&oh_locA, H5O_NAME_ID, 0, 0, &short_name, H5P_DATASET_XFER_DEFAULT) < 0)
+	FAIL_STACK_ERROR
+
+    if(H5AC_flush(f, H5P_DATASET_XFER_DEFAULT, TRUE) < 0)
+	FAIL_STACK_ERROR
+
+    if(H5O_get_info(&oh_locA, H5P_DATASET_XFER_DEFAULT, FALSE, &oinfo) < 0)
+	FAIL_STACK_ERROR
+    nchunks = oinfo.hdr.nchunks;
+
+    /* remove the 1st H5O_NAME_ID message */
+    if(H5O_msg_remove(&oh_locA, H5O_NAME_ID, 0, FALSE, H5P_DATASET_XFER_DEFAULT) < 0)
+	FAIL_STACK_ERROR
+
+    if(H5O_get_info(&oh_locA, H5P_DATASET_XFER_DEFAULT, FALSE, &oinfo) < 0)
+	FAIL_STACK_ERROR
+
+    if (oinfo.hdr.nchunks >= nchunks)
+	TEST_ERROR
+
+    if(H5O_close(&oh_locA) < 0)
+	FAIL_STACK_ERROR
+    if(H5O_close(&oh_locB) < 0)
+	FAIL_STACK_ERROR
+    if(H5Fclose(file) < 0)
+	FAIL_STACK_ERROR
+
+    PASSED();
+
+
+    return 0;
+
+error:
+    H5E_BEGIN_TRY {
+        H5O_close(&oh_locA);
+        H5O_close(&oh_locB);
+        H5Fclose (file);
+    } H5E_END_TRY;
+    return -1;
+} /* test_cont() */
+
 
 /*-------------------------------------------------------------------------
  * Function:	main
@@ -84,12 +179,16 @@ main(void)
         /* Display info about testing */
         if(b)
             HDputs("Using new file format:");
-        else
+        else	
             HDputs("Using default file format:");
 
         /* Set the format to use for the file */
         if (H5Pset_libver_bounds(fapl, (b ? H5F_LIBVER_LATEST : H5F_LIBVER_EARLIEST), H5F_LIBVER_LATEST) < 0) FAIL_STACK_ERROR
 
+	/* test on object continuation block */
+	if (test_cont(filename, fapl) < 0)
+            FAIL_STACK_ERROR
+
         /* Create the file to operate on */
         if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR
         if(NULL == (f = (H5F_t *)H5I_object(file))) FAIL_STACK_ERROR
diff --git a/test/tsohm.c b/test/tsohm.c
index 47c087e..0516590 100644
--- a/test/tsohm.c
+++ b/test/tsohm.c
@@ -2348,8 +2348,10 @@ static void test_sohm_size2(int close_reopen)
     if((list_index_big.attrs1 - list_index_big.interleaved) >=
             (list_index_small.attrs1 - list_index_small.interleaved))
         VERIFY(0, 1, "h5_get_file_size");
+
+    /* Give it some overhead (for checkin to move messages into continuation message) */
     if((list_index_small.attrs1 - list_index_small.interleaved) >
-            (btree_index.attrs1 - btree_index.interleaved))
+            ((btree_index.attrs1 - btree_index.interleaved) * OVERHEAD_ALLOWED))
         VERIFY(0, 1, "h5_get_file_size");