[svn-r22073] Port r22072 to 1.8 branch

Purpose: Fix rare corruption bug (HDFFV-7879)

Description:
When using the new object header format, it was possible for corruption to occur
if the first object header chunk changed size such that the lenght of the "chunk
0 size" field changed.  This only occurred if there were messages that had not
been decoded.  The original algorithm that changed the object header chunk size
marked all messages as dirty, causing those that had not been decoded to have
both the raw and native form invalidated.  Changed the algorithm to avoid
marking messages dirty and added assertions to catch the case where messages
are dirtied without being decoded (or recently created) first.

Tested: jam, koala, ostrich (h5committest), durandal

3 files changed, 21 insertions, 12 deletions

diff --git a/src/H5Oalloc.c b/src/H5Oalloc.c
index dfc844d..05322af 100644
--- a/src/H5Oalloc.c
+++ b/src/H5Oalloc.c
@@ -647,17 +647,18 @@ H5O_alloc_extend_chunk(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned chunkno,
     /* Wipe new space for chunk */
     HDmemset(oh->chunk[chunkno].image + old_size, 0, oh->chunk[chunkno].size - old_size);
 
+    /* Move chunk 0 data up if the size flags changed */
+    if(adjust_size_flags)
+        HDmemmove(oh->chunk[0].image + H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh),
+                oh->chunk[0].image + H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh) - extra_prfx_size,
+                old_size - (size_t)H5O_SIZEOF_HDR(oh) + extra_prfx_size);
+
     /* Spin through existing messages, adjusting them */
     for(u = 0; u < oh->nmesgs; u++) {
         /* Adjust raw addresses for messages in this chunk to reflect new 'image' address */
-        if(oh->mesg[u].chunkno == chunkno) {
+        if(oh->mesg[u].chunkno == chunkno)
             oh->mesg[u].raw = oh->chunk[chunkno].image + extra_prfx_size + (oh->mesg[u].raw - old_image);
 
-            /* Flag message as dirty directly */
-            /* (we mark the entire chunk dirty when we update its size) */
-            oh->mesg[u].dirty = TRUE;
-        } /* endif */
-
         /* Find continuation message which points to this chunk and adjust chunk's size */
         /* (Chunk 0 doesn't have a continuation message that points to it and
          * it's size is directly encoded in the object header) */
@@ -1331,8 +1332,9 @@ H5O_move_cont(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned cont_u)
             move_end = cont_msg->raw + cont_msg->raw_size;
             cont_chunkno = cont_msg->chunkno;
 
-            /* Convert continuation message into a null message */
-            if(H5O_release_mesg(f, dxpl_id, oh, cont_msg, TRUE) < 0)
+            /* Convert continuation message into a null message.  Do not delete
+             * the target chunk yet, so we can still copy messages from it. */
+            if(H5O_release_mesg(f, dxpl_id, oh, cont_msg, FALSE) < 0)
                 HGOTO_ERROR(H5E_OHDR, H5E_CANTDELETE, FAIL, "unable to convert into null message")
 
             /* Protect chunk */
@@ -1354,7 +1356,6 @@ H5O_move_cont(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned cont_u)
                         HDmemcpy(move_start, curr_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh), move_size);
                         curr_msg->raw = move_start + H5O_SIZEOF_MSGHDR_OH(oh);
                         curr_msg->chunkno = cont_chunkno;
-                        curr_msg->dirty = TRUE;
                         chk_dirtied = TRUE;
 
                         /* Adjust location to move messages to */
@@ -1362,6 +1363,10 @@ H5O_move_cont(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned cont_u)
                     } /* end else */
                 } /* end if */
 
+            /* Delete the target chunk */
+            if(H5O_chunk_delete(f, dxpl_id, oh, deleted_chunkno) < 0)
+                HGOTO_ERROR(H5E_OHDR, H5E_CANTDELETE, FAIL, "unable to remove chunk from cache")
+
             HDassert(move_start <= (move_end + gap_size));
 
             /* Check if there is space remaining in the continuation message */
diff --git a/src/H5Odbg.c b/src/H5Odbg.c
index ce8b870..5c07b64 100644
--- a/src/H5Odbg.c
+++ b/src/H5Odbg.c
@@ -167,6 +167,8 @@ H5O_assert(const H5O_t *oh)
             H5O_cont_t *cont = (H5O_cont_t *)curr_msg->native;
             hbool_t found_chunk = FALSE;        /* Found a chunk that matches */
 
+            HDassert(cont);
+
             /* Increment # of continuation messages found */
             cont_msgs_found++;
 
@@ -186,6 +188,9 @@ H5O_assert(const H5O_t *oh)
         else {
             meta_space += (size_t)H5O_SIZEOF_MSGHDR_OH(oh);
             mesg_space += curr_msg->raw_size;
+
+            /* Make sure the message has a native form if it is marked dirty */
+            HDassert(curr_msg->native || !curr_msg->dirty);
         } /* end else */
 
         /* Make certain that the message is in a valid chunk */
diff --git a/src/H5Omessage.c b/src/H5Omessage.c
index 13ad73d..8d1c318 100644
--- a/src/H5Omessage.c
+++ b/src/H5Omessage.c
@@ -2171,9 +2171,8 @@ H5O_msg_flush(H5F_t *f, H5O_t *oh, H5O_mesg_t *mesg)
     /* Make certain that null messages aren't in chunks w/gaps */
     if(H5O_NULL_ID == msg_id)
         HDassert(oh->chunk[mesg->chunkno].gap == 0);
-
-    /* Unknown messages should always have a native pointer */
-    if(mesg->type == H5O_MSG_UNKNOWN)
+    else
+        /* Non-null messages should always have a native pointer */
         HDassert(mesg->native);
 #endif /* NDEBUG */