[svn-r24263] Changes to do re-reads for metadata with checksums when a file is opened with SWMR access.

There are debugging printfs which will be removed when coding is finalized.
Also some bug fixes:
1) accum.c--clean up some warning messages and use new_argv/new_envp for the call to execve.
2) hl/tools/h5watch--clean up some warning messages and a bug fix for h5watch.c.

This checkin is awaiting code review feedback.

1 files changed, 96 insertions, 51 deletions

diff --git a/src/H5Ocache.c b/src/H5Ocache.c
index 80f5c05..3a087e5 100644
--- a/src/H5Ocache.c
+++ b/src/H5Ocache.c
@@ -92,7 +92,6 @@ static herr_t H5O_chunk_serialize(const H5F_t *f, H5O_t *oh, unsigned chunkno);
 /* Misc. routines */
 static herr_t H5O_add_cont_msg(H5O_cont_msgs_t *cont_msg_info,
     const H5O_cont_t *cont);
-
 static herr_t H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata);
 
 
@@ -315,12 +314,15 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)
     H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata;       /* User data for callback */
     H5WB_t      *wb = NULL;     /* Wrapped buffer for prefix data */
     uint8_t     read_buf[H5O_SPEC_READ_SIZE];       /* Buffer for speculative read */
-    const uint8_t *p;           /* Pointer into buffer to decode */
     uint8_t     *buf;           /* Buffer to decode */
     size_t	spec_read_size; /* Size of buffer to speculatively read in */
     size_t	buf_size;       /* Size of prefix+chunk #0 buffer */
     haddr_t     eoa;		/* Relative end of file address	*/
-    H5O_t	*ret_value;     /* Return value */
+    size_t 	tries, max_tries;	/* The # of read attempts */
+    size_t 	fixed_tries; 		/* The # of read attempts for prefix */
+    uint32_t 	stored_chksum;     	/* Stored metadata checksum value */
+    uint32_t 	computed_chksum;   	/* Computed metadata checksum value */
+    H5O_t	*ret_value;     	/* Return value */
 
     FUNC_ENTER_NOAPI_NOINIT
 
@@ -338,15 +340,89 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)
     /* Compute the size of the speculative object header buffer */
     H5_ASSIGN_OVERFLOW(spec_read_size, MIN(eoa - addr, H5O_SPEC_READ_SIZE), /* From: */ hsize_t, /* To: */ size_t);
 
-    /* Attempt to speculatively read both object header prefix and first chunk */
-    if(H5F_block_read(f, H5FD_MEM_OHDR, addr, spec_read_size, dxpl_id, read_buf) < 0)
-	HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header")
-    p = read_buf;
-
     /* Allocate space for the object header data structure */
     if(NULL == (oh = H5FL_CALLOC(H5O_t)))
 	HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed")
 
+    /* Wrap the local buffer for serialized header info */
+    if(NULL == (wb = H5WB_wrap(read_buf, sizeof(read_buf))))
+	HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't wrap buffer")
+
+    /* Get the # of read attempts */
+    tries = max_tries = H5F_GET_READ_ATTEMPTS(f);
+    do {
+	/* Get a pointer to a buffer that's large enough for serialized header */
+	if(NULL == (buf = (uint8_t *)H5WB_actual(wb, spec_read_size)))
+	    HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "can't get actual buffer")
+
+	fixed_tries = max_tries;
+	do {
+	    /* Attempt to speculatively read both object header prefix and first chunk */
+	    if(H5F_block_read(f, H5FD_MEM_OHDR, addr, spec_read_size, dxpl_id, read_buf) < 0)
+		HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header")
+
+	    /* Decode header prefix */
+	    if(H5O_decode_prefix(f, oh, read_buf, udata) >= 0)
+		break;
+
+	} while (--fixed_tries);
+
+	if(fixed_tries == 0)
+            /* After all tries (for SWMR access) or after 1 try (for non-SWMR) */
+            HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad object header prefix after all tries")
+        else if((max_tries - fixed_tries + 1) > 1)
+            HDfprintf(stderr, "%s: SUCCESS after %u attempts for fixed data\n", FUNC, max_tries - fixed_tries + 1);
+
+	/* Compute the size of the buffer used */
+	buf_size = oh->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh);
+
+	/* Check if the speculative read was large enough to parse the first chunk */
+	if(spec_read_size < buf_size) {
+
+	    /* Get a pointer to a buffer that's large enough for serialized header */
+	    if(NULL == (buf = (uint8_t *)H5WB_actual(wb, buf_size)))
+		HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "can't get actual buffer")
+
+	    /* SWMR access */
+	    if(H5F_INTENT(f) & H5F_ACC_SWMR_READ || H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) {
+		/* Read the chunk that is bigger than the speculative read size */
+		if(H5F_block_read(f, H5FD_MEM_OHDR, addr, buf_size, dxpl_id, buf) < 0)
+		    HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header")
+		/* See if the data read now is the same as what is read initially */
+		if(HDmemcmp(buf, read_buf, spec_read_size)) {
+		    HDmemset(oh, 0, sizeof(oh));
+		    /* Decode the prefix again */
+		    if(H5O_decode_prefix(f, oh, buf, udata) < 0)
+			HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't decode object header prefix")
+		}
+	    } else {
+		/* Copy existing raw data into new buffer */
+		HDmemcpy(buf, read_buf, spec_read_size);
+
+		/* Read rest of the raw data */
+		if(H5F_block_read(f, H5FD_MEM_OHDR, (addr + spec_read_size), (buf_size - spec_read_size), dxpl_id, (buf + spec_read_size)) < 0)
+		    HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data")
+	    }
+	} else
+	    buf = read_buf;
+
+	/* There is no checksum for version 1 */
+	if(oh->version == H5O_VERSION_1)
+	    break;
+	
+	/* For version 2: retrieve the stored checksum and compute the checksum */
+	H5F_get_checksums(buf, buf_size, &stored_chksum, &computed_chksum);
+
+        /* Verify checksums */
+        if(stored_chksum == computed_chksum)
+	    break;
+    } while (--tries);
+
+    if(tries == 0)
+       HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "incorrect metadata checksum for object header chunk after all read tries (%u) for %u bytes", max_tries, buf_size)
+    else if((max_tries - tries + 1) > 1)
+        HDfprintf(stderr, "%s: SUCCESS after %u attempts\n", FUNC, max_tries - tries + 1);
+
     /* File-specific, non-stored information */
     oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f);
     oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f);
@@ -355,49 +431,11 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)
     /* Create object header proxy if doing SWMR writes */
     HDassert(!oh->proxy_present);
     if(H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) {
-        if(H5O_proxy_create(f, dxpl_id, oh) < 0)
-            HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, NULL, "can't create object header proxy")
+	if(H5O_proxy_create(f, dxpl_id, oh) < 0)
+	    HGOTO_ERROR(H5E_OHDR, H5E_CANTCREATE, NULL, "can't create object header proxy")
     } /* end if */
     else
-        oh->proxy_addr = HADDR_UNDEF;
-
-    if(H5O_decode_prefix(f, oh, read_buf, udata) < 0)
-        HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't decode object header prefix")
-
-    /* Compute the size of the buffer used */
-    buf_size = oh->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh);
-
-    /* Check if the speculative read was large enough to parse the first chunk */
-    if(spec_read_size < buf_size) {
-        /* Wrap the local buffer for serialized header info */
-        if(NULL == (wb = H5WB_wrap(read_buf, sizeof(read_buf))))
-            HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't wrap buffer")
-
-        /* Get a pointer to a buffer that's large enough for serialized header */
-        if(NULL == (buf = (uint8_t *)H5WB_actual(wb, buf_size)))
-            HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "can't get actual buffer")
-
-	/* SWMR access */
-        if(H5F_INTENT(f) & H5F_ACC_SWMR_READ || H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) {
-            /* Read the chunk that is bigger than the speculative read size */
-            if(H5F_block_read(f, H5FD_MEM_OHDR, addr, buf_size, dxpl_id, buf) < 0)
-                HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header")
-            /* See if the data read now is the same as what is read initially */
-            if(HDmemcmp(buf, read_buf, spec_read_size)) {
-		HDmemset(oh, 0, sizeof(oh));
-                if(H5O_decode_prefix(f, oh, buf, udata) < 0)
-                    HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't decode object header prefix")
-            }
-        } else {
-            /* Copy existing raw data into new buffer */
-            HDmemcpy(buf, read_buf, spec_read_size);
-
-            /* Read rest of the raw data */
-            if(H5F_block_read(f, H5FD_MEM_OHDR, (addr + spec_read_size), (buf_size - spec_read_size), dxpl_id, (buf + spec_read_size)) < 0)
-                HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data")
-	}
-    } else
-        buf = read_buf;
+	oh->proxy_addr = HADDR_UNDEF;
 
     /* Parse the first chunk */
     if(H5O_chunk_deserialize(oh, udata->common.addr, oh->chunk0_size, buf, &(udata->common), &oh->cache_info.is_dirty) < 0)
@@ -778,8 +816,15 @@ H5O_cache_chk_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)
         HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "can't get actual buffer")
 
     /* Read rest of the raw data */
-    if(H5F_block_read(f, H5FD_MEM_OHDR, addr, udata->size, dxpl_id, buf) < 0)
-        HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header continuation chunk")
+    if(udata->oh->version == H5O_VERSION_2 && udata->decoding) {
+	/* Read and validate object header continuation chunk */
+	if(H5F_read_check_metadata(f, H5FD_MEM_OHDR, addr, udata->size, udata->size, dxpl_id, buf, NULL) < 0)
+	    HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "incorrect metadata checksum for object header continuation chunk")
+    } else {
+	/* Read the header object continuation chunk */
+	if(H5F_block_read(f, H5FD_MEM_OHDR, addr, udata->size, dxpl_id, buf) < 0)
+	    HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header continuation chunk")
+    }
 
     /* Check if we are still decoding the object header */
     /* (as opposed to bringing a piece of it back from the file) */