From f7a8a14bfd9d2c103c5d9be8adb578bec4a1b015 Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Mon, 16 Oct 2006 18:04:45 -0500 Subject: [svn-r12770] Description: Overhaul usage of object header chunks to reduce I/O operations and memory allocations. The object header prefix is now stored in the first object header chunk's "image" in memory. Also, lots of formatting cleanups. Taught h5debug tool about new object header format (which isn't enabled just yet). Tested on: Linux/32 2.6 (chicago) Linux/64 2.6 (chicago2) --- src/H5O.c | 178 +++++++++++-------- src/H5Ocache.c | 488 +++++++++++++++++++++++---------------------------- src/H5Opkg.h | 67 ++++--- tools/misc/h5debug.c | 9 +- 4 files changed, 367 insertions(+), 375 deletions(-) diff --git a/src/H5O.c b/src/H5O.c index 39548fb..80b963f 100644 --- a/src/H5O.c +++ b/src/H5O.c @@ -130,7 +130,7 @@ typedef struct H5O_addr_map_t { static hid_t H5O_open_by_loc(H5G_loc_t *obj_loc, hid_t dxpl_id); static H5O_loc_t * H5O_get_oloc(hid_t id); -static herr_t H5O_new(H5F_t *f, hid_t dxpl_id, size_t size_hint, +static herr_t H5O_new(H5F_t *f, hid_t dxpl_id, size_t chunk_size, H5O_loc_t *loc/*out*/, haddr_t header); static herr_t H5O_reset_real(const H5O_msg_class_t *type, void *native); static void * H5O_copy_real(const H5O_msg_class_t *type, const void *mesg, @@ -713,7 +713,7 @@ done: * write access and should eventually be closed by calling * H5O_close(). * - * Return: Success: SUCCEED, the ENT argument contains + * Return: Success: SUCCEED, the LOC argument contains * information about the object header, * including its address. * Failure: FAIL @@ -724,10 +724,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5O_new(H5F_t *f, hid_t dxpl_id, size_t size_hint, H5O_loc_t *loc/*out*/, +H5O_new(H5F_t *f, hid_t dxpl_id, size_t chunk_size, H5O_loc_t *loc/*out*/, haddr_t header) { H5O_t *oh = NULL; + size_t oh_size; /* Size of initial object header */ herr_t ret_value = SUCCEED; /* return value */ FUNC_ENTER_NOAPI_NOINIT(H5O_new) @@ -736,49 +737,61 @@ H5O_new(H5F_t *f, hid_t dxpl_id, size_t size_hint, H5O_loc_t *loc/*out*/, HDassert(f); HDassert(loc); + /* Set up object location */ loc->file = f; loc->addr = header; - /* allocate the object header and fill in header fields */ + /* Allocate the object header and fill in header fields */ if(NULL == (oh = H5FL_MALLOC(H5O_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + /* Initialize rudimentary information object object header */ oh->version = H5O_VERSION_1; oh->nlink = 0; - /* create the chunk list and initialize the first chunk */ + /* Compute total size of initial object header */ + /* (i.e. object header prefix and first chunk) */ + oh_size = H5O_SIZEOF_HDR_OH(oh) + chunk_size; + + /* Create the chunk list */ oh->nchunks = 1; oh->alloc_nchunks = H5O_NCHUNKS; - if(NULL == (oh->chunk = H5FL_SEQ_MALLOC(H5O_chunk_t, (size_t)oh->alloc_nchunks))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + /* Initialize the first chunk */ oh->chunk[0].dirty = TRUE; - oh->chunk[0].addr = loc->addr + (hsize_t)H5O_SIZEOF_HDR_OH(oh); - oh->chunk[0].size = size_hint; + oh->chunk[0].addr = loc->addr; + oh->chunk[0].size = oh_size; - if(NULL == (oh->chunk[0].image = H5FL_BLK_CALLOC(chunk_image, size_hint))) + /* Allocate enough space for the first chunk */ + /* (including space for serializing the object header prefix */ + if(NULL == (oh->chunk[0].image = H5FL_BLK_CALLOC(chunk_image, oh_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - /* create the message list and initialize the first message */ + /* Put magic # for object header in first chunk */ + if(oh->version > H5O_VERSION_1) + HDmemcpy(oh->chunk[0].image, H5O_HDR_MAGIC, (size_t)H5O_SIZEOF_MAGIC); + + /* Create the message list */ oh->nmesgs = 1; oh->alloc_nmesgs = H5O_NMESGS; - - if(NULL == (oh->mesg = H5FL_SEQ_CALLOC(H5O_mesg_t, (size_t)oh->alloc_nmesgs))) + if(NULL == (oh->mesg = H5FL_SEQ_CALLOC(H5O_mesg_t, oh->alloc_nmesgs))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + /* Initialize the first message */ oh->mesg[0].type = H5O_MSG_NULL; oh->mesg[0].dirty = TRUE; oh->mesg[0].native = NULL; - oh->mesg[0].raw = oh->chunk[0].image + H5O_SIZEOF_MSGHDR_OH(oh); - oh->mesg[0].raw_size = size_hint - H5O_SIZEOF_MSGHDR_OH(oh); + oh->mesg[0].raw = oh->chunk[0].image + (H5O_SIZEOF_HDR_OH(oh) - H5O_SIZEOF_CHKSUM_OH(oh)) + H5O_SIZEOF_MSGHDR_OH(oh); + oh->mesg[0].raw_size = chunk_size - H5O_SIZEOF_MSGHDR_OH(oh); oh->mesg[0].chunkno = 0; - /* cache it */ + /* Cache object header */ if(H5AC_set(f, dxpl_id, H5AC_OHDR, loc->addr, oh, H5AC__NO_FLAGS_SET) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to cache object header") - /* open it */ + /* Open it */ if(H5O_open(loc) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTOPENOBJ, FAIL, "unable to open object header") @@ -1819,9 +1832,11 @@ H5O_protect(H5O_loc_t *loc, hid_t dxpl_id) HDassert(loc->file); HDassert(H5F_addr_defined(loc->addr)); + /* Check for write access on the file */ if(0 == (loc->file->intent & H5F_ACC_RDWR)) HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, NULL, "no write intent on file") + /* Lock the object header into the cache */ if(NULL == (ret_value = H5AC_protect(loc->file, dxpl_id, H5AC_OHDR, loc->addr, NULL, NULL, H5AC_WRITE))) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "unable to load object header") @@ -1860,6 +1875,7 @@ H5O_unprotect(H5O_loc_t *loc, H5O_t *oh, hid_t dxpl_id, unsigned oh_flags) HDassert(H5F_addr_defined(loc->addr)); HDassert(oh); + /* Release the object header from the cache */ if(H5AC_unprotect(loc->file, dxpl_id, H5AC_OHDR, loc->addr, oh, oh_flags) < 0) HGOTO_ERROR(H5E_OHDR, H5E_PROTECT, FAIL, "unable to release object header") @@ -2153,7 +2169,7 @@ H5O_touch_oh(H5F_t *f, } /* end if */ /* Update the message */ - *((time_t*)(oh->mesg[idx].native)) = now; + *((time_t *)(oh->mesg[idx].native)) = now; /* Mark the message & object header as dirty */ oh->mesg[idx].dirty = TRUE; @@ -2581,7 +2597,7 @@ H5O_alloc_msgs(H5O_t *oh, size_t min_alloc) /* Initialize number of messages information */ old_alloc = oh->alloc_nmesgs; - na = oh->alloc_nmesgs + MAX (H5O_NMESGS, min_alloc); + na = oh->alloc_nmesgs + MAX(oh->alloc_nmesgs, min_alloc); /* Attempt to allocate more memory */ if(NULL == (new_mesg = H5FL_SEQ_REALLOC(H5O_mesg_t, oh->mesg, na))) @@ -2626,8 +2642,8 @@ H5O_move_msgs_forward(H5F_t *f, H5O_t *oh, hid_t dxpl_id) HDassert(oh); /* Loop until no messages packed */ - /* (Double loop is not very efficient, but it would be some extra work to add - * a list of messages to each chunk -QAK) + /* (Double loop is not very efficient, but it would be some extra work to + * add a list of messages to each chunk -QAK) */ do { H5O_mesg_t *curr_msg; /* Pointer to current message to operate on */ @@ -2643,7 +2659,7 @@ H5O_move_msgs_forward(H5F_t *f, H5O_t *oh, hid_t dxpl_id) /* Check if null message is not last in chunk */ chunk = &(oh->chunk[curr_msg->chunkno]); - if((curr_msg->raw + curr_msg->raw_size) != (chunk->image + chunk->size)) { + if((curr_msg->raw + curr_msg->raw_size + H5O_SIZEOF_CHKSUM_OH(oh)) != (chunk->image + chunk->size)) { H5O_mesg_t *nonnull_msg; /* Pointer to current message to operate on */ unsigned v; /* Local index variable */ @@ -2939,7 +2955,7 @@ H5O_remove_empty_chunks(H5F_t *f, H5O_t *oh, hid_t dxpl_id) * its not the "base" chunk), delete that chunk from object header */ if(H5O_NULL_ID == null_msg->type->id && null_msg->chunkno > 0 && - (H5O_SIZEOF_MSGHDR_OH(oh) + null_msg->raw_size) == oh->chunk[null_msg->chunkno].size) { + (H5O_SIZEOF_MSGHDR_OH(oh) + null_msg->raw_size + H5O_SIZEOF_CHKHDR_OH(oh)) == oh->chunk[null_msg->chunkno].size) { H5O_mesg_t *curr_msg; /* Pointer to current message to operate on */ unsigned null_msg_no; /* Message # for null message */ unsigned deleted_chunkno; /* Chunk # to delete */ @@ -3152,17 +3168,16 @@ H5O_alloc_extend_chunk(H5F_t *f, HDassert(chunkno < oh->nchunks); HDassert(size > 0); HDassert(msg_idx != NULL); - - if(!H5F_addr_defined(oh->chunk[chunkno].addr)) - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, FAIL, "chunk isn't on disk") + HDassert(H5F_addr_defined(oh->chunk[chunkno].addr)); /* Test to see if the specified chunk ends with a null messages. * If successful, set the index of the the null message in extend_msg. */ for(u = 0; u < oh->nmesgs; u++) { /* Check for null message at end of proper chunk */ + /* (account for possible checksum at end of chunk) */ if(oh->mesg[u].chunkno == chunkno && H5O_NULL_ID == oh->mesg[u].type->id && - (oh->mesg[u].raw + oh->mesg[u].raw_size == oh->chunk[chunkno].image + oh->chunk[chunkno].size)) { + ((oh->mesg[u].raw + oh->mesg[u].raw_size + H5O_SIZEOF_CHKSUM_OH(oh)) == oh->chunk[chunkno].image + oh->chunk[chunkno].size)) { extend_msg = u; break; @@ -3370,6 +3385,13 @@ H5O_alloc_new_chunk(H5F_t *f, } /* end if */ /* + * The total chunk size must include enough space for the checksum + * on the chunk and the continuation chunk magic #. (which are only present + * in later versions of the object header) + */ + size += H5O_SIZEOF_CHKHDR_OH(oh); + + /* * The total chunk size must include the requested space plus enough * for the message header. This must be at least some minimum and a * multiple of the alignment size. @@ -3386,8 +3408,8 @@ H5O_alloc_new_chunk(H5F_t *f, * Create the new chunk giving it a file address. */ if(oh->nchunks >= oh->alloc_nchunks) { - unsigned na = oh->alloc_nchunks + H5O_NCHUNKS; - H5O_chunk_t *x = H5FL_SEQ_REALLOC (H5O_chunk_t, oh->chunk, (size_t)na); + unsigned na = MAX(H5O_NCHUNKS, oh->alloc_nchunks * 2); /* Double # of chunks allocated */ + H5O_chunk_t *x = H5FL_SEQ_REALLOC(H5O_chunk_t, oh->chunk, (size_t)na); if(!x) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, UFAIL, "memory allocation failed") @@ -3402,6 +3424,14 @@ H5O_alloc_new_chunk(H5F_t *f, if(NULL == (oh->chunk[chunkno].image = p = H5FL_BLK_CALLOC(chunk_image, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, UFAIL, "memory allocation failed") + /* If this is a later version of the object header format, put the magic + * # at the beginning of the chunk + */ + if(oh->version > H5O_VERSION_1) { + HDmemcpy(p, H5O_CHK_MAGIC, (size_t)H5O_SIZEOF_MAGIC); + p += H5O_SIZEOF_MAGIC; + } /* end if */ + /* * Make sure we have enough space for all possible new messages * that could be generated below. @@ -3410,10 +3440,11 @@ H5O_alloc_new_chunk(H5F_t *f, if(H5O_alloc_msgs(oh, (size_t)3) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, UFAIL, "can't allocate more space for messages") - /* - * Describe the messages of the new chunk. + /* Move message (that will be replaced with continuation message) + * to new chunk, if necessary. */ if(found_null < 0) { + /* Create null message for space that message to copy currently occupies */ found_null = u = oh->nmesgs++; oh->mesg[u].type = H5O_MSG_NULL; oh->mesg[u].dirty = TRUE; @@ -3422,22 +3453,28 @@ H5O_alloc_new_chunk(H5F_t *f, oh->mesg[u].raw_size = oh->mesg[found_other].raw_size; oh->mesg[u].chunkno = oh->mesg[found_other].chunkno; - oh->mesg[found_other].dirty = TRUE; /* Copy the message to the new location */ HDmemcpy(p + H5O_SIZEOF_MSGHDR_OH(oh), oh->mesg[found_other].raw, oh->mesg[found_other].raw_size); oh->mesg[found_other].raw = p + H5O_SIZEOF_MSGHDR_OH(oh); oh->mesg[found_other].chunkno = chunkno; + oh->mesg[found_other].dirty = TRUE; + + /* Account for copied message in new chunk */ p += H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_other].raw_size; size -= H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_other].raw_size; } /* end if */ + + /* Create null message for [rest of] space in new chunk */ + /* (except for chunk's magic # & checksum) */ idx = oh->nmesgs++; oh->mesg[idx].type = H5O_MSG_NULL; oh->mesg[idx].dirty = TRUE; oh->mesg[idx].native = NULL; oh->mesg[idx].raw = p + H5O_SIZEOF_MSGHDR_OH(oh); - oh->mesg[idx].raw_size = size - H5O_SIZEOF_MSGHDR_OH(oh); + oh->mesg[idx].raw_size = size - + (H5O_SIZEOF_CHKHDR_OH(oh) + H5O_SIZEOF_MSGHDR_OH(oh)); oh->mesg[idx].chunkno = chunkno; /* @@ -3460,9 +3497,7 @@ H5O_alloc_new_chunk(H5F_t *f, oh->mesg[found_null].raw_size = cont_size; } /* end if */ - /* - * Initialize the continuation message. - */ + /* Initialize the continuation message */ oh->mesg[found_null].type = H5O_MSG_CONT; oh->mesg[found_null].dirty = TRUE; if(NULL == (cont = H5FL_MALLOC(H5O_cont_t))) @@ -3815,7 +3850,7 @@ H5O_delete_oh(H5F_t *f, hid_t dxpl_id, H5O_t *oh) } /* end for */ /* Free main (first) object header "chunk" */ - if(H5MF_xfree(f, H5FD_MEM_OHDR, dxpl_id, (oh->chunk[0].addr - H5O_SIZEOF_HDR_OH(oh)), (hsize_t)(oh->chunk[0].size + H5O_SIZEOF_HDR_OH(oh))) < 0) + if(H5MF_xfree(f, H5FD_MEM_OHDR, dxpl_id, oh->chunk[0].addr, (hsize_t)oh->chunk[0].size) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to free object header") done: @@ -3919,7 +3954,7 @@ H5O_get_info(H5O_loc_t *loc, H5O_stat_t *ostat, hid_t dxpl_id) /* Check for this message being free space */ if(H5O_NULL_ID == curr_msg->type->id) - free_space+= H5O_SIZEOF_MSGHDR_OH(oh) + curr_msg->raw_size; + free_space += H5O_SIZEOF_MSGHDR_OH(oh) + curr_msg->raw_size; } /* end for */ /* Set the information for this object header */ @@ -4581,24 +4616,11 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, for(chunkno = 0; chunkno < oh_src->nchunks; chunkno++) { size_t chunk_size = oh_src->chunk[chunkno].size; - /* '0th' chunk is preceded by object header prefix */ - if(0 == chunkno) { - size_t hdr_size; - - /* get the size of the file header of the destination file */ - hdr_size = H5O_SIZEOF_HDR_OH(oh_dst); - - /* Allocate file space for the first chunk & object header prefix */ - if(HADDR_UNDEF == (addr_new = H5MF_alloc(oloc_dst->file, H5FD_MEM_OHDR, dxpl_id, (hsize_t)hdr_size + chunk_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for object header") - - /* Set first chunk's address */ - oh_dst->chunk[0].addr = addr_new + (hsize_t)hdr_size; - } /* end if */ - else { - if(HADDR_UNDEF == (oh_dst->chunk[chunkno].addr = H5MF_alloc(oloc_dst->file, H5FD_MEM_OHDR, dxpl_id, (hsize_t)chunk_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for object header") - } /* end else */ + /* Allocate space for chunk in destination file */ + if(HADDR_UNDEF == (oh_dst->chunk[chunkno].addr = H5MF_alloc(oloc_dst->file, H5FD_MEM_OHDR, dxpl_id, (hsize_t)chunk_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for object header") + if(chunkno == 0) + addr_new = oh_dst->chunk[chunkno].addr; /* Create memory image for the new chunk */ if(NULL == (oh_dst->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, chunk_size))) @@ -5234,7 +5256,6 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i unsigned i, chunkno; size_t mesg_total = 0, chunk_total = 0; int *sequence; - haddr_t tmp_addr; void *(*decode)(H5F_t*, hid_t, const uint8_t*); herr_t (*debug)(H5F_t*, hid_t, const void*, FILE*, int, int)=NULL; herr_t ret_value = SUCCEED; @@ -5252,15 +5273,15 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i /* debug */ HDfprintf(stream, "%*sObject Header...\n", indent, ""); - HDfprintf(stream, "%*s%-*s %d\n", indent, "", fwidth, + HDfprintf(stream, "%*s%-*s %t\n", indent, "", fwidth, "Dirty:", - (int) (oh->cache_info.is_dirty)); + oh->cache_info.is_dirty); HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, "Version:", oh->version); HDfprintf(stream, "%*s%-*s %u\n", indent, "", fwidth, "Header size (in bytes):", - (unsigned) H5O_SIZEOF_HDR_OH(oh)); + (unsigned)H5O_SIZEOF_HDR_OH(oh)); HDfprintf(stream, "%*s%-*s %d\n", indent, "", fwidth, "Number of links:", oh->nlink); @@ -5273,22 +5294,29 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i /* debug each chunk */ for(i = 0, chunk_total = 0; i < oh->nchunks; i++) { + size_t chunk_size; + chunk_total += oh->chunk[i].size; HDfprintf(stream, "%*sChunk %d...\n", indent, "", i); - HDfprintf(stream, "%*s%-*s %d\n", indent + 3, "", MAX(0, fwidth - 3), + HDfprintf(stream, "%*s%-*s %t\n", indent + 3, "", MAX(0, fwidth - 3), "Dirty:", - (int) (oh->chunk[i].dirty)); + oh->chunk[i].dirty); HDfprintf(stream, "%*s%-*s %a\n", indent + 3, "", MAX(0, fwidth - 3), - "Address:", oh->chunk[i].addr); + "Address:", + oh->chunk[i].addr); - tmp_addr = addr + (hsize_t)H5O_SIZEOF_HDR_OH(oh); - if(0 == i && H5F_addr_ne(oh->chunk[i].addr, tmp_addr)) - HDfprintf(stream, "*** WRONG ADDRESS!\n"); - HDfprintf(stream, "%*s%-*s %lu\n", indent + 3, "", MAX(0, fwidth - 3), + if(0 == i) { + if(H5F_addr_ne(oh->chunk[i].addr, addr)) + HDfprintf(stream, "*** WRONG ADDRESS!\n"); + chunk_size = oh->chunk[i].size - H5O_SIZEOF_HDR_OH(oh); + } /* end if */ + else + chunk_size = oh->chunk[i].size; + HDfprintf(stream, "%*s%-*s %Zu\n", indent + 3, "", MAX(0, fwidth - 3), "Size in bytes:", - (unsigned long) (oh->chunk[i].size)); + chunk_size); } /* end for */ /* debug each message */ @@ -5312,9 +5340,9 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i (unsigned) (oh->mesg[i].type->id), oh->mesg[i].type->name, sequence[oh->mesg[i].type->id]++); - HDfprintf (stream, "%*s%-*s %d\n", indent+3, "", MAX (0, fwidth-3), + HDfprintf (stream, "%*s%-*s %t\n", indent+3, "", MAX (0, fwidth-3), "Dirty:", - (int)(oh->mesg[i].dirty)); + oh->mesg[i].dirty); HDfprintf (stream, "%*s%-*s %s\n", indent+3, "", MAX (0, fwidth-3), "Shared:", (oh->mesg[i].flags & H5O_FLAG_SHARED) ? "Yes" : "No"); @@ -5326,18 +5354,18 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i "*** ADDITIONAL UNKNOWN FLAGS --->", oh->mesg[i].flags & ~H5O_FLAG_BITS); } /* end if */ - HDfprintf(stream, "%*s%-*s %lu bytes\n", indent+3, "", MAX(0,fwidth-3), + HDfprintf(stream, "%*s%-*s %Zu bytes\n", indent+3, "", MAX(0,fwidth-3), "Raw size in obj header:", - (unsigned long) (oh->mesg[i].raw_size)); - HDfprintf(stream, "%*s%-*s %d\n", indent + 3, "", MAX(0, fwidth - 3), + oh->mesg[i].raw_size); + HDfprintf(stream, "%*s%-*s %u\n", indent + 3, "", MAX(0, fwidth - 3), "Chunk number:", - (int) (oh->mesg[i].chunkno)); + oh->mesg[i].chunkno); chunkno = oh->mesg[i].chunkno; if(chunkno >= oh->nchunks) HDfprintf(stream, "*** BAD CHUNK NUMBER\n"); - HDfprintf(stream, "%*s%-*s %u\n", indent + 3, "", MAX(0, fwidth - 3), + HDfprintf(stream, "%*s%-*s %Zu\n", indent + 3, "", MAX(0, fwidth - 3), "Raw data offset in chunk:", - (unsigned) (oh->mesg[i].raw - oh->chunk[chunkno].image)); + (size_t)(oh->mesg[i].raw - oh->chunk[chunkno].image)); /* check the size */ if((oh->mesg[i].raw + oh->mesg[i].raw_size > diff --git a/src/H5Ocache.c b/src/H5Ocache.c index e3ce6ac..d4e00a3 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -109,10 +109,7 @@ const H5AC_class_t H5AC_OHDR[1] = {{ herr_t H5O_flush_msgs(H5F_t *f, H5O_t *oh) { - uint8_t *p; /* Temporary pointer to encode with */ - int id; /* ID of message to encode */ H5O_mesg_t *curr_msg; /* Pointer to current message being operated on */ - herr_t (*encode)(H5F_t*, uint8_t*, const void*) = NULL; unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ @@ -125,29 +122,29 @@ H5O_flush_msgs(H5F_t *f, H5O_t *oh) /* Encode any dirty messages */ for(u = 0, curr_msg = &oh->mesg[0]; u < oh->nmesgs; u++, curr_msg++) { if(curr_msg->dirty) { + uint8_t *p; /* Temporary pointer to encode with */ + + /* Point into message's chunk's image */ p = curr_msg->raw - H5O_SIZEOF_MSGHDR_OH(oh); /* Encode the message prefix */ - id = curr_msg->type->id; - UINT16ENCODE(p, id); + UINT16ENCODE(p, curr_msg->type->id); HDassert(curr_msg->raw_size < H5O_MESG_MAX_SIZE); UINT16ENCODE(p, curr_msg->raw_size); *p++ = curr_msg->flags; - *p++ = 0; /*reserved*/ - *p++ = 0; /*reserved*/ - *p++ = 0; /*reserved*/ + + /* Only encode reserved bytes for version 1 of format */ + if(oh->version == H5O_VERSION_1) { + *p++ = 0; /*reserved*/ + *p++ = 0; /*reserved*/ + *p++ = 0; /*reserved*/ + } /* end for */ /* Encode the message itself */ if(curr_msg->native) { - HDassert(curr_msg->type->encode); + herr_t (*encode)(H5F_t*, uint8_t*, const void*); - /* allocate file space for chunks that have none yet */ - if(H5O_CONT_ID == id && !H5F_addr_defined(((H5O_cont_t *)(curr_msg->native))->addr)) - /* We now allocate disk space on insertion, instead - * of on flush from the cache, so this case is now an - * error. -- JRM - */ - HGOTO_ERROR(H5E_OHDR, H5E_SYSTEM, FAIL, "File space for message not allocated!?!") + HDassert(curr_msg->type->encode); /* * Encode the message. If the message is shared then we @@ -178,186 +175,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_flush_msgs() */ -#ifdef OLD_WAY - -/*------------------------------------------------------------------------- - * Function: H5O_load - * - * Purpose: Loads an object header from disk. - * - * Return: Success: Pointer to the new object header. - * - * Failure: NULL - * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Aug 5 1997 - * - *------------------------------------------------------------------------- - */ -static H5O_t * -H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, - void UNUSED * _udata2) -{ - H5O_t *oh = NULL; - H5O_t *ret_value; - uint8_t buf[16], *p; - size_t mesg_size; - size_t hdr_size; - unsigned id; - int mesgno; - unsigned curmesg = 0, nmesgs; - unsigned chunkno; - unsigned skipped_msgs = 0; /* Number of unknown messages skipped */ - unsigned merged_null_msgs = 0; /* Number of null messages merged together */ - haddr_t chunk_addr; - size_t chunk_size; - uint8_t flags; - - FUNC_ENTER_NOAPI(H5O_load, NULL) - - /* check args */ - HDassert(f); - HDassert(H5F_addr_defined(addr)); - HDassert(!_udata1); - HDassert(!_udata2); - - /* allocate ohdr and init chunk list */ - if (NULL==(oh = H5FL_CALLOC(H5O_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* read fixed-length part of object header */ - hdr_size = H5O_SIZEOF_HDR(f); - assert(hdr_size<=sizeof(buf)); - if (H5F_block_read(f, H5FD_MEM_OHDR, addr, hdr_size, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header") - p = buf; - - /* decode version */ - oh->version = *p++; - if (H5O_VERSION != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header version number") - - /* reserved */ - p++; - - /* decode number of messages */ - UINT16DECODE(p, nmesgs); - - /* decode link count */ - UINT32DECODE(p, oh->nlink); - - /* decode first chunk info */ - chunk_addr = addr + (hsize_t)hdr_size; - UINT32DECODE(p, chunk_size); - - /* build the message array */ - oh->alloc_nmesgs = nmesgs; - if (NULL==(oh->mesg=H5FL_SEQ_MALLOC(H5O_mesg_t,(size_t)oh->alloc_nmesgs))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* read each chunk from disk */ - while(H5F_addr_defined(chunk_addr)) { - /* increase chunk array size */ - if(oh->nchunks >= oh->alloc_nchunks) { - unsigned na = oh->alloc_nchunks + H5O_NCHUNKS; - H5O_chunk_t *x = H5FL_SEQ_REALLOC (H5O_chunk_t, oh->chunk, (size_t)na); - - if(!x) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - oh->alloc_nchunks = na; - oh->chunk = x; - } /* end if */ - - /* read the chunk raw data */ - chunkno = oh->nchunks++; - oh->chunk[chunkno].dirty = FALSE; - oh->chunk[chunkno].addr = chunk_addr; - oh->chunk[chunkno].size = chunk_size; - if(NULL==(oh->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, chunk_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - if(H5F_block_read(f, H5FD_MEM_OHDR, chunk_addr, chunk_size, dxpl_id, oh->chunk[chunkno].image) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data") - - /* load messages from this chunk */ - for(p = oh->chunk[chunkno].image; p < oh->chunk[chunkno].image + chunk_size; p += mesg_size) { - UINT16DECODE(p, id); - UINT16DECODE(p, mesg_size); - HDassert(mesg_size == H5O_ALIGN_OH(mesg_size)); - flags = *p++; - p += 3; /*reserved*/ - - /* Try to detect invalidly formatted object header messages */ - if(p + mesg_size > oh->chunk[chunkno].image + chunk_size) - HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "corrupt object header") - - /* Skip header messages we don't know about */ - /* (Usually from future versions of the library */ - if(id >= NELMTS(H5O_msg_class_g) || NULL == H5O_msg_class_g[id]) { - skipped_msgs++; - continue; - } /* end if */ - - if((H5F_get_intent(f) & H5F_ACC_RDWR) && - H5O_NULL_ID == id && oh->nmesgs > 0 && - H5O_NULL_ID == oh->mesg[oh->nmesgs - 1].type->id && - oh->mesg[oh->nmesgs - 1].chunkno == chunkno) { - /* combine adjacent null messages */ - mesgno = oh->nmesgs - 1; - oh->mesg[mesgno].raw_size += H5O_SIZEOF_MSGHDR(f) + mesg_size; - oh->mesg[mesgno].dirty = TRUE; - merged_null_msgs++; - } else { - /* new message */ - if (oh->nmesgs >= nmesgs) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "corrupt object header - too many messages") - mesgno = oh->nmesgs++; - oh->mesg[mesgno].type = H5O_msg_class_g[id]; - oh->mesg[mesgno].dirty = FALSE; - oh->mesg[mesgno].flags = flags; - oh->mesg[mesgno].native = NULL; - oh->mesg[mesgno].raw = p; - oh->mesg[mesgno].raw_size = mesg_size; - oh->mesg[mesgno].chunkno = chunkno; - } /* end else */ - } /* end for */ - - HDassert(p == oh->chunk[chunkno].image + chunk_size); - - /* decode next object header continuation message */ - for(chunk_addr = HADDR_UNDEF; !H5F_addr_defined(chunk_addr) && curmesg < oh->nmesgs; ++curmesg) { - if(H5O_CONT_ID == oh->mesg[curmesg].type->id) { - H5O_cont_t *cont; - - cont = (H5O_MSG_CONT->decode) (f, dxpl_id, oh->mesg[curmesg].raw); - oh->mesg[curmesg].native = cont; - chunk_addr = cont->addr; - chunk_size = cont->size; - cont->chunkno = oh->nchunks; /*the next chunk to allocate */ - } /* end if */ - } /* end for */ - } /* end while */ - - /* Mark the object header dirty if we've merged a message */ - if(merged_null_msgs) - oh->cache_info.is_dirty = TRUE; - - /* Sanity check for the correct # of messages in object header */ - if((oh->nmesgs + skipped_msgs + merged_null_msgs) != nmesgs) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "corrupt object header - too few messages") - - /* Set return value */ - ret_value = oh; - -done: - if(!ret_value && oh) { - if(H5O_dest(f,oh) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CANTFREE, NULL, "unable to destroy object header data") - } /* end if */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O_load() */ -#else /* OLD_WAY */ /*------------------------------------------------------------------------- * Function: H5O_load @@ -389,8 +206,9 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, unsigned merged_null_msgs = 0; /* Number of null messages merged together */ haddr_t chunk_addr; /* Address of first chunk */ size_t chunk_size; /* Size of first chunk */ - haddr_t abs_eoa; /* Absolute end of file address */ - haddr_t rel_eoa; /* Relative end of file address */ + haddr_t abs_eoa; /* Absolute end of file address */ + haddr_t rel_eoa; /* Relative end of file address */ + uint32_t prefix_chksum = 0; /* Checksum of object header prefix */ H5O_t *ret_value; /* Return value */ FUNC_ENTER_NOAPI(H5O_load, NULL) @@ -420,13 +238,26 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, if(NULL == (oh = H5FL_CALLOC(H5O_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Version */ - oh->version = *p++; - if(H5O_VERSION_1 != oh->version) - HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header version number") + /* Check for magic number */ + /* (indicates version 2 or later) */ + if(!HDmemcmp(p, H5O_HDR_MAGIC, (size_t)H5O_SIZEOF_MAGIC)) { + /* Magic number */ + p += H5O_SIZEOF_MAGIC; - /* Reserved */ - p++; + /* Version */ + oh->version = *p++; + if(H5O_VERSION_2 != oh->version) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header version number") + } /* end if */ + else { + /* Version */ + oh->version = *p++; + if(H5O_VERSION_1 != oh->version) + HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header version number") + + /* Reserved */ + p++; + } /* end else */ /* Number of messages */ UINT16DECODE(p, nmesgs); @@ -437,11 +268,18 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, /* First chunk size */ UINT32DECODE(p, chunk_size); - /* Reserved */ - p += 4; + /* Reserved, in version 1 */ + if(H5O_VERSION_1 == oh->version) + p += 4; - /* Compute first chunk address */ + /* Determine object header prefix length */ prefix_size = (size_t)(p - read_buf); + + /* Compute partial checksum, for later versions of the format */ + if(oh->version > H5O_VERSION_1) + prefix_chksum = H5_checksum_lookup3(read_buf, prefix_size, 0); + + /* Compute first chunk address */ chunk_addr = addr + (hsize_t)prefix_size; /* Allocate the message array */ @@ -452,11 +290,10 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, /* Read each chunk from disk */ while(H5F_addr_defined(chunk_addr)) { unsigned chunkno; /* Current chunk's index */ - size_t mesg_size; /* Size of message read in */ /* Increase chunk array size, if necessary */ if(oh->nchunks >= oh->alloc_nchunks) { - unsigned na = oh->alloc_nchunks + H5O_NCHUNKS; + unsigned na = MAX(H5O_NCHUNKS, oh->alloc_nchunks * 2); /* Double # of chunks allocated */ H5O_chunk_t *x = H5FL_SEQ_REALLOC(H5O_chunk_t, oh->chunk, (size_t)na); if(!x) @@ -465,43 +302,86 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, oh->chunk = x; } /* end if */ - /* Init the chunk raw data info */ + /* Init the chunk data info */ chunkno = oh->nchunks++; oh->chunk[chunkno].dirty = FALSE; - oh->chunk[chunkno].addr = chunk_addr; - oh->chunk[chunkno].size = chunk_size; - if(NULL == (oh->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, chunk_size))) + if(chunkno == 0) { + /* First chunk's 'image' includes room for the object header prefix */ + oh->chunk[0].addr = addr; + oh->chunk[0].size = chunk_size + H5O_SIZEOF_HDR_OH(oh); + } /* end if */ + else { + oh->chunk[chunkno].addr = chunk_addr; + oh->chunk[chunkno].size = chunk_size; + } /* end else */ + if(NULL == (oh->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, oh->chunk[chunkno].size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Check for speculative read of first chunk containing all the data needed */ - if(chunkno == 0 && (spec_read_size - prefix_size) >= chunk_size) - HDmemcpy(oh->chunk[chunkno].image, p, chunk_size); + /* Handle chunk 0 as special case */ + if(chunkno == 0) { + /* Check for speculative read of first chunk containing all the data needed */ + if(spec_read_size >= oh->chunk[0].size) + HDmemcpy(oh->chunk[0].image, read_buf, oh->chunk[0].size); + else { + /* Copy the object header prefix into chunk 0's image */ + HDmemcpy(oh->chunk[0].image, read_buf, prefix_size); + + /* Read the chunk raw data */ + if(H5F_block_read(f, H5FD_MEM_OHDR, chunk_addr, (oh->chunk[0].size - prefix_size), + dxpl_id, (oh->chunk[0].image + prefix_size)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data") + } /* end else */ + + /* Point into chunk image to decode */ + p = oh->chunk[0].image + prefix_size; + } /* end if */ else { /* Read the chunk raw data */ - if(H5F_block_read(f, H5FD_MEM_OHDR, chunk_addr, chunk_size, dxpl_id, oh->chunk[chunkno].image) < 0) + if(H5F_block_read(f, H5FD_MEM_OHDR, chunk_addr, chunk_size, + dxpl_id, oh->chunk[chunkno].image) < 0) HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data") + + /* Point into chunk image to decode */ + p = oh->chunk[chunkno].image; } /* end else */ - /* Load messages from this chunk */ - for(p = oh->chunk[chunkno].image; p < oh->chunk[chunkno].image + chunk_size; p += mesg_size) { + /* Check for magic # on chunks > 0 in later versions of the format */ + if(chunkno > 0 && oh->version > H5O_VERSION_1) { + /* Magic number */ + if(!HDmemcmp(p, H5O_CHK_MAGIC, (size_t)H5O_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "wrong object header chunk signature") + p += H5O_SIZEOF_MAGIC; + } /* end if */ + + /* Decode messages from this chunk */ + while(p < (oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM_OH(oh)))) { unsigned mesgno; /* Current message to operate on */ + size_t mesg_size; /* Size of message read in */ unsigned id; /* ID (type) of current message */ uint8_t flags; /* Flags for current message */ + /* Decode message prefix info */ UINT16DECODE(p, id); UINT16DECODE(p, mesg_size); HDassert(mesg_size == H5O_ALIGN_OH(oh, mesg_size)); flags = *p++; - p += 3; /*reserved*/ + if(oh->version == H5O_VERSION_1) + p += 3; /*reserved*/ /* Try to detect invalidly formatted object header messages */ - if(p + mesg_size > oh->chunk[chunkno].image + chunk_size) + if(p + mesg_size > oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM_OH(oh))) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "corrupt object header") /* Skip header messages we don't know about */ /* (Usually from future versions of the library) */ if(id >= NELMTS(H5O_msg_class_g) || NULL == H5O_msg_class_g[id]) { + /* Increment skipped messages counter */ skipped_msgs++; + + /* Advance decode pointer past message */ + p += mesg_size; + + /* Go get next message */ continue; } /* end if */ @@ -530,9 +410,32 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, oh->mesg[mesgno].raw_size = mesg_size; oh->mesg[mesgno].chunkno = chunkno; } /* end else */ - } /* end for */ - HDassert(p == oh->chunk[chunkno].image + chunk_size); + /* Advance decode pointer past message */ + p += mesg_size; + } /* end while */ + + /* Check for correct checksum on chunks, in later versions of the format */ + if(oh->version > H5O_VERSION_1) { + uint32_t stored_chksum; /* Checksum from file */ + uint32_t computed_chksum; /* Checksum computed in memory */ + + /* Metadata checksum */ + UINT32DECODE(p, stored_chksum); + + /* Compute checksum on entire header */ + if(chunkno == 0) + computed_chksum = H5_checksum_metadata(oh->chunk[chunkno].image + prefix_size, chunk_size, prefix_chksum); + else + computed_chksum = H5_checksum_metadata(oh->chunk[chunkno].image, (chunk_size - H5O_SIZEOF_CHKSUM), 0); + + /* Verify checksum */ + if(stored_chksum != computed_chksum) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "incorrect metadata checksum for object header chunk") + } /* end if */ + + /* Sanity check */ + HDassert(p == oh->chunk[chunkno].image + oh->chunk[chunkno].size); /* Check for another chunk to read in & parse */ for(chunk_addr = HADDR_UNDEF; !H5F_addr_defined(chunk_addr) && curmesg < oh->nmesgs; ++curmesg) { @@ -574,7 +477,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_load() */ -#endif /* OLD_WAY */ /*------------------------------------------------------------------------- @@ -591,11 +493,8 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5O_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5O_t *oh) +H5O_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, H5O_t *oh) { - uint8_t buf[16], *p; - hbool_t combine = FALSE; /* Whether to combine the object header prefix & the first chunk */ - unsigned u; /* Local index variable */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5O_flush, FAIL) @@ -607,69 +506,112 @@ H5O_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5O_t *oh) /* flush */ if(oh->cache_info.is_dirty) { + uint8_t *p; /* Pointer to object header prefix buffer */ + unsigned u; /* Local index variable */ + /* Encode any dirty messages */ if(H5O_flush_msgs(f, oh) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTFLUSH, FAIL, "unable to flush object header messages") - /* Encode header prefix */ - p = buf; + /* Point to raw data 'image' for first chunk, which has room for the prefix */ + p = oh->chunk[0].image; + + /* Later versions of object header prefix have different format and + * also require that chunk 0 always be updated, since the checksum + * on the entire block of memory needs to be updated if anything is + * modified */ + if(oh->version > H5O_VERSION_1) { + size_t prefix_size; /* Length of object header prefix */ + uint32_t prefix_chksum; /* Prefix checksum value */ + uint32_t full_chksum; /* Full checksum value */ + size_t raw_size; /* Size of raw data in first chunk */ - /* encode version */ - *p++ = oh->version; + /* Verify magic number */ + HDassert(!HDmemcmp(oh->chunk[u].image, H5O_HDR_MAGIC, H5O_SIZEOF_MAGIC)); + p += H5O_SIZEOF_MAGIC; - /* reserved */ - *p++ = 0; + /* Version */ + *p++ = oh->version; - /* encode number of messages */ - UINT16ENCODE(p, oh->nmesgs); + /* Number of messages */ + UINT16ENCODE(p, oh->nmesgs); - /* encode link count */ - UINT32ENCODE(p, oh->nlink); + /* Link count */ + UINT32ENCODE(p, oh->nlink); - /* encode body size */ - UINT32ENCODE(p, oh->chunk[0].size); + /* Chunk size */ + UINT32ENCODE(p, (oh->chunk[0].size - H5O_SIZEOF_HDR_OH(oh))); - /* zero to alignment */ - HDmemset(p, 0, (size_t)(H5O_SIZEOF_HDR_OH(oh) - 12)); + /* Determine object header prefix length */ + prefix_size = (size_t)(p - oh->chunk[0].image); - /* write the object header prefix */ + /* Compute partial checksum for later */ + /* (checksum performed in this odd way in order to accomodate + * reading in the header & first chunk in reasonable way) + */ + prefix_chksum = H5_checksum_lookup3(oh->chunk[0].image, prefix_size, 0); - /* Check if we can combine the object header prefix & the first chunk into one I/O operation */ - if(oh->chunk[0].dirty && (addr + H5O_SIZEOF_HDR_OH(oh)) == oh->chunk[0].addr) - combine = TRUE; + /* Finish full checksum, over chunk data */ + raw_size = oh->chunk[0].size - H5O_SIZEOF_HDR_OH(oh); + full_chksum = H5_checksum_metadata(p, raw_size, prefix_chksum); + p += raw_size; + + /* Metadata checksum */ + UINT32ENCODE(p, full_chksum); + HDassert((size_t)(p - oh->chunk[0].image) == oh->chunk[0].size); + } /* end if */ else { - if(H5F_block_write(f, H5FD_MEM_OHDR, addr, (size_t)H5O_SIZEOF_HDR_OH(oh), dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to write object header hdr to disk") + /* Version */ + *p++ = oh->version; + + /* Reserved */ + *p++ = 0; + + /* Number of messages */ + UINT16ENCODE(p, oh->nmesgs); + + /* Link count */ + UINT32ENCODE(p, oh->nlink); + + /* First chunk size */ + UINT32ENCODE(p, (oh->chunk[0].size - H5O_SIZEOF_HDR_OH(oh))); + + /* Zero to alignment */ + HDmemset(p, 0, (size_t)(H5O_SIZEOF_HDR_OH(oh) - 12)); } /* end else */ - /* write each chunk to disk */ - for(u = 0; u < oh->nchunks; u++) { - if(oh->chunk[u].dirty) { - HDassert(H5F_addr_defined(oh->chunk[u].addr)); - if(u == 0 && combine) { - /* Allocate space for the combined prefix and first chunk */ - if((p = H5FL_BLK_MALLOC(chunk_image, (H5O_SIZEOF_HDR_OH(oh) + oh->chunk[u].size))) == NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + /* Mark chunk 0 as dirty, since the object header prefix has been updated */ + /* (this could be more sophisticated and track whether any prefix fields + * have been changed, which could save I/O accesses - QAK) + */ + HDassert(H5F_addr_eq(addr, oh->chunk[0].addr)); + oh->chunk[0].dirty = TRUE; - /* Copy in the prefix */ - HDmemcpy(p, buf, (size_t)H5O_SIZEOF_HDR_OH(oh)); + /* Write each chunk to disk, if it's dirty */ + for(u = 0; u < oh->nchunks; u++) { + /* Sanity check - make certain the magic # is present */ + if(oh->version > H5O_VERSION_1) + HDassert(!HDmemcmp(oh->chunk[u].image, (u == 0 ? H5O_HDR_MAGIC : H5O_CHK_MAGIC), H5O_SIZEOF_MAGIC)); - /* Copy in the first chunk */ - HDmemcpy(p + H5O_SIZEOF_HDR_OH(oh), oh->chunk[u].image, oh->chunk[u].size); + /* Write out chunk, if it's dirty */ + if(oh->chunk[u].dirty) { + /* Compute checksum, for chunks > 0 & later versions of format */ + if(u > 0 && oh->version > H5O_VERSION_1) { + uint32_t metadata_chksum; /* Computed metadata checksum value */ - /* Write the combined prefix/chunk out */ - if(H5F_block_write(f, H5FD_MEM_OHDR, addr, - (H5O_SIZEOF_HDR_OH(oh) + oh->chunk[u].size), dxpl_id, p) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to write object header data to disk") + /* Compute metadata checksum */ + metadata_chksum = H5_checksum_metadata(oh->chunk[u].image, (oh->chunk[u].size - H5O_SIZEOF_CHKSUM), 0); - /* Release the memory for the combined prefix/chunk */ - p = H5FL_BLK_FREE(chunk_image, p); + /* Metadata checksum */ + p = oh->chunk[u].image + (oh->chunk[u].size - H5O_SIZEOF_CHKSUM); + UINT32ENCODE(p, metadata_chksum); } /* end if */ - else { - if(H5F_block_write(f, H5FD_MEM_OHDR, oh->chunk[u].addr, - (oh->chunk[u].size), dxpl_id, oh->chunk[u].image) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to write object header data to disk") - } /* end else */ + + /* Write the chunk out */ + HDassert(H5F_addr_defined(oh->chunk[u].addr)); + if(H5F_block_write(f, H5FD_MEM_OHDR, oh->chunk[u].addr, + oh->chunk[u].size, dxpl_id, oh->chunk[u].image) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to write object header chunk to disk") /* Mark chunk as clean now */ oh->chunk[u].dirty = FALSE; @@ -680,10 +622,10 @@ H5O_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5O_t *oh) oh->cache_info.is_dirty = FALSE; } /* end if */ - if(destroy) { + /* Destroy the object header, if requested */ + if(destroy) if(H5O_dest(f,oh) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to destroy object header data") - } /* end if */ done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Opkg.h b/src/H5Opkg.h index 49a0cf9..4886153 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -48,13 +48,9 @@ * object header chunks directly into memory and operate on them there, even * on 64-bit architectures. This allows us to reduce the number of disk I/O * requests with a minimum amount of mem-to-mem copies. + * + * Note: We no longer attempt to do this. - QAK, 10/16/06 */ -/* - * Note: We no longer attempt to do this. - */ -#ifdef OLD_WAY -#define H5O_ALIGN(X) (8 * (((X) + 7) / 8)) -#else /* OLD_WAY */ #define H5O_ALIGN_OLD(X) (8 * (((X) + 7) / 8)) #define H5O_ALIGN_VERS(V, X) \ (((V) == H5O_VERSION_1) ? \ @@ -66,19 +62,20 @@ H5O_ALIGN_VERS((O)->version, X) #define H5O_ALIGN_F(F, X) \ H5O_ALIGN_VERS((H5F_USE_LATEST_FORMAT(F) ? H5O_VERSION_LATEST : H5O_VERSION_1), X) -#endif /* OLD_WAY */ + +/* Size of signature information (on disk) */ +#define H5O_SIZEOF_MAGIC 4 + +/* Object header signatures */ +#define H5O_HDR_MAGIC "OHDR" /* Header */ +#define H5O_CHK_MAGIC "OCHK" /* Continuation chunk */ + +/* Size of checksum (on disk) */ +#define H5O_SIZEOF_CHKSUM 4 /* * Size of object header prefix. */ -#ifdef OLD_WAY -#define H5O_SIZEOF_HDR(F) \ - H5O_ALIGN(1 + /*version number */ \ - 1 + /*reserved */ \ - 2 + /*number of messages */ \ - 4 + /*reference count */ \ - 4) /*header data size */ -#else /* OLD_WAY */ #define H5O_SIZEOF_HDR_VERS(V) \ (((V) == H5O_VERSION_1) ? \ H5O_ALIGN_OLD(1 + /*version number */ \ @@ -87,28 +84,21 @@ 4 + /*reference count */ \ 4) /*header data size */ \ : \ - (4 + /*magic number */ \ + (H5O_SIZEOF_MAGIC + /*magic number */ \ 1 + /*version number */ \ 2 + /*number of messages */ \ 4 + /*reference count */ \ - 4) /*header data size */ \ + 4 + /*header data size */ \ + H5O_SIZEOF_CHKSUM) /*checksum size */ \ ) #define H5O_SIZEOF_HDR_OH(O) \ H5O_SIZEOF_HDR_VERS((O)->version) #define H5O_SIZEOF_HDR_F(F) \ H5O_SIZEOF_HDR_VERS(H5F_USE_LATEST_FORMAT(F) ? H5O_VERSION_LATEST : H5O_VERSION_1) -#endif /* OLD_WAY */ /* * Size of object header message prefix */ -#ifdef OLD_WAY -#define H5O_SIZEOF_MSGHDR(F) \ - H5O_ALIGN(2 + /*message type */ \ - 2 + /*sizeof message data */ \ - 1 + /*flags */ \ - 3) /*reserved */ -#else /* OLD_WAY */ #define H5O_SIZEOF_MSGHDR_VERS(V) \ (((V) == H5O_VERSION_1) ? \ H5O_ALIGN_OLD(2 + /*message type */ \ @@ -123,7 +113,32 @@ H5O_SIZEOF_MSGHDR_VERS((O)->version) #define H5O_SIZEOF_MSGHDR_F(F) \ H5O_SIZEOF_MSGHDR_VERS(H5F_USE_LATEST_FORMAT(F) ? H5O_VERSION_LATEST : H5O_VERSION_1) -#endif /* OLD_WAY */ + +/* + * Size of chunk "header" for each chunk + */ +#define H5O_SIZEOF_CHKHDR_VERS(V) \ + (((V) == H5O_VERSION_1) ? \ + 0 + /*no magic # */ \ + 0 /*no checksum */ \ + : \ + H5O_SIZEOF_MAGIC + /*magic # */ \ + H5O_SIZEOF_CHKSUM /*checksum */ \ + ) +#define H5O_SIZEOF_CHKHDR_OH(O) \ + H5O_SIZEOF_CHKHDR_VERS((O)->version) + +/* + * Size of checksum for each chunk + */ +#define H5O_SIZEOF_CHKSUM_VERS(V) \ + (((V) == H5O_VERSION_1) ? \ + 0 /*no checksum */ \ + : \ + H5O_SIZEOF_CHKSUM /*checksum */ \ + ) +#define H5O_SIZEOF_CHKSUM_OH(O) \ + H5O_SIZEOF_CHKSUM_VERS((O)->version) struct H5O_msg_class_t { unsigned id; /*message type ID on disk */ diff --git a/tools/misc/h5debug.c b/tools/misc/h5debug.c index 5b33c65..1518fb6 100644 --- a/tools/misc/h5debug.c +++ b/tools/misc/h5debug.c @@ -380,9 +380,16 @@ main(int argc, char *argv[]) status = H5FS_sects_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL, extra, extra2); + } else if(!HDmemcmp(sig, H5O_HDR_MAGIC, H5O_SIZEOF_MAGIC)) { + /* + * Debug v2 object header (which have signatures). + */ + + status = H5O_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL); + } else if(sig[0] == H5O_VERSION_1) { /* - * This could be an object header. Since they don't have a signature + * This could be a v1 object header. Since they don't have a signature * it's a somewhat "ify" detection. */ status = H5O_debug(f, H5P_DATASET_XFER_DEFAULT, addr, stdout, 0, VCOL); -- cgit v0.12