diff options
author | Quincey Koziol <koziol@hdfgroup.org> | 2008-07-20 02:32:47 (GMT) |
---|---|---|
committer | Quincey Koziol <koziol@hdfgroup.org> | 2008-07-20 02:32:47 (GMT) |
commit | d08619bfe1a3cbc593bdaae5a5bd393e1d33f150 (patch) | |
tree | 57ec6ef4e2167462f2a10b5739b0f79f2299584d /src/H5Ocache.c | |
parent | 90497050b818e914ab814e40811b1c0ff98384ea (diff) | |
download | hdf5-d08619bfe1a3cbc593bdaae5a5bd393e1d33f150.zip hdf5-d08619bfe1a3cbc593bdaae5a5bd393e1d33f150.tar.gz hdf5-d08619bfe1a3cbc593bdaae5a5bd393e1d33f150.tar.bz2 |
[svn-r15388] Description:
Convert object header cache client to use the new metadata journaling
cache, which included adding a new client for handling continuation chunks.
Added "real" protect calls around modifying chunks in object headers.
Switched a few more metadata cache library API routines to drop the
file pointer, when it is not needed (pinning/unpinning entries, etc.)
Fixed bug in journaling cache handling of 'image_len' callbacks and
also changed cache to retry deserializing entries when the entry's size is
larger than the speculative size initially tried.
Retrying for 'image_len' callbacks has problems with the 'multi' VFD,
so the h5dump and FORTRAN 'multi' tests are commented out, until the changes to
the 'multi' VFD from the file free space branch are brought back into the
trunk.
Currently, the 'h5recover' tool has a bug which requires it to be run
twice before replaying the journal "sticks". However, this is from an earlier
checkin, since the code in the branch already has this behavior... :-(
Tested on:
FreeBSD/32 6.2 (duty) in debug mode
FreeBSD/64 6.2 (liberty) w/C++ & FORTRAN, in debug mode
Linux/32 2.6 (kagiso) w/PGI compilers, w/C++ & FORTRAN, w/threadsafe,
in debug mode
Linux/64-amd64 2.6 (smirom) w/default API=1.6.x, w/C++ & FORTRAN,
in production mode
Linux/64-ia64 2.6 (cobalt) w/Intel compilers, w/C++ & FORTRAN,
in production mode
Solaris/32 2.10 (linew) w/deprecated symbols disabled, w/C++ & FORTRAN,
w/szip filter, in production mode
Mac OS X/32 10.5.4 (amazon) in debug mode
Linux/64-ia64 2.4 (tg-login3) w/parallel, w/FORTRAN, in production mode
Diffstat (limited to 'src/H5Ocache.c')
-rw-r--r-- | src/H5Ocache.c | 1446 |
1 files changed, 814 insertions, 632 deletions
diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 0d5765e..4df43d8 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -44,12 +44,6 @@ /* Local Macros */ /****************/ -/* Set the object header size to speculatively read in */ -/* (needs to be more than the object header prefix size to work at all and - * should be larger than the largest object type's default object header - * size to save the extra I/O operations) */ -#define H5O_SPEC_READ_SIZE 512 - /******************/ /* Local Typedefs */ @@ -66,106 +60,129 @@ /********************/ /* Metadata cache callbacks */ -static H5O_t *H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *_udata1, - void *_udata2); -static herr_t H5O_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5O_t *oh, unsigned UNUSED * flags_ptr); -static herr_t H5O_clear(H5F_t *f, H5O_t *oh, hbool_t destroy); -static herr_t H5O_size(const H5F_t *f, const H5O_t *oh, size_t *size_ptr); +static void *H5O_cache_deserialize(haddr_t addr, size_t len, const void *image, + void *udata, hbool_t *dirty); +static herr_t H5O_cache_image_len(const void *thing, size_t *image_len_ptr); +static herr_t H5O_cache_serialize(const H5F_t *f, haddr_t addr, size_t len, + void *image, void *thing, unsigned *flags, haddr_t *new_addr, + size_t *new_len, void **new_image); +static herr_t H5O_cache_free_icr(haddr_t addr, size_t len, void *thing); + +static void *H5O_cache_chk_deserialize(haddr_t addr, size_t len, const void *image, + void *udata, hbool_t *dirty); +static herr_t H5O_cache_chk_serialize(const H5F_t *f, haddr_t addr, size_t len, + void *image, void *thing, unsigned *flags, haddr_t *new_addr, + size_t *new_len, void **new_image); +static herr_t H5O_cache_chk_free_icr(haddr_t addr, size_t len, void *thing); + +/* Chunk proxy routines */ +static herr_t H5O_chunk_proxy_dest(H5O_chunk_proxy_t *chunk_proxy); + +/* Chunk routines */ +static herr_t H5O_chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, + const uint8_t *image, H5O_common_cache_ud_t *udata, hbool_t *dirty); +static herr_t H5O_chunk_serialize(const H5F_t *f, H5O_t *oh, unsigned chunkno, + uint8_t *image); + +/* Misc. routines */ +static herr_t H5O_add_cont_msg(H5O_cont_msgs_t *cont_msg_info, + const H5O_cont_t *cont); /*********************/ /* Package Variables */ /*********************/ +/* H5O object header prefix inherits cache-like properties from H5AC */ +const H5AC2_class_t H5AC2_OHDR[1] = {{ + H5AC2_OHDR_ID, + "object header", + H5FD_MEM_OHDR, + H5O_cache_deserialize, + H5O_cache_image_len, + H5O_cache_serialize, + H5O_cache_free_icr, + NULL, +}}; -/*****************************/ -/* Library Private Variables */ -/*****************************/ +/* H5O object header chunk inherits cache-like properties from H5AC */ +const H5AC2_class_t H5AC2_OHDR_CHK[1] = {{ + H5AC2_OHDR_CHK_ID, + "object header chunk", + H5FD_MEM_OHDR, + H5O_cache_chk_deserialize, + NULL, + H5O_cache_chk_serialize, + H5O_cache_chk_free_icr, + NULL, +}}; /* Declare external the free list for H5O_unknown_t's */ H5FL_EXTERN(H5O_unknown_t); +/* Declare external the free list for H5O_cont_t sequences */ +H5FL_SEQ_DEFINE(H5O_cont_t); + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + /*******************/ /* Local Variables */ /*******************/ -/* H5O inherits cache-like properties from H5AC */ -const H5AC_class_t H5AC_OHDR[1] = {{ - H5AC_OHDR_ID, - (H5AC_load_func_t)H5O_load, - (H5AC_flush_func_t)H5O_flush, - (H5AC_dest_func_t)H5O_dest, - (H5AC_clear_func_t)H5O_clear, - (H5AC_size_func_t)H5O_size, -}}; +/* Declare extern the free list for H5O_chunk_proxy_t's */ +H5FL_EXTERN(H5O_chunk_proxy_t); /*------------------------------------------------------------------------- - * Function: H5O_load - * - * Purpose: Loads an object header from disk. + * Function: H5O_cache_deserialize * - * Return: Success: Pointer to the new object header. + * Purpose: Deserializes an object header prefix + first chunk * + * Return: Success: Pointer to a new object header * Failure: NULL * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Aug 5 1997 + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Jul 12 2008 * *------------------------------------------------------------------------- */ -static H5O_t * -H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, - void UNUSED * _udata2) +static void * +H5O_cache_deserialize(haddr_t addr, size_t len, const void *image, + void *_udata, hbool_t *dirty) { - H5O_t *oh = NULL; /* Object header read in */ - uint8_t read_buf[H5O_SPEC_READ_SIZE]; /* Buffer for speculative read */ - const uint8_t *p; /* Pointer into buffer to decode */ - size_t spec_read_size; /* Size of buffer to speculatively read in */ - size_t prefix_size; /* Size of object header prefix */ - unsigned nmesgs; /* Total # of messages in this object header */ - unsigned curmesg = 0; /* Current message being decoded in object header */ - unsigned merged_null_msgs = 0; /* Number of null messages merged together */ - haddr_t chunk_addr; /* Address of first chunk */ - size_t chunk_size; /* Size of first chunk */ - haddr_t abs_eoa; /* Absolute end of file address */ - haddr_t rel_eoa; /* Relative end of file address */ - H5O_t *ret_value; /* Return value */ - - FUNC_ENTER_NOAPI(H5O_load, NULL) - - /* check args */ - HDassert(f); - HDassert(H5F_addr_defined(addr)); - HDassert(!_udata1); - HDassert(!_udata2); + H5O_t *oh = NULL; /* Object header info */ + H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ + const uint8_t *p; /* Pointer into raw data buffer */ + size_t prefix_size; /* Size of object header prefix */ + H5O_t *ret_value; /* Return value */ - /* Make certain we don't speculatively read off the end of the file */ - if(HADDR_UNDEF == (abs_eoa = H5F_get_eoa(f))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, NULL, "unable to determine file size") + FUNC_ENTER_NOAPI_NOINIT(H5O_cache_deserialize) - /* Adjust absolute EOA address to relative EOA address */ - rel_eoa = abs_eoa - H5F_get_base_addr(f); - - /* Compute the size of the speculative object header buffer */ - H5_ASSIGN_OVERFLOW(spec_read_size, MIN(rel_eoa - addr, H5O_SPEC_READ_SIZE), /* From: */ hsize_t, /* To: */ size_t); - - /* Attempt to speculatively read both object header prefix and first chunk */ - if(H5F_block_read(f, H5FD_MEM_OHDR, addr, spec_read_size, dxpl_id, read_buf) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header") - p = read_buf; + /* Check arguments */ + HDassert(H5F_addr_defined(addr)); + HDassert(len > 0); + HDassert(image); + HDassert(udata); + HDassert(udata->common.f); + HDassert(udata->common.cont_msg_info); - /* allocate ohdr and init chunk list */ + /* Allocate space for the object header data structure */ if(NULL == (oh = H5FL_CALLOC(H5O_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* File-specific, non-stored information */ - oh->sizeof_size = H5F_SIZEOF_SIZE(f); - oh->sizeof_addr = H5F_SIZEOF_ADDR(f); + oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); + oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f); - /* Check for magic number */ + /* Get temporary pointer to serialized header */ + p = image; + + /* Check for presence of magic number */ /* (indicates version 2 or later) */ if(!HDmemcmp(p, H5O_HDR_MAGIC, (size_t)H5O_SIZEOF_MAGIC)) { /* Magic number */ @@ -181,9 +198,6 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, if(oh->flags & ~H5O_HDR_ALL_FLAGS) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "unknown object header status flag(s)") - /* Number of messages (to allocate initially) */ - nmesgs = 1; - /* Number of links to object (unless overridden by refcount message) */ oh->nlink = 1; @@ -212,25 +226,25 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, /* First chunk size */ switch(oh->flags & H5O_HDR_CHUNK0_SIZE) { case 0: /* 1 byte size */ - chunk_size = *p++; + oh->chunk0_size = *p++; break; case 1: /* 2 byte size */ - UINT16DECODE(p, chunk_size); + UINT16DECODE(p, oh->chunk0_size); break; case 2: /* 4 byte size */ - UINT32DECODE(p, chunk_size); + UINT32DECODE(p, oh->chunk0_size); break; case 3: /* 8 byte size */ - UINT64DECODE(p, chunk_size); + UINT64DECODE(p, oh->chunk0_size); break; default: HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "bad size for chunk 0") } /* end switch */ - if(chunk_size > 0 && chunk_size < H5O_SIZEOF_MSGHDR_OH(oh)) + if(oh->chunk0_size > 0 && oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header chunk size") } /* end if */ else { @@ -246,7 +260,7 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, p++; /* Number of messages */ - UINT16DECODE(p, nmesgs); + UINT16DECODE(p, udata->v1_pfx_nmesgs); /* Link count */ UINT32DECODE(p, oh->nlink); @@ -259,691 +273,859 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * _udata1, oh->min_dense = 0; /* First chunk size */ - UINT32DECODE(p, chunk_size); - if((nmesgs > 0 && chunk_size < H5O_SIZEOF_MSGHDR_OH(oh)) || - (nmesgs == 0 && chunk_size > 0)) + UINT32DECODE(p, oh->chunk0_size); + if((udata->v1_pfx_nmesgs > 0 && oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || + (udata->v1_pfx_nmesgs == 0 && oh->chunk0_size > 0)) HGOTO_ERROR(H5E_OHDR, H5E_VERSION, NULL, "bad object header chunk size") - /* Reserved, in version 1 */ + /* Reserved, in version 1 (for 8-byte alignment padding) */ p += 4; } /* end else */ /* Determine object header prefix length */ - prefix_size = (size_t)(p - read_buf); - HDassert((size_t)prefix_size == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + prefix_size = (size_t)(p - (const uint8_t *)image); + HDassert(prefix_size == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); - /* Compute first chunk address */ - chunk_addr = addr + (hsize_t)prefix_size; + /* Check if the length is large enough to parse the first chunk also */ + /* (If not, we'll return the object header struct that we have now and + * count on the cache to query it for the correct size and retry with the + * full first chunk available) + */ + if(len >= (oh->chunk0_size + H5O_SIZEOF_HDR(oh))) { + /* Parse the first chunk */ + if(H5O_chunk_deserialize(oh, addr, oh->chunk0_size, image, &(udata->common), dirty) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk") + } /* end if */ + else + /* Make certain the length was OK on the retry */ + HDassert(!udata->made_attempt); - /* Allocate the message array */ - oh->alloc_nmesgs = (nmesgs > 0) ? nmesgs : 1; - if(NULL == (oh->mesg = H5FL_SEQ_MALLOC(H5O_mesg_t, oh->alloc_nmesgs))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + /* Note that we've made one attempt at decoding the object header already */ + /* (useful when the length is incorrect and the cache will retry with a larger one) */ + udata->made_attempt = TRUE; - /* Read each chunk from disk */ - while(H5F_addr_defined(chunk_addr)) { - unsigned chunkno; /* Current chunk's index */ -#ifndef NDEBUG - unsigned nullcnt; /* Count of null messages (for sanity checking gaps in chunks) */ -#endif /* NDEBUG */ - uint8_t *eom_ptr; /* Pointer to end of messages for a chunk */ - - /* Increase chunk array size, if necessary */ - if(oh->nchunks >= oh->alloc_nchunks) { - unsigned na = MAX(H5O_NCHUNKS, oh->alloc_nchunks * 2); /* Double # of chunks allocated */ - H5O_chunk_t *x = H5FL_SEQ_REALLOC(H5O_chunk_t, oh->chunk, (size_t)na); - - if(!x) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - oh->alloc_nchunks = na; - oh->chunk = x; - } /* end if */ - - /* Init the chunk data info */ - chunkno = oh->nchunks++; - oh->chunk[chunkno].dirty = FALSE; - oh->chunk[chunkno].gap = 0; - if(chunkno == 0) { - /* First chunk's 'image' includes room for the object header prefix */ - oh->chunk[0].addr = addr; - oh->chunk[0].size = chunk_size + H5O_SIZEOF_HDR(oh); - } /* end if */ - else { - oh->chunk[chunkno].addr = chunk_addr; - oh->chunk[chunkno].size = chunk_size; - } /* end else */ - if(NULL == (oh->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, oh->chunk[chunkno].size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* Handle chunk 0 as special case */ - if(chunkno == 0) { - /* Check for speculative read of first chunk containing all the data needed */ - if(spec_read_size >= oh->chunk[0].size) - HDmemcpy(oh->chunk[0].image, read_buf, oh->chunk[0].size); - else { - /* Copy the object header prefix into chunk 0's image */ - HDmemcpy(oh->chunk[0].image, read_buf, prefix_size); - - /* Read the chunk raw data */ - /* (probably re-reads some data we already retrieved, but since - * we have to do the I/O operation anyway, we might as - * well avoid memcpy()ing the data in our buffer already) - */ - if(H5F_block_read(f, H5FD_MEM_OHDR, chunk_addr, (oh->chunk[0].size - prefix_size), - dxpl_id, (oh->chunk[0].image + prefix_size)) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data") - } /* end else */ - - /* Point into chunk image to decode */ - p = oh->chunk[0].image + prefix_size; - } /* end if */ - else { - /* Read the chunk raw data */ - if(H5F_block_read(f, H5FD_MEM_OHDR, chunk_addr, chunk_size, dxpl_id, oh->chunk[chunkno].image) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data") + /* Set return value */ + ret_value = oh; - /* Point into chunk image to decode */ - p = oh->chunk[chunkno].image; - } /* end else */ +done: + /* Release the [possibly partially initialized] object header on errors */ + if(!ret_value && oh) + if(H5O_dest(oh) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, NULL, "unable to destroy object header data") - /* Check for magic # on chunks > 0 in later versions of the format */ - if(chunkno > 0 && oh->version > H5O_VERSION_1) { - /* Magic number */ - if(HDmemcmp(p, H5O_CHK_MAGIC, (size_t)H5O_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "wrong object header chunk signature") - p += H5O_SIZEOF_MAGIC; - } /* end if */ + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_cache_deserialize() */ - /* Decode messages from this chunk */ - eom_ptr = oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM_OH(oh)); -#ifndef NDEBUG - nullcnt = 0; -#endif /* NDEBUG */ - while(p < eom_ptr) { - unsigned mesgno; /* Current message to operate on */ - size_t mesg_size; /* Size of message read in */ - unsigned id; /* ID (type) of current message */ - uint8_t flags; /* Flags for current message */ - H5O_msg_crt_idx_t crt_idx = 0; /* Creation index for current message */ - - /* Decode message prefix info */ - - /* Version # */ - if(oh->version == H5O_VERSION_1) - UINT16DECODE(p, id) - else - id = *p++; - - /* Check for unknown message ID getting encoded in file */ - if(id == H5O_UNKNOWN_ID) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "'unknown' message ID encoded in file?!?") - - /* Message size */ - UINT16DECODE(p, mesg_size); - HDassert(mesg_size == H5O_ALIGN_OH(oh, mesg_size)); - - /* Message flags */ - flags = *p++; - if(flags & ~H5O_MSG_FLAG_BITS) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "unknown flag for message") - if((flags & H5O_MSG_FLAG_SHARED) && (flags & H5O_MSG_FLAG_DONTSHARE)) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad flag combination for message") - if((flags & H5O_MSG_FLAG_WAS_UNKNOWN) && (flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN)) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad flag combination for message") - if((flags & H5O_MSG_FLAG_WAS_UNKNOWN) && !(flags & H5O_MSG_FLAG_MARK_IF_UNKNOWN)) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "bad flag combination for message") - - /* Reserved bytes/creation index */ - if(oh->version == H5O_VERSION_1) - p += 3; /*reserved*/ - else { - /* Only encode creation index if they are being tracked */ - if(oh->flags & H5O_HDR_ATTR_CRT_ORDER_TRACKED) - UINT16DECODE(p, crt_idx); - } /* end else */ - - /* Try to detect invalidly formatted object header message that - * extends past end of chunk. - */ - if(p + mesg_size > eom_ptr) - HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "corrupt object header") + +/*------------------------------------------------------------------------- + * Function: H5O_cache_image_len + * + * Purpose: Tell the metadata cache about the actual size of the object + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Jul 12 2008 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_cache_image_len(const void *thing, size_t *image_len_ptr) +{ + const H5O_t *oh = (const H5O_t *)thing; /* The object header */ -#ifndef NDEBUG - /* Increment count of null messages */ - if(H5O_NULL_ID == id) - nullcnt++; -#endif /* NDEBUG */ + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_cache_image_len) - /* Check for combining two adjacent 'null' messages */ - if((H5F_get_intent(f) & H5F_ACC_RDWR) && - H5O_NULL_ID == id && oh->nmesgs > 0 && - H5O_NULL_ID == oh->mesg[oh->nmesgs - 1].type->id && - oh->mesg[oh->nmesgs - 1].chunkno == chunkno) { - - /* Combine adjacent null messages */ - mesgno = oh->nmesgs - 1; - oh->mesg[mesgno].raw_size += H5O_SIZEOF_MSGHDR_OH(oh) + mesg_size; - oh->mesg[mesgno].dirty = TRUE; - merged_null_msgs++; - } /* end if */ - else { - /* Check if we need to extend message table to hold the new message */ - if(oh->nmesgs >= oh->alloc_nmesgs) - if(H5O_alloc_msgs(oh, (size_t)1) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "can't allocate more space for messages") - - /* Get index for message */ - mesgno = oh->nmesgs++; - - /* Initialize information about message */ - oh->mesg[mesgno].dirty = FALSE; - oh->mesg[mesgno].flags = flags; - oh->mesg[mesgno].crt_idx = crt_idx; - oh->mesg[mesgno].native = NULL; - oh->mesg[mesgno].raw = (uint8_t *)p; /* Casting away const OK - QAK */ - oh->mesg[mesgno].raw_size = mesg_size; - oh->mesg[mesgno].chunkno = chunkno; - - /* Point unknown messages at 'unknown' message class */ - /* (Usually from future versions of the library) */ - if(id >= NELMTS(H5O_msg_class_g) || NULL == H5O_msg_class_g[id]) { - H5O_unknown_t *unknown; /* Pointer to "unknown" message info */ - - /* Allocate "unknown" message info */ - if(NULL == (unknown = H5FL_MALLOC(H5O_unknown_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* Save the original message type ID */ - *unknown = id; - - /* Save 'native' form of continuation message */ - oh->mesg[mesgno].native = unknown; - - /* Set message to "unknown" class */ - oh->mesg[mesgno].type = H5O_msg_class_g[H5O_UNKNOWN_ID]; - - /* Check for "fail if unknown" message flag */ - if(flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN) - HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, NULL, "unknown message with 'fail if unknown' flag found") - /* Check for "mark if unknown" message flag, etc. */ - else if((flags & H5O_MSG_FLAG_MARK_IF_UNKNOWN) && - !(flags & H5O_MSG_FLAG_WAS_UNKNOWN) && - (H5F_get_intent(f) & H5F_ACC_RDWR)) { - - /* Mark the message as "unknown" */ - /* This is a bit aggressive, since the application may - * never change anything about the object (metadata or - * raw data), but we can sort out the finer details - * when/if we start using the flag - QAK - */ - /* Also, it's possible that this functionality may not - * get invoked if the object header is brought into - * the metadata cache in some other "weird" way, like - * using H5Ocopy() - QAK - */ - oh->mesg[mesgno].flags |= H5O_MSG_FLAG_WAS_UNKNOWN; - - /* Mark the message and object header as dirty */ - oh->mesg[mesgno].dirty = TRUE; - oh->cache_info.is_dirty = TRUE; - } /* end if */ - } /* end if */ - else - /* Set message class for "known" messages */ - oh->mesg[mesgno].type = H5O_msg_class_g[id]; - } /* end else */ + /* Check arguments */ + HDassert(oh); + HDassert(image_len_ptr); - /* Advance decode pointer past message */ - p += mesg_size; + /* Report the object header's prefix+first chunk length */ + *image_len_ptr = H5O_SIZEOF_HDR(oh) + oh->chunk0_size; - /* Check for 'gap' at end of chunk */ - if((eom_ptr - p) > 0 && (eom_ptr - p) < H5O_SIZEOF_MSGHDR_OH(oh)) { - /* Gaps can only occur in later versions of the format */ - HDassert(oh->version > H5O_VERSION_1); + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5O_cache_image_len() */ - /* Gaps should only occur in chunks with no null messages */ - HDassert(nullcnt == 0); + +/*------------------------------------------------------------------------- + * Function: H5O_cache_serialize + * + * Purpose: Serializes an object header prefix+first chunk + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Jul 12 2008 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_cache_serialize(const H5F_t *f, haddr_t UNUSED addr, size_t UNUSED len, + void *image, void *_thing, unsigned *flags, haddr_t UNUSED *new_addr, + size_t UNUSED *new_len, void UNUSED **new_image) +{ + H5O_t *oh = (H5O_t *)_thing; /* Pointer to the object header */ + uint8_t *p; /* Pointer into raw data buffer */ + herr_t ret_value = SUCCEED; /* Return value */ - /* Set gap information for chunk */ - oh->chunk[chunkno].gap = (eom_ptr - p); + FUNC_ENTER_NOAPI(H5O_cache_serialize, FAIL) - /* Increment location in chunk */ - p += oh->chunk[chunkno].gap; - } /* end if */ - } /* end while */ + /* check arguments */ + HDassert(f); + HDassert(image); + HDassert(oh); + HDassert(flags); + + /* Get temporary pointer to chunk zero's buffer */ + p = oh->chunk[0].image; - /* Check for correct checksum on chunks, in later versions of the format */ - if(oh->version > H5O_VERSION_1) { - uint32_t stored_chksum; /* Checksum from file */ - uint32_t computed_chksum; /* Checksum computed in memory */ + /* Later versions of object header prefix have different format and + * also require that chunk 0 always be updated, since the checksum + * on the entire block of memory needs to be updated if anything is + * modified */ + if(oh->version > H5O_VERSION_1) { + uint64_t chunk0_size = oh->chunk[0].size - H5O_SIZEOF_HDR(oh); /* Size of chunk 0's data */ + + /* Magic number */ + HDmemcpy(p, H5O_HDR_MAGIC, (size_t)H5O_SIZEOF_MAGIC); + p += H5O_SIZEOF_MAGIC; - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Version */ + *p++ = oh->version; - /* Compute checksum on chunk */ - computed_chksum = H5_checksum_metadata(oh->chunk[chunkno].image, (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM), 0); + /* Flags */ + *p++ = oh->flags; - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "incorrect metadata checksum for object header chunk") + /* Time fields */ + if(oh->flags & H5O_HDR_STORE_TIMES) { + UINT32ENCODE(p, oh->atime); + UINT32ENCODE(p, oh->mtime); + UINT32ENCODE(p, oh->ctime); + UINT32ENCODE(p, oh->btime); } /* end if */ - /* Sanity check */ - HDassert(p == oh->chunk[chunkno].image + oh->chunk[chunkno].size); + /* Attribute fields */ + if(oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { + UINT16ENCODE(p, oh->max_compact); + UINT16ENCODE(p, oh->min_dense); + } /* end if */ - /* Check for another chunk to read in & parse */ - for(chunk_addr = HADDR_UNDEF; !H5F_addr_defined(chunk_addr) && curmesg < oh->nmesgs; ++curmesg) { - /* Check if next message to examine is a continuation message */ - if(H5O_CONT_ID == oh->mesg[curmesg].type->id) { - H5O_cont_t *cont; + /* First chunk size */ + switch(oh->flags & H5O_HDR_CHUNK0_SIZE) { + case 0: /* 1 byte size */ + HDassert(chunk0_size < 256); + *p++ = chunk0_size; + break; - /* Decode continuation message */ - cont = (H5O_cont_t *)(H5O_MSG_CONT->decode)(f, dxpl_id, 0, oh->mesg[curmesg].raw); - cont->chunkno = oh->nchunks; /*the next chunk to allocate */ + case 1: /* 2 byte size */ + HDassert(chunk0_size < 65536); + UINT16ENCODE(p, chunk0_size); + break; - /* Save 'native' form of continuation message */ - oh->mesg[curmesg].native = cont; + case 2: /* 4 byte size */ + /* use <= 2**32 -1 to stay within 4 bytes integer range */ + HDassert(chunk0_size <= 4294967295UL); + UINT32ENCODE(p, chunk0_size); + break; - /* Set up to read in next chunk */ - chunk_addr = cont->addr; - chunk_size = cont->size; - } /* end if */ - /* Check if next message to examine is a ref. count message */ - else if(H5O_REFCOUNT_ID == oh->mesg[curmesg].type->id) { - H5O_refcount_t *refcount; + case 3: /* 8 byte size */ + UINT64ENCODE(p, chunk0_size); + break; + + default: + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") + } /* end switch */ + } /* end if */ + else { + /* Version */ + *p++ = oh->version; - /* Decode ref. count message */ - HDassert(oh->version > H5O_VERSION_1); - refcount = (H5O_refcount_t *)(H5O_MSG_REFCOUNT->decode)(f, dxpl_id, 0, oh->mesg[curmesg].raw); + /* Reserved */ + *p++ = 0; - /* Save 'native' form of ref. count message */ - oh->mesg[curmesg].native = refcount; + /* Number of messages */ +#ifdef H5O_ENABLE_BAD_MESG_COUNT + if(oh->store_bad_mesg_count) + UINT16ENCODE(p, (oh->nmesgs - 1)) + else +#endif /* H5O_ENABLE_BAD_MESG_COUNT */ + UINT16ENCODE(p, oh->nmesgs); - /* Set object header values */ - oh->has_refcount_msg = TRUE; - oh->nlink = *refcount; - } /* end if */ - /* Check if next message to examine is a link message */ - else if(H5O_LINK_ID == oh->mesg[curmesg].type->id) { - /* Increment the count of link messages */ - oh->link_msgs_seen++; - } /* end if */ - /* Check if next message to examine is an attribute message */ - else if(H5O_ATTR_ID == oh->mesg[curmesg].type->id) { - /* Increment the count of attribute messages */ - oh->attr_msgs_seen++; - } /* end if */ - } /* end for */ - } /* end while */ + /* Link count */ + UINT32ENCODE(p, oh->nlink); + + /* First chunk size */ + UINT32ENCODE(p, (oh->chunk[0].size - H5O_SIZEOF_HDR(oh))); + + /* Zero to alignment */ + HDmemset(p, 0, (size_t)(H5O_SIZEOF_HDR(oh) - 12)); + p += (size_t)(H5O_SIZEOF_HDR(oh) - 12); + } /* end else */ + HDassert((size_t)(p - (uint8_t *)oh->chunk[0].image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); - /* Mark the object header dirty if we've merged a message */ - if(merged_null_msgs) - oh->cache_info.is_dirty = TRUE; + /* Serialize messages for this chunk */ + if(H5O_chunk_serialize(f, oh, (unsigned)0, image) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "unable to serialize first object header chunk") + + /* Reset the cache flags for this operation */ + *flags = 0; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_cache_serialize() */ -/* Don't check for the incorrect # of object header messages bug unless we've - * enabled strict format checking. This allows for older files, created with - * a version of the library that had a bug in tracking the correct # of header - * messages to be read in without the library fussing about things. -QAK + +/*------------------------------------------------------------------------- + * Function: H5O_cache_free_icr + * + * Purpose: Destroy/release an "in core representation" of a data + * structure + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * July 12, 2008 + * + *------------------------------------------------------------------------- */ -#ifdef H5_STRICT_FORMAT_CHECKS - /* Sanity check for the correct # of messages in object header */ - if(oh->version == H5O_VERSION_1) - if((oh->nmesgs + merged_null_msgs) != nmesgs) - HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "corrupt object header - too few messages") -#else /* H5_STRICT_FORMAT_CHECKS */ - /* Check for incorrect # of messages in object header and if we have write - * access on the file, flag the object header as dirty, so it gets fixed. - */ - if(oh->version == H5O_VERSION_1) - if((oh->nmesgs + merged_null_msgs) != nmesgs && - (H5F_get_intent(f) & H5F_ACC_RDWR)) - oh->cache_info.is_dirty = TRUE; -#endif /* H5_STRICT_FORMAT_CHECKS */ +static herr_t +H5O_cache_free_icr(haddr_t UNUSED addr, size_t UNUSED len, void *thing) +{ + H5O_t *oh = (H5O_t *)thing; /* Object header to destroy */ + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_cache_free_icr) + + /* Check arguments */ + HDassert(oh); + HDassert(oh->rc == 0); + + /* Destroy object header */ + H5O_dest(oh); -#ifdef H5O_DEBUG -H5O_assert(oh); -#endif /* H5O_DEBUG */ + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5O_cache_free_icr() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_cache_chk_deserialize + * + * Purpose: Deserializes an object header continuation chunk + * + * Return: Success: Pointer to a new chunk proxy + * Failure: NULL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Jul 12 2008 + * + *------------------------------------------------------------------------- + */ +static void * +H5O_cache_chk_deserialize(haddr_t addr, size_t len, const void *image, + void *_udata, hbool_t *dirty) +{ + H5O_chunk_proxy_t *chk_proxy = NULL; /* Chunk proxy object */ + H5O_chk_cache_ud_t *udata = (H5O_chk_cache_ud_t *)_udata; /* User data for callback */ + H5O_chunk_proxy_t *ret_value; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5O_cache_chk_deserialize) + + /* Check arguments */ + HDassert(H5F_addr_defined(addr)); + HDassert(len > 0); + HDassert(image); + HDassert(udata); + HDassert(udata->oh); + + /* Allocate space for the object header data structure */ + if(NULL == (chk_proxy = H5FL_CALLOC(H5O_chunk_proxy_t))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, NULL, "memory allocation failed") + + /* Check if we are still decoding the object header */ + if(udata->decoding) { + /* Sanity check */ + HDassert(udata->common.f); + HDassert(udata->common.cont_msg_info); + + /* Parse the chunk */ + if(H5O_chunk_deserialize(udata->oh, addr, len, image, &(udata->common), dirty) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize object header chunk") + + /* Set the fields for the chunk proxy */ + chk_proxy->oh = udata->oh; + chk_proxy->chunkno = udata->oh->nchunks - 1; + } /* end if */ + else { + /* Sanity check */ + HDassert(udata->chunkno < udata->oh->nchunks); + + /* Set the fields for the chunk proxy */ + chk_proxy->oh = udata->oh; + chk_proxy->chunkno = udata->chunkno; + + /* Sanity check that the chunk representation we have in memory is the same + * as the one being brought in from disk. + */ + HDassert(0 == HDmemcmp(image, chk_proxy->oh->chunk[chk_proxy->chunkno].image, chk_proxy->oh->chunk[chk_proxy->chunkno].size)); + } /* end else */ + + /* Increment reference count of object header */ + if(H5O_inc_rc(udata->oh) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINC, NULL, "can't increment reference count on object header") /* Set return value */ - ret_value = oh; + ret_value = chk_proxy; done: /* Release the [possibly partially initialized] object header on errors */ - if(!ret_value && oh) - if(H5O_dest(f, oh) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CANTFREE, NULL, "unable to destroy object header data") + if(!ret_value && chk_proxy) + if(H5O_chunk_proxy_dest(chk_proxy) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, NULL, "unable to destroy object header chunk proxy") FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O_load() */ +} /* end H5O_cache_chk_deserialize() */ /*------------------------------------------------------------------------- - * Function: H5O_flush + * Function: H5O_cache_chk_serialize * - * Purpose: Flushes (and destroys) an object header. + * Purpose: Serializes an object header chunk * * Return: Non-negative on success/Negative on failure * - * Programmer: Robb Matzke - * matzke@llnl.gov - * Aug 5 1997 - * - * Changes: JRM -- 8/21/06 - * Added the flags_ptr parameter. This parameter exists to - * allow the flush routine to report to the cache if the - * entry is resized or renamed as a result of the flush. - * *flags_ptr is set to H5C_CALLBACK__NO_FLAGS_SET on entry. + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * Jul 12 2008 * *------------------------------------------------------------------------- */ static herr_t -H5O_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t UNUSED addr, H5O_t *oh, unsigned UNUSED * flags_ptr) +H5O_cache_chk_serialize(const H5F_t *f, haddr_t UNUSED addr, size_t UNUSED len, + void *image, void *_thing, unsigned *flags, haddr_t UNUSED *new_addr, + size_t UNUSED *new_len, void UNUSED **new_image) { - herr_t ret_value = SUCCEED; /* Return value */ + H5O_chunk_proxy_t *chk_proxy = (H5O_chunk_proxy_t *)_thing; /* Pointer to the object header chunk proxy */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI(H5O_flush, FAIL) + FUNC_ENTER_NOAPI(H5O_cache_chk_serialize, FAIL) - /* check args */ + /* check arguments */ HDassert(f); - HDassert(H5F_addr_defined(addr)); - HDassert(oh); + HDassert(image); + HDassert(chk_proxy); + HDassert(flags); - /* flush */ - if(oh->cache_info.is_dirty) { - uint8_t *p; /* Pointer to object header prefix buffer */ - unsigned u; /* Local index variable */ - -#ifdef H5O_DEBUG -H5O_assert(oh); -#endif /* H5O_DEBUG */ - - /* Point to raw data 'image' for first chunk, which has room for the prefix */ - p = oh->chunk[0].image; - - /* Later versions of object header prefix have different format and - * also require that chunk 0 always be updated, since the checksum - * on the entire block of memory needs to be updated if anything is - * modified */ - if(oh->version > H5O_VERSION_1) { - uint64_t chunk0_size = oh->chunk[0].size - H5O_SIZEOF_HDR(oh); /* Size of chunk 0's data */ - - /* Verify magic number */ - HDassert(!HDmemcmp(p, H5O_HDR_MAGIC, H5O_SIZEOF_MAGIC)); - p += H5O_SIZEOF_MAGIC; - - /* Version */ - *p++ = oh->version; - - /* Flags */ - *p++ = oh->flags; - - /* Time fields */ - if(oh->flags & H5O_HDR_STORE_TIMES) { - UINT32ENCODE(p, oh->atime); - UINT32ENCODE(p, oh->mtime); - UINT32ENCODE(p, oh->ctime); - UINT32ENCODE(p, oh->btime); - } /* end if */ + /* Serialize messages for this chunk */ + if(H5O_chunk_serialize(f, chk_proxy->oh, chk_proxy->chunkno, image) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "unable to serialize object header chunk") - /* Attribute fields */ - if(oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { - UINT16ENCODE(p, oh->max_compact); - UINT16ENCODE(p, oh->min_dense); - } /* end if */ + /* Reset the cache flags for this operation */ + *flags = 0; - /* First chunk size */ - switch(oh->flags & H5O_HDR_CHUNK0_SIZE) { - case 0: /* 1 byte size */ - HDassert(chunk0_size < 256); - *p++ = chunk0_size; - break; - - case 1: /* 2 byte size */ - HDassert(chunk0_size < 65536); - UINT16ENCODE(p, chunk0_size); - break; - - case 2: /* 4 byte size */ - /* use <= 2**32 -1 to stay within 4 bytes integer range */ - HDassert(chunk0_size <= 4294967295UL); - UINT32ENCODE(p, chunk0_size); - break; - - case 3: /* 8 byte size */ - UINT64ENCODE(p, chunk0_size); - break; - - default: - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") - } /* end switch */ - } /* end if */ - else { - /* Version */ - *p++ = oh->version; +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_cache_chk_serialize() */ - /* Reserved */ - *p++ = 0; + +/*------------------------------------------------------------------------- + * Function: H5O_cache_chk_free_icr + * + * Purpose: Destroy/release an "in core representation" of a data + * structure + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * July 12, 2008 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_cache_chk_free_icr(haddr_t UNUSED addr, size_t UNUSED len, void *thing) +{ + H5O_chunk_proxy_t *chk_proxy = (H5O_chunk_proxy_t *)thing; /* Object header chunk proxy to destroy */ - /* Number of messages */ -#ifdef H5O_ENABLE_BAD_MESG_COUNT - if(oh->store_bad_mesg_count) - UINT16ENCODE(p, (oh->nmesgs - 1)) - else -#endif /* H5O_ENABLE_BAD_MESG_COUNT */ - UINT16ENCODE(p, oh->nmesgs); + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_cache_chk_free_icr) - /* Link count */ - UINT32ENCODE(p, oh->nlink); + /* Check arguments */ + HDassert(chk_proxy); - /* First chunk size */ - UINT32ENCODE(p, (oh->chunk[0].size - H5O_SIZEOF_HDR(oh))); + /* Destroy object header chunk proxy */ + H5O_chunk_proxy_dest(chk_proxy); - /* Zero to alignment */ - HDmemset(p, 0, (size_t)(H5O_SIZEOF_HDR(oh) - 12)); - p += (size_t)(H5O_SIZEOF_HDR(oh) - 12); - } /* end else */ - HDassert((size_t)(p - oh->chunk[0].image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5O_cache_chk_free_icr() */ - /* Mark chunk 0 as dirty, since the object header prefix has been updated */ - /* (this could be more sophisticated and track whether any prefix fields - * have been changed, which could save I/O accesses if the - * messages in chunk 0 haven't changed - QAK) - */ - HDassert(H5F_addr_eq(addr, oh->chunk[0].addr)); - oh->chunk[0].dirty = TRUE; - - /* Encode any dirty messages */ - if(H5O_flush_msgs(f, oh) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTFLUSH, FAIL, "unable to flush object header messages") - - /* Write each chunk to disk, if it's dirty */ - for(u = 0; u < oh->nchunks; u++) { - /* Sanity checks */ - if(oh->version > H5O_VERSION_1) - /* Make certain the magic # is present */ - HDassert(!HDmemcmp(oh->chunk[u].image, (u == 0 ? H5O_HDR_MAGIC : H5O_CHK_MAGIC), H5O_SIZEOF_MAGIC)); - else - /* Gaps should never occur in version 1 of the format */ - HDassert(oh->chunk[u].gap == 0); - - /* Write out chunk, if it's dirty */ - if(oh->chunk[u].dirty) { - /* Extra work, for later versions of the format */ - if(oh->version > H5O_VERSION_1) { - uint32_t metadata_chksum; /* Computed metadata checksum value */ - - /* Check for gap in chunk & zero it out */ - if(oh->chunk[u].gap) - HDmemset((oh->chunk[u].image + oh->chunk[u].size) - - (H5O_SIZEOF_CHKSUM + oh->chunk[u].gap), 0, oh->chunk[u].gap); - - /* Compute metadata checksum */ - metadata_chksum = H5_checksum_metadata(oh->chunk[u].image, (oh->chunk[u].size - H5O_SIZEOF_CHKSUM), 0); - - /* Metadata checksum */ - p = oh->chunk[u].image + (oh->chunk[u].size - H5O_SIZEOF_CHKSUM); - UINT32ENCODE(p, metadata_chksum); - } /* end if */ + +/*------------------------------------------------------------------------- + * Function: H5O_add_cont_msg + * + * Purpose: Add information from a continuation message to the list of + * continuation messages in the object header + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * July 12, 2008 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_add_cont_msg(H5O_cont_msgs_t *cont_msg_info, const H5O_cont_t *cont) +{ + unsigned contno; /* Continuation message index */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5O_add_cont_msg) - /* Write the chunk out */ - HDassert(H5F_addr_defined(oh->chunk[u].addr)); - if(H5F_block_write(f, H5FD_MEM_OHDR, oh->chunk[u].addr, - oh->chunk[u].size, dxpl_id, oh->chunk[u].image) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to write object header chunk to disk") + /* Check arguments */ + HDassert(cont_msg_info); + HDassert(cont); - /* Mark chunk as clean now */ - oh->chunk[u].dirty = FALSE; - } /* end if */ - } /* end for */ + /* Increase chunk array size, if necessary */ + if(cont_msg_info->nmsgs >= cont_msg_info->alloc_nmsgs) { + unsigned na = MAX(H5O_NCHUNKS, cont_msg_info->alloc_nmsgs * 2); /* Double # of messages allocated */ + H5O_cont_t *x = H5FL_SEQ_REALLOC(H5O_cont_t, cont_msg_info->msgs, (size_t)na); - /* Mark object header as clean now */ - oh->cache_info.is_dirty = FALSE; + if(!x) + HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, FAIL, "memory allocation failed") + cont_msg_info->alloc_nmsgs = na; + cont_msg_info->msgs = x; } /* end if */ - /* Destroy the object header, if requested */ - if(destroy) - if(H5O_dest(f,oh) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to destroy object header data") + /* Init the continuation message info */ + contno = cont_msg_info->nmsgs++; + cont_msg_info->msgs[contno].addr = cont->addr; + cont_msg_info->msgs[contno].size = cont->size; + cont_msg_info->msgs[contno].chunkno = cont->chunkno; done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O_flush() */ +} /* H5O_add_cont_msg() */ /*------------------------------------------------------------------------- - * Function: H5O_dest + * Function: H5O_chunk_deserialize * - * Purpose: Destroys an object header. + * Purpose: Deserialize a chunk for an object header * - * Return: Non-negative on success/Negative on failure + * Return: Success: SUCCEED + * Failure: FAIL * * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Jan 15 2003 + * koziol@hdfgroup.org + * July 12, 2008 * *------------------------------------------------------------------------- */ -herr_t -H5O_dest(H5F_t UNUSED *f, H5O_t *oh) +static herr_t +H5O_chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image, + H5O_common_cache_ud_t *udata, hbool_t *dirty) { - unsigned u; /* Local index variable */ + const uint8_t *p; /* Pointer into buffer to decode */ + uint8_t *eom_ptr; /* Pointer to end of messages for a chunk */ + unsigned curmesg; /* Current message being decoded in object header */ + unsigned merged_null_msgs = 0; /* Number of null messages merged together */ + unsigned chunkno; /* Current chunk's index */ +#ifndef NDEBUG + unsigned nullcnt; /* Count of null messages (for sanity checking gaps in chunks) */ +#endif /* NDEBUG */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_dest) + FUNC_ENTER_NOAPI_NOINIT(H5O_chunk_deserialize) - /* check args */ + /* Check arguments */ HDassert(oh); + HDassert(image); + HDassert(udata->f); + HDassert(udata->cont_msg_info); + + /* Increase chunk array size, if necessary */ + if(oh->nchunks >= oh->alloc_nchunks) { + unsigned na = MAX(H5O_NCHUNKS, oh->alloc_nchunks * 2); /* Double # of chunks allocated */ + H5O_chunk_t *x = H5FL_SEQ_REALLOC(H5O_chunk_t, oh->chunk, (size_t)na); + + if(!x) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") + oh->alloc_nchunks = na; + oh->chunk = x; + } /* end if */ - /* Verify that node is clean */ - HDassert(oh->cache_info.is_dirty == FALSE); + /* Init the chunk data info */ + chunkno = oh->nchunks++; + oh->chunk[chunkno].gap = 0; + if(chunkno == 0) { + /* First chunk's 'image' includes room for the object header prefix */ + oh->chunk[0].addr = addr; + oh->chunk[0].size = len + H5O_SIZEOF_HDR(oh); + } /* end if */ + else { + oh->chunk[chunkno].addr = addr; + oh->chunk[chunkno].size = len; + } /* end else */ + if(NULL == (oh->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, oh->chunk[chunkno].size))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") - /* destroy chunks */ - if(oh->chunk) { - for(u = 0; u < oh->nchunks; u++) { - /* Verify that chunk is clean */ - HDassert(oh->chunk[u].dirty == 0); + /* Copy disk image into chunk's image */ + HDmemcpy(oh->chunk[chunkno].image, image, oh->chunk[chunkno].size); - oh->chunk[u].image = H5FL_BLK_FREE(chunk_image, oh->chunk[u].image); - } /* end for */ + /* Point into chunk image to decode */ + p = oh->chunk[chunkno].image; - oh->chunk = (H5O_chunk_t *)H5FL_SEQ_FREE(H5O_chunk_t, oh->chunk); + /* Handle chunk 0 as special case */ + if(chunkno == 0) + /* Skip over [already decoded] prefix */ + p += (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh)); + /* Check for magic # on chunks > 0 in later versions of the format */ + else if(chunkno > 0 && oh->version > H5O_VERSION_1) { + /* Magic number */ + if(HDmemcmp(p, H5O_CHK_MAGIC, (size_t)H5O_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "wrong object header chunk signature") + p += H5O_SIZEOF_MAGIC; } /* end if */ - /* destroy messages */ - if(oh->mesg) { - for(u = 0; u < oh->nmesgs; u++) { - /* Verify that message is clean */ - HDassert(oh->mesg[u].dirty == 0); + /* Save # of messages already inspected */ + curmesg = oh->nmesgs; + + /* Decode messages from this chunk */ + eom_ptr = oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM_OH(oh)); +#ifndef NDEBUG + nullcnt = 0; +#endif /* NDEBUG */ + while(p < eom_ptr) { + unsigned mesgno; /* Current message to operate on */ + size_t mesg_size; /* Size of message read in */ + unsigned id; /* ID (type) of current message */ + uint8_t flags; /* Flags for current message */ + H5O_msg_crt_idx_t crt_idx = 0; /* Creation index for current message */ + + /* Decode message prefix info */ + + /* Version # */ + if(oh->version == H5O_VERSION_1) + UINT16DECODE(p, id) + else + id = *p++; + + /* Check for unknown message ID getting encoded in file */ + if(id == H5O_UNKNOWN_ID) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "'unknown' message ID encoded in file?!?") + + /* Message size */ + UINT16DECODE(p, mesg_size); + HDassert(mesg_size == H5O_ALIGN_OH(oh, mesg_size)); + + /* Message flags */ + flags = *p++; + if(flags & ~H5O_MSG_FLAG_BITS) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "unknown flag for message") + if((flags & H5O_MSG_FLAG_SHARED) && (flags & H5O_MSG_FLAG_DONTSHARE)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "bad flag combination for message") + if((flags & H5O_MSG_FLAG_WAS_UNKNOWN) && (flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "bad flag combination for message") + if((flags & H5O_MSG_FLAG_WAS_UNKNOWN) && !(flags & H5O_MSG_FLAG_MARK_IF_UNKNOWN)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "bad flag combination for message") + + /* Reserved bytes/creation index */ + if(oh->version == H5O_VERSION_1) + p += 3; /*reserved*/ + else { + /* Only decode creation index if they are being tracked */ + if(oh->flags & H5O_HDR_ATTR_CRT_ORDER_TRACKED) + UINT16DECODE(p, crt_idx); + } /* end else */ + + /* Try to detect invalidly formatted object header message that + * extends past end of chunk. + */ + if(p + mesg_size > eom_ptr) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "corrupt object header") + +#ifndef NDEBUG + /* Increment count of null messages */ + if(H5O_NULL_ID == id) + nullcnt++; +#endif /* NDEBUG */ + + /* Check for combining two adjacent 'null' messages */ + if((udata->file_intent & H5F_ACC_RDWR) && + H5O_NULL_ID == id && oh->nmesgs > 0 && + H5O_NULL_ID == oh->mesg[oh->nmesgs - 1].type->id && + oh->mesg[oh->nmesgs - 1].chunkno == chunkno) { + + /* Combine adjacent null messages */ + mesgno = oh->nmesgs - 1; + oh->mesg[mesgno].raw_size += H5O_SIZEOF_MSGHDR_OH(oh) + mesg_size; + oh->mesg[mesgno].dirty = TRUE; + merged_null_msgs++; + udata->merged_null_msgs++; + } /* end if */ + else { + /* Check if we need to extend message table to hold the new message */ + if(oh->nmesgs >= oh->alloc_nmesgs) + if(H5O_alloc_msgs(oh, (size_t)1) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate more space for messages") + + /* Get index for message */ + mesgno = oh->nmesgs++; + + /* Initialize information about message */ + oh->mesg[mesgno].dirty = FALSE; + oh->mesg[mesgno].flags = flags; + oh->mesg[mesgno].crt_idx = crt_idx; + oh->mesg[mesgno].native = NULL; + oh->mesg[mesgno].raw = (uint8_t *)p; /* Casting away const OK - QAK */ + oh->mesg[mesgno].raw_size = mesg_size; + oh->mesg[mesgno].chunkno = chunkno; + + /* Point unknown messages at 'unknown' message class */ + /* (Usually from future versions of the library) */ + if(id >= NELMTS(H5O_msg_class_g) || NULL == H5O_msg_class_g[id]) { + H5O_unknown_t *unknown; /* Pointer to "unknown" message info */ + + /* Allocate "unknown" message info */ + if(NULL == (unknown = H5FL_MALLOC(H5O_unknown_t))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") + + /* Save the original message type ID */ + *unknown = id; + + /* Save 'native' form of unknown message */ + oh->mesg[mesgno].native = unknown; + + /* Set message to "unknown" class */ + oh->mesg[mesgno].type = H5O_msg_class_g[H5O_UNKNOWN_ID]; + + /* Check for "fail if unknown" message flag */ + if(flags & H5O_MSG_FLAG_FAIL_IF_UNKNOWN) + HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, FAIL, "unknown message with 'fail if unknown' flag found") + /* Check for "mark if unknown" message flag, etc. */ + else if((flags & H5O_MSG_FLAG_MARK_IF_UNKNOWN) && + !(flags & H5O_MSG_FLAG_WAS_UNKNOWN) && + (udata->file_intent & H5F_ACC_RDWR)) { + + /* Mark the message as "unknown" */ + /* This is a bit aggressive, since the application may + * never change anything about the object (metadata or + * raw data), but we can sort out the finer details + * when/if we start using the flag - QAK + */ + /* Also, it's possible that this functionality may not + * get invoked if the object header is brought into + * the metadata cache in some other "weird" way, like + * using H5Ocopy() - QAK + */ + oh->mesg[mesgno].flags |= H5O_MSG_FLAG_WAS_UNKNOWN; + + /* Mark the message and chunk as dirty */ + oh->mesg[mesgno].dirty = TRUE; + udata->mesgs_modified = TRUE; + *dirty = TRUE; + } /* end if */ + } /* end if */ + else + /* Set message class for "known" messages */ + oh->mesg[mesgno].type = H5O_msg_class_g[id]; + } /* end else */ + + /* Advance decode pointer past message */ + p += mesg_size; + + /* Check for 'gap' at end of chunk */ + if((eom_ptr - p) > 0 && (eom_ptr - p) < H5O_SIZEOF_MSGHDR_OH(oh)) { + /* Gaps can only occur in later versions of the format */ + HDassert(oh->version > H5O_VERSION_1); + + /* Gaps should only occur in chunks with no null messages */ + HDassert(nullcnt == 0); + + /* Set gap information for chunk */ + oh->chunk[chunkno].gap = (eom_ptr - p); + + /* Increment location in chunk */ + p += oh->chunk[chunkno].gap; + } /* end if */ + } /* end while */ + + /* Check for correct checksum on chunks, in later versions of the format */ + if(oh->version > H5O_VERSION_1) { + uint32_t stored_chksum; /* Checksum from file */ + uint32_t computed_chksum; /* Checksum computed in memory */ + + /* Metadata checksum */ + UINT32DECODE(p, stored_chksum); - H5O_msg_free_mesg(&oh->mesg[u]); - } /* end for */ + /* Compute checksum on chunk */ + computed_chksum = H5_checksum_metadata(oh->chunk[chunkno].image, (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM), 0); - oh->mesg = (H5O_mesg_t *)H5FL_SEQ_FREE(H5O_mesg_t, oh->mesg); + /* Verify checksum */ + if(stored_chksum != computed_chksum) + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "incorrect metadata checksum for object header chunk") } /* end if */ - /* destroy object header */ - H5FL_FREE(H5O_t, oh); + /* Sanity check */ + HDassert(p == oh->chunk[chunkno].image + oh->chunk[chunkno].size); - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5O_dest() */ + /* Do some inspection/interpretation of new messages from this chunk */ + /* (detect continuation messages, ref. count messages, etc.) */ + while(curmesg < oh->nmesgs) { + /* Check if next message to examine is a continuation message */ + if(H5O_CONT_ID == oh->mesg[curmesg].type->id) { + H5O_cont_t *cont; + + /* Decode continuation message */ + cont = (H5O_cont_t *)(H5O_MSG_CONT->decode)(udata->f, udata->dxpl_id, 0, oh->mesg[curmesg].raw); + cont->chunkno = udata->cont_msg_info->nmsgs + 1; /*the next continuation message/chunk */ + + /* Save 'native' form of continuation message */ + oh->mesg[curmesg].native = cont; + + /* Add to continuation messages left to interpret */ + if(H5O_add_cont_msg(udata->cont_msg_info, cont) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add continuation message") + } /* end if */ + /* Check if next message to examine is a ref. count message */ + else if(H5O_REFCOUNT_ID == oh->mesg[curmesg].type->id) { + H5O_refcount_t *refcount; + + /* Decode ref. count message */ + HDassert(oh->version > H5O_VERSION_1); + refcount = (H5O_refcount_t *)(H5O_MSG_REFCOUNT->decode)(udata->f, udata->dxpl_id, 0, oh->mesg[curmesg].raw); + + /* Save 'native' form of ref. count message */ + oh->mesg[curmesg].native = refcount; + + /* Set object header values */ + oh->has_refcount_msg = TRUE; + oh->nlink = *refcount; + } /* end if */ + /* Check if next message to examine is a link message */ + else if(H5O_LINK_ID == oh->mesg[curmesg].type->id) { + /* Increment the count of link messages */ + oh->link_msgs_seen++; + } /* end if */ + /* Check if next message to examine is an attribute message */ + else if(H5O_ATTR_ID == oh->mesg[curmesg].type->id) { + /* Increment the count of attribute messages */ + oh->attr_msgs_seen++; + } /* end if */ + + /* Advance to next message */ + curmesg++; + } /* end while */ + + /* Mark the chunk dirty if we've merged null messages */ + if(merged_null_msgs) { + udata->mesgs_modified = TRUE; + *dirty = TRUE; + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_chunk_deserialize() */ /*------------------------------------------------------------------------- - * Function: H5O_clear + * Function: H5O_chunk_serialize * - * Purpose: Mark a object header in memory as non-dirty. + * Purpose: Serialize a chunk for an object header * - * Return: Non-negative on success/Negative on failure + * Return: Success: SUCCEED + * Failure: FAIL * * Programmer: Quincey Koziol - * koziol@ncsa.uiuc.edu - * Mar 20 2003 + * koziol@hdfgroup.org + * July 12, 2008 * *------------------------------------------------------------------------- */ static herr_t -H5O_clear(H5F_t *f, H5O_t *oh, hbool_t destroy) +H5O_chunk_serialize(const H5F_t *f, H5O_t *oh, unsigned chunkno, uint8_t *image) { - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; + H5O_mesg_t *curr_msg; /* Pointer to current message being operated on */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5O_clear) + FUNC_ENTER_NOAPI_NOINIT(H5O_chunk_serialize) - /* check args */ + /* Check arguments */ + HDassert(f); HDassert(oh); + HDassert(image); + + /* Encode any dirty messages in this chunk */ + for(u = 0, curr_msg = &oh->mesg[0]; u < oh->nmesgs; u++, curr_msg++) + if(curr_msg->dirty && curr_msg->chunkno == chunkno) + /* Casting away const OK -QAK */ + if(H5O_msg_flush((H5F_t *)f, oh, curr_msg) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "unable to encode object header message") + + /* Extra work, for later versions of the format */ + if(oh->version > H5O_VERSION_1) { + uint32_t metadata_chksum; /* Computed metadata checksum value */ + uint8_t *p; /* Pointer into object header chunk */ + + /* Check for gap in chunk & zero it out */ + if(oh->chunk[chunkno].gap) + HDmemset((oh->chunk[chunkno].image + oh->chunk[chunkno].size) - + (H5O_SIZEOF_CHKSUM + oh->chunk[chunkno].gap), 0, oh->chunk[chunkno].gap); + + /* Compute metadata checksum */ + metadata_chksum = H5_checksum_metadata(oh->chunk[chunkno].image, (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM), 0); + + /* Metadata checksum */ + p = oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM); + UINT32ENCODE(p, metadata_chksum); + } /* end if */ - /* Mark chunks as clean */ - for(u = 0; u < oh->nchunks; u++) - oh->chunk[u].dirty = FALSE; - - /* Mark messages as clean */ - for(u = 0; u < oh->nmesgs; u++) - oh->mesg[u].dirty = FALSE; - - /* Mark whole header as clean */ - oh->cache_info.is_dirty = FALSE; - - if(destroy) - if(H5O_dest(f, oh) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTFREE, FAIL, "unable to destroy object header data") + /* Copy the chunk's image into the cache's image */ + HDmemcpy(image, oh->chunk[chunkno].image, oh->chunk[chunkno].size); done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O_clear() */ +} /* H5O_chunk_serialize() */ /*------------------------------------------------------------------------- - * Function: H5O_size + * Function: H5O_chunk_proxy_dest * - * Purpose: Compute the size in bytes of the specified instance of - * H5O_t on disk, and return it in *len_ptr. On failure, - * the value of *len_ptr is undefined. + * Purpose: Destroy a chunk proxy object * - * Return: Non-negative on success/Negative on failure + * Return: Success: SUCCEED + * Failure: FAIL * - * Programmer: John Mainzer - * 5/13/04 + * Programmer: Quincey Koziol + * koziol@hdfgroup.org + * July 13, 2008 * *------------------------------------------------------------------------- */ static herr_t -H5O_size(const H5F_t UNUSED *f, const H5O_t *oh, size_t *size_ptr) +H5O_chunk_proxy_dest(H5O_chunk_proxy_t *chk_proxy) { - size_t size; /* Running sum of the object header's size */ - unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5O_size) + FUNC_ENTER_NOAPI_NOINIT(H5O_chunk_proxy_dest) - /* check args */ - HDassert(oh); - HDassert(size_ptr); + /* Check arguments */ + HDassert(chk_proxy); - /* Add sizes of all the chunks */ - /* (includes size of prefix, in chunk 0) */ - for(u = 0, size = 0; u < oh->nchunks; u++) - size += oh->chunk[u].size; + /* Decrement reference count of object header */ + if(H5O_dec_rc(chk_proxy->oh) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTDEC, FAIL, "can't decrement reference count on object header") - *size_ptr = size; + /* Release the chunk proxy object */ + H5FL_FREE(H5O_chunk_proxy_t, chk_proxy); - FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5O_size() */ +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_chunk_proxy_dest() */ |