diff options
Diffstat (limited to 'src/H5O.c')
| -rw-r--r-- | src/H5O.c | 713 |
1 files changed, 593 insertions, 120 deletions
@@ -41,6 +41,8 @@ #include "H5MFprivate.h" /* File memory management */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ +#include "H5Tprivate.h" +#include "H5SMprivate.h" /* Shared object header messages */ #ifdef H5_HAVE_GETTIMEOFDAY #include <sys/time.h> @@ -70,6 +72,7 @@ HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, ERR, "unable to decode message") \ } /* end if */ +/* Private typedefs */ /******************/ /* Local Typedefs */ @@ -170,8 +173,6 @@ static herr_t H5O_iterate_real(const H5O_loc_t *loc, const H5O_msg_class_t *type static const H5O_obj_class_t *H5O_obj_class(H5O_loc_t *loc, hid_t dxpl_id); static H5G_obj_t H5O_obj_type_real(H5O_t *oh); static const H5O_obj_class_t *H5O_obj_class_real(H5O_t *oh); -static void * H5O_copy_mesg_file(const H5O_msg_class_t *type, H5F_t *file_src, void *mesg_src, - H5F_t *file_dst, hid_t dxpl_id, H5O_copy_t *cpy_info, void *udata); static herr_t H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, hid_t dxpl_id, H5O_copy_t *cpy_info); static herr_t H5O_copy_free_addrmap_cb(void *item, void *key, void *op_data); @@ -249,7 +250,6 @@ H5FL_EXTERN(H5O_cont_t); /* Declare a free list to manage the H5O_addr_map_t struct */ H5FL_EXTERN(H5O_addr_map_t); - /*------------------------------------------------------------------------- * Function: H5Oopen @@ -1607,11 +1607,14 @@ done: * *------------------------------------------------------------------------- */ +/* JAMES: this will probably get put through its paces when extending shared + * dataspaces */ int H5O_modify(H5O_loc_t *loc, unsigned type_id, int overwrite, - unsigned flags, unsigned update_flags, const void *mesg, hid_t dxpl_id) + unsigned flags, unsigned update_flags, void *mesg, hid_t dxpl_id) { const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + htri_t shared_mess; int ret_value; /* Return value */ FUNC_ENTER_NOAPI(H5O_modify, FAIL) @@ -1626,6 +1629,15 @@ H5O_modify(H5O_loc_t *loc, unsigned type_id, int overwrite, HDassert(mesg); HDassert(0 == (flags & ~H5O_FLAG_BITS)); + /* Should this message be written as a SOHM? */ + if((shared_mess = H5SM_try_share(loc->file, dxpl_id, type_id, mesg)) >0) + { + /* Mark the message as shared */ + flags |= H5O_FLAG_SHARED; + + } else if(shared_mess < 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, FAIL, "error while trying to share message"); + /* Call the "real" modify routine */ if((ret_value = H5O_modify_real(loc, type, overwrite, flags, update_flags, mesg, dxpl_id)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to write object header message") @@ -1679,6 +1691,8 @@ H5O_modify_real(H5O_loc_t *loc, const H5O_msg_class_t *type, int overwrite, unsigned idx; /* Index of message to modify */ H5O_mesg_t *idx_msg; /* Pointer to message to modify */ H5O_shared_t sh_mesg; + const H5O_msg_class_t *write_type = type; /* Type of message to be written */ + const void *write_mesg = mesg; /* Actual message being written */ int ret_value; FUNC_ENTER_NOAPI_NOINIT(H5O_modify_real) @@ -1709,7 +1723,7 @@ H5O_modify_real(H5O_loc_t *loc, const H5O_msg_class_t *type, int overwrite, /* Was the right message found? */ if(overwrite >= 0 && (idx >= oh->nmesgs || sequence != overwrite)) { - /* But can we insert a new one with this sequence number? */ + /* No, but can we insert a new one with this sequence number? */ if(overwrite == sequence + 1) overwrite = -1; else @@ -1719,6 +1733,7 @@ H5O_modify_real(H5O_loc_t *loc, const H5O_msg_class_t *type, int overwrite, /* Check for creating new message */ if(overwrite < 0) { /* Create a new message */ + /* JAMES: why is sh_mesg passed in here? Is it ever used? */ if((idx = H5O_new_mesg(loc->file, oh, &flags, type, mesg, &sh_mesg, &type, &mesg, dxpl_id, &oh_flags)) == UFAIL) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to create new message") @@ -1728,11 +1743,41 @@ H5O_modify_real(H5O_loc_t *loc, const H5O_msg_class_t *type, int overwrite, } else if(oh->mesg[idx].flags & H5O_FLAG_CONSTANT) { HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to modify constant message") } else if(oh->mesg[idx].flags & H5O_FLAG_SHARED) { - HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to modify shared (constant) message") + /* This message is shared, but it's being modified. This is valid if + * it's shared in the heap . + * First, make sure it's not a committed message; these can't ever + * be modified. + */ + if(((H5O_shared_t*) oh->mesg[idx].native)->flags & H5O_COMMITTED_FLAG) + HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to modify committed message") + + /* Remove the old message from the SOHM index */ + if(H5SM_try_delete(loc->file, dxpl_id, oh->mesg[idx].type->id, oh->mesg[idx].native) < 0) + HGOTO_ERROR (H5E_OHDR, H5E_CANTFREE, FAIL, "unable to delete message from SOHM table") + + /* Now this message is no longer shared and we can safely overwrite + * it. + * We need to make sure that the message we're writing is shared, + * though, and that the library doesn't try to reset the current + * message like it would in a normal overwrite (this message is + * realy a shared pointer, not a real + * message). + * JAMES: will this break if a shared message is overwritten with a larger + * non-shared message? + */ + HDassert(H5O_is_shared(type->id, mesg) > 0); /* JAMES: this should work with + * replacement messages that aren't shared, too. */ + + if(H5O_get_share(type->id, loc->file, mesg, &sh_mesg)<0) + HGOTO_ERROR (H5E_OHDR, H5E_BADMESG, FAIL, "can't get shared message") + + /* Instead of writing the original message, write a shared message */ + write_type = H5O_msg_class_g[H5O_SHARED_ID]; + write_mesg = &sh_mesg; } /* Write the information to the message */ - if(H5O_write_mesg(oh, idx, type, mesg, flags, update_flags, &oh_flags) < 0) + if(H5O_write_mesg(oh, idx, write_type, write_mesg, flags, update_flags, &oh_flags) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to write message") /* Update the modification time message if any */ @@ -1854,9 +1899,10 @@ done: */ int H5O_append(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned type_id, unsigned flags, - const void *mesg, unsigned * oh_flags_ptr) + void *mesg, unsigned * oh_flags_ptr) { - const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + htri_t shared_mess; /* Should this message be stored in the Shared Message table? */ int ret_value; /* Return value */ FUNC_ENTER_NOAPI(H5O_append, FAIL) @@ -1871,7 +1917,15 @@ H5O_append(H5F_t *f, hid_t dxpl_id, H5O_t *oh, unsigned type_id, unsigned flags, HDassert(mesg); HDassert(oh_flags_ptr); - /* Call the "real" append routine */ + /* Should this message be written as a SOHM? */ + if((shared_mess = H5SM_try_share(f, dxpl_id, type_id, mesg)) >0) + { + /* Mark the message as shared */ + flags |= H5O_FLAG_SHARED; + } + else if(shared_mess < 0) + HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "error determining if message should be shared"); + if((ret_value = H5O_append_real( f, dxpl_id, oh, type, flags, mesg, oh_flags_ptr)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_WRITEERROR, FAIL, "unable to append to object header") @@ -1953,6 +2007,7 @@ H5O_new_mesg(H5F_t *f, H5O_t *oh, unsigned *flags, const H5O_msg_class_t *orig_t const void **new_mesg, hid_t dxpl_id, unsigned * oh_flags_ptr) { size_t size; /* Size of space allocated for object header */ + htri_t is_shared; /* Is this a shared message? */ unsigned ret_value = UFAIL; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5O_new_mesg) @@ -1972,20 +2027,25 @@ H5O_new_mesg(H5F_t *f, H5O_t *oh, unsigned *flags, const H5O_msg_class_t *orig_t if(*flags & H5O_FLAG_SHARED) { HDmemset(sh_mesg, 0, sizeof(H5O_shared_t)); - if(NULL == orig_type->get_share) - HGOTO_ERROR(H5E_OHDR, H5E_UNSUPPORTED, UFAIL, "message class is not sharable") - if((orig_type->get_share)(f, orig_mesg, sh_mesg/*out*/) < 0) { + if ((NULL == orig_type->is_shared) || (NULL == orig_type->get_share)) + HGOTO_ERROR(H5E_OHDR, H5E_UNSUPPORTED, UFAIL, "message class is not sharable"); + if ((is_shared = (orig_type->is_shared)(orig_mesg)) == FALSE) { /* * If the message isn't shared then turn off the shared bit * and treat it as an unshared message. */ - H5E_clear_stack(NULL); *flags &= ~H5O_FLAG_SHARED; - } else { - /* Change type & message to use shared information */ + } else if(is_shared > 0) { + /* Message is shared. Get shared message, change message type, + * and use shared information */ + if ((orig_type->get_share)(f, orig_mesg, sh_mesg/*out*/) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, UFAIL, "can't get shared message") + *new_type = H5O_MSG_SHARED; *new_mesg = sh_mesg; - } /* end else */ + } else { + HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, UFAIL, "can't determine if message is shared") + }/* end else */ } /* end if */ else { *new_type = orig_type; @@ -1996,7 +2056,7 @@ H5O_new_mesg(H5F_t *f, H5O_t *oh, unsigned *flags, const H5O_msg_class_t *orig_t if((size = ((*new_type)->raw_size)(f, *new_mesg)) >= H5O_MESG_MAX_SIZE) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, UFAIL, "object header message is too large") - /* Allocate space in the object headed for the message */ + /* Allocate space in the object header for the message */ if((ret_value = H5O_alloc(f, dxpl_id, oh, orig_type, size, oh_flags_ptr)) == UFAIL) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, UFAIL, "unable to allocate space for message") @@ -3455,9 +3515,11 @@ H5O_alloc_new_chunk(H5F_t *f, * that could be moved to make room for the continuation message. * * Don't ever move continuation message from one chunk to another. + * Prioritize link messages moving to later chunks, instead of + * more "important" messages. + * Avoid moving attributes when possible to preserve their + * ordering (although ordering is *not* guaranteed!). * - * Prioritize moving attribute and link messages to later chunks, - * instead of more "important" messages. */ cont_size = H5O_ALIGN_OH(oh, H5F_SIZEOF_ADDR(f) + H5F_SIZEOF_SIZE(f)); for(u = 0; u < oh->nmesgs; u++) { @@ -3498,15 +3560,17 @@ H5O_alloc_new_chunk(H5F_t *f, * message, then make sure the new chunk has enough room for that * other message. * - * Move attributes first, then link messages, then other messages. + * Move link messages first, then other messages, and attributes + * only as a last resort. * */ if(found_null < 0) { - if(found_attr >= 0) - found_other = found_attr; - else if(found_link >= 0) + if(found_link >= 0) found_other = found_link; + if(found_other < 0) + found_other = found_attr; + HDassert(found_other >= 0); size += H5O_SIZEOF_MSGHDR_OH(oh) + oh->mesg[found_other].raw_size; } /* end if */ @@ -4030,7 +4094,7 @@ H5O_get_share(unsigned type_id, H5F_t *f, const void *mesg, H5O_shared_t *share) HDassert(mesg); HDassert(share); - /* Compute the raw data size for the mesg */ + /* Get shared data for the mesg */ if((ret_value = (type->get_share)(f, mesg, share)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "unable to retrieve shared message information") @@ -4039,6 +4103,143 @@ done: } /* end H5O_get_share() */ + +/*------------------------------------------------------------------------- + * Function: H5O_is_shared + * + * Purpose: Call the 'is_shared' method for a + * particular class of object header. + * + * Return: Object is shared: TRUE + * Object is not shared: FALSE + * + * Programmer: James Laird + * jlaird@ncsa.uiuc.edu + * April 5 2006 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +htri_t +H5O_is_shared(unsigned type_id, const void *mesg) +{ + const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + htri_t ret_value; + + FUNC_ENTER_NOAPI_NOFUNC(H5O_is_shared) + + /* Check args */ + HDassert(type_id < NELMTS(H5O_msg_class_g)); + type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */ + HDassert(type); + HDassert(mesg); + + HDassert(type_id != H5O_SHARED_ID); /* JAMES: check for this mistake elsewhere, too */ + + /* If there is no is_shared function, then obviously it's not a shared message! */ + if( !(type->is_shared)) + ret_value = FALSE; + else + ret_value = (type->is_shared)(mesg); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_is_shared() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_set_share + * + * Purpose: Set the shared information for an object header message. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: James Laird + * jlaird@hdfgroup.org + * November 1 2006 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_set_share(H5F_t *f, hid_t dxpl_id, H5O_shared_t *share, + unsigned type_id, void *mesg) +{ + const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + herr_t ret_value; /* Return value */ + + FUNC_ENTER_NOAPI(H5O_set_share,FAIL) + + /* Check args */ + HDassert(f); + HDassert(share); + HDassert(type_id < NELMTS(H5O_msg_class_g)); + type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */ + HDassert(type); + HDassert(type->set_share); + HDassert(mesg); + HDassert(share->flags != H5O_NOT_SHARED); + + /* Set this message as the shared message for the message, wiping out + * any information that was there before + */ + if((ret_value = (type->set_share)(f, mesg, share)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "unable to set shared message information") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_set_share() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_reset_share + * + * Purpose: Reset the shared information for an object header message. + * + * Return: Success: Non-negative + * Failure: Negative + * + * Programmer: James Laird + * jlaird@hdfgroup.org + * Oct 17 2006 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_reset_share(H5F_t *f, unsigned type_id, void *mesg) +{ + const H5O_msg_class_t *type; /* Actual H5O class type for the ID */ + H5O_shared_t sh_mesg; /* Shared message */ + herr_t ret_value; /* Return value */ + + FUNC_ENTER_NOAPI(H5O_reset_share,FAIL) + + /* Check args */ + HDassert(type_id < NELMTS(H5O_msg_class_g)); + type = H5O_msg_class_g[type_id]; /* map the type ID to the actual type object */ + HDassert(type); + HDassert(type->set_share); + HDassert(mesg); + + /* Initialize the shared message to zero. */ + HDmemset(&sh_mesg, 0, sizeof(H5O_shared_t)); + + /* Set this message as the shared message for the message, wiping out + * any information that was there before + */ + if((ret_value = (type->set_share)(f, mesg, &sh_mesg)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "unable to reset shared message information") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_reset_share() */ + + + /*------------------------------------------------------------------------- * Function: H5O_delete * @@ -4160,7 +4361,9 @@ H5O_delete_mesg(H5F_t *f, hid_t dxpl_id, H5O_mesg_t *mesg, hbool_t adj_link) /* Get the message to free's type */ if(mesg->flags & H5O_FLAG_SHARED) + { type = H5O_MSG_SHARED; + } else type = mesg->type; @@ -4176,6 +4379,22 @@ H5O_delete_mesg(H5F_t *f, hid_t dxpl_id, H5O_mesg_t *mesg, hbool_t adj_link) HGOTO_ERROR(H5E_OHDR, H5E_CANTDECODE, FAIL, "unable to decode message") } /* end if */ + /* Check if this message needs to be removed from the SOHM table */ + /* JAMES: there should be a callback, maybe in H5O_shared_delete, to fiddle w/ the ref. count. + * We shouldn't need to do a search in the SOHM table on delete. */ + if(type == H5O_MSG_SHARED) + { + /* JAMES ugly! And not quite correct. */ + void * mesg_orig; + if(NULL == (mesg_orig = H5O_shared_read(f, dxpl_id, mesg->native, mesg->type, NULL))) + HGOTO_ERROR (H5E_OHDR, H5E_BADMESG, FAIL, "unable to read shared message") + + if(H5SM_try_delete(f, dxpl_id, mesg->type->id, mesg->native) < 0) + HGOTO_ERROR (H5E_OHDR, H5E_CANTFREE, FAIL, "unable to delete message from SOHM table") + + H5O_free(mesg->type->id, mesg_orig); + } + if((type->del)(f, dxpl_id, mesg->native, adj_link) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTDELETE, FAIL, "unable to delete file space for object header message") } /* end if */ @@ -4444,12 +4663,23 @@ H5O_iterate_real(const H5O_loc_t *loc, const H5O_msg_class_t *type, H5AC_protect /* Iterate over messages */ for(sequence = 0, idx = 0, idx_msg = &oh->mesg[0]; idx < oh->nmesgs && !ret_value; idx++, idx_msg++) { if(type->id == idx_msg->type->id) { + void * unshared_mesg; /* JAMES */ + /* * Decode the message if necessary. If the message is shared then decode * a shared message, ignoring the message type. */ LOAD_NATIVE(loc->file, dxpl_id, idx_msg, FAIL) + /* JAMES: test */ + if(idx_msg->flags & H5O_FLAG_SHARED) + { + if(NULL == (unshared_mesg = H5O_shared_read(loc->file, dxpl_id, idx_msg->native, idx_msg->type, NULL))) + HGOTO_ERROR(H5E_OHDR, H5E_BADMESG, FAIL, "unable to read shared message"); + } + else + unshared_mesg = idx_msg->native; + /* Check for making an "internal" (i.e. within the H5O package) callback */ if(internal) { /* Call the "internal" iterator callback */ @@ -4458,10 +4688,19 @@ H5O_iterate_real(const H5O_loc_t *loc, const H5O_msg_class_t *type, H5AC_protect } /* end if */ else { /* Call the iterator callback */ - if((ret_value = (op.app_op)(idx_msg->native, sequence, op_data)) != 0) +/* JAMES if((ret_value = (op.app_op)(idx_msg->native, sequence, op_data)) != 0) + break; +*/ + if((ret_value = (op.app_op)(unshared_mesg, sequence, op_data)) != 0) break; } /* end else */ + /* JAMES again */ + if(idx_msg->flags & H5O_FLAG_SHARED) + { + H5O_free_real(idx_msg->type, unshared_mesg); + } + /* Check for error from iterator */ if(ret_value < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "iterator function failed") @@ -4820,8 +5059,8 @@ done: * *------------------------------------------------------------------------- */ -static void * -H5O_copy_mesg_file(const H5O_msg_class_t *type, H5F_t *file_src, void *native_src, +void * +H5O_copy_mesg_file(const H5O_msg_class_t *copy_type, const H5O_msg_class_t *mesg_type, H5F_t *file_src, void *native_src, H5F_t *file_dst, hid_t dxpl_id, H5O_copy_t *cpy_info, void *udata) { void *ret_value; @@ -4829,14 +5068,15 @@ H5O_copy_mesg_file(const H5O_msg_class_t *type, H5F_t *file_src, void *native_sr FUNC_ENTER_NOAPI_NOINIT(H5O_copy_mesg_file) /* check args */ - HDassert(type); - HDassert(type->copy_file); + HDassert(copy_type); + HDassert(mesg_type); + HDassert(copy_type->copy_file); HDassert(file_src); HDassert(native_src); HDassert(file_dst); HDassert(cpy_info); - if(NULL == (ret_value = (type->copy_file)(file_src, native_src, file_dst, dxpl_id, cpy_info, udata))) + if(NULL == (ret_value = (copy_type->copy_file)(file_src, mesg_type, native_src, file_dst, dxpl_id, cpy_info, udata))) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "unable to copy object header message to file") done: @@ -4847,7 +5087,15 @@ done: /*------------------------------------------------------------------------- * Function: H5O_copy_header_real * - * Purpose: copy header object from one location to another. + * Purpose: Copy header object from one location to another using + * pre-copy, copy, and post-copy callbacks for each message + * type. + * + * The source header object is compressed into a single chunk + * (since we know how big it is) and any continuation messages + * are converted into NULL messages. + * + * By default, NULL messages are not copied. * * Return: Non-negative on success/Negative on failure * @@ -4865,11 +5113,17 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, H5O_t *oh_dst = NULL; /* Object header for destination object */ unsigned chunkno = 0, mesgno = 0; haddr_t addr_new = HADDR_UNDEF; + hbool_t *deleted = NULL; /* Array of flags indicating whether messages should be copied */ + size_t null_msgs; /* Number of NULL messages found in each loop */ H5O_mesg_t *mesg_src; /* Message in source object header */ H5O_mesg_t *mesg_dst; /* Message in source object header */ const H5O_msg_class_t *copy_type; /* Type of message to use for copying */ const H5O_obj_class_t *obj_class = NULL; /* Type of object we are copying */ - void *udata = NULL; /* User data for passing to message callbacks */ + void *udata = NULL; /* User data for passing to message + callbacks */ + size_t dst_oh_size; /* Total size of the destination OH */ + uint8_t *current_pos; /* Current position in destination image */ + size_t msghdr_size; herr_t ret_value = SUCCEED; FUNC_ENTER_NOAPI_NOINIT(H5O_copy_header_real) @@ -4905,57 +5159,27 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, oh_dst->version = oh_src->version; oh_dst->nlink = 0; - /* Initialize size of chunk array */ - oh_dst->alloc_nchunks = oh_dst->nchunks = oh_src->nchunks; + /* Initialize size of chunk array. The destination always has only one + * chunk. + */ + oh_dst->alloc_nchunks = oh_dst->nchunks = 1; /* Allocate memory for the chunk array */ if(NULL == (oh_dst->chunk = H5FL_SEQ_MALLOC(H5O_chunk_t, oh_dst->alloc_nchunks))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - /* need to allocate all the chunks for the destination before copy the chunk message - because continuation chunk message will need to know the chunk address of address of - continuation block. + /* Allocate memory for "deleted" array. This array marks the message in + * the source that shouldn't be copied to the destination. */ - for(chunkno = 0; chunkno < oh_src->nchunks; chunkno++) { - size_t chunk_size = oh_src->chunk[chunkno].size; - - /* Allocate space for chunk in destination file */ - if(HADDR_UNDEF == (oh_dst->chunk[chunkno].addr = H5MF_alloc(oloc_dst->file, H5FD_MEM_OHDR, dxpl_id, (hsize_t)chunk_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for object header") - if(chunkno == 0) - addr_new = oh_dst->chunk[chunkno].addr; - - /* Create memory image for the new chunk */ - if(NULL == (oh_dst->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, chunk_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - - /* Copy the chunk image from source to destination in memory */ - /* (This copies over all the messages which don't require special - * callbacks to fix them up.) - */ - HDmemcpy(oh_dst->chunk[chunkno].image, oh_src->chunk[chunkno].image, chunk_size); - - /* Set dest. chunk information */ - oh_dst->chunk[chunkno].dirty = TRUE; - oh_dst->chunk[chunkno].size = chunk_size; - oh_dst->chunk[chunkno].gap = oh_src->chunk[chunkno].gap; - } /* end for */ - - - /* Initialize size of message list */ - oh_dst->alloc_nmesgs = oh_dst->nmesgs = oh_src->nmesgs; - - /* Allocate memory for message array */ - if(NULL == (oh_dst->mesg = H5FL_SEQ_CALLOC(H5O_mesg_t, oh_dst->alloc_nmesgs))) + if(NULL == (deleted = HDmalloc(sizeof(hbool_t) * oh_src->nmesgs))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + HDmemset(deleted, FALSE, sizeof(hbool_t) * oh_src->nmesgs); - /* Copy basic information about messages */ - HDmemcpy(oh_dst->mesg, oh_src->mesg, oh_dst->alloc_nmesgs * sizeof(H5O_mesg_t)); - - /* Set up destination message raw image pointer information */ - /* (must be done before "pre copy" pass, so that if a message is deleted, - * the raw message information in a chunk can be moved around safely) + /* "pre copy" pass over messages, to gather information for actual message copy operation + * (for messages which depend on information from other messages) + * Keep track of how many NULL or deleted messages we find (or create) */ + null_msgs = 0; for(mesgno = 0; mesgno < oh_src->nmesgs; mesgno++) { /* Set up convenience variables */ mesg_src = &(oh_src->mesg[mesgno]); @@ -4964,28 +5188,23 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, /* Sanity check */ HDassert(!mesg_src->dirty); /* Should be cleared by earlier call to flush messages */ - /* Fix up destination message pointers */ - mesg_dst->raw = oh_dst->chunk[mesg_dst->chunkno].image + (mesg_src->raw - oh_src->chunk[mesg_src->chunkno].image); - mesg_dst->native = NULL; - } /* end for */ - - /* "pre copy" pass over messages, to gather information for actual message copy operation */ - /* (for messages which depend on information from other messages) */ - for(mesgno = 0; mesgno < oh_src->nmesgs; mesgno++) { - /* Set up convenience variables */ - mesg_src = &(oh_src->mesg[mesgno]); - mesg_dst = &(oh_dst->mesg[mesgno]); - /* Check for shared message to operate on */ if(mesg_src->flags & H5O_FLAG_SHARED) copy_type = H5O_MSG_SHARED; else copy_type = mesg_src->type; - HDassert(copy_type); - if(copy_type->pre_copy_file) { - hbool_t deleted = FALSE; /* Flag to indicate that the message should be deleted from the destination */ + /* Check for continuation message; these are converted to NULL + * messages because the destination OH will have only one chunk + */ + if(H5O_CONT_ID == mesg_src->type->id || H5O_NULL_ID == mesg_src->type->id) { + deleted[mesgno] = TRUE; + ++null_msgs; + copy_type = H5O_MSG_NULL; + } + HDassert(copy_type); + if(copy_type->pre_copy_file ) { /* * Decode the message if necessary. If the message is shared then do * a shared message, ignoring the message type. @@ -4998,28 +5217,68 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, } /* end if (NULL == mesg_src->native) */ /* Perform "pre copy" operation on message */ - if((copy_type->pre_copy_file)(oloc_src->file, mesg_src->type, mesg_src->native, &deleted, cpy_info, udata) < 0) + if((copy_type->pre_copy_file)(oloc_src->file, mesg_src->type, mesg_src->native, &(deleted[mesgno]), cpy_info, udata) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to perform 'pre copy' operation on message") /* Check if the message should be deleted in the destination */ - if(deleted) { - /* Convert message into a null message */ - if(H5O_release_mesg(oloc_dst->file, dxpl_id, oh_dst, mesg_dst, FALSE, FALSE) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTDELETE, FAIL, "unable to convert into null message") - } /* end if */ - } /* end if */ + if(deleted[mesgno]) { + /* Mark message as deleted */ + ++null_msgs; + } /* end if(deleted) */ + } /* end if(copy_type->pre_copy_file) */ } /* end for */ + /* Initialize size of message list. It may or may not include the NULL messages + * detected above. + */ + if(cpy_info->preserve_null) + oh_dst->alloc_nmesgs = oh_dst->nmesgs = oh_src->nmesgs; + else + oh_dst->alloc_nmesgs = oh_dst->nmesgs = (oh_src->nmesgs - null_msgs); + + /* Allocate memory for destination message array */ + if(oh_dst->alloc_nmesgs > 0) { + if(NULL == (oh_dst->mesg = H5FL_SEQ_CALLOC(H5O_mesg_t, oh_dst->alloc_nmesgs))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + } + /* "copy" pass over messages, to perform main message copying */ - for(mesgno = 0; mesgno < oh_src->nmesgs; mesgno++) { + null_msgs = 0; + for(mesgno = 0; mesgno < oh_dst->nmesgs; mesgno++) { + /* Skip any deleted or NULL messages in the source unless the + * preserve_null flag is set + */ + if(FALSE == cpy_info->preserve_null) { + while(deleted[mesgno + null_msgs]) { + ++null_msgs; + HDassert(mesgno + null_msgs < oh_src->nmesgs); + } + } + /* Set up convenience variables */ - mesg_src = &(oh_src->mesg[mesgno]); + mesg_src = &(oh_src->mesg[mesgno + null_msgs]); mesg_dst = &(oh_dst->mesg[mesgno]); + /* Initialize on destination message */ + mesg_dst->chunkno = 0; + mesg_dst->dirty = FALSE; + mesg_dst->flags = mesg_src->flags; + mesg_dst->native = NULL; + mesg_dst->raw = NULL; + mesg_dst->raw_size = mesg_src->raw_size; + mesg_dst->type = mesg_src->type; + + /* If we're preserving deleted messages, set their types to 'NULL' + * in the destination. + */ + if(cpy_info->preserve_null && deleted[mesgno]) { + mesg_dst->type = H5O_MSG_NULL; + } + /* Check for shared message to operate on */ /* (Use destination message, in case the message has been removed (i.e - * converted to a nil message) in the destination -QAK) - */ + * converted to a nil message) in the destination -QAK) + */ if(mesg_dst->flags & H5O_FLAG_SHARED) copy_type = H5O_MSG_SHARED; else @@ -5029,9 +5288,9 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, /* copy this message into destination file */ if(copy_type->copy_file) { /* - * Decode the message if necessary. If the message is shared then do - * a shared message, ignoring the message type. - */ + * Decode the message if necessary. If the message is shared then do + * a shared message, ignoring the message type. + */ if(NULL == mesg_src->native) { /* Decode the message if necessary */ HDassert(copy_type->decode); @@ -5040,16 +5299,10 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, } /* end if (NULL == mesg_src->native) */ /* Copy the source message */ - if(H5O_CONT_ID == mesg_src->type->id) { - if((mesg_dst->native = H5O_copy_mesg_file(copy_type, oloc_src->file, mesg_src->native, - oloc_dst->file, dxpl_id, cpy_info, oh_dst->chunk)) == NULL) - HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object header message") - } /* end if */ - else { - if((mesg_dst->native = H5O_copy_mesg_file(copy_type, oloc_src->file, mesg_src->native, - oloc_dst->file, dxpl_id, cpy_info, udata)) == NULL) - HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object header message") - } /* end else */ + if((mesg_dst->native = H5O_copy_mesg_file(copy_type, mesg_dst->type, + oloc_src->file, mesg_src->native, oloc_dst->file, dxpl_id, + cpy_info, udata)) == NULL) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object header message") /* Mark the message in the destination as dirty, so it'll get encoded when the object header is flushed */ mesg_dst->dirty = TRUE; @@ -5057,6 +5310,86 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, } /* end of mesgno loop */ + /* Allocate the destination header and copy any messages that didn't have + * copy callbacks. They get copied directly from the source image to the + * destination image. + */ + + /* Calculate how big the destination object header will be on disk. + * This isn't necessarily the same size as the original. + */ + dst_oh_size = H5O_SIZEOF_HDR_OH(oh_dst); + + /* Add space for messages. */ + for(mesgno = 0; mesgno < oh_dst->nmesgs; mesgno++) { + dst_oh_size += H5O_SIZEOF_MSGHDR_OH(oh_dst); + dst_oh_size += H5O_ALIGN_OH(oh_dst, oh_dst->mesg[mesgno].raw_size); + } + + /* Allocate space for chunk in destination file */ + if(HADDR_UNDEF == (oh_dst->chunk[0].addr = H5MF_alloc(oloc_dst->file, H5FD_MEM_OHDR, dxpl_id, (hsize_t)dst_oh_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "file allocation failed for object header") + addr_new = oh_dst->chunk[0].addr; + + /* Create memory image for the new chunk */ + if(NULL == (oh_dst->chunk[0].image = H5FL_BLK_MALLOC(chunk_image, dst_oh_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Set dest. chunk information */ + oh_dst->chunk[0].dirty = TRUE; + oh_dst->chunk[0].size = dst_oh_size; + oh_dst->chunk[0].gap = 0; + + /* Set up raw pointers and copy messages that didn't need special + * treatment. This has to happen after the destination header has been + * allocated. + */ + HDassert(H5O_SIZEOF_HDR_OH(oh_src) == H5O_SIZEOF_HDR_OH(oh_dst)); + HDassert(H5O_SIZEOF_MSGHDR_OH(oh_src) == H5O_SIZEOF_MSGHDR_OH(oh_dst)); + msghdr_size = H5O_SIZEOF_MSGHDR_OH(oh_src); + + current_pos = oh_dst->chunk[0].image; + + /* Copy the message header. Most of this will be overwritten when + * the header is flushed to disk, but later versions have a + * magic number that isn't. + */ + HDmemcpy(current_pos, oh_src->chunk[0].image, + H5O_SIZEOF_HDR_OH(oh_dst) - H5O_SIZEOF_CHKSUM_OH(oh_dst)); + current_pos += H5O_SIZEOF_HDR_OH(oh_dst) - H5O_SIZEOF_CHKSUM_OH(oh_dst); + + /* JAMES: include this in loop above? Doesn't take deleted messages + * into account + */ + /* Copy each message that wasn't dirtied above */ + null_msgs = 0; + for(mesgno = 0; mesgno < oh_dst->nmesgs; mesgno++) { + /* Skip any deleted or NULL messages in the source unless the + * preserve_null flag is set + */ + if(FALSE == cpy_info->preserve_null) { + while(deleted[mesgno + null_msgs]) { + ++null_msgs; + HDassert(mesgno + null_msgs < oh_src->nmesgs); + } + } + + /* Set up convenience variables */ + mesg_src = &(oh_src->mesg[mesgno + null_msgs]); + mesg_dst = &(oh_dst->mesg[mesgno]); + + if(! mesg_dst->dirty) { + /* Copy the message header plus the message's raw data. */ + HDmemcpy(current_pos, mesg_src->raw - msghdr_size, + msghdr_size + mesg_src->raw_size); + } + mesg_dst->raw = current_pos + msghdr_size; + current_pos += mesg_dst->raw_size + msghdr_size; + } + + /* Make sure we filled the chunk, except for room at the end for a checksum */ + HDassert(current_pos + H5O_SIZEOF_CHKSUM_OH(oh_dst) == dst_oh_size + oh_dst->chunk[0].image); + /* Set the dest. object location to the first chunk address */ HDassert(H5F_addr_defined(addr_new)); oloc_dst->addr = addr_new; @@ -5078,9 +5411,20 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, /* "post copy" loop over messages, to fix up any messages which require a complete * object header for destination object */ - for(mesgno = 0; mesgno < oh_src->nmesgs; mesgno++) { + null_msgs = 0; + for(mesgno = 0; mesgno < oh_dst->nmesgs; mesgno++) { + /* Skip any deleted or NULL messages in the source unless the + * preserve_null flag is set + */ + if(FALSE == cpy_info->preserve_null) { + while(deleted[mesgno + null_msgs]) { + ++null_msgs; + HDassert(mesgno + null_msgs < oh_src->nmesgs); + } + } + /* Set up convenience variables */ - mesg_src = &(oh_src->mesg[mesgno]); + mesg_src = &(oh_src->mesg[mesgno + null_msgs]); mesg_dst = &(oh_dst->mesg[mesgno]); /* Check for shared message to operate on */ @@ -5119,7 +5463,12 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to cache object header") done: - /* Release pointer to source object header and it's derived objects */ + /* Free deleted array */ + if(deleted) { + HDfree(deleted); + } + + /* Release pointer to source object header and its derived objects */ if(oh_src != NULL) { /* Unprotect the source object header */ if(H5AC_unprotect(oloc_src->file, dxpl_id, H5AC_OHDR, oloc_src->addr, oh_src, H5AC__NO_FLAGS_SET) < 0) @@ -5180,7 +5529,7 @@ H5O_copy_header_map(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, /* Copy object for the first time */ /* Check for incrementing the depth of copy */ - /* (Can't do this for all copies, since shared datatypes should always be copied) */ + /* (Can't do this for all copies, since committed datatypes should always be copied) */ if(inc_depth) cpy_info->curr_depth++; @@ -5199,7 +5548,7 @@ H5O_copy_header_map(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, ret_value++; } /* end if */ else { - /* Object has already been copied, set it's address in destination file */ + /* Object has already been copied, set its address in destination file */ oloc_dst->addr = addr_map->dst_addr; /* If the object is locked currently (because we are copying a group @@ -5301,6 +5650,8 @@ H5O_copy_header(const H5O_loc_t *oloc_src, H5O_loc_t *oloc_dst /*out */, cpy_info.expand_ref = TRUE; if((cpy_option & H5O_COPY_WITHOUT_ATTR_FLAG) > 0) cpy_info.copy_without_attr = TRUE; + if((cpy_option & H5O_COPY_PRESERVE_NULL_FLAG) > 0) + cpy_info.preserve_null = TRUE; /* Create a skip list to keep track of which objects are copied */ if((cpy_info.map_list = H5SL_create(H5SL_TYPE_HADDR, 0.5, (size_t)16)) == NULL) @@ -5317,6 +5668,128 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_copy_header() */ + +/*------------------------------------------------------------------------- + * Function: H5O_mesg_hash + * + * Purpose: Returns a hash value for an object header message. + * + * Return: Non-H5O_HASH_UNDEF hash value on success + * H5O_HASH_UNDEF on failure + * + * Programmer: James Laird + * April 13 2006 + * + *------------------------------------------------------------------------- + */ +uint32_t +H5O_mesg_hash(unsigned type_id, H5F_t *f, const void *mesg) +{ + size_t buf_size; + unsigned char * buf = NULL; /* Buffer to be hashed */ + uint32_t hash; + uint32_t ret_value; + + FUNC_ENTER_NOAPI(H5O_mesg_hash, FAIL) + + /* Check args */ + HDassert(type_id < NELMTS(H5O_msg_class_g)); + HDassert(mesg); + HDassert(f); + + /* Find out the size of buffer needed */ + if((buf_size = H5O_raw_size(type_id, f, mesg)) <= 0) + HGOTO_ERROR(H5E_OHDR, H5E_BADSIZE, FAIL, "can't find message size"); + + /* JAMES: revisit this! Some messages don't use as much space as they say + * they need. Quincey may have fixed this. + */ + if((buf = HDmalloc(buf_size)) == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate buffer for message"); + HDmemset(buf, 0, buf_size); + + if(H5O_encode(f, buf, mesg, type_id) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTENCODE, FAIL, "can't encode OH message"); + + /* + * Compute the hash value for this message. type_id is used here to + * initialize the hash algorithm, and affects the resulting value. + */ + hash = H5_checksum_lookup3(buf, buf_size, type_id); + + /* JAMES: this is a pretty good hash function. Do we need to version it? + * If so, we'd do so here. */ + + /* A hash value of H5O_HASH_UNDEF indicates failure. If we naturally + * generated this value, reset it to some valid value. */ + if(hash == H5O_HASH_UNDEF) + hash = (uint32_t) 1; + + ret_value = hash; +done: + if(buf) + HDfree(buf); + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_mesg_hash() */ + + +/*------------------------------------------------------------------------- + * Function: H5O_copy_obj_by_ref + * + * Purpose: Copy the object pointed by _src_ref. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Peter Cao + * Aug 7 2006 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_copy_obj_by_ref(H5O_loc_t *src_oloc, hid_t dxpl_id, H5O_loc_t *dst_oloc, + H5G_loc_t *dst_root_loc, H5O_copy_t *cpy_info) +{ + herr_t ret_value = SUCCEED; + + FUNC_ENTER_NOAPI(H5O_copy_obj_by_ref, FAIL) + + HDassert(src_oloc); + HDassert(dst_oloc); + + /* Perform the copy, or look up existing copy */ + if((ret_value = H5O_copy_header_map(src_oloc, dst_oloc, dxpl_id, cpy_info, FALSE)) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, FAIL, "unable to copy object") + + /* Check if a new valid object is copied to the destination */ + if(H5F_addr_defined(dst_oloc->addr) && (ret_value > SUCCEED)) { + char tmp_obj_name[80]; + H5G_name_t new_path; + H5O_loc_t new_oloc; + H5G_loc_t new_loc; + + /* Set up group location for new object */ + new_loc.oloc = &new_oloc; + new_loc.path = &new_path; + H5G_loc_reset(&new_loc); + new_oloc.file = dst_oloc->file; + new_oloc.addr = dst_oloc->addr; + + /* Pick a default name for the new object */ + sprintf(tmp_obj_name, "~obj_pointed_by_%llu", (unsigned long_long)dst_oloc->addr); + + /* Create a link to the newly copied object */ + if(H5L_link(dst_root_loc, tmp_obj_name, &new_loc, H5P_DEFAULT, H5P_DEFAULT, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to insert link") + + H5G_loc_free(&new_loc); + } /* if (H5F_addr_defined(dst_oloc.addr)) */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_copy_obj_by_ref() */ + + + #ifdef H5O_DEBUG /*------------------------------------------------------------------------- @@ -5603,7 +6076,7 @@ H5O_debug_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh, haddr_t addr, FILE *stream, i H5O_shared_t *shared = (H5O_shared_t*)(oh->mesg[i].native); void *mesg; - mesg = H5O_read_real(&(shared->oloc), oh->mesg[i].type, 0, NULL, dxpl_id); + mesg = H5O_shared_read(f, dxpl_id, shared, oh->mesg[i].type, NULL); if(oh->mesg[i].type->debug) (oh->mesg[i].type->debug)(f, dxpl_id, mesg, stream, indent + 3, MAX (0, fwidth - 3)); H5O_free_real(oh->mesg[i].type, mesg); |