diff options
44 files changed, 3888 insertions, 2935 deletions
diff --git a/java/src/hdf/hdf5lib/H5.java b/java/src/hdf/hdf5lib/H5.java index 789fea5..50faa6d 100644 --- a/java/src/hdf/hdf5lib/H5.java +++ b/java/src/hdf/hdf5lib/H5.java @@ -2854,6 +2854,7 @@ public class H5 implements java.io.Serializable { // /////// unimplemented //////// // ssize_t H5Fget_file_image(hid_t file_id, void * buf_ptr, size_t buf_len); + // herr_t H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info); // ssize_t H5Fget_free_sections(hid_t file_id, H5F_mem_t type, size_t nsects, H5F_sect_info_t *sect_info/*out*/); // /** @@ -5672,6 +5673,41 @@ public class H5 implements java.io.Serializable { public synchronized static native void H5Pset_elink_file_cache_size(long fapl_id, int efc_size) throws HDF5LibraryException; + /** + /** + * H5Pget_metadata_read_attempts retrieves the number of read attempts that is set in the file access property list plist_id. + * + * @param plist_id + * IN: File access property list identifier + * + * @return The number of read attempts. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native long H5Pget_metadata_read_attempts(long plist_id) throws HDF5LibraryException; + + /** + * H5Pset_metadata_read_attempts sets the number of reads that the library will try when reading checksummed + * metadata in an HDF5 file opened with SWMR access. When reading such metadata, the library will compare the + * checksum computed for the metadata just read with the checksum stored within the piece of checksum. When + * performing SWMR operations on a file, the checksum check might fail when the library reads data on a system + * that is not atomic. To remedy such situations, the library will repeatedly read the piece of metadata until + * the check passes or finally fails the read when the allowed number of attempts is reached. + * + * @param plist_id + * IN: File access property list identifier + * @param attempts + * IN: The number of read attempts which is a value greater than 0. + * + * @exception HDF5LibraryException + * - Error from the HDF-5 Library. + * + **/ + public synchronized static native void H5Pset_metadata_read_attempts(long plist_id, long attempts) + throws HDF5LibraryException; + // Dataset creation property list (DCPL) routines // /** diff --git a/java/src/jni/h5pImp.c b/java/src/jni/h5pImp.c index 7c71da1..01da210 100644 --- a/java/src/jni/h5pImp.c +++ b/java/src/jni/h5pImp.c @@ -5542,6 +5542,40 @@ Java_hdf_hdf5lib_H5_H5Piterate return (jint)status; } /* end Java_hdf_hdf5lib_H5_H5Piterate */ +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_metadata_read_attempts + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1metadata_1read_1attempts + (JNIEnv *env, jclass clss, jlong plist_id) +{ + unsigned attempts; + if (H5Pget_metadata_read_attempts((hid_t)plist_id, &attempts) < 0) + h5libraryError(env); + + return (jlong) attempts; +} /* end Java_hdf_hdf5lib_H5_H5Pget_1metadata_1read_1attempts */ + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_metadata_read_attempts + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1metadata_1read_1attempts + (JNIEnv *env, jclass clss, jlong plist_id, jlong attempts) +{ + if (attempts <= 0) { + h5badArgument(env, "H5Pset_metadata_read_attempts: attempts <= 0"); + } /* end if */ + else { + if(H5Pset_metadata_read_attempts((hid_t)plist_id, (unsigned)attempts) < 0) + h5libraryError(env); + } /* end else */ +} /* end Java_hdf_hdf5lib_H5_H5Pset_1metadata_1read_1attempts */ + #ifdef __cplusplus } /* end extern "C" */ #endif /* __cplusplus */ diff --git a/java/src/jni/h5pImp.h b/java/src/jni/h5pImp.h index 2269624..d606b7a 100644 --- a/java/src/jni/h5pImp.h +++ b/java/src/jni/h5pImp.h @@ -1473,6 +1473,25 @@ JNIEXPORT jint JNICALL Java_hdf_hdf5lib_H5_H5Piterate (JNIEnv*, jclass, jlong, jintArray, jobject, jobject); +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pget_metadata_read_attempts + * Signature: (J)J + */ +JNIEXPORT jlong JNICALL +Java_hdf_hdf5lib_H5_H5Pget_1metadata_1read_1attempts +(JNIEnv *, jclass, jlong); + + +/* + * Class: hdf_hdf5lib_H5 + * Method: H5Pset_metadata_read_attempts + * Signature: (JJ)V + */ +JNIEXPORT void JNICALL +Java_hdf_hdf5lib_H5_H5Pset_1metadata_1read_1attempts +(JNIEnv *, jclass, jlong, jlong); + #ifdef __cplusplus } /* end extern "C" */ @@ -567,7 +567,7 @@ H5AC_dest(H5F_t *f, hid_t dxpl_id) #ifdef H5_HAVE_PARALLEL /* destroying the cache, so clear all collective entries */ - if(H5C_clear_coll_entries(f->shared->cache, 0) < 0) + if(H5C_clear_coll_entries(f->shared->cache, FALSE) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed.") aux_ptr = (H5AC_aux_t *)H5C_get_aux_ptr(f->shared->cache); @@ -726,7 +726,7 @@ H5AC_flush(H5F_t *f, hid_t dxpl_id) #ifdef H5_HAVE_PARALLEL /* flushing the cache, so clear all collective entries */ - if(H5C_clear_coll_entries(f->shared->cache, 0) < 0) + if(H5C_clear_coll_entries(f->shared->cache, FALSE) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, FAIL, "H5C_clear_coll_entries() failed.") /* Attempt to flush all entries from rank 0 & Bcast clean list to other ranks */ @@ -1639,11 +1639,9 @@ H5AC_unprotect(H5F_t *f, hid_t dxpl_id, const H5AC_class_t *type, haddr_t addr, * the entry. */ if(dirtied && !deleted) { - hbool_t curr_compressed = FALSE; /* dummy for call */ size_t curr_size = 0; - size_t curr_compressed_size = 0; /* dummy for call */ - if((type->image_len)(thing, &curr_size, &curr_compressed, &curr_compressed_size) < 0) + if((type->image_len)(thing, &curr_size) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGETSIZE, FAIL, "Can't get size of thing") if(((H5AC_info_t *)thing)->size != curr_size) diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 01b7125..2251af4 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -142,7 +142,6 @@ typedef enum { #define H5AC__SERIALIZE_RESIZED_FLAG H5C__SERIALIZE_RESIZED_FLAG #define H5AC__SERIALIZE_MOVED_FLAG H5C__SERIALIZE_MOVED_FLAG -#define H5AC__SERIALIZE_COMPRESSED_FLAG H5C__SERIALIZE_COMPRESSED_FLAG /* Cork actions: cork/uncork/get cork status of an object */ #define H5AC__SET_CORK H5C__SET_CORK @@ -171,13 +170,14 @@ typedef H5C_notify_action_t H5AC_notify_action_t; #define H5AC__CLASS_NO_FLAGS_SET H5C__CLASS_NO_FLAGS_SET #define H5AC__CLASS_SPECULATIVE_LOAD_FLAG H5C__CLASS_SPECULATIVE_LOAD_FLAG -#define H5AC__CLASS_COMPRESSED_FLAG H5C__CLASS_COMPRESSED_FLAG /* The following flags should only appear in test code */ #define H5AC__CLASS_SKIP_READS H5C__CLASS_SKIP_READS #define H5AC__CLASS_SKIP_WRITES H5C__CLASS_SKIP_WRITES -typedef H5C_get_load_size_func_t H5AC_get_load_size_func_t; +typedef H5C_get_initial_load_size_func_t H5AC_get_initial_load_size_func_t; +typedef H5C_get_final_load_size_func_t H5AC_get_final_load_size_func_t; +typedef H5C_verify_chksum_func_t H5AC_verify_chksum_func_t; typedef H5C_deserialize_func_t H5AC_deserialize_func_t; typedef H5C_image_len_func_t H5AC_image_len_func_t; @@ -189,7 +189,6 @@ typedef H5C_pre_serialize_func_t H5AC_pre_serialize_func_t; typedef H5C_serialize_func_t H5AC_serialize_func_t; typedef H5C_notify_func_t H5AC_notify_func_t; typedef H5C_free_icr_func_t H5AC_free_icr_func_t; -typedef H5C_clear_func_t H5AC_clear_func_t; typedef H5C_get_fsf_size_t H5AC_get_fsf_size_t; typedef H5C_class_t H5AC_class_t; diff --git a/src/H5B2cache.c b/src/H5B2cache.c index b899f6a..51ad32b 100644 --- a/src/H5B2cache.c +++ b/src/H5B2cache.c @@ -65,29 +65,29 @@ /********************/ /* Metadata cache callbacks */ -static herr_t H5B2__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5B2__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5B2__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata); static void *H5B2__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5B2__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5B2__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5B2__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5B2__cache_hdr_free_icr(void *thing); -static herr_t H5B2__cache_int_get_load_size(const void *udata, size_t *image_len); +static herr_t H5B2__cache_int_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5B2__cache_int_verify_chksum(const void *image_ptr, size_t len, void *udata); static void *H5B2__cache_int_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5B2__cache_int_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5B2__cache_int_image_len(const void *thing, size_t *image_len); static herr_t H5B2__cache_int_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5B2__cache_int_free_icr(void *thing); -static herr_t H5B2__cache_leaf_get_load_size(const void *udata, size_t *image_len); +static herr_t H5B2__cache_leaf_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5B2__cache_leaf_verify_chksum(const void *image_ptr, size_t len, void *udata); static void *H5B2__cache_leaf_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5B2__cache_leaf_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5B2__cache_leaf_image_len(const void *thing, size_t *image_len); static herr_t H5B2__cache_leaf_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5B2__cache_leaf_free_icr(void *thing); @@ -102,14 +102,15 @@ const H5AC_class_t H5AC_BT2_HDR[1] = {{ "v2 B-tree header", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5B2__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5B2__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5B2__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5B2__cache_hdr_deserialize, /* 'deserialize' callback */ H5B2__cache_hdr_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5B2__cache_hdr_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5B2__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -119,14 +120,15 @@ const H5AC_class_t H5AC_BT2_INT[1] = {{ "v2 B-tree internal node", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5B2__cache_int_get_load_size, /* 'get_load_size' callback */ + H5B2__cache_int_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5B2__cache_int_verify_chksum, /* 'verify_chksum' callback */ H5B2__cache_int_deserialize, /* 'deserialize' callback */ H5B2__cache_int_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5B2__cache_int_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5B2__cache_int_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -136,14 +138,15 @@ const H5AC_class_t H5AC_BT2_LEAF[1] = {{ "v2 B-tree leaf node", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5B2__cache_leaf_get_load_size, /* 'get_load_size' callback */ + H5B2__cache_leaf_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5B2__cache_leaf_verify_chksum, /* 'verify_chksum' callback */ H5B2__cache_leaf_deserialize, /* 'deserialize' callback */ H5B2__cache_leaf_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5B2__cache_leaf_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5B2__cache_leaf_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -160,7 +163,7 @@ const H5AC_class_t H5AC_BT2_LEAF[1] = {{ /*------------------------------------------------------------------------- - * Function: H5B2__cache_hdr_get_load_size + * Function: H5B2__cache_hdr_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -173,21 +176,58 @@ const H5AC_class_t H5AC_BT2_LEAF[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_hdr_get_load_size(const void *_udata, size_t *image_len) +H5B2__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len) { - const H5B2_hdr_cache_ud_t *udata = (const H5B2_hdr_cache_ud_t *)_udata; /* User data for callback */ + H5B2_hdr_cache_ud_t *udata = (H5B2_hdr_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR /* Check arguments */ HDassert(udata); + HDassert(udata->f); HDassert(image_len); /* Set the image length size */ *image_len = H5B2_HEADER_SIZE_FILE(udata->f); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B2__cache_hdr_get_load_size() */ +} /* end H5B2__cache_hdr_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__cache_hdr_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5B2__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__cache_hdr_verify_chksum() */ /*------------------------------------------------------------------------- @@ -214,7 +254,6 @@ H5B2__cache_hdr_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, H5B2_subid_t id; /* ID of B-tree class, as found in file */ uint16_t depth; /* Depth of B-tree */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ H5B2_hdr_t *ret_value = NULL; /* Return value */ @@ -260,19 +299,14 @@ H5B2__cache_hdr_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, UINT16DECODE(image, hdr->root.node_nrec); H5F_DECODE_LENGTH(udata->f, image, hdr->root.all_nrec); + /* checksum verification already done in verify_chksum cb */ + /* Metadata checksum */ UINT32DECODE(image, stored_chksum); /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == hdr->hdr_size); - /* Compute checksum on entire header */ - computed_chksum = H5_checksum_metadata(_image, (hdr->hdr_size - H5B2_SIZEOF_CHKSUM), 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 B-tree header") - /* Initialize B-tree header info */ cparam.cls = H5B2_client_class_g[id]; if(H5B2__hdr_init(hdr, &cparam, udata->ctx_udata, depth) < 0) @@ -310,8 +344,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5B2__cache_hdr_image_len(const void *_thing, size_t *image_len) { const H5B2_hdr_t *hdr = (const H5B2_hdr_t *)_thing; /* Pointer to the B-tree header */ @@ -433,7 +466,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5B2__cache_int_get_load_size + * Function: H5B2__cache_int_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -446,9 +479,9 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_int_get_load_size(const void *_udata, size_t *image_len) +H5B2__cache_int_get_initial_load_size(void *_udata, size_t *image_len) { - const H5B2_internal_cache_ud_t *udata = (const H5B2_internal_cache_ud_t *)_udata; /* User data for callback */ + H5B2_internal_cache_ud_t *udata = (H5B2_internal_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -461,7 +494,49 @@ H5B2__cache_int_get_load_size(const void *_udata, size_t *image_len) *image_len = udata->hdr->node_size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B2__cache_int_get_load_size() */ +} /* end H5B2__cache_int_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__cache_int_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5B2__cache_int_verify_chksum(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5B2_internal_cache_ud_t *udata = (H5B2_internal_cache_ud_t *)_udata; /* Pointer to user data */ + size_t chk_size; /* Exact size of the node with checksum at the end */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* Internal node prefix header + records + child pointer triplets: size with checksum at the end */ + chk_size = H5B2_INT_PREFIX_SIZE + (udata->nrec * udata->hdr->rrec_size) + ((size_t)(udata->nrec + 1) * H5B2_INT_POINTER_SIZE(udata->hdr, udata->depth)); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, chk_size, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__cache_int_verify_chksum() */ /*------------------------------------------------------------------------- @@ -488,7 +563,6 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, uint8_t *native; /* Pointer to native record info */ H5B2_node_ptr_t *int_node_ptr; /* Pointer to node pointer info */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned u; /* Local index variable */ H5B2_internal_t *ret_value = NULL; /* Return value */ @@ -562,8 +636,7 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, int_node_ptr++; } /* end for */ - /* Compute checksum on internal node */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -571,10 +644,6 @@ H5B2__cache_int_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, /* Sanity check parsing */ HDassert((size_t)(image - (const uint8_t *)_image) <= len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 internal node") - /* Set return value */ ret_value = internal; @@ -601,8 +670,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_int_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5B2__cache_int_image_len(const void *_thing, size_t *image_len) { const H5B2_internal_t *internal = (const H5B2_internal_t *)_thing; /* Pointer to the B-tree internal node */ @@ -721,17 +789,18 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_int_free_icr(void *thing) +H5B2__cache_int_free_icr(void *_thing) { + H5B2_internal_t *internal = (H5B2_internal_t *)_thing; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Check arguments */ - HDassert(thing); + HDassert(internal); /* Release v2 B-tree internal node */ - if(H5B2__internal_free((H5B2_internal_t *)thing) < 0) + if(H5B2__internal_free(internal) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to release v2 B-tree internal node") done: @@ -740,7 +809,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5B2__cache_leaf_get_load_size + * Function: H5B2__cache_leaf_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -753,9 +822,9 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_leaf_get_load_size(const void *_udata, size_t *image_len) +H5B2__cache_leaf_get_initial_load_size(void *_udata, size_t *image_len) { - const H5B2_leaf_cache_ud_t *udata = (const H5B2_leaf_cache_ud_t *)_udata; /* User data for callback */ + H5B2_leaf_cache_ud_t *udata = (H5B2_leaf_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -768,7 +837,49 @@ H5B2__cache_leaf_get_load_size(const void *_udata, size_t *image_len) *image_len = udata->hdr->node_size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B2__cache_leaf_get_load_size() */ +} /* end H5B2__cache_leaf_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5B2__cache_leaf_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5B2__cache_leaf_verify_chksum(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5B2_internal_cache_ud_t *udata = (H5B2_internal_cache_ud_t *)_udata; /* Pointer to user data */ + size_t chk_size; /* Exact size of the node with checksum at the end */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* Leaf node prefix header + records: size with checksum at the end */ + chk_size = H5B2_LEAF_PREFIX_SIZE + (udata->nrec * udata->hdr->rrec_size); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, chk_size, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B2__cache_leaf_verify_chksum() */ /*------------------------------------------------------------------------- @@ -794,7 +905,6 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint8_t *native; /* Pointer to native keys */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned u; /* Local index variable */ H5B2_leaf_t *ret_value = NULL; /* Return value */ @@ -848,8 +958,7 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, native += udata->hdr->cls->nrec_size; } /* end for */ - /* Compute checksum on leaf node */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -857,10 +966,6 @@ H5B2__cache_leaf_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, /* Sanity check parsing */ HDassert((size_t)(image - (const uint8_t *)_image) <= udata->hdr->node_size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 leaf node") - /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) <= len); @@ -890,8 +995,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_leaf_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5B2__cache_leaf_image_len(const void *_thing, size_t *image_len) { const H5B2_leaf_t *leaf = (const H5B2_leaf_t *)_thing; /* Pointer to the B-tree leaf node */ @@ -996,17 +1100,18 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B2__cache_leaf_free_icr(void *thing) +H5B2__cache_leaf_free_icr(void *_thing) { + H5B2_leaf_t *leaf = (H5B2_leaf_t *)_thing; herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC /* Check arguments */ - HDassert(thing); + HDassert(leaf); /* Destroy v2 B-tree leaf node */ - if(H5B2__leaf_free((H5B2_leaf_t *)thing) < 0) + if(H5B2__leaf_free(leaf) < 0) HGOTO_ERROR(H5E_BTREE, H5E_CANTFREE, FAIL, "unable to destroy B-tree leaf node") done: diff --git a/src/H5Bcache.c b/src/H5Bcache.c index 8354e8e..b2be829 100644 --- a/src/H5Bcache.c +++ b/src/H5Bcache.c @@ -54,14 +54,13 @@ /********************/ /* Metadata cache callbacks */ -static herr_t H5B__get_load_size(const void *udata, size_t *image_len); -static void *H5B__deserialize(const void *image, size_t len, void *udata, +static herr_t H5B__cache_get_initial_load_size(void *udata, size_t *image_len); +static void *H5B__cache_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5B__image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); -static herr_t H5B__serialize(const H5F_t *f, void *image, size_t len, +static herr_t H5B__cache_image_len(const void *thing, size_t *image_len); +static herr_t H5B__cache_serialize(const H5F_t *f, void *image, size_t len, void *thing); -static herr_t H5B__free_icr(void *thing); +static herr_t H5B__cache_free_icr(void *thing); /*********************/ @@ -74,14 +73,15 @@ const H5AC_class_t H5AC_BT[1] = {{ "v1 B-tree", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5B__get_load_size, /* 'get_load_size' callback */ - H5B__deserialize, /* 'deserialize' callback */ - H5B__image_len, /* 'image_len' callback */ + H5B__cache_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ + H5B__cache_deserialize, /* 'deserialize' callback */ + H5B__cache_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ - H5B__serialize, /* 'serialize' callback */ + H5B__cache_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ - H5B__free_icr, /* 'free_icr' callback */ - NULL, /* 'clear" callback */ + H5B__cache_free_icr, /* 'free_icr' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -92,7 +92,7 @@ const H5AC_class_t H5AC_BT[1] = {{ /*------------------------------------------------------------------------- - * Function: H5B__get_load_size + * Function: H5B__cache_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -105,9 +105,9 @@ const H5AC_class_t H5AC_BT[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5B__get_load_size(const void *_udata, size_t *image_len) +H5B__cache_get_initial_load_size(void *_udata, size_t *image_len) { - const H5B_cache_ud_t *udata = (const H5B_cache_ud_t *)_udata; /* User data for callback */ + H5B_cache_ud_t *udata = (H5B_cache_ud_t *)_udata; /* User data for callback */ H5B_shared_t *shared; /* Pointer to shared B-tree info */ FUNC_ENTER_STATIC_NOERR @@ -124,11 +124,11 @@ H5B__get_load_size(const void *_udata, size_t *image_len) *image_len = shared->sizeof_rnode; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B__get_load_size() */ +} /* end H5B__cache_get_initial_load_size() */ /*------------------------------------------------------------------------- - * Function: H5B__deserialize + * Function: H5B__cache_deserialize * * Purpose: Deserialize the data structure from disk. * @@ -142,7 +142,7 @@ H5B__get_load_size(const void *_udata, size_t *image_len) *------------------------------------------------------------------------- */ static void * -H5B__deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata, +H5B__cache_deserialize(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty) { H5B_t *bt = NULL; /* Pointer to the deserialized B-tree node */ @@ -231,11 +231,11 @@ done: HDONE_ERROR(H5E_BTREE, H5E_CANTFREE, NULL, "unable to destroy B-tree node") FUNC_LEAVE_NOAPI(ret_value) -} /* end H5B__deserialize() */ +} /* end H5B__cache_deserialize() */ /*------------------------------------------------------------------------- - * Function: H5B__image_len + * Function: H5B__cache_image_len * * Purpose: Compute the size of the data structure on disk. * @@ -248,8 +248,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B__image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5B__cache_image_len(const void *_thing, size_t *image_len) { const H5B_t *bt = (const H5B_t *)_thing; /* Pointer to the B-tree node */ H5B_shared_t *shared; /* Pointer to shared B-tree info */ @@ -268,11 +267,11 @@ H5B__image_len(const void *_thing, size_t *image_len, *image_len = shared->sizeof_rnode; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5B__image_len() */ +} /* end H5B__cache_image_len() */ /*------------------------------------------------------------------------- - * Function: H5B__serialize + * Function: H5B__cache_serialize * * Purpose: Serialize the data structure for writing to disk. * @@ -285,7 +284,7 @@ H5B__image_len(const void *_thing, size_t *image_len, *------------------------------------------------------------------------- */ static herr_t -H5B__serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len, +H5B__cache_serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len, void *_thing) { H5B_t *bt = (H5B_t *)_thing; /* Pointer to the B-tree node */ @@ -354,11 +353,11 @@ H5B__serialize(const H5F_t *f, void *_image, size_t H5_ATTR_UNUSED len, done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5B__serialize() */ +} /* end H5B__cache_serialize() */ /*------------------------------------------------------------------------- - * Function: H5B__free_icr + * Function: H5B__cache_free_icr * * Purpose: Destroy/release an "in core representation" of a data structure * @@ -371,7 +370,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5B__free_icr(void *thing) +H5B__cache_free_icr(void *thing) { herr_t ret_value = SUCCEED; /* Return value */ @@ -386,5 +385,5 @@ H5B__free_icr(void *thing) done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5B__free_icr() */ +} /* end H5B__cache_free_icr() */ @@ -167,6 +167,9 @@ static herr_t H5C__mark_flush_dep_dirty(H5C_cache_entry_t * entry); static herr_t H5C__mark_flush_dep_clean(H5C_cache_entry_t * entry); +static herr_t H5C__verify_len_eoa(H5F_t *f, const H5C_class_t * type, + haddr_t addr, size_t *len, hbool_t actual); + static herr_t H5C__generate_image(const H5F_t *f, H5C_t * cache_ptr, H5C_cache_entry_t *entry_ptr, hid_t dxpl_id); @@ -776,8 +779,7 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, #if H5C_DO_EXTREME_SANITY_CHECKS if(H5C_validate_lru_list(cache_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, \ - "LRU extreme sanity check failed on entry.\n"); + HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "LRU extreme sanity check failed on entry.\n"); #endif /* H5C_DO_EXTREME_SANITY_CHECKS */ /* Look for entry in cache */ @@ -806,7 +808,7 @@ H5C_expunge_entry(H5F_t *f, hid_t dxpl_id, const H5C_class_t *type, * This will clear the entry, and then delete it from the cache. */ - /* Pass along 'free file space' flag to cache client. */ + /* Pass along 'free file space' flag */ flush_flags |= (flags & H5C__FREE_FILE_SPACE_FLAG); /* Delete the entry from the skip list on destroy */ @@ -1273,24 +1275,10 @@ H5C_insert_entry(H5F_t * f, /* not protected, so can't be dirtied */ entry_ptr->dirtied = FALSE; - /* Retrieve the size of the thing. Set the compressed field to FALSE - * and the compressed_size field to zero first, as they may not be - * initialized by the image_len call. - */ - entry_ptr->compressed = FALSE; - entry_ptr->compressed_size = 0; - if((type->image_len)(thing, &(entry_ptr->size), &(entry_ptr->compressed), &(entry_ptr->compressed_size)) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGETSIZE, FAIL, "Can't get size of thing") + /* Retrieve the size of the thing */ + if((type->image_len)(thing, &(entry_ptr->size)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGETSIZE, FAIL, "can't get size of thing") HDassert(entry_ptr->size > 0 && entry_ptr->size < H5C_MAX_ENTRY_SIZE); - HDassert(((type->flags & H5C__CLASS_COMPRESSED_FLAG) != 0) || - (entry_ptr->compressed == FALSE)); - - /* entry has just been inserted -- thus compressed size cannot have - * been computed yet. Thus if entry_ptr->compressed is TRUE, - * entry_ptr->size must equal entry_ptr->compressed_size. - */ - HDassert(!entry_ptr->compressed || (entry_ptr->size == entry_ptr->compressed_size)); - HDassert(entry_ptr->compressed || (entry_ptr->compressed_size == 0)); entry_ptr->in_slist = FALSE; @@ -2119,22 +2107,14 @@ H5C_protect(H5F_t * f, if(entry_ptr->image_ptr == NULL) { int mpi_rank; - size_t image_size; if((mpi_rank = H5F_mpi_get_rank(f)) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "Can't get MPI rank") - if(entry_ptr->compressed) - image_size = entry_ptr->compressed_size; - else - image_size = entry_ptr->size; - HDassert(image_size > 0); - - if(NULL == (entry_ptr->image_ptr = H5MM_malloc(image_size + H5C_IMAGE_EXTRA_SPACE))) + if(NULL == (entry_ptr->image_ptr = H5MM_malloc(entry_ptr->size + H5C_IMAGE_EXTRA_SPACE))) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "memory allocation failed for on disk image buffer") #if H5C_DO_MEMORY_SANITY_CHECKS - HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + image_size, - H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); + HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ if(0 == mpi_rank) if(H5C__generate_image(f, cache_ptr, entry_ptr, dxpl_id) < 0) @@ -5862,19 +5842,10 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ #endif /* H5C_DO_SANITY_CHECKS */ if(NULL == entry_ptr->image_ptr) { - size_t image_size; - - if(entry_ptr->compressed) - image_size = entry_ptr->compressed_size; - else - image_size = entry_ptr->size; - HDassert(image_size > 0); - - - if(NULL == (entry_ptr->image_ptr = H5MM_malloc(image_size + H5C_IMAGE_EXTRA_SPACE))) + if(NULL == (entry_ptr->image_ptr = H5MM_malloc(entry_ptr->size + H5C_IMAGE_EXTRA_SPACE))) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") #if H5C_DO_MEMORY_SANITY_CHECKS - HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + image_size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); + HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ } /* end if */ @@ -5890,19 +5861,7 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ * in the entry's type, we silently skip the write. This * flag should only be used in test code. */ - if(((entry_ptr->type->flags) & H5C__CLASS_SKIP_WRITES) == 0) - { - /* If compression is not enabled, the size of the entry on - * disk is entry_prt->size. However if entry_ptr->compressed - * is TRUE, the on disk size is entry_ptr->compressed_size. - */ - size_t image_size; - - if(entry_ptr->compressed) - image_size = entry_ptr->compressed_size; - else - image_size = entry_ptr->size; - + if(((entry_ptr->type->flags) & H5C__CLASS_SKIP_WRITES) == 0) { #ifdef H5_HAVE_PARALLEL if(collective_write_list) { H5C_collective_write_t *item; @@ -5910,7 +5869,7 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ if(NULL == (item = (H5C_collective_write_t *)H5FL_MALLOC(H5C_collective_write_t))) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "unable to allocate skip list item") - item->length = image_size; + item->length = entry_ptr->size; item->free_buf = FALSE; item->buf = entry_ptr->image_ptr; item->offset = entry_ptr->addr; @@ -5922,10 +5881,9 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ } /* end if */ else #endif /* H5_HAVE_PARALLEL */ - if(H5F_block_write(f, entry_ptr->type->mem_type, entry_ptr->addr, - image_size, dxpl_id, entry_ptr->image_ptr) < 0) + if(H5F_block_write(f, entry_ptr->type->mem_type, entry_ptr->addr, entry_ptr->size, dxpl_id, entry_ptr->image_ptr) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "Can't write image to file.") - } + } /* end if */ /* if the entry has a notify callback, notify it that we have * just flushed the entry. @@ -6004,7 +5962,7 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ /* verify that the entry is no longer part of any flush dependencies */ HDassert(entry_ptr->flush_dep_nparents == 0); HDassert(entry_ptr->flush_dep_nchildren == 0); - } + } /* end if */ else { HDassert(clear_only || write_entry); HDassert(entry_ptr->is_dirty); @@ -6031,9 +5989,6 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ H5C__UPDATE_INDEX_FOR_ENTRY_CLEAN(cache_ptr, entry_ptr); - if(entry_ptr->type->clear && (entry_ptr->type->clear)(f, (void *)entry_ptr, FALSE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear entry") - /* Check for entry changing status and do notifications, etc. */ if(was_dirty) { /* If the entry's type has a 'notify' callback send a 'entry cleaned' @@ -6082,30 +6037,22 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ HDassert(!H5F_IS_TMP_ADDR(f, entry_ptr->addr)); #ifndef NDEBUG { - hbool_t curr_compressed = FALSE; size_t curr_len; - size_t curr_compressed_len = 0; /* Get the actual image size for the thing again */ - entry_ptr->type->image_len((void *)entry_ptr, &curr_len, &curr_compressed, &curr_compressed_len); + entry_ptr->type->image_len((void *)entry_ptr, &curr_len); HDassert(curr_len == entry_ptr->size); - HDassert(curr_compressed == entry_ptr->compressed); - HDassert(curr_compressed_len == entry_ptr->compressed_size); } #endif /* NDEBUG */ - /* if the file space free size callback is defined, use + /* If the file space free size callback is defined, use * it to get the size of the block of file space to free. - * Otherwise use entry_ptr->compressed_size if - * entry_ptr->compressed == TRUE, and entry_ptr->size - * if entry_ptr->compressed == FALSE. + * Otherwise use entry_ptr->size. */ if(entry_ptr->type->fsf_size) { if((entry_ptr->type->fsf_size)((void *)entry_ptr, &fsf_size) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFREE, FAIL, "unable to get file space free size") } /* end if */ - else if(entry_ptr->compressed) /* use compressed size */ - fsf_size = entry_ptr->compressed_size; else /* no file space free size callback -- use entry size */ fsf_size = entry_ptr->size; @@ -6140,19 +6087,10 @@ H5C__flush_single_entry(const H5F_t *f, hid_t dxpl_id, H5C_cache_entry_t *entry_ /* Check for actually destroying the entry in memory */ /* (As opposed to taking ownership of it) */ if(destroy_entry) { - /* if the entry is dirty and it has a clear callback, - * call this callback now. Since this callback exists, - * it follows tht the client maintains its own dirty bits, - * which must be cleared before the entry is freed to avoid - * sanity check failures. Also clear the dirty flag for - * the same reason. - */ if(entry_ptr->is_dirty) { + /* Reset dirty flag */ entry_ptr->is_dirty = FALSE; - if(entry_ptr->type->clear && (entry_ptr->type->clear)(f, (void *)entry_ptr, TRUE) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to clear entry") - /* If the entry's type has a 'notify' callback send a 'entry cleaned' * notice now that the entry is fully integrated into the cache. */ @@ -6201,6 +6139,70 @@ done: /*------------------------------------------------------------------------- * + * Function: H5C__verify_len_eoa + * + * Purpose: Verify that 'len' does not exceed eoa when 'actual' is + * false i.e. 'len" is the initial speculative length from + * get_load_size callback with null image pointer. + * If exceed, adjust 'len' accordingly. + * + * Verify that 'len' should not exceed eoa when 'actual' is + * true i.e. 'len' is the actual length from get_load_size + * callback with non-null image pointer. + * If exceed, return error. + * + * Return: FAIL if error is detected, SUCCEED otherwise. + * + * Programmer: Vailin Choi + * 9/6/15 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5C__verify_len_eoa(H5F_t *f, const H5C_class_t *type, haddr_t addr, + size_t *len, hbool_t actual) +{ + H5FD_mem_t cooked_type; /* Modified type, accounting for switching global heaps */ + haddr_t eoa; /* End-of-allocation in the file */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* if type == H5FD_MEM_GHEAP, H5F_block_read() forces + * type to H5FD_MEM_DRAW via its call to H5F__accum_read(). + * Thus we do the same for purposes of computing the EOA + * for sanity checks. + */ + cooked_type = (type->mem_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type->mem_type; + + /* Get the file's end-of-allocation value */ + eoa = H5F_get_eoa(f, cooked_type); + if(!H5F_addr_defined(eoa)) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "invalid EOA address for file") + + /* Check for bad address in general */ + if(H5F_addr_gt(addr, eoa)) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "address of object past end of allocation") + + /* Check if the amount of data to read will be past the EOA */ + if(H5F_addr_gt((addr + *len), eoa)) { + if(actual) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "actual len exceeds EOA.") + else + /* Trim down the length of the metadata */ + *len = (size_t)(eoa - addr); + } /* end if */ + + if(*len <= 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, FAIL, "len not positive after adjustment for EOA.") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5C__verify_len_eoa() */ + + +/*------------------------------------------------------------------------- + * * Function: H5C_load_entry * * Purpose: Attempt to load the entry at the specified disk address @@ -6228,15 +6230,7 @@ H5C_load_entry(H5F_t * f, void * udata) { hbool_t dirty = FALSE; /* Flag indicating whether thing was dirtied during deserialize */ - hbool_t compressed = FALSE; /* flag indicating whether thing */ - /* will be run through filters on */ - /* on read and write. Usually FALSE */ - /* set to true if appropriate. */ - size_t compressed_size = 0; /* entry compressed size if */ - /* known -- otherwise uncompressed. */ - /* Zero indicates compression not */ - /* enabled. */ - void * image = NULL; /* Buffer for disk image */ + uint8_t * image = NULL; /* Buffer for disk image */ void * thing = NULL; /* Pointer to thing loaded */ H5C_cache_entry_t *entry = NULL; /* Alias for thing loaded, as cache entry */ size_t len; /* Size of image in file */ @@ -6249,76 +6243,38 @@ H5C_load_entry(H5F_t * f, FUNC_ENTER_NOAPI_NOINIT + /* Sanity checks */ HDassert(f); HDassert(f->shared); HDassert(f->shared->cache); HDassert(type); - - /* for now, we do not combine the speculative load and compressed flags */ - HDassert(!((type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) && - (type->flags & H5C__CLASS_COMPRESSED_FLAG))); + HDassert(H5F_addr_defined(addr)); + HDassert(type->get_initial_load_size); + if(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) + HDassert(type->get_final_load_size); + else + HDassert(NULL == type->get_final_load_size); + HDassert(type->deserialize); /* Can't see how skip reads could be usefully combined with - * either the speculative read or compressed flags. Hence disallow. + * the speculative read flag. Hence disallow. */ HDassert(!((type->flags & H5C__CLASS_SKIP_READS) && (type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG))); - HDassert(!((type->flags & H5C__CLASS_SKIP_READS) && - (type->flags & H5C__CLASS_COMPRESSED_FLAG))); - HDassert(H5F_addr_defined(addr)); - HDassert(type->get_load_size); - HDassert(type->deserialize); - - /* Call the get_load_size callback, to retrieve the initial - * size of image - */ - if(type->get_load_size(udata, &len) < 0) + /* Call the get_initial_load_size callback, to retrieve the initial size of image */ + if(type->get_initial_load_size(udata, &len) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "can't retrieve image size") - HDassert(len > 0); /* Check for possible speculative read off the end of the file */ - if(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) { - - haddr_t eoa; /* End-of-allocation in the file */ - H5FD_mem_t cooked_type; - - /* if type == H5FD_MEM_GHEAP, H5F_block_read() forces - * type to H5FD_MEM_DRAW via its call to H5F__accum_read(). - * Thus we do the same for purposes of computing the eoa - * for sanity checks. - */ - cooked_type = (type->mem_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type->mem_type; - - /* Get the file's end-of-allocation value */ - eoa = H5F_get_eoa(f, cooked_type); - - HDassert(H5F_addr_defined(eoa)); - - /* Check for bad address in general */ - if ( H5F_addr_gt(addr, eoa) ) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "address of object past end of allocation") - - /* Check if the amount of data to read will be past the eoa */ - if( H5F_addr_gt((addr + len), eoa) ) { - - /* Trim down the length of the metadata */ - /* Note that for some cache clients, this will cause an - * assertion failure. JRM -- 8/29/14 - */ - len = (size_t)(eoa - addr); - } /* end if */ - - if ( len <= 0 ) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "len not positive after adjustment for EOA.") - - } /* end if */ + if(type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) + if(H5C__verify_len_eoa(f, type, addr, &len, FALSE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "invalid len with respect to EOA") /* Allocate the buffer for reading the on-disk entry image */ - if(NULL == (image = H5MM_malloc(len + H5C_IMAGE_EXTRA_SPACE))) + if(NULL == (image = (uint8_t *)H5MM_malloc(len + H5C_IMAGE_EXTRA_SPACE))) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "memory allocation failed for on disk image buffer.") - #if H5C_DO_MEMORY_SANITY_CHECKS HDmemcpy(image + len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ @@ -6334,194 +6290,144 @@ H5C_load_entry(H5F_t * f, /* Get the on-disk entry image */ if(0 == (type->flags & H5C__CLASS_SKIP_READS)) { + unsigned tries, max_tries; /* The # of read attempts */ + unsigned retries; /* The # of retries */ + htri_t chk_ret; /* return from verify_chksum callback */ + size_t actual_len = len; /* The actual length, after speculative reads have been resolved */ + uint64_t nanosec = 1; /* # of nanoseconds to sleep between retries */ + void *new_image; /* Pointer to image */ + hbool_t len_changed = TRUE; /* Whether to re-check speculative entries */ + + /* Get the # of read attempts */ + max_tries = tries = H5F_GET_READ_ATTEMPTS(f); + + /* + * This do/while loop performs the following till the metadata checksum + * is correct or the file's number of allowed read attempts are reached. + * --read the metadata + * --determine the actual size of the metadata + * --perform checksum verification + */ + do { + if(actual_len != len) { + if(NULL == (new_image = H5MM_realloc(image, len + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "image null after H5MM_realloc()") + image = (uint8_t *)new_image; +#if H5C_DO_MEMORY_SANITY_CHECKS + HDmemcpy(image + len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); +#endif /* H5C_DO_MEMORY_SANITY_CHECKS */ + } /* end if */ + #ifdef H5_HAVE_PARALLEL - if(!coll_access || 0 == mpi_rank) { + if(!coll_access || 0 == mpi_rank) { #endif /* H5_HAVE_PARALLEL */ - - if(H5F_block_read(f, type->mem_type, addr, len, dxpl_id, image) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_READERROR, NULL, "Can't read image*") - + if(H5F_block_read(f, type->mem_type, addr, len, dxpl_id, image) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_READERROR, NULL, "Can't read image*") #ifdef H5_HAVE_PARALLEL - } /* end if */ - - /* if the collective metadata read optimization is turned on, - * bcast the metadata read from process 0 to all ranks in the file - * communicator - */ - if(coll_access) { - int buf_size; + } /* end if */ + /* if the collective metadata read optimization is turned on, + * bcast the metadata read from process 0 to all ranks in the file + * communicator + */ + if(coll_access) { + int buf_size; - H5_CHECKED_ASSIGN(buf_size, int, len, size_t); - if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image, buf_size, MPI_BYTE, 0, comm))) - HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) - } /* end if */ + H5_CHECKED_ASSIGN(buf_size, int, len, size_t); + if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image, buf_size, MPI_BYTE, 0, comm))) + HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) + } /* end if */ #endif /* H5_HAVE_PARALLEL */ - } /* end if */ - - /* Deserialize the on-disk image into the native memory form */ - if(NULL == (thing = type->deserialize(image, len, udata, &dirty))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't deserialize image") - - /* If the client's cache has an image_len callback, check it */ - if(type->image_len) { - size_t new_len; /* New size of on-disk image */ - - /* set magic and type field in *entry_ptr. While the image_len - * callback shouldn't touch the cache specific fields, it may check - * these fields to ensure that it it has received the expected - * value. - * - * Note that this initialization is repeated below on the off - * chance that we had to re-try the deserialization. - */ - entry = (H5C_cache_entry_t *)thing; - entry->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC; - entry->type = type; - - /* verify that compressed and compressed_len are initialized */ - HDassert(compressed == FALSE); - HDassert(compressed_size == 0); - - /* Get the actual image size for the thing */ - if(type->image_len(thing, &new_len, &compressed, &compressed_size) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "can't retrieve image length") - - if(new_len == 0) - HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "image length is 0") - - HDassert(((type->flags & H5C__CLASS_COMPRESSED_FLAG) != 0) || - ((compressed == FALSE) && (compressed_size == 0))); - HDassert((compressed == TRUE) || (compressed_size == 0)); - - if(new_len != len) { - - if(type->flags & H5C__CLASS_COMPRESSED_FLAG) { - - /* if new_len != len, then compression must be - * enabled on the entry. In this case, the image_len - * callback should have set compressed to TRUE, set - * new_len equal to the uncompressed size of the - * entry, and compressed_len equal to the compressed - * size -- which must equal len. - * - * We can't verify the uncompressed size, but we can - * verify the rest with the following assertions. - */ - HDassert(compressed); - HDassert(compressed_size == len); - - /* new_len should contain the uncompressed size. Set len - * equal to new_len, so that the cache will use the - * uncompressed size for purposes of space allocation, etc. - */ - len = new_len; - - } else if (type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) { - - void *new_image; /* Buffer for disk image */ - - /* compressed must be FALSE, and compressed_size - * must be zero. - */ - HDassert(!compressed); - HDassert(compressed_size == 0); - - /* Adjust the size of the image to match new_len */ - if(NULL == (new_image = H5MM_realloc(image, - new_len + H5C_IMAGE_EXTRA_SPACE))) - - HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "image null after H5MM_realloc()") - - image = new_image; + /* If the entry could be read speculatively and the length is still + * changing, check for updating the actual size + */ + if((type->flags & H5C__CLASS_SPECULATIVE_LOAD_FLAG) && len_changed) { + /* Retrieve the actual length */ + actual_len = len; + if(type->get_final_load_size(image, len, udata, &actual_len) < 0) + continue; /* Transfer control to while() and count towards retries */ + + /* Check for the length changing */ + if(actual_len != len) { + /* Verify that the length isn't past the EOA for the file */ + if(H5C__verify_len_eoa(f, type, addr, &actual_len, TRUE) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "actual_len exceeds EOA.") + + /* Expand buffer to new size */ + if(NULL == (new_image = H5MM_realloc(image, actual_len + H5C_IMAGE_EXTRA_SPACE))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, NULL, "image null after H5MM_realloc()") + image = (uint8_t *)new_image; #if H5C_DO_MEMORY_SANITY_CHECKS - HDmemcpy(((uint8_t *)image) + new_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); + HDmemcpy(image + actual_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ - /* If the thing's image needs to be bigger for a speculatively - * loaded thing, free the thing and retry with new length - */ - if (new_len > len) { - - /* Release previous (possibly partially initialized) - * thing. Note that we must set entry->magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC and set one or - * two other fields before the call to free_icr - * so as to avoid sanity check failures. - */ - entry->magic = H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC; - - entry->addr = addr; - - if ( type->free_icr(thing) < 0 ) - - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "free_icr callback failed") - + if(actual_len > len) { #ifdef H5_HAVE_PARALLEL - if(!coll_access || 0 == mpi_rank) { + if(!coll_access || 0 == mpi_rank) { #endif /* H5_HAVE_PARALLEL */ - - /* Go get the on-disk image again */ - if(H5F_block_read(f, type->mem_type, addr, - new_len, dxpl_id, image) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't read image") - + /* If the thing's image needs to be bigger for a speculatively + * loaded thing, go get the on-disk image again (the extra portion). + */ + if(H5F_block_read(f, type->mem_type, addr + len, actual_len - len, dxpl_id, image + len) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "can't read image") #ifdef H5_HAVE_PARALLEL - } - /* if the collective metadata read optimization is turned on, - bcast the metadata read from process 0 to all ranks in the file - communicator */ - if(coll_access) { - int buf_size; - - H5_CHECKED_ASSIGN(buf_size, int, new_len, size_t); - if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image, buf_size, MPI_BYTE, 0, comm))) - HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) - } /* end if */ + } + /* If the collective metadata read optimization is turned on, + * Bcast the metadata read from process 0 to all ranks in the file + * communicator */ + if(coll_access) { + int buf_size; + + H5_CHECKED_ASSIGN(buf_size, int, actual_len - len, size_t); + if(MPI_SUCCESS != (mpi_code = MPI_Bcast(image + len, buf_size, MPI_BYTE, 0, comm))) + HMPI_GOTO_ERROR(NULL, "MPI_Bcast failed", mpi_code) + } /* end if */ #endif /* H5_HAVE_PARALLEL */ + } /* end if */ + } /* end if (actual_len != len) */ + else { + /* The length has stabilized */ + len_changed = FALSE; - /* Deserialize on-disk image into native memory form again */ - if(NULL == (thing = type->deserialize(image, new_len, udata, &dirty))) - HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't deserialize image") + /* Set the final length */ + len = actual_len; + } /* else */ + } /* end if */ -#ifndef NDEBUG - { - /* new_compressed and new_compressed_size must be - * initialize to FALSE / 0 respectively, as clients - * that don't use compression may ignore these two - * parameters. - */ - hbool_t new_compressed = FALSE; - size_t new_compressed_size = 0; - size_t new_new_len; - - /* Get the actual image size for the thing again. Note - * that since this is a new thing, we have to set - * the magic and type fields again so as to avoid - * failing sanity checks. - */ - entry = (H5C_cache_entry_t *)thing; - entry->magic = H5C__H5C_CACHE_ENTRY_T_MAGIC; - entry->type = type; - - type->image_len(thing, &new_new_len, &new_compressed, &new_compressed_size); - HDassert(new_new_len == new_len); - HDassert(!new_compressed); - HDassert(new_compressed_size == 0); - } /* end block */ -#endif /* NDEBUG */ - } /* end if (new_len > len) */ + /* If there's no way to verify the checksum for a piece of metadata + * (usually because there's no checksum in the file), leave now + */ + if(type->verify_chksum == NULL) + break; - /* Retain adjusted size */ - len = new_len; + /* Verify the checksum for the metadata image */ + if((chk_ret = type->verify_chksum(image, actual_len, udata)) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTGET, NULL, "failure from verify_chksum callback") + if(chk_ret == TRUE) + break; - } else { /* throw an error */ + /* Sleep for some time */ + H5_nanosleep(nanosec); + nanosec *= 2; /* Double the sleep time next time */ + } while(--tries); - HGOTO_ERROR(H5E_CACHE, H5E_UNSUPPORTED, NULL, \ - "size of non-speculative, non-compressed object changed") - } - } /* end if (new_len != len) */ - } /* end if */ + /* Check for too many tries */ + if(tries == 0) + HGOTO_ERROR(H5E_CACHE, H5E_READERROR, NULL, "incorrect metadatda checksum after all read attempts") + + /* Calculate and track the # of retries */ + retries = max_tries - tries; + if(retries) /* Does not track 0 retry */ + if(H5F_track_metadata_read_retries(f, (unsigned)type->mem_type, retries) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_BADVALUE, NULL, "cannot track read tries = %u ", retries) + + /* Set the final length (in case it wasn't set earlier) */ + len = actual_len; + } /* end if !H5C__CLASS_SKIP_READS */ + + /* Deserialize the on-disk image into the native memory form */ + if(NULL == (thing = type->deserialize(image, len, udata, &dirty))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTLOAD, NULL, "Can't deserialize image") entry = (H5C_cache_entry_t *)thing; @@ -6551,8 +6457,6 @@ H5C_load_entry(H5F_t * f, entry->addr = addr; entry->size = len; HDassert(entry->size < H5C_MAX_ENTRY_SIZE); - entry->compressed = compressed; - entry->compressed_size = compressed_size; entry->image_ptr = image; entry->image_up_to_date = TRUE; entry->type = type; @@ -6605,7 +6509,7 @@ done: if(thing && type->free_icr(thing) < 0) HDONE_ERROR(H5E_CACHE, H5E_CANTFLUSH, NULL, "free_icr callback failed") if(image) - image = H5MM_xfree(image); + image = (uint8_t *)H5MM_xfree(image); } /* end if */ FUNC_LEAVE_NOAPI(ret_value) @@ -7710,7 +7614,6 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p haddr_t new_addr = HADDR_UNDEF; haddr_t old_addr = HADDR_UNDEF; size_t new_len = 0; - size_t new_compressed_len = 0; unsigned serialize_flags = H5C__SERIALIZE_NO_FLAGS_SET; herr_t ret_value = SUCCEED; @@ -7724,18 +7627,15 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p /* Call client's pre-serialize callback, if there's one */ if(entry_ptr->type->pre_serialize && - (entry_ptr->type->pre_serialize)(f, dxpl_id, - (void *)entry_ptr, entry_ptr->addr, entry_ptr->size, - entry_ptr->compressed_size, &new_addr, &new_len, - &new_compressed_len, &serialize_flags) < 0) + (entry_ptr->type->pre_serialize)(f, dxpl_id, (void *)entry_ptr, + entry_ptr->addr, entry_ptr->size, &new_addr, &new_len, &serialize_flags) < 0) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to pre-serialize entry") /* Check for any flags set in the pre-serialize callback */ if(serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) { /* Check for unexpected flags from serialize callback */ if(serialize_flags & ~(H5C__SERIALIZE_RESIZED_FLAG | - H5C__SERIALIZE_MOVED_FLAG | - H5C__SERIALIZE_COMPRESSED_FLAG)) + H5C__SERIALIZE_MOVED_FLAG)) HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unknown serialize flag(s)") #ifdef H5_HAVE_PARALLEL @@ -7769,49 +7669,31 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p HGOTO_ERROR(H5E_CACHE, H5E_SYSTEM, FAIL, "resize/move in serialize occured in parallel case.") #endif - /* Resize the buffer if required */ - if(((!entry_ptr->compressed) && (serialize_flags & H5C__SERIALIZE_RESIZED_FLAG)) || - ((entry_ptr->compressed) && (serialize_flags & H5C__SERIALIZE_COMPRESSED_FLAG))) { - size_t new_image_size; - - if(entry_ptr->compressed) - new_image_size = new_compressed_len; - else - new_image_size = new_len; - HDassert(new_image_size > 0); - - /* Release the current image */ - if(entry_ptr->image_ptr) - entry_ptr->image_ptr = H5MM_xfree(entry_ptr->image_ptr); - + /* If required, resize the buffer and update the entry and the cache + * data structures */ + if(serialize_flags & H5C__SERIALIZE_RESIZED_FLAG) { /* Allocate a new image buffer */ - if(NULL == (entry_ptr->image_ptr = H5MM_malloc(new_image_size + H5C_IMAGE_EXTRA_SPACE))) + if(NULL == (entry_ptr->image_ptr = H5MM_realloc(entry_ptr->image_ptr, new_len + H5C_IMAGE_EXTRA_SPACE))) HGOTO_ERROR(H5E_CACHE, H5E_CANTALLOC, FAIL, "memory allocation failed for on disk image buffer") - #if H5C_DO_MEMORY_SANITY_CHECKS - HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + new_image_size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); + HDmemcpy(((uint8_t *)entry_ptr->image_ptr) + new_len, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ - } /* end if */ - /* If required, update the entry and the cache data structures - * for a resize. - */ - if(serialize_flags & H5C__SERIALIZE_RESIZED_FLAG) { + /* Update statistics for resizing the entry */ H5C__UPDATE_STATS_FOR_ENTRY_SIZE_CHANGE(cache_ptr, entry_ptr, new_len); - /* update the hash table for the size change*/ + /* update the hash table for the size change */ H5C__UPDATE_INDEX_FOR_SIZE_CHANGE(cache_ptr, entry_ptr->size, \ new_len, entry_ptr, !(entry_ptr->is_dirty)); - /* The entry can't be protected since we are - * in the process of flushing it. Thus we must - * update the replacement policy data - * structures for the size change. The macro - * deals with the pinned case. + /* The entry can't be protected since we are in the process of + * flushing it. Thus we must update the replacement policy data + * structures for the size change. The macro deals with the pinned + * case. */ H5C__UPDATE_RP_FOR_SIZE_CHANGE(cache_ptr, entry_ptr, new_len); - /* as we haven't updated the cache data structures for + /* As we haven't updated the cache data structures for * for the flush or flush destroy yet, the entry should * be in the slist. Thus update it for the size change. */ @@ -7828,12 +7710,9 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p if(serialize_flags & H5C__SERIALIZE_MOVED_FLAG) { H5C__UPDATE_STATS_FOR_MOVE(cache_ptr, entry_ptr); + /* We must update cache data structures for the change in address */ if(entry_ptr->addr == old_addr) { - /* we must update cache data structures for the - * change in address. - */ - - /* delete the entry from the hash table and the slist */ + /* Delete the entry from the hash table and the slist */ H5C__DELETE_FROM_INDEX(cache_ptr, entry_ptr); H5C__REMOVE_ENTRY_FROM_SLIST(cache_ptr, entry_ptr, FALSE); @@ -7847,35 +7726,15 @@ H5C__generate_image(const H5F_t *f, H5C_t *cache_ptr, H5C_cache_entry_t *entry_p else /* move is already done for us -- just do sanity checks */ HDassert(entry_ptr->addr == new_addr); } /* end if */ - - if(serialize_flags & H5C__SERIALIZE_COMPRESSED_FLAG) { - /* just save the new compressed entry size in - * entry_ptr->compressed_size. We don't need to - * do more, as compressed size is only used for I/O. - */ - HDassert(entry_ptr->compressed); - entry_ptr->compressed_size = new_compressed_len; - } /* end if */ } /* end if(serialize_flags != H5C__SERIALIZE_NO_FLAGS_SET) */ /* Serialize object into buffer */ - { - size_t image_len; - - if(entry_ptr->compressed) - image_len = entry_ptr->compressed_size; - else - image_len = entry_ptr->size; - - if(entry_ptr->type->serialize(f, entry_ptr->image_ptr, image_len, (void *)entry_ptr) < 0) - HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to serialize entry") - + if(entry_ptr->type->serialize(f, entry_ptr->image_ptr, entry_ptr->size, (void *)entry_ptr) < 0) + HGOTO_ERROR(H5E_CACHE, H5E_CANTFLUSH, FAIL, "unable to serialize entry") #if H5C_DO_MEMORY_SANITY_CHECKS - HDassert(0 == HDmemcmp(((uint8_t *)entry_ptr->image_ptr) + image_len, - H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE)); + HDassert(0 == HDmemcmp(((uint8_t *)entry_ptr->image_ptr) + entry_ptr->size, H5C_IMAGE_SANITY_VALUE, H5C_IMAGE_EXTRA_SPACE)); #endif /* H5C_DO_MEMORY_SANITY_CHECKS */ - entry_ptr->image_up_to_date = TRUE; - } /* end block */ + entry_ptr->image_up_to_date = TRUE; done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Cepoch.c b/src/H5Cepoch.c index 7c5e441..3726aa1 100644 --- a/src/H5Cepoch.c +++ b/src/H5Cepoch.c @@ -63,22 +63,23 @@ * epochs so that they can be evicted from the cache. * ****************************************************************************/ -static herr_t H5C__epoch_marker_get_load_size(const void *udata_ptr, +static herr_t H5C__epoch_marker_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t H5C__epoch_marker_get_final_load_size(const void *image_ptr, + size_t image_len_ptr, void *udata_ptr, size_t *actual_len); +static htri_t H5C__epoch_marker_verify_chksum(const void *image_ptr, + size_t len, void *udata_ptr); static void * H5C__epoch_marker_deserialize(const void * image_ptr, size_t len, void * udata, hbool_t * dirty_ptr); static herr_t H5C__epoch_marker_image_len(const void * thing, - size_t *image_len_ptr, hbool_t *compressed_ptr, size_t *compressed_len_ptr); + size_t *image_len_ptr); static herr_t H5C__epoch_marker_pre_serialize(const H5F_t *f, hid_t dxpl_id, void * thing, haddr_t addr, size_t len, - size_t compressed_len, haddr_t * new_addr_ptr, size_t * new_len_ptr, - size_t * new_compressed_len_ptr, unsigned * flags_ptr); + haddr_t * new_addr_ptr, size_t * new_len_ptr, unsigned * flags_ptr); static herr_t H5C__epoch_marker_serialize(const H5F_t *f, void * image_ptr, size_t len, void * thing); static herr_t H5C__epoch_marker_notify(H5C_notify_action_t action, void *thing); static herr_t H5C__epoch_marker_free_icr(void * thing); -static herr_t H5C__epoch_marker_clear(const H5F_t *f, void * thing, - hbool_t about_to_destroy); static herr_t H5C__epoch_marker_fsf_size(const void H5_ATTR_UNUSED * thing, size_t H5_ATTR_UNUSED * fsf_size_ptr); @@ -104,14 +105,15 @@ const H5C_class_t H5C__epoch_marker_class = /* name = */ "epoch marker", /* mem_type = */ H5FD_MEM_DEFAULT, /* value doesn't matter */ /* flags = */ H5C__CLASS_NO_FLAGS_SET, - /* get_load_size = */ H5C__epoch_marker_get_load_size, + /* get_initial_load_size = */ H5C__epoch_marker_get_initial_load_size, + /* get_final_load_size = */ H5C__epoch_marker_get_final_load_size, + /* verify_chksum = */ H5C__epoch_marker_verify_chksum, /* deserialize = */ H5C__epoch_marker_deserialize, /* image_len = */ H5C__epoch_marker_image_len, /* pre_serialize = */ H5C__epoch_marker_pre_serialize, /* serialize = */ H5C__epoch_marker_serialize, /* notify = */ H5C__epoch_marker_notify, /* free_icr = */ H5C__epoch_marker_free_icr, - /* clear = */ H5C__epoch_marker_clear, /* fsf_size = */ H5C__epoch_marker_fsf_size, }; @@ -127,7 +129,7 @@ const H5C_class_t H5C__epoch_marker_class = static herr_t -H5C__epoch_marker_get_load_size(const void H5_ATTR_UNUSED *udata_ptr, +H5C__epoch_marker_get_initial_load_size(void H5_ATTR_UNUSED *udata_ptr, size_t H5_ATTR_UNUSED *image_len_ptr) { FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ @@ -135,7 +137,32 @@ H5C__epoch_marker_get_load_size(const void H5_ATTR_UNUSED *udata_ptr, HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn."); FUNC_LEAVE_NOAPI(FAIL) -} /* end H5C__epoch_marker_get_load_size() */ +} /* end H5C__epoch_marker_get_initial_load_size() */ + + +static herr_t +H5C__epoch_marker_get_final_load_size(const void H5_ATTR_UNUSED *image_ptr, + size_t H5_ATTR_UNUSED image_len, void H5_ATTR_UNUSED *udata_ptr, + size_t H5_ATTR_UNUSED *actual_len) +{ + FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ + + HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn."); + + FUNC_LEAVE_NOAPI(FAIL) +} /* end H5C__epoch_marker_final_get_load_size() */ + + +static htri_t +H5C__epoch_marker_verify_chksum(const void H5_ATTR_UNUSED *image_ptr, size_t H5_ATTR_UNUSED len, + void H5_ATTR_UNUSED *udata_ptr) +{ + FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ + + HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn."); + + FUNC_LEAVE_NOAPI(FALSE) +} /* end H5C__epoch_marker_verify_chksum() */ static void * @@ -152,8 +179,7 @@ H5C__epoch_marker_deserialize(const void H5_ATTR_UNUSED * image_ptr, size_t H5_A static herr_t H5C__epoch_marker_image_len(const void H5_ATTR_UNUSED *thing, - size_t H5_ATTR_UNUSED *image_len_ptr, hbool_t H5_ATTR_UNUSED *compressed_ptr, - size_t H5_ATTR_UNUSED *compressed_len_ptr) + size_t H5_ATTR_UNUSED *image_len_ptr) { FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ @@ -166,8 +192,7 @@ H5C__epoch_marker_image_len(const void H5_ATTR_UNUSED *thing, static herr_t H5C__epoch_marker_pre_serialize(const H5F_t H5_ATTR_UNUSED *f, hid_t H5_ATTR_UNUSED dxpl_id, void H5_ATTR_UNUSED *thing, haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t H5_ATTR_UNUSED *new_addr_ptr, - size_t H5_ATTR_UNUSED *new_len_ptr, size_t H5_ATTR_UNUSED *new_compressed_len_ptr, + haddr_t H5_ATTR_UNUSED *new_addr_ptr, size_t H5_ATTR_UNUSED *new_len_ptr, unsigned H5_ATTR_UNUSED *flags_ptr) { FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ @@ -214,17 +239,6 @@ H5C__epoch_marker_free_icr(void H5_ATTR_UNUSED * thing) static herr_t -H5C__epoch_marker_clear(const H5F_t H5_ATTR_UNUSED *f, void H5_ATTR_UNUSED * thing, hbool_t H5_ATTR_UNUSED about_to_destroy) -{ - FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ - - HERROR(H5E_CACHE, H5E_SYSTEM, "called unreachable fcn."); - - FUNC_LEAVE_NOAPI(FAIL) -} /* end H5C__epoch_marker_clear() */ - - -static herr_t H5C__epoch_marker_fsf_size(const void H5_ATTR_UNUSED * thing, size_t H5_ATTR_UNUSED *fsf_size_ptr) { FUNC_ENTER_STATIC_NOERR /* Yes, even though this pushes an error on the stack */ diff --git a/src/H5Cprivate.h b/src/H5Cprivate.h index e98bc07..c03e2e7 100644 --- a/src/H5Cprivate.h +++ b/src/H5Cprivate.h @@ -68,16 +68,14 @@ /* Flags for cache client class behavior */ #define H5C__CLASS_NO_FLAGS_SET ((unsigned)0x0) #define H5C__CLASS_SPECULATIVE_LOAD_FLAG ((unsigned)0x1) -#define H5C__CLASS_COMPRESSED_FLAG ((unsigned)0x2) /* The following flags may only appear in test code */ -#define H5C__CLASS_SKIP_READS ((unsigned)0x4) -#define H5C__CLASS_SKIP_WRITES ((unsigned)0x8) +#define H5C__CLASS_SKIP_READS ((unsigned)0x2) +#define H5C__CLASS_SKIP_WRITES ((unsigned)0x4) /* Flags for pre-serialize callback */ #define H5C__SERIALIZE_NO_FLAGS_SET ((unsigned)0) #define H5C__SERIALIZE_RESIZED_FLAG ((unsigned)0x1) #define H5C__SERIALIZE_MOVED_FLAG ((unsigned)0x2) -#define H5C__SERIALIZE_COMPRESSED_FLAG ((unsigned)0x4) /* Upper and lower limits on cache size. These limits are picked * out of a hat -- you should be able to change them as necessary. @@ -286,19 +284,12 @@ typedef struct H5C_t H5C_t; * * flags: Flags indicating class-specific behavior. * - * Whoever created the flags field neglected to document the meanings - * of the flags he created. Hence the following discussion of the - * H5C__CLASS_SPECULATIVE_LOAD_FLAG and (to a lesser extent) - * H5C__CLASS_COMPRESSED_FLAG should be viewed with suspicion, - * as the meanings are divined from the source code, and thus may be - * inaccurate. Please correct any errors you find. - * * Possible flags are: * * H5C__CLASS_NO_FLAGS_SET: No special processing. * - * H5C__CLASS_SPECULATIVE_LOAD_FLAG: This flag appears to be used - * only in H5C_load_entry(). When it is set, entries are + * H5C__CLASS_SPECULATIVE_LOAD_FLAG: This flag is used only in + * H5C_load_entry(). When it is set, entries are * permitted to change their sizes on the first attempt * to load. * @@ -312,56 +303,15 @@ typedef struct H5C_t H5C_t; * entry in the cache. * * If the new size is smaller than the old, no new loads - * or desearializes are performed, but the new size becomes + * or deserializes are performed, but the new size becomes * the size of the entry in the cache. * * When this flag is set, an attempt to read past the - * end of file is pemitted. In this case, if the size + * end of file could occur. In this case, if the size * returned get_load_size callback would result in a - * read past the end of file, the size is trunkated to + * read past the end of file, the size is truncated to * avoid this, and processing proceeds as normal. * - * H5C__CLASS_COMPRESSED_FLAG: This flags indicates that the entry - * may be compressed -- i.e. its on disk image is run through - * filters on the way to and from disk. Thus the uncompressed - * (or unfiltered) size of the entry may be different from the - * size of the entry in file. - * - * This has the following implications: - * - * On load, uncompressed size and load size may be different. - * Presumably, load size will be smaller than uncompressed - * size, but there is no requirement for this in the code - * (but note that I have inserted an assertion to this effect, - * which has not been triggered to date). - * - * On insertion, compressed (AKA filtered, AKA on disk) size - * is unknown, as the entry has yet to be run through filters. - * Compressed size is computed whenever the entry is - * written (or the image is updated -- not relevant until - * journaling is brought back). - * - * On dirty (of a clean entry), compressed (AKA filtered, - * AKA on disk) size becomes unknown. Thus, compressed size - * must be computed by the pre-serialize callback before the - * entry may be written. - * - * Once the compressed size becomes unknown, it remains so - * until the on disk image is constructed. - * - * Observe that the cache needs to know the size of the entry - * for space allocation purposes. Since the compressed size - * can change or become unknown, it uses the uncompressed - * size which may change, but which should always be known. - * The compressed size is used only for journaling and disk I/O. - * - * While there is no logical reason why they could not be - * combined, due to absence of need and for simplicity of code, - * the cache does not permit both the the - * H5C__CLASS_COMPRESSED_FLAG and the - * H5C__CLASS_SPECULATIVE_LOAD_FLAG to be set in the same - * instance of H5C_class_t. - * * The following flags may only appear in test code. * * H5C__CLASS_SKIP_READS: This flags is intended only for use in test @@ -373,54 +323,124 @@ typedef struct H5C_t H5C_t; * code. When it is set, writes of buffers prepared by the * serialize callback will be skipped. * - * GET_LOAD_SIZE: Pointer to the 'get load size' function. + * GET_INITIAL_LOAD_SIZE: Pointer to the 'get initial load size' function. * - * This function must be able to determine the size of the disk image of - * a metadata cache entry, given the 'udata' that will be passed to the - * 'deserialize' callback. + * This function determines the size based on the information in the + * parameter "udata" or an initial speculative guess. The size is + * returned in the parameter "image_len_ptr". * - * Note that if either the H5C__CLASS_SPECULATIVE_LOAD_FLAG or - * the H5C__CLASS_COMPRESSED_FLAG is set, the disk image size - * returned by this callback is either a first guess (if the - * H5C__CLASS_SPECULATIVE_LOAD_FLAG is set) or (if the - * H5C__CLASS_COMPRESSED_FLAG is set), the exact on disk size - * of the entry whether it has been run through filters or not. - * In all other cases, the value returned should be the correct - * uncompressed size of the entry. + * For an entry with H5C__CLASS_NO_FLAGS_SET: + * This function returns in "image_len_ptr" the on disk size of the + * entry. + * + * For an entry with H5C__CLASS_SPECULATIVE_LOAD_FLAG: + * This function returns in "image_len_ptr" an initial guess of the + * entry's on disk size. This many bytes will be loaded from + * the file and then passed to 'get_final_load_size' callback + * for the actual (final) image length to be determined. * - * The typedef for the deserialize callback is as follows: + * The typedef for the get_initial_load_size callback is as follows: * - * typedef herr_t (*H5C_get_load_size_func_t)(void *udata_ptr, - * size_t *image_len_ptr); + * typedef herr_t (*H5C_get_initial_load_size_func_t)(void *udata_ptr, + * size_t *image_len_ptr); * - * The parameters of the deserialize callback are as follows: + * The parameters of the get_initial_load_size callback are as follows: * * udata_ptr: Pointer to user data provided in the protect call, which - * will also be passed through to the deserialize callback. + * will also be passed through to the 'get_final_load_size', + * 'verify_chksum', and 'deserialize' callbacks. * - * image_len_ptr: Pointer to the location in which the length in bytes - * of the in file image to be deserialized is to be returned. + * image_len_ptr: Pointer to the length in bytes of the in-file image to + * be deserialized is to be returned. * * This value is used by the cache to determine the size of * the disk image for the metadata, in order to read the disk * image from the file. - * + * * Processing in the get_load_size function should proceed as follows: * - * If successful, the function will place the length of the on disk - * image associated with supplied user data in *image_len_ptr, and - * then return SUCCEED. + * If successful, the function will place the length in the *image_len_ptr + * associated with supplied user data and then return SUCCEED. + * + * On failure, the function must return FAIL and push error information + * onto the error stack with the error API routines, without modifying + * the value pointed to by image_len_ptr. + * + * + * GET_FINAL_LOAD_SIZE: Pointer to the 'get final load size' function. + * + * This function determines the final size of a speculatively loaded + * metadata cache entry based on the parameter "image" and the "udata" + * parameters. This callback _must_ be implemented for cache clients + * which set the H5C__CLASS_SPECULATIVE_LOAD_FLAG and must return the + * actual length of on-disk image after being called once. + * + * This function might deserialize the needed metadata information to + * determine the actual size. The size is returned in the parameter + * "actual_len_ptr". + * + * The typedef for the get_load_size callback is as follows: + * + * typedef herr_t (*H5C_get_final_load_size_func_t)(const void *image_ptr, + * size_t image_len, + * void *udata_ptr, + * size_t *actual_len_ptr); + * + * The parameters of the get_load_size callback are as follows: + * + * image_ptr: Pointer to a buffer containing the (possibly partial) + * metadata read in. + * + * image_len: The length in bytes of the (possibly partial) in-file image + * to be queried for an actual length. + * + * udata_ptr: Pointer to user data provided in the protect call, which + * will also be passed through to the 'verify_chksum' and + * 'deserialize' callbacks. + * + * actual_len_ptr: Pointer to the location containing the actual length + * of the metadata entry on disk. + * + * Processing in the get_final_load_size function should proceed as follows: + * + * If successful, the function will place the length in the *actual_len_ptr + * associated with supplied image and/or user data and then return SUCCEED. * * On failure, the function must return FAIL and push error information * onto the error stack with the error API routines, without modifying - * the value pointed to by the image_len_ptr. + * the value pointed to by actual_len_ptr. + * + * + * VERIFY_CHKSUM: Pointer to the verify_chksum function. + * + * This function verifies the checksum computed for the metadata is + * the same as the checksum stored in the metadata. + * + * It computes the checksum based on the metadata stored in the + * parameter "image_ptr" and the actual length of the metadata in the + * parameter "len" which is obtained from the "get_load_size" callback. + * + * The typedef for the verify_chksum callback is as follows: + * + * typedef htri_t (*H5C_verify_chksum_func_t)(const void *image_ptr, + * size_t len, + * void *udata_ptr); + * + * The parameters of the verify_chksum callback are as follows: + * + * image_ptr: Pointer to a buffer containing the metadata read in. + * + * len: The actual length of the metadata. + * + * udata_ptr: Pointer to user data. * * * DESERIALIZE: Pointer to the deserialize function. * - * This function must be able to read a buffer containing the on disk - * image of a metadata cache entry, allocate and load the equivalent - * in core representation, and return a pointer to that representation. + * This function must be able to deserialize a buffer containing the + * on-disk image of a metadata cache entry, allocate and initialize the + * equivalent in core representation, and return a pointer to that + * representation. * * The typedef for the deserialize callback is as follows: * @@ -453,17 +473,17 @@ typedef struct H5C_t H5C_t; * Processing in the deserialize function should proceed as follows: * * If the image contains valid data, and is of the correct length, - * the deserialize function must allocate space for an in core - * representation of that data, load the contents of the image into - * the space allocated for the in core representation, and return + * the deserialize function must allocate space for an in-core + * representation of that data, deserialize the contents of the image + * into the space allocated for the in-core representation, and return * a pointer to the in core representation. Observe that an * instance of H5C_cache_entry_t must be the first item in this * representation. The cache will initialize it after the callback * returns. * - * Note that the structure of the in core representation is otherwise + * Note that the structure of the in-core representation is otherwise * up to the cache client. All that is required is that the pointer - * returned be sufficient for the clients purposes when it is returned + * returned be sufficient for the client's purposes when it is returned * on a protect call. * * If the deserialize function has to clean up file corruption @@ -475,61 +495,16 @@ typedef struct H5C_t H5C_t; * buffer length, malloc failure, etc.) the function must return NULL and * push error information on the error stack with the error API routines. * - * Exceptions to the above: - * - * If the H5C__CLASS_SPECULATIVE_LOAD_FLAG is set, the buffer supplied - * to the function need not be currect on the first invocation of the - * callback in any single attempt to load the entry. - * - * In this case, if the buffer is larger than necessary, the function - * should load the entry as described above and not flag an error due - * to the oversized buffer. The cache will correct its mis-apprehension - * of the entry size with a subsequent call to the image_len callback. - * - * If the buffer is too small, and this is the first deserialize call - * in the entry load operation, the function should not flag an error. - * Instead, it must compute the correct size of the entry, allocate an - * in core representation and initialize it to the extent that an - * immediate call to the image len callback will return the correct - * image size. - * - * In this case, when the deserialize callback returns, the cache will - * call the image length callback, realize that the supplied buffer was - * too small, discard the returned in core representation, allocate - * and load a new buffer of the correct size from file, and then call - * the deserialize callback again. - * - * If the H5C__CLASS_COMPRESSED_FLAG is set, exceptions are as per the - * H5C__CLASS_SPECULATIVE_LOAD_FLAG, save that only oversized buffers - * are permitted. - * * * IMAGE_LEN: Pointer to the image length callback. * - * This callback exists primarily to support - * H5C__CLASS_SPECULATIVE_LOAD_FLAG and H5C__CLASS_COMPRESSED_FLAG - * discussed above, although it is also used to obtain the size of - * newly inserted entries. - * - * In the case of the H5C__CLASS_SPECULATIVE_LOAD_FLAG, it is used to - * allow the client to change the size of an entry in the deserialize - * callback. - * - * For the H5C__CLASS_COMPRESSED_FLAG, it is used to allow the client - * to indicate whether the entry is compressed (i.e. whether entries - * are run through filters) and if so, to report both the uncompressed - * and the compressed entry size (i.e. the actual on disk size after - * the entry has been run through filters) if that value is known. - * - * The callback is also used in H5C_insert_entry() to obtain the - * size of the newly inserted entry. + * The image_len callback is used to obtain the size of newly inserted + * entries and assert verification. * * The typedef for the image_len callback is as follows: * * typedef herr_t (*H5C_image_len_func_t)(void *thing, - * size_t *image_len_ptr, - * hbool_t *compressed_ptr, - * size_t *compressed_image_len_ptr); + * size_t *image_len_ptr); * * The parameters of the image_len callback are as follows: * @@ -538,84 +513,16 @@ typedef struct H5C_t H5C_t; * image_len_ptr: Pointer to size_t in which the callback will return * the length (in bytes) of the cache entry. * - * If the H5C__CLASS_COMPRESSED_FLAG is not set in the - * associated instance of H5C_class_t, or if the flag is - * set, and the callback sets *compressed_ptr to FALSE, - * this size is the actual size of the entry on disk. - * - * Otherwise, this size is the uncompressed size of the - * entry -- which the cache will use for all purposes OTHER - * than journal writes and disk I/O. - * - * compressed_ptr: Pointer to a boolean flag indicating whether - * the cache entry will be compressed / uncompressed on - * disk writes / reads. - * - * If the H5C__CLASS_COMPRESSED_FLAG is not set in the - * associated instance of H5C_class_t, *compressed_ptr - * must be set to FALSE. - * - * If the H5C__CLASS_COMPRESSED_FLAG is set in the - * associated instance of H5C_class_t, and filters are - * not enabled, *compressed_ptr must be set to FALSE. - * - * If the H5C__CLASS_COMPRESSED_FLAG is set in the - * associated instance of H5C_class_t, and filters are - * enabled, the callback must set *compressed_ptr to TRUE. - * - * Note that *compressed_ptr will always be set to FALSE - * by the caller prior to invocation of the callback. Thus - * callbacks for clients that don't set the - * H5C__CLASS_COMPRESSED_FLAG can ignore this parameter. - * - * compressed_image_len_ptr: Pointer to size_t in which the callback - * may return the length (in bytes) of the compressed on - * disk image of the entry, or the uncompressed size if the - * compressed size has not yet been calculated. - * - * Since computing the compressed image len is expensive, - * the callback should only report the most recently computed - * value -- which will typically be incorrect if the entry - * is dirty. - * - * If *compressed_ptr is set to FALSE, *compressed_image_len_ptr - * should be set to zero. However, as *compressed_image_len_ptr - * will be initialize to zero prior to the call, the callback - * need not modify it if the H5C__CLASS_COMPRESSED_FLAG is - * not set. - * - * If the H5C__CLASS_COMPRESSED_FLAG is not set in the associated - * instance of H5C_class_t, processing in the image_len function - * should proceed as follows: + * Processing in the image_len function should proceed as follows: * * If successful, the function will place the length of the on disk * image associated with the in core representation provided in the - * thing parameter in *image_len_ptr, and then return SUCCEED. Since - * *compressed_ptr and *compressed_image_len_ptr will be initialized to - * FALSE and zero respectively before the call, the callback need not - * modify these values, and may declare the associated parameters as - * UNUSED. - * - * If the H5C__CLASS_COMPRESSED_FLAG is set in the associated - * instance of H5C_class_t, processing in the image_len function - * should proceed as follows: - * - * If successful, the function will place the uncompressed length of - * the on disk image associated with the in core representation - * provided in the thing parameter in *image_len_ptr. If filters - * are not enabled for the entry, it will set *compressed_ptr to FALSE, - * and *compressed_image_len_ptr to zero. If filters are enabled, - * it will set *compressed_ptr to TRUE. In this case, it must set - * *compressed_image_len_ptr equal to the last computed compressed - * if the compressed size, or to the uncompressed size if that value - * is yet to be computed. In all cases, it will return SUCCEED if - * successful. - * - * In either case, if the function fails, it must return FAIL and - * push error information onto the error stack with the error API - * routines, and return without modifying the values pointed to by - * the image_len_ptr, compressed_ptr, and compressed_image_len_ptr - * parameters. + * thing parameter in *image_len_ptr, and then return SUCCEED. + * + * If the function fails, it must return FAIL and push error information + * onto the error stack with the error API routines, and return without + * modifying the values pointed to by the image_len_ptr parameter. + * * * PRE_SERIALIZE: Pointer to the pre-serialize callback. * @@ -623,11 +530,11 @@ typedef struct H5C_t H5C_t; * it needs a current on-disk image of the metadata entry for purposes * either constructing a journal or flushing the entry to disk. * - * If the client needs to change the address or compressed or - * uncompressed length of the entry prior to flush, the pre-serialize - * callback is responsible for these actions, so that the actual - * serialize callback (described below) is only responsible for - * serializing the data structure, not moving it on disk or resizing it. + * If the client needs to change the address or length of the entry prior + * to flush, the pre-serialize callback is responsible for these actions, + * so that the actual serialize callback (described below) is only + * responsible for serializing the data structure, not moving it on disk + * or resizing it. * * In addition, the client may use the pre-serialize callback to * ensure that the entry is ready to be flushed -- in particular, @@ -641,7 +548,7 @@ typedef struct H5C_t H5C_t; * However, that need not be the case as free space section info * entries will change size (and possibly location) depending on the * number of blocks of free space being manages, and fractal heap - * direct blocks can change compressed size (and possibly location) + * direct blocks can change compressed size (and possibly location) * on serialization if compression is enabled. Similarly, it may * be necessary to move entries from temporary to real file space. * @@ -655,10 +562,8 @@ typedef struct H5C_t H5C_t; * void * thing, * haddr_t addr, * size_t len, - * size_t compressed_len, * haddr_t * new_addr_ptr, * size_t * new_len_ptr, - * size_t * new_compressed_len_ptr, * unsigned * flags_ptr); * * The parameters of the pre-serialize callback are as follows: @@ -686,22 +591,8 @@ typedef struct H5C_t H5C_t; * * len: Length in bytes of the in file image of the entry to be * serialized. Also the size the image passed to the - * serialize callback (discussed below) unless either that - * value is altered by this function, or the entry will be - * compressed. In the latter case, the compressed size - * of the entry will be reported in *new_compressed_len_ptr. - * - * This parameter is supplied mainly for sanity checking. - * Sanity checks should be performed when compiled in debug - * mode, but the parameter may be unused when compiled in - * production mode. - * - * compressed_len: If the entry is to be compressed (i.e. run through - * filters) prior to flush, Length in bytes of the last know - * compressed size of the entry -- or the uncompressed size - * if no such value exists (i.e. the entry has been inserted, - * but never flushed). This parameter should be set to zero - * in all other cases. + * serialize callback (discussed below) unless that + * value is altered by this function. * * This parameter is supplied mainly for sanity checking. * Sanity checks should be performed when compiled in debug @@ -726,20 +617,12 @@ typedef struct H5C_t H5C_t; * *new_len_ptr is undefined on pre-serialize callback * return. * - * new_compressed_len_ptr: Pointer to size_t. If the image will be - * compressed (i.e. run through filters) prior to being - * written to disk, the compressed size (in bytes) of the - * on disk image must be stored in *new_compressed_len_ptr, - * and the appropriate flag set in *flags_ptr. - * * flags_ptr: Pointer to an unsigned integer used to return flags * indicating whether the preserialize function resized or moved - * the entry, or computed its compressed size. If the entry was - * neither resized or moved, nor will be compressed, - * the serialize function must set *flags_ptr to zero. - * H5C__SERIALIZE_RESIZED_FLAG, H5C__SERIALIZE_MOVED_FLAG - * and H5C__SERIALIZE_COMPRESSED_FLAG must be set to indicate - * a resize, a move, or compression respectively. + * the entry. If the entry was neither resized or moved, the + * serialize function must set *flags_ptr to zero. The + * H5C__SERIALIZE_RESIZED_FLAG or H5C__SERIALIZE_MOVED_FLAG must + * be set to indicate a resize or move respectively. * * If the H5C__SERIALIZE_RESIZED_FLAG is set, the new length * must be stored in *new_len_ptr. @@ -747,40 +630,21 @@ typedef struct H5C_t H5C_t; * If the H5C__SERIALIZE_MOVED_FLAG flag is set, the * new image base address must be stored in *new_addr_ptr. * - * If the H5C__SERIALIZE_COMPRESSED_FLAG is set, the - * compressed size of the new image must be stored in - * *new_compressed_len_ptr. - * * Processing in the pre-serialize function should proceed as follows: * * The pre-serialize function must examine the in core representation * indicated by the thing parameter, if the pre-serialize function does - * not need to change the size or location of the on-disk image, or - * compute its compress size, it must set *flags_ptr to zero. + * not need to change the size or location of the on-disk image, it must + * set *flags_ptr to zero. * - * If the (uncompressed) size of the on-disk image must be changed, - * the pre-serialize function must load the length of the new image - * into *new_len_ptr, and set the H5C__SERIALIZE_RESIZED_FLAG in - * *flags_ptr. + * If the size of the on-disk image must be changed, the pre-serialize + * function must load the length of the new image into *new_len_ptr, and + * set the H5C__SERIALIZE_RESIZED_FLAG in *flags_ptr. * * If the base address of the on disk image must be changed, the * pre-serialize function must set *new_addr_ptr to the new base address, * and set the H5C__SERIALIZE_MOVED_FLAG in *flags_ptr. * - * If the H5C__CLASS_COMPRESSED_FLAG is set in the assocated instance - * of H5C_class_t, and filters (i.e. compression) are enabled, the - * pre-serialize function must compute the compressed size of the - * on disk image, and if it has changed, load this value into - * *new_compressed_len_ptr, and set H5C__SERIALIZE_COMPRESSED_FLAG in - * *flags_ptr. - * - * Note that to do this, the preserialize function will typically have - * to serialize the entry, and run it through the filters to obtain - * the compressed size. For efficiency, the compressed image may - * be stored to be copied into the supplied buffer by the - * serialize callback. Needless to say this is awkward. We may - * want to re-work the API for cache clients to simplify this. - * * In addition, the pre-serialize callback may perform any other * processing required before the entry is written to disk * @@ -829,11 +693,7 @@ typedef struct H5C_t H5C_t; * the buffer. * * len: Length in bytes of the in file image of the entry to be - * serialized. Also the size of *image_ptr (below). If - * compression is not enabled, this value is simply the - * uncompressed size of the entry's image on disk. If - * compression is enabled, this value is the size of the - * compressed image. + * serialized. Also the size of *image_ptr (below). * * This parameter is supplied mainly for sanity checking. * Sanity checks should be performed when compiled in debug @@ -852,8 +712,7 @@ typedef struct H5C_t H5C_t; * * The serialize function must then examine the in core * representation indicated by the thing parameter, and write a - * serialized (and possibly compressed) image of its contents into - * the provided buffer. + * serialized image of its contents into the provided buffer. * * If it is successful, the function must return SUCCEED. * @@ -925,65 +784,7 @@ typedef struct H5C_t H5C_t; * * At least when compiled with debug, it would be useful if the * free ICR call would fail if the in core representation has been - * modified since the last serialize of clear callback. - * - * CLEAR: Pointer to the clear callback. - * - * In principle, there should be no need for the clear callback, - * as the dirty flag should be maintained by the metadata cache. - *. - * However, some clients maintain dirty bits on internal data, - * and we need some way of keeping these dirty bits in sync with - * those maintained by the metadata cache. This callback exists - * to serve this purpose. If defined, it is called whenever the - * cache marks dirty entry clean, or when the cache is about to - * discard a dirty entry without writing it to disk (This - * happens as the result of an unprotect call with the - * H5AC__DELETED_FLAG set, and the H5C__TAKE_OWNERSHIP_FLAG not - * set.) - * - * Arguably, this functionality should be in the NOTIFY callback. - * However, this callback is specific to only a few clients, and - * it will be called relatively frequently. Hence it is made its - * own callback to minimize overhead. - * - * The typedef for the clear callback is as follows: - * - * typedef herr_t (*H5C_clear_func_t)(const H5F_t *f, - * void * thing, - * hbool_t about_to_destroy); - * - * The parameters of the clear callback are as follows: - * - * f: File pointer. - * - * thing: Pointer to void containing the address of the in core - * representation of the target metadata cache entry. This - * is the same pointer that would be returned by a protect() - * call of the associated addr and len. - * - * about_to_destroy: Boolean flag used to indicate whether the - * metadata cache is about to destroy the target metadata - * cache entry. The callback may use this flag to omit - * operations that are irrelevant it the entry is about - * to be destroyed. - * - * Processing in the clear function should proceed as follows: - * - * Reset all internal dirty bits in the target metadata cache entry. - * - * If the about_to_destroy flag is TRUE, the clear function may - * ommit any dirty bit that will not trigger a sanity check failure - * or otherwise cause problems in the subsequent free icr call. - * In particular, the call must ensure that the free icr call will - * not fail due to changes prior to this call, and after the - * last serialize or clear call. - * - * If the function is successful, it must return SUCCEED. - * - * If it fails for any reason, the function must return FAIL and - * push error information on the error stack with the error API - * routines. + * modified since the last serialize callback. * * GET_FSF_SIZE: Pointer to the get file space free size callback. * @@ -1019,12 +820,12 @@ typedef struct H5C_t H5C_t; * At present this callback is used only by the H5FA and H5EA dblock * and dblock page client classes. * - * The typedef for the clear callback is as follows: + * The typedef for the get_fsf_size callback is as follows: * * typedef herr_t (*H5C_get_fsf_size_t)(const void * thing, * size_t *fsf_size_ptr); * - * The parameters of the clear callback are as follows: + * The parameters of the get_fsf_size callback are as follows: * * thing: Pointer to void containing the address of the in core * representation of the target metadata cache entry. This @@ -1068,22 +869,20 @@ typedef enum H5C_notify_action_t { } H5C_notify_action_t; /* Cache client callback function pointers */ -typedef herr_t (*H5C_get_load_size_func_t)(const void *udata_ptr, - size_t *image_len_ptr); +typedef herr_t (*H5C_get_initial_load_size_func_t)(void *udata_ptr, size_t *image_len_ptr); +typedef herr_t (*H5C_get_final_load_size_func_t)(const void *image_ptr, + size_t image_len, void *udata_ptr, size_t *actual_len_ptr); +typedef htri_t (*H5C_verify_chksum_func_t)(const void *image_ptr, size_t len, void *udata_ptr); typedef void *(*H5C_deserialize_func_t)(const void *image_ptr, size_t len, void *udata_ptr, hbool_t *dirty_ptr); -typedef herr_t (*H5C_image_len_func_t)(const void *thing, - size_t *image_len_ptr, hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +typedef herr_t (*H5C_image_len_func_t)(const void *thing, size_t *image_len_ptr); typedef herr_t (*H5C_pre_serialize_func_t)(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr_ptr, size_t *new_len_ptr, size_t *new_compressed_len_ptr, - unsigned *flags_ptr); + void *thing, haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); typedef herr_t (*H5C_serialize_func_t)(const H5F_t *f, void *image_ptr, size_t len, void *thing); typedef herr_t (*H5C_notify_func_t)(H5C_notify_action_t action, void *thing); typedef herr_t (*H5C_free_icr_func_t)(void *thing); -typedef herr_t (*H5C_clear_func_t)(const H5F_t *f, void * thing, - hbool_t about_to_destroy); typedef herr_t (*H5C_get_fsf_size_t)(const void * thing, size_t *fsf_size_ptr); /* Metadata cache client class definition */ @@ -1092,14 +891,15 @@ typedef struct H5C_class_t { const char * name; H5FD_mem_t mem_type; unsigned flags; - H5C_get_load_size_func_t get_load_size; + H5C_get_initial_load_size_func_t get_initial_load_size; + H5C_get_final_load_size_func_t get_final_load_size; + H5C_verify_chksum_func_t verify_chksum; H5C_deserialize_func_t deserialize; H5C_image_len_func_t image_len; H5C_pre_serialize_func_t pre_serialize; H5C_serialize_func_t serialize; H5C_notify_func_t notify; H5C_free_icr_func_t free_icr; - H5C_clear_func_t clear; H5C_get_fsf_size_t fsf_size; } H5C_class_t; @@ -1204,51 +1004,11 @@ typedef int H5C_ring_t; * * addr: Base address of the cache entry on disk. * - * size: Length of the cache entry on disk in bytes(exception: if - * the entry is compressed on disk, this field contains the - * uncompressed size of the entry -- see discussion of - * compressed entries below). Note that unlike normal - * caches, the entries in this cache are of arbitrary size. - * - * With the exception of compressed entries, the file space - * allocations for cache entries implied by the addr and size - * fields must be disjoint. For compressed entries, - * the size field contains the uncompressed size -- thus in - * in this case, substitution of compressed size for size - * must result in disjoint file space allocations. However, - * as discussed below, the compressed size may not be know. - * - * Any entry whose associated instance of H5C_class_t has the - * H5C__CLASS_COMPRESSED_FLAG set may be compressed. When - * an entry is compressed (that is, when filters are enabled - * on it), the compressed flag (see below) must be set, and - * the compressed size (if known), must be stored in - * the compressed_size field. - * - * Since the compressed size will be unknown unless the - * entry is clean, or has an up to date image (see the - * image_ptr and image_up_to_date fields below), we use the - * uncompressed size for all purposes other than disk I/O. - * - * compressed: Boolean flag that is set iff the instance of H5C_class_t - * associated with the entry has the H5C__CLASS_COMPRESSED_FLAG - * set, and filters are enabled on the entry. - * - * compressed_size: If compressed is TRUE, this field contains the actual - * compressed size of the entry in bytes, which is also its - * true size on disk -- or the uncompressed size if the - * compressed size is unknown (i.e. the entry has been - * inserted in the cache, but it has not been compressed yet). - * Note that this value will usually be incorrect if the - * entry is dirty. - * - * Since this value is frequently out of date and expensive to - * compute, it is used only for disk I/O. The uncompressed - * size of the entry (stored in the size field above) is used - * for all other purposes (i.e. computing the sum of the sizes - * of all entries in the cache, etc.). - * - * If compressed is FALSE, this field should contain 0. + * size: Length of the cache entry on disk in bytes Note that unlike + * normal caches, the entries in this cache are of arbitrary size. + * + * The file space allocations for cache entries implied by the + * addr and size fields must be disjoint. * * image_ptr: Pointer to void. When not NULL, this field points to a * dynamically allocated block of size bytes in which the @@ -1273,25 +1033,6 @@ typedef int H5C_ring_t; * entry has been modified since the last time it was written * to disk. * - * NOTE: For historical reasons, this field is not maintained - * by the cache. Instead, the module using the cache - * sets this flag when it modifies the entry, and the - * flush and clear functions supplied by that module - * reset the dirty when appropriate. - * - * This is a bit quirky, so we may want to change this - * someday. However it will require a change in the - * cache interface. - * - * Update: Management of the is_dirty field has been largely - * moved into the cache. The only remaining exceptions - * are the flush and clear functions supplied by the - * modules using the cache. These still clear the - * is_dirty field as before. -- JRM 7/5/05 - * - * Update: Management of the is_dirty field is now entirely - * in the cache. -- JRM 7/5/07 - * * dirtied: Boolean flag used to indicate that the entry has been * dirtied while protected. * @@ -1604,8 +1345,6 @@ typedef struct H5C_cache_entry_t { H5C_t * cache_ptr; haddr_t addr; size_t size; - hbool_t compressed; - size_t compressed_size; void * image_ptr; hbool_t image_up_to_date; const H5C_class_t * type; diff --git a/src/H5EAcache.c b/src/H5EAcache.c index ea8bd0e..4210b07 100644 --- a/src/H5EAcache.c +++ b/src/H5EAcache.c @@ -73,55 +73,52 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5EA__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5EA__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5EA__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5EA__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5EA__cache_hdr_free_icr(void *thing); -static herr_t H5EA__cache_iblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_iblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_iblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_iblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5EA__cache_iblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5EA__cache_iblock_image_len(const void *thing, size_t *image_len); static herr_t H5EA__cache_iblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5EA__cache_iblock_notify(H5AC_notify_action_t action, void *thing); static herr_t H5EA__cache_iblock_free_icr(void *thing); -static herr_t H5EA__cache_sblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_sblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_sblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_sblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5EA__cache_sblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5EA__cache_sblock_image_len(const void *thing, size_t *image_len); static herr_t H5EA__cache_sblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5EA__cache_sblock_notify(H5AC_notify_action_t action, void *thing); static herr_t H5EA__cache_sblock_free_icr(void *thing); -static herr_t H5EA__cache_dblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_dblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_dblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_dblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5EA__cache_dblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5EA__cache_dblock_image_len(const void *thing, size_t *image_len); static herr_t H5EA__cache_dblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5EA__cache_dblock_notify(H5AC_notify_action_t action, void *thing); static herr_t H5EA__cache_dblock_free_icr(void *thing); static herr_t H5EA__cache_dblock_fsf_size(const void *thing, size_t *fsf_size); -static herr_t H5EA__cache_dblk_page_get_load_size(const void *udata, size_t *image_len); +static herr_t H5EA__cache_dblk_page_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5EA__cache_dblk_page_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5EA__cache_dblk_page_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); static herr_t H5EA__cache_dblk_page_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); + size_t *image_len); static herr_t H5EA__cache_dblk_page_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5EA__cache_dblk_page_notify(H5AC_notify_action_t action, void *thing); @@ -138,14 +135,15 @@ const H5AC_class_t H5AC_EARRAY_HDR[1] = {{ "Extensible Array Header", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_HDR, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_hdr_deserialize, /* 'deserialize' callback */ H5EA__cache_hdr_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_hdr_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5EA__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -155,14 +153,15 @@ const H5AC_class_t H5AC_EARRAY_IBLOCK[1] = {{ "Extensible Array Index Block", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_IBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_iblock_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_iblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_iblock_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_iblock_deserialize, /* 'deserialize' callback */ H5EA__cache_iblock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_iblock_serialize, /* 'serialize' callback */ H5EA__cache_iblock_notify, /* 'notify' callback */ H5EA__cache_iblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -172,14 +171,15 @@ const H5AC_class_t H5AC_EARRAY_SBLOCK[1] = {{ "Extensible Array Super Block", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_SBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_sblock_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_sblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_sblock_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_sblock_deserialize, /* 'deserialize' callback */ H5EA__cache_sblock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_sblock_serialize, /* 'serialize' callback */ H5EA__cache_sblock_notify, /* 'notify' callback */ H5EA__cache_sblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -189,14 +189,15 @@ const H5AC_class_t H5AC_EARRAY_DBLOCK[1] = {{ "Extensible Array Data Block", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_DBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_dblock_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_dblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_dblock_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_dblock_deserialize, /* 'deserialize' callback */ H5EA__cache_dblock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_dblock_serialize, /* 'serialize' callback */ H5EA__cache_dblock_notify, /* 'notify' callback */ H5EA__cache_dblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ H5EA__cache_dblock_fsf_size, /* 'fsf_size' callback */ }}; @@ -206,14 +207,15 @@ const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{ "Extensible Array Data Block Page", /* Metadata client name (for debugging) */ H5FD_MEM_EARRAY_DBLK_PAGE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5EA__cache_dblk_page_get_load_size, /* 'get_load_size' callback */ + H5EA__cache_dblk_page_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5EA__cache_dblk_page_verify_chksum, /* 'verify_chksum' callback */ H5EA__cache_dblk_page_deserialize, /* 'deserialize' callback */ H5EA__cache_dblk_page_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5EA__cache_dblk_page_serialize, /* 'serialize' callback */ H5EA__cache_dblk_page_notify, /* 'notify' callback */ H5EA__cache_dblk_page_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -230,7 +232,7 @@ const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{ /*------------------------------------------------------------------------- - * Function: H5EA__cache_hdr_get_load_size + * Function: H5EA__cache_hdr_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -244,21 +246,54 @@ const H5AC_class_t H5AC_EARRAY_DBLK_PAGE[1] = {{ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_hdr_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_hdr_cache_ud_t *udata = (const H5EA_hdr_cache_ud_t *)_udata; /* User data for callback */ + H5EA_hdr_cache_ud_t *udata = (H5EA_hdr_cache_ud_t *)_udata; /* User data for callback */ /* Check arguments */ HDassert(udata); HDassert(udata->f); - HDassert(H5F_addr_defined(udata->addr)); HDassert(image_len); /* Set the image length size */ *image_len = (size_t)H5EA_HEADER_SIZE_FILE(udata->f); -END_FUNC(STATIC) /* end H5EA__cache_hdr_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_hdr_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_hdr_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_hdr_verify_chksum() */ /*------------------------------------------------------------------------- @@ -286,7 +321,6 @@ H5EA__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, H5EA_hdr_cache_ud_t *udata = (H5EA_hdr_cache_ud_t *)_udata; const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Check arguments */ HDassert(image); @@ -361,17 +395,11 @@ H5EA__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* (allow for checksum not decoded yet) */ HDassert((size_t)(image - (const uint8_t *)_image) == (len - H5EA_SIZEOF_CHKSUM)); - /* Compute checksum on entire header */ - /* (including the filter information, if present) */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array header") - /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == len); @@ -408,8 +436,7 @@ END_FUNC(STATIC) /* end H5EA__cache_hdr_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_hdr_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_hdr_t *hdr = (const H5EA_hdr_t *)_thing; /* Pointer to the object */ @@ -524,7 +551,7 @@ END_FUNC(STATIC) /* end H5EA__cache_hdr_free_icr() */ /*------------------------------------------------------------------------- - * Function: H5EA__cache_iblock_get_load_size + * Function: H5EA__cache_iblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -538,11 +565,11 @@ END_FUNC(STATIC) /* end H5EA__cache_hdr_free_icr() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_iblock_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_iblock_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_hdr_t *hdr = (const H5EA_hdr_t *)_udata; /* User data for callback */ - H5EA_iblock_t iblock; /* Fake index block for computing size */ + H5EA_hdr_t *hdr = (H5EA_hdr_t *)_udata; /* User data for callback */ + H5EA_iblock_t iblock; /* Fake index block for computing size */ /* Check arguments */ HDassert(hdr); @@ -558,7 +585,41 @@ H5EA__cache_iblock_get_load_size(const void *_udata, size_t *image_len)) /* Set the image length size */ *image_len = (size_t)H5EA_IBLOCK_SIZE(&iblock); -END_FUNC(STATIC) /* end H5EA__cache_iblock_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_iblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_iblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_iblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_iblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -585,7 +646,6 @@ H5EA__cache_iblock_deserialize(const void *_image, size_t len, H5EA_hdr_t *hdr = (H5EA_hdr_t *)_udata; /* User data for callback */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ size_t u; /* Local index variable */ @@ -649,8 +709,7 @@ H5EA__cache_iblock_deserialize(const void *_image, size_t len, /* Save the index block's size */ iblock->size = len; - /* Compute checksum on index block */ - computed_chksum = H5_checksum_metadata((const uint8_t *)_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -658,10 +717,6 @@ H5EA__cache_iblock_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == iblock->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array index block") - /* Set return value */ ret_value = iblock; @@ -690,8 +745,7 @@ END_FUNC(STATIC) /* end H5EA__cache_iblock_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_iblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_iblock_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_iblock_t *iblock = (const H5EA_iblock_t *)_thing; /* Pointer to the object */ @@ -878,7 +932,7 @@ END_FUNC(STATIC) /* end H5EA__cache_iblock_free_icr() */ /*------------------------------------------------------------------------- - * Function: H5EA__cache_sblock_get_load_size + * Function: H5EA__cache_sblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -892,11 +946,11 @@ END_FUNC(STATIC) /* end H5EA__cache_iblock_free_icr() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_sblock_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_sblock_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_sblock_cache_ud_t *udata = (const H5EA_sblock_cache_ud_t *)_udata; /* User data */ - H5EA_sblock_t sblock; /* Fake super block for computing size */ + H5EA_sblock_cache_ud_t *udata = (H5EA_sblock_cache_ud_t *)_udata; /* User data */ + H5EA_sblock_t sblock; /* Fake super block for computing size */ /* Check arguments */ HDassert(udata); @@ -932,7 +986,41 @@ H5EA__cache_sblock_get_load_size(const void *_udata, size_t *image_len)) /* Set the image length size */ *image_len = (size_t)H5EA_SBLOCK_SIZE(&sblock); -END_FUNC(STATIC) /* end H5EA__cache_sblock_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_sblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_sblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_sblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_sblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -959,7 +1047,6 @@ H5EA__cache_sblock_deserialize(const void *_image, size_t len, H5EA_sblock_cache_ud_t *udata = (H5EA_sblock_cache_ud_t *)_udata; /* User data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ size_t u; /* Local index variable */ @@ -1020,8 +1107,7 @@ H5EA__cache_sblock_deserialize(const void *_image, size_t len, /* Save the super block's size */ sblock->size = len; - /* Compute checksum on super block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -1029,10 +1115,6 @@ H5EA__cache_sblock_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == sblock->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array super block") - /* Set return value */ ret_value = sblock; @@ -1061,8 +1143,7 @@ END_FUNC(STATIC) /* end H5EA__cache_sblock_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_sblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_sblock_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_sblock_t *sblock = (const H5EA_sblock_t *)_thing; /* Pointer to the object */ @@ -1251,7 +1332,7 @@ END_FUNC(STATIC) /* end H5EA__cache_sblock_free_icr() */ /*------------------------------------------------------------------------- - * Function: H5EA__cache_dblock_get_load_size + * Function: H5EA__cache_dblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -1265,11 +1346,11 @@ END_FUNC(STATIC) /* end H5EA__cache_sblock_free_icr() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_dblock_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_dblock_cache_ud_t *udata = (const H5EA_dblock_cache_ud_t *)_udata; /* User data */ - H5EA_dblock_t dblock; /* Fake data block for computing size */ + H5EA_dblock_cache_ud_t *udata = (H5EA_dblock_cache_ud_t *)_udata; /* User data */ + H5EA_dblock_t dblock; /* Fake data block for computing size */ /* Check arguments */ HDassert(udata); @@ -1306,7 +1387,41 @@ H5EA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) else *image_len = H5EA_DBLOCK_PREFIX_SIZE(&dblock); -END_FUNC(STATIC) /* end H5EA__cache_dblock_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_dblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_dblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_dblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_sblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -1333,7 +1448,6 @@ H5EA__cache_dblock_deserialize(const void *_image, size_t len, H5EA_dblock_cache_ud_t *udata = (H5EA_dblock_cache_ud_t *)_udata; /* User data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ /* Check arguments */ @@ -1393,8 +1507,7 @@ H5EA__cache_dblock_deserialize(const void *_image, size_t len, /* (Note: This is not the same as the image length, for paged data blocks) */ dblock->size = H5EA_DBLOCK_SIZE(dblock); - /* Compute checksum on data block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -1402,10 +1515,6 @@ H5EA__cache_dblock_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block") - /* Set return value */ ret_value = dblock; @@ -1434,8 +1543,7 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_dblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_dblock_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_dblock_t *dblock = (const H5EA_dblock_t *)_thing; /* Pointer to the object */ @@ -1632,7 +1740,7 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_free_icr() */ * file space block set. * * This function is needed when the data block is paged, as - * the datablock header and all its pages are allocted as a + * the datablock header and all its pages are allocated as a * single contiguous chunk of file space, and must be * deallocated the same way. * @@ -1672,7 +1780,7 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_fsf_size() */ /*------------------------------------------------------------------------- - * Function: H5EA__cache_dblk_page_get_load_size + * Function: H5EA__cache_dblk_page_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -1686,10 +1794,10 @@ END_FUNC(STATIC) /* end H5EA__cache_dblock_fsf_size() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len)) +H5EA__cache_dblk_page_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5EA_dblk_page_cache_ud_t *udata = (const H5EA_dblk_page_cache_ud_t *)_udata; /* User data */ + H5EA_dblk_page_cache_ud_t *udata = (H5EA_dblk_page_cache_ud_t *)_udata; /* User data */ /* Check arguments */ HDassert(udata); @@ -1697,9 +1805,44 @@ H5EA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len)) HDassert(udata->parent); HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)H5EA_DBLK_PAGE_SIZE(udata->hdr); -END_FUNC(STATIC) /* end H5EA__cache_dblk_page_get_load_size() */ +END_FUNC(STATIC) /* end H5EA__cache_dblk_page_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5EA__cache_dblk_page_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5EA__cache_dblk_page_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5EA__cache_dblk_page_verify_chksum() */ /*------------------------------------------------------------------------- @@ -1726,7 +1869,6 @@ H5EA__cache_dblk_page_deserialize(const void *_image, size_t len, H5EA_dblk_page_cache_ud_t *udata = (H5EA_dblk_page_cache_ud_t *)_udata; /* User data for loading data block page */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Sanity check */ HDassert(udata); @@ -1756,8 +1898,7 @@ H5EA__cache_dblk_page_deserialize(const void *_image, size_t len, /* Set the data block page's size */ dblk_page->size = len; - /* Compute checksum on data block page */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -1765,10 +1906,6 @@ H5EA__cache_dblk_page_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == dblk_page->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block page") - /* Set return value */ ret_value = dblk_page; @@ -1797,8 +1934,7 @@ END_FUNC(STATIC) /* end H5EA__cache_dblk_page_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5EA__cache_dblk_page_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5EA__cache_dblk_page_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5EA_dblk_page_t *dblk_page = (const H5EA_dblk_page_t *)_thing; /* Pointer to the object */ diff --git a/src/H5FAcache.c b/src/H5FAcache.c index 540004a..82b87dd 100644 --- a/src/H5FAcache.c +++ b/src/H5FAcache.c @@ -71,32 +71,30 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5FA__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FA__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FA__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FA__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FA__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5FA__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5FA__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5FA__cache_hdr_free_icr(void *thing); -static herr_t H5FA__cache_dblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FA__cache_dblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FA__cache_dblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FA__cache_dblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FA__cache_dblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5FA__cache_dblock_image_len(const void *thing, size_t *image_len); static herr_t H5FA__cache_dblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5FA__cache_dblock_free_icr(void *thing); static herr_t H5FA__cache_dblock_fsf_size(const void *thing, size_t *fsf_size); -static herr_t H5FA__cache_dblk_page_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FA__cache_dblk_page_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FA__cache_dblk_page_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FA__cache_dblk_page_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FA__cache_dblk_page_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5FA__cache_dblk_page_image_len(const void *thing, size_t *image_len); static herr_t H5FA__cache_dblk_page_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5FA__cache_dblk_page_free_icr(void *thing); @@ -112,14 +110,15 @@ const H5AC_class_t H5AC_FARRAY_HDR[1] = {{ "Fixed-array Header", /* Metadata client name (for debugging) */ H5FD_MEM_FARRAY_HDR, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FA__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5FA__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FA__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5FA__cache_hdr_deserialize, /* 'deserialize' callback */ H5FA__cache_hdr_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5FA__cache_hdr_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5FA__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -129,14 +128,15 @@ const H5AC_class_t H5AC_FARRAY_DBLOCK[1] = {{ "Fixed Array Data Block", /* Metadata client name (for debugging) */ H5FD_MEM_FARRAY_DBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FA__cache_dblock_get_load_size, /* 'get_load_size' callback */ + H5FA__cache_dblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FA__cache_dblock_verify_chksum, /* 'verify_chksum' callback */ H5FA__cache_dblock_deserialize, /* 'deserialize' callback */ H5FA__cache_dblock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5FA__cache_dblock_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5FA__cache_dblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ H5FA__cache_dblock_fsf_size, /* 'fsf_size' callback */ }}; @@ -146,14 +146,15 @@ const H5AC_class_t H5AC_FARRAY_DBLK_PAGE[1] = {{ "Fixed Array Data Block Page", /* Metadata client name (for debugging) */ H5FD_MEM_FARRAY_DBLK_PAGE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FA__cache_dblk_page_get_load_size, /* 'get_load_size' callback */ + H5FA__cache_dblk_page_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FA__cache_dblk_page_verify_chksum, /* 'verify_chksum' callback */ H5FA__cache_dblk_page_deserialize, /* 'deserialize' callback */ H5FA__cache_dblk_page_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5FA__cache_dblk_page_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5FA__cache_dblk_page_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -170,7 +171,7 @@ const H5AC_class_t H5AC_FARRAY_DBLK_PAGE[1] = {{ /*------------------------------------------------------------------------- - * Function: H5FA__cache_hdr_get_load_size + * Function: H5FA__cache_hdr_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -184,10 +185,10 @@ const H5AC_class_t H5AC_FARRAY_DBLK_PAGE[1] = {{ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_hdr_get_load_size(const void *_udata, size_t *image_len)) +H5FA__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5FA_hdr_cache_ud_t *udata = (const H5FA_hdr_cache_ud_t *)_udata; /* User data for callback */ + H5FA_hdr_cache_ud_t *udata = (H5FA_hdr_cache_ud_t *)_udata; /* User data for callback */ /* Check arguments */ HDassert(udata); @@ -197,7 +198,41 @@ H5FA__cache_hdr_get_load_size(const void *_udata, size_t *image_len)) /* Set the image length size */ *image_len = (size_t)H5FA_HEADER_SIZE_FILE(udata->f); -END_FUNC(STATIC) /* end H5FA__cache_hdr_get_load_size() */ +END_FUNC(STATIC) /* end H5FA__cache_hdr_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FA__cache_hdr_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5FA__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5FA__cache_hdr_verify_chksum() */ /*------------------------------------------------------------------------- @@ -225,7 +260,6 @@ H5FA__cache_hdr_deserialize(const void *_image, size_t len, H5FA_hdr_cache_ud_t *udata = (H5FA_hdr_cache_ud_t *)_udata; const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Check arguments */ HDassert(udata); @@ -289,9 +323,7 @@ H5FA__cache_hdr_deserialize(const void *_image, size_t len, /* (allow for checksum not decoded yet) */ HDassert((size_t)(image - (const uint8_t *)_image) == (len - H5FA_SIZEOF_CHKSUM)); - /* Compute checksum on entire header */ - /* (including the filter information, if present) */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -299,10 +331,6 @@ H5FA__cache_hdr_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array header") - /* Finish initializing fixed array header */ if(H5FA__hdr_init(hdr, udata->ctx_udata) < 0) H5E_THROW(H5E_CANTINIT, "initialization failed for fixed array header") @@ -336,8 +364,7 @@ END_FUNC(STATIC) /* end H5FA__cache_hdr_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5FA__cache_hdr_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5FA_hdr_t *hdr = (const H5FA_hdr_t *)_thing; /* Pointer to the object */ @@ -443,7 +470,7 @@ END_FUNC(STATIC) /* end H5FA__cache_hdr_free_icr() */ /*------------------------------------------------------------------------- - * Function: H5FA__cache_dblock_get_load_size + * Function: H5FA__cache_dblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -457,7 +484,7 @@ END_FUNC(STATIC) /* end H5FA__cache_hdr_free_icr() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) +H5FA__cache_dblock_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ H5FA_dblock_cache_ud_t *udata = (H5FA_dblock_cache_ud_t *)_udata; /* User data */ @@ -479,7 +506,6 @@ H5FA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) * dblock->npages * dblock->dblk_page_init_size */ - dblock.hdr = udata->hdr; dblk_page_nelmts = (size_t)1 << udata->hdr->cparam.max_dblk_page_nelmts_bits; if(udata->hdr->cparam.nelmts > dblk_page_nelmts) { @@ -493,7 +519,41 @@ H5FA__cache_dblock_get_load_size(const void *_udata, size_t *image_len)) else *image_len = (size_t)H5FA_DBLOCK_PREFIX_SIZE(&dblock); -END_FUNC(STATIC) /* end H5FA__cache_dblock_get_load_size() */ +END_FUNC(STATIC) /* end H5FA__cache_dblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FA__cache_dblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5FA__cache_dblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5FA__cache_dblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -520,7 +580,6 @@ H5FA__cache_dblock_deserialize(const void *_image, size_t len, H5FA_dblock_cache_ud_t *udata = (H5FA_dblock_cache_ud_t *)_udata; /* User data for loading data block */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ /* Sanity check */ @@ -577,8 +636,7 @@ H5FA__cache_dblock_deserialize(const void *_image, size_t len, /* Set the data block's size */ dblock->size = H5FA_DBLOCK_SIZE(dblock); - /* Compute checksum on data block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -586,10 +644,6 @@ H5FA__cache_dblock_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block") - /* Set return value */ ret_value = dblock; @@ -618,8 +672,7 @@ END_FUNC(STATIC) /* end H5FA__cache_dblock_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_dblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5FA__cache_dblock_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5FA_dblock_t *dblock = (const H5FA_dblock_t *)_thing; /* Pointer to the object */ @@ -790,7 +843,7 @@ END_FUNC(STATIC) /* end H5FA__cache_dblock_fsf_size() */ /*------------------------------------------------------------------------- - * Function: H5FA__cache_dblk_page_get_load_size + * Function: H5FA__cache_dblk_page_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -804,10 +857,10 @@ END_FUNC(STATIC) /* end H5FA__cache_dblock_fsf_size() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len)) +H5FA__cache_dblk_page_get_initial_load_size(void *_udata, size_t *image_len)) /* Local variables */ - const H5FA_dblk_page_cache_ud_t *udata = (const H5FA_dblk_page_cache_ud_t *)_udata; /* User data */ + H5FA_dblk_page_cache_ud_t *udata = (H5FA_dblk_page_cache_ud_t *)_udata; /* User data */ /* Check arguments */ HDassert(udata); @@ -815,9 +868,44 @@ H5FA__cache_dblk_page_get_load_size(const void *_udata, size_t *image_len)) HDassert(udata->nelmts > 0); HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)H5FA_DBLK_PAGE_SIZE(udata->hdr, udata->nelmts); -END_FUNC(STATIC) /* end H5FA__cache_dblk_page_get_load_size() */ +END_FUNC(STATIC) /* end H5FA__cache_dblk_page_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FA__cache_dblk_page_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +BEGIN_FUNC(STATIC, NOERR, +htri_t, TRUE, -, +H5FA__cache_dblk_page_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata)) + + /* Local variables */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + +END_FUNC(STATIC) /* end H5FA__cache_dblk_page_verify_chksum() */ /*------------------------------------------------------------------------- @@ -844,7 +932,6 @@ H5FA__cache_dblk_page_deserialize(const void *_image, size_t len, H5FA_dblk_page_cache_ud_t *udata = (H5FA_dblk_page_cache_ud_t *)_udata; /* User data for loading data block page */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Sanity check */ HDassert(udata); @@ -874,8 +961,7 @@ H5FA__cache_dblk_page_deserialize(const void *_image, size_t len, /* Set the data block page's size */ dblk_page->size = len; - /* Compute checksum on data block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -883,10 +969,6 @@ H5FA__cache_dblk_page_deserialize(const void *_image, size_t len, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == dblk_page->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block page") - /* Set return value */ ret_value = dblk_page; @@ -915,8 +997,7 @@ END_FUNC(STATIC) /* end H5FA__cache_dblk_page_deserialize() */ */ BEGIN_FUNC(STATIC, NOERR, herr_t, SUCCEED, -, -H5FA__cache_dblk_page_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr)) +H5FA__cache_dblk_page_image_len(const void *_thing, size_t *image_len)) /* Local variables */ const H5FA_dblk_page_t *dblk_page = (const H5FA_dblk_page_t *)_thing; /* Pointer to the object */ diff --git a/src/H5FScache.c b/src/H5FScache.c index c04ac7c..c1a9e0a 100644 --- a/src/H5FScache.c +++ b/src/H5FScache.c @@ -78,27 +78,25 @@ static herr_t H5FS__sinfo_serialize_sect_cb(void *_item, void H5_ATTR_UNUSED *ke static herr_t H5FS__sinfo_serialize_node_cb(void *_item, void H5_ATTR_UNUSED *key, void *_udata); /* Metadata cache callbacks */ -static herr_t H5FS__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FS__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FS__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FS__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FS__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5FS__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5FS__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5FS__cache_hdr_free_icr(void *thing); -static herr_t H5FS__cache_sinfo_get_load_size(const void *udata, size_t *image_len); +static herr_t H5FS__cache_sinfo_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5FS__cache_sinfo_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5FS__cache_sinfo_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5FS__cache_sinfo_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5FS__cache_sinfo_image_len(const void *thing, size_t *image_len); static herr_t H5FS__cache_sinfo_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5FS__cache_sinfo_serialize(const H5F_t *f, void *image, size_t len, void *thing); @@ -115,14 +113,15 @@ const H5AC_class_t H5AC_FSPACE_HDR[1] = {{ "Free Space Header", /* Metadata client name (for debugging) */ H5FD_MEM_FSPACE_HDR, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FS__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5FS__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FS__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5FS__cache_hdr_deserialize, /* 'deserialize' callback */ H5FS__cache_hdr_image_len, /* 'image_len' callback */ H5FS__cache_hdr_pre_serialize, /* 'pre_serialize' callback */ H5FS__cache_hdr_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5FS__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -132,14 +131,15 @@ const H5AC_class_t H5AC_FSPACE_SINFO[1] = {{ "Free Space Section Info", /* Metadata client name (for debugging) */ H5FD_MEM_FSPACE_SINFO, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5FS__cache_sinfo_get_load_size, /* 'get_load_size' callback */ + H5FS__cache_sinfo_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5FS__cache_sinfo_verify_chksum, /* 'verify_chksum' callback */ H5FS__cache_sinfo_deserialize, /* 'deserialize' callback */ H5FS__cache_sinfo_image_len, /* 'image_len' callback */ H5FS__cache_sinfo_pre_serialize, /* 'pre_serialize' callback */ H5FS__cache_sinfo_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5FS__cache_sinfo_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -156,7 +156,7 @@ const H5AC_class_t H5AC_FSPACE_SINFO[1] = {{ /*------------------------------------------------------------------------- - * Function: H5FS__cache_hdr_get_load_size + * Function: H5FS__cache_hdr_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -169,9 +169,9 @@ const H5AC_class_t H5AC_FSPACE_SINFO[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5FS__cache_hdr_get_load_size(const void *_udata, size_t *image_len) +H5FS__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len) { - const H5FS_hdr_cache_ud_t *udata = (const H5FS_hdr_cache_ud_t *)_udata; /* User-data for metadata cache callback */ + H5FS_hdr_cache_ud_t *udata = (H5FS_hdr_cache_ud_t *)_udata; /* User-data for metadata cache callback */ FUNC_ENTER_STATIC_NOERR @@ -184,7 +184,43 @@ H5FS__cache_hdr_get_load_size(const void *_udata, size_t *image_len) *image_len = (size_t)H5FS_HEADER_SIZE(udata->f); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FS__cache_hdr_get_load_size() */ +} /* end H5FS__cache_hdr_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FS__cache_hdr_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +htri_t +H5FS__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FS__cache_hdr_verify_chksum() */ /*------------------------------------------------------------------------- @@ -212,7 +248,6 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, H5FS_hdr_cache_ud_t *udata = (H5FS_hdr_cache_ud_t *)_udata; /* User data for callback */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned nclasses; /* Number of section classes */ H5FS_t *ret_value = NULL; /* Return value */ @@ -285,8 +320,7 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* Allocated size of serialized free space sections */ H5F_DECODE_LENGTH(udata->f, image, fspace->alloc_sect_size); - /* Compute checksum on indirect block */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -294,10 +328,6 @@ H5FS__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) <= len); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block") - /* Set return value */ ret_value = fspace; @@ -326,8 +356,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FS__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5FS__cache_hdr_image_len(const void *_thing, size_t *image_len) { const H5FS_t *fspace = (const H5FS_t *)_thing; /* Pointer to the object */ @@ -375,8 +404,7 @@ H5FS__cache_hdr_image_len(const void *_thing, size_t *image_len, */ static herr_t H5FS__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, - haddr_t addr, size_t H5_ATTR_UNUSED len, size_t H5_ATTR_UNUSED compressed_len, - haddr_t *new_addr, size_t *new_len, size_t H5_ATTR_UNUSED *new_compressed_len, + haddr_t addr, size_t H5_ATTR_UNUSED len, haddr_t *new_addr, size_t *new_len, unsigned *flags) { H5FS_t *fspace = (H5FS_t *)_thing; /* Pointer to the object */ @@ -729,10 +757,6 @@ H5FS__cache_hdr_serialize(const H5F_t *f, void *_image, size_t len, FUNC_LEAVE_NOAPI(ret_value) } /* H5FS__cache_hdr_serialize() */ -/***************************************/ -/* no H5FS__cache_hdr_notify() function */ -/***************************************/ - /*------------------------------------------------------------------------- * Function: H5FS__cache_hdr_free_icr @@ -776,13 +800,9 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5FS__cache_hdr_free_icr() */ -/********************************************************/ -/* metadata cache callback definitions for section info */ -/********************************************************/ - /*------------------------------------------------------------------------- - * Function: H5FS__cache_sinfo_get_load_size() + * Function: H5FS__cache_sinfo_get_initial_load_size() * * Purpose: Compute the size of the on disk image of the free space * manager section info, and place this value in *image_len. @@ -796,10 +816,10 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FS__cache_sinfo_get_load_size(const void *_udata, size_t *image_len) +H5FS__cache_sinfo_get_initial_load_size(void *_udata, size_t *image_len) { - const H5FS_t *fspace; /* free space manager */ - const H5FS_sinfo_cache_ud_t *udata = (const H5FS_sinfo_cache_ud_t *)_udata; /* User data for callback */ + const H5FS_t *fspace; /* free space manager */ + H5FS_sinfo_cache_ud_t *udata = (H5FS_sinfo_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -810,10 +830,47 @@ H5FS__cache_sinfo_get_load_size(const void *_udata, size_t *image_len) HDassert(fspace->sect_size > 0); HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)(fspace->sect_size); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5FS__cache_sinfo_get_load_size() */ +} /* end H5FS__cache_sinfo_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5FS__cache_sinfo_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +htri_t +H5FS__cache_sinfo_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5FS__cache_sinfo_verify_chksum() */ /*------------------------------------------------------------------------- @@ -843,7 +900,6 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, size_t old_sect_size; /* Old section size */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ uint32_t stored_chksum; /* Stored metadata checksum */ - uint32_t computed_chksum; /* Computed metadata checksum */ void * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -950,16 +1006,11 @@ H5FS__cache_sinfo_deserialize(const void *_image, size_t len, void *_udata, HDassert(old_tot_space == fspace->tot_space); } /* end if */ - /* Compute checksum on indirect block */ - computed_chksum = H5_checksum_metadata((const uint8_t *)_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block") - /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == old_sect_size); @@ -990,8 +1041,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5FS__cache_sinfo_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5FS__cache_sinfo_image_len(const void *_thing, size_t *image_len) { const H5FS_sinfo_t *sinfo = (const H5FS_sinfo_t *)_thing; /* Pointer to the object */ const H5FS_t *fspace; /* Free space header */ @@ -1033,8 +1083,7 @@ H5FS__cache_sinfo_image_len(const void *_thing, size_t *image_len, */ static herr_t H5FS__cache_sinfo_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, - haddr_t addr, size_t len, size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr, - size_t *new_len, size_t H5_ATTR_UNUSED *new_compressed_len, unsigned *flags) + haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags) { H5FS_sinfo_t *sinfo = (H5FS_sinfo_t *)_thing; /* Pointer to the object */ H5FS_t *fspace; /* Free space header */ diff --git a/src/H5Fint.c b/src/H5Fint.c index dab1900..dbaf3cc 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -686,6 +686,18 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t */ f->shared->use_tmp_space = !H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI); + /* Retrieve the # of read attempts here so that sohm in superblock will get the correct # of attempts */ + if(H5P_get(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &f->shared->read_attempts) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get the # of read attempts") + + /* If no value for read attempts has been set, use the default */ + if(!f->shared->read_attempts) + f->shared->read_attempts = H5F_METADATA_READ_ATTEMPTS; + + /* Determine the # of bins for metdata read retries */ + if(H5F_set_retries(f) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "can't set retries and retries_nbins") + /* Get the metadata cache log location (if we're logging) */ { char *mdc_log_location = NULL; /* location of metadata cache log location */ @@ -780,6 +792,7 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) HDassert(f->shared); if(1 == f->shared->nrefs) { + int actype; /* metadata cache type (enum value) */ H5F_io_info_t fio_info; /* I/O info for operation */ /* Flush at this point since the file will be closed. @@ -897,6 +910,11 @@ H5F_dest(H5F_t *f, hid_t dxpl_id, hbool_t flush) f->shared->mtab.child = (H5F_mount_t *)H5MM_xfree(f->shared->mtab.child); f->shared->mtab.nalloc = 0; + /* Clean up the metadata retries array */ + for(actype = 0; actype < (int)H5AC_NTYPES; actype++) + if(f->shared->retries[actype]) + f->shared->retries[actype] = (uint32_t *)H5MM_xfree(f->shared->retries[actype]); + /* Destroy shared file struct */ f->shared = (H5F_file_t *)H5FL_FREE(H5F_file_t, f->shared); @@ -2113,6 +2131,96 @@ done: /*------------------------------------------------------------------------- + * Function: H5F_track_metadata_read_retries + * + * Purpose: To track the # of a "retries" (log10) for a metadata item. + * This routine should be used only when: + * "retries" > 0 + * f->shared->read_attempts > 1 (does not have retry when 1) + * f->shared->retries_nbins > 0 (calculated based on f->shared->read_attempts) + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Vailin Choi; October 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_track_metadata_read_retries(H5F_t *f, unsigned actype, unsigned retries) +{ + unsigned log_ind; /* Index to the array of retries based on log10 of retries */ + double tmp; /* Temporary value, to keep compiler quiet */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(FAIL) + + /* Sanity check */ + HDassert(f); + HDassert(f->shared->read_attempts > 1); + HDassert(f->shared->retries_nbins > 0); + HDassert(retries > 0); + HDassert(retries < f->shared->read_attempts); + HDassert(actype < H5AC_NTYPES); + + /* Allocate memory for retries */ + if(NULL == f->shared->retries[actype]) + if(NULL == (f->shared->retries[actype] = (uint32_t *)H5MM_calloc((size_t)f->shared->retries_nbins * sizeof(uint32_t)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Index to retries based on log10 */ + tmp = HDlog10((double)retries); + log_ind = (unsigned)tmp; + HDassert(log_ind < f->shared->retries_nbins); + + /* Increment the # of the "retries" */ + f->shared->retries[actype][log_ind]++; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F_track_metadata_read_retries() */ + + +/*------------------------------------------------------------------------- + * Function: H5F_set_retries + * + * Purpose: To initialize data structures for read retries: + * --zero out "retries" + * --set up "retries_nbins" based on read_attempts + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Vailin Choi; November 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_set_retries(H5F_t *f) +{ + double tmp; /* Temporary variable */ + + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Sanity check */ + HDassert(f); + + /* Initialize the tracking for metadata read retries */ + HDmemset(f->shared->retries, 0, sizeof(f->shared->retries)); + + /* Initialize the # of bins for retries */ + f->shared->retries_nbins = 0; + if(f->shared->read_attempts > 1) { + tmp = HDlog10((double)(f->shared->read_attempts - 1)); + f->shared->retries_nbins = (unsigned)tmp + 1; + } + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* H5F_set_retries() */ + + +/*------------------------------------------------------------------------- * Function: H5F_object_flush_cb * * Purpose: To invoke the callback function for object flush that is set diff --git a/src/H5Fio.c b/src/H5Fio.c index 2ccd3f3..273ee74 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -282,3 +282,46 @@ done: FUNC_LEAVE_NOAPI(ret_value); } /* end H5F_evict_tagged_metadata */ + +/*------------------------------------------------------------------------- + * Function: H5F_get_checksums + * + * Purpose: Decode checksum stored in the buffer + * Calculate checksum for the data in the buffer + * + * Note: Assumes that the checksum is the last data in the buffer + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * Sept 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5F_get_checksums(const uint8_t *buf, size_t buf_size, uint32_t *s_chksum/*out*/, uint32_t *c_chksum/*out*/) +{ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Check arguments */ + HDassert(buf); + HDassert(buf_size); + + /* Return the stored checksum */ + if(s_chksum) { + const uint8_t *chk_p; /* Pointer into raw data buffer */ + + /* Offset to the checksum in the buffer */ + chk_p = buf + buf_size - H5_SIZEOF_CHKSUM; + + /* Decode the checksum stored in the buffer */ + UINT32DECODE(chk_p, *s_chksum); + } /* end if */ + + /* Return the computed checksum for the buffer */ + if(c_chksum) + *c_chksum = H5_checksum_metadata(buf, buf_size - H5_SIZEOF_CHKSUM, 0); + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5F_get_chksums() */ + diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 8d7dcbd..e9c45f3 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -152,6 +152,9 @@ typedef struct H5F_superblock_cache_ud_t { unsigned btree_k[H5B_NUM_BTREE_ID]; /* B-tree key values for each type */ haddr_t stored_eof; /* End-of-file in file */ hbool_t drvrinfo_removed; /* Indicate if the driver info was removed */ + unsigned super_vers; /* Superblock version obtained in get_load_size callback. + * It will be used later in verify_chksum callback + */ } H5F_superblock_cache_ud_t; /* Structure for passing 'user data' to driver info block cache callbacks */ @@ -299,6 +302,11 @@ struct H5F_file_t { /* Metadata accumulator information */ H5F_meta_accum_t accum; /* Metadata accumulator info */ + /* Metadata retry info */ + unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */ + unsigned retries_nbins; /* # of bins for each retries[] */ + uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metdata items with checksum */ + /* Object flush info */ H5F_object_flush_t object_flush; /* Information for object flush callback */ }; diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index fdf82ed..ee8a845 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -286,6 +286,7 @@ #define H5F_FILE_ID(F) ((F)->file_id) #define H5F_PARENT(F) ((F)->parent) #define H5F_NMOUNTS(F) ((F)->nmounts) +#define H5F_GET_READ_ATTEMPTS(F) ((F)->shared->read_attempts) #define H5F_DRIVER_ID(F) ((F)->shared->lf->driver_id) #define H5F_GET_FILENO(F,FILENUM) ((FILENUM) = (F)->shared->lf->fileno) #define H5F_HAS_FEATURE(F,FL) ((F)->shared->lf->feature_flags & (FL)) @@ -336,6 +337,7 @@ #define H5F_FILE_ID(F) (H5F_get_file_id(F)) #define H5F_PARENT(F) (H5F_get_parent(F)) #define H5F_NMOUNTS(F) (H5F_get_nmounts(F)) +#define H5F_GET_READ_ATTEMPTS(F) (H5F_get_read_attempts(F)) #define H5F_DRIVER_ID(F) (H5F_get_driver_id(F)) #define H5F_GET_FILENO(F,FILENUM) (H5F_get_fileno((F), &(FILENUM))) #define H5F_HAS_FEATURE(F,FL) (H5F_has_feature(F,FL)) @@ -469,6 +471,7 @@ #define H5F_ACS_MULTI_TYPE_NAME "multi_type" /* Data type in multi file driver */ #define H5F_ACS_LATEST_FORMAT_NAME "latest_format" /* 'Use latest format version' flag */ #define H5F_ACS_WANT_POSIX_FD_NAME "want_posix_fd" /* Internal: query the file descriptor from the core VFD, instead of the memory address */ +#define H5F_ACS_METADATA_READ_ATTEMPTS_NAME "metadata_read_attempts" /* # of metadata read attempts */ #define H5F_ACS_OBJECT_FLUSH_CB_NAME "object_flush_cb" /* Object flush callback */ #define H5F_ACS_EFC_SIZE_NAME "efc_size" /* Size of external file cache */ #define H5F_ACS_FILE_IMAGE_INFO_NAME "file_image_info" /* struct containing initial file image and callback info */ @@ -520,12 +523,19 @@ /* Default free space section threshold used by free-space managers */ #define H5F_FREE_SPACE_THRESHOLD_DEF 1 +/* Metadata read attempt values */ +#define H5F_METADATA_READ_ATTEMPTS 1 /* Default # of read attempts for non-SWMR access */ + /* Macros to define signatures of all objects in the file */ /* Size of signature information (on disk) */ /* (all on-disk signatures should be this length) */ #define H5_SIZEOF_MAGIC 4 +/* Size of checksum information (on disk) */ +/* (all on-disk checksums should be this length) */ +#define H5_SIZEOF_CHKSUM 4 + /* v1 B-tree node signature */ #define H5B_MAGIC "TREE" @@ -654,6 +664,7 @@ H5_DLL unsigned H5F_decr_nopen_objs(H5F_t *f); H5_DLL hid_t H5F_get_file_id(const H5F_t *f); H5_DLL H5F_t *H5F_get_parent(const H5F_t *f); H5_DLL unsigned H5F_get_nmounts(const H5F_t *f); +H5_DLL unsigned H5F_get_read_attempts(const H5F_t *f); H5_DLL hid_t H5F_get_access_plist(H5F_t *f, hbool_t app_ref); H5_DLL hid_t H5F_get_id(H5F_t *file, hbool_t app_ref); H5_DLL herr_t H5F_get_obj_count(const H5F_t *f, unsigned types, hbool_t app_ref, size_t *obj_id_count_ptr); @@ -719,6 +730,13 @@ H5_DLL herr_t H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr, H5_DLL herr_t H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id); H5_DLL herr_t H5F_evict_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id); +/* Functions that verify a piece of metadata with checksum */ +H5_DLL herr_t H5F_get_checksums(const uint8_t *buf, size_t chk_size, uint32_t *s_chksum, uint32_t *c_chksum); + +/* Routine to track the # of retries */ +H5_DLL herr_t H5F_track_metadata_read_retries(H5F_t *f, unsigned actype, unsigned retries); +H5_DLL herr_t H5F_set_retries(H5F_t *f); + /* Routine to invoke callback function upon object flush */ H5_DLL herr_t H5F_object_flush_cb(H5F_t *f, hid_t obj_id); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 6db5a69..018618e 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -175,6 +175,14 @@ typedef enum H5F_file_space_type_t { H5F_FILE_SPACE_NTYPES /* must be last */ } H5F_file_space_type_t; +/* Data structure to report the collection of read retries for metadata items with checksum */ +/* Used by public routine H5Fget_metadata_read_retry_info() */ +#define H5F_NUM_METADATA_READ_RETRY_TYPES 21 +typedef struct H5F_retry_info_t { + unsigned nbins; + uint32_t *retries[H5F_NUM_METADATA_READ_RETRY_TYPES]; +} H5F_retry_info_t; + /* Callback for H5Pset_object_flush_cb() in a file access property list */ typedef herr_t (*H5F_flush_cb_t)(hid_t object_id, void *udata); @@ -216,6 +224,7 @@ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id, H5_DLL herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id); H5_DLL ssize_t H5Fget_name(hid_t obj_id, char *name, size_t size); H5_DLL herr_t H5Fget_info2(hid_t obj_id, H5F_info2_t *finfo); +H5_DLL herr_t H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info); H5_DLL ssize_t H5Fget_free_sections(hid_t file_id, H5F_mem_t type, size_t nsects, H5F_sect_info_t *sect_info/*out*/); H5_DLL herr_t H5Fclear_elink_file_cache(hid_t file_id); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index e5e785e..2ffe766 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -322,6 +322,29 @@ H5F_get_nmounts(const H5F_t *f) /*------------------------------------------------------------------------- + * Function: H5F_get_read_attempts + * + * Purpose: Retrieve the file's 'read_attempts' value + * + * Return: '# of read attempts' on success/abort on failure (shouldn't fail) + * + * Programmer: Vaili Choi; Sept 2013 + * + *------------------------------------------------------------------------- + */ +unsigned +H5F_get_read_attempts(const H5F_t *f) +{ + /* Use FUNC_ENTER_NOAPI_NOINIT_NOERR here to avoid performance issues */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + HDassert(f); + + FUNC_LEAVE_NOAPI(f->shared->read_attempts) +} /* end H5F_get_read_attempts() */ + + +/*------------------------------------------------------------------------- * Function: H5F_get_fcpl * * Purpose: Retrieve the value of a file's FCPL. diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index 2592b90..6cfd9c7 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -67,26 +67,26 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5F__cache_superblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5F__cache_superblock_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5F__cache_superblock_get_final_load_size(const void *image_ptr, + size_t image_len, void *udata, size_t *actual_len); +static htri_t H5F__cache_superblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5F__cache_superblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5F__cache_superblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5F__cache_superblock_image_len(const void *thing, size_t *image_len); static herr_t H5F__cache_superblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, haddr_t addr, size_t len, - size_t compressed_len, haddr_t *new_addr, size_t *new_len, - size_t *new_compressed_len, unsigned *flags); + haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5F__cache_superblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5F__cache_superblock_free_icr(void *thing); -static herr_t H5F__cache_drvrinfo_get_load_size(const void *udata, size_t *image_len); +static herr_t H5F__cache_drvrinfo_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5F__cache_drvrinfo_get_final_load_size(const void *image_ptr, + size_t image_len, void *udata, size_t *actual_len); static void *H5F__cache_drvrinfo_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5F__cache_drvrinfo_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5F__cache_drvrinfo_image_len(const void *thing, size_t *image_len); static herr_t H5F__cache_drvrinfo_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5F__cache_drvrinfo_free_icr(void *thing); @@ -102,14 +102,15 @@ const H5AC_class_t H5AC_SUPERBLOCK[1] = {{ "Superblock", /* Metadata client name (for debugging) */ H5FD_MEM_SUPER, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5F__cache_superblock_get_load_size,/* 'get_load_size' callback */ + H5F__cache_superblock_get_initial_load_size,/* 'get_initial_load_size' callback */ + H5F__cache_superblock_get_final_load_size, /* 'get_final_load_size' callback */ + H5F__cache_superblock_verify_chksum, /* 'verify_chksum' callback */ H5F__cache_superblock_deserialize, /* 'deserialize' callback */ H5F__cache_superblock_image_len, /* 'image_len' callback */ H5F__cache_superblock_pre_serialize,/* 'pre_serialize' callback */ H5F__cache_superblock_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5F__cache_superblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -119,14 +120,15 @@ const H5AC_class_t H5AC_DRVRINFO[1] = {{ "Driver info block", /* Metadata client name (for debugging) */ H5FD_MEM_SUPER, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5F__cache_drvrinfo_get_load_size, /* 'get_load_size' callback */ + H5F__cache_drvrinfo_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5F__cache_drvrinfo_get_final_load_size, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5F__cache_drvrinfo_deserialize, /* 'deserialize' callback */ H5F__cache_drvrinfo_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5F__cache_drvrinfo_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5F__cache_drvrinfo_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -146,7 +148,7 @@ H5FL_EXTERN(H5F_super_t); /*------------------------------------------------------------------------- - * Function: H5F__cache_superblock_get_load_size + * Function: H5F__cache_superblock_get_initial_load_size * * Purpose: Compute the size of the data structure on disk. * @@ -159,7 +161,7 @@ H5FL_EXTERN(H5F_super_t); *------------------------------------------------------------------------- */ static herr_t -H5F__cache_superblock_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *image_len) +H5F__cache_superblock_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_STATIC_NOERR @@ -168,10 +170,138 @@ H5F__cache_superblock_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *im /* Set the initial image length size */ *image_len = H5F_SUPERBLOCK_FIXED_SIZE + /* Fixed size of superblock */ - H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE; + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5F__cache_superblock_get_load_size() */ +} /* end H5F__cache_superblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__cache_superblock_get_final_load_size + * + * Purpose: Compute the final size of the data structure on disk. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@lbl.gov + * November 17, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__cache_superblock_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ + unsigned super_vers; /* Superblock version */ + uint8_t sizeof_addr; /* Size of offsets in the file (in bytes) */ + uint8_t sizeof_size; /* Size of lengths in the file (in bytes) */ + size_t variable_size; /* Variable size of superblock */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check arguments */ + HDassert(image); + HDassert(udata); + HDassert(udata->f); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Skip over file signature */ + image += H5F_SIGNATURE_LEN; + + /* Superblock version */ + super_vers = *image++; + if(super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad superblock version number") + + /* Save the version to be used in verify_chksum callback */ + udata->super_vers = super_vers; + + /* Sanity check */ + HDassert(((size_t)(image - (const uint8_t *)_image)) == H5F_SUPERBLOCK_FIXED_SIZE); + HDassert(image_len >= H5F_SUPERBLOCK_FIXED_SIZE + 6); + + /* Determine the size of addresses & size of offsets, for computing the + * variable-sized portion of the superblock. + */ + if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { + sizeof_addr = image[4]; + sizeof_size = image[5]; + } /* end if */ + else { + sizeof_addr = image[0]; + sizeof_size = image[1]; + } /* end else */ + if(sizeof_addr != 2 && sizeof_addr != 4 && + sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number in an address") + if(sizeof_size != 2 && sizeof_size != 4 && + sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad byte number for object size") + + /* Determine the size of the variable-length part of the superblock */ + variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size); + HDassert(variable_size > 0); + + /* Sanity check */ + HDassert(image_len == (H5F_SUPERBLOCK_FIXED_SIZE + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE)); + + /* Make certain we can read the variable-sized portion of the superblock */ + if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + + /* Set the final size for the cache image */ + *actual_len = H5F_SUPERBLOCK_FIXED_SIZE + variable_size; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__cache_superblock_get_final_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__cache_superblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5F__cache_superblock_verify_chksum(const void *_image, size_t len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5F_superblock_cache_ud_t *udata = (H5F_superblock_cache_ud_t *)_udata; /* User data */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* No checksum for version 0 & 1 */ + if(udata->super_vers >= HDF5_SUPERBLOCK_VERSION_2) { + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__cache_superblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -251,176 +381,161 @@ H5F__cache_superblock_deserialize(const void *_image, size_t len, void *_udata, variable_size = (size_t)H5F_SUPERBLOCK_VARLEN_SIZE(super_vers, sizeof_addr, sizeof_size); HDassert(variable_size > 0); - /* Handle metadata cache retry for variable-sized portion of the superblock */ - if(len != (H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) { - /* Sanity check */ - HDassert(len == (H5F_SUPERBLOCK_FIXED_SIZE + H5F_SUPERBLOCK_MINIMAL_VARLEN_SIZE)); + HDassert(len == (H5F_SUPERBLOCK_FIXED_SIZE + variable_size)); - /* Make certain we can read the variabled-sized portion of the superblock */ - if(H5F__set_eoa(udata->f, H5FD_MEM_SUPER, (haddr_t)(H5F_SUPERBLOCK_FIXED_SIZE + variable_size)) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "set end of space allocation request failed") - } /* end if */ - else { - /* Check for older version of superblock format */ - if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { - uint32_t status_flags; /* File status flags */ - unsigned sym_leaf_k; /* Symbol table leaf node's 'K' value */ - unsigned snode_btree_k; /* B-tree symbol table internal node 'K' value */ - unsigned chunk_btree_k; /* B-tree chunk internal node 'K' value */ - - /* Freespace version (hard-wired) */ - if(HDF5_FREESPACE_VERSION != *image++) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad free space version number") - - /* Root group version number (hard-wired) */ - if(HDF5_OBJECTDIR_VERSION != *image++) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad object directory version number") - - /* Skip over reserved byte */ - image++; - - /* Shared header version number (hard-wired) */ - if(HDF5_SHAREDHEADER_VERSION != *image++) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad shared-header format version number") - - /* Size of file addresses */ - sizeof_addr = *image++; - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - sblock->sizeof_addr = sizeof_addr; - udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ - - /* Size of file sizes */ - sizeof_size = *image++; - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_size = sizeof_size; - udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ - - /* Skip over reserved byte */ - image++; - - /* Various B-tree sizes */ - UINT16DECODE(image, sym_leaf_k); - if(sym_leaf_k == 0) - HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad symbol table leaf node 1/2 rank") - udata->sym_leaf_k = sym_leaf_k; /* Keep a local copy also */ - - /* Need 'get' call to set other array values */ - UINT16DECODE(image, snode_btree_k); - if(snode_btree_k == 0) - HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad 1/2 rank for btree internal nodes") - udata->btree_k[H5B_SNODE_ID] = snode_btree_k; - - /* - * Delay setting the value in the property list until we've checked - * for the indexed storage B-tree internal 'K' value later. - */ - - /* File status flags (not really used yet) */ - UINT32DECODE(image, status_flags); - HDassert(status_flags <= 255); - sblock->status_flags = (uint8_t)status_flags; - if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") - - /* - * If the superblock version # is greater than 0, read in the indexed - * storage B-tree internal 'K' value - */ - if(super_vers > HDF5_SUPERBLOCK_VERSION_DEF) { - UINT16DECODE(image, chunk_btree_k); - - /* Reserved bytes are present only in version 1 */ - if(super_vers == HDF5_SUPERBLOCK_VERSION_1) - image += 2; /* reserved */ - } /* end if */ - else - chunk_btree_k = HDF5_BTREE_CHUNK_IK_DEF; - udata->btree_k[H5B_CHUNK_ID] = chunk_btree_k; - - /* Remainder of "variable-sized" portion of superblock */ - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->driver_addr/*out*/); - - /* Allocate space for the root group symbol table entry */ - HDassert(!sblock->root_ent); - if(NULL == (sblock->root_ent = (H5G_entry_t *)H5MM_calloc(sizeof(H5G_entry_t)))) - HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for root group symbol table entry") - - /* decode the root group symbol table entry */ - if(H5G_ent_decode(udata->f, (const uint8_t **)&image, sblock->root_ent) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode root group symbol table entry") - - /* Set the root group address to the correct value */ - sblock->root_addr = sblock->root_ent->header; - - /* This step is for h5repart tool only. If user wants to change file driver - * from family to sec2 while using h5repart, set the driver address to - * undefined to let the library ignore the family driver information saved - * in the superblock. - */ - if(udata->ignore_drvrinfo && H5F_addr_defined(sblock->driver_addr)) { - /* Eliminate the driver info */ - sblock->driver_addr = HADDR_UNDEF; - udata->drvrinfo_removed = TRUE; - } /* end if */ + /* Check for older version of superblock format */ + if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { + uint32_t status_flags; /* File status flags */ + unsigned sym_leaf_k; /* Symbol table leaf node's 'K' value */ + unsigned snode_btree_k; /* B-tree symbol table internal node 'K' value */ + unsigned chunk_btree_k; /* B-tree chunk internal node 'K' value */ + + /* Freespace version (hard-wired) */ + if(HDF5_FREESPACE_VERSION != *image++) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad free space version number") + + /* Root group version number (hard-wired) */ + if(HDF5_OBJECTDIR_VERSION != *image++) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad object directory version number") + + /* Skip over reserved byte */ + image++; + + /* Shared header version number (hard-wired) */ + if(HDF5_SHAREDHEADER_VERSION != *image++) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad shared-header format version number") + + /* Size of file addresses */ + sizeof_addr = *image++; + if(sizeof_addr != 2 && sizeof_addr != 4 && + sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") + sblock->sizeof_addr = sizeof_addr; + udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ + + /* Size of file sizes */ + sizeof_size = *image++; + if(sizeof_size != 2 && sizeof_size != 4 && + sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") + sblock->sizeof_size = sizeof_size; + udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ + + /* Skip over reserved byte */ + image++; + + /* Various B-tree sizes */ + UINT16DECODE(image, sym_leaf_k); + if(sym_leaf_k == 0) + HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad symbol table leaf node 1/2 rank") + udata->sym_leaf_k = sym_leaf_k; /* Keep a local copy also */ + + /* Need 'get' call to set other array values */ + UINT16DECODE(image, snode_btree_k); + if(snode_btree_k == 0) + HGOTO_ERROR(H5E_FILE, H5E_BADRANGE, NULL, "bad 1/2 rank for btree internal nodes") + udata->btree_k[H5B_SNODE_ID] = snode_btree_k; + + /* + * Delay setting the value in the property list until we've checked + * for the indexed storage B-tree internal 'K' value later. + */ - /* NOTE: Driver info block is decoded separately, later */ + /* File status flags (not really used yet) */ + UINT32DECODE(image, status_flags); + HDassert(status_flags <= 255); + sblock->status_flags = (uint8_t)status_flags; + if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") - } /* end if */ - else { - uint32_t computed_chksum; /* Computed checksum */ - uint32_t read_chksum; /* Checksum read from file */ - - /* Size of file addresses */ - sizeof_addr = *image++; - if(sizeof_addr != 2 && sizeof_addr != 4 && - sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") - sblock->sizeof_addr = sizeof_addr; - udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ - - /* Size of file sizes */ - sizeof_size = *image++; - if(sizeof_size != 2 && sizeof_size != 4 && - sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") - sblock->sizeof_size = sizeof_size; - udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ - - /* File status flags (not really used yet) */ - sblock->status_flags = *image++; - if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS) - HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") - - /* Base, superblock extension, end of file & root group object header addresses */ - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/); - H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->root_addr/*out*/); - - /* Compute checksum for superblock */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); - - /* Decode checksum */ - UINT32DECODE(image, read_chksum); - - /* Verify correct checksum */ - if(read_chksum != computed_chksum) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad checksum on driver information block") - - /* The Driver Information Block may not appear with the version - * 2 super block. Thus we set the driver_addr field of the in - * core representation of the super block HADDR_UNDEF to prevent - * any attempt to load the Driver Information Block. - */ - sblock->driver_addr = HADDR_UNDEF; - } /* end else */ + /* + * If the superblock version # is greater than 0, read in the indexed + * storage B-tree internal 'K' value + */ + if(super_vers > HDF5_SUPERBLOCK_VERSION_DEF) { + UINT16DECODE(image, chunk_btree_k); + + /* Reserved bytes are present only in version 1 */ + if(super_vers == HDF5_SUPERBLOCK_VERSION_1) + image += 2; /* reserved */ + } /* end if */ + else + chunk_btree_k = HDF5_BTREE_CHUNK_IK_DEF; + udata->btree_k[H5B_CHUNK_ID] = chunk_btree_k; + + /* Remainder of "variable-sized" portion of superblock */ + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->driver_addr/*out*/); + + /* Allocate space for the root group symbol table entry */ + HDassert(!sblock->root_ent); + if(NULL == (sblock->root_ent = (H5G_entry_t *)H5MM_calloc(sizeof(H5G_entry_t)))) + HGOTO_ERROR(H5E_FILE, H5E_CANTALLOC, NULL, "can't allocate space for root group symbol table entry") + + /* decode the root group symbol table entry */ + if(H5G_ent_decode(udata->f, (const uint8_t **)&image, sblock->root_ent) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "can't decode root group symbol table entry") + + /* Set the root group address to the correct value */ + sblock->root_addr = sblock->root_ent->header; + + /* This step is for h5repart tool only. If user wants to change file driver + * from family to sec2 while using h5repart, set the driver address to + * undefined to let the library ignore the family driver information saved + * in the superblock. + */ + if(udata->ignore_drvrinfo && H5F_addr_defined(sblock->driver_addr)) { + /* Eliminate the driver info */ + sblock->driver_addr = HADDR_UNDEF; + udata->drvrinfo_removed = TRUE; + } /* end if */ + + /* NOTE: Driver info block is decoded separately, later */ + + } /* end if */ + else { + uint32_t read_chksum; /* Checksum read from file */ + + /* Size of file addresses */ + sizeof_addr = *image++; + if(sizeof_addr != 2 && sizeof_addr != 4 && + sizeof_addr != 8 && sizeof_addr != 16 && sizeof_addr != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number in an address") + sblock->sizeof_addr = sizeof_addr; + udata->f->shared->sizeof_addr = sizeof_addr; /* Keep a local copy also */ + + /* Size of file sizes */ + sizeof_size = *image++; + if(sizeof_size != 2 && sizeof_size != 4 && + sizeof_size != 8 && sizeof_size != 16 && sizeof_size != 32) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad byte number for object size") + sblock->sizeof_size = sizeof_size; + udata->f->shared->sizeof_size = sizeof_size; /* Keep a local copy also */ + + /* File status flags (not really used yet) */ + sblock->status_flags = *image++; + if(sblock->status_flags & ~H5F_SUPER_ALL_FLAGS) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad flag value for superblock") + + /* Base, superblock extension, end of file & root group object header addresses */ + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->base_addr/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->ext_addr/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &udata->stored_eof/*out*/); + H5F_addr_decode(udata->f, (const uint8_t **)&image, &sblock->root_addr/*out*/); + + /* checksum verification already done in verify_chksum cb */ + + /* Decode checksum */ + UINT32DECODE(image, read_chksum); + + /* The Driver Information Block may not appear with the version + * 2 super block. Thus we set the driver_addr field of the in + * core representation of the super block HADDR_UNDEF to prevent + * any attempt to load the Driver Information Block. + */ + sblock->driver_addr = HADDR_UNDEF; } /* end else */ /* Sanity check */ @@ -453,7 +568,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__cache_superblock_image_len(const void *_thing, size_t *image_len, hbool_t *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5F__cache_superblock_image_len(const void *_thing, size_t *image_len) { const H5F_super_t *sblock = (const H5F_super_t *)_thing; /* Pointer to the object */ @@ -464,14 +579,10 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len, hbool_t * HDassert(sblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(sblock->cache_info.type == H5AC_SUPERBLOCK); HDassert(image_len); - HDassert(compressed_ptr); /* Set the image length size */ *image_len = (size_t)H5F_SUPERBLOCK_SIZE(sblock); - /* Set *compressed_ptr to FALSE unconditionally */ - *compressed_ptr = FALSE; - FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5F__cache_superblock_image_len() */ @@ -511,8 +622,7 @@ H5F__cache_superblock_image_len(const void *_thing, size_t *image_len, hbool_t * static herr_t H5F__cache_superblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, haddr_t H5_ATTR_UNUSED addr, size_t H5_ATTR_UNUSED len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t H5_ATTR_UNUSED *new_addr, - size_t H5_ATTR_UNUSED *new_len, size_t H5_ATTR_UNUSED *new_compressed_len, + haddr_t H5_ATTR_UNUSED *new_addr, size_t H5_ATTR_UNUSED *new_len, unsigned H5_ATTR_UNUSED *flags) { H5P_genplist_t *dxpl = NULL; /* DXPL for setting ring */ @@ -771,9 +881,9 @@ done: /*------------------------------------------------------------------------- - * Function: H5F__cache_drvrinfo_get_load_size + * Function: H5F__cache_drvrinfo_get_initial_load_size * - * Purpose: Compute the size of the data structure on disk. + * Purpose: Compute the intiial size of the data structure on disk. * * Return: Non-negative on success/Negative on failure * @@ -784,7 +894,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__cache_drvrinfo_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *image_len) +H5F__cache_drvrinfo_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_STATIC_NOERR @@ -795,7 +905,76 @@ H5F__cache_drvrinfo_get_load_size(const void H5_ATTR_UNUSED *udata, size_t *imag *image_len = H5F_DRVINFOBLOCK_HDR_SIZE; /* Fixed size portion of driver info block */ FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5F__cache_drvrinfo_get_load_size() */ +} /* end H5F__cache_drvrinfo_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5F__cache_drvrinfo_get_final_load_size + * + * Purpose: Compute the final size of the data structure on disk. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * koziol@lbl.gov + * November 17, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5F__cache_drvrinfo_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5F_drvrinfo_cache_ud_t *udata = (H5F_drvrinfo_cache_ud_t *)_udata; /* User data */ + unsigned drv_vers; /* Version of driver info block */ + size_t drvinfo_len; /* Length of encoded buffer */ + haddr_t eoa; /* Current EOA for the file */ + haddr_t min_eoa; /* Minimum EOA needed for reading the driver info */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Check arguments */ + HDassert(image); + HDassert(udata); + HDassert(udata->f); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Version number */ + drv_vers = *image++; + if(drv_vers != HDF5_DRIVERINFO_VERSION_0) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, FAIL, "bad driver information block version number") + + image += 3; /* reserved bytes */ + + /* Driver info size */ + UINT32DECODE(image, drvinfo_len); + + /* Sanity check */ + HDassert(image_len == H5F_DRVINFOBLOCK_HDR_SIZE); + + /* Extend the EOA if required so that we can read the complete driver info block */ + + /* Get current EOA... */ + if((eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "driver get_eoa request failed") + + /* ... if it is too small, extend it. */ + min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len; + + /* If it grew, set it */ + if(H5F_addr_gt(min_eoa, eoa)) + if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "set end of space allocation request failed") + + /* Set the final size for the cache image */ + *actual_len = H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo_len; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5F__cache_drvrinfo_get_final_load_size() */ /*------------------------------------------------------------------------- @@ -850,35 +1029,11 @@ H5F__cache_drvrinfo_deserialize(const void *_image, size_t len, void *_udata, drv_name[8] = '\0'; image += 8; /* advance past name/version */ - /* Handle metadata cache retry for variable-sized portion of the driver info block */ - if(len != (H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len)) { - /* Sanity check */ - HDassert(len == H5F_DRVINFOBLOCK_HDR_SIZE); - - /* extend the eoa if required so that we can read the complete driver info block */ - { - haddr_t eoa; - haddr_t min_eoa; - - /* get current eoa... */ - if ((eoa = H5FD_get_eoa(udata->f->shared->lf, H5FD_MEM_SUPER)) == HADDR_UNDEF) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, NULL, "driver get_eoa request failed") - - /* ... if it is too small, extend it. */ - min_eoa = udata->driver_addr + H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len; + HDassert(len == (H5F_DRVINFOBLOCK_HDR_SIZE + drvinfo->len)); - if ( H5F_addr_gt(min_eoa, eoa) ) - if(H5FD_set_eoa(udata->f->shared->lf, H5FD_MEM_SUPER, min_eoa) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, \ - "set end of space allocation request failed") - } - - } /* end if */ - else { - /* Validate and decode driver information */ - if(H5FD_sb_load(udata->f->shared->lf, drv_name, image) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "unable to decode driver information") - } /* end if */ + /* Validate and decode driver information */ + if(H5FD_sb_load(udata->f->shared->lf, drv_name, image) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTDECODE, NULL, "unable to decode driver information") /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) <= len); @@ -909,8 +1064,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5F__cache_drvrinfo_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5F__cache_drvrinfo_image_len(const void *_thing, size_t *image_len) { const H5O_drvinfo_t *drvinfo = (const H5O_drvinfo_t *)_thing; /* Pointer to the object */ @@ -921,11 +1075,10 @@ H5F__cache_drvrinfo_image_len(const void *_thing, size_t *image_len, HDassert(drvinfo->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(drvinfo->cache_info.type == H5AC_DRVRINFO); HDassert(image_len); - HDassert(compressed_ptr); /* Set the image length size */ *image_len = (size_t)(H5F_DRVINFOBLOCK_HDR_SIZE + /* Fixed-size portion of driver info block */ - drvinfo->len); /* Variable-size portion of driver info block */ + drvinfo->len); /* Variable-size portion of driver info block */ FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5F__cache_drvrinfo_image_len() */ diff --git a/src/H5Gcache.c b/src/H5Gcache.c index e3303d1..d153560 100644 --- a/src/H5Gcache.c +++ b/src/H5Gcache.c @@ -63,11 +63,10 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5G__cache_node_get_load_size(const void *udata, size_t *image_len); +static herr_t H5G__cache_node_get_initial_load_size(void *udata, size_t *image_len); static void *H5G__cache_node_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5G__cache_node_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5G__cache_node_image_len(const void *thing, size_t *image_len); static herr_t H5G__cache_node_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5G__cache_node_free_icr(void *thing); @@ -93,14 +92,15 @@ const H5AC_class_t H5AC_SNODE[1] = {{ "Symbol table node", /* Metadata client name (for debugging) */ H5FD_MEM_BTREE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5G__cache_node_get_load_size, /* 'get_load_size' callback */ + H5G__cache_node_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5G__cache_node_deserialize, /* 'deserialize' callback */ H5G__cache_node_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5G__cache_node_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5G__cache_node_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -113,17 +113,11 @@ H5FL_SEQ_EXTERN(H5G_entry_t); /*------------------------------------------------------------------------- - * Function: H5G__cache_node_get_load_size() + * Function: H5G__cache_node_get_initial_load_size() * - * Purpose: Determine the size of the on disk image of the node, and + * Purpose: Determine the size of the on-disk image of the node, and * return this value in *image_len. * - * Note that this computation requires access to the file pointer, - * which is not provided in the parameter list for this callback. - * Finesse this issue by passing in the file pointer twice to the - * H5AC_protect() call -- once as the file pointer proper, and - * again as the user data. - * * Return: Success: SUCCEED * Failure: FAIL * @@ -133,9 +127,9 @@ H5FL_SEQ_EXTERN(H5G_entry_t); *------------------------------------------------------------------------- */ static herr_t -H5G__cache_node_get_load_size(const void *_udata, size_t *image_len) +H5G__cache_node_get_initial_load_size(void *_udata, size_t *image_len) { - const H5F_t *f = (const H5F_t *)_udata; /* User data for callback */ + H5F_t *f = (H5F_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -143,11 +137,11 @@ H5G__cache_node_get_load_size(const void *_udata, size_t *image_len) HDassert(f); HDassert(image_len); - /* report image length */ + /* Set the image length size */ *image_len = (size_t)(H5G_NODE_SIZE(f)); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5G__cache_node_get_load_size() */ +} /* end H5G__cache_node_get_initial_load_size() */ /*------------------------------------------------------------------------- @@ -241,8 +235,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5G__cache_node_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5G__cache_node_image_len(const void *_thing, size_t *image_len) { const H5G_node_t *sym = (const H5G_node_t *)_thing; /* Pointer to object */ @@ -259,11 +252,6 @@ H5G__cache_node_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5G__cache_node_image_len() */ - -/*************************************/ -/* no H5G__cache_node_pre_serialize() */ -/*************************************/ - /*------------------------------------------------------------------------- * Function: H5G__cache_node_serialize @@ -323,11 +311,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5G__cache_node_serialize() */ - -/***************************************/ -/* no H5G__cache_node_notify() function */ -/***************************************/ - /*------------------------------------------------------------------------- * Function: H5G__cache_node_free_icr diff --git a/src/H5HFcache.c b/src/H5HFcache.c index 3308f25..04f1459 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -73,43 +73,40 @@ static herr_t H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *d static herr_t H5HF__dtable_decode(H5F_t *f, const uint8_t **pp, H5HF_dtable_t *dtable); /* Metadata cache (H5AC) callbacks */ -static herr_t H5HF__cache_hdr_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HF__cache_hdr_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5HF__cache_hdr_get_final_load_size(const void *image_ptr, + size_t image_len, void *udata, size_t *actual_len); +static htri_t H5HF__cache_hdr_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5HF__cache_hdr_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HF__cache_hdr_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5HF__cache_hdr_image_len(const void *thing, size_t *image_len); static herr_t H5HF__cache_hdr_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5HF__cache_hdr_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HF__cache_hdr_free_icr(void *thing); -static herr_t H5HF__cache_iblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HF__cache_iblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5HF__cache_iblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5HF__cache_iblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HF__cache_iblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5HF__cache_iblock_image_len(const void *thing, size_t *image_len); static herr_t H5HF__cache_iblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5HF__cache_iblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HF__cache_iblock_notify(H5C_notify_action_t action, void *thing); static herr_t H5HF__cache_iblock_free_icr(void *thing); -static herr_t H5HF__cache_dblock_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HF__cache_dblock_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5HF__cache_dblock_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5HF__cache_dblock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HF__cache_dblock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5HF__cache_dblock_image_len(const void *thing, size_t *image_len); static herr_t H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, - void *thing, haddr_t addr, size_t len, size_t compressed_len, - haddr_t *new_addr, size_t *new_len, size_t *new_compressed_len, + void *thing, haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags); static herr_t H5HF__cache_dblock_serialize(const H5F_t *f, void *image, size_t len, void *thing); @@ -139,14 +136,15 @@ const H5AC_class_t H5AC_FHEAP_HDR[1] = {{ "fractal heap header", /* Metadata client name (for debugging) */ H5FD_MEM_FHEAP_HDR, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5HF__cache_hdr_get_load_size, /* 'get_load_size' callback */ + H5HF__cache_hdr_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5HF__cache_hdr_get_final_load_size, /* 'get_final_load_size' callback */ + H5HF__cache_hdr_verify_chksum, /* 'verify_chksum' callback */ H5HF__cache_hdr_deserialize, /* 'deserialize' callback */ H5HF__cache_hdr_image_len, /* 'image_len' callback */ H5HF__cache_hdr_pre_serialize, /* 'pre_serialize' callback */ H5HF__cache_hdr_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5HF__cache_hdr_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -156,14 +154,15 @@ const H5AC_class_t H5AC_FHEAP_IBLOCK[1] = {{ "fractal heap indirect block", /* Metadata client name (for debugging) */ H5FD_MEM_FHEAP_IBLOCK, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5HF__cache_iblock_get_load_size, /* 'get_load_size' callback */ + H5HF__cache_iblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5HF__cache_iblock_verify_chksum, /* 'verify_chksum' callback */ H5HF__cache_iblock_deserialize, /* 'deserialize' callback */ H5HF__cache_iblock_image_len, /* 'image_len' callback */ H5HF__cache_iblock_pre_serialize, /* 'pre_serialize' callback */ H5HF__cache_iblock_serialize, /* 'serialize' callback */ H5HF__cache_iblock_notify, /* 'notify' callback */ H5HF__cache_iblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -172,15 +171,16 @@ const H5AC_class_t H5AC_FHEAP_DBLOCK[1] = {{ H5AC_FHEAP_DBLOCK_ID, /* Metadata client ID */ "fractal heap direct block", /* Metadata client name (for debugging) */ H5FD_MEM_FHEAP_DBLOCK, /* File space memory type for client */ - H5C__CLASS_COMPRESSED_FLAG, /* Client class behavior flags */ - H5HF__cache_dblock_get_load_size, /* 'get_load_size' callback */ + H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ + H5HF__cache_dblock_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5HF__cache_dblock_verify_chksum, /* 'verify_chksum' callback */ H5HF__cache_dblock_deserialize, /* 'deserialize' callback */ H5HF__cache_dblock_image_len, /* 'image_len' callback */ H5HF__cache_dblock_pre_serialize, /* 'pre_serialize' callback */ H5HF__cache_dblock_serialize, /* 'serialize' callback */ H5HF__cache_dblock_notify, /* 'notify' callback */ H5HF__cache_dblock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -298,21 +298,13 @@ H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HF__dtable_encode() */ -/**************************************************/ -/* metadata cache callback definitions for header */ -/**************************************************/ - /*------------------------------------------------------------------------- - * Function: H5HF__cache_hdr_get_load_size() + * Function: H5HF__cache_hdr_get_initial_load_size() * * Purpose: Determine the size of the fractal heap header on disk, * and set *image_len to this value. * - * This code is based on the old H5HF_cache_hdr_load() routine - * that was used with the version 2 metadata cache. Note the - * use of a dummy header to compute the on disk size of the header. - * * Note also that the value returned by this function presumes that * there is no I/O filtering data in the header. If there is, the * size reported will be too small, and H5C_load_entry() @@ -327,10 +319,10 @@ H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable) *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_hdr_get_load_size(const void *_udata, size_t *image_len) +H5HF__cache_hdr_get_initial_load_size(void *_udata, size_t *image_len) { - const H5HF_hdr_cache_ud_t *udata = (const H5HF_hdr_cache_ud_t *)_udata; /* pointer to user data */ - H5HF_hdr_t dummy_hdr; /* dummy header -- to compute size */ + H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* Pointer to user data */ + H5HF_hdr_t dummy_hdr; /* Dummy header -- to compute size */ FUNC_ENTER_STATIC_NOERR @@ -347,40 +339,108 @@ H5HF__cache_hdr_get_load_size(const void *_udata, size_t *image_len) *image_len = (size_t)H5HF_HEADER_SIZE(&dummy_hdr); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HF__cache_hdr_get_load_size() */ +} /* end H5HF__cache_hdr_get_initial_load_size() */ /*------------------------------------------------------------------------- - * Function: H5HF__cache_hdr_deserialize + * Function: H5HF__cache_hdr_get_final_load_size() * - * Purpose: Given a buffer containing an on disk image of a fractal heap - * header block, allocate an instance of H5HF_hdr_t, load the contents - * of the buffer into into the new instance of H5HF_hdr_t, and then - * return a pointer to the new instance. + * Purpose: Determine the final size of the fractal heap header on disk, + * and set *actual_len to this value. * - * Since H5HF__cache_hdr_get_load_size() reports header on disk size - * base on the assumption that the header contains no I/O filtering - * data, it is possible that the provided image will be too small. + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * November 18, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HF__cache_hdr_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* pointer to user data */ + unsigned filter_len; /* Size of I/O filter information (in bytes) */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(image); + HDassert(udata); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Magic number */ + if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "wrong fractal heap header signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(*image++ != H5HF_HDR_VERSION) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong fractal heap header version") + + /* General heap information */ + image += 2; /* Heap ID length */ + UINT16DECODE(image, filter_len); /* I/O filters' encoded length */ + + /* Check for I/O filter info on this heap */ + if(filter_len > 0) + /* Compute the extra heap header size */ + *actual_len += (size_t)(H5F_SIZEOF_SIZE(udata->f) /* Size of size for filtered root direct block */ + + (unsigned)4 /* Size of filter mask for filtered root direct block */ + + filter_len); /* Size of encoded I/O filter info */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__cache_hdr_get_final_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF__cache_hdr_verify_chksum * - * In this case, we DO NOT flag an error when this is discovered. - * Instead, we make note of the correct image size, and report - * success. + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. * - * Since H5HF__cache_hdr_image_len() callback is defined, - * H5C_load_entry() will call H5HF__cache_hdr_image_len() and - * obtain the correct image length. + * Return: Success: TRUE/FALSE + * Failure: Negative * - * Since the H5AC__CLASS_SPECULATIVE_LOAD_FLAG is set, - * H5C_load_entry() will load an image of the correct size, and - * then call this function again to deserialize it. Before doing - * so, it will also call H5HF__cache_hdr_free_icr() to discard the - * result of the first deserialize call. + * Programmer: Vailin Choi; Aug 2015 * - * Note that the v2 B-tree and free space manager associated - * with the fractal heap (roots stored in the huge_bt2 and fspace - * fields respectively) are not loaded at this time. As best I can - * tell from reviewing the code, they are loaded or created when - * they are accessed. + *------------------------------------------------------------------------- + */ +static htri_t +H5HF__cache_hdr_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__cache_hdr_verify_chksum() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF__cache_hdr_deserialize + * + * Purpose: Given a buffer containing an on disk image of a fractal heap + * header block, allocate an instance of H5HF_hdr_t, load the contents + * of the buffer into into the new instance of H5HF_hdr_t, and then + * return a pointer to the new instance. * * Return: Success: Pointer to in core representation * Failure: NULL @@ -397,9 +457,7 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, H5HF_hdr_t *hdr = NULL; /* Fractal heap info */ H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; /* User data for callback */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into into supplied image */ - size_t size; /* Header size */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ uint8_t heap_flags; /* Status flags for heap */ void * ret_value = NULL; /* Return value */ @@ -415,28 +473,6 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (hdr = H5HF_hdr_alloc(udata->f))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Compute the 'base' size of the fractal heap header on disk */ - size = (size_t)H5HF_HEADER_SIZE(hdr); - - /* the size we have just calculated presumes that there is no I/O - * filter information in the header. If there is no filter information, - * the deserialize operation should succeed. - * - * If there is filter information, the first attempt to deserialize - * the header will reveal this. In this case, we will be unable to - * deserialize the header as the supplied image will be too small. - * However, we will make note of the correct size and report success - * anyway. - * - * When H5C_load_entry() calls H5HF__cache_hdr_image_len(), we will report - * the correct size. Since the H5C__CLASS_SPECULATIVE_LOAD_FLAG is set, - * this will prompt H5C_load_entry() to load the correct size image, - * discard the result of the first attempt at deserialization, and - * call this routine a second time to deserialize the correct size - * buffer. - */ - HDassert(size <= len); - /* Magic number */ if(HDmemcmp(image, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "wrong fractal heap header signature") @@ -480,32 +516,24 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, if(H5HF__dtable_decode(hdr->f, &image, &(hdr->man_dtable)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTENCODE, NULL, "unable to encode managed obj. doubling table info") + /* Set the fractal heap header's 'base' size */ + hdr->heap_size = (size_t)H5HF_HEADER_SIZE(hdr); + /* Sanity check */ /* (allow for checksum not decoded yet) */ - HDassert((size_t)(image - (const uint8_t *)_image) == (size - H5HF_SIZEOF_CHKSUM)); + HDassert((size_t)(image - (const uint8_t *)_image) == (hdr->heap_size - H5HF_SIZEOF_CHKSUM)); /* Check for I/O filter information to decode */ if(hdr->filter_len > 0) { - size_t filter_info_size; /* Size of filter information */ H5O_pline_t *pline; /* Pipeline information from the header on disk */ - /* Compute the size of the extra filter information */ - filter_info_size = (size_t)(hdr->sizeof_size /* Size of size for filtered root direct block */ - + (unsigned)4 /* Size of filter mask for filtered root direct block */ - + hdr->filter_len); /* Size of encoded I/O filter info */ + /* Sanity check */ + HDassert(len > hdr->heap_size); /* A header with filter info is > than a standard header */ /* Compute the heap header's size */ - hdr->heap_size = size + filter_info_size; - - if(size == len) - /* we were supplied with too small a buffer -- goto done - * and let H5C_load_entry() retry with a larger buffer - */ - HGOTO_DONE((void *)hdr) - - else - if((size + filter_info_size) != len) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, NULL, "bad image len") + hdr->heap_size += (size_t)(hdr->sizeof_size /* Size of size for filtered root direct block */ + + (unsigned)4 /* Size of filter mask for filtered root direct block */ + + hdr->filter_len); /* Size of encoded I/O filter info */ /* Decode the size of a filtered root direct block */ H5F_DECODE_LENGTH(udata->f, image, hdr->pline_root_direct_size); @@ -517,6 +545,7 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, if(NULL == (pline = (H5O_pline_t *)H5O_msg_decode(hdr->f, udata->dxpl_id, NULL, H5O_PLINE_ID, image))) HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode I/O pipeline filters") + /* Advance past filter info to checksum */ image += hdr->filter_len; /* Copy the information into the header's I/O pipeline structure */ @@ -526,13 +555,6 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* Release the space allocated for the I/O pipeline filters */ H5O_msg_free(H5O_PLINE_ID, pline); } /* end if */ - else - /* Set the heap header's size */ - hdr->heap_size = size; - - /* Compute checksum on entire header */ - /* (including the filter information, if present) */ - computed_chksum = H5_checksum_metadata(_image, (size_t)(image - (const uint8_t *)_image), 0); /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -540,10 +562,6 @@ H5HF__cache_hdr_deserialize(const void *_image, size_t len, void *_udata, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == hdr->heap_size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap header") - /* Finish initialization of heap header */ if(H5HF_hdr_finish_init(hdr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, NULL, "can't finish initializing shared fractal heap header") @@ -567,7 +585,7 @@ done: * disk image. * * If the header contains filter information, this size will be - * larger than the value returned by H5HF__cache_hdr_get_load_size(). + * larger than the value returned by H5HF__cache_hdr_get_initial_load_size(). * * Return: Success: SUCCEED * Failure: FAIL @@ -578,8 +596,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_hdr_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HF__cache_hdr_image_len(const void *_thing, size_t *image_len) { const H5HF_hdr_t *hdr = (const H5HF_hdr_t *)_thing; /* Fractal heap info */ @@ -619,9 +636,8 @@ H5HF__cache_hdr_image_len(const void *_thing, size_t *image_len, */ static herr_t H5HF__cache_hdr_pre_serialize(const H5F_t *f, hid_t H5_ATTR_UNUSED dxpl_id, - void *_thing, haddr_t addr, size_t len, size_t H5_ATTR_UNUSED compressed_len, - haddr_t H5_ATTR_UNUSED *new_addr, size_t H5_ATTR_UNUSED *new_len, - size_t H5_ATTR_UNUSED *new_compressed_len, unsigned *flags) + void *_thing, haddr_t addr, size_t len, haddr_t H5_ATTR_UNUSED *new_addr, + size_t H5_ATTR_UNUSED *new_len, unsigned *flags) { H5HF_hdr_t *hdr = (H5HF_hdr_t *)_thing; /* Fractal heap info */ herr_t ret_value = SUCCEED; /* Return value */ @@ -783,10 +799,6 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF__cache_hdr_serialize() */ -/***************************************/ -/* no H5HF__cache_hdr_notify() function */ -/***************************************/ - /*------------------------------------------------------------------------- * Function: H5HF__cache_hdr_free_icr @@ -833,13 +845,9 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF__cache_hdr_free_icr() */ -/***********************************************************/ -/* metadata cache callback definitions for indirect blocks */ -/***********************************************************/ - /*------------------------------------------------------------------------- - * Function: H5HF__cache_iblock_get_load_size() + * Function: H5HF__cache_iblock_get_initial_load_size() * * Purpose: Compute the size of the on disk image of the indirect * block, and place this value in *image_len. @@ -853,24 +861,60 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_iblock_get_load_size(const void *_udata, size_t *image_len) +H5HF__cache_iblock_get_initial_load_size(void *_udata, size_t *image_len) { - const H5HF_iblock_cache_ud_t *udata = (const H5HF_iblock_cache_ud_t *)_udata; /* User data for callback */ + H5HF_iblock_cache_ud_t *udata = (H5HF_iblock_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(udata); + HDassert(udata->par_info); + HDassert(udata->par_info->hdr); HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)H5HF_MAN_INDIRECT_SIZE(udata->par_info->hdr, *udata->nrows); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HF__cache_iblock_get_load_size() */ +} /* end H5HF__cache_iblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF__cache_iblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5HF__cache_iblock_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__cache_iblock_verify_chksum() */ -/***********************************************************/ -/* metadata cache callback definitions for indirect blocks */ -/***********************************************************/ /*------------------------------------------------------------------------- @@ -903,7 +947,6 @@ H5HF__cache_iblock_deserialize(const void *_image, size_t len, void *_udata, const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ haddr_t heap_addr; /* Address of heap header in the file */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned u; /* Local index variable */ void * ret_value = NULL; /* Return value */ @@ -1030,8 +1073,7 @@ H5HF__cache_iblock_deserialize(const void *_image, size_t len, void *_udata, /* Sanity check */ HDassert(iblock->nchildren); /* indirect blocks w/no children should have been deleted */ - /* Compute checksum on indirect block */ - computed_chksum = H5_checksum_metadata((const uint8_t *)_image, (size_t)(image - (const uint8_t *)_image), 0); + /* checksum verification already done by verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); @@ -1039,10 +1081,6 @@ H5HF__cache_iblock_deserialize(const void *_image, size_t len, void *_udata, /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == iblock->size); - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block") - /* Check if we have any indirect block children */ if(iblock->nrows > hdr->man_dtable.max_direct_rows) { unsigned indir_rows;/* Number of indirect rows in this indirect block */ @@ -1083,8 +1121,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_iblock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HF__cache_iblock_image_len(const void *_thing, size_t *image_len) { const H5HF_indirect_t *iblock = (const H5HF_indirect_t *)_thing; /* Indirect block info */ @@ -1123,9 +1160,8 @@ H5HF__cache_iblock_image_len(const void *_thing, size_t *image_len, */ static herr_t H5HF__cache_iblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, - haddr_t addr, size_t H5_ATTR_UNUSED len, size_t H5_ATTR_UNUSED compressed_len, - haddr_t *new_addr, size_t H5_ATTR_UNUSED *new_len, - size_t H5_ATTR_UNUSED *new_compressed_len, unsigned *flags) + haddr_t addr, size_t H5_ATTR_UNUSED len, haddr_t *new_addr, + size_t H5_ATTR_UNUSED *new_len, unsigned *flags) { H5HF_hdr_t *hdr; /* Shared fractal heap information */ H5HF_indirect_t *iblock = (H5HF_indirect_t *)_thing; /* Indirect block info */ @@ -1494,13 +1530,9 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF__cache_iblock_free_icr() */ -/*********************************************************/ -/* metadata cache callback definitions for direct blocks */ -/*********************************************************/ - /*------------------------------------------------------------------------- - * Function: H5HF__cache_dblock_get_load_size() + * Function: H5HF__cache_dblock_get_initial_load_size() * * Purpose: Determine the size of the direct block on disk image, and * return it in *image_len. @@ -1514,44 +1546,158 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_dblock_get_load_size(const void *_udata, size_t *image_len) +H5HF__cache_dblock_get_initial_load_size(void *_udata, size_t *image_len) { const H5HF_dblock_cache_ud_t *udata = (const H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */ - const H5HF_parent_t *par_info; /* Pointer to parent information */ - const H5HF_hdr_t *hdr; /* Shared fractal heap information */ - size_t size; + const H5HF_parent_t *par_info; /* Pointer to parent information */ + const H5HF_hdr_t *hdr; /* Shared fractal heap information */ FUNC_ENTER_STATIC_NOERR /* Sanity checks */ HDassert(udata); HDassert(image_len); + + /* Convenience variables */ par_info = (const H5HF_parent_t *)(&(udata->par_info)); HDassert(par_info); hdr = par_info->hdr; HDassert(hdr); - HDassert(hdr->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); - HDassert(hdr->cache_info.type == H5AC_FHEAP_HDR); /* Check for I/O filters on this heap */ if(hdr->filter_len > 0) { /* Check for root direct block */ if(par_info->iblock == NULL) - size = hdr->pline_root_direct_size; + /* filtered root direct block */ + *image_len = hdr->pline_root_direct_size; else - size = par_info->iblock->filt_ents[par_info->entry].size; + /* filtered direct block */ + *image_len = par_info->iblock->filt_ents[par_info->entry].size; + } /* end if */ + else + *image_len = udata->dblock_size; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5HF__cache_dblock_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HF__cache_dblock_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +static htri_t +H5HF__cache_dblock_verify_chksum(const void *_image, size_t len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5HF_dblock_cache_ud_t *udata = (H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */ + void *read_buf = NULL; /* Pointer to buffer to read in */ + H5HF_hdr_t *hdr; /* Shared fractal heap information */ + H5HF_parent_t *par_info; /* Pointer to parent information */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + size_t chk_size; /* The size for validating checksum */ + uint8_t *chk_p; /* Pointer to the area for validating checksum */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(image); + HDassert(udata); + par_info = (H5HF_parent_t *)(&(udata->par_info)); + HDassert(par_info); + hdr = par_info->hdr; + HDassert(hdr); + + /* Reset callback context info */ + udata->decompressed = FALSE; + udata->dblk = NULL; + + /* Get out if data block is not checksummed */ + if(!(hdr->checksum_dblocks)) + HGOTO_DONE(TRUE); + + if(hdr->filter_len > 0) { + size_t nbytes; /* Number of bytes used in buffer, after applying reverse filters */ + unsigned filter_mask; /* Excluded filters for direct block */ + H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ + + /* Allocate buffer to perform I/O filtering on and copy image into + * it. Must do this as H5Z_pipeline() may re-size the buffer + * provided to it. + */ + if(NULL == (read_buf = H5MM_malloc(len))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline buffer") + + /* Set up parameters for filter pipeline */ + nbytes = len; + filter_mask = udata->filter_mask; + HDmemcpy(read_buf, image, len); + + /* Push direct block data through I/O filter pipeline */ + if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, filter_cb, &nbytes, &len, &read_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, FAIL, "output pipeline failed") + + /* Update info about direct block */ + udata->decompressed = TRUE; + len = nbytes; } /* end if */ else - size = udata->dblock_size; + read_buf = (void *)image; /* Casting away const OK - QAK */ - *image_len = size; + /* Decode checksum */ + chk_size = (size_t)(H5HF_MAN_ABS_DIRECT_OVERHEAD(hdr) - H5HF_SIZEOF_CHKSUM); + chk_p = (uint8_t *)read_buf + chk_size; - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HF__cache_dblock_get_load_size() */ + /* Metadata checksum */ + UINT32DECODE(chk_p, stored_chksum); -/*********************************************************/ -/* metadata cache callback definitions for direct blocks */ -/*********************************************************/ + chk_p -= H5HF_SIZEOF_CHKSUM; + + /* Reset checksum field, for computing the checksum */ + /* (Casting away const OK - QAK) */ + HDmemset(chk_p, 0, (size_t)H5HF_SIZEOF_CHKSUM); + + /* Compute checksum on entire direct block */ + computed_chksum = H5_checksum_metadata(read_buf, len, 0); + + /* Restore the checksum */ + UINT32ENCODE(chk_p, stored_chksum) + + /* Verify checksum */ + if(stored_chksum != computed_chksum) + HGOTO_DONE(FALSE); + + /* Save the decompressed data to be used later in deserialize callback */ + if(hdr->filter_len > 0) { + /* Sanity check */ + HDassert(udata->decompressed); + HDassert(len == udata->dblock_size); + + /* Allocate block buffer */ + if(NULL == (udata->dblk = H5FL_BLK_MALLOC(direct_block, (size_t)len))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Copy un-filtered data into block's buffer */ + HDmemcpy(udata->dblk, read_buf, len); + } /* end if */ + +done: + /* Release the read buffer */ + if(read_buf && read_buf != image) + H5MM_xfree(read_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF__cache_dblock_verify_chksum() */ /*------------------------------------------------------------------------- @@ -1582,14 +1728,15 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, H5HF_dblock_cache_ud_t *udata = (H5HF_dblock_cache_ud_t *)_udata; /* User data for callback */ H5HF_parent_t *par_info; /* Pointer to parent information */ H5HF_direct_t *dblock = NULL; /* Direct block info */ - const uint8_t *image; /* Pointer into raw data buffer */ + const uint8_t *image = _image;/* Pointer into raw data buffer */ + void *read_buf = NULL; /* Pointer to buffer to decompress */ haddr_t heap_addr; /* Address of heap header in the file */ void * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC /* Sanity checks */ - HDassert(_image); + HDassert(image); HDassert(udata); par_info = (H5HF_parent_t *)(&(udata->par_info)); HDassert(par_info); @@ -1600,7 +1747,7 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, HDassert(dirty); /* Allocate space for the fractal heap direct block */ - if(NULL == (dblock = H5FL_MALLOC(H5HF_direct_t))) + if(NULL == (dblock = H5FL_CALLOC(H5HF_direct_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") HDmemset(&dblock->cache_info, 0, sizeof(H5AC_info_t)); @@ -1614,61 +1761,62 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, /* Set block's internal information */ dblock->size = udata->dblock_size; - dblock->file_size = 0; - - /* initialize fields used in serialization */ - dblock->write_buf = NULL; - dblock->write_size = 0; - - /* Allocate block buffer */ -/* XXX: Change to using free-list factories */ - if(NULL == (dblock->blk = H5FL_BLK_MALLOC(direct_block, (size_t)dblock->size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Check for I/O filters on this heap */ if(hdr->filter_len > 0) { - H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ - size_t nbytes; /* Number of bytes used in buffer, after applying reverse filters */ - void *read_buf; /* Pointer to buffer to read in */ - size_t read_size; /* Size of filtered direct block to read */ - unsigned filter_mask; /* Excluded filters for direct block */ + /* Direct block is already decompressed in verify_chksum callback */ + if(udata->decompressed) { + /* Sanity check */ + HDassert(udata->dblk); - /* Check for root direct block */ - if(par_info->iblock == NULL) - /* Set up parameters to read filtered direct block */ - read_size = hdr->pline_root_direct_size; - else - /* Set up parameters to read filtered direct block */ - read_size = par_info->iblock->filt_ents[par_info->entry].size; - HDassert(len == read_size); + /* Take ownership of the decompressed direct block */ + dblock->blk = udata->dblk; + udata->dblk = NULL; + } /* end if */ + else { + H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ + size_t nbytes; /* Number of bytes used in buffer, after applying reverse filters */ + unsigned filter_mask; /* Excluded filters for direct block */ - /* Allocate buffer to perform I/O filtering on and copy image into - * it. Must do this as H5Z_pipeline() may re-sized the buffer - * provided to it. - */ - if(NULL == (read_buf = H5MM_malloc(read_size))) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "memory allocation failed for pipeline buffer") - HDmemcpy(read_buf, _image, len); + /* Sanity check */ + HDassert(udata->dblk == NULL); - /* Push direct block data through I/O filter pipeline */ - nbytes = read_size; - filter_mask = udata->filter_mask; - if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, filter_cb, &nbytes, &read_size, &read_buf) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, NULL, "output pipeline failed") + /* Allocate buffer to perform I/O filtering on and copy image into + * it. Must do this as H5Z_pipeline() may resize the buffer + * provided to it. + */ + if(NULL == (read_buf = H5MM_malloc(len))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "memory allocation failed for pipeline buffer") - /* Sanity check */ - HDassert(nbytes == dblock->size); + /* Copy compressed image into buffer */ + HDmemcpy(read_buf, image, len); - /* Copy un-filtered data into block's buffer */ - HDmemcpy(dblock->blk, read_buf, dblock->size); + /* Push direct block data through I/O filter pipeline */ + nbytes = len; + filter_mask = udata->filter_mask; + if(H5Z_pipeline(&(hdr->pline), H5Z_FLAG_REVERSE, &filter_mask, H5Z_ENABLE_EDC, filter_cb, &nbytes, &len, &read_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFILTER, NULL, "output pipeline failed") - /* Release the read buffer */ - H5MM_xfree(read_buf); + /* Sanity check */ + HDassert(nbytes == dblock->size); + + /* Copy un-filtered data into block's buffer */ + HDmemcpy(dblock->blk, read_buf, dblock->size); + } /* end if */ } /* end if */ else { - /* copy image to dblock->blk */ + /* Sanity checks */ + HDassert(udata->dblk == NULL); + HDassert(!udata->decompressed); + + /* Allocate block buffer */ +/* XXX: Change to using free-list factories */ + if(NULL == (dblock->blk = H5FL_BLK_MALLOC(direct_block, (size_t)dblock->size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + /* Copy image to dblock->blk */ HDassert(dblock->size == len); - HDmemcpy(dblock->blk, _image, dblock->size); + HDmemcpy(dblock->blk, image, dblock->size); } /* end else */ /* Start decoding direct block */ @@ -1703,22 +1851,12 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, /* Decode checksum on direct block, if requested */ if(hdr->checksum_dblocks) { - uint32_t stored_chksum; /* Metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ + uint32_t stored_chksum; /* Metadata checksum value */ + + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(image, stored_chksum); - - /* Reset checksum field, for computing the checksum */ - /* (Casting away const OK - QAK) */ - HDmemset((uint8_t *)image - H5HF_SIZEOF_CHKSUM, 0, (size_t)H5HF_SIZEOF_CHKSUM); - - /* Compute checksum on entire direct block */ - computed_chksum = H5_checksum_metadata(dblock->blk, dblock->size, 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap direct block") } /* end if */ /* Sanity check */ @@ -1728,6 +1866,11 @@ H5HF__cache_dblock_deserialize(const void *_image, size_t len, void *_udata, ret_value = (void *)dblock; done: + /* Release the read buffer */ + if(read_buf) + H5MM_xfree(read_buf); + + /* Cleanup on error */ if(!ret_value && dblock) if(H5HF_man_dblock_dest(dblock) < 0) HDONE_ERROR(H5E_HEAP, H5E_CANTFREE, NULL, "unable to destroy fractal heap direct block") @@ -1752,14 +1895,11 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *compressed_ptr, size_t *compressed_image_len_ptr) +H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len) { const H5HF_direct_t *dblock = (const H5HF_direct_t *)_thing; /* Direct block info */ - const H5HF_indirect_t *par_iblock; /* Parent iblock */ const H5HF_hdr_t *hdr; /* Shared fractal heap information */ - hbool_t compressed; size_t size; - size_t compressed_size; FUNC_ENTER_STATIC_NOERR @@ -1768,19 +1908,15 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com HDassert(dblock->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert(dblock->cache_info.type == H5AC_FHEAP_DBLOCK); HDassert(image_len); - HDassert(compressed_ptr); - HDassert(compressed_image_len_ptr); /* Set up convenience variables */ hdr = dblock->hdr; - par_iblock = dblock->parent; + HDassert(hdr); /* Check for I/O filters on this heap */ if(hdr->filter_len > 0) { - - /* Filters are enabled, so set compressed to TRUE, and set - * size equal to the uncompressed size of the direct block. - * If the data is available, set compressed_size to the compressed + /* + * If the data is available, set to the compressed * size of the direct block -- otherwise set it equal to the * uncompressed size. * @@ -1790,7 +1926,7 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com * case, both dblock->file_size and the size stored in the * parent (either the header or the parent iblock) will all * be zero. In this case, return the uncompressed size - * stored in dblock->size as the compressed size. + * stored in dblock->size as the size. * * Second, the block may have just been serialized, in which * case, dblock->file_size should be zero, and the correct @@ -1804,42 +1940,26 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com * and thus looking up the parent will likely return incorrect * data. */ - size = dblock->size; - compressed = TRUE; - compressed_size = dblock->size; /* will overwrite if compressed - * size is available. - */ - if(dblock->file_size != 0) - compressed_size = dblock->file_size; + size = dblock->file_size; else { - if(par_iblock) { - unsigned par_entry; /* Entry in parent indirect block */ + const H5HF_indirect_t *par_iblock = dblock->parent; /* Parent iblock */ - par_entry = dblock->par_entry; - compressed_size = par_iblock->filt_ents[par_entry].size; - - } /* end if */ - else { - compressed_size = hdr->pline_root_direct_size; - } - - if(compressed_size == 0) - compressed_size = dblock->size; + if(par_iblock) + size = par_iblock->filt_ents[dblock->par_entry].size; + else + size = hdr->pline_root_direct_size; + if(size == 0) + size = dblock->size; } /* end else */ } /* end if */ - else { + else size = dblock->size; - compressed = FALSE; - compressed_size = 0; /* a convenient, invalid value */ - } + /* Set the image size */ HDassert(size > 0); - - *image_len = size; - *compressed_ptr = compressed; - *compressed_image_len_ptr = compressed_size; + *image_len = size; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HF__cache_dblock_image_len() */ @@ -1851,7 +1971,7 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com * Purpose: In principle, the purpose of this function is to determine * the size and location of the disk image of the target direct * block. In this case, the uncompressed size of the block is - * fixed, but sined the H5C__CLASS_COMPRESSED_FLAG is set, + * fixed, but since the direct block could be compressed, * we may need to compute and report the compressed size. * * This is a bit sticky in the case of a direct block when I/O @@ -1863,8 +1983,8 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com * * To complicate matters further, the direct block may have been * initially allocated in temporary (AKA imaginary) file space. - * In this case, we must relocate the direct block's on disk - * image to real file space regardless of whether it has changed + * In this case, we must relocate the direct block's on-disk + * image to "real" file space regardless of whether it has changed * size. * * One simplifying factor is the direct block's "blk" field, @@ -1909,8 +2029,7 @@ H5HF__cache_dblock_image_len(const void *_thing, size_t *image_len, hbool_t *com */ static herr_t H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr, - size_t H5_ATTR_UNUSED *new_len, size_t *new_compressed_len, unsigned *flags) + haddr_t addr, size_t len, haddr_t *new_addr, size_t *new_len, unsigned *flags) { hbool_t at_tmp_addr; /* Flag to indicate direct block is */ /* at temporary address */ @@ -1936,18 +2055,14 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, HDassert(dblock->write_size == 0); HDassert(dblock->cache_info.size == len); HDassert(H5F_addr_defined(addr)); - HDassert(len == dblock->size); HDassert(new_addr); - HDassert(new_compressed_len); + HDassert(new_len); HDassert(flags); /* Set up local variables */ hdr = dblock->hdr; dblock_addr = addr; /* will update dblock_addr if we move the block */ - /* dblock->size must match dblock->cache_info.size */ - HDassert(dblock->cache_info.size == dblock->size); - /* Set the shared heap header's file context for this operation */ hdr->f = (H5F_t *)f; @@ -2035,9 +2150,10 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, /* Allocate buffer to perform I/O filtering on */ write_size = dblock->size; - if(NULL == (write_buf = H5MM_malloc(write_size))) HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline buffer") + + /* Copy the direct block's image into the buffer to compress */ HDmemcpy(write_buf, dblock->blk, write_size); /* Push direct block data through I/O filter pipeline */ @@ -2074,7 +2190,7 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * size matches the heap's last record. This value will * likely change shortly. */ - HDassert(compressed_len == hdr->pline_root_direct_size); + HDassert(len == hdr->pline_root_direct_size); /* Check if we need to re-size the block on disk */ if(hdr->pline_root_direct_size != write_size || at_tmp_addr) { @@ -2083,26 +2199,20 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * * (temp. file space does not need to be freed) */ - if(!at_tmp_addr) { + if(!at_tmp_addr) /* Release direct block's current disk space */ if(H5MF_xfree(f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, addr, (hsize_t)hdr->pline_root_direct_size) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to free fractal heap direct block") - } /* end if */ /* Allocate space for the compressed direct block */ if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") - /* Let the metadata cache know, if the block moved */ - if(!H5F_addr_eq(hdr->man_dtable.table_addr, dblock_addr)) - if(H5AC_move_entry((H5F_t *)f, H5AC_FHEAP_DBLOCK, hdr->man_dtable.table_addr, dblock_addr, dxpl_id) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTMOVE, FAIL, "unable to move direct block") - /* Update information about compressed direct block's * location & size */ HDassert(hdr->man_dtable.table_addr == addr); - HDassert(hdr->pline_root_direct_size == compressed_len); + HDassert(hdr->pline_root_direct_size == len); hdr->man_dtable.table_addr = dblock_addr; hdr->pline_root_direct_size = write_size; @@ -2132,7 +2242,7 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * size matches the heap's last record. This value will * likely change shortly. */ - HDassert(compressed_len == par_iblock->filt_ents[par_entry].size); + HDassert(len == par_iblock->filt_ents[par_entry].size); /* Check if we need to re-size the block on disk */ if(par_iblock->filt_ents[par_entry].size != write_size || at_tmp_addr) { @@ -2141,26 +2251,20 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * * (temp. file space does not need to be freed) */ - if(!at_tmp_addr) { + if(!at_tmp_addr) /* Release direct block's current disk space */ if(H5MF_xfree(f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, addr, (hsize_t)par_iblock->filt_ents[par_entry].size) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to free fractal heap direct block") - } /* end if */ /* Allocate space for the compressed direct block */ if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") - /* Let the metadata cache know, if the block moved */ - if(!H5F_addr_eq(par_iblock->ents[par_entry].addr, dblock_addr)) - if(H5AC_move_entry((H5F_t *)f, H5AC_FHEAP_DBLOCK, par_iblock->ents[par_entry].addr, dblock_addr, dxpl_id) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTMOVE, FAIL, "unable to move direct block") - /* Update information about compressed direct block's * location & size */ HDassert(par_iblock->ents[par_entry].addr == addr); - HDassert(par_iblock->filt_ents[par_entry].size == compressed_len); + HDassert(par_iblock->filt_ents[par_entry].size == len); par_iblock->ents[par_entry].addr = dblock_addr; par_iblock->filt_ents[par_entry].size = write_size; @@ -2192,22 +2296,16 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, * to 'normal' file space */ if(at_tmp_addr) { + /* Allocate 'normal' space for the direct block */ + if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") + /* Check for root direct block */ if(NULL == dblock->parent) { - /* Sanity check */ + /* Sanity checks */ HDassert(H5F_addr_eq(hdr->man_dtable.table_addr, addr)); - - /* Allocate 'normal' space for the direct block */ - if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") - - /* Sanity check */ HDassert(!H5F_addr_eq(hdr->man_dtable.table_addr, dblock_addr)); - /* Let the metadata cache know the block moved */ - if(H5AC_move_entry((H5F_t *)f, H5AC_FHEAP_DBLOCK, hdr->man_dtable.table_addr, dblock_addr, dxpl_id) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTMOVE, FAIL, "unable to move direct block") - /* Update information about direct block's location */ hdr->man_dtable.table_addr = dblock_addr; @@ -2216,22 +2314,12 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, HGOTO_ERROR(H5E_HEAP, H5E_CANTDIRTY, FAIL, "can't mark heap header as dirty") } /* end if */ else { /* the direct block's parent is an indirect block */ - /* Sanity check */ + /* Sanity checks */ HDassert(par_iblock); HDassert(par_iblock->ents); HDassert(H5F_addr_eq(par_iblock->ents[par_entry].addr, addr)); - - /* Allocate 'normal' space for the direct block */ - if(HADDR_UNDEF == (dblock_addr = H5MF_alloc((H5F_t *)f, H5FD_MEM_FHEAP_DBLOCK, dxpl_id, (hsize_t)write_size))) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "file allocation failed for fractal heap direct block") - - /* Sanity check */ HDassert(!H5F_addr_eq(par_iblock->ents[par_entry].addr, dblock_addr)); - /* Let the metadata cache know the block moved */ - if(H5AC_move_entry((H5F_t *)f, H5AC_FHEAP_DBLOCK, par_iblock->ents[par_entry].addr, dblock_addr, dxpl_id) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTMOVE, FAIL, "unable to move direct block") - /* Update information about direct block's location */ par_iblock->ents[par_entry].addr = dblock_addr; @@ -2261,9 +2349,9 @@ H5HF__cache_dblock_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *_thing, *new_addr = dblock_addr; } /* end if */ - if((hdr->filter_len > 0) && (compressed_len != write_size)) { - dblock_flags |= H5C__SERIALIZE_COMPRESSED_FLAG; - *new_compressed_len = write_size; + if((hdr->filter_len > 0) && (len != write_size)) { + dblock_flags |= H5C__SERIALIZE_RESIZED_FLAG; + *new_len = write_size; } /* end if */ *flags = dblock_flags; diff --git a/src/H5HFpkg.h b/src/H5HFpkg.h index 6253160..6abae65 100644 --- a/src/H5HFpkg.h +++ b/src/H5HFpkg.h @@ -544,6 +544,14 @@ typedef struct H5HF_dblock_cache_ud_t { * calls to it. */ unsigned filter_mask; /* Excluded filters for direct block */ + uint8_t *dblk; /* Pointer to the buffer containing the decompressed + * direct block data obtained in verify_chksum callback. + * It will be used later in deserialize callback. + */ + htri_t decompressed; /* Indicate that the direct block has been + * decompressed in verify_chksum callback. + * It will be used later in deserialize callback. + */ } H5HF_dblock_cache_ud_t; diff --git a/src/H5HGcache.c b/src/H5HGcache.c index c1eb8d9..6bc9860 100644 --- a/src/H5HGcache.c +++ b/src/H5HGcache.c @@ -62,11 +62,12 @@ /********************/ /* Metadata cache callbacks */ -static herr_t H5HG__cache_heap_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HG__cache_heap_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5HG__cache_heap_get_final_load_size(const void *_image, + size_t image_len, void *udata, size_t *actual_len); static void *H5HG__cache_heap_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HG__cache_heap_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5HG__cache_heap_image_len(const void *thing, size_t *image_len); static herr_t H5HG__cache_heap_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HG__cache_heap_free_icr(void *thing); @@ -82,14 +83,15 @@ const H5AC_class_t H5AC_GHEAP[1] = {{ "global heap", /* Metadata client name (for debugging) */ H5FD_MEM_GHEAP, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5HG__cache_heap_get_load_size, /* 'get_load_size' callback */ + H5HG__cache_heap_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5HG__cache_heap_get_final_load_size, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5HG__cache_heap_deserialize, /* 'deserialize' callback */ H5HG__cache_heap_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5HG__cache_heap_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5HG__cache_heap_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -106,13 +108,13 @@ const H5AC_class_t H5AC_GHEAP[1] = {{ /*------------------------------------------------------------------------- - * Function: H5HG__cache_heap_get_load_size() + * Function: H5HG__cache_heap_get_initial_load_size() * * Purpose: Return the initial speculative read size to the metadata * cache. This size will be used in the initial attempt to read * the global heap. If this read is too small, the cache will * try again with the correct value obtained from - * H5HG__cache_heap_image_len(). + * H5HG__cache_get_final_load_size(). * * Return: Success: SUCCEED * Failure: FAIL @@ -123,16 +125,74 @@ const H5AC_class_t H5AC_GHEAP[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5HG__cache_heap_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len) +H5HG__cache_heap_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_STATIC_NOERR + /* Sanity check */ HDassert(image_len); + /* Set the image length size */ *image_len = (size_t)H5HG_MINSIZE; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HG__cache_heap_get_load_size() */ +} /* end H5HG__cache_heap_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HG__cache_heap_get_initial_load_size() + * + * Purpose: Return the final read size for a speculatively ready heap to + * the metadata cache. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * November 18, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HG__cache_heap_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5F_t *f = (H5F_t *)_udata; /* File pointer -- obtained from user data */ + size_t heap_size = 0; /* Total size of collection */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(image); + HDassert(f); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Magic number */ + if(HDmemcmp(image, H5HG_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad global heap collection signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(H5HG_VERSION != *image++) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in global heap") + + /* Reserved */ + image += 3; + + /* Size */ + H5F_DECODE_LENGTH(f, image, heap_size); + HDassert(heap_size >= H5HG_MINSIZE); + HDassert(image_len == H5HG_MINSIZE); + + /* Set the final size for the cache image */ + *actual_len = heap_size; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HG__cache_heap_get_final_load_size() */ /*------------------------------------------------------------------------- @@ -142,10 +202,6 @@ H5HG__cache_heap_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_ * heap, deserialize it, load its contents into a newly allocated * instance of H5HG_heap_t, and return a pointer to the new instance. * - * Note that this heap client uses speculative reads. If the supplied - * buffer is too small, we simply make note of the correct size, and - * wait for the metadata cache to try again. - * * Return: Success: Pointer to in core representation * Failure: NULL * @@ -161,6 +217,8 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, H5F_t *f = (H5F_t *)_udata; /* File pointer -- obtained from user data */ H5HG_heap_t *heap = NULL; /* New global heap */ uint8_t *image; /* Pointer to image to decode */ + size_t max_idx = 0; /* Maximum heap object index seen */ + size_t nalloc; /* Number of objects allocated */ void *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -201,109 +259,98 @@ H5HG__cache_heap_deserialize(const void *_image, size_t len, void *_udata, HDassert((len == H5HG_MINSIZE) /* first try */ || ((len == heap->size) && (len > H5HG_MINSIZE))); /* second try */ - if(len == heap->size) { /* proceed with the deserialize */ - size_t max_idx = 0; - size_t nalloc; - - /* Decode each object */ - image = heap->chunk + H5HG_SIZEOF_HDR(f); - nalloc = H5HG_NOBJS(f, heap->size); - - /* Calloc the obj array because the file format spec makes no guarantee - * about the order of the objects, and unused slots must be set to zero. - */ - if(NULL == (heap->obj = H5FL_SEQ_CALLOC(H5HG_obj_t, nalloc))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - heap->nalloc = nalloc; - - while(image < (heap->chunk + heap->size)) { - if((image + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) { - /* - * The last bit of space is too tiny for an object header, so - * we assume that it's free space. - */ - HDassert(NULL == heap->obj[0].begin); - heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - image); - heap->obj[0].begin = image; - image += heap->obj[0].size; + /* Decode each object */ + image = heap->chunk + H5HG_SIZEOF_HDR(f); + nalloc = H5HG_NOBJS(f, heap->size); + + /* Calloc the obj array because the file format spec makes no guarantee + * about the order of the objects, and unused slots must be set to zero. + */ + if(NULL == (heap->obj = H5FL_SEQ_CALLOC(H5HG_obj_t, nalloc))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + heap->nalloc = nalloc; + + while(image < (heap->chunk + heap->size)) { + if((image + H5HG_SIZEOF_OBJHDR(f)) > (heap->chunk + heap->size)) { + /* + * The last bit of space is too tiny for an object header, so + * we assume that it's free space. + */ + HDassert(NULL == heap->obj[0].begin); + heap->obj[0].size = (size_t)(((const uint8_t *)heap->chunk + heap->size) - image); + heap->obj[0].begin = image; + image += heap->obj[0].size; + } /* end if */ + else { + size_t need; + unsigned idx; + uint8_t *begin = image; + + UINT16DECODE(image, idx); + + /* Check if we need more room to store heap objects */ + if(idx >= heap->nalloc) { + size_t new_alloc; /* New allocation number */ + H5HG_obj_t *new_obj; /* New array of object descriptions */ + + /* Determine the new number of objects to index */ + new_alloc = MAX(heap->nalloc * 2, (idx + 1)); + HDassert(idx < new_alloc); + + /* Reallocate array of objects */ + if(NULL == (new_obj = H5FL_SEQ_REALLOC(H5HG_obj_t, heap->obj, new_alloc))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + + /* Clear newly allocated space */ + HDmemset(&new_obj[heap->nalloc], 0, (new_alloc - heap->nalloc) * sizeof(heap->obj[0])); + + /* Update heap information */ + heap->nalloc = new_alloc; + heap->obj = new_obj; + HDassert(heap->nalloc > heap->nused); + } /* end if */ + + UINT16DECODE(image, heap->obj[idx].nrefs); + image += 4; /*reserved*/ + H5F_DECODE_LENGTH(f, image, heap->obj[idx].size); + heap->obj[idx].begin = begin; + + /* + * The total storage size includes the size of the object + * header and is zero padded so the next object header is + * properly aligned. The entire obj array was calloc'ed, + * so no need to zero the space here. The last bit of space + * is the free space object whose size is never padded and + * already includes the object header. + */ + if(idx > 0) { + need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(heap->obj[idx].size); + if(idx > max_idx) + max_idx = idx; } /* end if */ - else { - size_t need; - unsigned idx; - uint8_t *begin = image; - - UINT16DECODE(image, idx); - - /* Check if we need more room to store heap objects */ - if(idx >= heap->nalloc) { - size_t new_alloc; /* New allocation number */ - H5HG_obj_t *new_obj; /* New array of object descriptions */ - - /* Determine the new number of objects to index */ - new_alloc = MAX(heap->nalloc * 2, (idx + 1)); - HDassert(idx < new_alloc); - - /* Reallocate array of objects */ - if(NULL == (new_obj = H5FL_SEQ_REALLOC(H5HG_obj_t, heap->obj, new_alloc))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* Clear newly allocated space */ - HDmemset(&new_obj[heap->nalloc], 0, (new_alloc - heap->nalloc) * sizeof(heap->obj[0])); - - /* Update heap information */ - heap->nalloc = new_alloc; - heap->obj = new_obj; - HDassert(heap->nalloc > heap->nused); - } /* end if */ - - UINT16DECODE(image, heap->obj[idx].nrefs); - image += 4; /*reserved*/ - H5F_DECODE_LENGTH(f, image, heap->obj[idx].size); - heap->obj[idx].begin = begin; - - /* - * The total storage size includes the size of the object - * header and is zero padded so the next object header is - * properly aligned. The entire obj array was calloc'ed, - * so no need to zero the space here. The last bit of space - * is the free space object whose size is never padded and - * already includes the object header. - */ - if(idx > 0) { - need = H5HG_SIZEOF_OBJHDR(f) + H5HG_ALIGN(heap->obj[idx].size); - if(idx > max_idx) - max_idx = idx; - } /* end if */ - else - need = heap->obj[idx].size; - - image = begin + need; - } /* end else */ - } /* end while */ - - HDassert(image == heap->chunk + heap->size); - HDassert(H5HG_ISALIGNED(heap->obj[0].size)); - - /* Set the next index value to use */ - if(max_idx > 0) - heap->nused = max_idx + 1; - else - heap->nused = 1; - - HDassert(max_idx < heap->nused); - - /* Add the new heap to the CWFS list for the file */ - if(H5F_cwfs_add(f, heap) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "unable to add global heap collection to file's CWFS") - } /* end if ( len == heap->size ) */ + else + need = heap->obj[idx].size; + + image = begin + need; + } /* end else */ + } /* end while */ + + /* Sanity checks */ + HDassert(image == heap->chunk + heap->size); + HDassert(H5HG_ISALIGNED(heap->obj[0].size)); + + /* Set the next index value to use */ + if(max_idx > 0) + heap->nused = max_idx + 1; else - /* if len is less than heap size, then the initial speculative - * read was too small. In this case we return without reporting - * failure. H5C_load_entry() will call H5HG__cache_heap_image_len() - * to get the actual read size, and then repeat the read with the - * correct size, and call this function a second time. - */ - HDassert(len < heap->size); + heap->nused = 1; + + /* Sanity check */ + HDassert(max_idx < heap->nused); + + /* Add the new heap to the CWFS list for the file */ + if(H5F_cwfs_add(f, heap) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "unable to add global heap collection to file's CWFS") ret_value = heap; @@ -331,8 +378,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HG__cache_heap_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HG__cache_heap_image_len(const void *_thing, size_t *image_len) { const H5HG_heap_t *heap = (const H5HG_heap_t *)_thing; @@ -350,10 +396,6 @@ H5HG__cache_heap_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HG__cache_heap_image_len() */ -/**************************************/ -/* no H5HG_cache_heap_pre_serialize() */ -/**************************************/ - /*------------------------------------------------------------------------- * Function: H5HG__cache_heap_serialize @@ -393,10 +435,6 @@ H5HG__cache_heap_serialize(const H5F_t *f, void *image, size_t len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HG__cache_heap_serialize() */ -/****************************************/ -/* no H5HG_cache_heap_notify() function */ -/****************************************/ - /*------------------------------------------------------------------------- * Function: H5HG__cache_heap_free_icr @@ -341,12 +341,10 @@ H5HL_protect(H5F_t *f, hid_t dxpl_id, haddr_t addr, unsigned flags)) HDassert((flags & (unsigned)(~H5AC__READ_ONLY_FLAG)) == 0); /* Construct the user data for protect callback */ - prfx_udata.made_attempt = FALSE; prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f); prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f); prfx_udata.prfx_addr = addr; prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f); - prfx_udata.loaded = FALSE; /* Protect the local heap prefix */ if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, flags))) @@ -359,25 +357,14 @@ H5HL_protect(H5F_t *f, hid_t dxpl_id, haddr_t addr, unsigned flags)) /* (for re-entrant situation) */ if(heap->prots == 0) { /* Check if heap has separate data block */ - if(heap->single_cache_obj) { + if(heap->single_cache_obj) /* Set the flag for pinning the prefix when unprotecting it */ prfx_cache_flags |= H5AC__PIN_ENTRY_FLAG; - } /* end if */ else { - H5HL_cache_dblk_ud_t dblk_udata; /* User data for protecting local heap data block */ - - /* Construct the user data for protect callback */ - dblk_udata.heap = heap; - dblk_udata.loaded = FALSE; - /* Protect the local heap data block */ - if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, &dblk_udata, flags))) + if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, heap, flags))) H5E_THROW(H5E_CANTPROTECT, "unable to load heap data block"); - /* Pin the prefix, if the data block was loaded from file */ - if(dblk_udata.loaded) - prfx_cache_flags |= H5AC__PIN_ENTRY_FLAG; - /* Set the flag for pinning the data block when unprotecting it */ dblk_cache_flags |= H5AC__PIN_ENTRY_FLAG; } /* end if */ @@ -931,12 +918,10 @@ H5HL_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr)) HDassert(H5F_addr_defined(addr)); /* Construct the user data for protect callback */ - prfx_udata.made_attempt = FALSE; prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f); prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f); prfx_udata.prfx_addr = addr; prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f); - prfx_udata.loaded = FALSE; /* Protect the local heap prefix */ if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, H5AC__NO_FLAGS_SET))) @@ -946,24 +931,11 @@ H5HL_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr)) heap = prfx->heap; /* Check if heap has separate data block */ - if(!heap->single_cache_obj) { - H5HL_cache_dblk_ud_t dblk_udata; /* User data for protecting local heap data block */ - - /* Construct the user data for protect callback */ - dblk_udata.heap = heap; - dblk_udata.loaded = FALSE; - + if(!heap->single_cache_obj) /* Protect the local heap data block */ - if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, &dblk_udata, H5AC__NO_FLAGS_SET))) + if(NULL == (dblk = (H5HL_dblk_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_DBLK, heap->dblk_addr, heap, H5AC__NO_FLAGS_SET))) H5E_THROW(H5E_CANTPROTECT, "unable to load heap data block"); - /* Pin the prefix, if the data block was loaded from file */ - if(dblk_udata.loaded) { - if(FAIL == H5AC_pin_protected_entry(prfx)) - H5E_THROW(H5E_CANTPIN, "unable to pin local heap prefix"); - } /* end if */ - } /* end if */ - /* Set the flags for releasing the prefix and data block */ cache_flags |= H5AC__DIRTIED_FLAG | H5AC__DELETED_FLAG | H5AC__FREE_FILE_SPACE_FLAG; @@ -1005,12 +977,10 @@ H5HL_get_size(H5F_t *f, hid_t dxpl_id, haddr_t addr, size_t *size)) HDassert(size); /* Construct the user data for protect callback */ - prfx_udata.made_attempt = FALSE; prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f); prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f); prfx_udata.prfx_addr = addr; prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f); - prfx_udata.loaded = FALSE; /* Protect the local heap prefix */ if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, H5AC__READ_ONLY_FLAG))) @@ -1056,12 +1026,10 @@ H5HL_heapsize(H5F_t *f, hid_t dxpl_id, haddr_t addr, hsize_t *heap_size)) HDassert(heap_size); /* Construct the user data for protect callback */ - prfx_udata.made_attempt = FALSE; prfx_udata.sizeof_size = H5F_SIZEOF_SIZE(f); prfx_udata.sizeof_addr = H5F_SIZEOF_ADDR(f); prfx_udata.prfx_addr = addr; prfx_udata.sizeof_prfx = H5HL_SIZEOF_HDR(f); - prfx_udata.loaded = FALSE; /* Protect the local heap prefix */ if(NULL == (prfx = (H5HL_prfx_t *)H5AC_protect(f, dxpl_id, H5AC_LHEAP_PRFX, addr, &prfx_udata, H5AC__READ_ONLY_FLAG))) diff --git a/src/H5HLcache.c b/src/H5HLcache.c index 95ab65d..c53292a 100644 --- a/src/H5HLcache.c +++ b/src/H5HLcache.c @@ -71,26 +71,24 @@ /* Metadata cache callbacks */ /* Local heap prefix */ -static herr_t H5HL__cache_prefix_get_load_size(const void *udata, size_t *image_len); +static herr_t H5HL__cache_prefix_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5HL__cache_prefix_get_final_load_size(const void *_image, + size_t image_len, void *udata, size_t *actual_len); static void *H5HL__cache_prefix_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HL__cache_prefix_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5HL__cache_prefix_image_len(const void *thing, size_t *image_len); static herr_t H5HL__cache_prefix_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5HL__cache_prefix_free_icr(void *thing); /* Local heap data block */ -static herr_t H5HL__cache_datablock_get_load_size(const void *udata, - size_t *image_len); +static herr_t H5HL__cache_datablock_get_initial_load_size(void *udata, size_t *image_len); static void *H5HL__cache_datablock_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5HL__cache_datablock_image_len(const void *thing, - size_t *image_len, hbool_t *compressed_ptr, - size_t *compressed_image_len_ptr); +static herr_t H5HL__cache_datablock_image_len(const void *thing, size_t *image_len); static herr_t H5HL__cache_datablock_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5HL__cache_datablock_notify(H5C_notify_action_t action, void *_thing); static herr_t H5HL__cache_datablock_free_icr(void *thing); /* Free list de/serialization */ @@ -107,14 +105,15 @@ const H5AC_class_t H5AC_LHEAP_PRFX[1] = {{ "local heap prefix", /* Metadata client name (for debugging) */ H5FD_MEM_LHEAP, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5HL__cache_prefix_get_load_size, /* 'get_load_size' callback */ + H5HL__cache_prefix_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5HL__cache_prefix_get_final_load_size, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5HL__cache_prefix_deserialize, /* 'deserialize' callback */ H5HL__cache_prefix_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5HL__cache_prefix_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5HL__cache_prefix_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -123,14 +122,15 @@ const H5AC_class_t H5AC_LHEAP_DBLK[1] = {{ "local heap datablock", /* Metadata client name (for debugging) */ H5FD_MEM_LHEAP, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5HL__cache_datablock_get_load_size,/* 'get_load_size' callback */ + H5HL__cache_datablock_get_initial_load_size,/* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + NULL, /* 'verify_chksum' callback */ H5HL__cache_datablock_deserialize, /* 'deserialize' callback */ H5HL__cache_datablock_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5HL__cache_datablock_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5HL__cache_datablock_notify, /* 'notify' callback */ H5HL__cache_datablock_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -260,18 +260,11 @@ H5HL__fl_serialize(const H5HL_t *heap) /*------------------------------------------------------------------------- - * Function: H5HL__cache_prefix_get_load_size() + * Function: H5HL__cache_prefix_get_initial_load_size() * - * Purpose: Return the size of the buffer the metadata cache should + * Purpose: Return the initial size of the buffer the metadata cache should * load from file and pass to the deserialize routine. * - * The version 2 metadata cache callbacks included a test to - * ensure that the read did not pass the end of file, but this - * functionality has been moved to H5C_load_entry(). Thus - * all this function does is set *image_len equal to - * H5HL_SPEC_READ_SIZE, leaving it to the metadata cache to - * reduce the size of the read if appropriate. - * * Return: Success: SUCCEED * Failure: FAIL * @@ -281,16 +274,91 @@ H5HL__fl_serialize(const H5HL_t *heap) *------------------------------------------------------------------------- */ static herr_t -H5HL__cache_prefix_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len) +H5HL__cache_prefix_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { FUNC_ENTER_STATIC_NOERR + /* Sanity check */ HDassert(image_len); + /* Set the image length size */ *image_len = H5HL_SPEC_READ_SIZE; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HL__cache_prefix_get_load_size() */ +} /* end H5HL__cache_prefix_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5HL__cache_prefix_get_final_load_size() + * + * Purpose: Return the final size of the buffer the metadata cache should + * load from file and pass to the deserialize routine. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * November 18, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HL__cache_prefix_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5HL_cache_prfx_ud_t *udata = (H5HL_cache_prfx_ud_t *)_udata; /* User data for callback */ + H5HL_t heap; /* Local heap */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity checks */ + HDassert(image); + HDassert(udata); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Check magic number */ + if(HDmemcmp(image, H5HL_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad local heap signature") + image += H5_SIZEOF_MAGIC; + + /* Version */ + if(H5HL_VERSION != *image++) + HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong version number in local heap") + + /* Reserved */ + image += 3; + + /* Store the prefix's address & length */ + heap.prfx_addr = udata->prfx_addr; /* NEED */ + heap.prfx_size = udata->sizeof_prfx; /* NEED */ + + /* Heap data size */ + H5F_DECODE_LENGTH_LEN(image, heap.dblk_size, udata->sizeof_size); /* NEED */ + + /* Free list head */ + H5F_DECODE_LENGTH_LEN(image, heap.free_block, udata->sizeof_size); + if(heap.free_block != H5HL_FREE_NULL && heap.free_block >= heap.dblk_size) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, FAIL, "bad heap free list"); + + /* Heap data address */ + H5F_addr_decode_len(udata->sizeof_addr, &image, &(heap.dblk_addr)); /* NEED */ + + /* Set the final size for the cache image */ + *actual_len = heap.prfx_size; + + /* Check if heap block exists */ + if(heap.dblk_size) + /* Check if heap data block is contiguous with header */ + if(H5F_addr_eq((heap.prfx_addr + heap.prfx_size), heap.dblk_addr)) + /* Note that the heap should be a single object in the cache */ + *actual_len += heap.dblk_size; + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HL__cache_prefix_get_final_load_size() */ /*------------------------------------------------------------------------- @@ -359,7 +427,6 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t len, void *_udata, /* Free list head */ H5F_DECODE_LENGTH_LEN(image, heap->free_block, udata->sizeof_size); - if((heap->free_block != H5HL_FREE_NULL) && (heap->free_block >= heap->dblk_size)) HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "bad heap free list") @@ -373,53 +440,29 @@ H5HL__cache_prefix_deserialize(const void *_image, size_t len, void *_udata, /* Note that the heap should be a single object in the cache */ heap->single_cache_obj = TRUE; - /* Check if the current buffer from the speculative read - * already has the heap data - */ - if(len >= (heap->prfx_size + heap->dblk_size)) { - /* Allocate space for the heap data image */ - if(NULL == (heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, heap->dblk_size))) - HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "memory allocation failed") - - /* Set image to the start of the data block. This is necessary - * because there may be a gap between the used portion of the - * prefix and the data block due to alignment constraints. */ - image = ((const uint8_t *)_image) + heap->prfx_size; - - /* Copy the heap data from the speculative read buffer */ - HDmemcpy(heap->dblk_image, image, heap->dblk_size); - - /* Build free list */ - if(H5HL__fl_deserialize(heap) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't initialize free list") - } /* end if */ - else { - /* the supplied buffer is too small -- We have already made note - * of the correct size, so simply return success. H5C_load_entry() - * will notice the size discrepency, and re-try the load. - */ - - /* Make certain that this is the first try ... */ - HDassert(!udata->made_attempt); - - /* ... and mark the udata so that we know that we have used up - * our first try. - */ - udata->made_attempt = TRUE; - } /* end else */ + /* Allocate space for the heap data image */ + if(NULL == (heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, heap->dblk_size))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "memory allocation failed") + + /* Set image to the start of the data block. This is necessary + * because there may be a gap between the used portion of the + * prefix and the data block due to alignment constraints. */ + image = ((const uint8_t *)_image) + heap->prfx_size; + + /* Copy the heap data from the speculative read buffer */ + HDmemcpy(heap->dblk_image, image, heap->dblk_size); + + /* Build free list */ + if(H5HL__fl_deserialize(heap) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't initialize free list") } /* end if */ - else { + else /* Note that the heap should _NOT_ be a single * object in the cache */ heap->single_cache_obj = FALSE; - - } /* end else */ } /* end if */ - /* Set flag to indicate prefix from loaded from file */ - udata->loaded = TRUE; - /* Set return value */ ret_value = prfx; @@ -455,8 +498,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HL__cache_prefix_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HL__cache_prefix_image_len(const void *_thing, size_t *image_len) { const H5HL_prfx_t *prfx = (const H5HL_prfx_t *)_thing; /* Pointer to local heap prefix to query */ @@ -480,10 +522,6 @@ H5HL__cache_prefix_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HL__cache_prefix_image_len() */ -/****************************************/ -/* no H5HL_cache_prefix_pre_serialize() */ -/****************************************/ - /*------------------------------------------------------------------------- * Function: H5HL__cache_prefix_serialize @@ -579,10 +617,6 @@ H5HL__cache_prefix_serialize(const H5F_t *f, void *_image, size_t len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HL__cache_prefix_serialize() */ -/******************************************/ -/* no H5HL_cache_prefix_notify() function */ -/******************************************/ - /*------------------------------------------------------------------------- * Function: H5HL__cache_prefix_free_icr @@ -630,7 +664,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5HL__cache_datablock_get_load_size() + * Function: H5HL__cache_datablock_get_initial_load_size() * * Purpose: Tell the metadata cache how large a buffer to read from * file when loading a datablock. In this case, we simply lookup @@ -645,22 +679,22 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HL__cache_datablock_get_load_size(const void *_udata, size_t *image_len) +H5HL__cache_datablock_get_initial_load_size(void *_udata, size_t *image_len) { - const H5HL_cache_dblk_ud_t *udata = (const H5HL_cache_dblk_ud_t *)_udata; /* User data for callback */ + H5HL_t *heap = (H5HL_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR /* Check arguments */ - HDassert(udata); - HDassert(udata->heap); - HDassert(udata->heap->dblk_size > 0); + HDassert(heap); + HDassert(heap->dblk_size > 0); HDassert(image_len); - *image_len = udata->heap->dblk_size; + /* Set the image length size */ + *image_len = heap->dblk_size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5HL__cache_datablock_get_load_size() */ +} /* end H5HL__cache_datablock_get_initial_load_size() */ /*------------------------------------------------------------------------- @@ -682,8 +716,8 @@ static void * H5HL__cache_datablock_deserialize(const void *image, size_t len, void *_udata, hbool_t H5_ATTR_UNUSED *dirty) { - H5HL_dblk_t *dblk = NULL; /* Local heap data block deserialized */ - H5HL_cache_dblk_ud_t *udata = (H5HL_cache_dblk_ud_t *)_udata; /* User data for callback */ + H5HL_dblk_t *dblk = NULL; /* Local heap data block deserialized */ + H5HL_t *heap = (H5HL_t *)_udata; /* User data for callback */ void *ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -691,34 +725,30 @@ H5HL__cache_datablock_deserialize(const void *image, size_t len, void *_udata, /* Check arguments */ HDassert(image); HDassert(len > 0); - HDassert(udata); - HDassert(udata->heap); - HDassert(udata->heap->dblk_size == len); - HDassert(!udata->heap->single_cache_obj); - HDassert(NULL == udata->heap->dblk); + HDassert(heap); + HDassert(heap->dblk_size == len); + HDassert(!heap->single_cache_obj); + HDassert(NULL == heap->dblk); HDassert(dirty); /* Allocate space in memory for the heap data block */ - if(NULL == (dblk = H5HL__dblk_new(udata->heap))) + if(NULL == (dblk = H5HL__dblk_new(heap))) HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "memory allocation failed"); /* Check for heap still retaining image */ - if(NULL == udata->heap->dblk_image) { + if(NULL == heap->dblk_image) { /* Allocate space for the heap data image */ - if(NULL == (udata->heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, udata->heap->dblk_size))) + if(NULL == (heap->dblk_image = H5FL_BLK_MALLOC(lheap_chunk, heap->dblk_size))) HGOTO_ERROR(H5E_HEAP, H5E_CANTALLOC, NULL, "can't allocate data block image buffer"); /* copy the datablock from the read buffer */ - HDmemcpy(udata->heap->dblk_image, image, len); + HDmemcpy(heap->dblk_image, image, len); /* Build free list */ - if(FAIL == H5HL__fl_deserialize(udata->heap)) + if(FAIL == H5HL__fl_deserialize(heap)) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't initialize free list"); } /* end if */ - /* Set flag to indicate data block from loaded from file */ - udata->loaded = TRUE; - /* Set return value */ ret_value = dblk; @@ -746,8 +776,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5HL__cache_datablock_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5HL__cache_datablock_image_len(const void *_thing, size_t *image_len) { const H5HL_dblk_t *dblk = (const H5HL_dblk_t *)_thing; /* Pointer to the local heap data block */ @@ -766,10 +795,6 @@ H5HL__cache_datablock_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HL__cache_datablock_image_len() */ -/*******************************************/ -/* no H5HL_cache_datablock_pre_serialize() */ -/*******************************************/ - /*------------------------------------------------------------------------- * Function: H5HL__cache_datablock_serialize @@ -817,9 +842,73 @@ H5HL__cache_datablock_serialize(const H5F_t *f, void *image, size_t len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HL__cache_datablock_serialize() */ -/*********************************************/ -/* no H5HL_cache_datablock_notify() function */ -/*********************************************/ + +/*------------------------------------------------------------------------- + * Function: H5HL__cache_datablock_notify + * + * Purpose: This function is used to create and destroy pinned + * relationships between datablocks and their prefix parent. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * November 19, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5HL__cache_datablock_notify(H5C_notify_action_t action, void *_thing) +{ + H5HL_dblk_t *dblk = (H5HL_dblk_t *)_thing; /* Pointer to the local heap data block */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* Sanity check */ + HDassert(dblk); + + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + /* Sanity checks */ + HDassert(dblk->heap); + HDassert(dblk->heap->prfx); + + /* Pin the heap's prefix */ + if(FAIL == H5AC_pin_protected_entry(dblk->heap->prfx)) + HGOTO_ERROR(H5E_HEAP, H5E_CANTPIN, FAIL, "unable to pin local heap prefix") + break; + + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* Sanity checks */ + HDassert(dblk->heap); + HDassert(dblk->heap->prfx); + + /* Unpin the local heap prefix */ + if(FAIL == H5AC_unpin_entry(dblk->heap->prfx)) + HGOTO_ERROR(H5E_HEAP, H5E_CANTUNPIN, FAIL, "unable to unpin local heap prefix") + break; + + default: + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "unknown action from metadata cache") + break; + } /* end switch */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HL__cache_datablock_notify() */ /*------------------------------------------------------------------------- @@ -859,3 +948,4 @@ H5HL__cache_datablock_free_icr(void *_thing) done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HL__cache_datablock_free_icr() */ + diff --git a/src/H5HLdblk.c b/src/H5HLdblk.c index 69e0334..f90562b 100644 --- a/src/H5HLdblk.c +++ b/src/H5HLdblk.c @@ -119,10 +119,9 @@ H5HL__dblk_new(H5HL_t *heap)) CATCH /* Ensure that the data block memory is deallocated on errors */ - if(!ret_value && dblk != NULL) { + if(!ret_value && dblk != NULL) /* H5FL_FREE always returns NULL so we can't check for errors */ dblk = H5FL_FREE(H5HL_dblk_t, dblk); - } END_FUNC(PKG) /* end H5HL__dblk_new() */ @@ -151,10 +150,6 @@ H5HL__dblk_dest(H5HL_dblk_t *dblk)) /* Unlink data block from heap */ dblk->heap->dblk = NULL; - /* Unpin the local heap prefix */ - if(FAIL == H5AC_unpin_entry(dblk->heap->prfx)) - H5E_THROW(H5E_CANTUNPIN, "can't unpin local heap prefix") - /* Decrement ref. count on heap data structure */ if(FAIL == H5HL__dec_rc(dblk->heap)) H5E_THROW(H5E_CANTDEC, "can't decrement heap ref. count") @@ -223,7 +218,6 @@ H5HL__dblk_realloc(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t new_heap_size)) /* Resize the heap prefix in the cache */ if(FAIL == H5AC_resize_entry(heap->prfx, (size_t)(heap->prfx_size + new_heap_size))) H5E_THROW(H5E_CANTRESIZE, "unable to resize heap in cache"); - } /* end if */ else { /* Sanity check */ @@ -233,7 +227,6 @@ H5HL__dblk_realloc(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t new_heap_size)) /* Resize the heap data block in the cache */ if(H5AC_resize_entry(heap->dblk, (size_t)new_heap_size) < 0) H5E_THROW(H5E_CANTRESIZE, "unable to resize heap (data block) in cache"); - } /* end else */ } /* end if */ else { @@ -280,3 +273,4 @@ CATCH } /* end if */ END_FUNC(PKG) /* end H5HL__dblk_realloc() */ + diff --git a/src/H5HLpkg.h b/src/H5HLpkg.h index fb8667b..7075b2a 100644 --- a/src/H5HLpkg.h +++ b/src/H5HLpkg.h @@ -131,28 +131,12 @@ struct H5HL_prfx_t { /* Callback information for loading local heap prefix from disk */ typedef struct H5HL_cache_prfx_ud_t { - /* Downwards */ - hbool_t made_attempt; /* Whether the deserialize routine */ - /* was already attempted */ size_t sizeof_size; /* Size of file sizes */ size_t sizeof_addr; /* Size of file addresses */ haddr_t prfx_addr; /* Address of prefix */ size_t sizeof_prfx; /* Size of heap prefix */ - - /* Upwards */ - hbool_t loaded; /* Whether prefix was loaded */ - /* from file */ } H5HL_cache_prfx_ud_t; -/* Callback information for loading local heap data block from disk */ -typedef struct H5HL_cache_dblk_ud_t { - /* Downwards */ - H5HL_t *heap; /* Local heap */ - - /* Upwards */ - hbool_t loaded; /* Whether data block was loaded from file */ -} H5HL_cache_dblk_ud_t; - /******************************/ /* Package Private Prototypes */ diff --git a/src/H5HLprfx.c b/src/H5HLprfx.c index 66c4dad..ed1c4db 100644 --- a/src/H5HLprfx.c +++ b/src/H5HLprfx.c @@ -118,10 +118,9 @@ H5HL__prfx_new(H5HL_t *heap)) CATCH /* Ensure that the prefix memory is deallocated on errors */ - if(!ret_value && prfx != NULL) { + if(!ret_value && prfx != NULL) /* H5FL_FREE always returns NULL so we can't check for errors */ prfx = H5FL_FREE(H5HL_prfx_t, prfx); - } END_FUNC(PKG) /* end H5HL__prfx_new() */ @@ -164,3 +163,4 @@ CATCH prfx = H5FL_FREE(H5HL_prfx_t, prfx); END_FUNC(PKG) /* end H5HL__prfx_dest() */ + @@ -1811,11 +1811,12 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags) /* Construct the user data for protect callback */ udata.made_attempt = FALSE; udata.v1_pfx_nmesgs = 0; + udata.chunk0_size = 0; + udata.oh = NULL; udata.common.f = loc->file; udata.common.dxpl_id = dxpl_id; udata.common.file_intent = file_intent; udata.common.merged_null_msgs = 0; - udata.common.mesgs_modified = FALSE; HDmemset(&cont_msg_info, 0, sizeof(cont_msg_info)); udata.common.cont_msg_info = &cont_msg_info; udata.common.addr = loc->addr; @@ -1843,10 +1844,12 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags) chk_udata.common.dxpl_id = dxpl_id; chk_udata.common.file_intent = file_intent; chk_udata.common.merged_null_msgs = udata.common.merged_null_msgs; - chk_udata.common.mesgs_modified = udata.common.mesgs_modified; chk_udata.common.cont_msg_info = &cont_msg_info; /* Read in continuation messages, until there are no more */ + /* (Note that loading chunks could increase the # of continuation + * messages if new ones are found - QAK, 19/11/2016) + */ curr_msg = 0; while(curr_msg < cont_msg_info.nmsgs) { H5O_chunk_proxy_t *chk_proxy; /* Proxy for chunk, to bring it into memory */ @@ -1879,16 +1882,12 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags) /* Pass back out some of the chunk's user data */ udata.common.merged_null_msgs = chk_udata.common.merged_null_msgs; - udata.common.mesgs_modified = chk_udata.common.mesgs_modified; } /* end if */ /* Check for incorrect # of object header messages, if we've just loaded * this object header from the file */ if(udata.made_attempt) { - /* Check for incorrect # of messages in v1 object header */ - if(oh->version == H5O_VERSION_1 && - (oh->nmesgs + udata.common.merged_null_msgs) != udata.v1_pfx_nmesgs) { /* Don't enforce the error on an incorrect # of object header messages bug * unless strict format checking is enabled. This allows for older * files, created with a version of the library that had a bug in tracking @@ -1896,79 +1895,11 @@ H5O_protect(const H5O_loc_t *loc, hid_t dxpl_id, unsigned prot_flags) * erroring out here. -QAK */ #ifdef H5_STRICT_FORMAT_CHECKS + /* Check for incorrect # of messages in v1 object header */ + if(oh->version == H5O_VERSION_1 && + (oh->nmesgs + udata.common.merged_null_msgs) != udata.v1_pfx_nmesgs) HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, NULL, "corrupt object header - incorrect # of messages") -#else /* H5_STRICT_FORMAT_CHECKS */ - /* Mark object header prefix dirty later if we don't have write access */ - /* (object header will have been marked dirty during protect, if we - * have write access -QAK) - */ - if((prot_flags & H5AC__READ_ONLY_FLAG) != 0) - oh->prefix_modified = TRUE; -#ifndef NDEBUG - else { - unsigned oh_status = 0; /* Object header entry cache status */ - - /* Check the object header's status in the metadata cache */ - if(H5AC_get_entry_status(loc->file, loc->addr, &oh_status) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, NULL, "unable to check metadata cache status for object header") - - /* Make certain that object header is not dirty */ - HDassert(!(oh_status & H5AC_ES__IS_DIRTY)); - } /* end else */ -#endif /* NDEBUG */ #endif /* H5_STRICT_FORMAT_CHECKS */ - } /* end if */ - - /* Check for any messages that were modified while being read in */ - if(udata.common.mesgs_modified && (0 == (prot_flags & H5AC__READ_ONLY_FLAG))) - oh->mesgs_modified = TRUE; - - /* Reset the field that contained chunk 0's size during speculative load */ - oh->chunk0_size = 0; - } /* end if */ - - /* Take care of loose ends for modifications made while bringing in the - * object header & chunks. - */ - if(0 == (prot_flags & H5AC__READ_ONLY_FLAG)) { - /* Check for the object header prefix being modified somehow */ - /* (usually through updating the # of object header messages) */ - if(oh->prefix_modified) { - /* Mark the header as dirty now */ - if(H5AC_mark_entry_dirty(oh) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTMARKDIRTY, NULL, "unable to mark object header as dirty") - - /* Reset flag */ - oh->prefix_modified = FALSE; - } /* end if */ - - /* Check for deferred dirty messages */ - if(oh->mesgs_modified) { - unsigned u; /* Local index variable */ - - /* Loop through all messages, marking their chunks as dirty */ - /* (slightly inefficient, since we don't know exactly which messages - * were modified when the object header & chunks were brought in - * from the file, but this only can happen once per load -QAK) - */ - for(u = 0; u < oh->nmesgs; u++) { - /* Mark each chunk with a dirty message as dirty also */ - if(oh->mesg[u].dirty) { - H5O_chunk_proxy_t *chk_proxy; /* Chunk that message is in */ - - /* Protect chunk */ - if(NULL == (chk_proxy = H5O_chunk_protect(loc->file, dxpl_id, oh, oh->mesg[u].chunkno))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTPROTECT, NULL, "unable to load object header chunk") - - /* Unprotect chunk, marking it dirty */ - if(H5O_chunk_unprotect(loc->file, dxpl_id, chk_proxy, TRUE) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, NULL, "unable to unprotect object header chunk") - } /* end if */ - } /* end for */ - - /* Reset flag */ - oh->mesgs_modified = FALSE; - } /* end if */ } /* end if */ #ifdef H5O_DEBUG @@ -3645,7 +3576,6 @@ herr_t H5O__free(H5O_t *oh) { unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_PACKAGE_NOERR diff --git a/src/H5Ocache.c b/src/H5Ocache.c index f1a128d..88a4d85 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -68,25 +68,27 @@ /********************/ /* Metadata cache callbacks */ -static herr_t H5O__cache_get_load_size(const void *udata, size_t *image_len); +static herr_t H5O__cache_get_initial_load_size(void *udata, size_t *image_len); +static herr_t H5O__cache_get_final_load_size(const void *image_ptr, size_t image_len, + void *udata, size_t *actual_len); +static htri_t H5O__cache_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5O__cache_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5O__cache_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5O__cache_image_len(const void *thing, size_t *image_len); static herr_t H5O__cache_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5O__cache_notify(H5AC_notify_action_t action, void *_thing); static herr_t H5O__cache_free_icr(void *thing); -static herr_t H5O__cache_clear(const H5F_t *f, void *thing, hbool_t about_to_destroy); -static herr_t H5O__cache_chk_get_load_size(const void *udata, size_t *image_len); +static herr_t H5O__cache_chk_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5O__cache_chk_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5O__cache_chk_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5O__cache_chk_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5O__cache_chk_image_len(const void *thing, size_t *image_len); static herr_t H5O__cache_chk_serialize(const H5F_t *f, void *image, size_t len, void *thing); +static herr_t H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing); static herr_t H5O__cache_chk_free_icr(void *thing); -static herr_t H5O__cache_chk_clear(const H5F_t *f, void *thing, hbool_t about_to_destroy); /* Chunk routines */ static herr_t H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, @@ -96,7 +98,6 @@ static herr_t H5O__chunk_serialize(const H5F_t *f, H5O_t *oh, unsigned chunkno); /* Misc. routines */ static herr_t H5O__add_cont_msg(H5O_cont_msgs_t *cont_msg_info, const H5O_cont_t *cont); -static herr_t H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata); /*********************/ @@ -109,14 +110,15 @@ const H5AC_class_t H5AC_OHDR[1] = {{ "object header", /* Metadata client name (for debugging) */ H5FD_MEM_OHDR, /* File space memory type for client */ H5AC__CLASS_SPECULATIVE_LOAD_FLAG, /* Client class behavior flags */ - H5O__cache_get_load_size, /* 'get_load_size' callback */ + H5O__cache_get_initial_load_size, /* 'get_initial_load_size' callback */ + H5O__cache_get_final_load_size, /* 'get_final_load_size' callback */ + H5O__cache_verify_chksum, /* 'verify_chksum' callback */ H5O__cache_deserialize, /* 'deserialize' callback */ H5O__cache_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5O__cache_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5O__cache_notify, /* 'notify' callback */ H5O__cache_free_icr, /* 'free_icr' callback */ - H5O__cache_clear, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -126,14 +128,15 @@ const H5AC_class_t H5AC_OHDR_CHK[1] = {{ "object header continuation chunk", /* Metadata client name (for debugging) */ H5FD_MEM_OHDR, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5O__cache_chk_get_load_size, /* 'get_load_size' callback */ + H5O__cache_chk_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5O__cache_chk_verify_chksum, /* 'verify_chksum' callback */ H5O__cache_chk_deserialize, /* 'deserialize' callback */ H5O__cache_chk_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5O__cache_chk_serialize, /* 'serialize' callback */ - NULL, /* 'notify' callback */ + H5O__cache_chk_notify, /* 'notify' callback */ H5O__cache_chk_free_icr, /* 'free_icr' callback */ - H5O__cache_chk_clear, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -156,48 +159,88 @@ H5FL_SEQ_DEFINE(H5O_cont_t); /* Local Variables */ /*******************/ + /*------------------------------------------------------------------------- - * Function: H5O_decode_prefix + * Function: H5O__cache_get_initial_load_size() * - * Purpose: To decode the object header prefix. - * The coding is extracted fromt H5O__cache_deserialize() to this routine. + * Purpose: Tell the metadata cache how much data to read from file in + * the first speculative read for the object header. * - * Return: Non-negative on success/Negative on failure + * Return: Success: SUCCEED + * Failure: FAIL * - * Programmer: Vailin Choi; Aug 2015 + * Programmer: John Mainzer + * 7/28/14 * *------------------------------------------------------------------------- */ static herr_t -H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata) +H5O__cache_get_initial_load_size(void H5_ATTR_UNUSED *_udata, size_t *image_len) { - H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - const uint8_t *p = buf; /* Pointer into buffer to decode */ - size_t prefix_size; /* Size of object header prefix */ - herr_t ret_value = SUCCEED; /* Return value */ + FUNC_ENTER_STATIC_NOERR - FUNC_ENTER_NOAPI_NOINIT + /* Check arguments */ + HDassert(image_len); + + /* Set the image length size */ + *image_len = H5O_SPEC_READ_SIZE; + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5O__cache_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5O__cache_get_final_load_size() + * + * Purpose: Tell the metadata cache the final size of an object header. + * + * Return: Success: SUCCEED + * Failure: FAIL + * + * Programmer: Quincey Koziol + * November 18, 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O__cache_get_final_load_size(const void *_image, size_t image_len, + void *_udata, size_t *actual_len) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ + H5O_t *oh = NULL; /* Object header read in */ + htri_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC /* Check arguments */ - HDassert(f); - HDassert(oh); - HDassert(buf); + HDassert(image); HDassert(udata); + HDassert(actual_len); + HDassert(*actual_len == image_len); + + /* Allocate space for the new object header data structure */ + if(NULL == (oh = H5FL_CALLOC(H5O_t))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") + + /* File-specific, non-stored information */ + oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); + oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f); /* Check for presence of magic number */ /* (indicates version 2 or later) */ - if(!HDmemcmp(p, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { + if(!HDmemcmp(image, H5O_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) { /* Magic number */ - p += H5_SIZEOF_MAGIC; + image += H5_SIZEOF_MAGIC; /* Version */ - oh->version = *p++; + oh->version = *image++; if(H5O_VERSION_2 != oh->version) HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") /* Flags */ - oh->flags = *p++; + oh->flags = *image++; if(oh->flags & ~H5O_HDR_ALL_FLAGS) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown object header status flag(s)") @@ -208,13 +251,13 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata) if(oh->flags & H5O_HDR_STORE_TIMES) { uint32_t tmp; /* Temporary value */ - UINT32DECODE(p, tmp); + UINT32DECODE(image, tmp); oh->atime = (time_t)tmp; - UINT32DECODE(p, tmp); + UINT32DECODE(image, tmp); oh->mtime = (time_t)tmp; - UINT32DECODE(p, tmp); + UINT32DECODE(image, tmp); oh->ctime = (time_t)tmp; - UINT32DECODE(p, tmp); + UINT32DECODE(image, tmp); oh->btime = (time_t)tmp; } /* end if */ else @@ -222,8 +265,8 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata) /* Attribute fields */ if(oh->flags & H5O_HDR_ATTR_STORE_PHASE_CHANGE) { - UINT16DECODE(p, oh->max_compact); - UINT16DECODE(p, oh->min_dense); + UINT16DECODE(image, oh->max_compact); + UINT16DECODE(image, oh->min_dense); if(oh->max_compact < oh->min_dense) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header attribute phase change values") } /* end if */ @@ -235,30 +278,30 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata) /* First chunk size */ switch(oh->flags & H5O_HDR_CHUNK0_SIZE) { case 0: /* 1 byte size */ - oh->chunk0_size = *p++; + udata->chunk0_size = *image++; break; case 1: /* 2 byte size */ - UINT16DECODE(p, oh->chunk0_size); + UINT16DECODE(image, udata->chunk0_size); break; case 2: /* 4 byte size */ - UINT32DECODE(p, oh->chunk0_size); + UINT32DECODE(image, udata->chunk0_size); break; case 3: /* 8 byte size */ - UINT64DECODE(p, oh->chunk0_size); + UINT64DECODE(image, udata->chunk0_size); break; default: HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad size for chunk 0") } /* end switch */ - if(oh->chunk0_size > 0 && oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) + if(udata->chunk0_size > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") } /* end if */ else { /* Version */ - oh->version = *p++; + oh->version = *image++; if(H5O_VERSION_1 != oh->version) HGOTO_ERROR(H5E_OHDR, H5E_VERSION, FAIL, "bad object header version number") @@ -266,13 +309,13 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata) oh->flags = H5O_CRT_OHDR_FLAGS_DEF; /* Reserved */ - p++; + image++; /* Number of messages */ - UINT16DECODE(p, udata->v1_pfx_nmesgs); + UINT16DECODE(image, udata->v1_pfx_nmesgs); /* Link count */ - UINT32DECODE(p, oh->nlink); + UINT32DECODE(image, oh->nlink); /* Reset unused time fields */ oh->atime = oh->mtime = oh->ctime = oh->btime = 0; @@ -282,52 +325,77 @@ H5O_decode_prefix(H5F_t *f, H5O_t *oh, const uint8_t *buf, void *_udata) oh->min_dense = 0; /* First chunk size */ - UINT32DECODE(p, oh->chunk0_size); - if((udata->v1_pfx_nmesgs > 0 && oh->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || - (udata->v1_pfx_nmesgs == 0 && oh->chunk0_size > 0)) + UINT32DECODE(image, udata->chunk0_size); + if((udata->v1_pfx_nmesgs > 0 && udata->chunk0_size < H5O_SIZEOF_MSGHDR_OH(oh)) || + (udata->v1_pfx_nmesgs == 0 && udata->chunk0_size > 0)) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "bad object header chunk size") /* Reserved, in version 1 (for 8-byte alignment padding) */ - p += 4; + image += 4; } /* end else */ /* Determine object header prefix length */ - prefix_size = (size_t)(p - buf); - HDassert((size_t)prefix_size == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + HDassert((size_t)((const uint8_t *)image - (const uint8_t *)_image) == (size_t)(H5O_SIZEOF_HDR(oh) - H5O_SIZEOF_CHKSUM_OH(oh))); + + /* Set the final size for the cache image */ + *actual_len = udata->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh); + + /* Save the object header for later use in 'deserialize' callback */ + udata->oh = oh; + oh = NULL; done: + /* Release the [possibly partially initialized] object header on errors */ + if(ret_value < 0 && oh) + if(H5O__free(oh) < 0) + HDONE_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header data") + FUNC_LEAVE_NOAPI(ret_value) -} /* H5O_decode_prefix() */ +} /* end H5O__cache_get_final_load_size() */ /*------------------------------------------------------------------------- - * Function: H5O__cache_get_load_size() + * Function: H5O__cache_verify_chksum * - * Purpose: Tell the metadata cache how much data to read from file in - * the first speculative read for the object header. Note that we do - * not have to be concerned about reading past the end of file, as the - * cache will clamp the read to avoid this if needed. + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. * - * Return: Success: SUCCEED - * Failure: FAIL + * Return: Success: TRUE/FALSE + * Failure: Negative * - * Programmer: John Mainzer - * 7/28/14 + * Programmer: Vailin Choi + * Aug 2015 * *------------------------------------------------------------------------- */ -static herr_t -H5O__cache_get_load_size(const void H5_ATTR_UNUSED *_udata, size_t *image_len) +static htri_t +H5O__cache_verify_chksum(const void *_image, size_t len, void *_udata) { + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ + htri_t ret_value = TRUE; /* Return value */ + FUNC_ENTER_STATIC_NOERR /* Check arguments */ - HDassert(image_len); + HDassert(image); + HDassert(udata); + HDassert(udata->oh); - *image_len = H5O_SPEC_READ_SIZE; + /* There is no checksum for version 1 */ + if(udata->oh->version != H5O_VERSION_1) { + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5O__cache_get_load_size() */ + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__cache_verify_chksum() */ /*------------------------------------------------------------------------- @@ -354,10 +422,9 @@ static void * H5O__cache_deserialize(const void *_image, size_t len, void *_udata, hbool_t *dirty) { - H5O_t *oh = NULL; /* Object header read in */ + H5O_t *oh; /* Object header read in */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into buffer to decode */ - size_t buf_size; /* Size of prefix+chunk #0 buffer */ void * ret_value = NULL; /* Return value */ FUNC_ENTER_STATIC @@ -366,38 +433,17 @@ H5O__cache_deserialize(const void *_image, size_t len, void *_udata, HDassert(image); HDassert(len > 0); HDassert(udata); + HDassert(udata->oh); HDassert(udata->common.f); HDassert(udata->common.cont_msg_info); HDassert(dirty); - /* Allocate space for the object header data structure */ - if(NULL == (oh = H5FL_CALLOC(H5O_t))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - - /* File-specific, non-stored information */ - oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); - oh->sizeof_addr = H5F_SIZEOF_ADDR(udata->common.f); - - /* Decode header prefix */ - if(H5O_decode_prefix(udata->common.f, oh, image, udata) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize object header prefix") - - /* Compute the size of the buffer used */ - buf_size = oh->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh); + /* Retrieve partially deserialized object header from user data */ + oh = udata->oh; - - /* Check to see if the buffer provided is large enough to contain both - * the prefix and the first chunk. If it isn't, make note of the desired - * size, but otherwise do nothing. H5C_load_entry() will notice the - * discrepency, load the correct size buffer, and retry the deserialize. - */ - if(len >= buf_size) { - /* Parse the first chunk */ - if(H5O__chunk_deserialize(oh, udata->common.addr, oh->chunk0_size, (const uint8_t *)_image, &(udata->common), dirty) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk") - } /* end if */ - else - HDassert(!udata->made_attempt); + /* Parse the first chunk */ + if(H5O__chunk_deserialize(oh, udata->common.addr, udata->chunk0_size, (const uint8_t *)_image, &(udata->common), dirty) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't deserialize first object header chunk") /* Note that we've loaded the object header from the file */ udata->made_attempt = TRUE; @@ -431,8 +477,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5O__cache_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5O__cache_image_len(const void *_thing, size_t *image_len) { const H5O_t *oh = (const H5O_t *)_thing; /* Object header to query */ @@ -445,18 +490,11 @@ H5O__cache_image_len(const void *_thing, size_t *image_len, HDassert(image_len); /* Report the object header's prefix+first chunk length */ - if(oh->chunk0_size) - *image_len = (size_t)H5O_SIZEOF_HDR(oh) + oh->chunk0_size; - else - *image_len = oh->chunk[0].size; + *image_len = oh->chunk[0].size; FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5O__cache_image_len() */ -/********************************/ -/* no H5O_cache_pre_serialize() */ -/********************************/ - /*------------------------------------------------------------------------- * Function: H5O__cache_serialize @@ -600,9 +638,72 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__cache_serialize() */ -/**********************************/ -/* no H5O_cache_notify() function */ -/**********************************/ + +/*------------------------------------------------------------------------- + * Function: H5O__cache_notify + * + * Purpose: Handle cache action notifications + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * Jul 23 2016 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O__cache_notify(H5AC_notify_action_t action, void *_thing) +{ + H5O_t *oh = (H5O_t *)_thing; + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_STATIC + + /* + * Check arguments. + */ + HDassert(oh); + + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + { + unsigned u; /* Local index variable */ + + /* Mark messages stored with the object header (i.e. messages in chunk 0) as clean */ + for(u = 0; u < oh->nmesgs; u++) + if(oh->mesg[u].chunkno == 0) + oh->mesg[u].dirty = FALSE; +#ifndef NDEBUG + /* Reset the number of messages dirtied by decoding */ + oh->ndecode_dirtied = 0; +#endif /* NDEBUG */ + } + break; + + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* do nothing */ + break; + + default: +#ifdef NDEBUG + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__cache_notify() */ /*------------------------------------------------------------------------- @@ -645,112 +746,79 @@ done: /*------------------------------------------------------------------------- - * Function: H5O__cache_clear - * - * Purpose: Clear all dirty bits associated with this cache entry. + * Function: H5O__cache_chk_get_initial_load_size() * - * This is ncessary as the object header cache client maintains - * its own dirty bits on individual messages. These dirty bits - * used to be cleared by the old V2 metadata cache flush callback, - * but now the metadata cache must clear them explicitly, as - * the serialize callback does not imply that the data has been - * written to disk. - * - * This callback is also necessary for the parallel case. + * Purpose: Tell the metadata cache how large the on disk image of the + * chunk proxy is, so it can load the image into a buffer for the + * deserialize call. * * Return: Success: SUCCEED * Failure: FAIL * * Programmer: John Mainzer - * 9/22/14 + * 7/28/14 * *------------------------------------------------------------------------- */ static herr_t -#ifdef H5_HAVE_PARALLEL -H5O__cache_clear(const H5F_t *f, void *_thing, hbool_t H5_ATTR_UNUSED about_to_destroy) -#else -H5O__cache_clear(const H5F_t H5_ATTR_UNUSED *f, void *_thing, hbool_t H5_ATTR_UNUSED about_to_destroy) -#endif /* H5_HAVE_PARALLEL */ -{ - H5O_t *oh = (H5O_t *)_thing; /* Object header to reset */ - unsigned u; /* Local index variable */ - herr_t ret_value = SUCCEED; /* Return value */ +H5O__cache_chk_get_initial_load_size(void *_udata, size_t *image_len) +{ + const H5O_chk_cache_ud_t *udata = (const H5O_chk_cache_ud_t *)_udata; /* User data for callback */ -#ifdef H5_HAVE_PARALLEL - FUNC_ENTER_STATIC -#else FUNC_ENTER_STATIC_NOERR -#endif /* H5_HAVE_PARALLEL */ /* Check arguments */ - HDassert(oh); - HDassert(oh->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); - HDassert(oh->cache_info.type == H5AC_OHDR); - -#ifdef H5_HAVE_PARALLEL - if((oh->nchunks > 0) && (!about_to_destroy)) { - /* Scan through chunk 0 (the chunk stored contiguously with this - * object header) and cause it to update its image of all entries - * currently marked dirty. Must do this in the parallel case, as - * it is possible that this processor may clear this object header - * several times before flushing it -- thus causing undefined - * sections of the image to be written to disk overwriting valid data. - */ - if(H5O__chunk_serialize(f, oh, 0) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "unable to serialize object header chunk") - } /* end if */ -#endif /* H5_HAVE_PARALLEL */ - - /* Mark messages stored with the object header (i.e. messages in chunk 0) as clean */ - for(u = 0; u < oh->nmesgs; u++) - if(oh->mesg[u].chunkno == 0) - oh->mesg[u].dirty = FALSE; + HDassert(udata); + HDassert(udata->oh); + HDassert(image_len); -#ifndef NDEBUG - /* Reset the number of messages dirtied by decoding */ - oh->ndecode_dirtied = 0; -#endif /* NDEBUG */ + /* Set the image length size */ + *image_len = udata->size; -#ifdef H5_HAVE_PARALLEL -done: -#endif /* H5_HAVE_PARALLEL */ - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O__cache_clear() */ + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5O__cache_chk_get_initial_load_size() */ /*------------------------------------------------------------------------- - * Function: H5O__cache_chk_get_load_size() + * Function: H5B2__cache_chk_verify_chksum * - * Purpose: Tell the metadata cache how large the on disk image of the - * chunk proxy is, so it can load the image into a buffer for the - * deserialize call. In this case, we simply look up the size in - * the user data, and return it in *image_len, + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. * - * Return: Success: SUCCEED - * Failure: FAIL + * Return: Success: TRUE/FALSE + * Failure: Negative * - * Programmer: John Mainzer - * 7/28/14 + * Programmer: Vailin Choi + * Aug 2015 * *------------------------------------------------------------------------- */ -static herr_t -H5O__cache_chk_get_load_size(const void *_udata, size_t *image_len) +static htri_t +H5O__cache_chk_verify_chksum(const void *_image, size_t len, void *_udata) { - const H5O_chk_cache_ud_t *udata = (const H5O_chk_cache_ud_t *)_udata; /* User data for callback */ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5O_chk_cache_ud_t *udata = (H5O_chk_cache_ud_t *)_udata; /* User data for callback */ + htri_t ret_value = TRUE; /* Return value */ FUNC_ENTER_STATIC_NOERR /* Check arguments */ - HDassert(udata); - HDassert(udata->oh); - HDassert(image_len); + HDassert(image); - *image_len = udata->size; + /* There is no checksum for version 1 */ + if(udata->oh->version != H5O_VERSION_1) { + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ - FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5O__cache_chk_get_load_size() */ + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O__cache_chk_verify_chksum() */ /*------------------------------------------------------------------------- @@ -848,8 +916,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5O__cache_chk_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5O__cache_chk_image_len(const void *_thing, size_t *image_len) { const H5O_chunk_proxy_t * chk_proxy = (const H5O_chunk_proxy_t *)_thing; /* Chunk proxy to query */ @@ -867,10 +934,6 @@ H5O__cache_chk_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5O__cache_chk_image_len() */ -/************************************/ -/* no H5O_cache_chk_pre_serialize() */ -/************************************/ - /*------------------------------------------------------------------------- * Function: H5O__cache_chk_serialize @@ -918,116 +981,115 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O__cache_chk_serialize() */ -/**************************************/ -/* no H5O_cache_chk_notify() function */ -/**************************************/ - /*------------------------------------------------------------------------- - * Function: H5O__cache_chk_free_icr + * Function: H5O__cache_chk_notify * - * Purpose: Free the in core memory associated with the supplied object - * header continuation chunk. + * Purpose: Handle cache action notifications * - * Note: The metadata cache sets the object's cache_info.magic to - * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr - * callback (checked in assert). + * Return: Non-negative on success/Negative on failure * - * Return: Success: SUCCEED - * Failure: FAIL - * - * Programmer: John Mainzer - * 7/28/14 + * Programmer: Neil Fortner + * Mar 20 2012 * *------------------------------------------------------------------------- */ static herr_t -H5O__cache_chk_free_icr(void *_thing) +H5O__cache_chk_notify(H5AC_notify_action_t action, void *_thing) { - H5O_chunk_proxy_t * chk_proxy = (H5O_chunk_proxy_t *)_thing; /* Object header chunk proxy to release */ - herr_t ret_value = SUCCEED; /* Return value */ + H5O_chunk_proxy_t *chk_proxy = (H5O_chunk_proxy_t *)_thing; + void *parent = NULL; /* Chunk containing continuation message that points to this chunk */ + H5O_chunk_proxy_t *cont_chk_proxy = NULL; /* Proxy for chunk containing continuation message that points to this chunk, if not chunk 0 */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC - /* Check arguments */ + /* + * Check arguments. + */ HDassert(chk_proxy); - HDassert(chk_proxy->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); - HDassert(chk_proxy->cache_info.type == H5AC_OHDR_CHK); + HDassert(chk_proxy->oh); - /* Destroy object header chunk proxy */ - if(H5O__chunk_dest(chk_proxy) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header chunk proxy") + switch(action) { + case H5AC_NOTIFY_ACTION_AFTER_INSERT: + case H5AC_NOTIFY_ACTION_AFTER_LOAD: + case H5AC_NOTIFY_ACTION_AFTER_FLUSH: + case H5AC_NOTIFY_ACTION_ENTRY_DIRTIED: + /* do nothing */ + break; + + case H5AC_NOTIFY_ACTION_ENTRY_CLEANED: + { + unsigned u; /* Local index variable */ + + /* Mark messages in chunk as clean */ + for(u = 0; u < chk_proxy->oh->nmesgs; u++) + if(chk_proxy->oh->mesg[u].chunkno == chk_proxy->chunkno) + chk_proxy->oh->mesg[u].dirty = FALSE; + } + break; + + case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: + case H5AC_NOTIFY_ACTION_CHILD_CLEANED: + case H5AC_NOTIFY_ACTION_BEFORE_EVICT: + /* do nothing */ + break; + + default: +#ifdef NDEBUG + HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "unknown action from metadata cache") +#else /* NDEBUG */ + HDassert(0 && "Unknown action?!?"); +#endif /* NDEBUG */ + } /* end switch */ done: + if(cont_chk_proxy) + if(H5O_chunk_unprotect(chk_proxy->f, H5AC_ind_read_dxpl_id, cont_chk_proxy, FALSE) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTUNPROTECT, FAIL, "unable to unprotect object header chunk") + FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O__cache_chk_free_icr() */ +} /* end H5O__cache_chk_notify() */ /*------------------------------------------------------------------------- - * Function: H5O__cache_chk_clear - * - * Purpose: Clear all dirty bits associated with this cache entry. + * Function: H5O__cache_chk_free_icr * - * This is ncessary as the object header cache client maintains - * its own dirty bits on individual messages. These dirty bits - * used to be cleared by the old V2 metadata cache flush callback, - * but now the metadata cache must clear them explicitly, as - * the serialize callback does not imply that the data has been - * written to disk. + * Purpose: Free the in core memory associated with the supplied object + * header continuation chunk. * - * This callback is also necessary for the parallel case. + * Note: The metadata cache sets the object's cache_info.magic to + * H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC before calling a free_icr + * callback (checked in assert). * * Return: Success: SUCCEED * Failure: FAIL * * Programmer: John Mainzer - * 9/22/14 + * 7/28/14 * *------------------------------------------------------------------------- */ static herr_t -#ifdef H5_HAVE_PARALLEL -H5O__cache_chk_clear(const H5F_t *f, void *_thing, hbool_t about_to_destroy) -#else -H5O__cache_chk_clear(const H5F_t H5_ATTR_UNUSED *f, void *_thing, hbool_t H5_ATTR_UNUSED about_to_destroy) -#endif /* H5_HAVE_PARALLEL */ -{ - H5O_chunk_proxy_t *chk_proxy = (H5O_chunk_proxy_t *)_thing; /* Object header chunk to reset */ - H5O_t *oh; /* Object header for chunk */ - unsigned u; /* Local index variable */ +H5O__cache_chk_free_icr(void *_thing) +{ + H5O_chunk_proxy_t * chk_proxy = (H5O_chunk_proxy_t *)_thing; /* Object header chunk proxy to release */ herr_t ret_value = SUCCEED; /* Return value */ -#ifdef H5_HAVE_PARALLEL FUNC_ENTER_STATIC -#else - FUNC_ENTER_STATIC_NOERR -#endif /* H5_HAVE_PARALLEL */ /* Check arguments */ HDassert(chk_proxy); - HDassert(chk_proxy->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); + HDassert(chk_proxy->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_BAD_MAGIC); HDassert(chk_proxy->cache_info.type == H5AC_OHDR_CHK); - oh = chk_proxy->oh; - HDassert(oh); - HDassert(oh->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); - HDassert(oh->cache_info.type == H5AC_OHDR); -#ifdef H5_HAVE_PARALLEL - if((chk_proxy->oh->cache_info.is_dirty) && (!about_to_destroy)) - if(H5O__chunk_serialize(f, chk_proxy->oh, chk_proxy->chunkno) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSERIALIZE, FAIL, "unable to serialize object header chunk") -#endif /* H5_HAVE_PARALLEL */ - - /* Mark messages in chunk as clean */ - for(u = 0; u < chk_proxy->oh->nmesgs; u++) - if(chk_proxy->oh->mesg[u].chunkno == chk_proxy->chunkno) - chk_proxy->oh->mesg[u].dirty = FALSE; + /* Destroy object header chunk proxy */ + if(H5O__chunk_dest(chk_proxy) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTRELEASE, FAIL, "unable to destroy object header chunk proxy") -#ifdef H5_HAVE_PARALLEL done: -#endif /* H5_HAVE_PARALLEL */ FUNC_LEAVE_NOAPI(ret_value) -} /* end H5O__cache_chk_clear() */ +} /* end H5O__cache_chk_free_icr() */ /*------------------------------------------------------------------------- @@ -1099,12 +1161,12 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image { const uint8_t *chunk_image; /* Pointer into buffer to decode */ uint8_t *eom_ptr; /* Pointer to end of messages for a chunk */ - size_t curmesg; /* Current message being decoded in object header */ unsigned merged_null_msgs = 0; /* Number of null messages merged together */ unsigned chunkno; /* Current chunk's index */ #ifndef NDEBUG unsigned nullcnt; /* Count of null messages (for sanity checking gaps in chunks) */ #endif /* NDEBUG */ + hbool_t mesgs_modified = FALSE; /* Whether any messages were modified when the object header was deserialized */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_STATIC @@ -1130,15 +1192,12 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image /* Init the chunk data info */ chunkno = (unsigned)oh->nchunks++; oh->chunk[chunkno].gap = 0; - if(chunkno == 0) { + oh->chunk[chunkno].addr = addr; + if(chunkno == 0) /* First chunk's 'image' includes room for the object header prefix */ - oh->chunk[0].addr = addr; oh->chunk[0].size = len + (size_t)H5O_SIZEOF_HDR(oh); - } /* end if */ - else { - oh->chunk[chunkno].addr = addr; + else oh->chunk[chunkno].size = len; - } /* end else */ if(NULL == (oh->chunk[chunkno].image = H5FL_BLK_MALLOC(chunk_image, oh->chunk[chunkno].size))) HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "memory allocation failed") @@ -1160,16 +1219,12 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image chunk_image += H5_SIZEOF_MAGIC; } /* end if */ - /* Save # of messages already inspected */ - curmesg = oh->nmesgs; - /* Decode messages from this chunk */ eom_ptr = oh->chunk[chunkno].image + (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM_OH(oh)); #ifndef NDEBUG nullcnt = 0; #endif /* NDEBUG */ while(chunk_image < eom_ptr) { - size_t mesgno; /* Current message to operate on */ size_t mesg_size; /* Size of message read in */ unsigned id; /* ID (type) of current message */ uint8_t flags; /* Flags for current message */ @@ -1228,39 +1283,45 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image H5O_NULL_ID == id && oh->nmesgs > 0 && H5O_NULL_ID == oh->mesg[oh->nmesgs - 1].type->id && oh->mesg[oh->nmesgs - 1].chunkno == chunkno) { + size_t mesgno; /* Current message to operate on */ /* Combine adjacent null messages */ mesgno = oh->nmesgs - 1; oh->mesg[mesgno].raw_size += (size_t)H5O_SIZEOF_MSGHDR_OH(oh) + mesg_size; oh->mesg[mesgno].dirty = TRUE; merged_null_msgs++; - udata->merged_null_msgs++; } /* end if */ else { + H5O_mesg_t *mesg; /* Pointer to new message */ + unsigned ioflags = 0; /* Flags for decode routine */ + /* Check if we need to extend message table to hold the new message */ if(oh->nmesgs >= oh->alloc_nmesgs) if(H5O_alloc_msgs(oh, (size_t)1) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, FAIL, "can't allocate more space for messages") - /* Get index for message */ - mesgno = oh->nmesgs++; + /* Get pointer to message to set up */ + mesg = &oh->mesg[oh->nmesgs]; + + /* Increment # of messages */ + oh->nmesgs++; /* Initialize information about message */ - oh->mesg[mesgno].dirty = FALSE; - oh->mesg[mesgno].flags = flags; - oh->mesg[mesgno].crt_idx = crt_idx; - oh->mesg[mesgno].native = NULL; - oh->mesg[mesgno].raw = (uint8_t *)chunk_image; /* Casting away const OK - QAK */ - oh->mesg[mesgno].raw_size = mesg_size; - oh->mesg[mesgno].chunkno = chunkno; + mesg->dirty = FALSE; + mesg->flags = flags; + mesg->crt_idx = crt_idx; + mesg->native = NULL; + mesg->raw = (uint8_t *)chunk_image; /* Casting away const OK - QAK */ + mesg->raw_size = mesg_size; + mesg->chunkno = chunkno; /* Point unknown messages at 'unknown' message class */ /* (Usually from future versions of the library) */ if(id >= H5O_UNKNOWN_ID || #ifdef H5O_ENABLE_BOGUS - id == H5O_BOGUS_VALID_ID || + id == H5O_BOGUS_VALID_ID || #endif - NULL == H5O_msg_class_g[id]) { + NULL == H5O_msg_class_g[id]) { H5O_unknown_t *unknown; /* Pointer to "unknown" message info */ @@ -1272,10 +1333,10 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image *unknown = id; /* Save 'native' form of unknown message */ - oh->mesg[mesgno].native = unknown; + mesg->native = unknown; /* Set message to "unknown" class */ - oh->mesg[mesgno].type = H5O_msg_class_g[H5O_UNKNOWN_ID]; + mesg->type = H5O_msg_class_g[H5O_UNKNOWN_ID]; /* Check for "fail if unknown" message flags */ if(((udata->file_intent & H5F_ACC_RDWR) && @@ -1298,17 +1359,66 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image * the metadata cache in some other "weird" way, like * using H5Ocopy() - QAK */ - oh->mesg[mesgno].flags |= H5O_MSG_FLAG_WAS_UNKNOWN; + mesg->flags |= H5O_MSG_FLAG_WAS_UNKNOWN; /* Mark the message and chunk as dirty */ - oh->mesg[mesgno].dirty = TRUE; - udata->mesgs_modified = TRUE; - *dirty = TRUE; + mesg->dirty = TRUE; + mesgs_modified = TRUE; } /* end if */ } /* end if */ else /* Set message class for "known" messages */ - oh->mesg[mesgno].type = H5O_msg_class_g[id]; + mesg->type = H5O_msg_class_g[id]; + + /* Do some inspection/interpretation of new messages from this chunk */ + /* (detect continuation messages, ref. count messages, etc.) */ + + /* Check if message is a continuation message */ + if(H5O_CONT_ID == id) { + H5O_cont_t *cont; + + /* Decode continuation message */ + cont = (H5O_cont_t *)(H5O_MSG_CONT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, mesg->raw); + H5_CHECKED_ASSIGN(cont->chunkno, unsigned, udata->cont_msg_info->nmsgs + 1, size_t); /* the next continuation message/chunk */ + + /* Save 'native' form of continuation message */ + mesg->native = cont; + + /* Add to continuation messages left to interpret */ + if(H5O__add_cont_msg(udata->cont_msg_info, cont) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add continuation message") + } /* end if */ + /* Check if message is a ref. count message */ + else if(H5O_REFCOUNT_ID == id) { + H5O_refcount_t *refcount; + + /* Decode ref. count message */ + HDassert(oh->version > H5O_VERSION_1); + refcount = (H5O_refcount_t *)(H5O_MSG_REFCOUNT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, mesg->raw); + + /* Save 'native' form of ref. count message */ + mesg->native = refcount; + + /* Set object header values */ + oh->has_refcount_msg = TRUE; + oh->nlink = *refcount; + } /* end if */ + /* Check if message is a link message */ + else if(H5O_LINK_ID == id) { + /* Increment the count of link messages */ + oh->link_msgs_seen++; + } /* end if */ + /* Check if message is an attribute message */ + else if(H5O_ATTR_ID == id) { + /* Increment the count of attribute messages */ + oh->attr_msgs_seen++; + } /* end if */ + + /* Mark the message & chunk as dirty if the message was changed by decoding */ + if((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) { + mesg->dirty = TRUE; + mesgs_modified = TRUE; + } /* end if */ } /* end else */ /* Advance decode pointer past message */ @@ -1333,89 +1443,23 @@ H5O__chunk_deserialize(H5O_t *oh, haddr_t addr, size_t len, const uint8_t *image /* Check for correct checksum on chunks, in later versions of the format */ if(oh->version > H5O_VERSION_1) { uint32_t stored_chksum; /* Checksum from file */ - uint32_t computed_chksum; /* Checksum computed in memory */ + + /* checksum verification already done in verify_chksum cb */ /* Metadata checksum */ UINT32DECODE(chunk_image, stored_chksum); - - /* Compute checksum on chunk */ - computed_chksum = H5_checksum_metadata(oh->chunk[chunkno].image, (oh->chunk[chunkno].size - H5O_SIZEOF_CHKSUM), 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "incorrect metadata checksum for object header chunk") } /* end if */ /* Sanity check */ HDassert(chunk_image == oh->chunk[chunkno].image + oh->chunk[chunkno].size); - /* Do some inspection/interpretation of new messages from this chunk */ - /* (detect continuation messages, ref. count messages, etc.) */ - while(curmesg < oh->nmesgs) { - /* Check if next message to examine is a continuation message */ - if(H5O_CONT_ID == oh->mesg[curmesg].type->id) { - H5O_cont_t *cont; - unsigned ioflags = 0; /* Flags for decode routine */ - - /* Decode continuation message */ - cont = (H5O_cont_t *)(H5O_MSG_CONT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, oh->mesg[curmesg].raw); - H5_CHECKED_ASSIGN(cont->chunkno, unsigned, udata->cont_msg_info->nmsgs + 1, size_t); /* the next continuation message/chunk */ - - /* Save 'native' form of continuation message */ - oh->mesg[curmesg].native = cont; - - /* Add to continuation messages left to interpret */ - if(H5O__add_cont_msg(udata->cont_msg_info, cont) < 0) - HGOTO_ERROR(H5E_OHDR, H5E_CANTSET, FAIL, "can't add continuation message") - - /* Mark the message & chunk as dirty if the message was changed by decoding */ - if((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) { - oh->mesg[curmesg].dirty = TRUE; - udata->mesgs_modified = TRUE; - *dirty = TRUE; - } /* end if */ - } /* end if */ - /* Check if next message to examine is a ref. count message */ - else if(H5O_REFCOUNT_ID == oh->mesg[curmesg].type->id) { - H5O_refcount_t *refcount; - unsigned ioflags = 0; /* Flags for decode routine */ - - /* Decode ref. count message */ - HDassert(oh->version > H5O_VERSION_1); - refcount = (H5O_refcount_t *)(H5O_MSG_REFCOUNT->decode)(udata->f, udata->dxpl_id, NULL, 0, &ioflags, oh->mesg[curmesg].raw); - - /* Save 'native' form of ref. count message */ - oh->mesg[curmesg].native = refcount; - - /* Set object header values */ - oh->has_refcount_msg = TRUE; - oh->nlink = *refcount; - - /* Mark the message & chunk as dirty if the message was changed by decoding */ - if((ioflags & H5O_DECODEIO_DIRTY) && (udata->file_intent & H5F_ACC_RDWR)) { - oh->mesg[curmesg].dirty = TRUE; - udata->mesgs_modified = TRUE; - *dirty = TRUE; - } /* end if */ - } /* end if */ - /* Check if next message to examine is a link message */ - else if(H5O_LINK_ID == oh->mesg[curmesg].type->id) { - /* Increment the count of link messages */ - oh->link_msgs_seen++; - } /* end if */ - /* Check if next message to examine is an attribute message */ - else if(H5O_ATTR_ID == oh->mesg[curmesg].type->id) { - /* Increment the count of attribute messages */ - oh->attr_msgs_seen++; - } /* end if */ - - /* Advance to next message */ - curmesg++; - } /* end while */ + /* Mark the chunk dirty if we've modified messages */ + if(mesgs_modified) + *dirty = TRUE; /* Mark the chunk dirty if we've merged null messages */ - if(merged_null_msgs) { - udata->mesgs_modified = TRUE; + if(merged_null_msgs > 0) { + udata->merged_null_msgs += merged_null_msgs; *dirty = TRUE; } /* end if */ diff --git a/src/H5Opkg.h b/src/H5Opkg.h index 659bf6e..b80f736 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -292,9 +292,6 @@ struct H5O_t { /* Chunk management information (not stored) */ size_t rc; /* Reference count of [continuation] chunks using this structure */ - size_t chunk0_size; /* Size of serialized first chunk */ - hbool_t mesgs_modified; /* Whether any messages were modified when the object header was deserialized */ - hbool_t prefix_modified; /* Whether prefix was modified when the object header was deserialized */ /* Object information (stored) */ hbool_t has_refcount_msg; /* Whether the object has a ref. count message */ @@ -362,7 +359,6 @@ typedef struct H5O_common_cache_ud_t { hid_t dxpl_id; /* DXPL for operation */ unsigned file_intent; /* Read/write intent for file */ unsigned merged_null_msgs; /* Number of null messages merged together */ - hbool_t mesgs_modified; /* Whether any messages were modified when the object header was deserialized */ H5O_cont_msgs_t *cont_msg_info; /* Pointer to continuation messages to work on */ haddr_t addr; /* Address of the prefix or chunk */ } H5O_common_cache_ud_t; @@ -371,6 +367,8 @@ typedef struct H5O_common_cache_ud_t { typedef struct H5O_cache_ud_t { hbool_t made_attempt; /* Whether the deserialize routine was already attempted */ unsigned v1_pfx_nmesgs; /* Number of messages from v1 prefix header */ + size_t chunk0_size; /* Size of serialized first chunk */ + H5O_t *oh; /* Partially deserialized object header, for later use */ H5O_common_cache_ud_t common; /* Common object header cache callback info */ } H5O_cache_ud_t; diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 15662cc..81cd260 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -178,6 +178,11 @@ #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF 524288 #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_ENC H5P__encode_size_t #define H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEC H5P__decode_size_t +/* Definition for # of metadata read attempts */ +#define H5F_ACS_METADATA_READ_ATTEMPTS_SIZE sizeof(unsigned) +#define H5F_ACS_METADATA_READ_ATTEMPTS_DEF 0 +#define H5F_ACS_METADATA_READ_ATTEMPTS_ENC H5P__encode_unsigned +#define H5F_ACS_METADATA_READ_ATTEMPTS_DEC H5P__decode_unsigned /* Definition for object flush callback */ #define H5F_ACS_OBJECT_FLUSH_CB_SIZE sizeof(H5F_object_flush_t) #define H5F_ACS_OBJECT_FLUSH_CB_DEF {NULL, NULL} @@ -327,6 +332,7 @@ static const unsigned H5F_def_efc_size_g = H5F_ACS_EFC_SIZE_DEF; static const H5FD_file_image_info_t H5F_def_file_image_info_g = H5F_ACS_FILE_IMAGE_INFO_DEF; /* Default file image info and callbacks */ static const hbool_t H5F_def_core_write_tracking_flag_g = H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF; /* Default setting for core VFD write tracking */ static const size_t H5F_def_core_write_tracking_page_size_g = H5F_ACS_CORE_WRITE_TRACKING_PAGE_SIZE_DEF; /* Default core VFD write tracking page size */ +static const unsigned H5F_def_metadata_read_attempts_g = H5F_ACS_METADATA_READ_ATTEMPTS_DEF; /* Default setting for the # of metadata read attempts */ static const H5F_object_flush_t H5F_def_object_flush_cb_g = H5F_ACS_OBJECT_FLUSH_CB_DEF; /* Default setting for object flush callback */ static const hbool_t H5F_def_use_mdc_logging_g = H5F_ACS_USE_MDC_LOGGING_DEF; /* Default metadata cache logging flag */ static const char *H5F_def_mdc_log_location_g = H5F_ACS_MDC_LOG_LOCATION_DEF; /* Default mdc log location */ @@ -492,6 +498,12 @@ H5P__facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register the # of read attempts */ + if(H5P_register_real(pclass, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, H5F_ACS_METADATA_READ_ATTEMPTS_SIZE, &H5F_def_metadata_read_attempts_g, + NULL, NULL, NULL, H5F_ACS_METADATA_READ_ATTEMPTS_ENC, H5F_ACS_METADATA_READ_ATTEMPTS_DEC, + NULL, NULL, NULL, NULL) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + /* Register object flush callback */ /* (Note: this property should not have an encode/decode callback -QAK) */ if(H5P_register_real(pclass, H5F_ACS_OBJECT_FLUSH_CB_NAME, H5F_ACS_OBJECT_FLUSH_CB_SIZE, &H5F_def_object_flush_cb_g, @@ -3570,6 +3582,90 @@ done: /*------------------------------------------------------------------------- + * Function: H5Pset_metadata_read_attempts + * + * Purpose: Sets the # of read attempts in the file access property list + * when reading metadata with checksum. + * The # of read attempts set via this routine will only apply + * when opening a file with SWMR access. + * The # of read attempts set via this routine does not have + * any effect when opening a file with non-SWMR access; for this + * case, the # of read attempts will be always be 1. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Sept 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts) +{ + H5P_genplist_t *plist; /* Property list pointer */ + herr_t ret_value = SUCCEED; /* return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "iIu", plist_id, attempts); + + /* Cannot set the # of attempts to 0 */ + if(attempts == 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "number of metadatata read attempts must be greater than 0"); + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Set values */ + if(H5P_set(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &attempts) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set # of metadata read attempts") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Pset_metadata_read_attempts() */ + + +/*------------------------------------------------------------------------- + * Function: H5Pget_metadata_read_attempts + * + * Purpose: Returns the # of metadata read attempts set in the file access property list. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi; Sept 2013 + * + *------------------------------------------------------------------------- + */ +herr_t +H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts/*out*/) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(FAIL) + H5TRACE2("e", "ix", plist_id, attempts); + + /* Get values */ + if(attempts) { + H5P_genplist_t *plist; /* Property list pointer */ + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + + /* Get the # of read attempts set */ + if(H5P_get(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, attempts) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get the number of metadata read attempts") + + /* If not set, return the default value */ + if(*attempts == H5F_ACS_METADATA_READ_ATTEMPTS_DEF) /* 0 */ + *attempts = H5F_METADATA_READ_ATTEMPTS; + } /* end if */ + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Pget_metadata_read_attempts() */ + + +/*------------------------------------------------------------------------- * Function: H5Pset_obj_flush_cb * * Purpose: Sets the callback function to invoke and the user data when an diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 1f0d734..c736d7b 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -351,6 +351,8 @@ H5_DLL herr_t H5Pget_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callbacks_ptr); H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size); H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size); +H5_DLL herr_t H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts); +H5_DLL herr_t H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts); H5_DLL herr_t H5Pset_object_flush_cb(hid_t plist_id, H5F_flush_cb_t func, void *udata); H5_DLL herr_t H5Pget_object_flush_cb(hid_t plist_id, H5F_flush_cb_t *func, void **udata); H5_DLL herr_t H5Pset_mdc_log_options(hid_t plist_id, hbool_t is_enabled, const char *location, hbool_t start_on_access); diff --git a/src/H5SMcache.c b/src/H5SMcache.c index 4de86e2..455dd1a 100644 --- a/src/H5SMcache.c +++ b/src/H5SMcache.c @@ -58,20 +58,20 @@ /********************/ /* Metadata cache (H5AC) callbacks */ -static herr_t H5SM__cache_table_get_load_size(const void *udata, size_t *image_len); +static herr_t H5SM__cache_table_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5SM__cache_table_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5SM__cache_table_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5SM__cache_table_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5SM__cache_table_image_len(const void *thing, size_t *image_len); static herr_t H5SM__cache_table_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5SM__cache_table_free_icr(void *thing); -static herr_t H5SM__cache_list_get_load_size(const void *udata, size_t *image_len); +static herr_t H5SM__cache_list_get_initial_load_size(void *udata, size_t *image_len); +static htri_t H5SM__cache_list_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *H5SM__cache_list_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty); -static herr_t H5SM__cache_list_image_len(const void *thing, size_t *image_len, - hbool_t *compressed_ptr, size_t *compressed_image_len_ptr); +static herr_t H5SM__cache_list_image_len(const void *thing, size_t *image_len); static herr_t H5SM__cache_list_serialize(const H5F_t *f, void *image, size_t len, void *thing); static herr_t H5SM__cache_list_free_icr(void *thing); @@ -87,14 +87,15 @@ const H5AC_class_t H5AC_SOHM_TABLE[1] = {{ "shared message table", /* Metadata client name (for debugging) */ H5FD_MEM_SOHM_TABLE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5SM__cache_table_get_load_size, /* 'get_load_size' callback */ + H5SM__cache_table_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5SM__cache_table_verify_chksum, /* 'verify_chksum' callback */ H5SM__cache_table_deserialize, /* 'deserialize' callback */ H5SM__cache_table_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5SM__cache_table_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5SM__cache_table_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -103,14 +104,15 @@ const H5AC_class_t H5AC_SOHM_LIST[1] = {{ "shared message list", /* Metadata client name (for debugging) */ H5FD_MEM_SOHM_TABLE, /* File space memory type for client */ H5AC__CLASS_NO_FLAGS_SET, /* Client class behavior flags */ - H5SM__cache_list_get_load_size, /* 'get_load_size' callback */ + H5SM__cache_list_get_initial_load_size, /* 'get_initial_load_size' callback */ + NULL, /* 'get_final_load_size' callback */ + H5SM__cache_list_verify_chksum, /* 'verify_chksum' callback */ H5SM__cache_list_deserialize, /* 'deserialize' callback */ H5SM__cache_list_image_len, /* 'image_len' callback */ NULL, /* 'pre_serialize' callback */ H5SM__cache_list_serialize, /* 'serialize' callback */ NULL, /* 'notify' callback */ H5SM__cache_list_free_icr, /* 'free_icr' callback */ - NULL, /* 'clear' callback */ NULL, /* 'fsf_size' callback */ }}; @@ -127,11 +129,10 @@ const H5AC_class_t H5AC_SOHM_LIST[1] = {{ /*------------------------------------------------------------------------- - * Function: H5SM__cache_table_get_load_size() + * Function: H5SM__cache_table_get_initial_load_size() * * Purpose: Return the size of the master table of Shared Object Header - * Message indexes on disk. As this cache client doesn't use - * speculative reads, this value should be accurate. + * Message indexes on disk. * * Return: Success: SUCCEED * Failure: FAIL @@ -142,9 +143,9 @@ const H5AC_class_t H5AC_SOHM_LIST[1] = {{ *------------------------------------------------------------------------- */ static herr_t -H5SM__cache_table_get_load_size(const void *_udata, size_t *image_len) +H5SM__cache_table_get_initial_load_size(void *_udata, size_t *image_len) { - const H5SM_table_cache_ud_t *udata = (const H5SM_table_cache_ud_t *)_udata; /* User data for callback */ + const H5SM_table_cache_ud_t *udata = (const H5SM_table_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -153,10 +154,47 @@ H5SM__cache_table_get_load_size(const void *_udata, size_t *image_len) HDassert(udata->f); HDassert(image_len); + /* Set the image length size */ *image_len = H5SM_TABLE_SIZE(udata->f); FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5SM__cache_table_get_load_size() */ +} /* end H5SM__cache_table_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5SM__cache_table_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +htri_t +H5SM__cache_table_verify_chksum(const void *_image, size_t len, void H5_ATTR_UNUSED *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, len, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5SM__cache_table_verify_chksum() */ /*------------------------------------------------------------------------- @@ -184,7 +222,6 @@ H5SM__cache_table_deserialize(const void *_image, size_t len, void *_udata, H5SM_table_cache_ud_t *udata = (H5SM_table_cache_ud_t *)_udata; /* Pointer to user data */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into input buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ size_t u; /* Counter variable for index headers */ void *ret_value = NULL; /* Return value */ @@ -260,19 +297,14 @@ H5SM__cache_table_deserialize(const void *_image, size_t len, void *_udata, table->indexes[u].list_size = H5SM_LIST_SIZE(f, table->indexes[u].list_max); } /* end for */ + /* checksum verification already done in verify_chksum cb */ + /* Read in checksum */ UINT32DECODE(image, stored_chksum); /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) == table->table_size); - /* Compute checksum on entire header */ - computed_chksum = H5_checksum_metadata(_image, (table->table_size - H5SM_SIZEOF_CHECKSUM), 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message table") - /* Set return value */ ret_value = table; @@ -301,8 +333,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5SM__cache_table_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5SM__cache_table_image_len(const void *_thing, size_t *image_len) { const H5SM_master_table_t *table = (const H5SM_master_table_t *)_thing; /* Shared message table to query */ @@ -319,10 +350,6 @@ H5SM__cache_table_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5SM__cache_table_image_len() */ -/***************************************/ -/* no H5SM_cache_table_pre_serialize() */ -/***************************************/ - /*------------------------------------------------------------------------- * Function: H5SM__cache_table_serialize @@ -451,7 +478,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5SM__cache_list_get_load_size() + * Function: H5SM__cache_list_get_initial_load_size() * * Purpose: Return the on disk size of list of SOHM messages. In this case, * we simply look up the size in the user data, and return that value @@ -466,9 +493,9 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5SM__cache_list_get_load_size(const void *_udata, size_t *image_len) +H5SM__cache_list_get_initial_load_size(void *_udata, size_t *image_len) { - const H5SM_list_cache_ud_t *udata = (const H5SM_list_cache_ud_t *)_udata; /* User data for callback */ + const H5SM_list_cache_ud_t *udata = (const H5SM_list_cache_ud_t *)_udata; /* User data for callback */ FUNC_ENTER_STATIC_NOERR @@ -478,10 +505,53 @@ H5SM__cache_list_get_load_size(const void *_udata, size_t *image_len) HDassert(udata->header->list_size > 0); HDassert(image_len); + /* Set the image length size */ *image_len = udata->header->list_size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* end H5SM__cache_list_get_load_size() */ +} /* end H5SM__cache_list_get_initial_load_size() */ + + +/*------------------------------------------------------------------------- + * Function: H5SM__cache_list_verify_chksum + * + * Purpose: Verify the computed checksum of the data structure is the + * same as the stored chksum. + * + * Return: Success: TRUE/FALSE + * Failure: Negative + * + * Programmer: Vailin Choi; Aug 2015 + * + *------------------------------------------------------------------------- + */ +htri_t +H5SM__cache_list_verify_chksum(const void *_image, size_t H5_ATTR_UNUSED len, void *_udata) +{ + const uint8_t *image = (const uint8_t *)_image; /* Pointer into raw data buffer */ + H5SM_list_cache_ud_t *udata = (H5SM_list_cache_ud_t *)_udata; /* User data for callback */ + size_t chk_size; /* Exact size of the node with checksum at the end */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + htri_t ret_value = TRUE; /* Return value */ + + FUNC_ENTER_STATIC_NOERR + + /* Check arguments */ + HDassert(image); + HDassert(udata); + + /* Exact size with checksum at the end */ + chk_size = H5SM_LIST_SIZE(udata->f, udata->header->num_messages); + + /* Get stored and computed checksums */ + H5F_get_checksums(image, chk_size, &stored_chksum, &computed_chksum); + + if(stored_chksum != computed_chksum) + ret_value = FALSE; + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5SM__cache_list_verify_chksum() */ /*------------------------------------------------------------------------- @@ -508,7 +578,6 @@ H5SM__cache_list_deserialize(const void *_image, size_t len, void *_udata, H5SM_bt2_ctx_t ctx; /* Message encoding context */ const uint8_t *image = (const uint8_t *)_image; /* Pointer into input buffer */ uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ size_t u; /* Counter variable for messages in list */ void *ret_value = NULL; /* Return value */ @@ -546,19 +615,14 @@ H5SM__cache_list_deserialize(const void *_image, size_t len, void *_udata, image += H5SM_SOHM_ENTRY_SIZE(udata->f); } /* end for */ + /* checksum verification already done in verify_chksum cb */ + /* Read in checksum */ UINT32DECODE(image, stored_chksum); /* Sanity check */ HDassert((size_t)(image - (const uint8_t *)_image) <= udata->header->list_size); - /* Compute checksum on entire header */ - computed_chksum = H5_checksum_metadata(_image, ((size_t)(image - (const uint8_t *)_image) - H5SM_SIZEOF_CHECKSUM), 0); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message list") - /* Initialize the rest of the array */ for(u = udata->header->num_messages; u < udata->header->list_max; u++) list->messages[u].location = H5SM_NO_LOC; @@ -591,8 +655,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5SM__cache_list_image_len(const void *_thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_image_len_ptr) +H5SM__cache_list_image_len(const void *_thing, size_t *image_len) { const H5SM_list_t *list = (const H5SM_list_t *)_thing; /* Shared message list to query */ @@ -610,10 +673,6 @@ H5SM__cache_list_image_len(const void *_thing, size_t *image_len, FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5SM__cache_list_image_len() */ -/**************************************/ -/* no H5SM_cache_list_pre_serialize() */ -/**************************************/ - /*------------------------------------------------------------------------- * Function: H5SM__cache_list_serialize diff --git a/src/H5private.h b/src/H5private.h index 4c40965..47f6d78 100644 --- a/src/H5private.h +++ b/src/H5private.h @@ -813,7 +813,8 @@ H5_DLL H5_ATTR_CONST int Nflock(int fd, int operation); /* NOTE: flock(2) is not present on all POSIX systems. * If it is not present, we try a flock() equivalent based on * fcntl(2), then fall back to a function that always fails if - * it is not present at all. + * it is not present at all (Windows uses a separate Wflock() + * function). */ #if defined(H5_HAVE_FLOCK) #define HDflock(F,L) flock(F,L) @@ -1114,6 +1115,9 @@ typedef off_t h5_stat_size_t; #ifndef HDmodf #define HDmodf(X,Y) modf(X,Y) #endif /* HDmodf */ +#ifndef HDnanosleep + #define HDnanosleep(N, O) nanosleep(N, O) +#endif /* HDnanosleep */ #ifndef HDopen #ifdef _O_BINARY #define HDopen(S,F,M) open(S,F|_O_BINARY,M) @@ -2597,6 +2601,8 @@ H5_DLL uint32_t H5_hash_string(const char *str); /* Time related routines */ H5_DLL time_t H5_make_time(struct tm *tm); +H5_DLL void H5_nanosleep(uint64_t nanosec); +H5_DLL double H5_get_time(void); /* Functions for building paths, etc. */ H5_DLL herr_t H5_build_extpath(const char *name, char **extpath /*out*/); diff --git a/src/H5system.c b/src/H5system.c index ab08d2c..5205d08 100644 --- a/src/H5system.c +++ b/src/H5system.c @@ -911,6 +911,27 @@ Wflock(int fd, int operation) { return 0; } /* end Wflock() */ + + /*-------------------------------------------------------------------------- + * Function: Wnanosleep + * + * Purpose: Sleep for a given # of nanoseconds (Windows version) + * + * Return: SUCCEED/FAIL + * + * Programmer: Dana Robinson + * Fall 2016 + *-------------------------------------------------------------------------- + */ +void +Wnanosleep(uint64_t nanosec) +{ + /* XXX: Currently just a placeholder */ + FUNC_ENTER_NOAPI_NOINIT_NOERR + + FUNC_LEAVE_NOAPI_VOID +} /* end Wnanosleep() */ + #endif /* H5_HAVE_WIN32_API */ @@ -1108,3 +1129,63 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5_combine_path() */ + +/*-------------------------------------------------------------------------- + * Function: H5_nanosleep + * + * Purpose: Sleep for a given # of nanoseconds + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * October 01, 2016 + *-------------------------------------------------------------------------- + */ +void +H5_nanosleep(uint64_t nanosec) +{ + struct timespec sleeptime; /* Struct to hold time to sleep */ + + FUNC_ENTER_NOAPI_NOINIT_NOERR + + /* Set up time to sleep */ + sleeptime.tv_sec = 0; + sleeptime.tv_nsec = (long)nanosec; + + HDnanosleep(&sleeptime, NULL); + + FUNC_LEAVE_NOAPI_VOID +} /* end H5_nanosleep() */ + + +/*-------------------------------------------------------------------------- + * Function: H5_get_time + * + * Purpose: Get the current time, as the time of seconds after the UNIX epoch + * + * Return: SUCCEED/FAIL + * + * Programmer: Quincey Koziol + * October 05, 2016 + *-------------------------------------------------------------------------- + */ +double +H5_get_time(void) +{ +#ifdef H5_HAVE_GETTIMEOFDAY + struct timeval curr_time; +#endif /* H5_HAVE_GETTIMEOFDAY */ + double ret_value = (double)0.0f; + + FUNC_ENTER_NOAPI_NOINIT_NOERR + +#ifdef H5_HAVE_GETTIMEOFDAY + HDgettimeofday(&curr_time, NULL); + + ret_value = (double)curr_time.tv_sec + ((double)curr_time.tv_usec / (double)1000000.0f); +#endif /* H5_HAVE_GETTIMEOFDAY */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5_get_time() */ + + diff --git a/src/H5win32defs.h b/src/H5win32defs.h index a757832..662b97b 100644 --- a/src/H5win32defs.h +++ b/src/H5win32defs.h @@ -42,6 +42,7 @@ typedef __int64 h5_stat_size_t; #define HDlseek(F,O,W) _lseeki64(F,O,W) #define HDlstat(S,B) _lstati64(S,B) #define HDmkdir(S,M) _mkdir(S) +#define HDnanosleep(S) Wnanosleep(S) #define HDoff_t __int64 /* _O_BINARY must be set in Windows to avoid CR-LF <-> LF EOL * transformations when performing I/O. @@ -65,13 +66,23 @@ typedef __int64 h5_stat_size_t; */ #define HDmemset(X,C,Z) memset((void*)(X),C,Z) -#endif /* H5_HAVE_VISUAL_STUDIO */ - struct timezone { int tz_minuteswest; int tz_dsttime; }; +/* time.h before VS2015 does not include timespec */ +#if (_MSC_VER < 1900) +struct timespec +{ + time_t tv_sec; // Seconds - >= 0 + long tv_nsec; // Nanoseconds - [0, 999999999] +}; +#endif /* MSC_VER < 1900 */ + +#endif /* H5_HAVE_VISUAL_STUDIO */ + + #ifdef __cplusplus extern "C" { #endif /* __cplusplus */ diff --git a/test/cache.c b/test/cache.c index c89a1c7..3c2cf5c 100644 --- a/test/cache.c +++ b/test/cache.c @@ -17149,6 +17149,135 @@ check_protect_ro_rw_err(void) /*------------------------------------------------------------------------- + * Function: check_protect_retries() + * + * Purpose: To exercise checksum verification retries for an entry with + * a speculative load. + * + * Return: + * + * Programmer: + * + *------------------------------------------------------------------------- + */ +static unsigned +check_protect_retries(void) +{ + H5F_t * file_ptr = NULL; + H5C_t *cache_ptr = NULL; + test_entry_t *base_addr = NULL; + test_entry_t *entry_ptr = NULL; + H5C_cache_entry_t * cache_entry_ptr = NULL; + int32_t type; + int32_t idx; + + TESTING("protect an entry to verify retries"); + + pass = TRUE; + + /* Set up the cache */ + if(pass) { + + reset_entries(); + + file_ptr = setup_cache((size_t)(2 * 1024), + (size_t)(1 * 1024)); + + /* Set up read attempts for verifying checksum */ + file_ptr->shared->read_attempts = 10; + file_ptr->shared->retries_nbins = 1; + } + + /* Test only for this type which has a speculative load */ + type = VARIABLE_ENTRY_TYPE; + idx = 0; + + if(pass) { + + cache_ptr = file_ptr->shared->cache; + base_addr = entries[type]; + entry_ptr = &(base_addr[idx]); + + /* test case (1): + * --actual_len is smaller the initial length from get_load_size() + * --verify_chksum() returns TRUE after max_verify_ct is reached + * + */ + entry_ptr->actual_len = entry_ptr->size/2; + entry_ptr->max_verify_ct = 3; + entry_ptr->verify_ct = 0; + + cache_entry_ptr = (H5C_cache_entry_t *)H5C_protect(file_ptr, H5AC_ind_read_dxpl_id, + &(types[type]), entry_ptr->addr, &entry_ptr->addr, H5C__READ_ONLY_FLAG); + + if((cache_entry_ptr != (void *)entry_ptr) || + (!(entry_ptr->header.is_protected)) || + (!(entry_ptr->header.is_read_only)) || + (entry_ptr->header.ro_ref_count <= 0) || + (entry_ptr->header.type != &(types[type])) || + (entry_ptr->size != entry_ptr->header.size) || + (entry_ptr->addr != entry_ptr->header.addr) || + (entry_ptr->verify_ct != entry_ptr->max_verify_ct)) { + + pass = FALSE; + failure_mssg = "error from H5C_protect()."; + + } else { + + HDassert((entry_ptr->cache_ptr == NULL) || + (entry_ptr->cache_ptr == cache_ptr)); + + entry_ptr->cache_ptr = cache_ptr; + entry_ptr->file_ptr = file_ptr; + entry_ptr->is_protected = TRUE; + entry_ptr->is_read_only = TRUE; + entry_ptr->ro_ref_count++; + } + + HDassert(((entry_ptr->header).type)->id == type); + } + + if(pass) + unprotect_entry(file_ptr, VARIABLE_ENTRY_TYPE, idx, H5C__NO_FLAGS_SET); + + if(pass) { + entry_ptr = &(base_addr[++idx]); + + /* test case (2): + * --actual_len is greater the initial length from get_load_size() + * --verify_chksum() returns FALSE even after all tries is reached + * (file_ptr->shared->read_attempts is smaller then max_verify_ct) + */ + entry_ptr->actual_len = entry_ptr->size*2; + entry_ptr->max_verify_ct = 11; + entry_ptr->verify_ct = 0; + + cache_entry_ptr = (H5C_cache_entry_t *)H5C_protect(file_ptr, H5AC_ind_read_dxpl_id, + &(types[type]), entry_ptr->addr, &entry_ptr->addr, H5C__READ_ONLY_FLAG); + + /* H5C_protect() should fail after all retries fail */ + if(cache_entry_ptr != NULL) + pass = FALSE; + } + + + takedown_cache(file_ptr, FALSE, FALSE); + reset_entries(); + + if(pass) { PASSED(); } else { H5_FAILED(); } + + if(!pass) { + + HDfprintf(stdout, "%s: failure_msg = \"%s\".\n", + FUNC, failure_mssg); + } + + return (unsigned)!pass; + +} /* check_protect_retries() */ + + +/*------------------------------------------------------------------------- * Function: check_evictions_enabled_err() * * Purpose: Verify that H5C_get_evictions_enabled() and @@ -36190,6 +36319,7 @@ main(void) nerrs += check_resize_entry_errs(); nerrs += check_unprotect_ro_dirty_err(); nerrs += check_protect_ro_rw_err(); + nerrs += check_protect_retries(); nerrs += check_check_evictions_enabled_err(); nerrs += check_auto_cache_resize(FALSE); nerrs += check_auto_cache_resize(TRUE); diff --git a/test/cache_common.c b/test/cache_common.c index 94034eb..c0bee24 100644 --- a/test/cache_common.c +++ b/test/cache_common.c @@ -76,17 +76,22 @@ static test_entry_t *notify_entries = NULL, *orig_notify_entries = NULL; hbool_t orig_entry_arrays_init = FALSE; -static herr_t pico_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t nano_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t micro_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t tiny_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t small_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t medium_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t large_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t huge_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t monster_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t variable_get_load_size(const void *udata_ptr, size_t *image_len_ptr); -static herr_t notify_get_load_size(const void *udata_ptr, size_t *image_len_ptr); +static herr_t pico_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t nano_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t micro_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t tiny_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t small_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t medium_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t large_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t huge_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t monster_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t variable_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); +static herr_t notify_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); + +static herr_t variable_get_final_load_size(const void *image, size_t image_len, + void *udata, size_t *actual_len); + +static htri_t variable_verify_chksum(const void *image_ptr, size_t len, void *udata_ptr); static void *pico_deserialize(const void *image_ptr, size_t len, void *udata_ptr, hbool_t *dirty_ptr); @@ -111,62 +116,51 @@ static void *variable_deserialize(const void *image_ptr, size_t len, void *udata static void *notify_deserialize(const void *image_ptr, size_t len, void *udata_ptr, hbool_t *dirty_ptr); -static herr_t pico_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t nano_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t micro_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t tiny_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t small_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t medium_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t large_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t huge_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t monster_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t variable_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); -static herr_t notify_image_len(void *thing, size_t *image_len_ptr, - hbool_t *compressed_ptr, size_t * compressed_len_ptr); +static herr_t pico_image_len(const void *thing, size_t *image_len_ptr); +static herr_t nano_image_len(const void *thing, size_t *image_len_ptr); +static herr_t micro_image_len(const void *thing, size_t *image_len_ptr); +static herr_t tiny_image_len(const void *thing, size_t *image_len_ptr); +static herr_t small_image_len(const void *thing, size_t *image_len_ptr); +static herr_t medium_image_len(const void *thing, size_t *image_len_ptr); +static herr_t large_image_len(const void *thing, size_t *image_len_ptr); +static herr_t huge_image_len(const void *thing, size_t *image_len_ptr); +static herr_t monster_image_len(const void *thing, size_t *image_len_ptr); +static herr_t variable_image_len(const void *thing, size_t *image_len_ptr); +static herr_t notify_image_len(const void *thing, size_t *image_len_ptr); static herr_t pico_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t nano_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t micro_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t tiny_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t small_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t medium_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t large_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t huge_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t monster_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t variable_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t notify_pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, - haddr_t addr, size_t len, size_t compressed_len, haddr_t *new_addr_ptr, - size_t *new_len_ptr, size_t *new_compressed_len_ptr, unsigned *flags_ptr); + haddr_t addr, size_t len, haddr_t *new_addr_ptr, + size_t *new_len_ptr, unsigned *flags_ptr); static herr_t pico_serialize(const H5F_t *f, void *image_ptr, size_t len, void *thing); @@ -210,11 +204,13 @@ static void mark_flush_dep_dirty(test_entry_t * entry_ptr); static void mark_flush_dep_clean(test_entry_t * entry_ptr); /* Generic callback routines */ -static herr_t get_load_size(const void *udata_ptr, size_t *image_len_ptr, +static herr_t get_initial_load_size(void *udata_ptr, size_t *image_len_ptr, int32_t entry_type); +static herr_t get_final_load_size(const void *image, size_t image_len, + void *udata, size_t *actual_len, int32_t entry_type); static void *deserialize(const void *image_ptr, size_t len, void *udata_ptr, hbool_t *dirty_ptr, int32_t entry_type); -static herr_t image_len(void *thing, size_t *image_len_ptr, int32_t entry_type); +static herr_t image_len(const void *thing, size_t *image_len_ptr, int32_t entry_type); static herr_t pre_serialize(const H5F_t *f, hid_t dxpl_id, void *thing, haddr_t addr, size_t len, haddr_t *new_addr_ptr, size_t *new_len_ptr, unsigned *flags_ptr); @@ -317,165 +313,176 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] = "pico_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)pico_get_load_size, - (H5C_deserialize_func_t)pico_deserialize, - (H5C_image_len_func_t)pico_image_len, - (H5AC_pre_serialize_func_t)pico_pre_serialize, - (H5C_serialize_func_t)pico_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)pico_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + pico_get_initial_load_size, + NULL, + NULL, + pico_deserialize, + pico_image_len, + pico_pre_serialize, + pico_serialize, + NULL, + pico_free_icr, + NULL, }, { NANO_ENTRY_TYPE, "nano_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)nano_get_load_size, - (H5C_deserialize_func_t)nano_deserialize, - (H5C_image_len_func_t)nano_image_len, - (H5AC_pre_serialize_func_t)nano_pre_serialize, - (H5C_serialize_func_t)nano_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)nano_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + nano_get_initial_load_size, + NULL, + NULL, + nano_deserialize, + nano_image_len, + nano_pre_serialize, + nano_serialize, + NULL, + nano_free_icr, + NULL, }, { MICRO_ENTRY_TYPE, "micro_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)micro_get_load_size, - (H5C_deserialize_func_t)micro_deserialize, - (H5C_image_len_func_t)micro_image_len, - (H5AC_pre_serialize_func_t)micro_pre_serialize, - (H5C_serialize_func_t)micro_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)micro_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + micro_get_initial_load_size, + NULL, + NULL, + micro_deserialize, + micro_image_len, + micro_pre_serialize, + micro_serialize, + NULL, + micro_free_icr, + NULL, }, { TINY_ENTRY_TYPE, "tiny_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)tiny_get_load_size, - (H5C_deserialize_func_t)tiny_deserialize, - (H5C_image_len_func_t)tiny_image_len, - (H5AC_pre_serialize_func_t)tiny_pre_serialize, - (H5C_serialize_func_t)tiny_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)tiny_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + tiny_get_initial_load_size, + NULL, + NULL, + tiny_deserialize, + tiny_image_len, + tiny_pre_serialize, + tiny_serialize, + NULL, + tiny_free_icr, + NULL, }, { SMALL_ENTRY_TYPE, "small_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)small_get_load_size, - (H5C_deserialize_func_t)small_deserialize, - (H5C_image_len_func_t)small_image_len, - (H5AC_pre_serialize_func_t)small_pre_serialize, - (H5C_serialize_func_t)small_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)small_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + small_get_initial_load_size, + NULL, + NULL, + small_deserialize, + small_image_len, + small_pre_serialize, + small_serialize, + NULL, + small_free_icr, + NULL, }, { MEDIUM_ENTRY_TYPE, "medium_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)medium_get_load_size, - (H5C_deserialize_func_t)medium_deserialize, - (H5C_image_len_func_t)medium_image_len, - (H5AC_pre_serialize_func_t)medium_pre_serialize, - (H5C_serialize_func_t)medium_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)medium_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + medium_get_initial_load_size, + NULL, + NULL, + medium_deserialize, + medium_image_len, + medium_pre_serialize, + medium_serialize, + NULL, + medium_free_icr, + NULL, }, { LARGE_ENTRY_TYPE, "large_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)large_get_load_size, - (H5C_deserialize_func_t)large_deserialize, - (H5C_image_len_func_t)large_image_len, - (H5AC_pre_serialize_func_t)large_pre_serialize, - (H5C_serialize_func_t)large_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)large_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + large_get_initial_load_size, + NULL, + NULL, + large_deserialize, + large_image_len, + large_pre_serialize, + large_serialize, + NULL, + large_free_icr, + NULL, }, { HUGE_ENTRY_TYPE, "huge_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)huge_get_load_size, - (H5C_deserialize_func_t)huge_deserialize, - (H5C_image_len_func_t)huge_image_len, - (H5AC_pre_serialize_func_t)huge_pre_serialize, - (H5C_serialize_func_t)huge_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)huge_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + huge_get_initial_load_size, + NULL, + NULL, + huge_deserialize, + huge_image_len, + huge_pre_serialize, + huge_serialize, + NULL, + huge_free_icr, + NULL, }, { MONSTER_ENTRY_TYPE, "monster_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)monster_get_load_size, - (H5C_deserialize_func_t)monster_deserialize, - (H5C_image_len_func_t)monster_image_len, - (H5AC_pre_serialize_func_t)monster_pre_serialize, - (H5C_serialize_func_t)monster_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)monster_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + monster_get_initial_load_size, + NULL, + NULL, + monster_deserialize, + monster_image_len, + monster_pre_serialize, + monster_serialize, + NULL, + monster_free_icr, + NULL, }, { VARIABLE_ENTRY_TYPE, "variable_entry", H5FD_MEM_DEFAULT, H5C__CLASS_SPECULATIVE_LOAD_FLAG, - (H5C_get_load_size_func_t)variable_get_load_size, - (H5C_deserialize_func_t)variable_deserialize, - (H5C_image_len_func_t)variable_image_len, - (H5AC_pre_serialize_func_t)variable_pre_serialize, - (H5C_serialize_func_t)variable_serialize, - (H5C_notify_func_t)NULL, - (H5C_free_icr_func_t)variable_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + variable_get_initial_load_size, + variable_get_final_load_size, + variable_verify_chksum, + variable_deserialize, + variable_image_len, + variable_pre_serialize, + variable_serialize, + NULL, + variable_free_icr, + NULL, }, { NOTIFY_ENTRY_TYPE, "notify_entry", H5FD_MEM_DEFAULT, H5C__CLASS_NO_FLAGS_SET, - (H5C_get_load_size_func_t)notify_get_load_size, - (H5C_deserialize_func_t)notify_deserialize, - (H5C_image_len_func_t)notify_image_len, - (H5AC_pre_serialize_func_t)notify_pre_serialize, - (H5C_serialize_func_t)notify_serialize, - (H5C_notify_func_t)notify_notify, - (H5C_free_icr_func_t)notify_free_icr, - (H5C_clear_func_t)NULL, - (H5C_get_fsf_size_t)NULL, + notify_get_initial_load_size, + NULL, + NULL, + notify_deserialize, + notify_image_len, + notify_pre_serialize, + notify_serialize, + notify_notify, + notify_free_icr, + NULL, } }; @@ -592,10 +599,10 @@ check_write_permitted(const H5F_t H5_ATTR_UNUSED *f, hbool_t *write_permitted_pt /*------------------------------------------------------------------------- - * Function: get_load_size & friends + * Function: get_initial_load_size & friends * * Purpose: Query the image size for loading an entry. The helper - * functions funnel into get_load_size proper. + * functions funnel into get_initial_load_size proper. * * Return: SUCCEED * @@ -605,7 +612,7 @@ check_write_permitted(const H5F_t H5_ATTR_UNUSED *f, hbool_t *write_permitted_pt *------------------------------------------------------------------------- */ static herr_t -get_load_size(const void *udata, size_t *image_length, int32_t entry_type) +get_initial_load_size(void *udata, size_t *image_length, int32_t entry_type) { test_entry_t *entry; test_entry_t *base_addr; @@ -631,72 +638,185 @@ get_load_size(const void *udata, size_t *image_length, int32_t entry_type) *image_length = entry->size; return(SUCCEED); -} /* get_load_size() */ +} /* get_initial_load_size() */ static herr_t -pico_get_load_size(const void *udata, size_t *image_length) +pico_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, PICO_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, PICO_ENTRY_TYPE); } static herr_t -nano_get_load_size(const void *udata, size_t *image_length) +nano_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, NANO_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, NANO_ENTRY_TYPE); } static herr_t -micro_get_load_size(const void *udata, size_t *image_length) +micro_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, MICRO_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, MICRO_ENTRY_TYPE); } static herr_t -tiny_get_load_size(const void *udata, size_t *image_length) +tiny_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, TINY_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, TINY_ENTRY_TYPE); } static herr_t -small_get_load_size(const void *udata, size_t *image_length) +small_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, SMALL_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, SMALL_ENTRY_TYPE); } static herr_t -medium_get_load_size(const void *udata, size_t *image_length) +medium_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, MEDIUM_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, MEDIUM_ENTRY_TYPE); } static herr_t -large_get_load_size(const void *udata, size_t *image_length) +large_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, LARGE_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, LARGE_ENTRY_TYPE); } static herr_t -huge_get_load_size(const void *udata, size_t *image_length) +huge_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, HUGE_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, HUGE_ENTRY_TYPE); } static herr_t -monster_get_load_size(const void *udata, size_t *image_length) +monster_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, MONSTER_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, MONSTER_ENTRY_TYPE); } static herr_t -variable_get_load_size(const void *udata, size_t *image_length) +variable_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, VARIABLE_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, VARIABLE_ENTRY_TYPE); } static herr_t -notify_get_load_size(const void *udata, size_t *image_length) +notify_get_initial_load_size(void *udata, size_t *image_length) { - return get_load_size(udata, image_length, NOTIFY_ENTRY_TYPE); + return get_initial_load_size(udata, image_length, NOTIFY_ENTRY_TYPE); +} + + +/*------------------------------------------------------------------------- + * Function: get_final_load_size & friends + * + * Purpose: Query the final image size for loading an entry. The helper + * functions funnel into get_final_load_size proper. + * + * Return: SUCCEED + * + * Programmer: Quincey Koziol + * 11/18/16 + * + *------------------------------------------------------------------------- + */ +static herr_t +get_final_load_size(const void *image, size_t image_len, void *udata, + size_t *actual_len, int32_t entry_type) +{ + test_entry_t *entry; + test_entry_t *base_addr; + haddr_t addr = *(const haddr_t *)udata; + int32_t type; + int32_t idx; + + addr_to_type_and_index(addr, &type, &idx); + + base_addr = entries[type]; + entry = &(base_addr[idx]); + + HDassert(entry->type >= 0); + HDassert(entry->type == type); + HDassert(entry->type == entry_type); + HDassert(entry->type < NUMBER_OF_ENTRY_TYPES); + HDassert(entry->index == idx); + HDassert(entry->index >= 0); + HDassert(entry->index <= max_indices[type]); + HDassert(entry == entry->self); + HDassert(entry->addr == addr); + HDassert(type == VARIABLE_ENTRY_TYPE); + + /* Simulate SPECULATIVE read with a specified actual_len */ + if(entry->actual_len) { + *actual_len = entry->actual_len; + entry->size = entry->actual_len; + } /* end if */ + else + *actual_len = entry->size; + + return(SUCCEED); +} /* get_final_load_size() */ + +static herr_t +variable_get_final_load_size(const void *image, size_t image_len, + void *udata, size_t *actual_len) +{ + return get_final_load_size(image, image_len, udata, actual_len, VARIABLE_ENTRY_TYPE); +} + + +/*------------------------------------------------------------------------- + * Function: verify_chksum & friends + * (only done for VARIABLE_ENTRY_TYPE which has a speculative read) + * + * Purpose: Simulate checksum verification: + * --check is ok only after 'max_verify_ct' is reached + * --otherwise check is not ok + * + * Return: TRUE: checksum is ok + * FALSE: checksum is not ok + * + * Programmer: + * + *------------------------------------------------------------------------- + */ + +static htri_t +verify_chksum(const void H5_ATTR_UNUSED *image, size_t H5_ATTR_UNUSED len, void *udata, int32_t entry_type) +{ + test_entry_t *entry; + test_entry_t *base_addr; + haddr_t addr = *(const haddr_t *)udata; + int32_t type; + int32_t idx; + + addr_to_type_and_index(addr, &type, &idx); + + base_addr = entries[type]; + entry = &(base_addr[idx]); + + HDassert(entry->type >= 0); + HDassert(entry->type == type); + HDassert(entry->type == entry_type); + HDassert(entry->type < NUMBER_OF_ENTRY_TYPES); + HDassert(type == VARIABLE_ENTRY_TYPE); + HDassert(entry->index == idx); + HDassert(entry->index >= 0); + HDassert(entry->index <= max_indices[type]); + HDassert(entry == entry->self); + HDassert(entry->addr == addr); + + if(++entry->verify_ct >= entry->max_verify_ct) + return(TRUE); + else + return(FALSE); + +} /* verify_chksum() */ + +static htri_t +variable_verify_chksum(const void *image, size_t len, void *udata) +{ + return verify_chksum(image, len, udata, VARIABLE_ENTRY_TYPE); } @@ -872,9 +992,9 @@ notify_deserialize(const void *image, size_t len, void *udata, hbool_t *dirty) *------------------------------------------------------------------------- */ herr_t -image_len(void *thing, size_t *image_length, int32_t entry_type) +image_len(const void *thing, size_t *image_length, int32_t entry_type) { - test_entry_t *entry; + const test_entry_t *entry; test_entry_t *base_addr; int32_t type; int32_t idx; @@ -882,7 +1002,7 @@ image_len(void *thing, size_t *image_length, int32_t entry_type) HDassert(thing); HDassert(image_length); - entry = (test_entry_t *)thing; + entry = (const test_entry_t *)thing; HDassert(entry->self == entry); @@ -909,78 +1029,67 @@ image_len(void *thing, size_t *image_length, int32_t entry_type) } /* image_len() */ herr_t -pico_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +pico_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, PICO_ENTRY_TYPE); } herr_t -nano_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +nano_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, NANO_ENTRY_TYPE); } herr_t -micro_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +micro_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, MICRO_ENTRY_TYPE); } herr_t -tiny_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +tiny_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, TINY_ENTRY_TYPE); } herr_t -small_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +small_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, SMALL_ENTRY_TYPE); } herr_t -medium_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +medium_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, MEDIUM_ENTRY_TYPE); } herr_t -large_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +large_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, LARGE_ENTRY_TYPE); } herr_t -huge_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +huge_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, HUGE_ENTRY_TYPE); } herr_t -monster_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +monster_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, MONSTER_ENTRY_TYPE); } herr_t -variable_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +variable_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, VARIABLE_ENTRY_TYPE); } herr_t -notify_image_len(void *thing, size_t *image_length, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +notify_image_len(const void *thing, size_t *image_length) { return image_len(thing, image_length, NOTIFY_ENTRY_TYPE); } @@ -1095,10 +1204,8 @@ pico_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1111,10 +1218,8 @@ nano_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1127,10 +1232,8 @@ micro_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1143,10 +1246,8 @@ tiny_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1159,10 +1260,8 @@ small_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1175,10 +1274,8 @@ medium_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1191,10 +1288,8 @@ large_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1207,10 +1302,8 @@ huge_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1223,10 +1316,8 @@ monster_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1239,10 +1330,8 @@ variable_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -1255,10 +1344,8 @@ notify_pre_serialize(const H5F_t *f, void *thing, haddr_t addr, size_t len, - size_t H5_ATTR_UNUSED compressed_len, haddr_t *new_addr_ptr, size_t *new_len_ptr, - size_t H5_ATTR_UNUSED *new_compressed_len_ptr, unsigned *flags_ptr) { return pre_serialize(f, dxpl_id, thing, addr, len, @@ -2395,6 +2482,10 @@ reset_entries(void) base_addr[j].notify_after_insert_count = 0; base_addr[j].notify_before_evict_count = 0; + base_addr[j].actual_len = 0; + base_addr[j].max_verify_ct = 0; + base_addr[j].verify_ct = 0; + addr += (haddr_t)entry_size; alt_addr += (haddr_t)entry_size; } /* end for */ @@ -3958,6 +4049,10 @@ protect_entry(H5F_t * file_ptr, int32_t type, int32_t idx) HDfprintf(stdout, "entry_ptr->addr = %d, entry_ptr->header.addr = %d\n", (int)(entry_ptr->addr), (int)(entry_ptr->header.addr)); + HDfprintf(stdout, + "entry_ptr->verify_ct = %d, entry_ptr->max_verify_ct = %d\n", + entry_ptr->verify_ct, entry_ptr->max_verify_ct); + H5Eprint2(H5E_DEFAULT, stdout); #endif pass = FALSE; failure_mssg = "error in H5C_protect()."; diff --git a/test/cache_common.h b/test/cache_common.h index 811c35e..3abcab5 100644 --- a/test/cache_common.h +++ b/test/cache_common.h @@ -360,6 +360,9 @@ typedef struct test_entry_t unsigned notify_after_insert_count; /* Count of times that entry was inserted in cache */ unsigned notify_before_evict_count; /* Count of times that entry was removed in cache */ + unsigned actual_len; /* Simulate the entry's actual size for a speculative load */ + unsigned max_verify_ct; /* Maximum # of times to verify an entry's checksum */ + unsigned verify_ct; /* Count the # of checksum verification for an entry */ } test_entry_t; /* The following are cut down test versions of the hash table manipulation diff --git a/test/earray.c b/test/earray.c index 44d16c5..07acbb5 100644 --- a/test/earray.c +++ b/test/earray.c @@ -160,25 +160,12 @@ typedef struct earray_test_t { /* Local prototypes */ /* Metadata cache (H5AC) callbacks */ - -static herr_t earray_cache_test_get_load_size(const void *udata_ptr, - size_t *image_len_ptr); - -static void *earray_cache_test_deserialize(const void *image_ptr, - size_t len, - void *udata_ptr, - hbool_t *dirty_ptr); - -static herr_t earray_cache_test_image_len(const void *thing, - size_t *image_len_ptr, - hbool_t *compressed_ptr, - size_t * compressed_len_ptr); - -static herr_t earray_cache_test_serialize(const H5F_t *f, - void *image_ptr, - size_t len, - void *thing); - +static herr_t earray_cache_test_get_initial_load_size(void *udata, size_t *image_len); +static void *earray_cache_test_deserialize(const void *image_ptr, size_t len, + void *udata_ptr, hbool_t *dirty_ptr); +static herr_t earray_cache_test_image_len(const void *thing, size_t *image_len_ptr); +static herr_t earray_cache_test_serialize(const H5F_t *f, void *image_ptr, + size_t len, void *thing); static herr_t earray_cache_test_free_icr(void *thing); @@ -201,14 +188,15 @@ const H5AC_class_t H5AC_EARRAY_TEST[1] = {{ /* name */ "earray test", /* mem_type */ H5FD_MEM_DEFAULT, /* flags */ H5AC__CLASS_SKIP_READS | H5AC__CLASS_SKIP_WRITES, - /* get_load_size */ (H5AC_get_load_size_func_t)earray_cache_test_get_load_size, - /* deserialize */ (H5AC_deserialize_func_t)earray_cache_test_deserialize, - /* image_len */ (H5AC_image_len_func_t)earray_cache_test_image_len, - /* pre_serialize */ (H5AC_pre_serialize_func_t)NULL, - /* serialize */ (H5AC_serialize_func_t)earray_cache_test_serialize, - /* notify */ (H5AC_notify_func_t)NULL, - /* free_icr */ (H5AC_free_icr_func_t)earray_cache_test_free_icr, - /* clear */ NULL, + /* get_initial_load_size */ earray_cache_test_get_initial_load_size, + /* get_final_load_size */ NULL, + /* verify_chksum */ NULL, + /* deserialize */ earray_cache_test_deserialize, + /* image_len */ earray_cache_test_image_len, + /* pre_serialize */ NULL, + /* serialize */ earray_cache_test_serialize, + /* notify */ NULL, + /* free_icr */ earray_cache_test_free_icr, /* fsf_size */ NULL, }}; @@ -631,13 +619,12 @@ error: /*------------------------------------------------------------------------- - * Function: earray_cache_test_get_load_size() + * Function: earray_cache_test_get_initial_load_size() * * Purpose: place holder function -- should never be called * - * * A generic discussion of metadata cache callbacks of this type - * may be found in H5Cprivate.h: + * may be found in H5Cprivate.h. * * Return: Success: SUCCEED * Failure: FAIL @@ -648,19 +635,18 @@ error: *------------------------------------------------------------------------- */ static herr_t -earray_cache_test_get_load_size(const void *udata_ptr, size_t *image_len_ptr) +earray_cache_test_get_initial_load_size( void *udata, size_t *image_len) { - HDassert(udata_ptr); - HDassert(image_len_ptr); + HDassert(udata); + HDassert(image_len); /* Should never be called */ HDassert(0 && "Can't be called!"); - *image_len_ptr = 0; + *image_len = 0; return(SUCCEED); - -} /* end earray_cache_test_get_load_size() */ +} /* end earray_cache_test_get_initial_load_size() */ /*------------------------------------------------------------------------- @@ -691,12 +677,10 @@ earray_cache_test_deserialize(const void *image_ptr, HDassert(udata_ptr); HDassert(dirty_ptr); - /* Should never be called */ HDassert(0 && "Can't be called!"); return(NULL); - } /* end earray_cache_test_deserialize() */ @@ -719,10 +703,8 @@ earray_cache_test_deserialize(const void *image_ptr, *------------------------------------------------------------------------- */ static herr_t -earray_cache_test_image_len(const void *thing, size_t *image_len_ptr, - hbool_t H5_ATTR_UNUSED * compressed_ptr, size_t H5_ATTR_UNUSED * compressed_len_ptr) +earray_cache_test_image_len(const void *thing, size_t *image_len_ptr) { - HDassert(thing); HDassert(image_len_ptr); @@ -731,14 +713,9 @@ earray_cache_test_image_len(const void *thing, size_t *image_len_ptr, *image_len_ptr = 1; return(SUCCEED); - } /* end earray_cache_test_image_len() */ -/********************************/ -/* no H5O_cache_pre_serialize() */ -/********************************/ - /*------------------------------------------------------------------------- * Function: earray_cache_test_serialize @@ -764,14 +741,12 @@ earray_cache_test_serialize(const H5F_t *f, H5_ATTR_UNUSED size_t len, void *thing) { - earray_test_t *test = NULL; + earray_test_t *test; HDassert(f); HDassert(image_ptr); HDassert(thing); - test = (earray_test_t *)thing; - HDassert(test); HDassert(test->cache_info.magic == H5C__H5C_CACHE_ENTRY_T_MAGIC); HDassert((const H5AC_class_t *)(test->cache_info.type) == @@ -814,16 +789,10 @@ earray_cache_test_serialize(const H5F_t *f, return(SUCCEED); error: - return(FAIL); - } /* end earray_cache_test_serialize() */ -/******************************************/ -/* no earray_cache_test_notify() function */ -/******************************************/ - /*------------------------------------------------------------------------- * Function: earray_cache_test_free_icr @@ -846,12 +815,10 @@ error: static herr_t earray_cache_test_free_icr(void *thing) { - earray_test_t *test = NULL; + earray_test_t *test; HDassert(thing); - test = (earray_test_t *)thing; - HDassert(test); /* the metadata cache sets cache_info.magic to @@ -867,7 +834,6 @@ earray_cache_test_free_icr(void *thing) HDfree(test); return(SUCCEED); - } /* end earray_cache_test_free_icr() */ diff --git a/testpar/t_cache.c b/testpar/t_cache.c index 8b77ebe..066dd77 100644 --- a/testpar/t_cache.c +++ b/testpar/t_cache.c @@ -139,7 +139,8 @@ int total_writes = 0; * happen to overlap some collective operation. * * cleared: Boolean flag that is set to true whenever the entry is - * dirty, and is cleared via a call to datum_clear(). + * dirty, and is cleared via a call to datum_notify with the + * "entry cleaned" action. * * flushed: Boolean flag that is set to true whenever the entry is * dirty, and is flushed by the metadata cache. @@ -389,7 +390,7 @@ static hbool_t serve_rw_count_reset_request(struct mssg_t * mssg_ptr); /* call back functions & related data structures */ -static herr_t datum_get_load_size(const void * udata_ptr, +static herr_t datum_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr); static void * datum_deserialize(const void * image_ptr, @@ -397,10 +398,8 @@ static void * datum_deserialize(const void * image_ptr, void * udata_ptr, hbool_t * dirty_ptr); -static herr_t datum_image_len(void *thing, - size_t *image_len_ptr, - hbool_t *compressed_ptr, - size_t *compressed_len_ptr); +static herr_t datum_image_len(const void *thing, + size_t *image_len_ptr); static herr_t datum_serialize(const H5F_t *f, void *image_ptr, @@ -411,8 +410,6 @@ static herr_t datum_notify(H5C_notify_action_t action, void *thing); static herr_t datum_free_icr(void * thing); -static herr_t datum_clear(H5F_t * f, void * thing, hbool_t about_to_destroy); - #define DATUM_ENTRY_TYPE H5AC_TEST_ID #define NUMBER_OF_ENTRY_TYPES 1 @@ -439,15 +436,16 @@ const H5C_class_t types[NUMBER_OF_ENTRY_TYPES] = /* name */ "datum", /* mem_type */ H5FD_MEM_DEFAULT, /* flags */ H5AC__CLASS_SKIP_READS | H5AC__CLASS_SKIP_WRITES, - /* get_load_size */ (H5AC_get_load_size_func_t)datum_get_load_size, - /* deserialize */ (H5AC_deserialize_func_t)datum_deserialize, - /* image_len */ (H5AC_image_len_func_t)datum_image_len, - /* pre_serialize */ (H5AC_pre_serialize_func_t)NULL, - /* serialize */ (H5AC_serialize_func_t)datum_serialize, - /* notify */ (H5AC_notify_func_t)datum_notify, - /* free_icr */ (H5AC_free_icr_func_t)datum_free_icr, - /* clear */ (H5AC_clear_func_t)datum_clear, - /* fsf_size */ (H5AC_get_fsf_size_t)NULL, + /* get_initial_load_size */ datum_get_initial_load_size, + /* get_final_load_size */ NULL, + /* verify_chksum */ NULL, + /* deserialize */ datum_deserialize, + /* image_len */ datum_image_len, + /* pre_serialize */ NULL, + /* serialize */ datum_serialize, + /* notify */ datum_notify, + /* free_icr */ datum_free_icr, + /* fsf_size */ NULL, } }; @@ -2322,7 +2320,7 @@ serve_rw_count_reset_request(struct mssg_t * mssg_ptr) /*------------------------------------------------------------------------- - * Function: datum_get_load_size + * Function: datum_get_initial_load_size * * Purpose: Query the image size for an entry before deserializing it * @@ -2334,8 +2332,7 @@ serve_rw_count_reset_request(struct mssg_t * mssg_ptr) *------------------------------------------------------------------------- */ static herr_t -datum_get_load_size(const void * udata_ptr, - size_t *image_len_ptr) +datum_get_initial_load_size(void *udata_ptr, size_t *image_len_ptr) { haddr_t addr = *(haddr_t *)udata_ptr; int idx; @@ -2359,7 +2356,7 @@ datum_get_load_size(const void * udata_ptr, if ( callbacks_verbose ) { HDfprintf(stdout, - "%d: get_load_size() idx = %d, addr = %ld, len = %d.\n", + "%d: get_initial_load_size() idx = %d, addr = %ld, len = %d.\n", world_mpi_rank, idx, (long)addr, (int)entry_ptr->local_len); fflush(stdout); } @@ -2368,7 +2365,7 @@ datum_get_load_size(const void * udata_ptr, *image_len_ptr = entry_ptr->local_len; return(SUCCEED); -} /* get_load_size() */ +} /* get_initial_load_size() */ /*------------------------------------------------------------------------- @@ -2448,8 +2445,7 @@ datum_deserialize(const void * image_ptr, *------------------------------------------------------------------------- */ static herr_t -datum_image_len(void *thing, size_t *image_len, - hbool_t H5_ATTR_UNUSED *compressed_ptr, size_t H5_ATTR_UNUSED *compressed_len_ptr) +datum_image_len(const void *thing, size_t *image_len) { int idx; struct datum * entry_ptr; @@ -2896,7 +2892,16 @@ datum_notify(H5C_notify_action_t action, void *thing) fflush(stdout); } - /* do nothing */ + entry_ptr->cleared = TRUE; + entry_ptr->dirty = FALSE; + + datum_clears++; + + if(entry_ptr->header.is_pinned) { + datum_pinned_clears++; + HDassert( entry_ptr->global_pinned || entry_ptr->local_pinned ); + } /* end if */ + break; case H5AC_NOTIFY_ACTION_CHILD_DIRTIED: @@ -2994,62 +2999,6 @@ datum_free_icr(void * thing) } /* datum_free_icr() */ -/*------------------------------------------------------------------------- - * Function: datum_clear - * - * Purpose: Mark the datum as clean. - * - * Do not write it to the server, or increment the version. - * - * Return: SUCCEED - * - * Programmer: John Mainzer - * 12/29/05 - * - *------------------------------------------------------------------------- - */ -static herr_t -datum_clear(H5F_t H5_ATTR_UNUSED * f, - void * thing, - hbool_t H5_ATTR_UNUSED about_to_destroy) -{ - int idx; - struct datum * entry_ptr; - - HDassert( thing ); - - entry_ptr = (struct datum *)thing; - - idx = addr_to_datum_index(entry_ptr->base_addr); - - HDassert( idx >= 0 ); - HDassert( idx < NUM_DATA_ENTRIES ); - HDassert( idx < virt_num_data_entries ); - HDassert( &(data[idx]) == entry_ptr ); - - HDassert( entry_ptr->header.addr == entry_ptr->base_addr ); - HDassert( ( entry_ptr->header.size == entry_ptr->len ) || - ( entry_ptr->header.size == entry_ptr->local_len ) ); - - HDassert( ( entry_ptr->dirty ) || - ( entry_ptr->header.is_dirty == entry_ptr->dirty ) ); - - entry_ptr->cleared = TRUE; - entry_ptr->dirty = FALSE; - - datum_clears++; - - if ( entry_ptr->header.is_pinned ) { - - datum_pinned_clears++; - HDassert( entry_ptr->global_pinned || entry_ptr->local_pinned ); - } - - return(SUCCEED); - -} /* datum_clear() */ - - /*****************************************************************************/ /************************** test utility functions ***************************/ /*****************************************************************************/ diff --git a/tools/test/h5dump/h5dumpgentest.c b/tools/test/h5dump/h5dumpgentest.c index 315210b..335eafa 100644 --- a/tools/test/h5dump/h5dumpgentest.c +++ b/tools/test/h5dump/h5dumpgentest.c @@ -10129,75 +10129,75 @@ static void gent_compound_complex2(void) H5Sclose(space); } -// /* CompoundComplex2D */ -// if ((type = H5Tcreate(H5T_COMPOUND, sizeof(s_t))) >= 0) { -// H5Tinsert(type, "a", HOFFSET(s_t, a), H5T_STD_B8LE); -// H5Tinsert(type, "b", HOFFSET(s_t, b), H5T_STD_B16LE); -// H5Tinsert(type, "c", HOFFSET(s_t, c), H5T_STD_B32LE); -// H5Tinsert(type, "d", HOFFSET(s_t, d), H5T_STD_B64LE); -// if ((space = H5Screate_simple(F82_RANK2, &nelmts, NULL)) >= 0) { -// if ((dset = H5Dcreate2(file, "compound_1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) >= 0) { -// for(i = 0; i < nelmts; i++) { -// buf5[i].a = (unsigned char)0xff ^ (unsigned char)i; -// buf5[i].b = (unsigned int)0xffff ^ (unsigned int)(i * 16); -// buf5[i].c = (unsigned long)0xffffffff ^ (unsigned long)(i * 32); -// buf5[i].d = (unsigned long long)0xffffffffffffffff ^ (unsigned long long)(i * 64); -// } -// -// H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf5); -// H5Dclose(dset); -// } -// H5Sclose(space); -// } -// H5Tclose(type); -// } -// -// /* CompoundComplex3D */ -// if ((type = H5Tcreate(H5T_COMPOUND, sizeof(s_t))) >= 0) { -// H5Tinsert(type, "a", HOFFSET(s_t, a), H5T_STD_B8LE); -// H5Tinsert(type, "b", HOFFSET(s_t, b), H5T_STD_B16LE); -// H5Tinsert(type, "c", HOFFSET(s_t, c), H5T_STD_B32LE); -// H5Tinsert(type, "d", HOFFSET(s_t, d), H5T_STD_B64LE); -// if ((space = H5Screate_simple(F82_RANK3, &nelmts, NULL)) >= 0) { -// if ((dset = H5Dcreate2(file, "compound_1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) >= 0) { -// for(i = 0; i < nelmts; i++) { -// buf5[i].a = (unsigned char)0xff ^ (unsigned char)i; -// buf5[i].b = (unsigned int)0xffff ^ (unsigned int)(i * 16); -// buf5[i].c = (unsigned long)0xffffffff ^ (unsigned long)(i * 32); -// buf5[i].d = (unsigned long long)0xffffffffffffffff ^ (unsigned long long)(i * 64); -// } -// -// H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf5); -// H5Dclose(dset); -// } -// H5Sclose(space); -// } -// H5Tclose(type); -// } -// -// /* CompoundComplex4D */ -// if ((type = H5Tcreate(H5T_COMPOUND, sizeof(s_t))) >= 0) { -// H5Tinsert(type, "a", HOFFSET(s_t, a), H5T_STD_B8LE); -// H5Tinsert(type, "b", HOFFSET(s_t, b), H5T_STD_B16LE); -// H5Tinsert(type, "c", HOFFSET(s_t, c), H5T_STD_B32LE); -// H5Tinsert(type, "d", HOFFSET(s_t, d), H5T_STD_B64LE); -// if ((space = H5Screate_simple(F82_RANK4, &nelmts, NULL)) >= 0) { -// if ((dset = H5Dcreate2(file, "compound_1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) >= 0) { -// for(i = 0; i < nelmts; i++) { -// buf5[i].a = (unsigned char)0xff ^ (unsigned char)i; -// buf5[i].b = (unsigned int)0xffff ^ (unsigned int)(i * 16); -// buf5[i].c = (unsigned long)0xffffffff ^ (unsigned long)(i * 32); -// buf5[i].d = (unsigned long long)0xffffffffffffffff ^ (unsigned long long)(i * 64); -// } -// -// H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf5); -// H5Dclose(dset); -// } -// H5Sclose(space); -// } -// H5Tclose(type); -// } + /* CompoundComplex2D */ +/* if ((type = H5Tcreate(H5T_COMPOUND, sizeof(s_t))) >= 0) { + H5Tinsert(type, "a", HOFFSET(s_t, a), H5T_STD_B8LE); + H5Tinsert(type, "b", HOFFSET(s_t, b), H5T_STD_B16LE); + H5Tinsert(type, "c", HOFFSET(s_t, c), H5T_STD_B32LE); + H5Tinsert(type, "d", HOFFSET(s_t, d), H5T_STD_B64LE); + if ((space = H5Screate_simple(F82_RANK2, &nelmts, NULL)) >= 0) { + if ((dset = H5Dcreate2(file, "compound_1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) >= 0) { + for(i = 0; i < nelmts; i++) { + buf5[i].a = (unsigned char)0xff ^ (unsigned char)i; + buf5[i].b = (unsigned int)0xffff ^ (unsigned int)(i * 16); + buf5[i].c = (unsigned long)0xffffffff ^ (unsigned long)(i * 32); + buf5[i].d = (unsigned long long)0xffffffffffffffff ^ (unsigned long long)(i * 64); + } + + H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf5); + H5Dclose(dset); + } + H5Sclose(space); + } + H5Tclose(type); + } +*/ + /* CompoundComplex3D */ +/* if ((type = H5Tcreate(H5T_COMPOUND, sizeof(s_t))) >= 0) { + H5Tinsert(type, "a", HOFFSET(s_t, a), H5T_STD_B8LE); + H5Tinsert(type, "b", HOFFSET(s_t, b), H5T_STD_B16LE); + H5Tinsert(type, "c", HOFFSET(s_t, c), H5T_STD_B32LE); + H5Tinsert(type, "d", HOFFSET(s_t, d), H5T_STD_B64LE); + if ((space = H5Screate_simple(F82_RANK3, &nelmts, NULL)) >= 0) { + if ((dset = H5Dcreate2(file, "compound_1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) >= 0) { + for(i = 0; i < nelmts; i++) { + buf5[i].a = (unsigned char)0xff ^ (unsigned char)i; + buf5[i].b = (unsigned int)0xffff ^ (unsigned int)(i * 16); + buf5[i].c = (unsigned long)0xffffffff ^ (unsigned long)(i * 32); + buf5[i].d = (unsigned long long)0xffffffffffffffff ^ (unsigned long long)(i * 64); + } + H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf5); + H5Dclose(dset); + } + H5Sclose(space); + } + H5Tclose(type); + } +*/ + /* CompoundComplex4D */ +/* if ((type = H5Tcreate(H5T_COMPOUND, sizeof(s_t))) >= 0) { + H5Tinsert(type, "a", HOFFSET(s_t, a), H5T_STD_B8LE); + H5Tinsert(type, "b", HOFFSET(s_t, b), H5T_STD_B16LE); + H5Tinsert(type, "c", HOFFSET(s_t, c), H5T_STD_B32LE); + H5Tinsert(type, "d", HOFFSET(s_t, d), H5T_STD_B64LE); + if ((space = H5Screate_simple(F82_RANK4, &nelmts, NULL)) >= 0) { + if ((dset = H5Dcreate2(file, "compound_1", type, space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT)) >= 0) { + for(i = 0; i < nelmts; i++) { + buf5[i].a = (unsigned char)0xff ^ (unsigned char)i; + buf5[i].b = (unsigned int)0xffff ^ (unsigned int)(i * 16); + buf5[i].c = (unsigned long)0xffffffff ^ (unsigned long)(i * 32); + buf5[i].d = (unsigned long long)0xffffffffffffffff ^ (unsigned long long)(i * 64); + } + + H5Dwrite(dset, type, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf5); + H5Dclose(dset); + } + H5Sclose(space); + } + H5Tclose(type); + } +*/ H5Fclose(file); HDfree(buf); |