From dfd71ce12683526f1fdb6abeb43d4a1a107cfe49 Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Tue, 22 Aug 2006 12:13:17 -0500 Subject: [svn-r12610] Description: Add support for checksumming fractal heaps. This is always enabled for the heap header and indirect blocks (as they are "pure" metadata) and is optional for direct blocks, since they may be used for "raw" data. Also, rearrange direct block routines in H5HFcache.c to be in a more sensible location in the file. (probably should have been a separate checkin, since the diffs are mostly useless for this checkin... *sigh*) Tested on: Linux/32 2.6 (chicago) Linux/64 2.6 (chicago2) --- src/H5HFcache.c | 969 ++++++++++++++++++++++++++++-------------------------- src/H5HFhdr.c | 1 + src/H5HFpkg.h | 45 +-- src/H5HFprivate.h | 1 + test/fheap.c | 20 +- 5 files changed, 549 insertions(+), 487 deletions(-) diff --git a/src/H5HFcache.c b/src/H5HFcache.c index e0b0c16..2f3a78e 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -62,23 +62,23 @@ /* Local Prototypes */ /********************/ -/* Metadata cache callbacks */ +/* Local encode/decode routines */ +static herr_t H5HF_dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable); +static herr_t H5HF_dtable_decode(H5F_t *f, const uint8_t **pp, H5HF_dtable_t *dtable); + +/* Metadata cache (H5AC) callbacks */ static H5HF_hdr_t *H5HF_cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *udata, void *udata2); static herr_t H5HF_cache_hdr_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5HF_hdr_t *hdr); static herr_t H5HF_cache_hdr_clear(H5F_t *f, H5HF_hdr_t *hdr, hbool_t destroy); static herr_t H5HF_cache_hdr_size(const H5F_t *f, const H5HF_hdr_t *hdr, size_t *size_ptr); -static H5HF_direct_t *H5HF_cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *udata, void *udata2); -static herr_t H5HF_cache_dblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5HF_direct_t *dblock); -static herr_t H5HF_cache_dblock_clear(H5F_t *f, H5HF_direct_t *dblock, hbool_t destroy); -static herr_t H5HF_cache_dblock_size(const H5F_t *f, const H5HF_direct_t *dblock, size_t *size_ptr); static H5HF_indirect_t *H5HF_cache_iblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *udata, void *udata2); static herr_t H5HF_cache_iblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5HF_indirect_t *iblock); static herr_t H5HF_cache_iblock_clear(H5F_t *f, H5HF_indirect_t *iblock, hbool_t destroy); static herr_t H5HF_cache_iblock_size(const H5F_t *f, const H5HF_indirect_t *iblock, size_t *size_ptr); - -/* Local encode/decode routines */ -static herr_t H5HF_dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable); -static herr_t H5HF_dtable_decode(H5F_t *f, const uint8_t **pp, H5HF_dtable_t *dtable); +static H5HF_direct_t *H5HF_cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *udata, void *udata2); +static herr_t H5HF_cache_dblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5HF_direct_t *dblock); +static herr_t H5HF_cache_dblock_clear(H5F_t *f, H5HF_direct_t *dblock, hbool_t destroy); +static herr_t H5HF_cache_dblock_size(const H5F_t *f, const H5HF_direct_t *dblock, size_t *size_ptr); /*********************/ /* Package Variables */ @@ -94,16 +94,6 @@ const H5AC_class_t H5AC_FHEAP_HDR[1] = {{ (H5AC_size_func_t)H5HF_cache_hdr_size, }}; -/* H5HF direct block inherits cache-like properties from H5AC */ -const H5AC_class_t H5AC_FHEAP_DBLOCK[1] = {{ - H5AC_FHEAP_DBLOCK_ID, - (H5AC_load_func_t)H5HF_cache_dblock_load, - (H5AC_flush_func_t)H5HF_cache_dblock_flush, - (H5AC_dest_func_t)H5HF_cache_dblock_dest, - (H5AC_clear_func_t)H5HF_cache_dblock_clear, - (H5AC_size_func_t)H5HF_cache_dblock_size, -}}; - /* H5HF indirect block inherits cache-like properties from H5AC */ const H5AC_class_t H5AC_FHEAP_IBLOCK[1] = {{ H5AC_FHEAP_IBLOCK_ID, @@ -114,6 +104,16 @@ const H5AC_class_t H5AC_FHEAP_IBLOCK[1] = {{ (H5AC_size_func_t)H5HF_cache_iblock_size, }}; +/* H5HF direct block inherits cache-like properties from H5AC */ +const H5AC_class_t H5AC_FHEAP_DBLOCK[1] = {{ + H5AC_FHEAP_DBLOCK_ID, + (H5AC_load_func_t)H5HF_cache_dblock_load, + (H5AC_flush_func_t)H5HF_cache_dblock_flush, + (H5AC_dest_func_t)H5HF_cache_dblock_dest, + (H5AC_clear_func_t)H5HF_cache_dblock_clear, + (H5AC_size_func_t)H5HF_cache_dblock_size, +}}; + /*****************************/ /* Library Private Variables */ @@ -257,7 +257,8 @@ H5HF_cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED *ud size_t size; /* Header size */ uint8_t *buf = NULL; /* Temporary buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t metadata_chksum; /* Metadata checksum value */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ uint8_t heap_flags; /* Status flags for heap */ H5HF_hdr_t *ret_value; /* Return value */ @@ -277,7 +278,7 @@ HDfprintf(stderr, "%s: Load heap header, addr = %a\n", FUNC, addr); /* Set the heap header's address */ hdr->heap_addr = addr; - /* Compute the size of the fractal heap header on disk */ + /* Compute the 'base' size of the fractal heap header on disk */ size = H5HF_HEADER_SIZE(hdr); /* Allocate temporary buffer */ @@ -304,12 +305,6 @@ HDfprintf(stderr, "%s: Load heap header, addr = %a\n", FUNC, addr); if(*p++ != 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "unknown metadata flag in fractal heap header") - /* Metadata checksum (unused, currently) */ - UINT32DECODE(p, metadata_chksum); -/* XXX: Verify checksum */ - if(metadata_chksum != 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "incorrect metadata checksum for fractal heap header") - /* General heap information */ UINT16DECODE(p, hdr->id_len); /* Heap ID length */ UINT16DECODE(p, hdr->filter_len); /* I/O filters' encoded length */ @@ -318,6 +313,7 @@ HDfprintf(stderr, "%s: Load heap header, addr = %a\n", FUNC, addr); /* (bit 0: "huge" object IDs have wrapped) */ heap_flags = *p++; hdr->huge_ids_wrapped = heap_flags & H5HF_HDR_FLAGS_HUGE_ID_WRAPPED; + hdr->checksum_dblocks = heap_flags & H5HF_HDR_FLAGS_CHECKSUM_DBLOCKS; /* "Huge" object information */ UINT32DECODE(p, hdr->max_man_size); /* Max. size of "managed" objects */ @@ -343,29 +339,37 @@ HDfprintf(stderr, "%s: Load heap header, addr = %a\n", FUNC, addr); HGOTO_ERROR(H5E_HEAP, H5E_CANTENCODE, NULL, "unable to encode managed obj. doubling table info") /* Sanity check */ - HDassert((size_t)(p - buf) == size); + /* (allow for checksum not decoded yet) */ + HDassert((size_t)(p - buf) == (size - H5HF_SIZEOF_CHKSUM)); /* Check for I/O filter information to decode */ if(hdr->filter_len > 0) { - size_t filter_info_size; /* Size of filter information */ - H5O_pline_t *pline; /* Pipeline information from the header on disk */ + size_t filter_info_off; /* Offset in header of filter information */ + size_t filter_info_size; /* Size of filter information */ + H5O_pline_t *pline; /* Pipeline information from the header on disk */ + + /* Compute the offset of the filter info in the header */ + filter_info_off = p - buf; /* Compute the size of the extra filter information */ filter_info_size = hdr->sizeof_size /* Size of size for filtered root direct block */ + 4 /* Size of filter mask for filtered root direct block */ + hdr->filter_len; /* Size of encoded I/O filter info */ - /* Check if the current buffer can be re-used, or needs to be re-sized */ - if(filter_info_size > size) { - if((buf = H5FL_BLK_REALLOC(header_block, buf, filter_info_size)) == NULL) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "can't allocate space to decode I/O pipeline filters") - } /* end if */ + /* Compute the heap header's size */ + hdr->heap_size = size + filter_info_size; + + /* Re-size current buffer */ + if((buf = H5FL_BLK_REALLOC(header_block, buf, hdr->heap_size)) == NULL) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "can't allocate space to decode I/O pipeline filters") /* Read in I/O filter information */ - if(H5F_block_read(f, H5FD_MEM_FHEAP_HDR, (addr + size), filter_info_size, dxpl_id, buf) < 0) + /* (and the checksum) */ + if(H5F_block_read(f, H5FD_MEM_FHEAP_HDR, (addr + filter_info_off), (filter_info_size + H5HF_SIZEOF_CHKSUM), dxpl_id, (buf + filter_info_off)) < 0) HGOTO_ERROR(H5E_HEAP, H5E_READERROR, NULL, "can't read fractal heap header's I/O pipeline filter info") - p = buf; + /* Point at correct offset in header for the filter information */ + p = buf + filter_info_off; /* Decode the size of a filtered root direct block */ H5F_DECODE_LENGTH(f, p, hdr->pline_root_direct_size); @@ -376,6 +380,7 @@ HDfprintf(stderr, "%s: Load heap header, addr = %a\n", FUNC, addr); /* Decode I/O filter information */ if(NULL == (pline = H5O_decode(hdr->f, p, H5O_PLINE_ID))) HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode I/O pipeline filters") + p += hdr->filter_len; /* Copy the information into the header's I/O pipeline structure */ if(NULL == H5O_copy(H5O_PLINE_ID, pline, &(hdr->pline))) @@ -383,14 +388,25 @@ HDfprintf(stderr, "%s: Load heap header, addr = %a\n", FUNC, addr); /* Release the space allocated for the I/O pipeline filters */ H5O_free(H5O_PLINE_ID, pline); - - /* Compute the heap header's size */ - hdr->heap_size = size + filter_info_size; } /* end if */ else /* Set the heap header's size */ hdr->heap_size = size; + /* Metadata checksum */ + UINT32DECODE(p, stored_chksum); + + /* Sanity check */ + HDassert((size_t)(p - buf) == hdr->heap_size); + + /* Compute checksum on entire header */ + /* (including the filter information, if present) */ + computed_chksum = H5_fletcher32(buf, (hdr->heap_size - H5HF_SIZEOF_CHKSUM)); + + /* Verify checksum */ + if(stored_chksum != computed_chksum) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "incorrect metadata checksum for fractal heap header") + /* Finish initialization of heap header */ if(H5HF_hdr_finish_init(hdr) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, NULL, "can't finish initializing shared fractal heap header") @@ -444,6 +460,7 @@ HDfprintf(stderr, "%s: Flushing heap header, addr = %a, destroy = %u\n", FUNC, a uint8_t *p; /* Pointer into raw data buffer */ size_t size; /* Header size on disk */ uint8_t heap_flags; /* Status flags for heap */ + uint32_t metadata_chksum; /* Computed metadata checksum value */ /* Sanity check */ HDassert(hdr->dirty); @@ -468,11 +485,6 @@ HDfprintf(stderr, "%s: Flushing heap header, addr = %a, destroy = %u\n", FUNC, a /* XXX: Set this? */ *p++ = 0; - /* Metadata checksum */ -/* XXX: Set this! (After all the metadata is in the buffer) */ - HDmemset(p, 0, (size_t)4); - p += 4; - /* General heap information */ UINT16ENCODE(p, hdr->id_len); /* Heap ID length */ UINT16ENCODE(p, hdr->filter_len); /* I/O filters' encoded length */ @@ -481,6 +493,7 @@ HDfprintf(stderr, "%s: Flushing heap header, addr = %a, destroy = %u\n", FUNC, a /* (bit 0: "huge" object IDs have wrapped) */ heap_flags = 0; heap_flags |= (hdr->huge_ids_wrapped ? H5HF_HDR_FLAGS_HUGE_ID_WRAPPED : 0); + heap_flags |= (hdr->checksum_dblocks ? H5HF_HDR_FLAGS_CHECKSUM_DBLOCKS : 0); *p++ = heap_flags; /* "Huge" object information */ @@ -520,6 +533,12 @@ HDfprintf(stderr, "%s: Flushing heap header, addr = %a, destroy = %u\n", FUNC, a p += hdr->filter_len; } /* end if */ + /* Compute metadata checksum */ + metadata_chksum = H5_fletcher32(buf, hdr->heap_size - H5HF_SIZEOF_CHKSUM); + + /* Metadata checksum */ + UINT32ENCODE(p, metadata_chksum); + /* Write the heap header. */ HDassert((size_t)(p - buf) == size); if(H5F_block_write(f, H5FD_MEM_FHEAP_HDR, addr, size, dxpl_id, buf) < 0) @@ -649,11 +668,11 @@ H5HF_cache_hdr_size(const H5F_t UNUSED *f, const H5HF_hdr_t *hdr, size_t *size_p /*------------------------------------------------------------------------- - * Function: H5HF_cache_dblock_load + * Function: H5HF_cache_iblock_load * - * Purpose: Loads a fractal heap direct block from the disk. + * Purpose: Loads a fractal heap indirect block from the disk. * - * Return: Success: Pointer to a new fractal heap direct block + * Return: Success: Pointer to a new fractal heap indirect block * * Failure: NULL * @@ -663,68 +682,71 @@ H5HF_cache_hdr_size(const H5F_t UNUSED *f, const H5HF_hdr_t *hdr, size_t *size_p * *------------------------------------------------------------------------- */ -static H5HF_direct_t * -H5HF_cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *_size, void *_par_info) +static H5HF_indirect_t * +H5HF_cache_iblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *_nrows, void *_par_info) { - const size_t *size = (const size_t *)_size; /* Size of block */ H5HF_hdr_t *hdr; /* Shared fractal heap information */ - H5HF_parent_t *par_info = (H5HF_parent_t *)_par_info; /* Pointer to parent information */ - H5HF_direct_t *dblock = NULL; /* Direct block info */ + const unsigned *nrows = (const unsigned *)_nrows; /* # of rows in indirect block */ + H5HF_parent_t *par_info = (H5HF_parent_t *)_par_info; /* Shared parent information */ + H5HF_indirect_t *iblock = NULL; /* Indirect block info */ + uint8_t *buf = NULL; /* Temporary buffer */ const uint8_t *p; /* Pointer into raw data buffer */ haddr_t heap_addr; /* Address of heap header in the file */ - uint32_t metadata_chksum; /* Metadata checksum value */ - H5HF_direct_t *ret_value; /* Return value */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + size_t u; /* Local index variable */ + H5HF_indirect_t *ret_value; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_dblock_load) + FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_iblock_load) +#ifdef QAK +HDfprintf(stderr, "%s: Load indirect block, addr = %a\n", FUNC, addr); +#endif /* QAK */ /* Check arguments */ HDassert(f); HDassert(H5F_addr_defined(addr)); HDassert(par_info); - /* Allocate space for the fractal heap direct block */ - if(NULL == (dblock = H5FL_MALLOC(H5HF_direct_t))) + /* Allocate space for the fractal heap indirect block */ + if(NULL == (iblock = H5FL_MALLOC(H5HF_indirect_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - HDmemset(&dblock->cache_info, 0, sizeof(H5AC_info_t)); + HDmemset(&iblock->cache_info, 0, sizeof(H5AC_info_t)); /* Get the pointer to the shared heap header */ hdr = par_info->hdr; /* Share common heap information */ - dblock->hdr = hdr; + iblock->hdr = hdr; if(H5HF_hdr_incr(hdr) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINC, NULL, "can't increment reference count on shared heap header") -#ifdef LATER - /* Check for I/O filters on this heap */ - if(hdr->filter_len > 0) { -HDfprintf(stderr, "%s: I/O filters not supported yet!\n", FUNC); -HGOTO_ERROR(H5E_HEAP, H5E_UNSUPPORTED, NULL, "I/O filters not supported yet") - } /* end if */ -#endif /* LATER */ - /* Set block's internal information */ - dblock->size = *size; - dblock->blk_off_size = H5HF_SIZEOF_OFFSET_LEN(dblock->size); + iblock->rc = 0; + iblock->nrows = *nrows; + iblock->addr = addr; + iblock->nchildren = 0; - /* Allocate block buffer */ -/* XXX: Change to using free-list factories */ - if((dblock->blk = H5FL_BLK_MALLOC(direct_block, (size_t)dblock->size)) == NULL) + /* Compute size of indirect block */ + iblock->size = H5HF_MAN_INDIRECT_SIZE(hdr, iblock); + + /* Allocate buffer to decode block */ +/* XXX: Use free list factories? */ + if((buf = H5FL_BLK_MALLOC(indirect_block, iblock->size)) == NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Read direct block from disk */ - if(H5F_block_read(f, H5FD_MEM_FHEAP_DBLOCK, addr, (size_t)dblock->size, dxpl_id, dblock->blk) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_READERROR, NULL, "can't read fractal heap direct block") + /* Read indirect block from disk */ + if(H5F_block_read(f, H5FD_MEM_FHEAP_IBLOCK, addr, iblock->size, dxpl_id, buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_READERROR, NULL, "can't read fractal heap indirect block") - p = dblock->blk; + p = buf; /* Magic number */ - if(HDmemcmp(p, H5HF_DBLOCK_MAGIC, (size_t)H5HF_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "wrong fractal heap direct block signature") + if(HDmemcmp(p, H5HF_IBLOCK_MAGIC, (size_t)H5HF_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "wrong fractal heap indirect block signature") p += H5HF_SIZEOF_MAGIC; /* Version */ - if(*p++ != H5HF_DBLOCK_VERSION) + if(*p++ != H5HF_IBLOCK_VERSION) HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "wrong fractal heap direct block version") /* Metadata flags (unused, currently) */ @@ -732,243 +754,370 @@ HGOTO_ERROR(H5E_HEAP, H5E_UNSUPPORTED, NULL, "I/O filters not supported yet") if(*p++ != 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "unknown metadata flag in fractal heap direct block") - /* Metadata checksum (unused, currently) */ - UINT32DECODE(p, metadata_chksum); -/* XXX: Verify checksum */ - if(metadata_chksum != 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "incorrect metadata checksum for fractal heap direct block") - - /* Address of heap that owns this block (skip) */ + /* Address of heap that owns this block */ H5F_addr_decode(f, &p, &heap_addr); if(H5F_addr_ne(heap_addr, hdr->heap_addr)) HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "incorrect heap header address for direct block") /* Address of parent block */ - dblock->parent = par_info->iblock; - dblock->par_entry = par_info->entry; - if(dblock->parent) { + iblock->parent = par_info->iblock; + iblock->par_entry = par_info->entry; + if(iblock->parent) { /* Share parent block */ - if(H5HF_iblock_incr(dblock->parent) < 0) + if(H5HF_iblock_incr(iblock->parent) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINC, NULL, "can't increment reference count on shared indirect block") + + /* Set max. # of rows in this block */ + iblock->max_rows = iblock->nrows; } /* end if */ + else { + /* Set max. # of rows in this block */ + iblock->max_rows = hdr->man_dtable.max_root_rows; + } /* end else */ /* Offset of heap within the heap's address space */ - UINT64DECODE_VAR(p, dblock->block_off, hdr->heap_off_size); + UINT64DECODE_VAR(p, iblock->block_off, hdr->heap_off_size); + + /* Allocate & decode child block entry tables */ + HDassert(iblock->nrows > 0); + if(NULL == (iblock->ents = H5FL_SEQ_MALLOC(H5HF_indirect_ent_t, (size_t)(iblock->nrows * hdr->man_dtable.cparam.width)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for direct entries") + if(hdr->filter_len > 0) { + unsigned dir_rows; /* Number of direct rows in this indirect block */ + + /* Compute the number of direct rows for this indirect block */ + dir_rows = MIN(iblock->nrows, hdr->man_dtable.max_direct_rows); + + /* Allocate indirect block filtered entry array */ + if(NULL == (iblock->filt_ents = H5FL_SEQ_MALLOC(H5HF_indirect_filt_ent_t, (size_t)(dir_rows * hdr->man_dtable.cparam.width)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for block entries") + } /* end if */ + else + iblock->filt_ents = NULL; + for(u = 0; u < (iblock->nrows * hdr->man_dtable.cparam.width); u++) { + /* Decode child block address */ + H5F_addr_decode(f, &p, &(iblock->ents[u].addr)); + + /* Check for heap with I/O filters */ + if(hdr->filter_len > 0) { + /* Sanity check */ + HDassert(iblock->filt_ents); + + /* Decode extra information for direct blocks */ + if(u < (hdr->man_dtable.max_direct_rows * hdr->man_dtable.cparam.width)) { + /* Size of filtered direct block */ + H5F_DECODE_LENGTH(f, p, iblock->filt_ents[u].size); + + /* Sanity check */ + /* (either both the address & size are defined or both are + * not defined) + */ + HDassert((H5F_addr_defined(iblock->ents[u].addr) && iblock->filt_ents[u].size) + || (!H5F_addr_defined(iblock->ents[u].addr) && iblock->filt_ents[u].size == 0)); + + /* I/O filter mask for filtered direct block */ + UINT32DECODE(p, iblock->filt_ents[u].filter_mask); + } /* end if */ + } /* end if */ + + /* Count child blocks */ + if(H5F_addr_defined(iblock->ents[u].addr)) { + iblock->nchildren++; + iblock->max_child = u; + } /* end if */ +#ifdef QAK +HDfprintf(stderr, "%s: iblock->ents[%Zu] = {%a}\n", FUNC, u, iblock->ents[u].addr); +#endif /* QAK */ + } /* end for */ + + /* Sanity check */ + HDassert(iblock->nchildren); /* indirect blocks w/no children should have been deleted */ + + /* Metadata checksum */ + UINT32DECODE(p, stored_chksum); + + /* Sanity check */ + HDassert((size_t)(p - buf) == iblock->size); + + /* Compute checksum on indirect block */ + computed_chksum = H5_fletcher32(buf, (iblock->size - H5HF_SIZEOF_CHKSUM)); + + /* Verify checksum */ + if(stored_chksum != computed_chksum) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "incorrect metadata checksum for fractal heap indirect block") + + /* Check if we have any indirect block children */ + if(iblock->nrows > hdr->man_dtable.max_direct_rows) { + unsigned indir_rows; /* Number of indirect rows in this indirect block */ + + /* Compute the number of indirect rows for this indirect block */ + indir_rows = iblock->nrows - hdr->man_dtable.max_direct_rows; + + /* Allocate & initialize child indirect block pointer array */ + if(NULL == (iblock->child_iblocks = H5FL_SEQ_CALLOC(H5HF_indirect_ptr_t, (size_t)(indir_rows * hdr->man_dtable.cparam.width)))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for block entries") + } /* end if */ + else + iblock->child_iblocks = NULL; /* Set return value */ - ret_value = dblock; + ret_value = iblock; done: - if(!ret_value && dblock) - (void)H5HF_cache_dblock_dest(f, dblock); + /* Free buffer */ +/* XXX: Keep buffer around? */ + H5FL_BLK_FREE(indirect_block, buf); + + if(!ret_value && iblock) + (void)H5HF_cache_iblock_dest(f, iblock); FUNC_LEAVE_NOAPI(ret_value) -} /* end H5HF_cache_dblock_load() */ +} /* end H5HF_cache_iblock_load() */ /*------------------------------------------------------------------------- - * Function: H5HF_cache_dblock_flush + * Function: H5HF_cache_iblock_flush * - * Purpose: Flushes a dirty fractal heap direct block to disk. + * Purpose: Flushes a dirty fractal heap indirect block to disk. * * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol * koziol@ncsa.uiuc.edu - * Feb 27 2006 + * Mar 6 2006 * *------------------------------------------------------------------------- */ static herr_t -H5HF_cache_dblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5HF_direct_t *dblock) +H5HF_cache_iblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5HF_indirect_t *iblock) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_dblock_flush) + FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_iblock_flush) +#ifdef QAK +HDfprintf(stderr, "%s: Flushing indirect block, addr = %a, destroy = %u\n", FUNC, addr, (unsigned)destroy); +#endif /* QAK */ /* check arguments */ HDassert(f); HDassert(H5F_addr_defined(addr)); - HDassert(dblock); - - if(dblock->cache_info.is_dirty) { - H5HF_hdr_t *hdr; /* Shared fractal heap information */ - void *write_buf; /* Pointer to buffer to write out */ - size_t write_size; /* Size of buffer to write out */ - uint8_t *p; /* Pointer into raw data buffer */ + HDassert(iblock); - /* Get the pointer to the shared heap header */ - hdr = dblock->hdr; + if(iblock->cache_info.is_dirty) { + H5HF_hdr_t *hdr; /* Shared fractal heap information */ + uint8_t *buf = NULL; /* Temporary buffer */ + uint8_t *p; /* Pointer into raw data buffer */ +#ifndef NDEBUG + unsigned nchildren = 0; /* Track # of children */ + unsigned max_child = 0; /* Track max. child entry used */ +#endif /* NDEBUG */ + uint32_t metadata_chksum; /* Computed metadata checksum value */ + size_t u; /* Local index variable */ - HDassert(dblock->blk); - p = dblock->blk; + /* Get the pointer to the shared heap header */ + hdr = iblock->hdr; + + /* Allocate buffer to encode block */ +/* XXX: Use free list factories? */ +#ifdef QAK +HDfprintf(stderr, "%s: iblock->nrows = %u\n", FUNC, iblock->nrows); +HDfprintf(stderr, "%s: iblock->size = %Zu\n", FUNC, iblock->size); +HDfprintf(stderr, "%s: iblock->block_off = %Hu\n", FUNC, iblock->block_off); +HDfprintf(stderr, "%s: hdr->man_dtable.cparam.width = %u\n", FUNC, hdr->man_dtable.cparam.width); +#endif /* QAK */ + if((buf = H5FL_BLK_MALLOC(indirect_block, iblock->size)) == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + p = buf; /* Magic number */ - HDmemcpy(p, H5HF_DBLOCK_MAGIC, (size_t)H5HF_SIZEOF_MAGIC); + HDmemcpy(p, H5HF_IBLOCK_MAGIC, (size_t)H5HF_SIZEOF_MAGIC); p += H5HF_SIZEOF_MAGIC; /* Version # */ - *p++ = H5HF_DBLOCK_VERSION; + *p++ = H5HF_IBLOCK_VERSION; /* Metadata status flags */ /* XXX: Set this? */ *p++ = 0; - /* Metadata checksum */ -/* XXX: Set this! (After all the metadata is in the buffer) */ - HDmemset(p, 0, (size_t)4); - p += 4; - /* Address of heap header for heap which owns this block */ H5F_addr_encode(f, &p, hdr->heap_addr); /* Offset of block in heap */ - UINT64ENCODE_VAR(p, dblock->block_off, hdr->heap_off_size); + UINT64ENCODE_VAR(p, iblock->block_off, hdr->heap_off_size); - /* Sanity check */ - HDassert((size_t)(p - dblock->blk) == H5HF_MAN_ABS_DIRECT_OVERHEAD(hdr)); + /* Encode indirect block-specific fields */ + for(u = 0; u < (iblock->nrows * hdr->man_dtable.cparam.width); u++) { +#ifdef QAK +HDfprintf(stderr, "%s: iblock->ents[%Zu] = {%a}\n", FUNC, u, iblock->ents[u].addr); +#endif /* QAK */ + /* Encode child block address */ + H5F_addr_encode(f, &p, iblock->ents[u].addr); -#ifdef LATER - /* Check for I/O filters on this heap */ - if(hdr->filter_len > 0) { - H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ - size_t nbytes; /* Number of bytes used */ - unsigned filter_mask; /* Filter mask for block */ + /* Check for heap with I/O filters */ + if(hdr->filter_len > 0) { + /* Sanity check */ + HDassert(iblock->filt_ents); - /* Allocate buffer to perform I/O filtering on */ - write_size = dblock->size; - if(NULL == (write_buf = H5MM_malloc(write_size))) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline buffer") - HDmemcpy(write_buf, dblock->blk, write_size); + /* Encode extra information for direct blocks */ + if(u < (hdr->man_dtable.max_direct_rows * hdr->man_dtable.cparam.width)) { +#ifdef QAK +HDfprintf(stderr, "%s: iblock->filt_ents[%Zu] = {%Zu, %x}\n", FUNC, u, iblock->filt_ents[u].size, iblock->filt_ents[u].filter_mask); +#endif /* QAK */ + /* Sanity check */ + /* (either both the address & size are defined or both are + * not defined) + */ + HDassert((H5F_addr_defined(iblock->ents[u].addr) && iblock->filt_ents[u].size) + || (!H5F_addr_defined(iblock->ents[u].addr) && iblock->filt_ents[u].size == 0)); - /* Push direct block data through I/O filter pipeline */ - nbytes = write_size; - if(H5Z_pipeline(&(hdr->pline), 0, &filter_mask, H5Z_ENABLE_EDC, - filter_cb, &nbytes, &write_size, &write_buf) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_WRITEERROR, FAIL, "output pipeline failed") -HDfprintf(stderr, "%s: nbytes = %Zu, write_size = %Zu, write_buf = %p\n", FUNC, nbytes, write_size, write_buf); -HDfprintf(stderr, "%s: dblock->size = %Zu, dblock->blk = %p\n", FUNC, dblock->size, dblock->blk); + /* Size of filtered direct block */ + H5F_ENCODE_LENGTH(f, p, iblock->filt_ents[u].size); -HDfprintf(stderr, "%s: I/O filters not supported yet!\n", FUNC); -HGOTO_ERROR(H5E_HEAP, H5E_UNSUPPORTED, FAIL, "I/O filters not supported yet") - } /* end if */ - else { -#endif /* LATER */ - write_buf = dblock->blk; - write_size = dblock->size; -#ifdef LATER - } /* end else */ -#endif /* LATER */ + /* I/O filter mask for filtered direct block */ + UINT32ENCODE(p, iblock->filt_ents[u].filter_mask); + } /* end if */ + } /* end if */ - /* Write the direct block */ - if(H5F_block_write(f, H5FD_MEM_FHEAP_DBLOCK, addr, write_size, dxpl_id, write_buf) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTFLUSH, FAIL, "unable to save fractal heap direct block to disk") +#ifndef NDEBUG + /* Count child blocks */ + if(H5F_addr_defined(iblock->ents[u].addr)) { + nchildren++; + if(u > max_child) + max_child = u; + } /* end if */ +#endif /* NDEBUG */ + } /* end for */ - /* Release the write buffer, if it was allocated */ - if(write_buf != dblock->blk) - H5MM_xfree(write_buf); + /* Compute checksum */ + metadata_chksum = H5_fletcher32(buf, iblock->size - H5HF_SIZEOF_CHKSUM); - dblock->cache_info.is_dirty = FALSE; + /* Metadata checksum */ + UINT32ENCODE(p, metadata_chksum); + + /* Sanity check */ + HDassert((size_t)(p - buf) == iblock->size); +#ifndef NDEBUG + HDassert(nchildren == iblock->nchildren); + HDassert(max_child == iblock->max_child); +#endif /* NDEBUG */ + + /* Write the indirect block */ + if(H5F_block_write(f, H5FD_MEM_FHEAP_IBLOCK, addr, iblock->size, dxpl_id, buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFLUSH, FAIL, "unable to save fractal heap indirect block to disk") + + /* Free buffer */ + H5FL_BLK_FREE(indirect_block, buf); + + /* Reset dirty flags */ + iblock->cache_info.is_dirty = FALSE; } /* end if */ if(destroy) - if(H5HF_cache_dblock_dest(f, dblock) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to destroy fractal heap direct block") + if(H5HF_cache_iblock_dest(f, iblock) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to destroy fractal heap indirect block") done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5HF_cache_dblock_flush() */ +} /* H5HF_cache_iblock_flush() */ /*------------------------------------------------------------------------- - * Function: H5HF_cache_dblock_dest + * Function: H5HF_cache_iblock_dest * - * Purpose: Destroys a fractal heap direct block in memory. + * Purpose: Destroys a fractal heap indirect block in memory. * * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol * koziol@ncsa.uiuc.edu - * Feb 27 2006 + * Mar 6 2006 * *------------------------------------------------------------------------- */ /* ARGSUSED */ herr_t -H5HF_cache_dblock_dest(H5F_t UNUSED *f, H5HF_direct_t *dblock) +H5HF_cache_iblock_dest(H5F_t UNUSED *f, H5HF_indirect_t *iblock) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_dblock_dest) + FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_iblock_dest) /* * Check arguments. */ - HDassert(dblock); + HDassert(iblock); + HDassert(iblock->rc == 0); #ifdef QAK -HDfprintf(stderr, "%s: Destroying direct block, dblock = %p\n", FUNC, dblock); +HDfprintf(stderr, "%s: Destroying indirect block\n", FUNC); #endif /* QAK */ - /* Decrement reference count on shared fractal heap info */ - HDassert(dblock->hdr); - if(H5HF_hdr_decr(dblock->hdr) < 0) + /* Decrement reference count on shared info */ + HDassert(iblock->hdr); + if(H5HF_hdr_decr(iblock->hdr) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTDEC, FAIL, "can't decrement reference count on shared heap header") - if(dblock->parent) - if(H5HF_iblock_decr(dblock->parent) < 0) + if(iblock->parent) + if(H5HF_iblock_decr(iblock->parent) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTDEC, FAIL, "can't decrement reference count on shared indirect block") - /* Free block's buffer */ - H5FL_BLK_FREE(direct_block, dblock->blk); + /* Release entry tables */ + if(iblock->ents) + H5FL_SEQ_FREE(H5HF_indirect_ent_t, iblock->ents); + if(iblock->filt_ents) + H5FL_SEQ_FREE(H5HF_indirect_filt_ent_t, iblock->filt_ents); + if(iblock->child_iblocks) + H5FL_SEQ_FREE(H5HF_indirect_ptr_t, iblock->child_iblocks); - /* Free fractal heap direct block info */ - H5FL_FREE(H5HF_direct_t, dblock); + /* Free fractal heap indirect block info */ + H5FL_FREE(H5HF_indirect_t, iblock); done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5HF_cache_dblock_dest() */ +} /* end H5HF_cache_iblock_dest() */ /*------------------------------------------------------------------------- - * Function: H5HF_cache_dblock_clear + * Function: H5HF_cache_iblock_clear * - * Purpose: Mark a fractal heap direct block in memory as non-dirty. + * Purpose: Mark a fractal heap indirect block in memory as non-dirty. * * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol * koziol@ncsa.uiuc.edu - * Feb 27 2006 + * Mar 6 2006 * *------------------------------------------------------------------------- */ static herr_t -H5HF_cache_dblock_clear(H5F_t *f, H5HF_direct_t *dblock, hbool_t destroy) +H5HF_cache_iblock_clear(H5F_t *f, H5HF_indirect_t *iblock, hbool_t destroy) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_dblock_clear) + FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_iblock_clear) /* * Check arguments. */ - HDassert(dblock); + HDassert(iblock); /* Reset the dirty flag. */ - dblock->cache_info.is_dirty = FALSE; + iblock->cache_info.is_dirty = FALSE; if(destroy) - if(H5HF_cache_dblock_dest(f, dblock) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to destroy fractal heap direct block") + if(H5HF_cache_iblock_dest(f, iblock) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to destroy fractal heap indirect block") done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5HF_cache_dblock_clear() */ +} /* end H5HF_cache_iblock_clear() */ /*------------------------------------------------------------------------- - * Function: H5HF_cache_dblock_size + * Function: H5HF_cache_iblock_size * - * Purpose: Compute the size in bytes of a fractal heap direct block + * Purpose: Compute the size in bytes of a fractal heap indirect block * on disk, and return it in *size_ptr. On failure, * the value of *size_ptr is undefined. * @@ -976,32 +1125,32 @@ done: * * Programmer: Quincey Koziol * koziol@ncsa.uiuc.edu - * Feb 24 2006 + * Mar 6 2006 * *------------------------------------------------------------------------- */ static herr_t -H5HF_cache_dblock_size(const H5F_t UNUSED *f, const H5HF_direct_t *dblock, size_t *size_ptr) +H5HF_cache_iblock_size(const H5F_t UNUSED *f, const H5HF_indirect_t *iblock, size_t *size_ptr) { - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5HF_cache_dblock_size) + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5HF_cache_iblock_size) /* check arguments */ - HDassert(dblock); + HDassert(iblock); HDassert(size_ptr); /* Set size value */ - *size_ptr = dblock->size; + *size_ptr = iblock->size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5HF_cache_dblock_size() */ +} /* H5HF_cache_iblock_size() */ /*------------------------------------------------------------------------- - * Function: H5HF_cache_iblock_load + * Function: H5HF_cache_dblock_load * - * Purpose: Loads a fractal heap indirect block from the disk. + * Purpose: Loads a fractal heap direct block from the disk. * - * Return: Success: Pointer to a new fractal heap indirect block + * Return: Success: Pointer to a new fractal heap direct block * * Failure: NULL * @@ -1011,70 +1160,67 @@ H5HF_cache_dblock_size(const H5F_t UNUSED *f, const H5HF_direct_t *dblock, size_ * *------------------------------------------------------------------------- */ -static H5HF_indirect_t * -H5HF_cache_iblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *_nrows, void *_par_info) +static H5HF_direct_t * +H5HF_cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void *_size, void *_par_info) { + const size_t *size = (const size_t *)_size; /* Size of block */ H5HF_hdr_t *hdr; /* Shared fractal heap information */ - const unsigned *nrows = (const unsigned *)_nrows; /* # of rows in indirect block */ - H5HF_parent_t *par_info = (H5HF_parent_t *)_par_info; /* Shared parent information */ - H5HF_indirect_t *iblock = NULL; /* Indirect block info */ - uint8_t *buf = NULL; /* Temporary buffer */ + H5HF_parent_t *par_info = (H5HF_parent_t *)_par_info; /* Pointer to parent information */ + H5HF_direct_t *dblock = NULL; /* Direct block info */ const uint8_t *p; /* Pointer into raw data buffer */ haddr_t heap_addr; /* Address of heap header in the file */ - uint32_t metadata_chksum; /* Metadata checksum value */ - size_t u; /* Local index variable */ - H5HF_indirect_t *ret_value; /* Return value */ + H5HF_direct_t *ret_value; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_iblock_load) -#ifdef QAK -HDfprintf(stderr, "%s: Load indirect block, addr = %a\n", FUNC, addr); -#endif /* QAK */ + FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_dblock_load) /* Check arguments */ HDassert(f); HDassert(H5F_addr_defined(addr)); HDassert(par_info); - /* Allocate space for the fractal heap indirect block */ - if(NULL == (iblock = H5FL_MALLOC(H5HF_indirect_t))) + /* Allocate space for the fractal heap direct block */ + if(NULL == (dblock = H5FL_MALLOC(H5HF_direct_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - HDmemset(&iblock->cache_info, 0, sizeof(H5AC_info_t)); + HDmemset(&dblock->cache_info, 0, sizeof(H5AC_info_t)); /* Get the pointer to the shared heap header */ hdr = par_info->hdr; /* Share common heap information */ - iblock->hdr = hdr; + dblock->hdr = hdr; if(H5HF_hdr_incr(hdr) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINC, NULL, "can't increment reference count on shared heap header") - /* Set block's internal information */ - iblock->rc = 0; - iblock->nrows = *nrows; - iblock->addr = addr; - iblock->nchildren = 0; +#ifdef LATER + /* Check for I/O filters on this heap */ + if(hdr->filter_len > 0) { +HDfprintf(stderr, "%s: I/O filters not supported yet!\n", FUNC); +HGOTO_ERROR(H5E_HEAP, H5E_UNSUPPORTED, NULL, "I/O filters not supported yet") + } /* end if */ +#endif /* LATER */ - /* Compute size of indirect block */ - iblock->size = H5HF_MAN_INDIRECT_SIZE(hdr, iblock); + /* Set block's internal information */ + dblock->size = *size; + dblock->blk_off_size = H5HF_SIZEOF_OFFSET_LEN(dblock->size); - /* Allocate buffer to decode block */ -/* XXX: Use free list factories? */ - if((buf = H5FL_BLK_MALLOC(indirect_block, iblock->size)) == NULL) + /* Allocate block buffer */ +/* XXX: Change to using free-list factories */ + if((dblock->blk = H5FL_BLK_MALLOC(direct_block, (size_t)dblock->size)) == NULL) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Read indirect block from disk */ - if(H5F_block_read(f, H5FD_MEM_FHEAP_IBLOCK, addr, iblock->size, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_READERROR, NULL, "can't read fractal heap indirect block") + /* Read direct block from disk */ + if(H5F_block_read(f, H5FD_MEM_FHEAP_DBLOCK, addr, (size_t)dblock->size, dxpl_id, dblock->blk) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_READERROR, NULL, "can't read fractal heap direct block") - p = buf; + p = dblock->blk; /* Magic number */ - if(HDmemcmp(p, H5HF_IBLOCK_MAGIC, (size_t)H5HF_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "wrong fractal heap indirect block signature") + if(HDmemcmp(p, H5HF_DBLOCK_MAGIC, (size_t)H5HF_SIZEOF_MAGIC)) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "wrong fractal heap direct block signature") p += H5HF_SIZEOF_MAGIC; /* Version */ - if(*p++ != H5HF_IBLOCK_VERSION) + if(*p++ != H5HF_DBLOCK_VERSION) HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "wrong fractal heap direct block version") /* Metadata flags (unused, currently) */ @@ -1082,362 +1228,269 @@ HDfprintf(stderr, "%s: Load indirect block, addr = %a\n", FUNC, addr); if(*p++ != 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "unknown metadata flag in fractal heap direct block") - /* Metadata checksum (unused, currently) */ - UINT32DECODE(p, metadata_chksum); -/* XXX: Verify checksum */ - if(metadata_chksum != 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "incorrect metadata checksum for fractal heap direct block") - - /* Address of heap that owns this block */ + /* Address of heap that owns this block (just for file integrity checks) */ H5F_addr_decode(f, &p, &heap_addr); if(H5F_addr_ne(heap_addr, hdr->heap_addr)) HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "incorrect heap header address for direct block") /* Address of parent block */ - iblock->parent = par_info->iblock; - iblock->par_entry = par_info->entry; - if(iblock->parent) { + dblock->parent = par_info->iblock; + dblock->par_entry = par_info->entry; + if(dblock->parent) { /* Share parent block */ - if(H5HF_iblock_incr(iblock->parent) < 0) + if(H5HF_iblock_incr(dblock->parent) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTINC, NULL, "can't increment reference count on shared indirect block") - - /* Set max. # of rows in this block */ - iblock->max_rows = iblock->nrows; } /* end if */ - else { - /* Set max. # of rows in this block */ - iblock->max_rows = hdr->man_dtable.max_root_rows; - } /* end else */ - /* Offset of heap within the heap's address space */ - UINT64DECODE_VAR(p, iblock->block_off, hdr->heap_off_size); - - /* Allocate & decode child block entry tables */ - HDassert(iblock->nrows > 0); - if(NULL == (iblock->ents = H5FL_SEQ_MALLOC(H5HF_indirect_ent_t, (size_t)(iblock->nrows * hdr->man_dtable.cparam.width)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for direct entries") - if(hdr->filter_len > 0) { - unsigned dir_rows; /* Number of direct rows in this indirect block */ - - /* Compute the number of direct rows for this indirect block */ - dir_rows = MIN(iblock->nrows, hdr->man_dtable.max_direct_rows); - - /* Allocate indirect block filtered entry array */ - if(NULL == (iblock->filt_ents = H5FL_SEQ_MALLOC(H5HF_indirect_filt_ent_t, (size_t)(dir_rows * hdr->man_dtable.cparam.width)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for block entries") - } /* end if */ - else - iblock->filt_ents = NULL; - for(u = 0; u < (iblock->nrows * hdr->man_dtable.cparam.width); u++) { - /* Decode child block address */ - H5F_addr_decode(f, &p, &(iblock->ents[u].addr)); - - /* Check for heap with I/O filters */ - if(hdr->filter_len > 0) { - /* Sanity check */ - HDassert(iblock->filt_ents); - - /* Decode extra information for direct blocks */ - if(u < (hdr->man_dtable.max_direct_rows * hdr->man_dtable.cparam.width)) { - /* Size of filtered direct block */ - H5F_DECODE_LENGTH(f, p, iblock->filt_ents[u].size); - - /* Sanity check */ - /* (either both the address & size are defined or both are - * not defined) - */ - HDassert((H5F_addr_defined(iblock->ents[u].addr) && iblock->filt_ents[u].size) - || (!H5F_addr_defined(iblock->ents[u].addr) && iblock->filt_ents[u].size == 0)); - - /* I/O filter mask for filtered direct block */ - UINT32DECODE(p, iblock->filt_ents[u].filter_mask); - } /* end if */ - } /* end if */ - - /* Count child blocks */ - if(H5F_addr_defined(iblock->ents[u].addr)) { - iblock->nchildren++; - iblock->max_child = u; - } /* end if */ -#ifdef QAK -HDfprintf(stderr, "%s: iblock->ents[%Zu] = {%a}\n", FUNC, u, iblock->ents[u].addr); -#endif /* QAK */ - } /* end for */ + /* Offset of heap within the heap's address space */ + UINT64DECODE_VAR(p, dblock->block_off, hdr->heap_off_size); - /* Check if we have any indirect block children */ - if(iblock->nrows > hdr->man_dtable.max_direct_rows) { - unsigned indir_rows; /* Number of indirect rows in this indirect block */ + /* Encode checksum on direct block, if requested */ + if(hdr->checksum_dblocks) { + uint32_t stored_chksum; /* Metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ - /* Compute the number of indirect rows for this indirect block */ - indir_rows = iblock->nrows - hdr->man_dtable.max_direct_rows; + /* Metadata checksum */ + UINT32DECODE(p, stored_chksum); - /* Allocate & initialize child indirect block pointer array */ - if(NULL == (iblock->child_iblocks = H5FL_SEQ_CALLOC(H5HF_indirect_ptr_t, (size_t)(indir_rows * hdr->man_dtable.cparam.width)))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed for block entries") + /* Reset checksum field, for computing the checksum */ + /* (Casting away const OK - QAK) */ + HDmemset((uint8_t *)p - H5HF_SIZEOF_CHKSUM, 0, (size_t)H5HF_SIZEOF_CHKSUM); + + /* Compute checksum on entire direct block */ + computed_chksum = H5_fletcher32(dblock->blk, dblock->size); + + /* Verify checksum */ + if(stored_chksum != computed_chksum) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, NULL, "incorrect metadata checksum for fractal heap direct block") } /* end if */ - else - iblock->child_iblocks = NULL; - /* Sanity checks */ - HDassert((size_t)(p - buf) == iblock->size); - HDassert(iblock->nchildren); /* indirect blocks w/no children should have been deleted */ + /* Sanity check */ + HDassert((size_t)(p - dblock->blk) == H5HF_MAN_ABS_DIRECT_OVERHEAD(hdr)); /* Set return value */ - ret_value = iblock; + ret_value = dblock; done: - /* Free buffer */ -/* XXX: Keep buffer around? */ - H5FL_BLK_FREE(indirect_block, buf); - - if(!ret_value && iblock) - (void)H5HF_cache_iblock_dest(f, iblock); + if(!ret_value && dblock) + (void)H5HF_cache_dblock_dest(f, dblock); FUNC_LEAVE_NOAPI(ret_value) -} /* end H5HF_cache_iblock_load() */ +} /* end H5HF_cache_dblock_load() */ /*------------------------------------------------------------------------- - * Function: H5HF_cache_iblock_flush + * Function: H5HF_cache_dblock_flush * - * Purpose: Flushes a dirty fractal heap indirect block to disk. + * Purpose: Flushes a dirty fractal heap direct block to disk. * * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol * koziol@ncsa.uiuc.edu - * Mar 6 2006 + * Feb 27 2006 * *------------------------------------------------------------------------- */ static herr_t -H5HF_cache_iblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5HF_indirect_t *iblock) +H5HF_cache_dblock_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5HF_direct_t *dblock) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_iblock_flush) -#ifdef QAK -HDfprintf(stderr, "%s: Flushing indirect block, addr = %a, destroy = %u\n", FUNC, addr, (unsigned)destroy); -#endif /* QAK */ + FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_dblock_flush) /* check arguments */ HDassert(f); HDassert(H5F_addr_defined(addr)); - HDassert(iblock); + HDassert(dblock); - if(iblock->cache_info.is_dirty) { - H5HF_hdr_t *hdr; /* Shared fractal heap information */ - uint8_t *buf = NULL; /* Temporary buffer */ - uint8_t *p; /* Pointer into raw data buffer */ -#ifndef NDEBUG - unsigned nchildren = 0; /* Track # of children */ - unsigned max_child = 0; /* Track max. child entry used */ -#endif /* NDEBUG */ - size_t u; /* Local index variable */ + if(dblock->cache_info.is_dirty) { + H5HF_hdr_t *hdr; /* Shared fractal heap information */ + void *write_buf; /* Pointer to buffer to write out */ + size_t write_size; /* Size of buffer to write out */ + uint8_t *p; /* Pointer into raw data buffer */ /* Get the pointer to the shared heap header */ - hdr = iblock->hdr; - - /* Allocate buffer to encode block */ -/* XXX: Use free list factories? */ -#ifdef QAK -HDfprintf(stderr, "%s: iblock->nrows = %u\n", FUNC, iblock->nrows); -HDfprintf(stderr, "%s: iblock->size = %Zu\n", FUNC, iblock->size); -HDfprintf(stderr, "%s: iblock->block_off = %Hu\n", FUNC, iblock->block_off); -HDfprintf(stderr, "%s: hdr->man_dtable.cparam.width = %u\n", FUNC, hdr->man_dtable.cparam.width); -#endif /* QAK */ - if((buf = H5FL_BLK_MALLOC(indirect_block, iblock->size)) == NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + hdr = dblock->hdr; - p = buf; + HDassert(dblock->blk); + p = dblock->blk; /* Magic number */ - HDmemcpy(p, H5HF_IBLOCK_MAGIC, (size_t)H5HF_SIZEOF_MAGIC); + HDmemcpy(p, H5HF_DBLOCK_MAGIC, (size_t)H5HF_SIZEOF_MAGIC); p += H5HF_SIZEOF_MAGIC; /* Version # */ - *p++ = H5HF_IBLOCK_VERSION; + *p++ = H5HF_DBLOCK_VERSION; /* Metadata status flags */ /* XXX: Set this? */ *p++ = 0; - /* Metadata checksum */ -/* XXX: Set this! (After all the metadata is in the buffer) */ - HDmemset(p, 0, (size_t)4); - p += 4; - /* Address of heap header for heap which owns this block */ H5F_addr_encode(f, &p, hdr->heap_addr); /* Offset of block in heap */ - UINT64ENCODE_VAR(p, iblock->block_off, hdr->heap_off_size); + UINT64ENCODE_VAR(p, dblock->block_off, hdr->heap_off_size); - /* Encode indirect block-specific fields */ - for(u = 0; u < (iblock->nrows * hdr->man_dtable.cparam.width); u++) { -#ifdef QAK -HDfprintf(stderr, "%s: iblock->ents[%Zu] = {%a}\n", FUNC, u, iblock->ents[u].addr); -#endif /* QAK */ - /* Encode child block address */ - H5F_addr_encode(f, &p, iblock->ents[u].addr); + /* Metadata checksum */ + if(hdr->checksum_dblocks) { + uint32_t metadata_chksum; /* Computed metadata checksum value */ - /* Check for heap with I/O filters */ - if(hdr->filter_len > 0) { - /* Sanity check */ - HDassert(iblock->filt_ents); + /* Clear the checksum field, to compute the checksum */ + HDmemset(p, 0, (size_t)H5HF_SIZEOF_CHKSUM); - /* Encode extra information for direct blocks */ - if(u < (hdr->man_dtable.max_direct_rows * hdr->man_dtable.cparam.width)) { -#ifdef QAK -HDfprintf(stderr, "%s: iblock->filt_ents[%Zu] = {%Zu, %x}\n", FUNC, u, iblock->filt_ents[u].size, iblock->filt_ents[u].filter_mask); -#endif /* QAK */ - /* Sanity check */ - /* (either both the address & size are defined or both are - * not defined) - */ - HDassert((H5F_addr_defined(iblock->ents[u].addr) && iblock->filt_ents[u].size) - || (!H5F_addr_defined(iblock->ents[u].addr) && iblock->filt_ents[u].size == 0)); + /* Compute checksum on entire direct block */ + metadata_chksum = H5_fletcher32(dblock->blk, dblock->size); - /* Size of filtered direct block */ - H5F_ENCODE_LENGTH(f, p, iblock->filt_ents[u].size); + /* Metadata checksum */ + UINT32ENCODE(p, metadata_chksum); + } /* end if */ - /* I/O filter mask for filtered direct block */ - UINT32ENCODE(p, iblock->filt_ents[u].filter_mask); - } /* end if */ - } /* end if */ + /* Sanity check */ + HDassert((size_t)(p - dblock->blk) == H5HF_MAN_ABS_DIRECT_OVERHEAD(hdr)); -#ifndef NDEBUG - /* Count child blocks */ - if(H5F_addr_defined(iblock->ents[u].addr)) { - nchildren++; - if(u > max_child) - max_child = u; - } /* end if */ -#endif /* NDEBUG */ - } /* end for */ +#ifdef LATER + /* Check for I/O filters on this heap */ + if(hdr->filter_len > 0) { + H5Z_cb_t filter_cb = {NULL, NULL}; /* Filter callback structure */ + size_t nbytes; /* Number of bytes used */ + unsigned filter_mask; /* Filter mask for block */ - /* Sanity check */ - HDassert((size_t)(p - buf) == iblock->size); -#ifndef NDEBUG - HDassert(nchildren == iblock->nchildren); - HDassert(max_child == iblock->max_child); -#endif /* NDEBUG */ + /* Allocate buffer to perform I/O filtering on */ + write_size = dblock->size; + if(NULL == (write_buf = H5MM_malloc(write_size))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, FAIL, "memory allocation failed for pipeline buffer") + HDmemcpy(write_buf, dblock->blk, write_size); - /* Write the indirect block */ - if(H5F_block_write(f, H5FD_MEM_FHEAP_IBLOCK, addr, iblock->size, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTFLUSH, FAIL, "unable to save fractal heap indirect block to disk") + /* Push direct block data through I/O filter pipeline */ + nbytes = write_size; + if(H5Z_pipeline(&(hdr->pline), 0, &filter_mask, H5Z_ENABLE_EDC, + filter_cb, &nbytes, &write_size, &write_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_WRITEERROR, FAIL, "output pipeline failed") +HDfprintf(stderr, "%s: nbytes = %Zu, write_size = %Zu, write_buf = %p\n", FUNC, nbytes, write_size, write_buf); +HDfprintf(stderr, "%s: dblock->size = %Zu, dblock->blk = %p\n", FUNC, dblock->size, dblock->blk); - /* Free buffer */ - H5FL_BLK_FREE(indirect_block, buf); +HDfprintf(stderr, "%s: I/O filters not supported yet!\n", FUNC); +HGOTO_ERROR(H5E_HEAP, H5E_UNSUPPORTED, FAIL, "I/O filters not supported yet") + } /* end if */ + else { +#endif /* LATER */ + write_buf = dblock->blk; + write_size = dblock->size; +#ifdef LATER + } /* end else */ +#endif /* LATER */ - /* Reset dirty flags */ - iblock->cache_info.is_dirty = FALSE; + /* Write the direct block */ + if(H5F_block_write(f, H5FD_MEM_FHEAP_DBLOCK, addr, write_size, dxpl_id, write_buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFLUSH, FAIL, "unable to save fractal heap direct block to disk") + + /* Release the write buffer, if it was allocated */ + if(write_buf != dblock->blk) + H5MM_xfree(write_buf); + + dblock->cache_info.is_dirty = FALSE; } /* end if */ if(destroy) - if(H5HF_cache_iblock_dest(f, iblock) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to destroy fractal heap indirect block") + if(H5HF_cache_dblock_dest(f, dblock) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to destroy fractal heap direct block") done: FUNC_LEAVE_NOAPI(ret_value) -} /* H5HF_cache_iblock_flush() */ +} /* H5HF_cache_dblock_flush() */ /*------------------------------------------------------------------------- - * Function: H5HF_cache_iblock_dest + * Function: H5HF_cache_dblock_dest * - * Purpose: Destroys a fractal heap indirect block in memory. + * Purpose: Destroys a fractal heap direct block in memory. * * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol * koziol@ncsa.uiuc.edu - * Mar 6 2006 + * Feb 27 2006 * *------------------------------------------------------------------------- */ /* ARGSUSED */ herr_t -H5HF_cache_iblock_dest(H5F_t UNUSED *f, H5HF_indirect_t *iblock) +H5HF_cache_dblock_dest(H5F_t UNUSED *f, H5HF_direct_t *dblock) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_iblock_dest) + FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_dblock_dest) /* * Check arguments. */ - HDassert(iblock); - HDassert(iblock->rc == 0); + HDassert(dblock); #ifdef QAK -HDfprintf(stderr, "%s: Destroying indirect block\n", FUNC); +HDfprintf(stderr, "%s: Destroying direct block, dblock = %p\n", FUNC, dblock); #endif /* QAK */ - /* Decrement reference count on shared info */ - HDassert(iblock->hdr); - if(H5HF_hdr_decr(iblock->hdr) < 0) + /* Decrement reference count on shared fractal heap info */ + HDassert(dblock->hdr); + if(H5HF_hdr_decr(dblock->hdr) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTDEC, FAIL, "can't decrement reference count on shared heap header") - if(iblock->parent) - if(H5HF_iblock_decr(iblock->parent) < 0) + if(dblock->parent) + if(H5HF_iblock_decr(dblock->parent) < 0) HGOTO_ERROR(H5E_HEAP, H5E_CANTDEC, FAIL, "can't decrement reference count on shared indirect block") - /* Release entry tables */ - if(iblock->ents) - H5FL_SEQ_FREE(H5HF_indirect_ent_t, iblock->ents); - if(iblock->filt_ents) - H5FL_SEQ_FREE(H5HF_indirect_filt_ent_t, iblock->filt_ents); - if(iblock->child_iblocks) - H5FL_SEQ_FREE(H5HF_indirect_ptr_t, iblock->child_iblocks); + /* Free block's buffer */ + H5FL_BLK_FREE(direct_block, dblock->blk); - /* Free fractal heap indirect block info */ - H5FL_FREE(H5HF_indirect_t, iblock); + /* Free fractal heap direct block info */ + H5FL_FREE(H5HF_direct_t, dblock); done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5HF_cache_iblock_dest() */ +} /* end H5HF_cache_dblock_dest() */ /*------------------------------------------------------------------------- - * Function: H5HF_cache_iblock_clear + * Function: H5HF_cache_dblock_clear * - * Purpose: Mark a fractal heap indirect block in memory as non-dirty. + * Purpose: Mark a fractal heap direct block in memory as non-dirty. * * Return: Non-negative on success/Negative on failure * * Programmer: Quincey Koziol * koziol@ncsa.uiuc.edu - * Mar 6 2006 + * Feb 27 2006 * *------------------------------------------------------------------------- */ static herr_t -H5HF_cache_iblock_clear(H5F_t *f, H5HF_indirect_t *iblock, hbool_t destroy) +H5HF_cache_dblock_clear(H5F_t *f, H5HF_direct_t *dblock, hbool_t destroy) { herr_t ret_value = SUCCEED; /* Return value */ - FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_iblock_clear) + FUNC_ENTER_NOAPI_NOINIT(H5HF_cache_dblock_clear) /* * Check arguments. */ - HDassert(iblock); + HDassert(dblock); /* Reset the dirty flag. */ - iblock->cache_info.is_dirty = FALSE; + dblock->cache_info.is_dirty = FALSE; if(destroy) - if(H5HF_cache_iblock_dest(f, iblock) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to destroy fractal heap indirect block") + if(H5HF_cache_dblock_dest(f, dblock) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTFREE, FAIL, "unable to destroy fractal heap direct block") done: FUNC_LEAVE_NOAPI(ret_value) -} /* end H5HF_cache_iblock_clear() */ +} /* end H5HF_cache_dblock_clear() */ /*------------------------------------------------------------------------- - * Function: H5HF_cache_iblock_size + * Function: H5HF_cache_dblock_size * - * Purpose: Compute the size in bytes of a fractal heap indirect block + * Purpose: Compute the size in bytes of a fractal heap direct block * on disk, and return it in *size_ptr. On failure, * the value of *size_ptr is undefined. * @@ -1445,22 +1498,22 @@ done: * * Programmer: Quincey Koziol * koziol@ncsa.uiuc.edu - * Mar 6 2006 + * Feb 24 2006 * *------------------------------------------------------------------------- */ static herr_t -H5HF_cache_iblock_size(const H5F_t UNUSED *f, const H5HF_indirect_t *iblock, size_t *size_ptr) +H5HF_cache_dblock_size(const H5F_t UNUSED *f, const H5HF_direct_t *dblock, size_t *size_ptr) { - FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5HF_cache_iblock_size) + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5HF_cache_dblock_size) /* check arguments */ - HDassert(iblock); + HDassert(dblock); HDassert(size_ptr); /* Set size value */ - *size_ptr = iblock->size; + *size_ptr = dblock->size; FUNC_LEAVE_NOAPI(SUCCEED) -} /* H5HF_cache_iblock_size() */ +} /* H5HF_cache_dblock_size() */ diff --git a/src/H5HFhdr.c b/src/H5HFhdr.c index e239aeb..5b9022c 100644 --- a/src/H5HFhdr.c +++ b/src/H5HFhdr.c @@ -393,6 +393,7 @@ H5HF_hdr_create(H5F_t *f, hid_t dxpl_id, const H5HF_create_t *cparam) /* Set the creation parameters for the heap */ hdr->max_man_size = cparam->max_man_size; + hdr->checksum_dblocks = cparam->checksum_dblocks; HDmemcpy(&(hdr->man_dtable.cparam), &(cparam->managed), sizeof(H5HF_dtable_cparam_t)); /* Set root table address to indicate that the heap is empty currently */ diff --git a/src/H5HFpkg.h b/src/H5HFpkg.h index bc9e23a..14a9baa 100644 --- a/src/H5HFpkg.h +++ b/src/H5HFpkg.h @@ -49,12 +49,15 @@ #define H5HF_DBLOCK_MAGIC "FHDB" /* Direct block */ #define H5HF_IBLOCK_MAGIC "FHIB" /* Indirect block */ +/* Size of checksum information (on disk) */ +#define H5HF_SIZEOF_CHKSUM 4 + /* "Standard" size of prefix information for fractal heap metadata */ -#define H5HF_METADATA_PREFIX_SIZE ( \ +#define H5HF_METADATA_PREFIX_SIZE(c) ( \ 4 /* Signature */ \ + 1 /* Version */ \ + 1 /* Metadata flags */ \ - + 4 /* Metadata checksum */ \ + + ((c) ? H5HF_SIZEOF_CHKSUM : 0) /* Metadata checksum */ \ ) /* Size of doubling-table information */ @@ -70,6 +73,7 @@ /* Flags for status byte */ #define H5HF_HDR_FLAGS_HUGE_ID_WRAPPED 0x01 +#define H5HF_HDR_FLAGS_CHECKSUM_DBLOCKS 0x02 /* Size of the fractal heap header on disk */ /* (this is the fixed-len portion, the variable-len I/O filter information @@ -77,7 +81,7 @@ */ #define H5HF_HEADER_SIZE(h) ( \ /* General metadata fields */ \ - H5HF_METADATA_PREFIX_SIZE \ + H5HF_METADATA_PREFIX_SIZE(TRUE) \ \ /* Fractal Heap Header specific fields */ \ \ @@ -112,7 +116,7 @@ /* Size of overhead for a direct block */ #define H5HF_MAN_ABS_DIRECT_OVERHEAD(h) ( \ /* General metadata fields */ \ - H5HF_METADATA_PREFIX_SIZE \ + H5HF_METADATA_PREFIX_SIZE(h->checksum_dblocks) \ \ /* Fractal heap managed, absolutely mapped direct block specific fields */ \ + (h)->sizeof_addr /* File address of heap owning the block */ \ @@ -129,7 +133,7 @@ /* Size of managed indirect block */ #define H5HF_MAN_INDIRECT_SIZE(h, i) ( \ /* General metadata fields */ \ - H5HF_METADATA_PREFIX_SIZE \ + H5HF_METADATA_PREFIX_SIZE(TRUE) \ \ /* Fractal heap managed, absolutely mapped indirect block specific fields */ \ + (h)->sizeof_addr /* File address of heap owning the block */ \ @@ -308,6 +312,7 @@ typedef struct H5HF_hdr_t { hbool_t debug_objs; /* Is the heap storing objects in 'debug' format */ hbool_t write_once; /* Is heap being written in "write once" mode? */ hbool_t huge_ids_wrapped; /* Have "huge" object IDs wrapped around? */ + hbool_t checksum_dblocks; /* Should the direct blocks in the heap be checksummed? */ /* Doubling table information (partially stored in header) */ /* (Partially set by user, partially derived/updated internally) */ @@ -475,12 +480,12 @@ typedef struct { /* H5HF header inherits cache-like properties from H5AC */ H5_DLLVAR const H5AC_class_t H5AC_FHEAP_HDR[1]; -/* H5HF direct block inherits cache-like properties from H5AC */ -H5_DLLVAR const H5AC_class_t H5AC_FHEAP_DBLOCK[1]; - /* H5HF indirect block inherits cache-like properties from H5AC */ H5_DLLVAR const H5AC_class_t H5AC_FHEAP_IBLOCK[1]; +/* H5HF direct block inherits cache-like properties from H5AC */ +H5_DLLVAR const H5AC_class_t H5AC_FHEAP_DBLOCK[1]; + /* The v2 B-tree class for tracking indirectly accessed 'huge' objects */ H5_DLLVAR const H5B2_class_t H5HF_BT2_INDIR[1]; @@ -508,12 +513,6 @@ H5_DLLVAR H5FS_section_class_t H5HF_FSPACE_SECT_CLS_INDIRECT[1]; /* Declare a free list to manage the H5HF_hdr_t struct */ H5FL_EXTERN(H5HF_hdr_t); -/* Declare a free list to manage the H5HF_direct_t struct */ -H5FL_EXTERN(H5HF_direct_t); - -/* Declare a free list to manage heap direct block data to/from disk */ -H5FL_BLK_EXTERN(direct_block); - /* Declare a free list to manage the H5HF_indirect_t struct */ H5FL_EXTERN(H5HF_indirect_t); @@ -527,18 +526,17 @@ H5FL_SEQ_EXTERN(H5HF_indirect_filt_ent_t); typedef H5HF_indirect_t *H5HF_indirect_ptr_t; H5FL_SEQ_EXTERN(H5HF_indirect_ptr_t); +/* Declare a free list to manage the H5HF_direct_t struct */ +H5FL_EXTERN(H5HF_direct_t); + +/* Declare a free list to manage heap direct block data to/from disk */ +H5FL_BLK_EXTERN(direct_block); + /******************************/ /* Package Private Prototypes */ /******************************/ -/* Routines for managing shared fractal heap header */ -H5_DLL H5HF_hdr_t * H5HF_hdr_alloc(H5F_t *f); -H5_DLL haddr_t H5HF_hdr_create(H5F_t *f, hid_t dxpl_id, const H5HF_create_t *cparam); -H5_DLL herr_t H5HF_hdr_finish_init_phase1(H5HF_hdr_t *hdr); -H5_DLL herr_t H5HF_hdr_finish_init_phase2(H5HF_hdr_t *hdr); -H5_DLL herr_t H5HF_hdr_finish_init(H5HF_hdr_t *hdr); - /* Doubling table routines */ H5_DLL herr_t H5HF_dtable_init(H5HF_dtable_t *dtable); H5_DLL herr_t H5HF_dtable_dest(H5HF_dtable_t *dtable); @@ -550,6 +548,11 @@ H5_DLL hsize_t H5HF_dtable_span_size(const H5HF_dtable_t *dtable, unsigned start unsigned start_col, unsigned num_entries); /* Heap header routines */ +H5_DLL H5HF_hdr_t * H5HF_hdr_alloc(H5F_t *f); +H5_DLL haddr_t H5HF_hdr_create(H5F_t *f, hid_t dxpl_id, const H5HF_create_t *cparam); +H5_DLL herr_t H5HF_hdr_finish_init_phase1(H5HF_hdr_t *hdr); +H5_DLL herr_t H5HF_hdr_finish_init_phase2(H5HF_hdr_t *hdr); +H5_DLL herr_t H5HF_hdr_finish_init(H5HF_hdr_t *hdr); H5_DLL herr_t H5HF_hdr_incr(H5HF_hdr_t *hdr); H5_DLL herr_t H5HF_hdr_decr(H5HF_hdr_t *hdr); H5_DLL herr_t H5HF_hdr_dirty(H5HF_hdr_t *hdr); diff --git a/src/H5HFprivate.h b/src/H5HFprivate.h index 20415c1..5e25a58f 100644 --- a/src/H5HFprivate.h +++ b/src/H5HFprivate.h @@ -62,6 +62,7 @@ typedef struct H5HF_dtable_cparam_t { /* Fractal heap creation parameters */ typedef struct H5HF_create_t { H5HF_dtable_cparam_t managed;/* Mapped object doubling-table creation parameters */ + hbool_t checksum_dblocks; /* Whether the direct blocks should be checksummed */ uint32_t max_man_size; /* Max. size of object to manage in doubling table */ /* (i.e. min. size of object to store standalone) */ uint16_t id_len; /* Length of IDs to use for heap objects */ diff --git a/test/fheap.c b/test/fheap.c index 721ad89..209a2a0 100644 --- a/test/fheap.c +++ b/test/fheap.c @@ -40,6 +40,7 @@ /* "Small" heap creation parameters */ #define SMALL_DBLOCK_OVERHEAD 22 /* Overhead for direct blocks */ +#define SMALL_CHECKSUM_DBLOCKS TRUE /* Whether to checksum direct blocks */ #define SMALL_MAN_WIDTH 4 /* Managed obj. table width */ #define SMALL_MAN_START_BLOCK_SIZE 512 /* Managed obj. starting block size */ #define SMALL_MAN_MAX_DIRECT_SIZE (64 * 1024) /* Managed obj. max. direct block size */ @@ -49,7 +50,8 @@ #define SMALL_STAND_SIZE (SMALL_MAN_MAX_DIRECT_SIZE - SMALL_DBLOCK_OVERHEAD) /* Standalone obj. min. size */ /* "Large" heap creation parameters */ -#define LARGE_DBLOCK_OVERHEAD 26 /* Overhead for direct blocks */ +#define LARGE_DBLOCK_OVERHEAD 22 /* Overhead for direct blocks */ +#define LARGE_CHECKSUM_DBLOCKS FALSE /* Whether to checksum direct blocks */ #define LARGE_MAN_WIDTH 32 /* Managed obj. table width */ #define LARGE_MAN_START_BLOCK_SIZE 4096 /* Managed obj. starting block size */ #define LARGE_MAN_MAX_DIRECT_SIZE (1024 * 1024) /* Managed obj. max. direct block size */ @@ -113,7 +115,7 @@ typedef struct fheap_test_param_t { fheap_test_del_dir_t del_dir; /* Whether to delete objects forward or reverse */ fheap_test_del_drain_t drain_half; /* Whether to drain half of the objects & refill, when deleting objects */ fheap_test_fill_t fill; /* How to "bulk" fill heap blocks */ - unsigned actual_id_len; /* The actual length of heap IDs for a test */ + size_t actual_id_len; /* The actual length of heap IDs for a test */ } fheap_test_param_t; /* Heap state information */ @@ -178,6 +180,7 @@ init_small_cparam(H5HF_create_t *cparam) /* General parameters */ cparam->id_len = SMALL_ID_LEN; cparam->max_man_size = SMALL_STAND_SIZE; + cparam->checksum_dblocks = SMALL_CHECKSUM_DBLOCKS; /* Managed object doubling-table parameters */ cparam->managed.width = SMALL_MAN_WIDTH; @@ -214,6 +217,7 @@ init_large_cparam(H5HF_create_t *cparam) /* General parameters */ cparam->id_len = LARGE_ID_LEN; cparam->max_man_size = LARGE_STAND_SIZE; + cparam->checksum_dblocks = LARGE_CHECKSUM_DBLOCKS; /* Managed object doubling-table parameters */ cparam->managed.width = LARGE_MAN_WIDTH; @@ -2173,7 +2177,7 @@ test_id_limits(hid_t fapl, H5HF_create_t *cparam) /* Set an I/O filter for heap data */ deflate_level = 6; - if(H5Z_append(&tmp_cparam.pline, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 1, &deflate_level) < 0) + if(H5Z_append(&tmp_cparam.pline, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, (size_t)1, &deflate_level) < 0) FAIL_STACK_ERROR /* Create absolute heap */ @@ -2449,7 +2453,7 @@ test_filtered_create(hid_t fapl, H5HF_create_t *cparam) /* Set an I/O filter for heap data */ deflate_level = 6; - if(H5Z_append(&tmp_cparam.pline, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 1, &deflate_level) < 0) + if(H5Z_append(&tmp_cparam.pline, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, (size_t)1, &deflate_level) < 0) FAIL_STACK_ERROR /* Create absolute heap */ @@ -2587,14 +2591,14 @@ test_man_insert_weird(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpa /* Attempt to insert 0-sized object into heap */ H5E_BEGIN_TRY { - ret = H5HF_insert(fh, dxpl, 0, shared_wobj_g, heap_id); + ret = H5HF_insert(fh, dxpl, (size_t)0, shared_wobj_g, heap_id); } H5E_END_TRY; if(ret >= 0) TEST_ERROR H5Eclear_stack(H5E_DEFAULT); /* Insert a 1-sized object into heap (should be a 'tiny' object) */ - if(add_obj(fh, dxpl, (size_t)10, 1, &state, NULL)) + if(add_obj(fh, dxpl, (size_t)10, (size_t)1, &state, NULL)) TEST_ERROR /* Check for closing & re-opening the heap */ @@ -12851,7 +12855,7 @@ test_filtered_huge(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tparam /* Set an I/O filter for heap data */ deflate_level = 6; - if(H5Z_append(&tmp_cparam.pline, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 1, &deflate_level) < 0) + if(H5Z_append(&tmp_cparam.pline, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, (size_t)1, &deflate_level) < 0) FAIL_STACK_ERROR /* Adjust actual ID length, if asking for IDs that can directly access 'huge' objects */ @@ -14009,7 +14013,7 @@ test_filtered_man_one(hid_t fapl, H5HF_create_t *cparam, fheap_test_param_t *tpa /* Set an I/O filter for heap data */ deflate_level = 6; - if(H5Z_append(&tmp_cparam.pline, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, 1, &deflate_level) < 0) + if(H5Z_append(&tmp_cparam.pline, H5Z_FILTER_DEFLATE, H5Z_FLAG_OPTIONAL, (size_t)1, &deflate_level) < 0) FAIL_STACK_ERROR /* Perform common file & heap open operations */ -- cgit v0.12