diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/H5ACprivate.h | 60 | ||||
| -rw-r--r-- | src/H5B2cache.c | 39 | ||||
| -rw-r--r-- | src/H5EAcache.c | 65 | ||||
| -rw-r--r-- | src/H5F.c | 34 | ||||
| -rw-r--r-- | src/H5FAcache.c | 41 | ||||
| -rw-r--r-- | src/H5FScache.c | 27 | ||||
| -rw-r--r-- | src/H5Fint.c | 103 | ||||
| -rw-r--r-- | src/H5Fio.c | 61 | ||||
| -rw-r--r-- | src/H5Fpkg.h | 8 | ||||
| -rw-r--r-- | src/H5Fprivate.h | 23 | ||||
| -rw-r--r-- | src/H5Fpublic.h | 12 | ||||
| -rw-r--r-- | src/H5Fquery.c | 2 | ||||
| -rw-r--r-- | src/H5Fsuper_cache.c | 71 | ||||
| -rw-r--r-- | src/H5HFcache.c | 353 | ||||
| -rw-r--r-- | src/H5HFpkg.h | 8 | ||||
| -rw-r--r-- | src/H5Ocache.c | 61 | ||||
| -rw-r--r-- | src/H5Pfapl.c | 28 | ||||
| -rw-r--r-- | src/H5Ppublic.h | 2 | ||||
| -rw-r--r-- | src/H5SMcache.c | 36 |
19 files changed, 404 insertions, 630 deletions
diff --git a/src/H5ACprivate.h b/src/H5ACprivate.h index 625d77f..7b55237 100644 --- a/src/H5ACprivate.h +++ b/src/H5ACprivate.h @@ -53,36 +53,36 @@ /* Types of metadata objects cached */ typedef enum { - H5AC_BT_ID = 0, /*B-tree nodes */ - H5AC_SNODE_ID = 1, /*symbol table nodes */ - H5AC_LHEAP_PRFX_ID = 2, /*local heap prefix */ - H5AC_LHEAP_DBLK_ID = 3, /*local heap data block */ - H5AC_GHEAP_ID = 4, /*global heap */ - H5AC_OHDR_ID = 5, /*object header */ - H5AC_OHDR_CHK_ID = 6, /*object header chunk */ - H5AC_OHDR_PROXY_ID = 7, /*object header proxy */ - H5AC_BT2_HDR_ID = 8, /*v2 B-tree header */ - H5AC_BT2_INT_ID = 9, /*v2 B-tree internal node */ - H5AC_BT2_LEAF_ID = 10, /*v2 B-tree leaf node */ - H5AC_FHEAP_HDR_ID = 11, /*fractal heap header */ - H5AC_FHEAP_DBLOCK_ID = 12, /*fractal heap direct block */ - H5AC_FHEAP_IBLOCK_ID = 13, /*fractal heap indirect block */ - H5AC_FSPACE_HDR_ID = 14, /*free space header */ - H5AC_FSPACE_SINFO_ID = 15, /*free space sections */ - H5AC_SOHM_TABLE_ID = 16, /*shared object header message master table */ - H5AC_SOHM_LIST_ID = 17, /*shared message index stored as a list */ - H5AC_EARRAY_HDR_ID = 18, /*extensible array header */ - H5AC_EARRAY_IBLOCK_ID = 19, /*extensible array index block */ - H5AC_EARRAY_SBLOCK_ID = 20, /*extensible array super block */ - H5AC_EARRAY_DBLOCK_ID = 21, /*extensible array data block */ - H5AC_EARRAY_DBLK_PAGE_ID = 22, /*extensible array data block page */ - H5AC_CHUNK_PROXY_ID = 23, /*chunk proxy */ - H5AC_FARRAY_HDR_ID = 24, /*fixed array header */ - H5AC_FARRAY_DBLOCK_ID = 25, /*fixed array data block */ - H5AC_FARRAY_DBLK_PAGE_ID = 26, /*fixed array data block page */ - H5AC_SUPERBLOCK_ID = 27, /*file superblock */ - H5AC_TEST_ID = 28, /*test entry -- not used for actual files */ - H5AC_NTYPES = 29 /*Number of types, must be last */ + H5AC_BT_ID = 0, /* ( 0) B-tree nodes */ + H5AC_SNODE_ID, /* ( 1) symbol table nodes */ + H5AC_LHEAP_PRFX_ID, /* ( 2) local heap prefix */ + H5AC_LHEAP_DBLK_ID, /* ( 3) local heap data block */ + H5AC_GHEAP_ID, /* ( 4) global heap */ + H5AC_OHDR_ID, /* ( 5) object header */ + H5AC_OHDR_CHK_ID, /* ( 6) object header chunk */ + H5AC_OHDR_PROXY_ID, /* ( 7) object header proxy */ + H5AC_BT2_HDR_ID, /* ( 8) v2 B-tree header */ + H5AC_BT2_INT_ID, /* ( 9) v2 B-tree internal node */ + H5AC_BT2_LEAF_ID, /* (10) v2 B-tree leaf node */ + H5AC_FHEAP_HDR_ID, /* (11) fractal heap header */ + H5AC_FHEAP_DBLOCK_ID, /* (12) fractal heap direct block */ + H5AC_FHEAP_IBLOCK_ID, /* (13) fractal heap indirect block */ + H5AC_FSPACE_HDR_ID, /* (14) free space header */ + H5AC_FSPACE_SINFO_ID, /* (15) free space sections */ + H5AC_SOHM_TABLE_ID, /* (16) shared object header message master table */ + H5AC_SOHM_LIST_ID, /* (17) shared message index stored as a list */ + H5AC_EARRAY_HDR_ID, /* (18) extensible array header */ + H5AC_EARRAY_IBLOCK_ID, /* (19) extensible array index block */ + H5AC_EARRAY_SBLOCK_ID, /* (20) extensible array super block */ + H5AC_EARRAY_DBLOCK_ID, /* (21) extensible array data block */ + H5AC_EARRAY_DBLK_PAGE_ID, /* (22) extensible array data block page */ + H5AC_CHUNK_PROXY_ID, /* (23) chunk proxy */ + H5AC_FARRAY_HDR_ID, /* (24) fixed array header */ + H5AC_FARRAY_DBLOCK_ID, /* (25) fixed array data block */ + H5AC_FARRAY_DBLK_PAGE_ID, /* (26) fixed array data block page */ + H5AC_SUPERBLOCK_ID, /* (27) file superblock */ + H5AC_TEST_ID, /* (28) test entry -- not used for actual files */ + H5AC_NTYPES /* Number of types, must be last */ } H5AC_type_t; /* H5AC_DUMP_STATS_ON_CLOSE should always be FALSE when diff --git a/src/H5B2cache.c b/src/H5B2cache.c index cde6a82..8e5554a 100644 --- a/src/H5B2cache.c +++ b/src/H5B2cache.c @@ -160,8 +160,6 @@ H5B2__cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) H5B2_create_t cparam; /* B-tree creation parameters */ H5B2_subid_t id; /* ID of B-tree class, as found in file */ uint16_t depth; /* Depth of B-tree */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ H5WB_t *wb = NULL; /* Wrapped buffer for header data */ uint8_t hdr_buf[H5B2_HDR_BUF_SIZE]; /* Buffer for header */ uint8_t *buf; /* Pointer to header buffer */ @@ -188,7 +186,7 @@ H5B2__cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) HGOTO_ERROR(H5E_BTREE, H5E_NOSPACE, NULL, "can't get actual buffer") /* Read and validate header from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_BTREE, H5AC_BT2_HDR_ID, addr, hdr->hdr_size, hdr->hdr_size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_BTREE, H5AC_BT2_HDR_ID, addr, hdr->hdr_size, hdr->hdr_size, buf) < 0) HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 B-tree header") /* Get temporary pointer to serialized header */ @@ -226,15 +224,10 @@ H5B2__cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) UINT16DECODE(p, hdr->root.node_nrec); H5F_DECODE_LENGTH(udata->f, p, hdr->root.all_nrec); - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - (const uint8_t *)buf) == hdr->hdr_size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 B-tree header") + HDassert((size_t)(p - (const uint8_t *)buf) == (hdr->hdr_size - H5_SIZEOF_CHKSUM)); /* Initialize B-tree header info */ cparam.cls = H5B2_client_class_g[id]; @@ -572,8 +565,6 @@ H5B2__cache_internal_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) const uint8_t *p; /* Pointer into raw data buffer */ uint8_t *native; /* Pointer to native record info */ H5B2_node_ptr_t *int_node_ptr; /* Pointer to node pointer info */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned u; /* Local index variable */ size_t chk_size; /* Exact size of the node with checksum at the end */ H5B2_internal_t *ret_value; /* Return value */ @@ -606,7 +597,7 @@ H5B2__cache_internal_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Internal node prefix header + records + child pointer triplets: size with checksum at the end */ chk_size = H5B2_INT_PREFIX_SIZE + (udata->nrec * udata->hdr->rrec_size) + ((size_t)(udata->nrec + 1) * H5B2_INT_POINTER_SIZE(udata->hdr, udata->depth)); /* Read and validate internal node from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_BTREE, H5AC_BT2_INT_ID, addr, (size_t)udata->hdr->node_size, chk_size, dxpl_id, udata->hdr->page, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_BTREE, H5AC_BT2_INT_ID, addr, (size_t)udata->hdr->node_size, chk_size, udata->hdr->page) < 0) HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 internal node") p = udata->hdr->page; @@ -663,15 +654,10 @@ H5B2__cache_internal_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) int_node_ptr++; } /* end for */ - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check parsing */ - HDassert((size_t)(p - (const uint8_t *)udata->hdr->page) <= udata->hdr->node_size); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 internal node") + HDassert((size_t)(p - (const uint8_t *)udata->hdr->page) <= (udata->hdr->node_size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = internal; @@ -994,8 +980,6 @@ H5B2__cache_leaf_load(H5F_t H5_ATTR_UNUSED *f, hid_t dxpl_id, haddr_t addr, void H5B2_leaf_t *leaf = NULL; /* Pointer to lead node loaded */ const uint8_t *p; /* Pointer into raw data buffer */ uint8_t *native; /* Pointer to native keys */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned u; /* Local index variable */ size_t chk_size; /* Exact size of the node with checksum at the end */ H5B2_leaf_t *ret_value; /* Return value */ @@ -1029,7 +1013,7 @@ H5B2__cache_leaf_load(H5F_t H5_ATTR_UNUSED *f, hid_t dxpl_id, haddr_t addr, void chk_size = H5B2_LEAF_PREFIX_SIZE + (udata->nrec * udata->hdr->rrec_size); /* Read and validate leaf node from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_BTREE, H5AC_BT2_LEAF_ID, addr, (size_t)udata->hdr->node_size, chk_size, dxpl_id, udata->hdr->page, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_BTREE, H5AC_BT2_LEAF_ID, addr, (size_t)udata->hdr->node_size, chk_size, udata->hdr->page) < 0) HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 leaf node") p = udata->hdr->page; @@ -1066,15 +1050,10 @@ H5B2__cache_leaf_load(H5F_t H5_ATTR_UNUSED *f, hid_t dxpl_id, haddr_t addr, void native += udata->hdr->cls->nrec_size; } /* end for */ - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check parsing */ - HDassert((size_t)(p - (const uint8_t *)udata->hdr->page) <= udata->hdr->node_size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for v2 leaf node") + HDassert((size_t)(p - (const uint8_t *)udata->hdr->page) <= (udata->hdr->node_size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = leaf; diff --git a/src/H5EAcache.c b/src/H5EAcache.c index c41ff24..3363701 100644 --- a/src/H5EAcache.c +++ b/src/H5EAcache.c @@ -208,8 +208,6 @@ H5EA__cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *udata)) uint8_t hdr_buf[H5EA_HDR_BUF_SIZE]; /* Buffer for header */ uint8_t *buf; /* Pointer to header buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Check arguments */ HDassert(f); @@ -234,7 +232,7 @@ H5EA__cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *udata)) H5E_THROW(H5E_CANTGET, "can't get actual buffer") /* Read and validate header from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_EARRAY_HDR, H5AC_EARRAY_HDR_ID, addr, size, size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_EARRAY_HDR, H5AC_EARRAY_HDR_ID, addr, size, size, buf) < 0) H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array header") /* Get temporary pointer to serialized header */ @@ -300,15 +298,10 @@ H5EA__cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *udata)) /* (allow for checksum not decoded yet) */ HDassert((size_t)(p - buf) == (size - H5EA_SIZEOF_CHKSUM)); - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - buf) == size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array header") + HDassert((size_t)(p - buf) == (size - H5_SIZEOF_CHKSUM)); /* Finish initializing extensible array header */ if(H5EA__hdr_init(hdr, udata) < 0) @@ -570,8 +563,6 @@ H5EA__cache_iblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) uint8_t iblock_buf[H5EA_IBLOCK_BUF_SIZE]; /* Buffer for index block */ uint8_t *buf; /* Pointer to index block buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ size_t u; /* Local index variable */ @@ -599,7 +590,7 @@ H5EA__cache_iblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) H5E_THROW(H5E_CANTGET, "can't get actual buffer") /* Read and validate index block from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_EARRAY_IBLOCK, H5AC_EARRAY_IBLOCK_ID, addr, size, size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_EARRAY_IBLOCK, H5AC_EARRAY_IBLOCK_ID, addr, size, size, buf) < 0) H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array index block") /* Get temporary pointer to serialized header */ @@ -654,15 +645,10 @@ H5EA__cache_iblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) /* Save the index block's size */ iblock->size = size; - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - buf) == iblock->size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array index block") + HDassert((size_t)(p - buf) == (iblock->size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = iblock; @@ -980,8 +966,6 @@ H5EA__cache_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) uint8_t sblock_buf[H5EA_IBLOCK_BUF_SIZE]; /* Buffer for super block */ uint8_t *buf; /* Pointer to super block buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ size_t u; /* Local index variable */ @@ -1009,7 +993,7 @@ H5EA__cache_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) H5E_THROW(H5E_CANTGET, "can't get actual buffer") /* Read and validate super block from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_EARRAY_SBLOCK, H5AC_EARRAY_SBLOCK_ID, addr, size, size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_EARRAY_SBLOCK, H5AC_EARRAY_SBLOCK_ID, addr, size, size, buf) < 0) H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array super block") /* Get temporary pointer to serialized header */ @@ -1058,15 +1042,10 @@ H5EA__cache_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) /* Save the super block's size */ sblock->size = size; - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - buf) == sblock->size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array super block") + HDassert((size_t)(p - buf) == (sblock->size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = sblock; @@ -1374,8 +1353,6 @@ H5EA__cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) uint8_t dblock_buf[H5EA_DBLOCK_BUF_SIZE]; /* Buffer for data block */ uint8_t *buf; /* Pointer to data block buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ /* Sanity check */ @@ -1405,7 +1382,7 @@ H5EA__cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) H5E_THROW(H5E_CANTGET, "can't get actual buffer") /* Read and validate data block from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_EARRAY_DBLOCK, H5AC_EARRAY_DBLOCK_ID, addr, size, size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_EARRAY_DBLOCK, H5AC_EARRAY_DBLOCK_ID, addr, size, size, buf) < 0) H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible data block") /* Get temporary pointer to serialized header */ @@ -1450,15 +1427,10 @@ H5EA__cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) /* Set the data block's size */ dblock->size = H5EA_DBLOCK_SIZE(dblock); - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - buf) == size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block - # pages = %d", dblock->npages) + HDassert((size_t)(p - buf) == (size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = dblock; @@ -1771,8 +1743,6 @@ H5EA__cache_dblk_page_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) uint8_t dblk_page_buf[H5EA_DBLK_PAGE_BUF_SIZE]; /* Buffer for data block page */ uint8_t *buf; /* Pointer to data block page buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Sanity check */ HDassert(f); @@ -1798,7 +1768,7 @@ H5EA__cache_dblk_page_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) H5E_THROW(H5E_CANTGET, "can't get actual buffer") /* Read and validate data block page from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_EARRAY_DBLK_PAGE, H5AC_EARRAY_DBLK_PAGE_ID, addr, size, size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_EARRAY_DBLK_PAGE, H5AC_EARRAY_DBLK_PAGE_ID, addr, size, size, buf) < 0) H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block page") /* Get temporary pointer to serialized header */ @@ -1819,15 +1789,10 @@ H5EA__cache_dblk_page_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) /* Set the data block page's size */ dblk_page->size = size; - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - buf) == dblk_page->size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for extensible array data block page") + HDassert((size_t)(p - buf) == (dblk_page->size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = dblk_page; @@ -61,7 +61,6 @@ /********************/ - /*********************/ /* Package Variables */ /*********************/ @@ -834,7 +833,6 @@ H5Freopen(hid_t file_id) { H5F_t *old_file = NULL; H5F_t *new_file = NULL; - unsigned i; hid_t ret_value; FUNC_ENTER_API(FAIL) @@ -848,14 +846,6 @@ H5Freopen(hid_t file_id) if(NULL == (new_file = H5F_new(old_file->shared, 0, H5P_FILE_CREATE_DEFAULT, H5P_FILE_ACCESS_DEFAULT, NULL))) HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, FAIL, "unable to reopen file") - /* Keep old file's read attempts in new file */ - new_file->read_attempts = old_file->read_attempts; - - new_file->retries_nbins = old_file->retries_nbins; - for(i = 0; i < H5AC_NTYPES; i++) - new_file->retries[i] = NULL; - - /* Duplicate old file's names */ new_file->open_name = H5MM_xstrdup(old_file->open_name); new_file->actual_name = H5MM_xstrdup(old_file->actual_name); @@ -1428,7 +1418,7 @@ done: /*------------------------------------------------------------------------- - * Function: H5Fget_metadata_read_retries_info + * Function: H5Fget_metadata_read_retry_info * * Purpose: To retrieve the collection of read retries for metadata items with checksum. * @@ -1440,7 +1430,7 @@ done: *------------------------------------------------------------------------- */ herr_t -H5Fget_metadata_read_retries_info(hid_t file_id, H5F_retries_info_t *info) +H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info) { H5F_t *file; /* File object for file ID */ unsigned i, j; /* Local index variable */ @@ -1459,11 +1449,10 @@ H5Fget_metadata_read_retries_info(hid_t file_id, H5F_retries_info_t *info) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file ID") /* Copy the # of bins for "retries" array */ - info->nbins = file->retries_nbins; + info->nbins = file->shared->retries_nbins; /* Initialize the array of "retries" */ - for(i = 0; i < NUM_METADATA_READ_RETRIES; i++) - info->retries[i] = NULL; + HDmemset(info->retries, 0, sizeof(info->retries)); /* Return if there are no bins -- no retries */ if(!info->nbins) @@ -1497,14 +1486,17 @@ H5Fget_metadata_read_retries_info(hid_t file_id, H5F_retries_info_t *info) case H5AC_FARRAY_DBLOCK_ID: case H5AC_FARRAY_DBLK_PAGE_ID: case H5AC_SUPERBLOCK_ID: - HDassert(j < NUM_METADATA_READ_RETRIES); - if(file->retries[i] != NULL) { + HDassert(j < H5F_NUM_METADATA_READ_RETRY_TYPES); + if(file->shared->retries[i] != NULL) { /* Allocate memory for retries[i] */ - if((info->retries[j] = (uint32_t *)HDmalloc(tot_size)) == NULL) + if(NULL == (info->retries[j] = (uint32_t *)HDmalloc(tot_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + /* Copy the information */ - HDmemcpy(info->retries[j], file->retries[i], tot_size); - } + HDmemcpy(info->retries[j], file->shared->retries[i], tot_size); + } /* end if */ + + /* Increment location in info->retries[] array */ j++; break; @@ -1515,7 +1507,7 @@ H5Fget_metadata_read_retries_info(hid_t file_id, H5F_retries_info_t *info) done: FUNC_LEAVE_API(ret_value) -} /* end H5Fget_metadata_read_retries_info() */ +} /* end H5Fget_metadata_read_retry_info() */ /*------------------------------------------------------------------------- diff --git a/src/H5FAcache.c b/src/H5FAcache.c index 0759213..bbfd94e 100644 --- a/src/H5FAcache.c +++ b/src/H5FAcache.c @@ -171,8 +171,6 @@ H5FA__cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *udata)) uint8_t hdr_buf[H5FA_HDR_BUF_SIZE]; /* Buffer for header */ uint8_t *buf; /* Pointer to header buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Check arguments */ HDassert(f); @@ -197,7 +195,7 @@ H5FA__cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *udata)) H5E_THROW(H5E_CANTGET, "can't get actual buffer") /* Read and vaildate header from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_FARRAY_HDR, H5AC_FARRAY_HDR_ID, addr, size, size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_FARRAY_HDR, H5AC_FARRAY_HDR_ID, addr, size, size, buf) < 0) H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array header") /* Get temporary pointer to serialized header */ @@ -254,15 +252,10 @@ H5FA__cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *udata)) /* (allow for checksum not decoded yet) */ HDassert((size_t)(p - buf) == (size - H5FA_SIZEOF_CHKSUM)); - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - buf) == size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array header") + HDassert((size_t)(p - buf) == (size - H5_SIZEOF_CHKSUM)); /* Finish initializing fixed array header */ if(H5FA__hdr_init(hdr, udata) < 0) @@ -510,8 +503,6 @@ H5FA__cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) uint8_t dblock_buf[H5FA_DBLOCK_BUF_SIZE]; /* Buffer for data block */ uint8_t *buf; /* Pointer to data block buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ haddr_t arr_addr; /* Address of array header in the file */ /* Sanity check */ @@ -540,8 +531,8 @@ H5FA__cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) if(NULL == (buf = (uint8_t *)H5WB_actual(wb, size))) H5E_THROW(H5E_CANTGET, "can't get actual buffer") - /* Read and validate data block from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_FARRAY_DBLOCK, H5AC_FARRAY_DBLOCK_ID, addr, size, size, dxpl_id, buf, &computed_chksum) < 0) + /* Read and validate data block from disk */ + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_FARRAY_DBLOCK, H5AC_FARRAY_DBLOCK_ID, addr, size, size, buf) < 0) H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block") /* Get temporary pointer to serialized header */ @@ -587,15 +578,10 @@ H5FA__cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) /* Set the data block's size */ dblock->size = H5FA_DBLOCK_SIZE(dblock); - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - buf) == size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block") + HDassert((size_t)(p - buf) == (size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = dblock; @@ -904,8 +890,6 @@ H5FA__cache_dblk_page_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) uint8_t dblk_page_buf[H5FA_DBLK_PAGE_BUF_SIZE]; /* Buffer for data block page */ uint8_t *buf; /* Pointer to data block page buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ /* Sanity check */ HDassert(f); @@ -931,7 +915,7 @@ H5FA__cache_dblk_page_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) H5E_THROW(H5E_CANTGET, "can't get actual buffer") /* Read and validate data block page from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_FARRAY_DBLK_PAGE, H5AC_FARRAY_DBLK_PAGE_ID, addr, size, size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_FARRAY_DBLK_PAGE, H5AC_FARRAY_DBLK_PAGE_ID, addr, size, size, buf) < 0) H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block page") /* Get temporary pointer to serialized header */ @@ -952,15 +936,10 @@ H5FA__cache_dblk_page_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata)) /* Set the data block page's size */ dblk_page->size = size; - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - buf) == dblk_page->size); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - H5E_THROW(H5E_BADVALUE, "incorrect metadata checksum for fixed array data block page") + HDassert((size_t)(p - buf) == (dblk_page->size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = dblk_page; diff --git a/src/H5FScache.c b/src/H5FScache.c index 9c1910d..f68bf0f 100644 --- a/src/H5FScache.c +++ b/src/H5FScache.c @@ -156,8 +156,6 @@ H5FS_cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) uint8_t hdr_buf[H5FS_HDR_BUF_SIZE]; /* Buffer for header */ uint8_t *hdr; /* Pointer to header buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned nclasses; /* Number of section classes */ H5FS_t *ret_value; /* Return value */ @@ -183,7 +181,7 @@ H5FS_cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) HGOTO_ERROR(H5E_FSPACE, H5E_NOSPACE, NULL, "can't get actual buffer") /* Read and validate header from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_FSPACE_HDR, H5AC_FSPACE_HDR_ID, addr, fspace->hdr_size, fspace->hdr_size, dxpl_id, hdr, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_FSPACE_HDR, H5AC_FSPACE_HDR_ID, addr, fspace->hdr_size, fspace->hdr_size, hdr) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for free space header") p = hdr; @@ -241,14 +239,10 @@ H5FS_cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Allocated size of serialized free space sections */ H5F_DECODE_LENGTH(udata->f, p, fspace->alloc_sect_size); - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ - HDassert((size_t)(p - (const uint8_t *)hdr) == fspace->hdr_size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for free space header") + /* Sanity check */ + HDassert((size_t)(p - (const uint8_t *)hdr) == (fspace->hdr_size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = fspace; @@ -555,8 +549,6 @@ H5FS_cache_sinfo_load(H5F_t *f, hid_t dxpl_id, haddr_t H5_ATTR_UNUSED addr, void size_t old_sect_size; /* Old section size */ uint8_t *buf = NULL; /* Temporary buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ H5FS_sinfo_t *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -575,7 +567,7 @@ H5FS_cache_sinfo_load(H5F_t *f, hid_t dxpl_id, haddr_t H5_ATTR_UNUSED addr, void HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") /* Read and validate free space sections from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_FSPACE_SINFO, H5AC_FSPACE_SINFO_ID, udata->fspace->sect_addr, (size_t)udata->fspace->sect_size, (size_t)udata->fspace->sect_size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_FSPACE_SINFO, H5AC_FSPACE_SINFO_ID, udata->fspace->sect_addr, (size_t)udata->fspace->sect_size, (size_t)udata->fspace->sect_size, buf) < 0) HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for free space sections") /* Deserialize free sections from buffer available */ @@ -668,15 +660,10 @@ H5FS_cache_sinfo_load(H5F_t *f, hid_t dxpl_id, haddr_t H5_ATTR_UNUSED addr, void HDassert(old_tot_space == udata->fspace->tot_space); } /* end if */ - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_FSPACE, H5E_BADVALUE, NULL, "incorrect metadata checksum for free space sections") + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - (const uint8_t *)buf) == old_sect_size); + HDassert((size_t)(p - (const uint8_t *)buf) == (old_sect_size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = sinfo; diff --git a/src/H5Fint.c b/src/H5Fint.c index 3887222..fa20e71 100644 --- a/src/H5Fint.c +++ b/src/H5Fint.c @@ -78,7 +78,7 @@ typedef struct H5F_olist_t { static int H5F_get_objects_cb(void *obj_ptr, hid_t obj_id, void *key); static herr_t H5F_build_actual_name(const H5F_t *f, const H5P_genplist_t *fapl, - const char *name, char ** /*out*/ actual_name);/* Declare a free list to manage the H5F_t struct */ + const char *name, char ** /*out*/ actual_name); /*********************/ @@ -194,7 +194,7 @@ H5F_get_access_plist(H5F_t *f, hbool_t app_ref) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set 'small data' cache size") if(H5P_set(new_plist, H5F_ACS_LATEST_FORMAT_NAME, &(f->shared->latest_format)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set 'latest format' flag") - if(H5P_set(new_plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &(f->read_attempts)) < 0) + if(H5P_set(new_plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &(f->shared->read_attempts)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, FAIL, "can't set 'latest format' flag") if(f->shared->efc) efc_size = H5F_efc_max_nfiles(f->shared->efc); @@ -655,6 +655,12 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get sieve buffer size") if(H5P_get(plist, H5F_ACS_LATEST_FORMAT_NAME, &(f->shared->latest_format)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get 'latest format' flag") + /* Require the latest format to use SWMR */ + /* (Need to revisit this when the 1.10 release is made, and require + * 1.10 or later -QAK) + */ + if(!H5F_USE_LATEST_FORMAT(f) && (H5F_INTENT(f) & H5F_ACC_SWMR_WRITE)) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "must use 'latest format' flag with SWMR write access") if(H5P_get(plist, H5F_ACS_META_BLOCK_SIZE_NAME, &(f->shared->meta_aggr.alloc_size)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get metadata cache size") f->shared->meta_aggr.feature_flag = H5FD_FEAT_AGGREGATE_METADATA; @@ -673,6 +679,10 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad maximum address from VFD") if(H5FD_get_feature_flags(lf, &f->shared->feature_flags) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get feature flags from VFD") + /* Require a POSIX compatible VFD to use SWMR feature */ + /* (It's reasonable to try to expand this to other VFDs eventually -QAK) */ + if(!H5F_HAS_FEATURE(f, H5FD_FEAT_POSIX_COMPAT_HANDLE) && (H5F_INTENT(f) & (H5F_ACC_SWMR_WRITE | H5F_ACC_SWMR_READ))) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "must use POSIX compatible VFD with SWMR write access") if(H5FD_get_fs_type_map(lf, f->shared->fs_type_map) < 0) HGOTO_ERROR(H5E_FILE, H5E_CANTGET, NULL, "can't get free space type mapping from VFD") if(H5MF_init_merge_flags(f) < 0) @@ -689,6 +699,37 @@ H5F_new(H5F_file_t *shared, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_t */ f->shared->use_tmp_space = !H5F_HAS_FEATURE(f, H5FD_FEAT_HAS_MPI); + /* Retrieve the # of read attempts here so that sohm in superblock will get the correct # of attempts */ + if(H5P_get(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &f->shared->read_attempts) < 0) + HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get the # of read attempts") + + /* When opening file with SWMR access, the # of read attempts is H5F_SWMR_METADATA_READ_ATTEMPTS if not set */ + /* When opening file without SWMR access, the # of read attempts is always H5F_METADATA_READ_ATTEMPTS (set or not set) */ + if(H5F_INTENT(f) & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE)) { + /* If no value for read attempts has been set, use the default */ + if(!f->shared->read_attempts) + f->shared->read_attempts = H5F_SWMR_METADATA_READ_ATTEMPTS; + + /* Turn off accumulator with SWMR */ + f->shared->feature_flags = lf->feature_flags & ~(unsigned)H5FD_FEAT_ACCUMULATE_METADATA; + if(H5FD_set_feature_flags(lf, f->shared->feature_flags) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "can't set feature_flags in VFD") + } /* end if */ + else + f->shared->read_attempts = H5F_METADATA_READ_ATTEMPTS; + + /* Determine the # of bins for metdata read retries */ + f->shared->retries_nbins = 0; + if(f->shared->read_attempts > 1) { + double tmp; + + tmp = HDlog10((double)(f->shared->read_attempts - 1)); + f->shared->retries_nbins = (unsigned)tmp + 1; + } /* end if */ + + /* Initialize the tracking for metadata read retries */ + HDmemset(f->shared->retries, 0, sizeof(f->shared->retries)); + /* * Create a metadata cache with the specified number of elements. * The cache might be created with a different number of elements and @@ -949,7 +990,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, H5FD_class_t *drvr; /*file driver class info */ H5P_genplist_t *a_plist; /*file access property list */ H5F_close_degree_t fc_degree; /*file close degree */ - unsigned i; /*local index variable */ H5F_t *ret_value; /*actual return value */ FUNC_ENTER_NOAPI(NULL) @@ -1065,40 +1105,6 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, shared = file->shared; lf = shared->lf; - file->open_name = H5MM_xstrdup(name); - - /* Get the file access property list, for future queries */ - if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") - - /* Retrieve the # of read attempts here so that sohm in superblock will get the correct # of attempts */ - if(H5P_get(a_plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, &file->read_attempts) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get the # of read attempts") - - /* When opening file with SWMR access, the # of read attempts is H5F_SWMR_METADATA_READ_ATTEMPTS if not set */ - /* When opening file with non-SWMR access, the # of read attempts is always H5F_METADATA_READ_ATTEMPTS (set or not set) */ - if(H5F_INTENT(file) & H5F_ACC_SWMR_READ || H5F_INTENT(file)& H5F_ACC_SWMR_WRITE) { - if(!file->read_attempts) - file->read_attempts = H5F_SWMR_METADATA_READ_ATTEMPTS; - /* Turn off accumulator */ - shared->feature_flags = lf->feature_flags & ~(unsigned)H5FD_FEAT_ACCUMULATE_METADATA; - if(H5FD_set_feature_flags(lf, shared->feature_flags) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTSET, NULL, "can't set feature_flags in VFD") - } else - file->read_attempts = H5F_METADATA_READ_ATTEMPTS; - - /* Determine the # of bins for metdata read retries */ - file->retries_nbins = 0; - if(file->read_attempts > 1) { - double tmp; - tmp = HDlog10((double)(file->read_attempts - 1)); - file->retries_nbins = (unsigned)tmp + 1; - } - - /* Initialize the tracking for metadata read retries */ - for(i = 0; i < H5AC_NTYPES; i++) - file->retries[i] = NULL; - /* * Read or write the file superblock, depending on whether the file is * empty or not. @@ -1130,6 +1136,9 @@ H5F_open(const char *name, unsigned flags, hid_t fcpl_id, hid_t fapl_id, HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "unable to read root group") } /* end if */ + /* Get the file access property list, for future queries */ + if(NULL == (a_plist = (H5P_genplist_t *)H5I_object(fapl_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, NULL, "not file access property list") /* * Decide the file close degree. If it's the first time to open the * file, set the degree to access property list value; if it's the @@ -2099,8 +2108,8 @@ done: * Purpose: To track the # of a "retries" (log10) for a metadata item. * This routine should be used only when: * "retries" > 0 - * f->read_attempts > 1 (does not have retry when 1) - * f->retries_nbins > 0 (calculated based on f->read_attempts) + * f->shared->read_attempts > 1 (does not have retry when 1) + * f->shared->retries_nbins > 0 (calculated based on f->shared->read_attempts) * * Return: Success: SUCCEED * Failure: FAIL @@ -2113,31 +2122,31 @@ herr_t H5F_track_metadata_read_retries(H5F_t *f, unsigned actype, unsigned retries) { unsigned log_ind; /* Index to the array of retries based on log10 of retries */ - double tmp; + double tmp; /* Temporary value, to keep compiler quiet */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) /* Sanity check */ HDassert(f); - HDassert(f->read_attempts > 1); - HDassert(f->retries_nbins > 0); + HDassert(f->shared->read_attempts > 1); + HDassert(f->shared->retries_nbins > 0); HDassert(retries > 0); - HDassert(retries < f->read_attempts); + HDassert(retries < f->shared->read_attempts); HDassert(actype < H5AC_NTYPES); /* Allocate memory for retries */ - if(f->retries[actype] == NULL) - if((f->retries[actype] = (uint32_t *)HDcalloc((size_t)f->retries_nbins, sizeof(uint32_t))) == NULL) + if(NULL == f->shared->retries[actype]) + if(NULL == (f->shared->retries[actype] = (uint32_t *)HDcalloc((size_t)f->shared->retries_nbins, sizeof(uint32_t)))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") /* Index to retries based on log10 */ tmp = HDlog10((double)retries); log_ind = (unsigned)tmp; - HDassert(log_ind < f->retries_nbins); + HDassert(log_ind < f->shared->retries_nbins); /* Increment the # of the "retries" */ - f->retries[actype][log_ind]++; + f->shared->retries[actype][log_ind]++; done: FUNC_LEAVE_NOAPI(ret_value) diff --git a/src/H5Fio.c b/src/H5Fio.c index 296a707..82e957c 100644 --- a/src/H5Fio.c +++ b/src/H5Fio.c @@ -266,6 +266,8 @@ done: * Purpose: Decode checksum stored in the buffer * Calculate checksum for the data in the buffer * + * Note: Assumes that the checksum is the last data in the buffer + * * Return: Non-negative on success/Negative on failure * * Programmer: Vailin Choi; Sept 2013 @@ -273,34 +275,28 @@ done: *------------------------------------------------------------------------- */ herr_t -H5F_get_checksums(uint8_t *buf, size_t chk_size, uint32_t *s_chksum/*out*/, uint32_t *c_chksum/*out*/) +H5F_get_checksums(const uint8_t *buf, size_t buf_size, uint32_t *s_chksum/*out*/, uint32_t *c_chksum/*out*/) { - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ - const uint8_t *chk_p; /* Pointer into raw data buffer */ - FUNC_ENTER_NOAPI_NOINIT_NOERR /* Check arguments */ HDassert(buf); - HDassert(chk_size); + HDassert(buf_size); - /* Compute checksum for the buffer */ - computed_chksum = H5_checksum_metadata(buf, chk_size - H5_SIZEOF_CHKSUM, 0); + /* Return the stored checksum */ + if(s_chksum) { + const uint8_t *chk_p; /* Pointer into raw data buffer */ - /* Offset to the checksum in the buffer */ - chk_p = buf + chk_size - H5_SIZEOF_CHKSUM; + /* Offset to the checksum in the buffer */ + chk_p = buf + buf_size - H5_SIZEOF_CHKSUM; - /* Decode the checksum stored in the buffer */ - UINT32DECODE(chk_p, stored_chksum); + /* Decode the checksum stored in the buffer */ + UINT32DECODE(chk_p, *s_chksum); + } /* end if */ - /* Return the stored checksum */ - if(s_chksum) - *s_chksum = stored_chksum; - - /* Return the computed checksum */ + /* Return the computed checksum for the buffer */ if(c_chksum) - *c_chksum = computed_chksum; + *c_chksum = H5_checksum_metadata(buf, buf_size - H5_SIZEOF_CHKSUM, 0); FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5F_get_chksums() */ @@ -311,13 +307,14 @@ H5F_get_checksums(uint8_t *buf, size_t chk_size, uint32_t *s_chksum/*out*/, uint * * Purpose: Attempts to read and validate a piece of meatadata that has * checksum as follows: - * a) read the piece of metadata - * b) calculate checksum for the buffer of metadata - * c) decode the checksum stored in the buffer of metadata - * d) compare the computed checksum with its stored checksum + * a) read the piece of metadata + * b) calculate checksum for the buffer of metadata + * c) decode the checksum stored in the buffer of metadata + * d) compare the computed checksum with its stored checksum + * * The library will perform (a) to (d) above for "f->read_attempts" * times or until the checksum comparison in (d) passes. - * This routine also tracks the # of retries via + * This routine also records the # of retries via * H5F_track_metadata_read_retries() * * Return: Non-negative on success/Negative on failure @@ -327,8 +324,9 @@ H5F_get_checksums(uint8_t *buf, size_t chk_size, uint32_t *s_chksum/*out*/, uint *------------------------------------------------------------------------- */ herr_t -H5F_read_check_metadata(H5F_t *f, H5FD_mem_t type, unsigned actype, haddr_t addr, size_t read_size, size_t chk_size, - hid_t dxpl_id, uint8_t *buf/*out*/, uint32_t *chksum/*out*/) +H5F_read_check_metadata(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, + unsigned actype, haddr_t addr, size_t read_size, size_t chk_size, + uint8_t *buf/*out*/) { unsigned tries, max_tries; /* The # of read attempts */ unsigned retries; /* The # of retries */ @@ -339,7 +337,7 @@ H5F_read_check_metadata(H5F_t *f, H5FD_mem_t type, unsigned actype, haddr_t addr FUNC_ENTER_NOAPI(FAIL) /* Get the # of read attempts */ - max_tries = tries = f->read_attempts; + max_tries = tries = H5F_GET_READ_ATTEMPTS(f); do { /* Read header from disk */ @@ -352,25 +350,20 @@ H5F_read_check_metadata(H5F_t *f, H5FD_mem_t type, unsigned actype, haddr_t addr /* Verify checksum */ if(stored_chksum == computed_chksum) break; - } while(--tries); + /* Check for too many tries */ if(tries == 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "incorrect metadatda checksum after all read attempts (%u) for %u bytes:c_chksum=%u, s_chkum=%u", max_tries, chk_size, computed_chksum, stored_chksum) /* Calculate and track the # of retries */ retries = max_tries - tries; - if(retries) { /* Does not track 0 retry */ - HDfprintf(stderr, "%s: SUCCESS after %u retries; actype=%u\n", FUNC, retries, actype); + if(retries) /* Does not track 0 retry */ if(H5F_track_metadata_read_retries(f, actype, retries) < 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, FAIL, "cannot track read tries = %u ", retries) - } - - /* Return the computed checksum */ - if(chksum) - *chksum = computed_chksum; done: FUNC_LEAVE_NOAPI(ret_value); } /* end H5F_read_check_metadata */ + diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index c74384d..051cd2b 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -251,6 +251,11 @@ struct H5F_file_t { /* Metadata accumulator information */ H5F_meta_accum_t accum; /* Metadata accumulator info */ + + /* Metadata retry info */ + unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */ + unsigned retries_nbins; /* # of bins for each retries[] */ + uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metdata items with checksum */ }; /* @@ -259,7 +264,6 @@ struct H5F_file_t { * to shared H5F_file_t structs. */ struct H5F_t { - unsigned read_attempts; /* The # of reads to try when reading metadata with checksum */ char *open_name; /* Name used to open file */ char *actual_name; /* Actual name of the file, after resolving symlinks, etc. */ char *extpath; /* Path for searching target external link file */ @@ -270,8 +274,6 @@ struct H5F_t { hbool_t closing; /* File is in the process of being closed */ struct H5F_t *parent; /* Parent file that this file is mounted to */ unsigned nmounts; /* Number of children mounted to this file */ - uint32_t *retries[H5AC_NTYPES]; /* Track # of read retries for metdata items with checksum */ - unsigned retries_nbins; /* # of bins for each retries[] */ }; diff --git a/src/H5Fprivate.h b/src/H5Fprivate.h index 839fa0a..6a47abc 100644 --- a/src/H5Fprivate.h +++ b/src/H5Fprivate.h @@ -283,7 +283,7 @@ #define H5F_FILE_ID(F) ((F)->file_id) #define H5F_PARENT(F) ((F)->parent) #define H5F_NMOUNTS(F) ((F)->nmounts) -#define H5F_GET_READ_ATTEMPTS(F) ((F)->read_attempts) +#define H5F_GET_READ_ATTEMPTS(F) ((F)->shared->read_attempts) #define H5F_DRIVER_ID(F) ((F)->shared->lf->driver_id) #define H5F_GET_FILENO(F,FILENUM) ((FILENUM) = (F)->shared->lf->fileno) #define H5F_HAS_FEATURE(F,FL) ((F)->shared->lf->feature_flags & (FL)) @@ -326,7 +326,7 @@ #define H5F_FILE_ID(F) (H5F_get_file_id(F)) #define H5F_PARENT(F) (H5F_get_parent(F)) #define H5F_NMOUNTS(F) (H5F_get_nmounts(F)) -#define H5F_GET_READ_ATTEMPTS(F) (H5F_get_read_attempts(F)) +#define H5F_GET_READ_ATTEMPTS(F) (H5F_get_read_attempts(F)) #define H5F_DRIVER_ID(F) (H5F_get_driver_id(F)) #define H5F_GET_FILENO(F,FILENUM) (H5F_get_fileno((F), &(FILENUM))) #define H5F_HAS_FEATURE(F,FL) (H5F_has_feature(F,FL)) @@ -498,17 +498,20 @@ /* Default free space section threshold used by free-space managers */ #define H5F_FREE_SPACE_THRESHOLD_DEF 1 +/* Metadata read attempt values */ +#define H5F_METADATA_READ_ATTEMPTS 1 /* Default # of read attempts for non-SWMR access */ +#define H5F_SWMR_METADATA_READ_ATTEMPTS 100 /* Default # of read attempts for SWMR access */ + /* Macros to define signatures of all objects in the file */ /* Size of signature information (on disk) */ /* (all on-disk signatures should be this length) */ #define H5_SIZEOF_MAGIC 4 +/* Size of checksum information (on disk) */ +/* (all on-disk checksums should be this length) */ #define H5_SIZEOF_CHKSUM 4 -#define H5F_METADATA_READ_ATTEMPTS 1 /* Default # of read attempts for non-SWMR access */ -#define H5F_SWMR_METADATA_READ_ATTEMPTS 100 /* Default # of read attempts for SWMR access */ - /* v1 B-tree node signature */ #define H5B_MAGIC "TREE" @@ -675,10 +678,12 @@ H5_DLL herr_t H5F_block_write(const H5F_t *f, H5FD_mem_t type, haddr_t addr, H5_DLL herr_t H5F_flush_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id); H5_DLL herr_t H5F_evict_tagged_metadata(H5F_t * f, haddr_t tag, hid_t dxpl_id); -/* Function that read and verify a piece of metadata with checksum */ -H5_DLL herr_t H5F_read_check_metadata(H5F_t *f, H5FD_mem_t type, unsigned actype, haddr_t addr, size_t read_size, size_t chk_size, - hid_t dxpl_id, uint8_t *buf/*out*/, uint32_t *chksum/*out*/); -H5_DLL herr_t H5F_get_checksums(uint8_t *buf, size_t chk_size, uint32_t *s_chksum, uint32_t *c_chksum); +/* Functions that read & verify a piece of metadata with checksum */ +H5_DLL herr_t H5F_read_check_metadata(H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, + unsigned actype, haddr_t addr, size_t read_size, size_t chk_size, + uint8_t *buf/*out*/); +H5_DLL herr_t H5F_get_checksums(const uint8_t *buf, size_t chk_size, uint32_t *s_chksum, uint32_t *c_chksum); + /* Routine to track the # of retries */ H5_DLL herr_t H5F_track_metadata_read_retries(H5F_t *f, unsigned actype, unsigned retries); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 37dcc93..36df854 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -188,12 +188,12 @@ typedef enum H5F_file_space_type_t { } H5F_file_space_type_t; /* Data structure to report the collection of read retries for metadata items with checksum */ -/* Used by public routine H5Fget_metadata_read_retries_info() */ -#define NUM_METADATA_READ_RETRIES 21 -typedef struct H5F_retries_info_t { +/* Used by public routine H5Fget_metadata_read_retry_info() */ +#define H5F_NUM_METADATA_READ_RETRY_TYPES 21 +typedef struct H5F_retry_info_t { unsigned nbins; - uint32_t *retries[NUM_METADATA_READ_RETRIES]; -} H5F_retries_info_t; + uint32_t *retries[H5F_NUM_METADATA_READ_RETRY_TYPES]; +} H5F_retry_info_t; #ifdef __cplusplus extern "C" { @@ -232,7 +232,7 @@ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id, H5_DLL herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id); H5_DLL ssize_t H5Fget_name(hid_t obj_id, char *name, size_t size); H5_DLL herr_t H5Fget_info2(hid_t obj_id, H5F_info2_t *finfo); -H5_DLL herr_t H5Fget_metadata_read_retries_info(hid_t file_id, H5F_retries_info_t *info); +H5_DLL herr_t H5Fget_metadata_read_retry_info(hid_t file_id, H5F_retry_info_t *info); H5_DLL ssize_t H5Fget_free_sections(hid_t file_id, H5F_mem_t type, size_t nsects, H5F_sect_info_t *sect_info/*out*/); H5_DLL herr_t H5Fclear_elink_file_cache(hid_t file_id); diff --git a/src/H5Fquery.c b/src/H5Fquery.c index 69fe193..fb3b490 100644 --- a/src/H5Fquery.c +++ b/src/H5Fquery.c @@ -340,7 +340,7 @@ H5F_get_read_attempts(const H5F_t *f) HDassert(f); - FUNC_LEAVE_NOAPI(f->read_attempts) + FUNC_LEAVE_NOAPI(f->shared->read_attempts) } /* end H5F_get_read_attempts() */ diff --git a/src/H5Fsuper_cache.c b/src/H5Fsuper_cache.c index 2b5239d..c55abbe 100644 --- a/src/H5Fsuper_cache.c +++ b/src/H5Fsuper_cache.c @@ -135,17 +135,16 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t H5_ATTR_UNUSED addr, void *_uda haddr_t eof; /* end of file address */ uint8_t sizeof_addr; /* Size of offsets in the file (in bytes) */ uint8_t sizeof_size; /* Size of lengths in the file (in bytes) */ - const size_t fixed_size = H5F_SUPERBLOCK_FIXED_SIZE; /*fixed sizeof superblock */ + const size_t fixed_size = H5F_SUPERBLOCK_FIXED_SIZE; /*fixed sizeof superblock */ size_t variable_size; /*variable sizeof superblock */ uint8_t *p; /* Temporary pointer into encoding buffer */ unsigned super_vers; /* Superblock version */ - hbool_t *dirtied = (hbool_t *)_udata; /* Set up dirtied out value */ + hbool_t *dirtied = (hbool_t *)_udata; /* Set up dirtied out value */ unsigned tries, max_tries; /* The # of read attempts to try */ - unsigned fixed_tries; /* The # of read attempts to try for the fixed-size portion */ unsigned retries; /* The # of retries */ uint32_t computed_chksum; /* Computed checksum */ uint32_t stored_chksum; /* Checksum read from file */ - H5F_super_t *ret_value; /* Return value */ + H5F_super_t *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -183,36 +182,23 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t H5_ATTR_UNUSED addr, void *_uda H5_CHECK_OVERFLOW(fixed_size, size_t, haddr_t); /* Get the # of read attempts */ - tries = max_tries = f->read_attempts; + tries = max_tries = H5F_GET_READ_ATTEMPTS(f); + retries = 0; do { - fixed_tries = max_tries; - do { - /* Read fixed-size portion of the superblock */ - p = sbuf; - if(H5FD_set_eoa(lf, H5FD_MEM_SUPER, (haddr_t)fixed_size) < 0) - HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "set end of space allocation request failed") - if(H5FD_read(lf, dxpl, H5FD_MEM_SUPER, (haddr_t)0, fixed_size, p) < 0) - HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock") - - /* Skip over signature (already checked when locating the superblock) */ - p += H5F_SIGNATURE_LEN; - - /* Superblock version */ - super_vers = *p++; - - /* A valid version # */ - if(super_vers <= HDF5_SUPERBLOCK_VERSION_LATEST) - break; - - } while (--fixed_tries); - - if(fixed_tries == 0) - /* After all tries (for SWMR access) or after 1 try (for non-SWMR) */ - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad superblock version number after all tries") - else if((max_tries - fixed_tries + 1) > 1) - HDfprintf(stderr, "%s: SUCCESS after %u attempts for fixed data\n", FUNC, max_tries - fixed_tries + 1); + /* Read fixed-size portion of the superblock */ + p = sbuf; + if(H5FD_set_eoa(lf, H5FD_MEM_SUPER, (haddr_t)fixed_size) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTINIT, NULL, "set end of space allocation request failed") + if(H5FD_read(lf, dxpl, H5FD_MEM_SUPER, (haddr_t)0, fixed_size, p) < 0) + HGOTO_ERROR(H5E_FILE, H5E_READERROR, NULL, "unable to read superblock") + + /* Skip over signature (already checked when locating the superblock) */ + p += H5F_SIGNATURE_LEN; - /* Set superblock version in property list */ + /* Superblock version */ + super_vers = *p++; + if(super_vers > HDF5_SUPERBLOCK_VERSION_LATEST) + HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "bad superblock version number") if(H5P_set(c_plist, H5F_CRT_SUPER_VERS_NAME, &super_vers) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, NULL, "unable to set superblock version") @@ -240,7 +226,7 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t H5_ATTR_UNUSED addr, void *_uda /* Decode size of file addresses at this point to get the correct H5F_SIZEOF_ADDR(f) for checksum */ sizeof_addr = *p++; if(sizeof_addr == 2 || sizeof_addr == 4 || - sizeof_addr == 8 || sizeof_addr == 16 || sizeof_addr == 32) { + sizeof_addr == 8 || sizeof_addr == 16 || sizeof_addr == 32) { if(H5P_set(c_plist, H5F_CRT_ADDR_BYTE_NUM_NAME, &sizeof_addr) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTSET, NULL, "unable to set byte number in an address") @@ -252,19 +238,17 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t H5_ATTR_UNUSED addr, void *_uda /* Verify correct checksum */ if(stored_chksum == computed_chksum) break; - } + } /* end if */ } while (--tries); if(tries == 0) /* After all tries (for SWMR access) or after 1 try (for non-SWMR) */ - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad checksum or bad byte number in superblock") + HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad checksum or bad byte value in superblock") - retries = max_tries - tries; - if(retries) { /* Does not track 0 retry */ - HDfprintf(stderr, "%s: SUCCESS after %u retries\n", FUNC, retries); + retries += (max_tries - tries); + if(retries) /* Does not track 0 retry */ if(H5F_track_metadata_read_retries(f, H5AC_SUPERBLOCK_ID, retries) < 0) HGOTO_ERROR(H5E_FILE, H5E_BADVALUE, NULL, "cannot track read retries = %u ", retries) - } /* Check for older version of superblock format */ if(super_vers < HDF5_SUPERBLOCK_VERSION_2) { @@ -451,7 +435,7 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t H5_ATTR_UNUSED addr, void *_uda } /* end if */ else { - /* Already decode sizeof_addr when validating checksum up there */ + /* Already decoded sizeof_addr (and set property) when validating checksum earlier */ /* Size of file sizes */ sizeof_size = *p++; @@ -473,12 +457,7 @@ H5F_sblock_load(H5F_t *f, hid_t dxpl_id, haddr_t H5_ATTR_UNUSED addr, void *_uda H5F_addr_decode(f, (const uint8_t **)&p, &stored_eof/*out*/); H5F_addr_decode(f, (const uint8_t **)&p, &sblock->root_addr/*out*/); - /* Decode checksum */ - UINT32DECODE(p, stored_chksum); - - /* Verify correct checksum with checksum computed via H5F_get_checksums() */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad checksum on superblock") + /* Already decoded and compared stored checksum earlier, when reading */ /* * Check if superblock address is different from base address and diff --git a/src/H5HFcache.c b/src/H5HFcache.c index 70dd569..4b02e06 100644 --- a/src/H5HFcache.c +++ b/src/H5HFcache.c @@ -1,4 +1,4 @@ -/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * +/* / * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * Copyright by The HDF Group. * * Copyright by the Board of Trustees of the University of Illinois. * * All rights reserved. * @@ -59,6 +59,14 @@ /* Size of stack buffer for serialized indirect blocks */ #define H5HF_IBLOCK_BUF_SIZE 4096 +/* Size of minimum header to decode for determining the full header size */ +#define H5HF_MIN_HEADER_SIZE ( \ + H5_SIZEOF_MAGIC + /* Signature */ \ + 1 + /* Version */ \ + 2 + /* Heap ID len */ \ + 2 /* I/O filters' encoded len */ \ + ) + /******************/ /* Local Typedefs */ @@ -78,10 +86,6 @@ static herr_t H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable); static herr_t H5HF__dtable_decode(H5F_t *f, const uint8_t **pp, H5HF_dtable_t *dtable); -/* Decode minimum header and full header routines */ -static herr_t H5HF_min_header_decode(const uint8_t **pp, H5HF_hdr_t *hdr); -static herr_t H5HF_full_header_decode(H5F_t *f, hid_t dxpl_id, const uint8_t **pp, H5HF_hdr_t *hdr); - /* Metadata cache (H5AC) callbacks */ static H5HF_hdr_t *H5HF_cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *udata); static herr_t H5HF_cache_hdr_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5HF_hdr_t *hdr, unsigned H5_ATTR_UNUSED * flags_ptr); @@ -266,138 +270,6 @@ H5HF__dtable_encode(H5F_t *f, uint8_t **pp, const H5HF_dtable_t *dtable) FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5HF__dtable_encode() */ -/**************************************************/ -/* metadata cache callback definitions for header */ -/**************************************************/ - - -/*------------------------------------------------------------------------- - * Function: H5HF_min_header_decode - * - * Purpose: Decodes enough info in the header to determine full header size - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Vailin Choi; Sept 2013 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5HF_min_header_decode(const uint8_t **pp, H5HF_hdr_t *hdr) -{ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT - - /* Check arguments */ - HDassert(pp && *pp); - HDassert(hdr); - - /* Magic number */ - if(HDmemcmp(*pp, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) - HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "wrong fractal heap header signature") - *pp += H5_SIZEOF_MAGIC; - - /* Version */ - if(**pp != H5HF_HDR_VERSION) - HGOTO_ERROR(H5E_HEAP, H5E_VERSION, FAIL, "wrong fractal heap header version") - (*pp)++; - - /* General heap information */ - UINT16DECODE(*pp, hdr->id_len); /* Heap ID length */ - UINT16DECODE(*pp, hdr->filter_len); /* I/O filters' encoded length */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5HF_min_header_decode() */ - - -/*------------------------------------------------------------------------- - * Function: H5HF_full_header_decode - * - * Purpose: Decodes info for the full header (fixed-len + variable-len) - * - * Return: Non-negative on success/Negative on failure - * - * Programmer: Vailin Choi; Sept 2013 - * - *------------------------------------------------------------------------- - */ -static herr_t -H5HF_full_header_decode(H5F_t *f, hid_t dxpl_id, const uint8_t **pp, H5HF_hdr_t *hdr) -{ - uint8_t heap_flags; /* Status flags for heap */ - herr_t ret_value = SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT - - /* Check arguments */ - HDassert(pp && *pp); - HDassert(hdr); - - /* Decode the minimum header info */ - if(H5HF_min_header_decode(pp, hdr) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode minimum header info") - - /* Heap status flags */ - /* (bit 0: "huge" object IDs have wrapped) */ - /* (bit 1: checksum direct blocks) */ - heap_flags = **pp; - (*pp)++; - - hdr->huge_ids_wrapped = heap_flags & H5HF_HDR_FLAGS_HUGE_ID_WRAPPED; - hdr->checksum_dblocks = heap_flags & H5HF_HDR_FLAGS_CHECKSUM_DBLOCKS; - - /* "Huge" object information */ - UINT32DECODE(*pp, hdr->max_man_size); /* Max. size of "managed" objects */ - H5F_DECODE_LENGTH(f, *pp, hdr->huge_next_id); /* Next ID to use for "huge" object */ - H5F_addr_decode(f, pp, &hdr->huge_bt2_addr); /* Address of "huge" object tracker B-tree */ - - /* "Managed" object free space information */ - H5F_DECODE_LENGTH(f, *pp, hdr->total_man_free); /* Internal free space in managed direct blocks */ - H5F_addr_decode(f, pp, &hdr->fs_addr); /* Address of free section header */ - - /* Heap statistics */ - H5F_DECODE_LENGTH(f, *pp, hdr->man_size); - H5F_DECODE_LENGTH(f, *pp, hdr->man_alloc_size); - H5F_DECODE_LENGTH(f, *pp, hdr->man_iter_off); - H5F_DECODE_LENGTH(f, *pp, hdr->man_nobjs); - H5F_DECODE_LENGTH(f, *pp, hdr->huge_size); - H5F_DECODE_LENGTH(f, *pp, hdr->huge_nobjs); - H5F_DECODE_LENGTH(f, *pp, hdr->tiny_size); - H5F_DECODE_LENGTH(f, *pp, hdr->tiny_nobjs); - - /* Managed objects' doubling-table info */ - if(H5HF__dtable_decode(f, pp, &(hdr->man_dtable)) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTENCODE, FAIL, "unable to encode managed obj. doubling table info") - - /* Check for I/O filter information to decode */ - if(hdr->filter_len > 0) { - H5O_pline_t *pline; /* Pipeline information from the header on disk */ - - /* Decode the size of a filtered root direct block */ - H5F_DECODE_LENGTH(f, *pp, hdr->pline_root_direct_size); - - /* Decode the filter mask for a filtered root direct block */ - UINT32DECODE(*pp, hdr->pline_root_direct_filter_mask); - - /* Decode I/O filter information */ - if(NULL == (pline = (H5O_pline_t *)H5O_msg_decode(f, dxpl_id, NULL, H5O_PLINE_ID, *pp))) - HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, FAIL, "can't decode I/O pipeline filters") - *pp += hdr->filter_len; - - /* Copy the information into the header's I/O pipeline structure */ - if(NULL == H5O_msg_copy(H5O_PLINE_ID, pline, &(hdr->pline))) - HGOTO_ERROR(H5E_HEAP, H5E_CANTCOPY, FAIL, "can't copy I/O filter pipeline") - - /* Release the space allocated for the I/O pipeline filters */ - H5O_msg_free(H5O_PLINE_ID, pline); - } /* end if */ - -done: - FUNC_LEAVE_NOAPI(ret_value) -} /* end H5HF_full_header_decode() */ - /*------------------------------------------------------------------------- * Function: H5HF_cache_hdr_load @@ -418,17 +290,18 @@ H5HF_cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) { H5HF_hdr_t *hdr = NULL; /* Fractal heap info */ H5HF_hdr_cache_ud_t *udata = (H5HF_hdr_cache_ud_t *)_udata; - size_t size, new_size; /* Header size */ + size_t min_size; /* Header size */ H5WB_t *wb = NULL; /* Wrapped buffer for header data */ uint8_t hdr_buf[H5HF_HDR_BUF_SIZE]; /* Buffer for header */ uint8_t *buf; /* Pointer to header buffer */ + size_t buf_size; /* Current size of temporary buffer */ const uint8_t *p; /* Pointer into raw data buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ - unsigned tries, max_tries; /* The # of read attempts */ - unsigned fixed_tries; /* The # of read attempts for the minimum portion */ - unsigned retries; /* The # of retries */ - H5HF_hdr_t *ret_value; /* Return value */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + unsigned tries, max_tries; /* The # of read attempts */ + unsigned retries; /* The # of retries */ + uint8_t heap_flags; /* Status flags for heap */ + H5HF_hdr_t *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -446,59 +319,64 @@ H5HF_cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, NULL, "can't wrap buffer") /* Compute the minimum size of the fractal heap header to determine filter info */ - size = (size_t)H5HF_MIN_HEADER_SIZE; + min_size = (size_t)H5HF_MIN_HEADER_SIZE; + + /* Get a pointer to a buffer that's large enough for serialized header */ + if(NULL == (buf = (uint8_t *)H5WB_actual(wb, min_size))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "can't get actual buffer") + buf_size = min_size; /* Get the # of read attempts */ - tries = max_tries = H5F_GET_READ_ATTEMPTS(f); + tries = max_tries = H5F_GET_READ_ATTEMPTS(udata->f); + retries = 0; do { - /* Get a pointer to a buffer that's large enough for serialized header */ - if(NULL == (buf = (uint8_t *)H5WB_actual(wb, size))) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "can't get actual buffer") + /* Read minimum header from disk */ + if(H5F_block_read(udata->f, H5FD_MEM_FHEAP_HDR, addr, min_size, dxpl_id, buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_READERROR, NULL, "can't read fractal heap header") - fixed_tries = max_tries; - do { - /* Read minimum header from disk */ - if(H5F_block_read(f, H5FD_MEM_FHEAP_HDR, addr, size, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_READERROR, NULL, "can't read fractal heap header") + /* Get temporary pointer to serialized header */ + p = buf; - /* Get temporary pointer to serialized header */ - p = buf; + /* Magic number */ + if(HDmemcmp(p, H5HF_HDR_MAGIC, (size_t)H5_SIZEOF_MAGIC)) + continue; /* Transfer control to while() and count this as a bad read, not an error */ + p += H5_SIZEOF_MAGIC; - /* Decode minimum header info */ - if(H5HF_min_header_decode(&p, hdr) >= 0) - break; - } while (--fixed_tries); + /* Version */ + if(*p++ != H5HF_HDR_VERSION) + continue; /* Transfer control to while() and count this as a bad read, not an error */ - if(fixed_tries == 0) - /* After all tries (for SWMR access) or after 1 try (for non-SWMR) */ - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad minimum header info in fractal heap header after all tries") - else if((max_tries - fixed_tries + 1) > 1) - HDfprintf(stderr, "%s: SUCCESS after %u attempts for minimum header in fractal heap\n", FUNC, max_tries - fixed_tries + 1); + /* General heap information */ + UINT16DECODE(p, hdr->id_len); /* Heap ID length */ + UINT16DECODE(p, hdr->filter_len); /* I/O filters' encoded length */ - /* Full header size */ - new_size = (size_t)H5HF_HEADER_SIZE(hdr); + /* Fixed-size header length */ + hdr->heap_size = (size_t)H5HF_HEADER_SIZE(hdr); /* Check for I/O filter information to decode */ if(hdr->filter_len > 0) - /* Compute the heap header's size: fixed-len header size + variable-len I/O filter information */ - hdr->heap_size = new_size - + (size_t)(hdr->sizeof_size /* Size of size for filtered root direct block */ + /* Add the variable-len I/O filter information */ + hdr->heap_size += (size_t)(hdr->sizeof_size /* Size of size for filtered root direct block */ + (unsigned)4 /* Size of filter mask for filtered root direct block */ + hdr->filter_len); /* Size of encoded I/O filter info */ - else - /* Set the heap header's size: fixed-len header size */ - hdr->heap_size = new_size; - /* Re-size current buffer */ - if(NULL == (buf = (uint8_t *)H5WB_actual(wb, hdr->heap_size))) - HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "can't get actual buffer") + /* Check if we need to resize the buffer */ + if(hdr->heap_size > buf_size) { + /* Re-size current buffer */ + if(NULL == (buf = (uint8_t *)H5WB_actual(wb, hdr->heap_size))) + HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "can't get actual buffer") + buf_size = hdr->heap_size; + } /* end if */ - /* Read the whole header with possibly filter info */ - if(H5F_block_read(f, H5FD_MEM_FHEAP_HDR, addr, hdr->heap_size, dxpl_id, buf) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_READERROR, NULL, "can't read fractal heap header's I/O pipeline filter info") + /* Read the whole header, possibly with filter info */ + /* (Can't use H5F_read_check_metadata() since the length from the + * minimum-sized header info might be incorrect. -QAK) + */ + if(H5F_block_read(udata->f, H5FD_MEM_FHEAP_HDR, addr, hdr->heap_size, dxpl_id, buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_READERROR, NULL, "can't read remainder of fractal heap header") /* Retrieve stored and computed checksums */ - H5F_get_checksums(buf, hdr->heap_size, &stored_chksum, &computed_chksum); + H5F_get_checksums(buf, hdr->heap_size, &stored_chksum, &computed_chksum); /* Verify checksum */ if(stored_chksum == computed_chksum) @@ -509,28 +387,72 @@ H5HF_cache_hdr_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum after all tries (%u) for fractal heap header", max_tries) /* Calculate and track the # of retry */ - retries = max_tries - tries; - if(retries) { /* Does not track 0 retry */ - HDfprintf(stderr, "%s: SUCCESS after %u retries\n", FUNC, retries); - if(H5F_track_metadata_read_retries(f, H5AC_FHEAP_HDR_ID, retries) < 0) + retries += (max_tries - tries); + if(retries) /* Does not track 0 retry */ + if(H5F_track_metadata_read_retries(udata->f, H5AC_FHEAP_HDR_ID, retries) < 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "cannot track read tries = %u ", retries) - } - p = buf; + /* Reset pointer into [possibly] re-sized buffer */ + /* (Skip over information decoded earlier) */ + p = buf + min_size; + + /* Heap status flags */ + /* (bit 0: "huge" object IDs have wrapped) */ + /* (bit 1: checksum direct blocks) */ + heap_flags = *p++; + hdr->huge_ids_wrapped = heap_flags & H5HF_HDR_FLAGS_HUGE_ID_WRAPPED; + hdr->checksum_dblocks = heap_flags & H5HF_HDR_FLAGS_CHECKSUM_DBLOCKS; - /* Decode full header info */ - if(H5HF_full_header_decode(hdr->f, udata->dxpl_id, &p, hdr) < 0) - HGOTO_ERROR(H5E_HEAP, H5E_CANTENCODE, NULL, "unable to decode full header info") + /* "Huge" object information */ + UINT32DECODE(p, hdr->max_man_size); /* Max. size of "managed" objects */ + H5F_DECODE_LENGTH(udata->f, p, hdr->huge_next_id); /* Next ID to use for "huge" object */ + H5F_addr_decode(udata->f, &p, &hdr->huge_bt2_addr); /* Address of "huge" object tracker B-tree */ - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* "Managed" object free space information */ + H5F_DECODE_LENGTH(udata->f, p, hdr->total_man_free); /* Internal free space in managed direct blocks */ + H5F_addr_decode(udata->f, &p, &hdr->fs_addr); /* Address of free section header */ - /* Sanity check */ - HDassert((size_t)(p - (const uint8_t *)buf) == hdr->heap_size); + /* Heap statistics */ + H5F_DECODE_LENGTH(udata->f, p, hdr->man_size); + H5F_DECODE_LENGTH(udata->f, p, hdr->man_alloc_size); + H5F_DECODE_LENGTH(udata->f, p, hdr->man_iter_off); + H5F_DECODE_LENGTH(udata->f, p, hdr->man_nobjs); + H5F_DECODE_LENGTH(udata->f, p, hdr->huge_size); + H5F_DECODE_LENGTH(udata->f, p, hdr->huge_nobjs); + H5F_DECODE_LENGTH(udata->f, p, hdr->tiny_size); + H5F_DECODE_LENGTH(udata->f, p, hdr->tiny_nobjs); + + /* Managed objects' doubling-table info */ + if(H5HF__dtable_decode(udata->f, &p, &(hdr->man_dtable)) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTENCODE, NULL, "unable to encode managed obj. doubling table info") + + /* Check for I/O filter information to decode */ + if(hdr->filter_len > 0) { + H5O_pline_t *pline; /* Pipeline information from the header on disk */ + + /* Decode the size of a filtered root direct block */ + H5F_DECODE_LENGTH(udata->f, p, hdr->pline_root_direct_size); + + /* Decode the filter mask for a filtered root direct block */ + UINT32DECODE(p, hdr->pline_root_direct_filter_mask); + + /* Decode I/O filter information */ + if(NULL == (pline = (H5O_pline_t *)H5O_msg_decode(udata->f, udata->dxpl_id, NULL, H5O_PLINE_ID, p))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTDECODE, NULL, "can't decode I/O pipeline filters") + p += hdr->filter_len; - /* Verify checksum with checksum computed via H5F_get_checksums() */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap header") + /* Copy the information into the header's I/O pipeline structure */ + if(NULL == H5O_msg_copy(H5O_PLINE_ID, pline, &(hdr->pline))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTCOPY, NULL, "can't copy I/O filter pipeline") + + /* Release the space allocated for the I/O pipeline filters */ + H5O_msg_free(H5O_PLINE_ID, pline); + } /* end if */ + + /* Already decoded and compared stored checksum earlier, when reading */ + + /* Sanity check */ + HDassert((size_t)(p - (const uint8_t *)buf) == (hdr->heap_size - H5_SIZEOF_CHKSUM)); /* Finish initialization of heap header */ if(H5HF_hdr_finish_init(hdr) < 0) @@ -854,8 +776,6 @@ H5HF_cache_iblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) uint8_t *buf; /* Temporary buffer */ const uint8_t *p; /* Pointer into raw data buffer */ haddr_t heap_addr; /* Address of heap header in the file */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ unsigned u; /* Local index variable */ H5HF_indirect_t *ret_value; /* Return value */ @@ -898,8 +818,8 @@ H5HF_cache_iblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) HGOTO_ERROR(H5E_HEAP, H5E_NOSPACE, NULL, "can't get actual buffer") /* Read and validate indirect block from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_FHEAP_IBLOCK, H5AC_FHEAP_IBLOCK_ID, addr, iblock->size, iblock->size, dxpl_id, buf, &computed_chksum) < 0) - HGOTO_ERROR(H5E_BTREE, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block") + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_FHEAP_IBLOCK, H5AC_FHEAP_IBLOCK_ID, addr, iblock->size, iblock->size, buf) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block") /* Get temporary pointer to serialized indirect block */ p = buf; @@ -995,15 +915,10 @@ H5HF_cache_iblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Sanity check */ HDassert(iblock->nchildren); /* indirect blocks w/no children should have been deleted */ - /* Metadata checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - (const uint8_t *)buf) == iblock->size); - - /* Verify checksum */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "incorrect metadata checksum for fractal heap indirect block") + HDassert((size_t)(p - (const uint8_t *)buf) == (iblock->size - H5_SIZEOF_CHKSUM)); /* Check if we have any indirect block children */ if(iblock->nrows > hdr->man_dtable.max_direct_rows) { @@ -1495,14 +1410,14 @@ H5HF_cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) H5HF_direct_t *dblock = NULL; /* Direct block info */ const uint8_t *p; /* Pointer into raw data buffer */ haddr_t heap_addr; /* Address of heap header in the file */ - uint32_t computed_chksum; /* Computed metadata checksum value */ - uint32_t stored_chksum; /* Metadata checksum value */ - unsigned tries, max_tries; /* The # of read attempts */ - unsigned retries; /* The # of retries */ - size_t chk_size; /* The size for validating checksum */ - uint8_t *chk_p; /* Pointer to the area for validating checksum */ - size_t read_size; /* Size of filtered direct block to read */ - H5HF_direct_t *ret_value; /* Return value */ + uint32_t computed_chksum; /* Computed metadata checksum value */ + uint32_t stored_chksum; /* Metadata checksum value */ + unsigned tries, max_tries; /* The # of read attempts */ + unsigned retries; /* The # of retries */ + size_t chk_size; /* The size for validating checksum */ + uint8_t *chk_p; /* Pointer to the area for validating checksum */ + size_t read_size; /* Size of filtered direct block to read */ + H5HF_direct_t *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -1603,10 +1518,12 @@ H5HF_cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Decode checksum on direct block, if requested */ chk_size = (size_t)(H5HF_MAN_ABS_DIRECT_OVERHEAD(hdr) - H5HF_SIZEOF_CHKSUM); chk_p = dblock->blk + chk_size; + /* Metadata checksum */ UINT32DECODE(chk_p, stored_chksum); chk_p -= H5HF_SIZEOF_CHKSUM; + /* Reset checksum field, for computing the checksum */ /* (Casting away const OK - QAK) */ HDmemset(chk_p, 0, (size_t)H5HF_SIZEOF_CHKSUM); @@ -1625,11 +1542,9 @@ H5HF_cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Calculate and track the # of retries */ retries = max_tries - tries; - if(retries) { /* Does not track 0 retry */ - HDfprintf(stderr, "%s: SUCCESS after %u retries\n", FUNC, retries); + if(retries) /* Does not track 0 retry */ if(H5F_track_metadata_read_retries(f, H5AC_FHEAP_DBLOCK_ID, retries) < 0) HGOTO_ERROR(H5E_HEAP, H5E_BADVALUE, NULL, "cannot track read retries = %u ", retries) - } /* Start decoding direct block */ p = dblock->blk; @@ -1661,7 +1576,7 @@ H5HF_cache_dblock_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Offset of heap within the heap's address space */ UINT64DECODE_VAR(p, dblock->block_off, hdr->heap_off_size); - /* Decode checksum on direct block, if requested */ + /* Advance past optional checksum on direct block, it was verified earlier */ if(hdr->checksum_dblocks) p += H5HF_SIZEOF_CHKSUM; diff --git a/src/H5HFpkg.h b/src/H5HFpkg.h index 4863edc..b0eba2f 100644 --- a/src/H5HFpkg.h +++ b/src/H5HFpkg.h @@ -105,14 +105,6 @@ + H5HF_DTABLE_INFO_SIZE(h) /* Size of managed obj. doubling-table info */ \ ) -/* Size of minimum header to decode for determining the full header size */ -#define H5HF_MIN_HEADER_SIZE ( \ - H5_SIZEOF_MAGIC + /* Signature */ \ - 1 + /* Version */ \ - 2 + /* Heap ID len */ \ - 2 /* I/O filters' encoded len */ \ - ) - /* Size of overhead for a direct block */ #define H5HF_MAN_ABS_DIRECT_OVERHEAD(h) ( \ /* General metadata fields */ \ diff --git a/src/H5Ocache.c b/src/H5Ocache.c index 6085222..8653817 100644 --- a/src/H5Ocache.c +++ b/src/H5Ocache.c @@ -317,18 +317,17 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) { H5O_t *oh = NULL; /* Object header read in */ H5O_cache_ud_t *udata = (H5O_cache_ud_t *)_udata; /* User data for callback */ - H5WB_t *wb = NULL; /* Wrapped buffer for prefix data */ uint8_t read_buf[H5O_SPEC_READ_SIZE]; /* Buffer for speculative read */ uint8_t *buf = NULL; /* Buffer to decode */ size_t spec_read_size; /* Size of buffer to speculatively read in */ size_t buf_size; /* Size of prefix+chunk #0 buffer */ haddr_t eoa; /* Relative end of file address */ unsigned tries, max_tries; /* The # of read attempts */ - unsigned retries; /* The # of retries */ - unsigned fixed_tries; /* The # of read attempts for prefix */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ - H5O_t *ret_value; /* Return value */ + unsigned retries; /* The # of retries */ + unsigned fixed_tries; /* The # of read attempts for prefix */ + uint32_t stored_chksum; /* Stored metadata checksum value */ + uint32_t computed_chksum;/* Computed metadata checksum value */ + H5O_t *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT @@ -350,17 +349,9 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) if(NULL == (oh = H5FL_CALLOC(H5O_t))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - /* Wrap the local buffer for serialized header info */ - if(NULL == (wb = H5WB_wrap(read_buf, sizeof(read_buf)))) - HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't wrap buffer") - /* Get the # of read attempts */ tries = max_tries = H5F_GET_READ_ATTEMPTS(f); do { - /* Get a pointer to a buffer that's large enough for serialized header */ - if(NULL == (buf = (uint8_t *)H5WB_actual(wb, spec_read_size))) - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "can't get actual buffer") - fixed_tries = max_tries; do { /* Attempt to speculatively read both object header prefix and first chunk */ @@ -370,45 +361,44 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Decode header prefix */ if(H5O_decode_prefix(f, oh, read_buf, udata) >= 0) break; - } while (--fixed_tries); if(fixed_tries == 0) /* After all tries (for SWMR access) or after 1 try (for non-SWMR) */ - HGOTO_ERROR(H5E_FILE, H5E_CANTOPENFILE, NULL, "bad object header prefix after all tries") - else if((max_tries - fixed_tries + 1) > 1) - HDfprintf(stderr, "%s: SUCCESS after %u attempts for fixed data\n", FUNC, max_tries - fixed_tries + 1); + HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "bad object header prefix after all tries") - /* Compute the size of the buffer used */ + /* Compute the size of the buffer required */ buf_size = oh->chunk0_size + (size_t)H5O_SIZEOF_HDR(oh); /* Check if the speculative read was large enough to parse the first chunk */ if(spec_read_size < buf_size) { - - /* Get a pointer to a buffer that's large enough for serialized header */ - if(NULL == (buf = (uint8_t *)H5WB_actual(wb, buf_size))) - HGOTO_ERROR(H5E_OHDR, H5E_NOSPACE, NULL, "can't get actual buffer") + /* Allocate space for a buffer that's large enough for serialized header */ + if(NULL == (buf = (uint8_t *)H5MM_malloc(buf_size))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTALLOC, NULL, "can't get actual buffer") /* SWMR access */ - if(H5F_INTENT(f) & H5F_ACC_SWMR_READ || H5F_INTENT(f) & H5F_ACC_SWMR_WRITE) { + if(H5F_INTENT(f) & (H5F_ACC_SWMR_READ | H5F_ACC_SWMR_WRITE)) { /* Read the chunk that is bigger than the speculative read size */ if(H5F_block_read(f, H5FD_MEM_OHDR, addr, buf_size, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header") + /* See if the data read now is the same as what is read initially */ if(HDmemcmp(buf, read_buf, spec_read_size)) { - HDmemset(oh, 0, sizeof(oh)); + HDmemset(oh, 0, sizeof(*oh)); + /* Decode the prefix again */ if(H5O_decode_prefix(f, oh, buf, udata) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, NULL, "can't decode object header prefix") - } - } else { + } /* end if */ + } /* end if */ + else { /* Copy existing raw data into new buffer */ HDmemcpy(buf, read_buf, spec_read_size); /* Read rest of the raw data */ if(H5F_block_read(f, H5FD_MEM_OHDR, (addr + spec_read_size), (buf_size - spec_read_size), dxpl_id, (buf + spec_read_size)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header data") - } + } /* end else */ } else buf = read_buf; @@ -429,11 +419,9 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Calculate and track the # of retries */ retries = max_tries - tries; - if(retries) { /* Does not track 0 retry */ - HDfprintf(stderr, "%s: SUCCESS after %u retries\n", FUNC, retries); + if(retries) /* Does not track 0 retry */ if(H5F_track_metadata_read_retries(f, H5AC_OHDR_ID, retries) < 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "cannot track read retries = %u ", retries) - } /* File-specific, non-stored information */ oh->sizeof_size = H5F_SIZEOF_SIZE(udata->common.f); @@ -461,8 +449,8 @@ H5O_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) done: /* Release resources */ - if(wb && H5WB_unwrap(wb) < 0) - HDONE_ERROR(H5E_OHDR, H5E_CLOSEERROR, NULL, "can't close wrapped buffer") + if(buf && buf != read_buf) + H5MM_free(buf); /* Release the [possibly partially initialized] object header on errors */ if(!ret_value && oh) @@ -827,13 +815,14 @@ H5O_cache_chk_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) /* Read rest of the raw data */ if(udata->oh->version == H5O_VERSION_2 && udata->decoding) { /* Read and validate object header continuation chunk */ - if(H5F_read_check_metadata(f, H5FD_MEM_OHDR, H5AC_OHDR_CHK_ID, addr, udata->size, udata->size, dxpl_id, buf, NULL) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_OHDR, H5AC_OHDR_CHK_ID, addr, udata->size, udata->size, buf) < 0) HGOTO_ERROR(H5E_OHDR, H5E_BADVALUE, NULL, "incorrect metadata checksum for object header continuation chunk") - } else { + } /* end if */ + else { /* Read the header object continuation chunk */ if(H5F_block_read(f, H5FD_MEM_OHDR, addr, udata->size, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_OHDR, H5E_READERROR, NULL, "unable to read object header continuation chunk") - } + } /* end else */ /* Check if we are still decoding the object header */ /* (as opposed to bringing a piece of it back from the file) */ diff --git a/src/H5Pfapl.c b/src/H5Pfapl.c index 163288c..8125e43 100644 --- a/src/H5Pfapl.c +++ b/src/H5Pfapl.c @@ -161,10 +161,10 @@ #define H5F_ACS_FILE_IMAGE_INFO_COPY H5P_file_image_info_copy #define H5F_ACS_FILE_IMAGE_INFO_CLOSE H5P_file_image_info_close /* Definition for # of metadata read attempts */ -#define H5F_ACS_METADATA_READ_ATTEMPTS_SIZE sizeof(unsigned) -#define H5F_ACS_METADATA_READ_ATTEMPTS_DEF 0 -#define H5F_ACS_METADATA_READ_ATTEMPTS_ENC H5P__encode_unsigned -#define H5F_ACS_METADATA_READ_ATTEMPTS_DEC H5P__decode_unsigned +#define H5F_ACS_METADATA_READ_ATTEMPTS_SIZE sizeof(unsigned) +#define H5F_ACS_METADATA_READ_ATTEMPTS_DEF 0 +#define H5F_ACS_METADATA_READ_ATTEMPTS_ENC H5P__encode_unsigned +#define H5F_ACS_METADATA_READ_ATTEMPTS_DEC H5P__decode_unsigned /* Definition of core VFD write tracking flag */ #define H5F_ACS_CORE_WRITE_TRACKING_FLAG_SIZE sizeof(hbool_t) #define H5F_ACS_CORE_WRITE_TRACKING_FLAG_DEF FALSE @@ -433,6 +433,7 @@ H5P_facc_reg_prop(H5P_genclass_t *pclass) NULL, NULL, NULL, H5F_ACS_METADATA_READ_ATTEMPTS_ENC, H5F_ACS_METADATA_READ_ATTEMPTS_DEC, NULL, NULL, NULL, NULL) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTINSERT, FAIL, "can't insert property into class") + done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5P_facc_reg_prop() */ @@ -3122,7 +3123,7 @@ H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts) H5TRACE2("e", "iIu", plist_id, attempts); /* Cannot set the # of attempts to 0 */ - if(attempts <= 0) + if(attempts == 0) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "number of metadatata read attempts must be greater than 0"); /* Get the plist structure */ @@ -3152,26 +3153,29 @@ done: herr_t H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts/*out*/) { - H5P_genplist_t *plist; /* Property list pointer */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_API(FAIL) H5TRACE2("e", "ix", plist_id, attempts); - /* Get the plist structure */ - if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) - HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") - /* Get values */ if(attempts) { + H5P_genplist_t *plist; /* Property list pointer */ + + /* Get the plist structure */ + if(NULL == (plist = H5P_object_verify(plist_id, H5P_FILE_ACCESS))) + HGOTO_ERROR(H5E_ATOM, H5E_BADATOM, FAIL, "can't find object for ID") + /* Get the # of read attempts set */ if(H5P_get(plist, H5F_ACS_METADATA_READ_ATTEMPTS_NAME, attempts) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't get the number of metadata read attempts") + /* If not set, return the default value */ - if(*attempts == H5F_ACS_METADATA_READ_ATTEMPTS_DEF) /* 0 */ + if(*attempts == H5F_ACS_METADATA_READ_ATTEMPTS_DEF) /* 0 */ *attempts = H5F_METADATA_READ_ATTEMPTS; - } + } /* end if */ done: FUNC_LEAVE_API(ret_value) } /* end H5Pget_metadata_read_attempts() */ + diff --git a/src/H5Ppublic.h b/src/H5Ppublic.h index 7f93dd7..4c29bd8 100644 --- a/src/H5Ppublic.h +++ b/src/H5Ppublic.h @@ -346,7 +346,7 @@ H5_DLL herr_t H5Pset_file_image_callbacks(hid_t fapl_id, H5_DLL herr_t H5Pget_file_image_callbacks(hid_t fapl_id, H5FD_file_image_callbacks_t *callbacks_ptr); H5_DLL herr_t H5Pset_metadata_read_attempts(hid_t plist_id, unsigned attempts); -H5_DLL herr_t H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts/*out*/); +H5_DLL herr_t H5Pget_metadata_read_attempts(hid_t plist_id, unsigned *attempts); H5_DLL herr_t H5Pset_core_write_tracking(hid_t fapl_id, hbool_t is_enabled, size_t page_size); H5_DLL herr_t H5Pget_core_write_tracking(hid_t fapl_id, hbool_t *is_enabled, size_t *page_size); diff --git a/src/H5SMcache.c b/src/H5SMcache.c index 72f5913..d4052ef 100644 --- a/src/H5SMcache.c +++ b/src/H5SMcache.c @@ -130,13 +130,11 @@ static H5SM_master_table_t * H5SM_table_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void H5_ATTR_UNUSED *udata) { H5SM_master_table_t *table = NULL; - H5WB_t *wb = NULL; /* Wrapped buffer for table data */ + H5WB_t *wb = NULL; /* Wrapped buffer for table data */ uint8_t tbl_buf[H5SM_TBL_BUF_SIZE]; /* Buffer for table */ - uint8_t *buf; /* Reading buffer */ - const uint8_t *p; /* Pointer into input buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ - size_t u; /* Counter variable for index headers */ + uint8_t *buf; /* Reading buffer */ + const uint8_t *p; /* Pointer into input buffer */ + size_t u; /* Counter variable for index headers */ H5SM_master_table_t *ret_value; FUNC_ENTER_NOAPI_NOINIT @@ -171,7 +169,7 @@ H5SM_table_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void H5_ATTR_UNUSED *udat HGOTO_ERROR(H5E_SOHM, H5E_NOSPACE, NULL, "can't get actual buffer") /* Read and validate shared message table from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_SOHM_TABLE, H5AC_SOHM_TABLE_ID, addr, table->table_size, table->table_size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_SOHM_TABLE, H5AC_SOHM_TABLE_ID, addr, table->table_size, table->table_size, buf) < 0) HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message table") /* Get temporary pointer to serialized table */ @@ -217,18 +215,12 @@ H5SM_table_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void H5_ATTR_UNUSED *udat H5F_addr_decode(f, &p, &(table->indexes[u].heap_addr)); /* Compute the size of a list index for this SOHM index */ - table->indexes[u].list_size = H5SM_LIST_SIZE(f, table->indexes[u].list_max); } /* end for */ - /* Read in checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - (const uint8_t *)buf) == table->table_size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message table") + HDassert((size_t)(p - (const uint8_t *)buf) == (table->table_size - H5_SIZEOF_CHKSUM)); /* Set return value */ ret_value = table; @@ -472,8 +464,6 @@ H5SM_list_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) uint8_t lst_buf[H5SM_LST_BUF_SIZE]; /* Buffer for list index */ uint8_t *buf; /* Reading buffer */ uint8_t *p; /* Pointer into input buffer */ - uint32_t stored_chksum; /* Stored metadata checksum value */ - uint32_t computed_chksum; /* Computed metadata checksum value */ size_t u; /* Counter variable for messages in list */ size_t chk_size; /* Exact size with checksum at the end */ H5SM_list_t *ret_value; /* Return value */ @@ -505,10 +495,9 @@ H5SM_list_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) chk_size = H5SM_LIST_SIZE(udata->f, udata->header->num_messages); /* Read and validate shared message list from disk */ - if(H5F_read_check_metadata(f, H5FD_MEM_SOHM_INDEX, H5AC_SOHM_LIST_ID, addr, udata->header->list_size, chk_size, dxpl_id, buf, &computed_chksum) < 0) + if(H5F_read_check_metadata(f, dxpl_id, H5FD_MEM_SOHM_INDEX, H5AC_SOHM_LIST_ID, addr, udata->header->list_size, chk_size, buf) < 0) HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message list") - /* Get temporary pointer to serialized list index */ p = buf; @@ -525,15 +514,10 @@ H5SM_list_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, void *_udata) p += H5SM_SOHM_ENTRY_SIZE(udata->f); } /* end for */ - /* Read in checksum */ - UINT32DECODE(p, stored_chksum); + /* Already decoded and compared stored checksum earlier, when reading */ /* Sanity check */ - HDassert((size_t)(p - buf) <= udata->header->list_size); - - /* Verify checksum with checksum computed via H5F_read_check_metadata() */ - if(stored_chksum != computed_chksum) - HGOTO_ERROR(H5E_SOHM, H5E_BADVALUE, NULL, "incorrect metadata checksum for shared message list") + HDassert((size_t)(p - buf) <= (udata->header->list_size - H5_SIZEOF_CHKSUM)); /* Initialize the rest of the array */ for(u = udata->header->num_messages; u < udata->header->list_max; u++) |