diff options
author | Vailin Choi <vchoi@hdfgroup.org> | 2007-07-13 21:12:25 (GMT) |
---|---|---|
committer | Vailin Choi <vchoi@hdfgroup.org> | 2007-07-13 21:12:25 (GMT) |
commit | d44b27ddcc66a8ed55f5069eabbf4c287ce1d99a (patch) | |
tree | ab22bfaa95add178894e6c649dc4b8aea0d196f4 | |
parent | 31a2af29ca7aa91fcd47eb018a02ff2e425c5b55 (diff) | |
download | hdf5-d44b27ddcc66a8ed55f5069eabbf4c287ce1d99a.zip hdf5-d44b27ddcc66a8ed55f5069eabbf4c287ce1d99a.tar.gz hdf5-d44b27ddcc66a8ed55f5069eabbf4c287ce1d99a.tar.bz2 |
[svn-r13978] purpose:
New feature.
Description:
Added routines to report on the amount of storage for:
1) 1.6 btree and heap storage info for groups
2) 1.8 btree, fractal heap storage info for groups, attributes and SOHM table
3) btree storage for chunked datasets
4) 1.8 superblock extension size.
Platform tested:
h5committested.
44 files changed, 1367 insertions, 79 deletions
@@ -2069,3 +2069,106 @@ done: FUNC_LEAVE_NOAPI(ret_value) } #endif /* H5B_DEBUG */ + +/*------------------------------------------------------------------------- + * Function: H5B_iterate_btree_size + * + * Purpose: Return the amount of storage used for the btree. + * Keep following the left-most child until reaching the leaf node. + * For each level, gather storage for all the nodes on that level. + * For 0 level, also gather storage for the SNODs. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 19, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B_iterate_btree_size(H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, H5B_operator_t op, haddr_t addr, H5B_info_ud_t *bh_udata) +{ + H5B_t *bt = NULL; /* Pointer to current B-tree node */ + herr_t ret_value; /* Return value */ + H5B_shared_t *shared; /* Pointer to shared B-tree info */ + H5B_t *start_bt; /* Pointer to next B-tree node */ + haddr_t start_addr; /* Address of next node to iterate over */ + + haddr_t *child; /* Pointer to node's child addresses */ + uint8_t *key; /* Pointer to node's native keys */ + unsigned u; /* Local index variable */ + + FUNC_ENTER_NOAPI(H5B_iterate_btree_size, FAIL) + + /* + * Check arguments. + */ + HDassert(f); + HDassert(type); + HDassert(H5F_addr_defined(addr)); + HDassert(bh_udata); + + /* Protect the initial/current node */ + if(NULL == (bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, addr, type, bh_udata->udata, H5AC_READ))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, FAIL, "unable to load B-tree node") + shared = H5RC_GET_OBJ(bt->rc_shared); + HDassert(shared); + + if (bt->level > 0) { + /* Keep following the left-most child until we reach a leaf node. */ + if((ret_value = H5B_iterate_btree_size(f, dxpl_id, type, op, bt->child[0], bh_udata)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTLIST, FAIL, "unable to list B-tree node") + } + ret_value = H5_ITER_CONT; + while(bt && ret_value == H5_ITER_CONT) { + /* count the SNOD sizes for leaf nodes */ + if (bt->level == 0) { /* for leaf node only */ + for(u = 0, child = bt->child, key = bt->native; u < bt->nchildren && ret_value == H5_ITER_CONT; u++, child++, key += type->sizeof_nkey) { + if (op != NULL) { /* only for symbol table nodes */ + ret_value = (*op)(f, dxpl_id, key, *child, key + type->sizeof_nkey, bh_udata->btree_size); + } + if(ret_value < 0) + HERROR(H5E_BTREE, H5E_CANTLIST, "iterator function failed"); + } /* end for */ + } /* end if */ + + /* Check for continuing iteration */ + if(ret_value == H5_ITER_CONT) { + H5B_t *next_bt; /* Pointer to next B-tree node */ + haddr_t next_addr; /* Address of next node to iterate over */ + + /* count the size of bt */ + *(bh_udata->btree_size) += H5B_nodesize(f, shared, NULL); + + /* Protect bt's next node to the right, if there is one */ + if(H5F_addr_defined(bt->right)) { + next_addr = bt->right; + if(NULL == (next_bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, next_addr, type, bh_udata->udata, H5AC_READ))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, FAIL, "B-tree node") + } else { + next_addr = HADDR_UNDEF; + next_bt = NULL; + } /* end if */ + + /* Unprotect bt */ + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, addr, bt, H5AC__NO_FLAGS_SET) < 0) { + if(next_bt) { + HDassert(H5F_addr_defined(next_addr)); + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, next_addr, next_bt, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_PROTECT, FAIL, "unable to release B-tree node") + } /* end if */ + HGOTO_ERROR(H5E_BTREE, H5E_PROTECT, FAIL, "unable to release B-tree node") + } /* end if */ + + /* Advance to the next node */ + bt = next_bt; + addr = next_addr; + } /* end if */ + } /* end while */ + +done: + if(bt && H5AC_unprotect(f, dxpl_id, H5AC_BT, addr, bt, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_PROTECT, FAIL, "unable to release B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B_iterate_btree_size() */ @@ -1187,4 +1187,76 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5B2_modify() */ + +/*------------------------------------------------------------------------- + * Function: H5B2_info_iterate + * + * Purpose: Iterate over all the records in the B-tree, collecting + * storage info. + * + * Return: non-negative on success, negative on error + * + * Programmer: Vailin Choi + * June 19 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2_info_iterate(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, haddr_t addr, hsize_t *btree_size) +{ + H5B2_t *bt2=NULL; /* Pointer to the B-tree header */ + H5B2_shared_t *shared; /* Pointer to B-tree's shared information */ + H5RC_t *bt2_shared=NULL; /* Pointer to ref-counter for shared B-tree info */ + hbool_t incr_rc=FALSE; /* Flag to indicate that we've incremented the B-tree's shared info reference count */ + H5B2_node_ptr_t root_ptr; /* Node pointer info for root node */ + unsigned depth; /* Current depth of the tree */ + herr_t ret_value=SUCCEED; + + + FUNC_ENTER_NOAPI(H5B2_info_iterate, FAIL) + + /* Check arguments. */ + HDassert(f); + HDassert(type); + HDassert(H5F_addr_defined(addr)); + HDassert(btree_size); + + /* Look up the B-tree header */ + if(NULL == (bt2 = H5AC_protect(f, dxpl_id, H5AC_BT2_HDR, addr, type, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree header") + + bt2_shared = bt2->shared; + H5RC_INC(bt2_shared); + incr_rc = TRUE; + + /* Get the pointer to the shared B-tree info */ + shared = H5RC_GET_OBJ(bt2->shared); + HDassert(shared); + + *btree_size += H5B2_HEADER_SIZE(f); + + /* Make copy of the root node pointer */ + root_ptr = bt2->root; + + /* Current depth of the tree */ + depth = shared->depth; + + /* Release header */ + if(H5AC_unprotect(f, dxpl_id, H5AC_BT2_HDR, addr, bt2, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree header info") + bt2 = NULL; + /* Iterate through records */ + if(root_ptr.node_nrec > 0) { + /* Iterate through nodes */ + if((ret_value = H5B2_info_iterate_node(f, dxpl_id, bt2_shared, depth, &root_ptr, btree_size)) < 0) + HERROR(H5E_BTREE, H5E_CANTLIST, "node iteration failed"); + } /* end if */ + +done: + /* Check if we need to decrement the reference count for the B-tree's shared info */ + if(incr_rc) + H5RC_DEC(bt2_shared); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2_info_iterate() */ diff --git a/src/H5B2int.c b/src/H5B2int.c index 5d71d43..20e4300 100644 --- a/src/H5B2int.c +++ b/src/H5B2int.c @@ -3304,4 +3304,63 @@ H5B2_assert_internal2(hsize_t parent_all_nrec, H5B2_shared_t *shared, H5B2_inter return(0); } /* end H5B2_assert_internal2() */ #endif /* H5B2_DEBUG */ + +/*------------------------------------------------------------------------- + * Function: H5B2_info_iterate_node + * + * Purpose: Iterate over all the records from a B-tree node, collecting + * btree storage info. + * + * Return: non-negative on success, negative on error + * + * Programmer: Vailin Choi + * July 12 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2_info_iterate_node(H5F_t *f, hid_t dxpl_id, H5RC_t *bt2_shared, unsigned depth, +const H5B2_node_ptr_t *curr_node, hsize_t *btree_size) +{ + H5B2_shared_t *shared; /* Pointer to B-tree's shared information */ + herr_t ret_value = SUCCEED; /* Iterator return value */ + H5B2_internal_t *internal = NULL; /* Pointer to internal node */ + unsigned u; /* Local index */ + + FUNC_ENTER_NOAPI(H5B2_info_iterate_node, FAIL) + + /* Check arguments. */ + HDassert(f); + HDassert(bt2_shared); + HDassert(curr_node); + HDassert(btree_size); + + /* Get the pointer to the shared B-tree info */ + shared=(H5B2_shared_t *)H5RC_GET_OBJ(bt2_shared); + HDassert(shared); + + + if(depth > 0) { + /* Lock the current B-tree node */ + if(NULL == (internal = H5B2_protect_internal(f, dxpl_id, bt2_shared, curr_node->addr, curr_node->node_nrec, depth, H5AC_READ))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node") + + if(depth>1) { + /* Descend into children */ + for(u=0; u<internal->nrec+1; u++) + if(H5B2_info_iterate_node(f, dxpl_id, bt2_shared, (depth-1), &(internal->node_ptrs[u]), btree_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTLIST, FAIL, "node iteration failed") + } /* end if */ + else /* depth is 1: count all the leaf nodes from this node */ + *btree_size += (internal->nrec + 1) * shared->node_size; + } /* end if */ + + /* Count this node */ + *btree_size += shared->node_size; +done: + if(internal && H5AC_unprotect(f, dxpl_id, H5AC_BT2_INT, curr_node->addr, internal, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2_info_iterate_node() */ diff --git a/src/H5B2pkg.h b/src/H5B2pkg.h index 232cfab..956abb1 100644 --- a/src/H5B2pkg.h +++ b/src/H5B2pkg.h @@ -270,6 +270,11 @@ H5_DLL herr_t H5B2_iterate_node(H5F_t *f, hid_t dxpl_id, H5RC_t *bt2_shared, unsigned depth, const H5B2_node_ptr_t *curr_node, H5B2_operator_t op, void *op_data); +H5_DLL herr_t H5B2_info_iterate_node(H5F_t *f, hid_t dxpl_id, H5RC_t *bt2_shared, + unsigned depth, const H5B2_node_ptr_t *curr_node, hsize_t *op_data); + + + /* Routines for locating records */ H5_DLL int H5B2_locate_record(const H5B2_class_t *type, unsigned nrec, size_t *rec_off, const uint8_t *native, const void *udata, unsigned *idx); diff --git a/src/H5B2private.h b/src/H5B2private.h index 0bcd83c..064743f 100644 --- a/src/H5B2private.h +++ b/src/H5B2private.h @@ -122,6 +122,10 @@ H5_DLL herr_t H5B2_insert(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, haddr_t addr, void *udata); H5_DLL herr_t H5B2_iterate(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, haddr_t addr, H5B2_operator_t op, void *op_data); + +H5_DLL herr_t H5B2_info_iterate(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, + haddr_t addr, hsize_t *op_data); + H5_DLL herr_t H5B2_find(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, haddr_t addr, void *udata, H5B2_found_t op, void *op_data); H5_DLL herr_t H5B2_index(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, diff --git a/src/H5Bprivate.h b/src/H5Bprivate.h index 6be236d..4d726be 100644 --- a/src/H5Bprivate.h +++ b/src/H5Bprivate.h @@ -124,6 +124,11 @@ typedef struct H5B_class_t { herr_t (*debug_key)(FILE*, H5F_t*, hid_t, int, int, const void*, const void*); } H5B_class_t; +typedef struct H5B_info_ud_t { + void *udata; + hsize_t *btree_size; +} H5B_info_ud_t; + /*****************************/ /* Library-private Variables */ /*****************************/ @@ -145,6 +150,8 @@ H5_DLL herr_t H5B_insert (H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, hadd void *udata); H5_DLL herr_t H5B_iterate (H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, H5B_operator_t op, haddr_t addr, void *udata); +H5_DLL herr_t H5B_iterate_btree_size (H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, H5B_operator_t + op, haddr_t addr, H5B_info_ud_t *bh_udata); H5_DLL herr_t H5B_remove(H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, haddr_t addr, void *udata); H5_DLL herr_t H5B_delete(H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, haddr_t addr, @@ -3288,4 +3288,3 @@ H5Ddebug(hid_t dset_id) done: FUNC_LEAVE_API(ret_value) } /* end H5Ddebug() */ - diff --git a/src/H5Distore.c b/src/H5Distore.c index 7e27d1e..297a480 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -4027,3 +4027,47 @@ H5D_istore_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int inden done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_debug() */ + +/*------------------------------------------------------------------------- + * Function: H5D_istore_btree_size + * + * Purpose: Retrieve the amount of B-tree storage for chunked dataset + * + * Return: Success: Non-negative + * Failure: negative + * + * Programmer: Vailin Choi + * June 8, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_obj_istore_bh_info(H5O_loc_t *oloc, H5O_layout_t *layout, hid_t dxpl_id, hsize_t *btree_size) +{ + H5D_istore_it_ud1_t udata; + herr_t ret_value=SUCCEED; + H5B_info_ud_t bh_udata; + + FUNC_ENTER_NOAPI(H5D_obj_istore_bh_info, FAIL) + + HDassert(oloc); + HDassert(layout); + HDassert(btree_size); + + if(H5D_istore_shared_create(oloc->file, layout)<0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") + + HDmemset(&udata, 0, sizeof udata); + udata.common.mesg = layout; + + bh_udata.udata = &udata; + bh_udata.btree_size = btree_size; + if ((ret_value = H5B_iterate_btree_size(oloc->file, dxpl_id, H5B_ISTORE, NULL, layout->u.chunk.addr, &bh_udata)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to iterate over chunk B-tree") +done: + if(layout->u.chunk.btree_shared==NULL) + HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil") + if(H5RC_DEC(layout->u.chunk.btree_shared)<0) + HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_obj_istore_bh_info() */ diff --git a/src/H5Doh.c b/src/H5Doh.c index e59fe8b..620554e0 100644 --- a/src/H5Doh.c +++ b/src/H5Doh.c @@ -327,3 +327,43 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_dset_get_oloc() */ +/*------------------------------------------------------------------------- + * Function: H5O_dset_bh_info + * + * Purpose: Returns the amount of btree storage that is used for chunked + * dataset. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi + * July 11, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_dset_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info) +{ + H5O_layout_t layout; /* Data storage layout message */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5O_dset_bh_info, FAIL) + + /* Sanity check */ + HDassert(oloc); + HDassert(oh); + HDassert(bh_info); + + if (NULL==H5O_msg_read_real(oloc->file, dxpl_id, oh, H5O_LAYOUT_ID, &layout)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't find LAYOUT message") + else { + if ((layout.type == H5D_CHUNKED) && (layout.u.chunk.addr != HADDR_UNDEF)) { + ret_value = H5D_obj_istore_bh_info(oloc, &layout, dxpl_id, &(bh_info->index_size)); + if (ret_value < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine chunked dataset btree info") + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_dset_bh_info() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index c4e4751..b1db412 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -329,6 +329,7 @@ H5_DLL herr_t H5D_istore_dest (H5D_t *dset, hid_t dxpl_id); H5_DLL herr_t H5D_istore_allocate (H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite); H5_DLL hsize_t H5D_istore_allocated(H5D_t *dset, hid_t dxpl_id); +H5_DLL herr_t H5D_obj_istore_bh_info(H5O_loc_t *oloc, H5O_layout_t *layout, hid_t dxpl_id, hsize_t *btree_size); H5_DLL herr_t H5D_istore_prune_by_extent(const H5D_io_info_t *io_info, const hsize_t *old_dims); H5_DLL herr_t H5D_istore_initialize_by_extent(H5D_io_info_t *io_info); @@ -3584,4 +3584,49 @@ H5Fget_name(hid_t obj_id, char *name/*out*/, size_t size) done: FUNC_LEAVE_API(ret_value) } /* end H5Fget_name() */ + +/*------------------------------------------------------------------------- + * Function: H5Fget_info + * 1. Get storage size for superblock extension if there is one + * 2. Get the amount of btree and heap storage for entries + * in the SOHM table if there is one. + * Consider success when there is no superblock extension and/or SOHM table + * + * Return: Success: non-negative on success + * Failure: Negative + * + * Programmer: Vailin Choi + * July 11, 2007 + *------------------------------------------------------------------------- + */ +herr_t +H5Fget_info(hid_t obj_id, H5F_info_t *finfo) +{ + H5F_t *f; + herr_t ret_value=SUCCEED; + + FUNC_ENTER_API(H5Fget_info, FAIL) + HDassert(finfo); + + HDmemset(finfo, 0, sizeof(H5F_info_t)); + if(H5I_get_type(obj_id) == H5I_FILE ) { + if(NULL == (f = H5I_object(obj_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file") + else { + HDassert(f->shared); + if (H5F_addr_defined(f->shared->extension_addr)) { + if (H5F_super_ext_info(f, finfo, H5AC_ind_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Unable to retrieve superblock extension size") + } + if(H5F_addr_defined(f->shared->sohm_addr)) { + if (H5SM_ih_info(f, finfo, H5AC_ind_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Unable to retrieve SOHM btree & heap storage info") + } + } + } else + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid object ID") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Fget_info() */ @@ -493,4 +493,51 @@ HDfprintf(stderr, "%s: fspace->hdr->tot_sect_count = %Hu\n", "H5FS_assert", fspa FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FS_assert() */ #endif /* H5FS_DEBUG */ + +/*------------------------------------------------------------------------- + * Function: H5FS_meta_info + * + * Purpose: Collect meta storage info used by the free space manager + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi + * June 19, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FS_meta_info(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, hsize_t *meta_size) +{ + H5FS_t *fspace = NULL; /* Free space header info */ + H5FS_prot_t fs_prot; /* Information for protecting free space manager */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5FS_meta_info, FAIL) + /* + * Check arguments. + */ + HDassert(f); + HDassert(H5F_addr_defined(fs_addr)); + HDassert(meta_size); + + /* Initialize user data for protecting the free space manager */ + fs_prot.nclasses = 0; + fs_prot.classes = NULL; + fs_prot.cls_init_udata = NULL; + + /* + * Load the free space header. + */ + if(NULL == (fspace = H5AC_protect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, &fs_prot, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTLOAD, FAIL, "unable to load free space header") + + *meta_size = H5FS_HEADER_SIZE(f) + fspace->alloc_sect_size; + +done: + if(fspace && H5AC_unprotect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, fspace, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_FSPACE, H5E_PROTECT, FAIL, "unable to release free space header") + FUNC_LEAVE_NOAPI(ret_value) +} diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h index 4840df2..e6a3ce4 100644 --- a/src/H5FSprivate.h +++ b/src/H5FSprivate.h @@ -161,6 +161,8 @@ H5_DLL H5FS_t *H5FS_open(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, H5_DLL herr_t H5FS_delete(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr); H5_DLL herr_t H5FS_close(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace); +H5_DLL herr_t H5FS_meta_info(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, hsize_t *meta_size); + /* Free space section routines */ H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *node, unsigned flags, void *op_data); diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 8c43ea2..60a81a9 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -178,6 +178,9 @@ H5_DLL herr_t H5F_super_init(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5F_super_write(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5F_super_read(H5F_t *f, hid_t dxpl_id, H5G_loc_t *root_loc); +H5_DLL herr_t H5F_super_ext_info(H5F_t *f, H5F_info_t *finfo, hid_t dxpl_id); + + /* Shared file list related routines */ H5_DLL herr_t H5F_sfile_add(H5F_file_t *shared); H5_DLL H5F_file_t * H5F_sfile_search(H5FD_t *lf); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 94f2bd7..db86101 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -97,6 +97,14 @@ typedef enum H5F_close_degree_t { H5F_CLOSE_STRONG = 3 } H5F_close_degree_t; +typedef struct H5F_info_t { + hsize_t super_ext_size; /* superblock extension size */ + struct { + hsize_t hdr_size; + H5_ih_info_t msgs_info; + } sohm; +} H5F_info_t; + #ifdef __cplusplus extern "C" { #endif @@ -132,6 +140,8 @@ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id, int * cur_num_entries_ptr); H5_DLL herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id); H5_DLL ssize_t H5Fget_name(hid_t obj_id, char *name, size_t size); +H5_DLL herr_t H5Fget_info(hid_t obj_id, H5F_info_t *bh_info); + #ifdef __cplusplus } diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index bf00584..4a516af 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -965,3 +965,30 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_super_write() */ +/*------------------------------------------------------------------------- + * Function: H5F_super_ext_info + * Get storage size of the superblock extension + * + * Return: Success: non-negative on success + * Failure: Negative + * + * Programmer: Vailin Choi + * July 11, 2007 + *------------------------------------------------------------------------- + */ +herr_t +H5F_super_ext_info(H5F_t *f, H5F_info_t *finfo, hid_t dxpl_id) +{ + herr_t ret_value=SUCCEED; + + FUNC_ENTER_NOAPI(H5F_super_ext_info, FAIL) + + HDassert(f); + HDassert(finfo); + + if (H5O_super_ext_size(f, &(finfo->super_ext_size), dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Unable to retrieve superblock extension size") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F_super_ext_info() */ diff --git a/src/H5Gnode.c b/src/H5Gnode.c index fb3c999..cf49c8b 100644 --- a/src/H5Gnode.c +++ b/src/H5Gnode.c @@ -2015,4 +2015,36 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5G_node_debug() */ + +/*------------------------------------------------------------------------- + * Function: H5G_btree_node_iterate + * + * Purpose: This function gets called by H5B_iterate_btree_size() + * to gather storage info for SNODs. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * Jun 19 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5G_btree_node_iterate(H5F_t *f, hid_t dxpl_id, const void UNUSED *_lt_key, haddr_t UNUSED addr, + const void UNUSED *_rt_key, void *_udata) +{ + herr_t ret_value=SUCCEED; + hsize_t *stab_size=(hsize_t *)_udata; + + FUNC_ENTER_NOAPI(H5G_btree_node_iterate, FAIL) + + /* + * Check arguments. + */ + HDassert(f); + HDassert(stab_size); + *stab_size += H5G_node_size(f); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5G_btree_node_iterate() */ diff --git a/src/H5Gobj.c b/src/H5Gobj.c index a3166ba..09eb3b1 100644 --- a/src/H5Gobj.c +++ b/src/H5Gobj.c @@ -1247,4 +1247,3 @@ H5G_obj_lookup_by_idx(H5O_loc_t *grp_oloc, H5_index_t idx_type, done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5G_obj_lookup_by_idx() */ - diff --git a/src/H5Goh.c b/src/H5Goh.c index 598cd5a..9b070a6 100644 --- a/src/H5Goh.c +++ b/src/H5Goh.c @@ -242,3 +242,64 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_group_get_oloc() */ +/*------------------------------------------------------------------------- + * Function: H5O_group_bh_info + * + * Purpose: Retrieve storage for 1.8 btree and heap + * Retrieve storage for 1.6 btree and heap via H5G_stab_bh_info() + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * July 12 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_group_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info) +{ + herr_t ret_value=SUCCEED; /* Return value */ + hsize_t huge_bt_size=0; + H5O_linfo_t linfo; /* Link info message */ + H5O_stab_t stabinfo; /* Info about symbol table */ + + + + FUNC_ENTER_NOAPI(H5O_group_bh_info, FAIL) + + /* Sanity check */ + HDassert(oloc); + HDassert(oh); + HDassert(bh_info); + + if(NULL==H5O_msg_read_real(oloc->file, dxpl_id, oh, H5O_LINFO_ID, &linfo)) { + H5E_clear_stack(NULL); + if(NULL==H5O_msg_read_real(oloc->file, dxpl_id, oh, H5O_STAB_ID, &stabinfo)) { + HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "can't find LINFO nor STAB messages") + } else { /* STAB */ + if (H5G_stab_bh_info(oloc, &stabinfo, dxpl_id, bh_info) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "can't retrieve pre-1.8 group btree & heap info") + } + } else { /* LINFO */ + if (H5F_addr_defined(linfo.corder_bt2_addr)) { + if (H5B2_info_iterate(oloc->file, dxpl_id, H5G_BT2_CORDER, + linfo.corder_bt2_addr, &bh_info->index_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + if (H5F_addr_defined(linfo.name_bt2_addr)) { + if (H5B2_info_iterate(oloc->file, dxpl_id, H5G_BT2_NAME, + linfo.name_bt2_addr, &bh_info->index_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + huge_bt_size = 0; + if (H5F_addr_defined(linfo.fheap_addr)) { + if (H5HF_fheap_info(oloc->file, dxpl_id, linfo.fheap_addr, + &bh_info->heap_size, &huge_bt_size) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't retrieve fractal heap storage info") + } + bh_info->index_size += huge_bt_size; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_group_bh_info() */ diff --git a/src/H5Gpkg.h b/src/H5Gpkg.h index 3da3417..cb36a0f 100644 --- a/src/H5Gpkg.h +++ b/src/H5Gpkg.h @@ -385,6 +385,10 @@ H5_DLL herr_t H5G_stab_lookup(H5O_loc_t *grp_oloc, const char *name, H5_DLL herr_t H5G_stab_lookup_by_idx(H5O_loc_t *grp_oloc, H5_iter_order_t order, hsize_t n, H5O_link_t *lnk, hid_t dxpl_id); +H5_DLL herr_t H5G_stab_bh_info(struct H5O_loc_t *grp_oloc, H5O_stab_t *stabinfo, + hid_t dxpl_id, H5_ih_info_t *bh_info); + + /* * Functions that understand symbol table entries. */ @@ -404,6 +408,10 @@ H5_DLL herr_t H5G_ent_debug(H5F_t *f, const H5G_entry_t *ent, H5_DLL herr_t H5G_node_init(H5F_t *f); H5_DLL int H5G_node_iterate(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, const void *_rt_key, void *_udata); + +H5_DLL herr_t H5G_btree_node_iterate(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, + const void *_rt_key, void *_udata); + H5_DLL int H5G_node_sumup(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, const void *_rt_key, void *_udata); H5_DLL int H5G_node_by_idx(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, diff --git a/src/H5Gprivate.h b/src/H5Gprivate.h index 40d2df8..665b597 100644 --- a/src/H5Gprivate.h +++ b/src/H5Gprivate.h @@ -175,6 +175,7 @@ H5_DLL herr_t H5G_obj_ent_decode(H5F_t *f, const uint8_t **pp, H5_DLL herr_t H5G_obj_ent_encode(const H5F_t *f, uint8_t **pp, const struct H5O_loc_t *oloc); + /* * These functions operate on group hierarchy names. */ diff --git a/src/H5Gstab.c b/src/H5Gstab.c index 22d90a7..9e9f91b 100644 --- a/src/H5Gstab.c +++ b/src/H5Gstab.c @@ -1046,4 +1046,43 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5G_stab_lookup_by_idx() */ + +/*------------------------------------------------------------------------- + * Function: H5G_stab_bh_info + * + * Purpose: Retrieve storage for btree and heap (1.6) + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 25 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5G_stab_bh_info(H5O_loc_t *oloc, H5O_stab_t *stab, hid_t dxpl_id, H5_ih_info_t *bh_info) +{ + hsize_t stab_size=0; /* storage used for symbol table nodes */ + herr_t ret_value = SUCCEED; + H5B_info_ud_t bh_udata; + + FUNC_ENTER_NOAPI(H5G_stab_bh_info, FAIL) + + /* Sanity check */ + HDassert(oloc); + HDassert(stab); + HDassert(bh_info); + + bh_udata.udata = NULL; + bh_udata.btree_size = &(bh_info->index_size); + + if (H5B_iterate_btree_size(oloc->file, dxpl_id, H5B_SNODE, H5G_btree_node_iterate, stab->btree_addr, &bh_udata) <0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "iteration operator failed") + + if (H5HL_heapsize(oloc->file, dxpl_id, stab->heap_addr, &(bh_info->heap_size)) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "iteration operator failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5G_stab_bh_info() */ @@ -926,3 +926,88 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF_delete() */ + +/*------------------------------------------------------------------------- + * Function: H5HF_fheap_info + * + * Purpose: Retrieve storage info for: + * 1. fractal heap + * 2. btree storage used by huge objects in fractal heap + * 3. free space storage info + * + * Return: non-negative on success, negative on error + * + * Programmer: Vailin Choi + * July 12 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_fheap_info(H5F_t *f, hid_t dxpl_id, haddr_t fh_addr, hsize_t *heap_size, hsize_t *huge_bt_size) +{ + H5HF_hdr_t *hdr = NULL; + herr_t ret_value = SUCCEED; + hsize_t meta_size = 0; + int reterr; + + FUNC_ENTER_NOAPI(H5HF_fheap_info, FAIL) + + /* + * Check arguments. + */ + HDassert(f); + HDassert(H5F_addr_defined(fh_addr)); + HDassert(heap_size); + HDassert(huge_bt_size); + + + /* Lock the heap header into memory */ + if(NULL == (hdr = H5AC_protect(f, dxpl_id, H5AC_FHEAP_HDR, fh_addr, NULL, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to load fractal heap header") + + *heap_size += hdr->heap_size; + *heap_size += hdr->man_alloc_size; + *heap_size += hdr->huge_size; + + if(H5F_addr_defined(hdr->man_dtable.table_addr)) { + if (hdr->man_dtable.curr_root_rows != 0) { + reterr = H5HF_indirect_info(f, dxpl_id, hdr, hdr->man_dtable.table_addr, hdr->man_dtable.curr_root_rows, heap_size); + if (reterr < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "unable to get fractal heap storage info for indirect block") + } + } + + /* get B-tree storage for huge objects in fractal heap */ + if (H5F_addr_defined(hdr->huge_bt2_addr)) { + if (hdr->huge_ids_direct) { + if (hdr->filter_len > 0) { + if (H5B2_info_iterate(f, dxpl_id, H5HF_BT2_FILT_DIR, hdr->huge_bt2_addr, huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } else { + if (H5B2_info_iterate(f, dxpl_id, H5HF_BT2_DIR, hdr->huge_bt2_addr, huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + } else { + if (hdr->filter_len > 0) { + if (H5B2_info_iterate(f, dxpl_id, H5HF_BT2_FILT_INDIR, hdr->huge_bt2_addr, huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } else { + if (H5B2_info_iterate(f, dxpl_id, H5HF_BT2_INDIR, hdr->huge_bt2_addr, huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + } + } + if (H5F_addr_defined(hdr->fs_addr)) { + meta_size = 0; + if (H5FS_meta_info(f, dxpl_id, hdr->fs_addr, &meta_size) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "can't retrieve FS meta storage info") + *heap_size += meta_size; + } + +done: + /* Unprotect the header, if an error occurred */ + if(hdr && H5AC_unprotect(f, dxpl_id, H5AC_FHEAP_HDR, fh_addr, hdr, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_HEAP, H5E_CANTUNPROTECT, FAIL, "unable to release fractal heap header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_fheap_info() */ diff --git a/src/H5HFiblock.c b/src/H5HFiblock.c index 7afa7c4..15c998c 100644 --- a/src/H5HFiblock.c +++ b/src/H5HFiblock.c @@ -1575,4 +1575,65 @@ HDfprintf(stderr, "%s: iblock_addr = %a, iblock_nrows = %u\n", FUNC, iblock_addr done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF_man_iblock_delete() */ + +/*------------------------------------------------------------------------- + * + * Function: H5HF_indirect_info + * + * Purpose: Gather storage used for the indirect block in fractal heap + * + * Return: non-negative on success, negative on error + * + * Programmer: Vailin Choi + * July 12 2007 + *------------------------------------------------------------------------- + */ +herr_t +H5HF_indirect_info(H5F_t *f, hid_t dxpl_id, H5HF_hdr_t *hdr, haddr_t iblock_addr, unsigned nrows, hsize_t *heap_size) +{ + hbool_t did_protect; + H5HF_indirect_t *iblock; /* Pointer to indirect block */ + size_t u, v; + int reterr; + int ret_value=SUCCEED; + FUNC_ENTER_NOAPI(H5HF_indirect_info, FAIL) + /* + * Check arguments. + */ + HDassert(f); + HDassert(hdr); + HDassert(H5F_addr_defined(iblock_addr)); + HDassert(heap_size); + + if (NULL == (iblock = H5HF_man_iblock_protect(hdr, dxpl_id, iblock_addr, nrows, NULL, 0, FALSE, H5AC_READ, &did_protect))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to load fractal heap indirect block") + + *heap_size += iblock->size; + + /* Indirect entries in this indirect block */ + if(iblock->nrows > hdr->man_dtable.max_direct_rows) { + unsigned first_row_bits; /* Number of bits used bit addresses in first row */ + unsigned num_indirect_rows; /* Number of rows of blocks in each indirect block */ + + first_row_bits = H5V_log2_of2((uint32_t)hdr->man_dtable.cparam.start_block_size) + + H5V_log2_of2(hdr->man_dtable.cparam.width); + for(u = hdr->man_dtable.max_direct_rows; u < iblock->nrows; u++) { + num_indirect_rows = (H5V_log2_gen(hdr->man_dtable.row_block_size[u]) - first_row_bits) + 1; + for(v = 0; v < hdr->man_dtable.cparam.width; v++) { + size_t off = (u * hdr->man_dtable.cparam.width) + v; + + if (H5F_addr_defined(iblock->ents[off].addr)) { + reterr = H5HF_indirect_info(f, dxpl_id, hdr, iblock->ents[off].addr, num_indirect_rows, heap_size); + if (reterr < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to get fractal heap storage info for indirect block") + } /* end if */ + } /* end for v */ + } /* end for u */ + } /* end if */ + +done: + if(H5HF_man_iblock_unprotect(iblock, dxpl_id, H5AC__NO_FLAGS_SET, did_protect) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTUNPROTECT, FAIL, "unable to release fractal heap indirect block") + FUNC_LEAVE_NOAPI(ret_value) +} diff --git a/src/H5HFpkg.h b/src/H5HFpkg.h index 6dc1498..bd3c4ec 100644 --- a/src/H5HFpkg.h +++ b/src/H5HFpkg.h @@ -722,6 +722,8 @@ H5_DLL herr_t H5HF_sect_indirect_add(H5HF_hdr_t *hdr, hid_t dxpl_id, /* Internal operator callbacks */ H5_DLL herr_t H5HF_op_read(const void *obj, size_t obj_len, void *op_data); H5_DLL herr_t H5HF_op_write(const void *obj, size_t obj_len, void *op_data); +H5_DLL herr_t H5HF_indirect_info(H5F_t *f, hid_t dxpl_id, H5HF_hdr_t *hdr, + haddr_t iblock_addr, unsigned nrows, hsize_t *heap_size/*out*/); /* Testing routines */ #ifdef H5HF_TESTING diff --git a/src/H5HFprivate.h b/src/H5HFprivate.h index 7d7c3f6..a8855de 100644 --- a/src/H5HFprivate.h +++ b/src/H5HFprivate.h @@ -126,6 +126,11 @@ H5_DLL herr_t H5HF_remove(H5HF_t *fh, hid_t dxpl_id, const void *id); H5_DLL herr_t H5HF_close(H5HF_t *fh, hid_t dxpl_id); H5_DLL herr_t H5HF_delete(H5F_t *f, hid_t dxpl_id, haddr_t fh_addr); +/* Routines to gather storage info for fractal heap */ +H5_DLL herr_t H5HF_fheap_info(H5F_t *f, hid_t dxpl_id, haddr_t fh_addr, + hsize_t *heap_size/*out*/, hsize_t *huge_bt_size/*out*/); + + /* Statistics routines */ H5_DLL herr_t H5HF_stat_info(const H5HF_t *fh, H5HF_stat_t *stats); @@ -1346,4 +1346,45 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HL_get_size() */ + +/*------------------------------------------------------------------------- + * Function: H5HL_heapsize + * + * Purpose: Compute the size in bytes of the specified instance of + * H5HL_t via H5HL_compute_size() + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 19 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HL_heapsize(H5F_t *f, hid_t dxpl_id, haddr_t addr, hsize_t *heap_size) +{ + H5HL_t *heap = NULL; /* Heap to query */ + herr_t ret_value = SUCCEED; /* Return value */ + size_t local_heap_size = 0; + + FUNC_ENTER_NOAPI(H5HL_heapsize, FAIL) + /* check arguments */ + HDassert(f); + HDassert(H5F_addr_defined(addr)); + HDassert(heap_size); + + /* Get heap pointer */ + if(NULL == (heap = H5AC_protect(f, dxpl_id, H5AC_LHEAP, addr, NULL, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to load heap") + + if (H5HL_compute_size(f, heap, &local_heap_size)<0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to compute size of local heap") + *heap_size = (hsize_t)local_heap_size; + +done: + if(heap && H5AC_unprotect(f, dxpl_id, H5AC_LHEAP, addr, heap, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_HEAP, H5E_PROTECT, FAIL, "unable to release local heap") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HL_get_size() */ diff --git a/src/H5HLprivate.h b/src/H5HLprivate.h index 4684d61..d8a4b49 100644 --- a/src/H5HLprivate.h +++ b/src/H5HLprivate.h @@ -73,6 +73,7 @@ H5_DLL size_t H5HL_insert(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t size, const void *buf); H5_DLL herr_t H5HL_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr); H5_DLL herr_t H5HL_get_size(H5F_t *f, hid_t dxpl_id, haddr_t addr, size_t *size); +H5_DLL herr_t H5HL_heapsize(H5F_t *f, hid_t dxpl_id, haddr_t addr, hsize_t *heap_size); /* Debugging functions */ H5_DLL herr_t H5HL_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int indent, @@ -488,7 +488,6 @@ H5Oget_info(hid_t loc_id, const char *name, H5O_info_t *oinfo, hid_t lapl_id) /* Retrieve the object's information */ if(H5G_loc_info(&loc, name, oinfo/*out*/, lapl_id, H5AC_ind_dxpl_id) < 0) HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, FAIL, "object not found") - done: FUNC_LEAVE_API(ret_value) } /* end H5Oget_info() */ @@ -1825,6 +1824,13 @@ done: * Programmer: Quincey Koziol * November 21 2006 * + * Modifications: + * 12 July 2007 by Vailin Choi + * Modified to retrieve the following information: + * 1. GROUP: storage for btree and heap (1.6 and 1.8) + * 2. DATASET: btree storage for chunked dataset + * 3. ATTRIBUTE: storage for 1.8 btree and fractal heap + * *------------------------------------------------------------------------- */ herr_t @@ -1857,6 +1863,16 @@ H5O_get_info(H5O_loc_t *oloc, H5O_info_t *oinfo, hid_t dxpl_id) if(H5O_obj_type_real(oh, &oinfo->type) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to determine object type") + /* Retrieve btree and heap storage info */ + HDmemset(&oinfo->meta_size.obj, 0, sizeof(H5_ih_info_t)); + if (oinfo->type == H5O_TYPE_GROUP) { + if (H5O_group_bh_info(oloc, oh, dxpl_id, &(oinfo->meta_size.obj)/*out*/) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't retrieve 1.8 group btree & heap info") + } else if (oinfo->type == H5O_TYPE_DATASET) { + if (H5O_dset_bh_info(oloc, oh, dxpl_id, &(oinfo->meta_size.obj)/*out*/) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't retrieve chunked dataset btree info") + } + /* Set the object's reference count */ oinfo->rc = oh->nlink; @@ -1927,6 +1943,9 @@ H5O_get_info(H5O_loc_t *oloc, H5O_info_t *oinfo, hid_t dxpl_id) /* Retrieve # of attributes */ oinfo->num_attrs = H5O_attr_count_real(oloc->file, dxpl_id, oh); + HDmemset(&oinfo->meta_size.attr, 0, sizeof(H5_ih_info_t)); + if ((oinfo->num_attrs > 0) && (H5O_attr_bh_info(oloc, oh, dxpl_id, &oinfo->meta_size.attr/*out*/) < 0)) + HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't retrieve attribute btree & heap info") /* Iterate over all the chunks, adding any gaps to the free space */ oinfo->hdr.space.total = 0; @@ -2121,3 +2140,47 @@ H5O_get_oh_addr(const H5O_t *oh) FUNC_LEAVE_NOAPI(oh->chunk[0].addr) } /* end H5O_get_oh_addr() */ +/*------------------------------------------------------------------------- + * Function: H5O_super_ext_size + * + * Purpose: Collect size of the superblock extension + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi + * July 12, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_super_ext_size(H5F_t *f, hsize_t *ext_size, hid_t dxpl_id) +{ + H5O_t *oh = NULL; /* Object header */ + H5O_chunk_t *curr_chunk; /* Pointer to current message being operated on */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5O_super_ext_size, FAIL) + + /* Check args */ + HDassert(f); + HDassert(ext_size); + HDassert(H5F_addr_defined(f->shared->extension_addr)); + + /* Get the object header */ + if(NULL == (oh = H5AC_protect(f, dxpl_id, H5AC_OHDR, f->shared->extension_addr, NULL, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "unable to load object header") + + *ext_size = 0; + for(u = 0, curr_chunk = &oh->chunk[0]; u < oh->nchunks; u++, curr_chunk++) { + /* Accumulate the size of the header on disk */ + *ext_size += curr_chunk->size; + } + +done: + if(oh && H5AC_unprotect(f, dxpl_id, H5AC_OHDR, f->shared->extension_addr, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_PROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_super_ext_size() */ diff --git a/src/H5Oattribute.c b/src/H5Oattribute.c index ddf5a35..f0bafc9 100644 --- a/src/H5Oattribute.c +++ b/src/H5Oattribute.c @@ -1687,4 +1687,63 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_attr_exists */ + +/*------------------------------------------------------------------------- + * Function: H5O_attr_bh_info + * + * Purpose: For 1.8 attribute, returns storage amount for btree and fractal heap + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 19, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_attr_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info) +{ + H5O_ainfo_t ainfo; /* Attribute information for object */ + herr_t ret_value=SUCCEED; /* Return value */ + hsize_t huge_bt_size=0; + FUNC_ENTER_NOAPI(H5O_attr_bh_info, FAIL) + + HDassert(oloc); + HDassert(oh); + HDassert(bh_info); + + /* Check for attribute info stored */ + ainfo.nattrs = 0; + ainfo.fheap_addr = HADDR_UNDEF; + ainfo.corder_bt2_addr = HADDR_UNDEF; + ainfo.name_bt2_addr = HADDR_UNDEF; + if(oh->version > H5O_VERSION_1 && NULL == H5A_get_ainfo(oloc->file, dxpl_id, oh, &ainfo)) + /* Clear error stack from not finding attribute info */ + H5E_clear_stack(NULL); + + if(oh->version > H5O_VERSION_1) { + if (H5F_addr_defined(ainfo.corder_bt2_addr)) { + if (H5B2_info_iterate(oloc->file, dxpl_id, H5A_BT2_CORDER, + ainfo.corder_bt2_addr, &(bh_info->index_size)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + + if (H5F_addr_defined(ainfo.name_bt2_addr)) { + if (H5B2_info_iterate(oloc->file, dxpl_id, H5A_BT2_NAME, + ainfo.name_bt2_addr, &(bh_info->index_size)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + + huge_bt_size = 0; + if (H5F_addr_defined(ainfo.fheap_addr)) { + HDassert(H5O_msg_count_real(oh, H5O_MSG_ATTR) == 0); + if (H5HF_fheap_info(oloc->file, dxpl_id, ainfo.fheap_addr, &(bh_info->heap_size), &huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + bh_info->index_size += huge_bt_size; + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_attr_bh_info() */ diff --git a/src/H5Opkg.h b/src/H5Opkg.h index fb17c0c..afde15a 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -479,6 +479,14 @@ H5_DLL void *H5O_msg_copy_file(const H5O_msg_class_t *type, H5F_t *file_src, H5_DLL herr_t H5O_msg_iterate_real(H5F_t *f, H5O_t *oh, const H5O_msg_class_t *type, const H5O_mesg_operator_t *op, void *op_data, hid_t dxpl_id); +/* collect storage info for btree and heap */ +H5_DLL herr_t H5O_group_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info); +H5_DLL herr_t H5O_dset_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info); +H5_DLL herr_t H5O_attr_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info); + + + + /* Object header allocation routines */ H5_DLL herr_t H5O_alloc_msgs(H5O_t *oh, size_t min_alloc); H5_DLL unsigned H5O_alloc(H5F_t *f, hid_t dxpl_id, H5O_t *oh, diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 2376113..584d9d5 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -532,6 +532,9 @@ H5_DLL herr_t H5O_bogus_oh(H5F_t *f, hid_t dxpl_id, struct H5O_t *oh, unsigned m #endif /* H5O_ENABLE_BOGUS */ H5_DLL herr_t H5O_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr); H5_DLL herr_t H5O_get_info(H5O_loc_t *oloc, H5O_info_t *oinfo, hid_t dxpl_id); + +H5_DLL herr_t H5O_super_ext_size(H5F_t *f, hsize_t *ext_size, hid_t dxpl_id); + H5_DLL herr_t H5O_obj_type(const H5O_loc_t *loc, H5O_type_t *obj_type, hid_t dxpl_id); H5_DLL herr_t H5O_get_create_plist(const H5O_loc_t *loc, hid_t dxpl_id, struct H5P_genplist_t *oc_plist); H5_DLL hid_t H5O_open_name(H5G_loc_t *loc, const char *name, hid_t lapl_id); @@ -618,5 +621,6 @@ H5_DLL herr_t H5O_set_shared(H5O_shared_t *dst, const H5O_shared_t *src); /* Attribute operators */ H5_DLL hsize_t H5O_attr_count_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh); + #endif /* _H5Oprivate_H */ diff --git a/src/H5Opublic.h b/src/H5Opublic.h index b3d0c36..c5f1b21 100644 --- a/src/H5Opublic.h +++ b/src/H5Opublic.h @@ -120,8 +120,12 @@ typedef struct H5O_info_t { uint64_t shared; /* Flags to indicate message type is shared in header */ } mesg; } hdr; - hsize_t meta_size; /* Size of additional metadata for an object */ - /* (B-tree & heap for groups, B-tree for chunked dataset, etc.) */ + /* btree and heap storage info for obj & attributes */ + /* (B-tree & heap for groups, B-tree for chunked dataset, 1.8 B-tree & heap for attributes) */ + struct { + H5_ih_info_t obj; + H5_ih_info_t attr; + } meta_size; } H5O_info_t; /* Typedef for message creation indexes */ @@ -2474,3 +2474,68 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5SM_list_debug() */ +/*------------------------------------------------------------------------- + * Function: H5SM_ih_info + * + * Purpose: Loop through the master SOHM table (if there is one) to: + * 1. collect storage used for header + * 1. collect storage used for B-tree and List + * (include btree storage used by huge objects in fractal heap) + * 2. collect fractal heap storage + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 19, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5SM_ih_info(H5F_t *f, H5F_info_t *finfo, hid_t dxpl_id) +{ + H5SM_master_table_t *table=NULL; /* SOHM master table */ + herr_t ret_value=SUCCEED; /* Return value */ + unsigned x; + hsize_t huge_bt_size=0; + + + FUNC_ENTER_NOAPI(H5SM_ih_info, FAIL) + + /* Sanity check */ + HDassert(f); + HDassert(H5F_addr_defined(f->shared->sohm_addr)); + HDassert(finfo); + + /* Look up the master SOHM table */ + if(NULL == (table = (H5SM_master_table_t *)H5AC_protect(f, dxpl_id, H5AC_SOHM_TABLE, f->shared->sohm_addr, NULL, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, FAIL, "unable to load SOHM master table") + + finfo->sohm.hdr_size = (hsize_t) H5SM_TABLE_SIZE(f) + + (hsize_t)(table->num_indexes * H5SM_INDEX_HEADER_SIZE(f)); + + for(x = 0; x < table->num_indexes; x++) { + if (table->indexes[x].index_type==H5SM_BTREE) { + if (H5F_addr_defined(table->indexes[x].index_addr)) { + if (H5B2_info_iterate(f, dxpl_id, H5SM_INDEX, + table->indexes[x].index_addr, &(finfo->sohm.msgs_info.index_size)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + } + if (table->indexes[x].index_type==H5SM_LIST) + finfo->sohm.msgs_info.index_size += H5SM_LIST_SIZE(f, table->indexes[x].list_max); + + huge_bt_size = 0; + if (H5F_addr_defined(table->indexes[x].heap_addr)) { + if (H5HF_fheap_info(f, dxpl_id, table->indexes[x].heap_addr, &(finfo->sohm.msgs_info.heap_size), &huge_bt_size) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't retrieve fractal heap storage info") + } + finfo->sohm.msgs_info.index_size += huge_bt_size; + } /* end for */ + + /* Release resources */ + if(table && H5AC_unprotect(f, dxpl_id, H5AC_SOHM_TABLE, f->shared->sohm_addr, table, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_CACHE, H5E_CANTRELEASE, FAIL, "unable to close SOHM master table") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5SM_ih_info() */ diff --git a/src/H5SMprivate.h b/src/H5SMprivate.h index 0f3d3e0..763c117 100755 --- a/src/H5SMprivate.h +++ b/src/H5SMprivate.h @@ -56,6 +56,8 @@ H5_DLL herr_t H5SM_reconstitute(H5O_shared_t *sh_mesg, H5F_t *f, unsigned msg_type_id, H5O_fheap_id_t heap_id); H5_DLL herr_t H5SM_get_refcount(H5F_t *f, hid_t dxpl_id, unsigned type_id, const H5O_shared_t *sh_mesg, hsize_t *ref_count); +H5_DLL herr_t H5SM_ih_info(H5F_t *f, H5F_info_t *bh_info, hid_t dxpl_id); + /* Debugging routines */ H5_DLL herr_t H5SM_table_debug(H5F_t *f, hid_t dxpl_id, haddr_t table_addr, diff --git a/src/H5public.h b/src/H5public.h index 09b2d5c..85a9d8b 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -300,6 +300,14 @@ typedef enum H5_index_t { H5_INDEX_N /* Number of indices defined */ } H5_index_t; +/* + * Storage info struct used by H5O_info_t and H5F_info_t + */ +typedef struct H5_ih_info_t { + hsize_t index_size; /* btree and/or list */ + hsize_t heap_size; +} H5_ih_info_t; + /* Functions in H5.c */ H5_DLL herr_t H5open(void); H5_DLL herr_t H5close(void); diff --git a/tools/misc/Makefile.am b/tools/misc/Makefile.am index 423a28b..fc097cd 100644 --- a/tools/misc/Makefile.am +++ b/tools/misc/Makefile.am @@ -24,7 +24,7 @@ include $(top_srcdir)/config/commence.am INCLUDES=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib #test script and program -TEST_PROG=h5repart_gentest +TEST_PROG=h5repart_gentest h5stat_gentest TEST_SCRIPT=testh5repart.sh testh5stat.sh $(srcdir)/testh5mkgrp.sh check_PROGRAMS=$(TEST_PROG) repart_test diff --git a/tools/misc/Makefile.in b/tools/misc/Makefile.in index 605c3c9..c8022c7 100644 --- a/tools/misc/Makefile.in +++ b/tools/misc/Makefile.in @@ -68,7 +68,7 @@ CONFIG_HEADER = $(top_builddir)/src/H5config.h CONFIG_CLEAN_FILES = h5cc testh5repart.sh testh5stat.sh am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)" binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) -am__EXEEXT_1 = h5repart_gentest$(EXEEXT) +am__EXEEXT_1 = h5repart_gentest$(EXEEXT) h5stat_gentest$(EXEEXT) PROGRAMS = $(bin_PROGRAMS) h5debug_SOURCES = h5debug.c h5debug_OBJECTS = h5debug.$(OBJEXT) @@ -90,6 +90,10 @@ h5stat_SOURCES = h5stat.c h5stat_OBJECTS = h5stat.$(OBJEXT) h5stat_LDADD = $(LDADD) h5stat_DEPENDENCIES = $(LIBH5TOOLS) $(LIBHDF5) +h5stat_gentest_SOURCES = h5stat_gentest.c +h5stat_gentest_OBJECTS = h5stat_gentest.$(OBJEXT) +h5stat_gentest_LDADD = $(LDADD) +h5stat_gentest_DEPENDENCIES = $(LIBH5TOOLS) $(LIBHDF5) repart_test_SOURCES = repart_test.c repart_test_OBJECTS = repart_test.$(OBJEXT) repart_test_LDADD = $(LDADD) @@ -109,9 +113,9 @@ LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \ $(LDFLAGS) -o $@ SOURCES = h5debug.c h5mkgrp.c h5repart.c h5repart_gentest.c h5stat.c \ - repart_test.c + h5stat_gentest.c repart_test.c DIST_SOURCES = h5debug.c h5mkgrp.c h5repart.c h5repart_gentest.c \ - h5stat.c repart_test.c + h5stat.c h5stat_gentest.c repart_test.c ETAGS = etags CTAGS = ctags DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) @@ -352,7 +356,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog *.h5 \ INCLUDES = -I$(top_srcdir)/src -I$(top_srcdir)/tools/lib #test script and program -TEST_PROG = h5repart_gentest +TEST_PROG = h5repart_gentest h5stat_gentest TEST_SCRIPT = testh5repart.sh testh5stat.sh $(srcdir)/testh5mkgrp.sh check_SCRIPTS = $(TEST_SCRIPT) SCRIPT_DEPEND = h5repart$(EXEEXT) h5stat$(EXEEXT) h5mkgrp$(EXEEXT) @@ -476,6 +480,9 @@ h5repart_gentest$(EXEEXT): $(h5repart_gentest_OBJECTS) $(h5repart_gentest_DEPEND h5stat$(EXEEXT): $(h5stat_OBJECTS) $(h5stat_DEPENDENCIES) @rm -f h5stat$(EXEEXT) $(LINK) $(h5stat_OBJECTS) $(h5stat_LDADD) $(LIBS) +h5stat_gentest$(EXEEXT): $(h5stat_gentest_OBJECTS) $(h5stat_gentest_DEPENDENCIES) + @rm -f h5stat_gentest$(EXEEXT) + $(LINK) $(h5stat_gentest_OBJECTS) $(h5stat_gentest_LDADD) $(LIBS) repart_test$(EXEEXT): $(repart_test_OBJECTS) $(repart_test_DEPENDENCIES) @rm -f repart_test$(EXEEXT) $(LINK) $(repart_test_OBJECTS) $(repart_test_LDADD) $(LIBS) @@ -510,6 +517,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repart.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repart_gentest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5stat.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5stat_gentest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/repart_test.Po@am__quote@ .c.o: diff --git a/tools/misc/h5stat.c b/tools/misc/h5stat.c index 93871bd..27566b9 100644 --- a/tools/misc/h5stat.c +++ b/tools/misc/h5stat.c @@ -24,6 +24,7 @@ /* Parameters to control statistics gathered */ #define SIZE_SMALL_GROUPS 10 +#define SIZE_SMALL_ATTRS 10 #define SIZE_SMALL_DSETS 10 #define H5_NFILTERS_IMPL 8 /* Number of currently implemented filters + one to @@ -63,6 +64,11 @@ typedef struct iter_t { unsigned long *group_bins; /* Pointer to array of bins for group counts */ ohdr_info_t group_ohdr_info; /* Object header information for groups */ + hsize_t max_attrs; /* Maximum attributes from a group */ + unsigned long num_small_attrs[SIZE_SMALL_ATTRS]; /* Size of small attributes tracked */ + unsigned attr_nbins; + unsigned long *attr_bins; + unsigned long max_dset_rank; /* Maximum rank of dataset */ unsigned long dset_rank_count[H5S_MAX_RANK]; /* Number of datasets of each rank */ hsize_t max_dset_dims; /* Maximum dimension size of dataset */ @@ -75,6 +81,15 @@ typedef struct iter_t { unsigned long *dset_dim_bins; /* Pointer to array of bins for dataset dimensions */ ohdr_info_t dset_ohdr_info; /* Object header information for datasets */ hsize_t dset_storage_size; /* Size of raw data for datasets */ + hsize_t groups_btree_storage_size; /* btree size for group */ + hsize_t groups_heap_storage_size; /* heap size for group */ + hsize_t attrs_btree_storage_size; /* btree size for attributes (1.8) */ + hsize_t attrs_heap_storage_size; /* fractal heap size for attributes (1.8) */ + hsize_t SM_hdr_storage_size; /* header size for SOHM table (1.8) */ + hsize_t SM_index_storage_size; /* index (btree & list) size for SOHM table (1.8) */ + hsize_t SM_heap_storage_size; /* fractal heap size for SOHM table (1.8) */ + hsize_t super_ext_size; /* superblock extension size */ + hsize_t datasets_btree_storage_size; /* btree size for chunked dataset */ unsigned long nexternal; /* Number of external files for a dataset */ int local; /* Flag to indicate iteration over the object*/ } iter_t; @@ -104,6 +119,7 @@ static int display_dtype_metadata = FALSE; static int display_dtype = FALSE; */ static int display_object = FALSE; +static int display_attr = FALSE; /* a structure for handling the order command-line parameters come in */ struct handler_t { @@ -113,21 +129,21 @@ struct handler_t { }; -static const char *s_opts ="FfhGgDdTO:V"; +static const char *s_opts ="AFfhGgDdTO:V"; static struct long_options l_opts[] = { {"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, {"file", no_arg, 'f'}, {"fil", no_arg, 'f'}, {"fi", no_arg, 'f'}, - {"filemetadata", no_arg, 'F'}, - {"filemetadat", no_arg, 'F'}, - {"filemetada", no_arg, 'F'}, - {"filemetad", no_arg, 'F'}, - {"filemeta", no_arg, 'F'}, - {"filemet", no_arg, 'F'}, - {"fileme", no_arg, 'F'}, - {"filem", no_arg, 'F'}, + {"FILEmetadata", no_arg, 'F'}, + {"FILEmetadat", no_arg, 'F'}, + {"FILEmetada", no_arg, 'F'}, + {"FILEmetad", no_arg, 'F'}, + {"FILEmeta", no_arg, 'F'}, + {"FILEmet", no_arg, 'F'}, + {"FILEme", no_arg, 'F'}, + {"FILEm", no_arg, 'F'}, {"group", no_arg, 'g'}, {"grou", no_arg, 'g'}, {"gro", no_arg, 'g'}, @@ -172,6 +188,15 @@ static struct long_options l_opts[] = { { "vers", no_arg, 'V' }, { "ver", no_arg, 'V' }, { "ve", no_arg, 'V' }, + { "attribute", no_arg, 'A' }, + { "attribut", no_arg, 'A' }, + { "attribu", no_arg, 'A' }, + { "attrib", no_arg, 'A' }, + { "attri", no_arg, 'A' }, + { "attr", no_arg, 'A' }, + { "att", no_arg, 'A' }, + { "at", no_arg, 'A' }, + { "a", no_arg, 'A' }, { NULL, 0, '\0' } }; @@ -205,6 +230,7 @@ static void usage(const char *prog) fprintf(stdout, " -d, --dset Print dataset information\n"); fprintf(stdout, " -D, --dsetmetadata Print dataset metadata\n"); fprintf(stdout, " -T, --dtypemetadata Print datatype metadata\n"); + fprintf(stdout, " -A, --attribute Print attribute information\n"); fprintf(stdout, "\n"); } @@ -369,31 +395,38 @@ fix_name(const char *path, const char *base) * Failure: -1 * * Programmer: Quincey Koziol - * Tuesday, August 16, 2005 + * Tuesday, August 16, 2005 * * Modifications: Refactored code from the walk_function * EIP, Wednesday, August 16, 2006 * + * Vailin Choi 12 July 2007 + * 1. Gathered storage info for btree and heap + * (groups and attributes) + * 2. Gathered info for attributes + * + * *------------------------------------------------------------------------- */ static herr_t group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * _sb, H5G_iterate_t _walk, iter_t *_iter) - - { - hid_t gid; /* Group ID */ - const char *last_container; - hsize_t num_objs; - unsigned bin; /* "bin" the number of objects falls in */ - iter_t *iter = (iter_t*)_iter; - H5G_stat_t *sb = _sb; - H5G_iterate_t walk = _walk; - herr_t ret; + hid_t gid; /* Group ID */ + const char *last_container; + hsize_t num_objs; + unsigned bin; /* "bin" the number of objects falls in */ + iter_t *iter = (iter_t*)_iter; + H5G_stat_t *sb = _sb; + H5G_iterate_t walk = _walk; + herr_t ret; + hsize_t num_attrs=0; + unsigned attr_bin; + H5O_info_t oinfo; /* Gather statistics about this type of object */ iter->uniq_groups++; if(iter->curr_depth > iter->max_depth) - iter->max_depth = iter->curr_depth; + iter->max_depth = iter->curr_depth; /* Get object header information */ iter->group_ohdr_info.total_size += sb->ohdr.size; @@ -403,6 +436,7 @@ group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * assert(gid > 0); H5Gget_num_objs(gid, &num_objs); + if(num_objs < SIZE_SMALL_GROUPS) (iter->num_small_groups[num_objs])++; if(num_objs > iter->max_fanout) @@ -411,11 +445,11 @@ group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * /* Add group count to proper bin */ bin = ceil_log10((unsigned long)num_objs); if((bin + 1) > iter->group_nbins) { - /* Allocate more storage for info about dataset's datatype */ + /* Allocate more storage for info about dataset's datatype */ iter->group_bins = realloc(iter->group_bins, (bin + 1) * sizeof(unsigned long)); assert(iter->group_bins); - /* Initialize counts for intermediate bins */ + /* Initialize counts for intermediate bins */ while(iter->group_nbins < bin) iter->group_bins[iter->group_nbins++] = 0; iter->group_nbins++; @@ -427,17 +461,51 @@ group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * (iter->group_bins[bin])++; } /* end else */ - ret = H5Gclose(gid); - assert(ret >= 0); + ret = H5Oget_info(gid, ".", &oinfo, H5P_DEFAULT); + if (ret < 0) { + warn_msg(progname, "Unable to retrieve object info for \"%s\"\n", name); + } else { + num_attrs = oinfo.num_attrs; + iter->groups_btree_storage_size += oinfo.meta_size.obj.index_size; + iter->groups_heap_storage_size += oinfo.meta_size.obj.heap_size; + iter->attrs_btree_storage_size += oinfo.meta_size.attr.index_size; + iter->attrs_heap_storage_size += oinfo.meta_size.attr.heap_size; + } + + if(num_attrs < SIZE_SMALL_ATTRS) + (iter->num_small_attrs[num_attrs])++; + if(num_attrs > iter->max_attrs) + iter->max_attrs = num_attrs; + + /* Add attribute count to proper bin */ + attr_bin = ceil_log10((unsigned long)num_attrs); + if((attr_bin + 1) > iter->attr_nbins) { + iter->attr_bins = realloc(iter->attr_bins, (attr_bin + 1) * sizeof(unsigned long)); + assert(iter->attr_bins); - last_container = iter->container; - iter->container = fullname; - iter->curr_depth++; + /* Initialize counts for intermediate bins */ + while(iter->attr_nbins < attr_bin) + iter->attr_bins[iter->attr_nbins++] = 0; + iter->attr_nbins++; - H5Giterate(group, name, NULL, walk, iter); + /* Initialize count for new bin */ + iter->attr_bins[attr_bin] = 1; + } /* end if */ + else { + (iter->attr_bins[attr_bin])++; + } /* end else */ + + ret = H5Gclose(gid); + assert(ret >= 0); + + last_container = iter->container; + iter->container = fullname; + iter->curr_depth++; - iter->container = last_container; - iter->curr_depth--; + H5Giterate(group, name, NULL, walk, iter); + + iter->container = last_container; + iter->curr_depth--; return 0; } @@ -457,32 +525,38 @@ group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * * Modifications: Refactored code from the walk_function * EIP, Wednesday, August 16, 2006 * + * Vailin Choi 12 July 2007 + * 1. Gathered storage info for btree and heap + * (chunked datasets and attributes) + * 2. Gathered info for attributes + * *------------------------------------------------------------------------- */ static herr_t dataset_stats (hid_t group, const char *name, H5G_stat_t * _sb, iter_t *_iter) - - { - unsigned bin; /* "bin" the number of objects falls in */ - iter_t *iter = (iter_t*)_iter; - H5G_stat_t *sb = _sb; - herr_t ret; - - hid_t did; /* Dataset ID */ - hid_t sid; /* Dataspace ID */ - hid_t tid; /* Datatype ID */ - hid_t dcpl; /* Dataset creation property list ID */ - hsize_t dims[H5S_MAX_RANK]; /* Dimensions of dataset */ - H5D_layout_t lout; /* Layout of dataset */ - unsigned type_found; /* Whether the dataset's datatype was */ - /* already found */ - int ndims; /* Number of dimensions of dataset */ - hsize_t storage; /* Size of dataset storage */ - unsigned u; /* Local index variable */ - int num_ext; /* Number of external files for a dataset */ - int nfltr; /* Number of filters for a dataset */ - H5Z_filter_t fltr; /* Filter identifier */ + unsigned bin; /* "bin" the number of objects falls in */ + iter_t *iter = (iter_t*)_iter; + H5G_stat_t *sb = _sb; + herr_t ret; + hid_t did; /* Dataset ID */ + hid_t sid; /* Dataspace ID */ + hid_t tid; /* Datatype ID */ + hid_t dcpl; /* Dataset creation property list ID */ + hsize_t dims[H5S_MAX_RANK];/* Dimensions of dataset */ + H5D_layout_t lout; /* Layout of dataset */ + unsigned type_found; /* Whether the dataset's datatype was */ + /* already found */ + int ndims; /* Number of dimensions of dataset */ + hsize_t storage; /* Size of dataset storage */ + unsigned u; /* Local index variable */ + int num_ext; /* Number of external files for a dataset */ + int nfltr; /* Number of filters for a dataset */ + H5Z_filter_t fltr; /* Filter identifier */ + + hsize_t num_attrs = 0; + unsigned attr_bin; + H5O_info_t oinfo; /* Gather statistics about this type of object */ iter->uniq_dsets++; @@ -494,6 +568,39 @@ dataset_stats (hid_t group, const char *name, H5G_stat_t * _sb, iter_t *_iter) did = H5Dopen(group, name); assert(did > 0); + ret = H5Oget_info(did, ".", &oinfo, H5P_DEFAULT); + if (ret < 0) { + warn_msg(progname, "Unable to retrieve object info for \"%s\"\n", name); + } else { + num_attrs = oinfo.num_attrs; + iter->datasets_btree_storage_size += oinfo.meta_size.obj.index_size; + iter->attrs_btree_storage_size += oinfo.meta_size.attr.index_size; + iter->attrs_heap_storage_size += oinfo.meta_size.attr.heap_size; + } + + if(num_attrs < SIZE_SMALL_ATTRS) + (iter->num_small_attrs[num_attrs])++; + if(num_attrs > iter->max_attrs) + iter->max_attrs = num_attrs; + + /* Add attribute count to proper bin */ + attr_bin = ceil_log10((unsigned long)num_attrs); + if((attr_bin + 1) > iter->attr_nbins) { + iter->attr_bins = realloc(iter->attr_bins, (attr_bin + 1) * sizeof(unsigned long)); + assert(iter->attr_bins); + + /* Initialize counts for intermediate bins */ + while(iter->attr_nbins < attr_bin) + iter->attr_bins[iter->attr_nbins++] = 0; + iter->attr_nbins++; + + /* Initialize count for new bin */ + iter->attr_bins[attr_bin] = 1; + } /* end if */ + else { + (iter->attr_bins[attr_bin])++; + } /* end else */ + /* Get storage info */ storage = H5Dget_storage_size(did); iter->dset_storage_size += storage; @@ -514,8 +621,6 @@ dataset_stats (hid_t group, const char *name, H5G_stat_t * _sb, iter_t *_iter) /* Only gather dim size statistics on 1-D datasets */ if(ndims == 1) { - - if(dims[0] > iter->max_dset_dims) iter->max_dset_dims = dims[0]; if(dims[0] < SIZE_SMALL_DSETS) (iter->small_dset_dims[dims[0]])++; @@ -633,11 +738,10 @@ dataset_stats (hid_t group, const char *name, H5G_stat_t * _sb, iter_t *_iter) * Purpose: Gather statistics about the file * * Return: Success: 0 - * - * Failure: -1 + * Failure: -1 * * Programmer: Quincey Koziol - * Tuesday, August 16, 2005 + * Tuesday, August 16, 2005 * * Modifications: * @@ -654,9 +758,6 @@ walk (hid_t group, const char *name, void *_iter) /* Get the full object name */ fullname = fix_name(iter->container, name); -/* -printf("walk: fullname = %s\n", fullname); -*/ /* Get object information */ ret = H5Gget_objinfo(group, name, FALSE, &sb); assert(ret >= 0); @@ -664,7 +765,7 @@ printf("walk: fullname = %s\n", fullname); /* If the object has already been printed then just show the object ID * and return. */ if ((s=sym_lookup(&sb))) { - printf("same as %s", s); + printf("%s same as %s\n", name, s); } else { sym_insert(&sb, fullname); @@ -717,7 +818,9 @@ printf("walk: fullname = %s\n", fullname); * Programmer: Elena Pourmal * Saturday, August 12, 2006 * - * Modifications: + * Modifications: + * Vailin Choi 12 July 2007 + * Added 'A' option to display attribute info * *------------------------------------------------------------------------- */ @@ -734,6 +837,10 @@ parse_command_line(int argc, const char *argv[]) /* parse command line options */ while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { + case 'A': + display_all = FALSE; + display_attr = TRUE; + break; case 'F': display_all = FALSE; display_file_metadata = TRUE; @@ -807,6 +914,12 @@ parse_command_line(int argc, const char *argv[]) * Saturday, August 12, 2006 * * Modifications: + * Vailin Choi 12 July 2007 + * Initialized storage info for: + * 1. btree/heap storage for groups and attributes + * 2. btree storage for chunked dataset + * 3. hdr/btree/list/heap storage for SOHM table + * 4. superblock extension size * *------------------------------------------------------------------------- */ @@ -838,6 +951,13 @@ iter_init(iter_t * _iter) iter->group_ohdr_info.total_size = 0; iter->group_ohdr_info.free_size = 0; + /* initialize attributes' information for groups and datasets */ + iter->max_attrs = 0; + for(u = 0; u < SIZE_SMALL_ATTRS; u++) + iter->num_small_attrs[u] = 0; + iter->attr_nbins = 0; + iter->attr_bins = NULL; + /* Initilaize datasets' metadata information */ iter->max_dset_rank = 0; for(u = 0; u < H5S_MAX_RANK; u++) @@ -849,6 +969,7 @@ iter_init(iter_t * _iter) iter->dset_layouts[u] = 0; for(u = 0; u < H5_NFILTERS_IMPL; u++) iter->dset_comptype[u] = 0; + iter->dset_ntypes = 0; iter->dset_type_info = NULL; iter->dset_dim_nbins = 0; @@ -856,6 +977,18 @@ iter_init(iter_t * _iter) iter->dset_ohdr_info.total_size = 0; iter->dset_ohdr_info.free_size = 0; iter->dset_storage_size = 0; + + /* Initialize storage info */ + iter->groups_btree_storage_size = 0; + iter->groups_heap_storage_size = 0; + iter->attrs_btree_storage_size = 0; + iter->attrs_heap_storage_size = 0; + iter->SM_hdr_storage_size = 0; + iter->SM_index_storage_size = 0; + iter->SM_heap_storage_size = 0; + iter->super_ext_size = 0; + iter->datasets_btree_storage_size = 0; + iter->nexternal = 0; iter->local = 0; @@ -911,6 +1044,12 @@ print_file_info(iter_t * _iter) * Saturday, August 12, 2006 * * Modifications: + * Vailin Choi 12 July 2007 + * Print storage info for: + * 1. btree/heap storage for groups and attributes + * 2. btree storage for chunked dataset + * 3. hdr/btree/list/heap storage for SOHM table + * 4. superblock extension size * *------------------------------------------------------------------------- */ @@ -926,6 +1065,24 @@ print_file_metadata(iter_t * _iter) HDfprintf(stdout, "\tDatasets: %Hu/%Hu\n", iter->dset_ohdr_info.total_size, iter->dset_ohdr_info.free_size); + printf("Storage information:\n"); + HDfprintf(stdout, "\tGroups:\n"); + HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->groups_btree_storage_size); + HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->groups_heap_storage_size); + + HDfprintf(stdout, "\tAttributes:\n"); + HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->attrs_btree_storage_size); + HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->attrs_heap_storage_size); + + HDfprintf(stdout, "\tChunked datasets:\n"); + HDfprintf(stdout, "\t\tB-tree: %Hu\n", iter->datasets_btree_storage_size); + + HDfprintf(stdout, "\tShared Messages:\n"); + HDfprintf(stdout, "\t\tHeader: %Hu\n", iter->SM_hdr_storage_size); + HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->SM_index_storage_size); + HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->SM_heap_storage_size); + + HDfprintf(stdout, "\tSuperblock extension: %Hu\n", iter->super_ext_size); return ret; } @@ -985,6 +1142,57 @@ print_group_info(iter_t * _iter) } /*------------------------------------------------------------------------- + * Function: print_attr_info + * + * Purpose: Prints information about attributes in the file + * + * Return: Success: 0 + * + * Failure: Never fails + * + * Programmer: Vailin Choi + * July 12, 2007 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +print_attr_info(iter_t * _iter) +{ + iter_t *iter = (iter_t*)_iter; + herr_t ret =0; /* Generic return value */ + unsigned u; /* Local index variable */ + unsigned long power; /* Temporary "power" for bins */ + unsigned long total; /* Total count for various statistics */ + + printf("Small # of attributes:\n"); + total = 0; + for(u = 1; u < SIZE_SMALL_ATTRS; u++) { + if(iter->num_small_attrs[u] > 0) { + printf("\t# of objects with %u attributes: %lu\n", u, iter->num_small_attrs[u]); + total += iter->num_small_attrs[u]; + } /* end if */ + } /* end for */ + printf("\tTotal # of objects with small # of attributes: %lu\n", total); + + printf("Attribute bins:\n"); + total = 0; + power = 1; + for(u = 1; u < iter->attr_nbins; u++) { + if(iter->attr_bins[u] > 0) { + printf("\t# of objects with %lu - %lu attributes: %lu\n", power, (power * 10) - 1, + iter->attr_bins[u]); + total += iter->attr_bins[u]; + } /* end if */ + power *= 10; + } /* end for */ + printf("\tTotal # of objects with attributes: %lu\n", total); + printf("\tMax. # of attributes to objects: %lu\n", (unsigned long)iter->max_attrs); + return ret; +} + +/*------------------------------------------------------------------------- * Function: print_dataset_info * * Purpose: Prints information about datasets in the file @@ -1125,6 +1333,7 @@ print_file_statistics(iter_t * _iter) display_group_metadata = TRUE; display_dset = TRUE; display_dtype_metadata = TRUE; + display_attr = TRUE; } @@ -1132,6 +1341,7 @@ print_file_statistics(iter_t * _iter) if(display_file_metadata) print_file_metadata(iter); if(display_group) print_group_info(iter); if(display_dset) print_dataset_info(iter); + if(display_attr) print_attr_info(iter); } @@ -1192,13 +1402,15 @@ print_statistics(char *name, iter_t * _iter) int main(int argc, const char *argv[]) { - iter_t iter; - const char *fname = NULL; - hid_t fid; + iter_t iter; + const char *fname = NULL; + hid_t fid; struct handler_t *hand; - herr_t status; - char root[] = "/"; - int i; + herr_t status, reterr; + char root[] = "/"; + int i; + H5F_info_t finfo; + /* Disable error reporting */ H5Eset_auto2(H5E_DEFAULT, NULL, NULL); @@ -1211,7 +1423,6 @@ main(int argc, const char *argv[]) leave(EXIT_FAILURE); } - fname = argv[opt_ind]; hand[opt_ind].obj = root; hand[opt_ind].flag = 1; @@ -1225,9 +1436,22 @@ main(int argc, const char *argv[]) leave(EXIT_FAILURE); } + /* Initialize iter structure */ status = iter_init(&iter); + /* Get storge info for SOHM's btree/list/heap and superblock extension */ + reterr = H5Fget_info(fid, &finfo); + if (reterr < 0) + warn_msg(progname, "Unable to retrieve SOHM info\n"); + else { + iter.super_ext_size = finfo.super_ext_size; + iter.SM_hdr_storage_size = finfo.sohm.hdr_size; + iter.SM_index_storage_size = finfo.sohm.msgs_info.index_size; + iter.SM_heap_storage_size = finfo.sohm.msgs_info.heap_size; + } + + /* Walk the objects or all file */ for (i = 0; i < argc; i++) { if (hand[i].obj) { diff --git a/tools/misc/testfiles/h5stat_filters-F.ddl b/tools/misc/testfiles/h5stat_filters-F.ddl index be95877..544d0c7 100644 --- a/tools/misc/testfiles/h5stat_filters-F.ddl +++ b/tools/misc/testfiles/h5stat_filters-F.ddl @@ -5,3 +5,17 @@ Filename: h5stat_filters.h5 Object header size: (total/unused) Groups: 48/8 Datasets: 4936/1344 +Storage information: + Groups: + B-tree/List: 1200 + Heap: 288 + Attributes: + B-tree/List: 0 + Heap: 0 + Chunked datasets: + B-tree: 31392 + Shared Messages: + Header: 0 + B-tree/List: 0 + Heap: 0 + Superblock extension: 0 diff --git a/tools/misc/testfiles/h5stat_filters.ddl b/tools/misc/testfiles/h5stat_filters.ddl index 891a62b..11eb76a 100644 --- a/tools/misc/testfiles/h5stat_filters.ddl +++ b/tools/misc/testfiles/h5stat_filters.ddl @@ -14,6 +14,20 @@ File information Object header size: (total/unused) Groups: 48/8 Datasets: 4936/1344 +Storage information: + Groups: + B-tree/List: 1200 + Heap: 288 + Attributes: + B-tree/List: 0 + Heap: 0 + Chunked datasets: + B-tree: 31392 + Shared Messages: + Header: 0 + B-tree/List: 0 + Heap: 0 + Superblock extension: 0 Small groups: Total # of small groups: 0 Group bins: @@ -57,3 +71,8 @@ Dataset datatype information: Count (total/named) = (1/0) Size (desc./elmt) = (14/4) Total dataset datatype count: 15 +Small # of attributes: + Total # of objects with small # of attributes: 0 +Attribute bins: + Total # of objects with attributes: 0 + Max. # of attributes to objects: 0 diff --git a/tools/misc/testfiles/h5stat_help1.ddl b/tools/misc/testfiles/h5stat_help1.ddl index c4ef2c1..0841572 100644 --- a/tools/misc/testfiles/h5stat_help1.ddl +++ b/tools/misc/testfiles/h5stat_help1.ddl @@ -20,4 +20,5 @@ Usage: h5stat [OPTIONS] file -d, --dset Print dataset information -D, --dsetmetadata Print dataset metadata -T, --dtypemetadata Print datatype metadata + -A, --attribute Print attribute information diff --git a/tools/misc/testfiles/h5stat_help2.ddl b/tools/misc/testfiles/h5stat_help2.ddl index 8d9d6d6..1e6295b 100644 --- a/tools/misc/testfiles/h5stat_help2.ddl +++ b/tools/misc/testfiles/h5stat_help2.ddl @@ -20,4 +20,5 @@ Usage: h5stat [OPTIONS] file -d, --dset Print dataset information -D, --dsetmetadata Print dataset metadata -T, --dtypemetadata Print datatype metadata + -A, --attribute Print attribute information diff --git a/tools/misc/testh5stat.sh.in b/tools/misc/testh5stat.sh.in index 0672366..66d05be 100644 --- a/tools/misc/testh5stat.sh.in +++ b/tools/misc/testh5stat.sh.in @@ -119,6 +119,11 @@ TOOLTEST h5stat_filters-F.ddl -F h5stat_filters.h5 TOOLTEST h5stat_filters-d.ddl -d h5stat_filters.h5 TOOLTEST h5stat_filters-g.ddl -g h5stat_filters.h5 TOOLTEST h5stat_filters-dT.ddl -dT h5stat_filters.h5 +# h5stat_tsohm.h5 is a copy of ../../../test/tsohm.h5 generated by tsohm.c +# as of release 1.8.0-alpha4 +TOOLTEST h5stat_tsohm.ddl h5stat_tsohm.h5 +# h5stat_newgrat.h5 is generated by h5stat_gentest.c +TOOLTEST h5stat_newgrat.ddl h5stat_newgrat.h5 echo |