diff options
44 files changed, 1367 insertions, 79 deletions
@@ -2069,3 +2069,106 @@ done: FUNC_LEAVE_NOAPI(ret_value) } #endif /* H5B_DEBUG */ + +/*------------------------------------------------------------------------- + * Function: H5B_iterate_btree_size + * + * Purpose: Return the amount of storage used for the btree. + * Keep following the left-most child until reaching the leaf node. + * For each level, gather storage for all the nodes on that level. + * For 0 level, also gather storage for the SNODs. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 19, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B_iterate_btree_size(H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, H5B_operator_t op, haddr_t addr, H5B_info_ud_t *bh_udata) +{ + H5B_t *bt = NULL; /* Pointer to current B-tree node */ + herr_t ret_value; /* Return value */ + H5B_shared_t *shared; /* Pointer to shared B-tree info */ + H5B_t *start_bt; /* Pointer to next B-tree node */ + haddr_t start_addr; /* Address of next node to iterate over */ + + haddr_t *child; /* Pointer to node's child addresses */ + uint8_t *key; /* Pointer to node's native keys */ + unsigned u; /* Local index variable */ + + FUNC_ENTER_NOAPI(H5B_iterate_btree_size, FAIL) + + /* + * Check arguments. + */ + HDassert(f); + HDassert(type); + HDassert(H5F_addr_defined(addr)); + HDassert(bh_udata); + + /* Protect the initial/current node */ + if(NULL == (bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, addr, type, bh_udata->udata, H5AC_READ))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, FAIL, "unable to load B-tree node") + shared = H5RC_GET_OBJ(bt->rc_shared); + HDassert(shared); + + if (bt->level > 0) { + /* Keep following the left-most child until we reach a leaf node. */ + if((ret_value = H5B_iterate_btree_size(f, dxpl_id, type, op, bt->child[0], bh_udata)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTLIST, FAIL, "unable to list B-tree node") + } + ret_value = H5_ITER_CONT; + while(bt && ret_value == H5_ITER_CONT) { + /* count the SNOD sizes for leaf nodes */ + if (bt->level == 0) { /* for leaf node only */ + for(u = 0, child = bt->child, key = bt->native; u < bt->nchildren && ret_value == H5_ITER_CONT; u++, child++, key += type->sizeof_nkey) { + if (op != NULL) { /* only for symbol table nodes */ + ret_value = (*op)(f, dxpl_id, key, *child, key + type->sizeof_nkey, bh_udata->btree_size); + } + if(ret_value < 0) + HERROR(H5E_BTREE, H5E_CANTLIST, "iterator function failed"); + } /* end for */ + } /* end if */ + + /* Check for continuing iteration */ + if(ret_value == H5_ITER_CONT) { + H5B_t *next_bt; /* Pointer to next B-tree node */ + haddr_t next_addr; /* Address of next node to iterate over */ + + /* count the size of bt */ + *(bh_udata->btree_size) += H5B_nodesize(f, shared, NULL); + + /* Protect bt's next node to the right, if there is one */ + if(H5F_addr_defined(bt->right)) { + next_addr = bt->right; + if(NULL == (next_bt = (H5B_t *)H5AC_protect(f, dxpl_id, H5AC_BT, next_addr, type, bh_udata->udata, H5AC_READ))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTLOAD, FAIL, "B-tree node") + } else { + next_addr = HADDR_UNDEF; + next_bt = NULL; + } /* end if */ + + /* Unprotect bt */ + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, addr, bt, H5AC__NO_FLAGS_SET) < 0) { + if(next_bt) { + HDassert(H5F_addr_defined(next_addr)); + if(H5AC_unprotect(f, dxpl_id, H5AC_BT, next_addr, next_bt, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_PROTECT, FAIL, "unable to release B-tree node") + } /* end if */ + HGOTO_ERROR(H5E_BTREE, H5E_PROTECT, FAIL, "unable to release B-tree node") + } /* end if */ + + /* Advance to the next node */ + bt = next_bt; + addr = next_addr; + } /* end if */ + } /* end while */ + +done: + if(bt && H5AC_unprotect(f, dxpl_id, H5AC_BT, addr, bt, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_BTREE, H5E_PROTECT, FAIL, "unable to release B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5B_iterate_btree_size() */ @@ -1187,4 +1187,76 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* H5B2_modify() */ + +/*------------------------------------------------------------------------- + * Function: H5B2_info_iterate + * + * Purpose: Iterate over all the records in the B-tree, collecting + * storage info. + * + * Return: non-negative on success, negative on error + * + * Programmer: Vailin Choi + * June 19 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2_info_iterate(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, haddr_t addr, hsize_t *btree_size) +{ + H5B2_t *bt2=NULL; /* Pointer to the B-tree header */ + H5B2_shared_t *shared; /* Pointer to B-tree's shared information */ + H5RC_t *bt2_shared=NULL; /* Pointer to ref-counter for shared B-tree info */ + hbool_t incr_rc=FALSE; /* Flag to indicate that we've incremented the B-tree's shared info reference count */ + H5B2_node_ptr_t root_ptr; /* Node pointer info for root node */ + unsigned depth; /* Current depth of the tree */ + herr_t ret_value=SUCCEED; + + + FUNC_ENTER_NOAPI(H5B2_info_iterate, FAIL) + + /* Check arguments. */ + HDassert(f); + HDassert(type); + HDassert(H5F_addr_defined(addr)); + HDassert(btree_size); + + /* Look up the B-tree header */ + if(NULL == (bt2 = H5AC_protect(f, dxpl_id, H5AC_BT2_HDR, addr, type, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree header") + + bt2_shared = bt2->shared; + H5RC_INC(bt2_shared); + incr_rc = TRUE; + + /* Get the pointer to the shared B-tree info */ + shared = H5RC_GET_OBJ(bt2->shared); + HDassert(shared); + + *btree_size += H5B2_HEADER_SIZE(f); + + /* Make copy of the root node pointer */ + root_ptr = bt2->root; + + /* Current depth of the tree */ + depth = shared->depth; + + /* Release header */ + if(H5AC_unprotect(f, dxpl_id, H5AC_BT2_HDR, addr, bt2, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree header info") + bt2 = NULL; + /* Iterate through records */ + if(root_ptr.node_nrec > 0) { + /* Iterate through nodes */ + if((ret_value = H5B2_info_iterate_node(f, dxpl_id, bt2_shared, depth, &root_ptr, btree_size)) < 0) + HERROR(H5E_BTREE, H5E_CANTLIST, "node iteration failed"); + } /* end if */ + +done: + /* Check if we need to decrement the reference count for the B-tree's shared info */ + if(incr_rc) + H5RC_DEC(bt2_shared); + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2_info_iterate() */ diff --git a/src/H5B2int.c b/src/H5B2int.c index 5d71d43..20e4300 100644 --- a/src/H5B2int.c +++ b/src/H5B2int.c @@ -3304,4 +3304,63 @@ H5B2_assert_internal2(hsize_t parent_all_nrec, H5B2_shared_t *shared, H5B2_inter return(0); } /* end H5B2_assert_internal2() */ #endif /* H5B2_DEBUG */ + +/*------------------------------------------------------------------------- + * Function: H5B2_info_iterate_node + * + * Purpose: Iterate over all the records from a B-tree node, collecting + * btree storage info. + * + * Return: non-negative on success, negative on error + * + * Programmer: Vailin Choi + * July 12 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5B2_info_iterate_node(H5F_t *f, hid_t dxpl_id, H5RC_t *bt2_shared, unsigned depth, +const H5B2_node_ptr_t *curr_node, hsize_t *btree_size) +{ + H5B2_shared_t *shared; /* Pointer to B-tree's shared information */ + herr_t ret_value = SUCCEED; /* Iterator return value */ + H5B2_internal_t *internal = NULL; /* Pointer to internal node */ + unsigned u; /* Local index */ + + FUNC_ENTER_NOAPI(H5B2_info_iterate_node, FAIL) + + /* Check arguments. */ + HDassert(f); + HDassert(bt2_shared); + HDassert(curr_node); + HDassert(btree_size); + + /* Get the pointer to the shared B-tree info */ + shared=(H5B2_shared_t *)H5RC_GET_OBJ(bt2_shared); + HDassert(shared); + + + if(depth > 0) { + /* Lock the current B-tree node */ + if(NULL == (internal = H5B2_protect_internal(f, dxpl_id, bt2_shared, curr_node->addr, curr_node->node_nrec, depth, H5AC_READ))) + HGOTO_ERROR(H5E_BTREE, H5E_CANTPROTECT, FAIL, "unable to load B-tree internal node") + + if(depth>1) { + /* Descend into children */ + for(u=0; u<internal->nrec+1; u++) + if(H5B2_info_iterate_node(f, dxpl_id, bt2_shared, (depth-1), &(internal->node_ptrs[u]), btree_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTLIST, FAIL, "node iteration failed") + } /* end if */ + else /* depth is 1: count all the leaf nodes from this node */ + *btree_size += (internal->nrec + 1) * shared->node_size; + } /* end if */ + + /* Count this node */ + *btree_size += shared->node_size; +done: + if(internal && H5AC_unprotect(f, dxpl_id, H5AC_BT2_INT, curr_node->addr, internal, H5AC__NO_FLAGS_SET) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTUNPROTECT, FAIL, "unable to release B-tree node") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5B2_info_iterate_node() */ diff --git a/src/H5B2pkg.h b/src/H5B2pkg.h index 232cfab..956abb1 100644 --- a/src/H5B2pkg.h +++ b/src/H5B2pkg.h @@ -270,6 +270,11 @@ H5_DLL herr_t H5B2_iterate_node(H5F_t *f, hid_t dxpl_id, H5RC_t *bt2_shared, unsigned depth, const H5B2_node_ptr_t *curr_node, H5B2_operator_t op, void *op_data); +H5_DLL herr_t H5B2_info_iterate_node(H5F_t *f, hid_t dxpl_id, H5RC_t *bt2_shared, + unsigned depth, const H5B2_node_ptr_t *curr_node, hsize_t *op_data); + + + /* Routines for locating records */ H5_DLL int H5B2_locate_record(const H5B2_class_t *type, unsigned nrec, size_t *rec_off, const uint8_t *native, const void *udata, unsigned *idx); diff --git a/src/H5B2private.h b/src/H5B2private.h index 0bcd83c..064743f 100644 --- a/src/H5B2private.h +++ b/src/H5B2private.h @@ -122,6 +122,10 @@ H5_DLL herr_t H5B2_insert(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, haddr_t addr, void *udata); H5_DLL herr_t H5B2_iterate(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, haddr_t addr, H5B2_operator_t op, void *op_data); + +H5_DLL herr_t H5B2_info_iterate(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, + haddr_t addr, hsize_t *op_data); + H5_DLL herr_t H5B2_find(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, haddr_t addr, void *udata, H5B2_found_t op, void *op_data); H5_DLL herr_t H5B2_index(H5F_t *f, hid_t dxpl_id, const H5B2_class_t *type, diff --git a/src/H5Bprivate.h b/src/H5Bprivate.h index 6be236d..4d726be 100644 --- a/src/H5Bprivate.h +++ b/src/H5Bprivate.h @@ -124,6 +124,11 @@ typedef struct H5B_class_t { herr_t (*debug_key)(FILE*, H5F_t*, hid_t, int, int, const void*, const void*); } H5B_class_t; +typedef struct H5B_info_ud_t { + void *udata; + hsize_t *btree_size; +} H5B_info_ud_t; + /*****************************/ /* Library-private Variables */ /*****************************/ @@ -145,6 +150,8 @@ H5_DLL herr_t H5B_insert (H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, hadd void *udata); H5_DLL herr_t H5B_iterate (H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, H5B_operator_t op, haddr_t addr, void *udata); +H5_DLL herr_t H5B_iterate_btree_size (H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, H5B_operator_t + op, haddr_t addr, H5B_info_ud_t *bh_udata); H5_DLL herr_t H5B_remove(H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, haddr_t addr, void *udata); H5_DLL herr_t H5B_delete(H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, haddr_t addr, @@ -3288,4 +3288,3 @@ H5Ddebug(hid_t dset_id) done: FUNC_LEAVE_API(ret_value) } /* end H5Ddebug() */ - diff --git a/src/H5Distore.c b/src/H5Distore.c index 7e27d1e..297a480 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -4027,3 +4027,47 @@ H5D_istore_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int inden done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_debug() */ + +/*------------------------------------------------------------------------- + * Function: H5D_istore_btree_size + * + * Purpose: Retrieve the amount of B-tree storage for chunked dataset + * + * Return: Success: Non-negative + * Failure: negative + * + * Programmer: Vailin Choi + * June 8, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_obj_istore_bh_info(H5O_loc_t *oloc, H5O_layout_t *layout, hid_t dxpl_id, hsize_t *btree_size) +{ + H5D_istore_it_ud1_t udata; + herr_t ret_value=SUCCEED; + H5B_info_ud_t bh_udata; + + FUNC_ENTER_NOAPI(H5D_obj_istore_bh_info, FAIL) + + HDassert(oloc); + HDassert(layout); + HDassert(btree_size); + + if(H5D_istore_shared_create(oloc->file, layout)<0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, FAIL, "can't create wrapper for shared B-tree info") + + HDmemset(&udata, 0, sizeof udata); + udata.common.mesg = layout; + + bh_udata.udata = &udata; + bh_udata.btree_size = btree_size; + if ((ret_value = H5B_iterate_btree_size(oloc->file, dxpl_id, H5B_ISTORE, NULL, layout->u.chunk.addr, &bh_udata)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "unable to iterate over chunk B-tree") +done: + if(layout->u.chunk.btree_shared==NULL) + HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "ref-counted page nil") + if(H5RC_DEC(layout->u.chunk.btree_shared)<0) + HGOTO_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_obj_istore_bh_info() */ diff --git a/src/H5Doh.c b/src/H5Doh.c index e59fe8b..620554e0 100644 --- a/src/H5Doh.c +++ b/src/H5Doh.c @@ -327,3 +327,43 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_dset_get_oloc() */ +/*------------------------------------------------------------------------- + * Function: H5O_dset_bh_info + * + * Purpose: Returns the amount of btree storage that is used for chunked + * dataset. + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi + * July 11, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_dset_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info) +{ + H5O_layout_t layout; /* Data storage layout message */ + herr_t ret_value=SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5O_dset_bh_info, FAIL) + + /* Sanity check */ + HDassert(oloc); + HDassert(oh); + HDassert(bh_info); + + if (NULL==H5O_msg_read_real(oloc->file, dxpl_id, oh, H5O_LAYOUT_ID, &layout)) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't find LAYOUT message") + else { + if ((layout.type == H5D_CHUNKED) && (layout.u.chunk.addr != HADDR_UNDEF)) { + ret_value = H5D_obj_istore_bh_info(oloc, &layout, dxpl_id, &(bh_info->index_size)); + if (ret_value < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't determine chunked dataset btree info") + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_dset_bh_info() */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index c4e4751..b1db412 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -329,6 +329,7 @@ H5_DLL herr_t H5D_istore_dest (H5D_t *dset, hid_t dxpl_id); H5_DLL herr_t H5D_istore_allocate (H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite); H5_DLL hsize_t H5D_istore_allocated(H5D_t *dset, hid_t dxpl_id); +H5_DLL herr_t H5D_obj_istore_bh_info(H5O_loc_t *oloc, H5O_layout_t *layout, hid_t dxpl_id, hsize_t *btree_size); H5_DLL herr_t H5D_istore_prune_by_extent(const H5D_io_info_t *io_info, const hsize_t *old_dims); H5_DLL herr_t H5D_istore_initialize_by_extent(H5D_io_info_t *io_info); @@ -3584,4 +3584,49 @@ H5Fget_name(hid_t obj_id, char *name/*out*/, size_t size) done: FUNC_LEAVE_API(ret_value) } /* end H5Fget_name() */ + +/*------------------------------------------------------------------------- + * Function: H5Fget_info + * 1. Get storage size for superblock extension if there is one + * 2. Get the amount of btree and heap storage for entries + * in the SOHM table if there is one. + * Consider success when there is no superblock extension and/or SOHM table + * + * Return: Success: non-negative on success + * Failure: Negative + * + * Programmer: Vailin Choi + * July 11, 2007 + *------------------------------------------------------------------------- + */ +herr_t +H5Fget_info(hid_t obj_id, H5F_info_t *finfo) +{ + H5F_t *f; + herr_t ret_value=SUCCEED; + + FUNC_ENTER_API(H5Fget_info, FAIL) + HDassert(finfo); + + HDmemset(finfo, 0, sizeof(H5F_info_t)); + if(H5I_get_type(obj_id) == H5I_FILE ) { + if(NULL == (f = H5I_object(obj_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "not a file") + else { + HDassert(f->shared); + if (H5F_addr_defined(f->shared->extension_addr)) { + if (H5F_super_ext_info(f, finfo, H5AC_ind_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Unable to retrieve superblock extension size") + } + if(H5F_addr_defined(f->shared->sohm_addr)) { + if (H5SM_ih_info(f, finfo, H5AC_ind_dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Unable to retrieve SOHM btree & heap storage info") + } + } + } else + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a valid object ID") + +done: + FUNC_LEAVE_API(ret_value) +} /* end H5Fget_info() */ @@ -493,4 +493,51 @@ HDfprintf(stderr, "%s: fspace->hdr->tot_sect_count = %Hu\n", "H5FS_assert", fspa FUNC_LEAVE_NOAPI(SUCCEED) } /* end H5FS_assert() */ #endif /* H5FS_DEBUG */ + +/*------------------------------------------------------------------------- + * Function: H5FS_meta_info + * + * Purpose: Collect meta storage info used by the free space manager + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi + * June 19, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5FS_meta_info(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, hsize_t *meta_size) +{ + H5FS_t *fspace = NULL; /* Free space header info */ + H5FS_prot_t fs_prot; /* Information for protecting free space manager */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5FS_meta_info, FAIL) + /* + * Check arguments. + */ + HDassert(f); + HDassert(H5F_addr_defined(fs_addr)); + HDassert(meta_size); + + /* Initialize user data for protecting the free space manager */ + fs_prot.nclasses = 0; + fs_prot.classes = NULL; + fs_prot.cls_init_udata = NULL; + + /* + * Load the free space header. + */ + if(NULL == (fspace = H5AC_protect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, &fs_prot, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTLOAD, FAIL, "unable to load free space header") + + *meta_size = H5FS_HEADER_SIZE(f) + fspace->alloc_sect_size; + +done: + if(fspace && H5AC_unprotect(f, dxpl_id, H5AC_FSPACE_HDR, fs_addr, fspace, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_FSPACE, H5E_PROTECT, FAIL, "unable to release free space header") + FUNC_LEAVE_NOAPI(ret_value) +} diff --git a/src/H5FSprivate.h b/src/H5FSprivate.h index 4840df2..e6a3ce4 100644 --- a/src/H5FSprivate.h +++ b/src/H5FSprivate.h @@ -161,6 +161,8 @@ H5_DLL H5FS_t *H5FS_open(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, H5_DLL herr_t H5FS_delete(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr); H5_DLL herr_t H5FS_close(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace); +H5_DLL herr_t H5FS_meta_info(H5F_t *f, hid_t dxpl_id, haddr_t fs_addr, hsize_t *meta_size); + /* Free space section routines */ H5_DLL herr_t H5FS_sect_add(H5F_t *f, hid_t dxpl_id, H5FS_t *fspace, H5FS_section_info_t *node, unsigned flags, void *op_data); diff --git a/src/H5Fpkg.h b/src/H5Fpkg.h index 8c43ea2..60a81a9 100644 --- a/src/H5Fpkg.h +++ b/src/H5Fpkg.h @@ -178,6 +178,9 @@ H5_DLL herr_t H5F_super_init(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5F_super_write(H5F_t *f, hid_t dxpl_id); H5_DLL herr_t H5F_super_read(H5F_t *f, hid_t dxpl_id, H5G_loc_t *root_loc); +H5_DLL herr_t H5F_super_ext_info(H5F_t *f, H5F_info_t *finfo, hid_t dxpl_id); + + /* Shared file list related routines */ H5_DLL herr_t H5F_sfile_add(H5F_file_t *shared); H5_DLL H5F_file_t * H5F_sfile_search(H5FD_t *lf); diff --git a/src/H5Fpublic.h b/src/H5Fpublic.h index 94f2bd7..db86101 100644 --- a/src/H5Fpublic.h +++ b/src/H5Fpublic.h @@ -97,6 +97,14 @@ typedef enum H5F_close_degree_t { H5F_CLOSE_STRONG = 3 } H5F_close_degree_t; +typedef struct H5F_info_t { + hsize_t super_ext_size; /* superblock extension size */ + struct { + hsize_t hdr_size; + H5_ih_info_t msgs_info; + } sohm; +} H5F_info_t; + #ifdef __cplusplus extern "C" { #endif @@ -132,6 +140,8 @@ H5_DLL herr_t H5Fget_mdc_size(hid_t file_id, int * cur_num_entries_ptr); H5_DLL herr_t H5Freset_mdc_hit_rate_stats(hid_t file_id); H5_DLL ssize_t H5Fget_name(hid_t obj_id, char *name, size_t size); +H5_DLL herr_t H5Fget_info(hid_t obj_id, H5F_info_t *bh_info); + #ifdef __cplusplus } diff --git a/src/H5Fsuper.c b/src/H5Fsuper.c index bf00584..4a516af 100644 --- a/src/H5Fsuper.c +++ b/src/H5Fsuper.c @@ -965,3 +965,30 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5F_super_write() */ +/*------------------------------------------------------------------------- + * Function: H5F_super_ext_info + * Get storage size of the superblock extension + * + * Return: Success: non-negative on success + * Failure: Negative + * + * Programmer: Vailin Choi + * July 11, 2007 + *------------------------------------------------------------------------- + */ +herr_t +H5F_super_ext_info(H5F_t *f, H5F_info_t *finfo, hid_t dxpl_id) +{ + herr_t ret_value=SUCCEED; + + FUNC_ENTER_NOAPI(H5F_super_ext_info, FAIL) + + HDassert(f); + HDassert(finfo); + + if (H5O_super_ext_size(f, &(finfo->super_ext_size), dxpl_id) < 0) + HGOTO_ERROR(H5E_FILE, H5E_CANTGET, FAIL, "Unable to retrieve superblock extension size") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5F_super_ext_info() */ diff --git a/src/H5Gnode.c b/src/H5Gnode.c index fb3c999..cf49c8b 100644 --- a/src/H5Gnode.c +++ b/src/H5Gnode.c @@ -2015,4 +2015,36 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5G_node_debug() */ + +/*------------------------------------------------------------------------- + * Function: H5G_btree_node_iterate + * + * Purpose: This function gets called by H5B_iterate_btree_size() + * to gather storage info for SNODs. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * Jun 19 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5G_btree_node_iterate(H5F_t *f, hid_t dxpl_id, const void UNUSED *_lt_key, haddr_t UNUSED addr, + const void UNUSED *_rt_key, void *_udata) +{ + herr_t ret_value=SUCCEED; + hsize_t *stab_size=(hsize_t *)_udata; + + FUNC_ENTER_NOAPI(H5G_btree_node_iterate, FAIL) + + /* + * Check arguments. + */ + HDassert(f); + HDassert(stab_size); + *stab_size += H5G_node_size(f); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5G_btree_node_iterate() */ diff --git a/src/H5Gobj.c b/src/H5Gobj.c index a3166ba..09eb3b1 100644 --- a/src/H5Gobj.c +++ b/src/H5Gobj.c @@ -1247,4 +1247,3 @@ H5G_obj_lookup_by_idx(H5O_loc_t *grp_oloc, H5_index_t idx_type, done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5G_obj_lookup_by_idx() */ - diff --git a/src/H5Goh.c b/src/H5Goh.c index 598cd5a..9b070a6 100644 --- a/src/H5Goh.c +++ b/src/H5Goh.c @@ -242,3 +242,64 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_group_get_oloc() */ +/*------------------------------------------------------------------------- + * Function: H5O_group_bh_info + * + * Purpose: Retrieve storage for 1.8 btree and heap + * Retrieve storage for 1.6 btree and heap via H5G_stab_bh_info() + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * July 12 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_group_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info) +{ + herr_t ret_value=SUCCEED; /* Return value */ + hsize_t huge_bt_size=0; + H5O_linfo_t linfo; /* Link info message */ + H5O_stab_t stabinfo; /* Info about symbol table */ + + + + FUNC_ENTER_NOAPI(H5O_group_bh_info, FAIL) + + /* Sanity check */ + HDassert(oloc); + HDassert(oh); + HDassert(bh_info); + + if(NULL==H5O_msg_read_real(oloc->file, dxpl_id, oh, H5O_LINFO_ID, &linfo)) { + H5E_clear_stack(NULL); + if(NULL==H5O_msg_read_real(oloc->file, dxpl_id, oh, H5O_STAB_ID, &stabinfo)) { + HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "can't find LINFO nor STAB messages") + } else { /* STAB */ + if (H5G_stab_bh_info(oloc, &stabinfo, dxpl_id, bh_info) < 0) + HGOTO_ERROR(H5E_SYM, H5E_CANTGET, FAIL, "can't retrieve pre-1.8 group btree & heap info") + } + } else { /* LINFO */ + if (H5F_addr_defined(linfo.corder_bt2_addr)) { + if (H5B2_info_iterate(oloc->file, dxpl_id, H5G_BT2_CORDER, + linfo.corder_bt2_addr, &bh_info->index_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + if (H5F_addr_defined(linfo.name_bt2_addr)) { + if (H5B2_info_iterate(oloc->file, dxpl_id, H5G_BT2_NAME, + linfo.name_bt2_addr, &bh_info->index_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + huge_bt_size = 0; + if (H5F_addr_defined(linfo.fheap_addr)) { + if (H5HF_fheap_info(oloc->file, dxpl_id, linfo.fheap_addr, + &bh_info->heap_size, &huge_bt_size) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't retrieve fractal heap storage info") + } + bh_info->index_size += huge_bt_size; + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_group_bh_info() */ diff --git a/src/H5Gpkg.h b/src/H5Gpkg.h index 3da3417..cb36a0f 100644 --- a/src/H5Gpkg.h +++ b/src/H5Gpkg.h @@ -385,6 +385,10 @@ H5_DLL herr_t H5G_stab_lookup(H5O_loc_t *grp_oloc, const char *name, H5_DLL herr_t H5G_stab_lookup_by_idx(H5O_loc_t *grp_oloc, H5_iter_order_t order, hsize_t n, H5O_link_t *lnk, hid_t dxpl_id); +H5_DLL herr_t H5G_stab_bh_info(struct H5O_loc_t *grp_oloc, H5O_stab_t *stabinfo, + hid_t dxpl_id, H5_ih_info_t *bh_info); + + /* * Functions that understand symbol table entries. */ @@ -404,6 +408,10 @@ H5_DLL herr_t H5G_ent_debug(H5F_t *f, const H5G_entry_t *ent, H5_DLL herr_t H5G_node_init(H5F_t *f); H5_DLL int H5G_node_iterate(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, const void *_rt_key, void *_udata); + +H5_DLL herr_t H5G_btree_node_iterate(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, + const void *_rt_key, void *_udata); + H5_DLL int H5G_node_sumup(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, const void *_rt_key, void *_udata); H5_DLL int H5G_node_by_idx(H5F_t *f, hid_t dxpl_id, const void *_lt_key, haddr_t addr, diff --git a/src/H5Gprivate.h b/src/H5Gprivate.h index 40d2df8..665b597 100644 --- a/src/H5Gprivate.h +++ b/src/H5Gprivate.h @@ -175,6 +175,7 @@ H5_DLL herr_t H5G_obj_ent_decode(H5F_t *f, const uint8_t **pp, H5_DLL herr_t H5G_obj_ent_encode(const H5F_t *f, uint8_t **pp, const struct H5O_loc_t *oloc); + /* * These functions operate on group hierarchy names. */ diff --git a/src/H5Gstab.c b/src/H5Gstab.c index 22d90a7..9e9f91b 100644 --- a/src/H5Gstab.c +++ b/src/H5Gstab.c @@ -1046,4 +1046,43 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5G_stab_lookup_by_idx() */ + +/*------------------------------------------------------------------------- + * Function: H5G_stab_bh_info + * + * Purpose: Retrieve storage for btree and heap (1.6) + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 25 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5G_stab_bh_info(H5O_loc_t *oloc, H5O_stab_t *stab, hid_t dxpl_id, H5_ih_info_t *bh_info) +{ + hsize_t stab_size=0; /* storage used for symbol table nodes */ + herr_t ret_value = SUCCEED; + H5B_info_ud_t bh_udata; + + FUNC_ENTER_NOAPI(H5G_stab_bh_info, FAIL) + + /* Sanity check */ + HDassert(oloc); + HDassert(stab); + HDassert(bh_info); + + bh_udata.udata = NULL; + bh_udata.btree_size = &(bh_info->index_size); + + if (H5B_iterate_btree_size(oloc->file, dxpl_id, H5B_SNODE, H5G_btree_node_iterate, stab->btree_addr, &bh_udata) <0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTINIT, FAIL, "iteration operator failed") + + if (H5HL_heapsize(oloc->file, dxpl_id, stab->heap_addr, &(bh_info->heap_size)) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTINIT, FAIL, "iteration operator failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5G_stab_bh_info() */ @@ -926,3 +926,88 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF_delete() */ + +/*------------------------------------------------------------------------- + * Function: H5HF_fheap_info + * + * Purpose: Retrieve storage info for: + * 1. fractal heap + * 2. btree storage used by huge objects in fractal heap + * 3. free space storage info + * + * Return: non-negative on success, negative on error + * + * Programmer: Vailin Choi + * July 12 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HF_fheap_info(H5F_t *f, hid_t dxpl_id, haddr_t fh_addr, hsize_t *heap_size, hsize_t *huge_bt_size) +{ + H5HF_hdr_t *hdr = NULL; + herr_t ret_value = SUCCEED; + hsize_t meta_size = 0; + int reterr; + + FUNC_ENTER_NOAPI(H5HF_fheap_info, FAIL) + + /* + * Check arguments. + */ + HDassert(f); + HDassert(H5F_addr_defined(fh_addr)); + HDassert(heap_size); + HDassert(huge_bt_size); + + + /* Lock the heap header into memory */ + if(NULL == (hdr = H5AC_protect(f, dxpl_id, H5AC_FHEAP_HDR, fh_addr, NULL, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to load fractal heap header") + + *heap_size += hdr->heap_size; + *heap_size += hdr->man_alloc_size; + *heap_size += hdr->huge_size; + + if(H5F_addr_defined(hdr->man_dtable.table_addr)) { + if (hdr->man_dtable.curr_root_rows != 0) { + reterr = H5HF_indirect_info(f, dxpl_id, hdr, hdr->man_dtable.table_addr, hdr->man_dtable.curr_root_rows, heap_size); + if (reterr < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "unable to get fractal heap storage info for indirect block") + } + } + + /* get B-tree storage for huge objects in fractal heap */ + if (H5F_addr_defined(hdr->huge_bt2_addr)) { + if (hdr->huge_ids_direct) { + if (hdr->filter_len > 0) { + if (H5B2_info_iterate(f, dxpl_id, H5HF_BT2_FILT_DIR, hdr->huge_bt2_addr, huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } else { + if (H5B2_info_iterate(f, dxpl_id, H5HF_BT2_DIR, hdr->huge_bt2_addr, huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + } else { + if (hdr->filter_len > 0) { + if (H5B2_info_iterate(f, dxpl_id, H5HF_BT2_FILT_INDIR, hdr->huge_bt2_addr, huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } else { + if (H5B2_info_iterate(f, dxpl_id, H5HF_BT2_INDIR, hdr->huge_bt2_addr, huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + } + } + if (H5F_addr_defined(hdr->fs_addr)) { + meta_size = 0; + if (H5FS_meta_info(f, dxpl_id, hdr->fs_addr, &meta_size) < 0) + HGOTO_ERROR(H5E_FSPACE, H5E_CANTGET, FAIL, "can't retrieve FS meta storage info") + *heap_size += meta_size; + } + +done: + /* Unprotect the header, if an error occurred */ + if(hdr && H5AC_unprotect(f, dxpl_id, H5AC_FHEAP_HDR, fh_addr, hdr, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_HEAP, H5E_CANTUNPROTECT, FAIL, "unable to release fractal heap header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HF_fheap_info() */ diff --git a/src/H5HFiblock.c b/src/H5HFiblock.c index 7afa7c4..15c998c 100644 --- a/src/H5HFiblock.c +++ b/src/H5HFiblock.c @@ -1575,4 +1575,65 @@ HDfprintf(stderr, "%s: iblock_addr = %a, iblock_nrows = %u\n", FUNC, iblock_addr done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HF_man_iblock_delete() */ + +/*------------------------------------------------------------------------- + * + * Function: H5HF_indirect_info + * + * Purpose: Gather storage used for the indirect block in fractal heap + * + * Return: non-negative on success, negative on error + * + * Programmer: Vailin Choi + * July 12 2007 + *------------------------------------------------------------------------- + */ +herr_t +H5HF_indirect_info(H5F_t *f, hid_t dxpl_id, H5HF_hdr_t *hdr, haddr_t iblock_addr, unsigned nrows, hsize_t *heap_size) +{ + hbool_t did_protect; + H5HF_indirect_t *iblock; /* Pointer to indirect block */ + size_t u, v; + int reterr; + int ret_value=SUCCEED; + FUNC_ENTER_NOAPI(H5HF_indirect_info, FAIL) + /* + * Check arguments. + */ + HDassert(f); + HDassert(hdr); + HDassert(H5F_addr_defined(iblock_addr)); + HDassert(heap_size); + + if (NULL == (iblock = H5HF_man_iblock_protect(hdr, dxpl_id, iblock_addr, nrows, NULL, 0, FALSE, H5AC_READ, &did_protect))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to load fractal heap indirect block") + + *heap_size += iblock->size; + + /* Indirect entries in this indirect block */ + if(iblock->nrows > hdr->man_dtable.max_direct_rows) { + unsigned first_row_bits; /* Number of bits used bit addresses in first row */ + unsigned num_indirect_rows; /* Number of rows of blocks in each indirect block */ + + first_row_bits = H5V_log2_of2((uint32_t)hdr->man_dtable.cparam.start_block_size) + + H5V_log2_of2(hdr->man_dtable.cparam.width); + for(u = hdr->man_dtable.max_direct_rows; u < iblock->nrows; u++) { + num_indirect_rows = (H5V_log2_gen(hdr->man_dtable.row_block_size[u]) - first_row_bits) + 1; + for(v = 0; v < hdr->man_dtable.cparam.width; v++) { + size_t off = (u * hdr->man_dtable.cparam.width) + v; + + if (H5F_addr_defined(iblock->ents[off].addr)) { + reterr = H5HF_indirect_info(f, dxpl_id, hdr, iblock->ents[off].addr, num_indirect_rows, heap_size); + if (reterr < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to get fractal heap storage info for indirect block") + } /* end if */ + } /* end for v */ + } /* end for u */ + } /* end if */ + +done: + if(H5HF_man_iblock_unprotect(iblock, dxpl_id, H5AC__NO_FLAGS_SET, did_protect) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTUNPROTECT, FAIL, "unable to release fractal heap indirect block") + FUNC_LEAVE_NOAPI(ret_value) +} diff --git a/src/H5HFpkg.h b/src/H5HFpkg.h index 6dc1498..bd3c4ec 100644 --- a/src/H5HFpkg.h +++ b/src/H5HFpkg.h @@ -722,6 +722,8 @@ H5_DLL herr_t H5HF_sect_indirect_add(H5HF_hdr_t *hdr, hid_t dxpl_id, /* Internal operator callbacks */ H5_DLL herr_t H5HF_op_read(const void *obj, size_t obj_len, void *op_data); H5_DLL herr_t H5HF_op_write(const void *obj, size_t obj_len, void *op_data); +H5_DLL herr_t H5HF_indirect_info(H5F_t *f, hid_t dxpl_id, H5HF_hdr_t *hdr, + haddr_t iblock_addr, unsigned nrows, hsize_t *heap_size/*out*/); /* Testing routines */ #ifdef H5HF_TESTING diff --git a/src/H5HFprivate.h b/src/H5HFprivate.h index 7d7c3f6..a8855de 100644 --- a/src/H5HFprivate.h +++ b/src/H5HFprivate.h @@ -126,6 +126,11 @@ H5_DLL herr_t H5HF_remove(H5HF_t *fh, hid_t dxpl_id, const void *id); H5_DLL herr_t H5HF_close(H5HF_t *fh, hid_t dxpl_id); H5_DLL herr_t H5HF_delete(H5F_t *f, hid_t dxpl_id, haddr_t fh_addr); +/* Routines to gather storage info for fractal heap */ +H5_DLL herr_t H5HF_fheap_info(H5F_t *f, hid_t dxpl_id, haddr_t fh_addr, + hsize_t *heap_size/*out*/, hsize_t *huge_bt_size/*out*/); + + /* Statistics routines */ H5_DLL herr_t H5HF_stat_info(const H5HF_t *fh, H5HF_stat_t *stats); @@ -1346,4 +1346,45 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5HL_get_size() */ + +/*------------------------------------------------------------------------- + * Function: H5HL_heapsize + * + * Purpose: Compute the size in bytes of the specified instance of + * H5HL_t via H5HL_compute_size() + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 19 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5HL_heapsize(H5F_t *f, hid_t dxpl_id, haddr_t addr, hsize_t *heap_size) +{ + H5HL_t *heap = NULL; /* Heap to query */ + herr_t ret_value = SUCCEED; /* Return value */ + size_t local_heap_size = 0; + + FUNC_ENTER_NOAPI(H5HL_heapsize, FAIL) + /* check arguments */ + HDassert(f); + HDassert(H5F_addr_defined(addr)); + HDassert(heap_size); + + /* Get heap pointer */ + if(NULL == (heap = H5AC_protect(f, dxpl_id, H5AC_LHEAP, addr, NULL, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to load heap") + + if (H5HL_compute_size(f, heap, &local_heap_size)<0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTLOAD, FAIL, "unable to compute size of local heap") + *heap_size = (hsize_t)local_heap_size; + +done: + if(heap && H5AC_unprotect(f, dxpl_id, H5AC_LHEAP, addr, heap, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_HEAP, H5E_PROTECT, FAIL, "unable to release local heap") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5HL_get_size() */ diff --git a/src/H5HLprivate.h b/src/H5HLprivate.h index 4684d61..d8a4b49 100644 --- a/src/H5HLprivate.h +++ b/src/H5HLprivate.h @@ -73,6 +73,7 @@ H5_DLL size_t H5HL_insert(H5F_t *f, hid_t dxpl_id, H5HL_t *heap, size_t size, const void *buf); H5_DLL herr_t H5HL_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr); H5_DLL herr_t H5HL_get_size(H5F_t *f, hid_t dxpl_id, haddr_t addr, size_t *size); +H5_DLL herr_t H5HL_heapsize(H5F_t *f, hid_t dxpl_id, haddr_t addr, hsize_t *heap_size); /* Debugging functions */ H5_DLL herr_t H5HL_debug(H5F_t *f, hid_t dxpl_id, haddr_t addr, FILE * stream, int indent, @@ -488,7 +488,6 @@ H5Oget_info(hid_t loc_id, const char *name, H5O_info_t *oinfo, hid_t lapl_id) /* Retrieve the object's information */ if(H5G_loc_info(&loc, name, oinfo/*out*/, lapl_id, H5AC_ind_dxpl_id) < 0) HGOTO_ERROR(H5E_SYM, H5E_NOTFOUND, FAIL, "object not found") - done: FUNC_LEAVE_API(ret_value) } /* end H5Oget_info() */ @@ -1825,6 +1824,13 @@ done: * Programmer: Quincey Koziol * November 21 2006 * + * Modifications: + * 12 July 2007 by Vailin Choi + * Modified to retrieve the following information: + * 1. GROUP: storage for btree and heap (1.6 and 1.8) + * 2. DATASET: btree storage for chunked dataset + * 3. ATTRIBUTE: storage for 1.8 btree and fractal heap + * *------------------------------------------------------------------------- */ herr_t @@ -1857,6 +1863,16 @@ H5O_get_info(H5O_loc_t *oloc, H5O_info_t *oinfo, hid_t dxpl_id) if(H5O_obj_type_real(oh, &oinfo->type) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINIT, FAIL, "unable to determine object type") + /* Retrieve btree and heap storage info */ + HDmemset(&oinfo->meta_size.obj, 0, sizeof(H5_ih_info_t)); + if (oinfo->type == H5O_TYPE_GROUP) { + if (H5O_group_bh_info(oloc, oh, dxpl_id, &(oinfo->meta_size.obj)/*out*/) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't retrieve 1.8 group btree & heap info") + } else if (oinfo->type == H5O_TYPE_DATASET) { + if (H5O_dset_bh_info(oloc, oh, dxpl_id, &(oinfo->meta_size.obj)/*out*/) < 0) + HGOTO_ERROR(H5E_OHDR, H5E_CANTGET, FAIL, "can't retrieve chunked dataset btree info") + } + /* Set the object's reference count */ oinfo->rc = oh->nlink; @@ -1927,6 +1943,9 @@ H5O_get_info(H5O_loc_t *oloc, H5O_info_t *oinfo, hid_t dxpl_id) /* Retrieve # of attributes */ oinfo->num_attrs = H5O_attr_count_real(oloc->file, dxpl_id, oh); + HDmemset(&oinfo->meta_size.attr, 0, sizeof(H5_ih_info_t)); + if ((oinfo->num_attrs > 0) && (H5O_attr_bh_info(oloc, oh, dxpl_id, &oinfo->meta_size.attr/*out*/) < 0)) + HGOTO_ERROR(H5E_ATTR, H5E_CANTGET, FAIL, "can't retrieve attribute btree & heap info") /* Iterate over all the chunks, adding any gaps to the free space */ oinfo->hdr.space.total = 0; @@ -2121,3 +2140,47 @@ H5O_get_oh_addr(const H5O_t *oh) FUNC_LEAVE_NOAPI(oh->chunk[0].addr) } /* end H5O_get_oh_addr() */ +/*------------------------------------------------------------------------- + * Function: H5O_super_ext_size + * + * Purpose: Collect size of the superblock extension + * + * Return: Success: non-negative + * Failure: negative + * + * Programmer: Vailin Choi + * July 12, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_super_ext_size(H5F_t *f, hsize_t *ext_size, hid_t dxpl_id) +{ + H5O_t *oh = NULL; /* Object header */ + H5O_chunk_t *curr_chunk; /* Pointer to current message being operated on */ + unsigned u; /* Local index variable */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5O_super_ext_size, FAIL) + + /* Check args */ + HDassert(f); + HDassert(ext_size); + HDassert(H5F_addr_defined(f->shared->extension_addr)); + + /* Get the object header */ + if(NULL == (oh = H5AC_protect(f, dxpl_id, H5AC_OHDR, f->shared->extension_addr, NULL, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_OHDR, H5E_CANTLOAD, FAIL, "unable to load object header") + + *ext_size = 0; + for(u = 0, curr_chunk = &oh->chunk[0]; u < oh->nchunks; u++, curr_chunk++) { + /* Accumulate the size of the header on disk */ + *ext_size += curr_chunk->size; + } + +done: + if(oh && H5AC_unprotect(f, dxpl_id, H5AC_OHDR, f->shared->extension_addr, oh, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_OHDR, H5E_PROTECT, FAIL, "unable to release object header") + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_super_ext_size() */ diff --git a/src/H5Oattribute.c b/src/H5Oattribute.c index ddf5a35..f0bafc9 100644 --- a/src/H5Oattribute.c +++ b/src/H5Oattribute.c @@ -1687,4 +1687,63 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5O_attr_exists */ + +/*------------------------------------------------------------------------- + * Function: H5O_attr_bh_info + * + * Purpose: For 1.8 attribute, returns storage amount for btree and fractal heap + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 19, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5O_attr_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info) +{ + H5O_ainfo_t ainfo; /* Attribute information for object */ + herr_t ret_value=SUCCEED; /* Return value */ + hsize_t huge_bt_size=0; + FUNC_ENTER_NOAPI(H5O_attr_bh_info, FAIL) + + HDassert(oloc); + HDassert(oh); + HDassert(bh_info); + + /* Check for attribute info stored */ + ainfo.nattrs = 0; + ainfo.fheap_addr = HADDR_UNDEF; + ainfo.corder_bt2_addr = HADDR_UNDEF; + ainfo.name_bt2_addr = HADDR_UNDEF; + if(oh->version > H5O_VERSION_1 && NULL == H5A_get_ainfo(oloc->file, dxpl_id, oh, &ainfo)) + /* Clear error stack from not finding attribute info */ + H5E_clear_stack(NULL); + + if(oh->version > H5O_VERSION_1) { + if (H5F_addr_defined(ainfo.corder_bt2_addr)) { + if (H5B2_info_iterate(oloc->file, dxpl_id, H5A_BT2_CORDER, + ainfo.corder_bt2_addr, &(bh_info->index_size)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + + if (H5F_addr_defined(ainfo.name_bt2_addr)) { + if (H5B2_info_iterate(oloc->file, dxpl_id, H5A_BT2_NAME, + ainfo.name_bt2_addr, &(bh_info->index_size)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + + huge_bt_size = 0; + if (H5F_addr_defined(ainfo.fheap_addr)) { + HDassert(H5O_msg_count_real(oh, H5O_MSG_ATTR) == 0); + if (H5HF_fheap_info(oloc->file, dxpl_id, ainfo.fheap_addr, &(bh_info->heap_size), &huge_bt_size) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + bh_info->index_size += huge_bt_size; + } + } + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* H5O_attr_bh_info() */ diff --git a/src/H5Opkg.h b/src/H5Opkg.h index fb17c0c..afde15a 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -479,6 +479,14 @@ H5_DLL void *H5O_msg_copy_file(const H5O_msg_class_t *type, H5F_t *file_src, H5_DLL herr_t H5O_msg_iterate_real(H5F_t *f, H5O_t *oh, const H5O_msg_class_t *type, const H5O_mesg_operator_t *op, void *op_data, hid_t dxpl_id); +/* collect storage info for btree and heap */ +H5_DLL herr_t H5O_group_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info); +H5_DLL herr_t H5O_dset_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info); +H5_DLL herr_t H5O_attr_bh_info(H5O_loc_t *oloc, H5O_t *oh, hid_t dxpl_id, H5_ih_info_t *bh_info); + + + + /* Object header allocation routines */ H5_DLL herr_t H5O_alloc_msgs(H5O_t *oh, size_t min_alloc); H5_DLL unsigned H5O_alloc(H5F_t *f, hid_t dxpl_id, H5O_t *oh, diff --git a/src/H5Oprivate.h b/src/H5Oprivate.h index 2376113..584d9d5 100644 --- a/src/H5Oprivate.h +++ b/src/H5Oprivate.h @@ -532,6 +532,9 @@ H5_DLL herr_t H5O_bogus_oh(H5F_t *f, hid_t dxpl_id, struct H5O_t *oh, unsigned m #endif /* H5O_ENABLE_BOGUS */ H5_DLL herr_t H5O_delete(H5F_t *f, hid_t dxpl_id, haddr_t addr); H5_DLL herr_t H5O_get_info(H5O_loc_t *oloc, H5O_info_t *oinfo, hid_t dxpl_id); + +H5_DLL herr_t H5O_super_ext_size(H5F_t *f, hsize_t *ext_size, hid_t dxpl_id); + H5_DLL herr_t H5O_obj_type(const H5O_loc_t *loc, H5O_type_t *obj_type, hid_t dxpl_id); H5_DLL herr_t H5O_get_create_plist(const H5O_loc_t *loc, hid_t dxpl_id, struct H5P_genplist_t *oc_plist); H5_DLL hid_t H5O_open_name(H5G_loc_t *loc, const char *name, hid_t lapl_id); @@ -618,5 +621,6 @@ H5_DLL herr_t H5O_set_shared(H5O_shared_t *dst, const H5O_shared_t *src); /* Attribute operators */ H5_DLL hsize_t H5O_attr_count_real(H5F_t *f, hid_t dxpl_id, H5O_t *oh); + #endif /* _H5Oprivate_H */ diff --git a/src/H5Opublic.h b/src/H5Opublic.h index b3d0c36..c5f1b21 100644 --- a/src/H5Opublic.h +++ b/src/H5Opublic.h @@ -120,8 +120,12 @@ typedef struct H5O_info_t { uint64_t shared; /* Flags to indicate message type is shared in header */ } mesg; } hdr; - hsize_t meta_size; /* Size of additional metadata for an object */ - /* (B-tree & heap for groups, B-tree for chunked dataset, etc.) */ + /* btree and heap storage info for obj & attributes */ + /* (B-tree & heap for groups, B-tree for chunked dataset, 1.8 B-tree & heap for attributes) */ + struct { + H5_ih_info_t obj; + H5_ih_info_t attr; + } meta_size; } H5O_info_t; /* Typedef for message creation indexes */ @@ -2474,3 +2474,68 @@ done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5SM_list_debug() */ +/*------------------------------------------------------------------------- + * Function: H5SM_ih_info + * + * Purpose: Loop through the master SOHM table (if there is one) to: + * 1. collect storage used for header + * 1. collect storage used for B-tree and List + * (include btree storage used by huge objects in fractal heap) + * 2. collect fractal heap storage + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Vailin Choi + * June 19, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5SM_ih_info(H5F_t *f, H5F_info_t *finfo, hid_t dxpl_id) +{ + H5SM_master_table_t *table=NULL; /* SOHM master table */ + herr_t ret_value=SUCCEED; /* Return value */ + unsigned x; + hsize_t huge_bt_size=0; + + + FUNC_ENTER_NOAPI(H5SM_ih_info, FAIL) + + /* Sanity check */ + HDassert(f); + HDassert(H5F_addr_defined(f->shared->sohm_addr)); + HDassert(finfo); + + /* Look up the master SOHM table */ + if(NULL == (table = (H5SM_master_table_t *)H5AC_protect(f, dxpl_id, H5AC_SOHM_TABLE, f->shared->sohm_addr, NULL, NULL, H5AC_READ))) + HGOTO_ERROR(H5E_CACHE, H5E_CANTPROTECT, FAIL, "unable to load SOHM master table") + + finfo->sohm.hdr_size = (hsize_t) H5SM_TABLE_SIZE(f) + + (hsize_t)(table->num_indexes * H5SM_INDEX_HEADER_SIZE(f)); + + for(x = 0; x < table->num_indexes; x++) { + if (table->indexes[x].index_type==H5SM_BTREE) { + if (H5F_addr_defined(table->indexes[x].index_addr)) { + if (H5B2_info_iterate(f, dxpl_id, H5SM_INDEX, + table->indexes[x].index_addr, &(finfo->sohm.msgs_info.index_size)) < 0) + HGOTO_ERROR(H5E_BTREE, H5E_CANTGET, FAIL, "can't retrieve B-tree storage info") + } + } + if (table->indexes[x].index_type==H5SM_LIST) + finfo->sohm.msgs_info.index_size += H5SM_LIST_SIZE(f, table->indexes[x].list_max); + + huge_bt_size = 0; + if (H5F_addr_defined(table->indexes[x].heap_addr)) { + if (H5HF_fheap_info(f, dxpl_id, table->indexes[x].heap_addr, &(finfo->sohm.msgs_info.heap_size), &huge_bt_size) < 0) + HGOTO_ERROR(H5E_HEAP, H5E_CANTGET, FAIL, "can't retrieve fractal heap storage info") + } + finfo->sohm.msgs_info.index_size += huge_bt_size; + } /* end for */ + + /* Release resources */ + if(table && H5AC_unprotect(f, dxpl_id, H5AC_SOHM_TABLE, f->shared->sohm_addr, table, H5AC__NO_FLAGS_SET) < 0) + HDONE_ERROR(H5E_CACHE, H5E_CANTRELEASE, FAIL, "unable to close SOHM master table") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5SM_ih_info() */ diff --git a/src/H5SMprivate.h b/src/H5SMprivate.h index 0f3d3e0..763c117 100755 --- a/src/H5SMprivate.h +++ b/src/H5SMprivate.h @@ -56,6 +56,8 @@ H5_DLL herr_t H5SM_reconstitute(H5O_shared_t *sh_mesg, H5F_t *f, unsigned msg_type_id, H5O_fheap_id_t heap_id); H5_DLL herr_t H5SM_get_refcount(H5F_t *f, hid_t dxpl_id, unsigned type_id, const H5O_shared_t *sh_mesg, hsize_t *ref_count); +H5_DLL herr_t H5SM_ih_info(H5F_t *f, H5F_info_t *bh_info, hid_t dxpl_id); + /* Debugging routines */ H5_DLL herr_t H5SM_table_debug(H5F_t *f, hid_t dxpl_id, haddr_t table_addr, diff --git a/src/H5public.h b/src/H5public.h index 09b2d5c..85a9d8b 100644 --- a/src/H5public.h +++ b/src/H5public.h @@ -300,6 +300,14 @@ typedef enum H5_index_t { H5_INDEX_N /* Number of indices defined */ } H5_index_t; +/* + * Storage info struct used by H5O_info_t and H5F_info_t + */ +typedef struct H5_ih_info_t { + hsize_t index_size; /* btree and/or list */ + hsize_t heap_size; +} H5_ih_info_t; + /* Functions in H5.c */ H5_DLL herr_t H5open(void); H5_DLL herr_t H5close(void); diff --git a/tools/misc/Makefile.am b/tools/misc/Makefile.am index 423a28b..fc097cd 100644 --- a/tools/misc/Makefile.am +++ b/tools/misc/Makefile.am @@ -24,7 +24,7 @@ include $(top_srcdir)/config/commence.am INCLUDES=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib #test script and program -TEST_PROG=h5repart_gentest +TEST_PROG=h5repart_gentest h5stat_gentest TEST_SCRIPT=testh5repart.sh testh5stat.sh $(srcdir)/testh5mkgrp.sh check_PROGRAMS=$(TEST_PROG) repart_test diff --git a/tools/misc/Makefile.in b/tools/misc/Makefile.in index 605c3c9..c8022c7 100644 --- a/tools/misc/Makefile.in +++ b/tools/misc/Makefile.in @@ -68,7 +68,7 @@ CONFIG_HEADER = $(top_builddir)/src/H5config.h CONFIG_CLEAN_FILES = h5cc testh5repart.sh testh5stat.sh am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)" binPROGRAMS_INSTALL = $(INSTALL_PROGRAM) -am__EXEEXT_1 = h5repart_gentest$(EXEEXT) +am__EXEEXT_1 = h5repart_gentest$(EXEEXT) h5stat_gentest$(EXEEXT) PROGRAMS = $(bin_PROGRAMS) h5debug_SOURCES = h5debug.c h5debug_OBJECTS = h5debug.$(OBJEXT) @@ -90,6 +90,10 @@ h5stat_SOURCES = h5stat.c h5stat_OBJECTS = h5stat.$(OBJEXT) h5stat_LDADD = $(LDADD) h5stat_DEPENDENCIES = $(LIBH5TOOLS) $(LIBHDF5) +h5stat_gentest_SOURCES = h5stat_gentest.c +h5stat_gentest_OBJECTS = h5stat_gentest.$(OBJEXT) +h5stat_gentest_LDADD = $(LDADD) +h5stat_gentest_DEPENDENCIES = $(LIBH5TOOLS) $(LIBHDF5) repart_test_SOURCES = repart_test.c repart_test_OBJECTS = repart_test.$(OBJEXT) repart_test_LDADD = $(LDADD) @@ -109,9 +113,9 @@ LINK = $(LIBTOOL) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) \ --mode=link $(CCLD) $(AM_CFLAGS) $(CFLAGS) $(AM_LDFLAGS) \ $(LDFLAGS) -o $@ SOURCES = h5debug.c h5mkgrp.c h5repart.c h5repart_gentest.c h5stat.c \ - repart_test.c + h5stat_gentest.c repart_test.c DIST_SOURCES = h5debug.c h5mkgrp.c h5repart.c h5repart_gentest.c \ - h5stat.c repart_test.c + h5stat.c h5stat_gentest.c repart_test.c ETAGS = etags CTAGS = ctags DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) @@ -352,7 +356,7 @@ CHECK_CLEANFILES = *.chkexe *.chklog *.clog *.h5 \ INCLUDES = -I$(top_srcdir)/src -I$(top_srcdir)/tools/lib #test script and program -TEST_PROG = h5repart_gentest +TEST_PROG = h5repart_gentest h5stat_gentest TEST_SCRIPT = testh5repart.sh testh5stat.sh $(srcdir)/testh5mkgrp.sh check_SCRIPTS = $(TEST_SCRIPT) SCRIPT_DEPEND = h5repart$(EXEEXT) h5stat$(EXEEXT) h5mkgrp$(EXEEXT) @@ -476,6 +480,9 @@ h5repart_gentest$(EXEEXT): $(h5repart_gentest_OBJECTS) $(h5repart_gentest_DEPEND h5stat$(EXEEXT): $(h5stat_OBJECTS) $(h5stat_DEPENDENCIES) @rm -f h5stat$(EXEEXT) $(LINK) $(h5stat_OBJECTS) $(h5stat_LDADD) $(LIBS) +h5stat_gentest$(EXEEXT): $(h5stat_gentest_OBJECTS) $(h5stat_gentest_DEPENDENCIES) + @rm -f h5stat_gentest$(EXEEXT) + $(LINK) $(h5stat_gentest_OBJECTS) $(h5stat_gentest_LDADD) $(LIBS) repart_test$(EXEEXT): $(repart_test_OBJECTS) $(repart_test_DEPENDENCIES) @rm -f repart_test$(EXEEXT) $(LINK) $(repart_test_OBJECTS) $(repart_test_LDADD) $(LIBS) @@ -510,6 +517,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repart.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5repart_gentest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5stat.Po@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/h5stat_gentest.Po@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/repart_test.Po@am__quote@ .c.o: diff --git a/tools/misc/h5stat.c b/tools/misc/h5stat.c index 93871bd..27566b9 100644 --- a/tools/misc/h5stat.c +++ b/tools/misc/h5stat.c @@ -24,6 +24,7 @@ /* Parameters to control statistics gathered */ #define SIZE_SMALL_GROUPS 10 +#define SIZE_SMALL_ATTRS 10 #define SIZE_SMALL_DSETS 10 #define H5_NFILTERS_IMPL 8 /* Number of currently implemented filters + one to @@ -63,6 +64,11 @@ typedef struct iter_t { unsigned long *group_bins; /* Pointer to array of bins for group counts */ ohdr_info_t group_ohdr_info; /* Object header information for groups */ + hsize_t max_attrs; /* Maximum attributes from a group */ + unsigned long num_small_attrs[SIZE_SMALL_ATTRS]; /* Size of small attributes tracked */ + unsigned attr_nbins; + unsigned long *attr_bins; + unsigned long max_dset_rank; /* Maximum rank of dataset */ unsigned long dset_rank_count[H5S_MAX_RANK]; /* Number of datasets of each rank */ hsize_t max_dset_dims; /* Maximum dimension size of dataset */ @@ -75,6 +81,15 @@ typedef struct iter_t { unsigned long *dset_dim_bins; /* Pointer to array of bins for dataset dimensions */ ohdr_info_t dset_ohdr_info; /* Object header information for datasets */ hsize_t dset_storage_size; /* Size of raw data for datasets */ + hsize_t groups_btree_storage_size; /* btree size for group */ + hsize_t groups_heap_storage_size; /* heap size for group */ + hsize_t attrs_btree_storage_size; /* btree size for attributes (1.8) */ + hsize_t attrs_heap_storage_size; /* fractal heap size for attributes (1.8) */ + hsize_t SM_hdr_storage_size; /* header size for SOHM table (1.8) */ + hsize_t SM_index_storage_size; /* index (btree & list) size for SOHM table (1.8) */ + hsize_t SM_heap_storage_size; /* fractal heap size for SOHM table (1.8) */ + hsize_t super_ext_size; /* superblock extension size */ + hsize_t datasets_btree_storage_size; /* btree size for chunked dataset */ unsigned long nexternal; /* Number of external files for a dataset */ int local; /* Flag to indicate iteration over the object*/ } iter_t; @@ -104,6 +119,7 @@ static int display_dtype_metadata = FALSE; static int display_dtype = FALSE; */ static int display_object = FALSE; +static int display_attr = FALSE; /* a structure for handling the order command-line parameters come in */ struct handler_t { @@ -113,21 +129,21 @@ struct handler_t { }; -static const char *s_opts ="FfhGgDdTO:V"; +static const char *s_opts ="AFfhGgDdTO:V"; static struct long_options l_opts[] = { {"help", no_arg, 'h'}, {"hel", no_arg, 'h'}, {"file", no_arg, 'f'}, {"fil", no_arg, 'f'}, {"fi", no_arg, 'f'}, - {"filemetadata", no_arg, 'F'}, - {"filemetadat", no_arg, 'F'}, - {"filemetada", no_arg, 'F'}, - {"filemetad", no_arg, 'F'}, - {"filemeta", no_arg, 'F'}, - {"filemet", no_arg, 'F'}, - {"fileme", no_arg, 'F'}, - {"filem", no_arg, 'F'}, + {"FILEmetadata", no_arg, 'F'}, + {"FILEmetadat", no_arg, 'F'}, + {"FILEmetada", no_arg, 'F'}, + {"FILEmetad", no_arg, 'F'}, + {"FILEmeta", no_arg, 'F'}, + {"FILEmet", no_arg, 'F'}, + {"FILEme", no_arg, 'F'}, + {"FILEm", no_arg, 'F'}, {"group", no_arg, 'g'}, {"grou", no_arg, 'g'}, {"gro", no_arg, 'g'}, @@ -172,6 +188,15 @@ static struct long_options l_opts[] = { { "vers", no_arg, 'V' }, { "ver", no_arg, 'V' }, { "ve", no_arg, 'V' }, + { "attribute", no_arg, 'A' }, + { "attribut", no_arg, 'A' }, + { "attribu", no_arg, 'A' }, + { "attrib", no_arg, 'A' }, + { "attri", no_arg, 'A' }, + { "attr", no_arg, 'A' }, + { "att", no_arg, 'A' }, + { "at", no_arg, 'A' }, + { "a", no_arg, 'A' }, { NULL, 0, '\0' } }; @@ -205,6 +230,7 @@ static void usage(const char *prog) fprintf(stdout, " -d, --dset Print dataset information\n"); fprintf(stdout, " -D, --dsetmetadata Print dataset metadata\n"); fprintf(stdout, " -T, --dtypemetadata Print datatype metadata\n"); + fprintf(stdout, " -A, --attribute Print attribute information\n"); fprintf(stdout, "\n"); } @@ -369,31 +395,38 @@ fix_name(const char *path, const char *base) * Failure: -1 * * Programmer: Quincey Koziol - * Tuesday, August 16, 2005 + * Tuesday, August 16, 2005 * * Modifications: Refactored code from the walk_function * EIP, Wednesday, August 16, 2006 * + * Vailin Choi 12 July 2007 + * 1. Gathered storage info for btree and heap + * (groups and attributes) + * 2. Gathered info for attributes + * + * *------------------------------------------------------------------------- */ static herr_t group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * _sb, H5G_iterate_t _walk, iter_t *_iter) - - { - hid_t gid; /* Group ID */ - const char *last_container; - hsize_t num_objs; - unsigned bin; /* "bin" the number of objects falls in */ - iter_t *iter = (iter_t*)_iter; - H5G_stat_t *sb = _sb; - H5G_iterate_t walk = _walk; - herr_t ret; + hid_t gid; /* Group ID */ + const char *last_container; + hsize_t num_objs; + unsigned bin; /* "bin" the number of objects falls in */ + iter_t *iter = (iter_t*)_iter; + H5G_stat_t *sb = _sb; + H5G_iterate_t walk = _walk; + herr_t ret; + hsize_t num_attrs=0; + unsigned attr_bin; + H5O_info_t oinfo; /* Gather statistics about this type of object */ iter->uniq_groups++; if(iter->curr_depth > iter->max_depth) - iter->max_depth = iter->curr_depth; + iter->max_depth = iter->curr_depth; /* Get object header information */ iter->group_ohdr_info.total_size += sb->ohdr.size; @@ -403,6 +436,7 @@ group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * assert(gid > 0); H5Gget_num_objs(gid, &num_objs); + if(num_objs < SIZE_SMALL_GROUPS) (iter->num_small_groups[num_objs])++; if(num_objs > iter->max_fanout) @@ -411,11 +445,11 @@ group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * /* Add group count to proper bin */ bin = ceil_log10((unsigned long)num_objs); if((bin + 1) > iter->group_nbins) { - /* Allocate more storage for info about dataset's datatype */ + /* Allocate more storage for info about dataset's datatype */ iter->group_bins = realloc(iter->group_bins, (bin + 1) * sizeof(unsigned long)); assert(iter->group_bins); - /* Initialize counts for intermediate bins */ + /* Initialize counts for intermediate bins */ while(iter->group_nbins < bin) iter->group_bins[iter->group_nbins++] = 0; iter->group_nbins++; @@ -427,17 +461,51 @@ group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * (iter->group_bins[bin])++; } /* end else */ - ret = H5Gclose(gid); - assert(ret >= 0); + ret = H5Oget_info(gid, ".", &oinfo, H5P_DEFAULT); + if (ret < 0) { + warn_msg(progname, "Unable to retrieve object info for \"%s\"\n", name); + } else { + num_attrs = oinfo.num_attrs; + iter->groups_btree_storage_size += oinfo.meta_size.obj.index_size; + iter->groups_heap_storage_size += oinfo.meta_size.obj.heap_size; + iter->attrs_btree_storage_size += oinfo.meta_size.attr.index_size; + iter->attrs_heap_storage_size += oinfo.meta_size.attr.heap_size; + } + + if(num_attrs < SIZE_SMALL_ATTRS) + (iter->num_small_attrs[num_attrs])++; + if(num_attrs > iter->max_attrs) + iter->max_attrs = num_attrs; + + /* Add attribute count to proper bin */ + attr_bin = ceil_log10((unsigned long)num_attrs); + if((attr_bin + 1) > iter->attr_nbins) { + iter->attr_bins = realloc(iter->attr_bins, (attr_bin + 1) * sizeof(unsigned long)); + assert(iter->attr_bins); - last_container = iter->container; - iter->container = fullname; - iter->curr_depth++; + /* Initialize counts for intermediate bins */ + while(iter->attr_nbins < attr_bin) + iter->attr_bins[iter->attr_nbins++] = 0; + iter->attr_nbins++; - H5Giterate(group, name, NULL, walk, iter); + /* Initialize count for new bin */ + iter->attr_bins[attr_bin] = 1; + } /* end if */ + else { + (iter->attr_bins[attr_bin])++; + } /* end else */ + + ret = H5Gclose(gid); + assert(ret >= 0); + + last_container = iter->container; + iter->container = fullname; + iter->curr_depth++; - iter->container = last_container; - iter->curr_depth--; + H5Giterate(group, name, NULL, walk, iter); + + iter->container = last_container; + iter->curr_depth--; return 0; } @@ -457,32 +525,38 @@ group_stats (hid_t group, const char *name, const char * fullname, H5G_stat_t * * Modifications: Refactored code from the walk_function * EIP, Wednesday, August 16, 2006 * + * Vailin Choi 12 July 2007 + * 1. Gathered storage info for btree and heap + * (chunked datasets and attributes) + * 2. Gathered info for attributes + * *------------------------------------------------------------------------- */ static herr_t dataset_stats (hid_t group, const char *name, H5G_stat_t * _sb, iter_t *_iter) - - { - unsigned bin; /* "bin" the number of objects falls in */ - iter_t *iter = (iter_t*)_iter; - H5G_stat_t *sb = _sb; - herr_t ret; - - hid_t did; /* Dataset ID */ - hid_t sid; /* Dataspace ID */ - hid_t tid; /* Datatype ID */ - hid_t dcpl; /* Dataset creation property list ID */ - hsize_t dims[H5S_MAX_RANK]; /* Dimensions of dataset */ - H5D_layout_t lout; /* Layout of dataset */ - unsigned type_found; /* Whether the dataset's datatype was */ - /* already found */ - int ndims; /* Number of dimensions of dataset */ - hsize_t storage; /* Size of dataset storage */ - unsigned u; /* Local index variable */ - int num_ext; /* Number of external files for a dataset */ - int nfltr; /* Number of filters for a dataset */ - H5Z_filter_t fltr; /* Filter identifier */ + unsigned bin; /* "bin" the number of objects falls in */ + iter_t *iter = (iter_t*)_iter; + H5G_stat_t *sb = _sb; + herr_t ret; + hid_t did; /* Dataset ID */ + hid_t sid; /* Dataspace ID */ + hid_t tid; /* Datatype ID */ + hid_t dcpl; /* Dataset creation property list ID */ + hsize_t dims[H5S_MAX_RANK];/* Dimensions of dataset */ + H5D_layout_t lout; /* Layout of dataset */ + unsigned type_found; /* Whether the dataset's datatype was */ + /* already found */ + int ndims; /* Number of dimensions of dataset */ + hsize_t storage; /* Size of dataset storage */ + unsigned u; /* Local index variable */ + int num_ext; /* Number of external files for a dataset */ + int nfltr; /* Number of filters for a dataset */ + H5Z_filter_t fltr; /* Filter identifier */ + + hsize_t num_attrs = 0; + unsigned attr_bin; + H5O_info_t oinfo; /* Gather statistics about this type of object */ iter->uniq_dsets++; @@ -494,6 +568,39 @@ dataset_stats (hid_t group, const char *name, H5G_stat_t * _sb, iter_t *_iter) did = H5Dopen(group, name); assert(did > 0); + ret = H5Oget_info(did, ".", &oinfo, H5P_DEFAULT); + if (ret < 0) { + warn_msg(progname, "Unable to retrieve object info for \"%s\"\n", name); + } else { + num_attrs = oinfo.num_attrs; + iter->datasets_btree_storage_size += oinfo.meta_size.obj.index_size; + iter->attrs_btree_storage_size += oinfo.meta_size.attr.index_size; + iter->attrs_heap_storage_size += oinfo.meta_size.attr.heap_size; + } + + if(num_attrs < SIZE_SMALL_ATTRS) + (iter->num_small_attrs[num_attrs])++; + if(num_attrs > iter->max_attrs) + iter->max_attrs = num_attrs; + + /* Add attribute count to proper bin */ + attr_bin = ceil_log10((unsigned long)num_attrs); + if((attr_bin + 1) > iter->attr_nbins) { + iter->attr_bins = realloc(iter->attr_bins, (attr_bin + 1) * sizeof(unsigned long)); + assert(iter->attr_bins); + + /* Initialize counts for intermediate bins */ + while(iter->attr_nbins < attr_bin) + iter->attr_bins[iter->attr_nbins++] = 0; + iter->attr_nbins++; + + /* Initialize count for new bin */ + iter->attr_bins[attr_bin] = 1; + } /* end if */ + else { + (iter->attr_bins[attr_bin])++; + } /* end else */ + /* Get storage info */ storage = H5Dget_storage_size(did); iter->dset_storage_size += storage; @@ -514,8 +621,6 @@ dataset_stats (hid_t group, const char *name, H5G_stat_t * _sb, iter_t *_iter) /* Only gather dim size statistics on 1-D datasets */ if(ndims == 1) { - - if(dims[0] > iter->max_dset_dims) iter->max_dset_dims = dims[0]; if(dims[0] < SIZE_SMALL_DSETS) (iter->small_dset_dims[dims[0]])++; @@ -633,11 +738,10 @@ dataset_stats (hid_t group, const char *name, H5G_stat_t * _sb, iter_t *_iter) * Purpose: Gather statistics about the file * * Return: Success: 0 - * - * Failure: -1 + * Failure: -1 * * Programmer: Quincey Koziol - * Tuesday, August 16, 2005 + * Tuesday, August 16, 2005 * * Modifications: * @@ -654,9 +758,6 @@ walk (hid_t group, const char *name, void *_iter) /* Get the full object name */ fullname = fix_name(iter->container, name); -/* -printf("walk: fullname = %s\n", fullname); -*/ /* Get object information */ ret = H5Gget_objinfo(group, name, FALSE, &sb); assert(ret >= 0); @@ -664,7 +765,7 @@ printf("walk: fullname = %s\n", fullname); /* If the object has already been printed then just show the object ID * and return. */ if ((s=sym_lookup(&sb))) { - printf("same as %s", s); + printf("%s same as %s\n", name, s); } else { sym_insert(&sb, fullname); @@ -717,7 +818,9 @@ printf("walk: fullname = %s\n", fullname); * Programmer: Elena Pourmal * Saturday, August 12, 2006 * - * Modifications: + * Modifications: + * Vailin Choi 12 July 2007 + * Added 'A' option to display attribute info * *------------------------------------------------------------------------- */ @@ -734,6 +837,10 @@ parse_command_line(int argc, const char *argv[]) /* parse command line options */ while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) { switch ((char)opt) { + case 'A': + display_all = FALSE; + display_attr = TRUE; + break; case 'F': display_all = FALSE; display_file_metadata = TRUE; @@ -807,6 +914,12 @@ parse_command_line(int argc, const char *argv[]) * Saturday, August 12, 2006 * * Modifications: + * Vailin Choi 12 July 2007 + * Initialized storage info for: + * 1. btree/heap storage for groups and attributes + * 2. btree storage for chunked dataset + * 3. hdr/btree/list/heap storage for SOHM table + * 4. superblock extension size * *------------------------------------------------------------------------- */ @@ -838,6 +951,13 @@ iter_init(iter_t * _iter) iter->group_ohdr_info.total_size = 0; iter->group_ohdr_info.free_size = 0; + /* initialize attributes' information for groups and datasets */ + iter->max_attrs = 0; + for(u = 0; u < SIZE_SMALL_ATTRS; u++) + iter->num_small_attrs[u] = 0; + iter->attr_nbins = 0; + iter->attr_bins = NULL; + /* Initilaize datasets' metadata information */ iter->max_dset_rank = 0; for(u = 0; u < H5S_MAX_RANK; u++) @@ -849,6 +969,7 @@ iter_init(iter_t * _iter) iter->dset_layouts[u] = 0; for(u = 0; u < H5_NFILTERS_IMPL; u++) iter->dset_comptype[u] = 0; + iter->dset_ntypes = 0; iter->dset_type_info = NULL; iter->dset_dim_nbins = 0; @@ -856,6 +977,18 @@ iter_init(iter_t * _iter) iter->dset_ohdr_info.total_size = 0; iter->dset_ohdr_info.free_size = 0; iter->dset_storage_size = 0; + + /* Initialize storage info */ + iter->groups_btree_storage_size = 0; + iter->groups_heap_storage_size = 0; + iter->attrs_btree_storage_size = 0; + iter->attrs_heap_storage_size = 0; + iter->SM_hdr_storage_size = 0; + iter->SM_index_storage_size = 0; + iter->SM_heap_storage_size = 0; + iter->super_ext_size = 0; + iter->datasets_btree_storage_size = 0; + iter->nexternal = 0; iter->local = 0; @@ -911,6 +1044,12 @@ print_file_info(iter_t * _iter) * Saturday, August 12, 2006 * * Modifications: + * Vailin Choi 12 July 2007 + * Print storage info for: + * 1. btree/heap storage for groups and attributes + * 2. btree storage for chunked dataset + * 3. hdr/btree/list/heap storage for SOHM table + * 4. superblock extension size * *------------------------------------------------------------------------- */ @@ -926,6 +1065,24 @@ print_file_metadata(iter_t * _iter) HDfprintf(stdout, "\tDatasets: %Hu/%Hu\n", iter->dset_ohdr_info.total_size, iter->dset_ohdr_info.free_size); + printf("Storage information:\n"); + HDfprintf(stdout, "\tGroups:\n"); + HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->groups_btree_storage_size); + HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->groups_heap_storage_size); + + HDfprintf(stdout, "\tAttributes:\n"); + HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->attrs_btree_storage_size); + HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->attrs_heap_storage_size); + + HDfprintf(stdout, "\tChunked datasets:\n"); + HDfprintf(stdout, "\t\tB-tree: %Hu\n", iter->datasets_btree_storage_size); + + HDfprintf(stdout, "\tShared Messages:\n"); + HDfprintf(stdout, "\t\tHeader: %Hu\n", iter->SM_hdr_storage_size); + HDfprintf(stdout, "\t\tB-tree/List: %Hu\n", iter->SM_index_storage_size); + HDfprintf(stdout, "\t\tHeap: %Hu\n", iter->SM_heap_storage_size); + + HDfprintf(stdout, "\tSuperblock extension: %Hu\n", iter->super_ext_size); return ret; } @@ -985,6 +1142,57 @@ print_group_info(iter_t * _iter) } /*------------------------------------------------------------------------- + * Function: print_attr_info + * + * Purpose: Prints information about attributes in the file + * + * Return: Success: 0 + * + * Failure: Never fails + * + * Programmer: Vailin Choi + * July 12, 2007 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static herr_t +print_attr_info(iter_t * _iter) +{ + iter_t *iter = (iter_t*)_iter; + herr_t ret =0; /* Generic return value */ + unsigned u; /* Local index variable */ + unsigned long power; /* Temporary "power" for bins */ + unsigned long total; /* Total count for various statistics */ + + printf("Small # of attributes:\n"); + total = 0; + for(u = 1; u < SIZE_SMALL_ATTRS; u++) { + if(iter->num_small_attrs[u] > 0) { + printf("\t# of objects with %u attributes: %lu\n", u, iter->num_small_attrs[u]); + total += iter->num_small_attrs[u]; + } /* end if */ + } /* end for */ + printf("\tTotal # of objects with small # of attributes: %lu\n", total); + + printf("Attribute bins:\n"); + total = 0; + power = 1; + for(u = 1; u < iter->attr_nbins; u++) { + if(iter->attr_bins[u] > 0) { + printf("\t# of objects with %lu - %lu attributes: %lu\n", power, (power * 10) - 1, + iter->attr_bins[u]); + total += iter->attr_bins[u]; + } /* end if */ + power *= 10; + } /* end for */ + printf("\tTotal # of objects with attributes: %lu\n", total); + printf("\tMax. # of attributes to objects: %lu\n", (unsigned long)iter->max_attrs); + return ret; +} + +/*------------------------------------------------------------------------- * Function: print_dataset_info * * Purpose: Prints information about datasets in the file @@ -1125,6 +1333,7 @@ print_file_statistics(iter_t * _iter) display_group_metadata = TRUE; display_dset = TRUE; display_dtype_metadata = TRUE; + display_attr = TRUE; } @@ -1132,6 +1341,7 @@ print_file_statistics(iter_t * _iter) if(display_file_metadata) print_file_metadata(iter); if(display_group) print_group_info(iter); if(display_dset) print_dataset_info(iter); + if(display_attr) print_attr_info(iter); } @@ -1192,13 +1402,15 @@ print_statistics(char *name, iter_t * _iter) int main(int argc, const char *argv[]) { - iter_t iter; - const char *fname = NULL; - hid_t fid; + iter_t iter; + const char *fname = NULL; + hid_t fid; struct handler_t *hand; - herr_t status; - char root[] = "/"; - int i; + herr_t status, reterr; + char root[] = "/"; + int i; + H5F_info_t finfo; + /* Disable error reporting */ H5Eset_auto2(H5E_DEFAULT, NULL, NULL); @@ -1211,7 +1423,6 @@ main(int argc, const char *argv[]) leave(EXIT_FAILURE); } - fname = argv[opt_ind]; hand[opt_ind].obj = root; hand[opt_ind].flag = 1; @@ -1225,9 +1436,22 @@ main(int argc, const char *argv[]) leave(EXIT_FAILURE); } + /* Initialize iter structure */ status = iter_init(&iter); + /* Get storge info for SOHM's btree/list/heap and superblock extension */ + reterr = H5Fget_info(fid, &finfo); + if (reterr < 0) + warn_msg(progname, "Unable to retrieve SOHM info\n"); + else { + iter.super_ext_size = finfo.super_ext_size; + iter.SM_hdr_storage_size = finfo.sohm.hdr_size; + iter.SM_index_storage_size = finfo.sohm.msgs_info.index_size; + iter.SM_heap_storage_size = finfo.sohm.msgs_info.heap_size; + } + + /* Walk the objects or all file */ for (i = 0; i < argc; i++) { if (hand[i].obj) { diff --git a/tools/misc/testfiles/h5stat_filters-F.ddl b/tools/misc/testfiles/h5stat_filters-F.ddl index be95877..544d0c7 100644 --- a/tools/misc/testfiles/h5stat_filters-F.ddl +++ b/tools/misc/testfiles/h5stat_filters-F.ddl @@ -5,3 +5,17 @@ Filename: h5stat_filters.h5 Object header size: (total/unused) Groups: 48/8 Datasets: 4936/1344 +Storage information: + Groups: + B-tree/List: 1200 + Heap: 288 + Attributes: + B-tree/List: 0 + Heap: 0 + Chunked datasets: + B-tree: 31392 + Shared Messages: + Header: 0 + B-tree/List: 0 + Heap: 0 + Superblock extension: 0 diff --git a/tools/misc/testfiles/h5stat_filters.ddl b/tools/misc/testfiles/h5stat_filters.ddl index 891a62b..11eb76a 100644 --- a/tools/misc/testfiles/h5stat_filters.ddl +++ b/tools/misc/testfiles/h5stat_filters.ddl @@ -14,6 +14,20 @@ File information Object header size: (total/unused) Groups: 48/8 Datasets: 4936/1344 +Storage information: + Groups: + B-tree/List: 1200 + Heap: 288 + Attributes: + B-tree/List: 0 + Heap: 0 + Chunked datasets: + B-tree: 31392 + Shared Messages: + Header: 0 + B-tree/List: 0 + Heap: 0 + Superblock extension: 0 Small groups: Total # of small groups: 0 Group bins: @@ -57,3 +71,8 @@ Dataset datatype information: Count (total/named) = (1/0) Size (desc./elmt) = (14/4) Total dataset datatype count: 15 +Small # of attributes: + Total # of objects with small # of attributes: 0 +Attribute bins: + Total # of objects with attributes: 0 + Max. # of attributes to objects: 0 diff --git a/tools/misc/testfiles/h5stat_help1.ddl b/tools/misc/testfiles/h5stat_help1.ddl index c4ef2c1..0841572 100644 --- a/tools/misc/testfiles/h5stat_help1.ddl +++ b/tools/misc/testfiles/h5stat_help1.ddl @@ -20,4 +20,5 @@ Usage: h5stat [OPTIONS] file -d, --dset Print dataset information -D, --dsetmetadata Print dataset metadata -T, --dtypemetadata Print datatype metadata + -A, --attribute Print attribute information diff --git a/tools/misc/testfiles/h5stat_help2.ddl b/tools/misc/testfiles/h5stat_help2.ddl index 8d9d6d6..1e6295b 100644 --- a/tools/misc/testfiles/h5stat_help2.ddl +++ b/tools/misc/testfiles/h5stat_help2.ddl @@ -20,4 +20,5 @@ Usage: h5stat [OPTIONS] file -d, --dset Print dataset information -D, --dsetmetadata Print dataset metadata -T, --dtypemetadata Print datatype metadata + -A, --attribute Print attribute information diff --git a/tools/misc/testh5stat.sh.in b/tools/misc/testh5stat.sh.in index 0672366..66d05be 100644 --- a/tools/misc/testh5stat.sh.in +++ b/tools/misc/testh5stat.sh.in @@ -119,6 +119,11 @@ TOOLTEST h5stat_filters-F.ddl -F h5stat_filters.h5 TOOLTEST h5stat_filters-d.ddl -d h5stat_filters.h5 TOOLTEST h5stat_filters-g.ddl -g h5stat_filters.h5 TOOLTEST h5stat_filters-dT.ddl -dT h5stat_filters.h5 +# h5stat_tsohm.h5 is a copy of ../../../test/tsohm.h5 generated by tsohm.c +# as of release 1.8.0-alpha4 +TOOLTEST h5stat_tsohm.ddl h5stat_tsohm.h5 +# h5stat_newgrat.h5 is generated by h5stat_gentest.c +TOOLTEST h5stat_newgrat.ddl h5stat_newgrat.h5 echo |