From 596d5b2662e0cd9003efdfce6dd0fa160395ba67 Mon Sep 17 00:00:00 2001 From: Vailin Choi Date: Wed, 6 Jun 2012 16:12:34 -0500 Subject: [svn-r22438] (1) fix for making MEM_GHEAP as MEM_DRAW (2) fix to extend into the aggregator based on threshold (3) fix for the "flavor" logging problem (4) fix for test failures due to the above fixes (5) bug fix for h5stat when h5stat against a non-existing file --Ths line, and th se below, will be ignored-- M test/mf.c M src/H5MFaggr.c M src/H5FDpublic.h M src/H5MF.c M src/H5FDmulti.c M src/H5FDlog.c M src/H5Faccum.c M tools/h5copy/testfiles/h5copytst.out.ls M tools/h5copy/testfiles/h5copy_ref.out.ls M tools/h5stat/testh5stat.sh.in M tools/h5stat/h5stat.c --- src/H5FDlog.c | 19 +++- src/H5FDmulti.c | 3 +- src/H5FDpublic.h | 2 +- src/H5Faccum.c | 27 +++-- src/H5MF.c | 20 +++- src/H5MFaggr.c | 93 ++++++++++++----- test/mf.c | 50 +++++++-- tools/h5copy/testfiles/h5copy_ref.out.ls | 2 +- tools/h5copy/testfiles/h5copytst.out.ls | 174 +++++++++++++++---------------- tools/h5stat/h5stat.c | 39 ++----- tools/h5stat/testh5stat.sh.in | 3 + 11 files changed, 260 insertions(+), 172 deletions(-) diff --git a/src/H5FDlog.c b/src/H5FDlog.c index e3d67359..bd53658 100644 --- a/src/H5FDlog.c +++ b/src/H5FDlog.c @@ -1020,12 +1020,29 @@ H5FD_log_get_eoa(const H5FD_t *_file, H5FD_mem_t UNUSED type) *------------------------------------------------------------------------- */ static herr_t -H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t UNUSED type, haddr_t addr) +H5FD_log_set_eoa(H5FD_t *_file, H5FD_mem_t type, haddr_t addr) { H5FD_log_t *file = (H5FD_log_t *)_file; FUNC_ENTER_NOAPI_NOINIT_NOERR + if(file->fa.flags != 0) { + if(H5F_addr_gt(addr, file->eoa) && H5F_addr_gt(addr, 0)) { + hsize_t size = addr - file->eoa; + + /* Retain the flavor of the space allocated by the extension */ + if(file->fa.flags & H5FD_LOG_FLAVOR) { + HDassert(addr < file->iosize); + H5_CHECK_OVERFLOW(size, hsize_t, size_t); + HDmemset(&file->flavor[file->eoa], (int)type, (size_t)size); + } /* end if */ + + /* Log the extension like an allocation */ + if(file->fa.flags & H5FD_LOG_ALLOC) + HDfprintf(file->logfp, "%10a-%10a (%10Hu bytes) (%s) Allocated\n", file->eoa, addr, size, flavors[type]); + } /* end if */ + } /* end if */ + file->eoa = addr; FUNC_LEAVE_NOAPI(SUCCEED) diff --git a/src/H5FDmulti.c b/src/H5FDmulti.c index 772384f..cdb874a 100644 --- a/src/H5FDmulti.c +++ b/src/H5FDmulti.c @@ -308,7 +308,8 @@ H5Pset_fapl_split(hid_t fapl, const char *meta_ext, hid_t meta_plist_id, /* Initialize */ ALL_MEMBERS(mt) { - memb_map[mt] = (H5FD_MEM_DRAW==mt?mt:H5FD_MEM_SUPER); + /* Treat global heap as raw data, not metadata */ + memb_map[mt] = ((mt == H5FD_MEM_DRAW || mt == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : H5FD_MEM_SUPER); memb_fapl[mt] = -1; memb_name[mt] = NULL; memb_addr[mt] = HADDR_UNDEF; diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 0c5605c..b70af68 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -100,7 +100,7 @@ typedef enum H5F_mem_t H5FD_mem_t; H5FD_MEM_SUPER, /*super*/ \ H5FD_MEM_SUPER, /*btree*/ \ H5FD_MEM_DRAW, /*draw*/ \ - H5FD_MEM_SUPER, /*gheap*/ \ + H5FD_MEM_DRAW, /*gheap*/ \ H5FD_MEM_SUPER, /*lheap*/ \ H5FD_MEM_SUPER /*ohdr*/ \ } diff --git a/src/H5Faccum.c b/src/H5Faccum.c index 1c2abd9..032bd80 100644 --- a/src/H5Faccum.c +++ b/src/H5Faccum.c @@ -115,6 +115,7 @@ herr_t H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, size_t size, void *buf/*out*/) { + H5FD_mem_t map_type; /* Mapped memory type */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -123,8 +124,11 @@ H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, HDassert(f->shared); HDassert(buf); + /* Treat global heap as raw data */ + map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; + /* Check if this information is in the metadata accumulator */ - if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && type != H5FD_MEM_DRAW) { + if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { if(size < H5F_ACCUM_MAX_SIZE) { /* Sanity check */ HDassert(!f->shared->accum.buf || (f->shared->accum.alloc_size >= f->shared->accum.size)); @@ -173,7 +177,7 @@ H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, f->shared->accum.dirty_off += amount_before; /* Dispatch to driver */ - if(H5FD_read(f->shared->lf, dxpl_id, type, addr, amount_before, f->shared->accum.buf) < 0) + if(H5FD_read(f->shared->lf, dxpl_id, map_type, addr, amount_before, f->shared->accum.buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end if */ else @@ -187,7 +191,7 @@ H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, H5_ASSIGN_OVERFLOW(amount_after, ((addr + size) - (f->shared->accum.loc + f->shared->accum.size)), hsize_t, size_t); /* Dispatch to driver */ - if(H5FD_read(f->shared->lf, dxpl_id, type, (f->shared->accum.loc + f->shared->accum.size), amount_after, (f->shared->accum.buf + f->shared->accum.size + amount_before)) < 0) + if(H5FD_read(f->shared->lf, dxpl_id, map_type, (f->shared->accum.loc + f->shared->accum.size), amount_after, (f->shared->accum.buf + f->shared->accum.size + amount_before)) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end if */ @@ -201,13 +205,13 @@ H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, /* Current read doesn't overlap with metadata accumulator, read it from file */ else { /* Dispatch to driver */ - if(H5FD_read(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) + if(H5FD_read(f->shared->lf, dxpl_id, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end else */ } /* end if */ else { /* Read the data */ - if(H5FD_read(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) + if(H5FD_read(f->shared->lf, dxpl_id, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") /* Check for overlap w/dirty accumulator */ @@ -250,7 +254,7 @@ H5F_accum_read(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, } /* end if */ else { /* Read the data */ - if(H5FD_read(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) + if(H5FD_read(f->shared->lf, dxpl_id, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "driver read request failed") } /* end else */ @@ -415,6 +419,7 @@ herr_t H5F_accum_write(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, size_t size, const void *buf) { + H5FD_mem_t map_type; /* Mapped memory type */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -424,8 +429,11 @@ H5F_accum_write(const H5F_t *f, hid_t dxpl_id, H5FD_mem_t type, haddr_t addr, HDassert(f->intent & H5F_ACC_RDWR); HDassert(buf); + /* Treat global heap as raw data */ + map_type = (type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : type; + /* Check for accumulating metadata */ - if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && type != H5FD_MEM_DRAW) { + if((f->shared->feature_flags & H5FD_FEAT_ACCUMULATE_METADATA) && map_type != H5FD_MEM_DRAW) { if(size < H5F_ACCUM_MAX_SIZE) { /* Sanity check */ HDassert(!f->shared->accum.buf || (f->shared->accum.alloc_size >= f->shared->accum.size)); @@ -719,7 +727,7 @@ HDmemset(f->shared->accum.buf + size, 0, (f->shared->accum.alloc_size - size)); } /* end if */ else { /* Write the data */ - if(H5FD_write(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) + if(H5FD_write(f->shared->lf, dxpl_id, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") /* Check for overlap w/accumulator */ @@ -804,7 +812,7 @@ HDmemset(f->shared->accum.buf + size, 0, (f->shared->accum.alloc_size - size)); } /* end if */ else { /* Write the data */ - if(H5FD_write(f->shared->lf, dxpl_id, type, addr, size, buf) < 0) + if(H5FD_write(f->shared->lf, dxpl_id, map_type, addr, size, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file write failed") } /* end else */ @@ -846,6 +854,7 @@ H5F_accum_free(H5F_t *f, hid_t dxpl_id, H5FD_mem_t UNUSED type, haddr_t addr, /* Sanity check */ /* (The metadata accumulator should not intersect w/raw data */ HDassert(H5FD_MEM_DRAW != type); + HDassert(H5FD_MEM_GHEAP != type); /* (global heap data is being treated as raw data currently) */ /* Check for overlapping the beginning of the accumulator */ if(H5F_addr_le(addr, f->shared->accum.loc)) { diff --git a/src/H5MF.c b/src/H5MF.c index 203c942..3c14027 100644 --- a/src/H5MF.c +++ b/src/H5MF.c @@ -153,7 +153,8 @@ H5MF_init_merge_flags(H5F_t *f) all_metadata_same = TRUE; for(type = H5FD_MEM_SUPER; type < H5FD_MEM_NTYPES; H5_INC_ENUM(H5FD_mem_t, type)) /* Skip checking raw data free list mapping */ - if(type != H5FD_MEM_DRAW) { + /* (global heap is treated as raw data) */ + if(type != H5FD_MEM_DRAW && type != H5FD_MEM_GHEAP) { /* Check for any different type mappings */ if(f->shared->fs_type_map[type] != f->shared->fs_type_map[H5FD_MEM_SUPER]) { all_metadata_same = FALSE; @@ -176,9 +177,12 @@ H5MF_init_merge_flags(H5F_t *f) HDmemset(f->shared->fs_aggr_merge, 0, sizeof(f->shared->fs_aggr_merge)); /* Check if merging raw data should be allowed */ + /* (treat global heaps as raw data) */ if(H5FD_MEM_DRAW == f->shared->fs_type_map[H5FD_MEM_DRAW] || - H5FD_MEM_DEFAULT == f->shared->fs_type_map[H5FD_MEM_DRAW]) + H5FD_MEM_DEFAULT == f->shared->fs_type_map[H5FD_MEM_DRAW]) { f->shared->fs_aggr_merge[H5FD_MEM_DRAW] = H5F_FS_MERGE_RAWDATA; + f->shared->fs_aggr_merge[H5FD_MEM_GHEAP] = H5F_FS_MERGE_RAWDATA; + } /* end if */ break; case H5MF_AGGR_MERGE_DICHOTOMY: @@ -186,7 +190,9 @@ H5MF_init_merge_flags(H5F_t *f) HDmemset(f->shared->fs_aggr_merge, H5F_FS_MERGE_METADATA, sizeof(f->shared->fs_aggr_merge)); /* Allow merging raw data allocations together */ + /* (treat global heaps as raw data) */ f->shared->fs_aggr_merge[H5FD_MEM_DRAW] = H5F_FS_MERGE_RAWDATA; + f->shared->fs_aggr_merge[H5FD_MEM_GHEAP] = H5F_FS_MERGE_RAWDATA; break; case H5MF_AGGR_MERGE_TOGETHER: @@ -717,6 +723,7 @@ H5MF_try_extend(H5F_t *f, hid_t dxpl_id, H5FD_mem_t alloc_type, haddr_t addr, hsize_t size, hsize_t extra_requested) { haddr_t end; /* End of block to extend */ + H5FD_mem_t map_type; /* Mapped type */ htri_t ret_value; /* Return value */ FUNC_ENTER_NOAPI(FAIL) @@ -728,18 +735,21 @@ HDfprintf(stderr, "%s: Entering: alloc_type = %u, addr = %a, size = %Hu, extra_r HDassert(f); HDassert(H5F_INTENT(f) & H5F_ACC_RDWR); + /* Set mapped type, treating global heap as raw data */ + map_type = (alloc_type == H5FD_MEM_GHEAP) ? H5FD_MEM_DRAW : alloc_type; + /* Compute end of block to extend */ end = addr + size; /* Check if the block is exactly at the end of the file */ - if((ret_value = H5FD_try_extend(f->shared->lf, alloc_type, f, end, extra_requested)) < 0) + if((ret_value = H5FD_try_extend(f->shared->lf, map_type, f, end, extra_requested)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") else if(ret_value == FALSE) { H5F_blk_aggr_t *aggr; /* Aggregator to use */ /* Check for test block able to extend aggregation block */ - aggr = (alloc_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr); - if((ret_value = H5MF_aggr_try_extend(f, aggr, alloc_type, end, extra_requested)) < 0) + aggr = (map_type == H5FD_MEM_DRAW) ? &(f->shared->sdata_aggr) : &(f->shared->meta_aggr); + if((ret_value = H5MF_aggr_try_extend(f, aggr, map_type, end, extra_requested)) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending aggregation block") else if(ret_value == FALSE) { H5FD_mem_t fs_type; /* Free space type (mapped from allocation type) */ diff --git a/src/H5MFaggr.c b/src/H5MFaggr.c index 8a25c8c..ad91e45 100644 --- a/src/H5MFaggr.c +++ b/src/H5MFaggr.c @@ -46,6 +46,7 @@ /******************/ /* Local Typedefs */ /******************/ +#define EXTEND_THRESHOLD .10 /********************/ @@ -108,14 +109,14 @@ HDfprintf(stderr, "%s: alloc_type = %u, size = %Hu\n", FUNC, (unsigned)alloc_typ HDassert(size > 0); /* Couldn't find anything from the free space manager, go allocate some */ - if(alloc_type != H5FD_MEM_DRAW) { + if(alloc_type != H5FD_MEM_DRAW && alloc_type != H5FD_MEM_GHEAP) { /* Handle metadata differently from "raw" data */ if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->meta_aggr), &(f->shared->sdata_aggr), alloc_type, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate metadata") } /* end if */ else { - /* Allocate "raw" data */ - if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), alloc_type, size))) + /* Allocate "raw" data: H5FD_MEM_DRAW and H5FD_MEM_GHEAP */ + if(HADDR_UNDEF == (ret_value = H5MF_aggr_alloc(f, dxpl_id, &(f->shared->sdata_aggr), &(f->shared->meta_aggr), H5FD_MEM_DRAW, size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate raw data") } /* end else */ @@ -242,7 +243,7 @@ HDfprintf(stderr, "%s: aggr = {%a, %Hu, %Hu}\n", FUNC, aggr->addr, aggr->tot_siz } /* end if */ /* Allocate space from the VFD (i.e. at the end of the file) */ - if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, type, f, size, &eoa_frag_addr, &eoa_frag_size))) + if(HADDR_UNDEF == (ret_value = H5FD_alloc(f->shared->lf, dxpl_id, alloc_type, f, size, &eoa_frag_addr, &eoa_frag_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTALLOC, HADDR_UNDEF, "can't allocate aggregation block") } /* end else */ } /* end if */ @@ -307,12 +308,12 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); /* Freeing any possible fragment due to file allocation */ if(eoa_frag_size) - if(H5MF_xfree(f, type, dxpl_id, eoa_frag_addr, eoa_frag_size) < 0) + if(H5MF_xfree(f, alloc_type, dxpl_id, eoa_frag_addr, eoa_frag_size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free eoa fragment") /* Freeing any possible fragment due to alignment in the block after extension */ if(extended && aggr_frag_size) - if(H5MF_xfree(f, type, dxpl_id, aggr_frag_addr, aggr_frag_size) < 0) + if(H5MF_xfree(f, alloc_type, dxpl_id, aggr_frag_addr, aggr_frag_size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation fragment") } /* end if */ else { @@ -323,7 +324,7 @@ HDfprintf(stderr, "%s: Allocating block\n", FUNC); /* free any possible fragment */ if(aggr_frag_size) - if(H5MF_xfree(f, type, dxpl_id, aggr_frag_addr, aggr_frag_size) < 0) + if(H5MF_xfree(f, alloc_type, dxpl_id, aggr_frag_addr, aggr_frag_size) < 0) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTFREE, HADDR_UNDEF, "can't free aggregation fragment") } /* end else */ } /* end if */ @@ -364,6 +365,16 @@ HDfprintf(stderr, "%s: ret_value = %a\n", FUNC, ret_value); * Purpose: Check if a block is inside an aggregator block and extend it * if possible. * + * Note: + * When the block to be extended adjoins the aggregator-- + * 1) When the aggregator is at end of file: + * A) If the request is below the threshold, extend the block into the aggregator + * B) If the request is above the threshold, + * a) extend the aggregator by aggr->alloc_size or the extended amount + * b) extend the block into the aggregator + * 2) When the aggregator is not at end of file: + * Extended the block into the aggregator if it has enough space to satisfy the request + * * Return: Success: TRUE(1) - Block was extended * FALSE(0) - Block could not be extended * Failure: FAIL @@ -388,26 +399,56 @@ H5MF_aggr_try_extend(H5F_t *f, H5F_blk_aggr_t *aggr, H5FD_mem_t type, /* Check if this aggregator is active */ if(f->shared->feature_flags & aggr->feature_flag) { - /* If the block being tested adjoins the beginning of the aggregator + /* + * If the block being tested adjoins the beginning of the aggregator * block, check if the aggregator can accomodate the extension. */ if(H5F_addr_eq(blk_end, aggr->addr)) { - /* If the aggregator block is at the end of the file, extend the - * file and "bubble" the aggregator up - */ - if((ret_value = H5FD_try_extend(f->shared->lf, type, f, (aggr->addr + aggr->size), extra_requested)) < 0) - HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") - else if(ret_value == TRUE) { - /* Shift the aggregator block by the extra requested */ - aggr->addr += extra_requested; - - /* Add extra requested to the aggregator block's total amount allocated */ - aggr->tot_size += extra_requested; - } /* end if */ - else { - /* Check if the aggregator block has enough internal space to satisfy - * extending the block. - */ + haddr_t eoa; /* EOA for the file */ + + /* Get the EOA for the file */ + if(HADDR_UNDEF == (eoa = H5F_get_eoa(f, type))) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTGET, FAIL, "Unable to get eoa") + + /* If the aggregator is at the end of file: */ + if(H5F_addr_eq(eoa, aggr->addr + aggr->size)) { + /* If extra_requested is below percentage threshold, extend block into the aggregator. */ + if(extra_requested <= (EXTEND_THRESHOLD * aggr->size)) { + aggr->size -= extra_requested; + aggr->addr += extra_requested; + + /* Indicate success */ + HGOTO_DONE(TRUE); + } + /* + * If extra_requested is above percentage threshold: + * 1) "bubble" up the aggregator by aggr->alloc_size or extra_requested + * 2) extend the block into the aggregator + */ + else { + hsize_t extra = (extra_requested < aggr->alloc_size) ? aggr->alloc_size : extra_requested; + + if((ret_value = H5FD_try_extend(f->shared->lf, type, f, (aggr->addr + aggr->size), extra)) < 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_CANTEXTEND, FAIL, "error extending file") + else if(ret_value == TRUE) { + /* Shift the aggregator block by the extra requested */ + /* (allocates the space for the extra_requested) */ + aggr->addr += extra_requested; + + /* Add extra to the aggregator's total allocated amount */ + aggr->tot_size += extra; + + /* Account for any space added to the aggregator */ + /* (either 0 (if extra_requested > aggr->alloc_size) or + * (aggr->alloc_size - extra_requested) -QAK + */ + aggr->size += extra; + aggr->size -= extra_requested; + } /* end if */ + } /* end if */ + } /* end if */ + else { /* The aggreator is not at end of file */ + /* Check if aggregator has enough internal space to satisfy the extension. */ if(aggr->size >= extra_requested) { /* Extend block into aggregator */ aggr->size -= extra_requested; @@ -416,8 +457,8 @@ H5MF_aggr_try_extend(H5F_t *f, H5F_blk_aggr_t *aggr, H5FD_mem_t type, /* Indicate success */ HGOTO_DONE(TRUE); } /* end if */ - } /* end else */ - } /* end if */ + } /* end else */ + } /* end if */ } /* end if */ done: diff --git a/test/mf.c b/test/mf.c index 005e5a0..e3c3082 100644 --- a/test/mf.c +++ b/test/mf.c @@ -3165,8 +3165,13 @@ error: * To verify that a block can be extended from the aggregator * * Test 1: Allocate block A from meta_aggr which is at end of file - * Try to extend a block which adjoins the aggregator - * H5MF_try_extend() succeeds: meta_aggr is extended by extended-request and meta_aggr's info is updated + * Try to extend the block which adjoins the aggregator that is at end of file + * a. block size < (% * aggr->alloc_size) + * The block is allocated from the aggregator + * b. block size > (% * aggr->alloc_size) but block size < aggr->alloc_size + * The block is extended by aggr->alloc_size and the block is allocated from the aggregator + * c. block size > (% * aggr->alloc_size) but block size > aggr->alloc_size + * The block is extended by extended-request and the block is allocated from the aggregator * * Test 2: Allocate block A from meta_aggr * Allocate block B from sdata_aggr so that meta_aggr is not at end of file @@ -3233,7 +3238,7 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) new_addr = addr - 10; - /* Try to extend the block */ + /* Try to extend the block by an amount < (% * aggr->alloc_size) */ extended = H5MF_try_extend(f, H5P_DATASET_XFER_DEFAULT, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE50)); /* should succeed */ @@ -3244,15 +3249,42 @@ test_mf_aggr_extend(const char *env_h5_drvr, hid_t fapl) if (new_ma_addr != (addr+TEST_BLOCK_SIZE50)) TEST_ERROR - if (new_ma_size != f->shared->meta_aggr.alloc_size) TEST_ERROR + if (new_ma_size != (f->shared->meta_aggr.alloc_size - TEST_BLOCK_SIZE50)) TEST_ERROR - /* Restore info for meta_aggr */ - f->shared->meta_aggr.addr = ma_addr; - f->shared->meta_aggr.size = ma_size; + /* Free the allocated blocks */ + H5MF_xfree(f, type, H5P_DATASET_XFER_DEFAULT, addr, (hsize_t)TEST_BLOCK_SIZE50); + + /* Try to extend the block by an amount > (% * aggr->alloc_size) but amount < aggr->alloc_size */ + extended = H5MF_try_extend(f, H5P_DATASET_XFER_DEFAULT, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE700)); + + /* should succeed */ + if(!extended) + TEST_ERROR + + H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); + + if (new_ma_addr != (addr + TEST_BLOCK_SIZE700)) + TEST_ERROR + if (new_ma_size != (f->shared->meta_aggr.alloc_size * 2 - TEST_BLOCK_SIZE700)) TEST_ERROR /* Free the allocated blocks */ - H5MF_xfree(f, type, H5P_DATASET_XFER_DEFAULT, addr, (hsize_t)TEST_BLOCK_SIZE30); - H5MF_xfree(f, type, H5P_DATASET_XFER_DEFAULT, (ma_addr+ma_size), (hsize_t)TEST_BLOCK_SIZE50); + H5MF_xfree(f, type, H5P_DATASET_XFER_DEFAULT, addr, (hsize_t)TEST_BLOCK_SIZE700); + + /* Try to extend the block by an amount > (% * aggr->alloc_size) but amount > aggr->alloc_size */ + extended = H5MF_try_extend(f, H5P_DATASET_XFER_DEFAULT, type, (haddr_t)new_addr, (hsize_t)10, (hsize_t)(TEST_BLOCK_SIZE2058)); + + /* should succeed */ + if(!extended) + TEST_ERROR + + H5MF_aggr_query(f, &(f->shared->meta_aggr), &new_ma_addr, &new_ma_size); + + if (new_ma_addr != (addr + TEST_BLOCK_SIZE2058)) + TEST_ERROR + if (new_ma_size != f->shared->meta_aggr.size) TEST_ERROR + + /* Free the allocated blocks */ + H5MF_xfree(f, type, H5P_DATASET_XFER_DEFAULT, addr, (hsize_t)TEST_BLOCK_SIZE2058); if(H5Fclose(file) < 0) FAIL_STACK_ERROR diff --git a/tools/h5copy/testfiles/h5copy_ref.out.ls b/tools/h5copy/testfiles/h5copy_ref.out.ls index 0769dc8..d685af2 100644 --- a/tools/h5copy/testfiles/h5copy_ref.out.ls +++ b/tools/h5copy/testfiles/h5copy_ref.out.ls @@ -21,7 +21,7 @@ Opened "./testfiles/h5copy_ref.out.h5" with sec2 driver. Storage:
Type: object reference /COPY/Dset_REGREF Dataset {2/2} - Location: 1:9400 + Location: 1:5304 Links: 1 Storage:
Type: dataset region reference diff --git a/tools/h5copy/testfiles/h5copytst.out.ls b/tools/h5copy/testfiles/h5copytst.out.ls index 62c860b..4044aaf 100644 --- a/tools/h5copy/testfiles/h5copytst.out.ls +++ b/tools/h5copy/testfiles/h5copytst.out.ls @@ -3,57 +3,57 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. Location: 1:96 Links: 1 /A Group - Location: 1:88336 + Location: 1:84304 Links: 1 /A/B1 Group - Location: 1:89040 + Location: 1:85008 Links: 1 /A/B1/simple Dataset {6/6} - Location: 1:88208 + Location: 1:84176 Links: 1 Storage:
Type: 32-bit little-endian integer /A/B2 Group - Location: 1:92576 + Location: 1:88544 Links: 1 /A/B2/simple2 Dataset {6/6} - Location: 1:92448 + Location: 1:88416 Links: 1 Storage:
Type: 32-bit little-endian integer /C Group - Location: 1:95784 + Location: 1:91752 Links: 1 /C/D Group - Location: 1:96488 + Location: 1:92456 Links: 1 /C/D/simple Dataset {6/6} - Location: 1:95656 + Location: 1:91624 Links: 1 Storage:
Type: 32-bit little-endian integer /E Group - Location: 1:110400 + Location: 1:106368 Links: 1 /E/F Group - Location: 1:111104 + Location: 1:107072 Links: 1 /E/F/grp_dsets Group - Location: 1:98600 + Location: 1:94568 Links: 1 /E/F/grp_dsets/chunk Dataset {6/6} - Location: 1:102784 + Location: 1:98752 Links: 1 Chunks: {2} 8 bytes Storage:
Type: 32-bit little-endian integer /E/F/grp_dsets/compact Dataset {6/6} - Location: 1:103240 + Location: 1:99208 Links: 1 Storage:
Type: 32-bit little-endian integer /E/F/grp_dsets/compound Dataset {2/2} - Location: 1:103376 + Location: 1:99344 Links: 1 Storage:
Type: struct { @@ -61,60 +61,60 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /E/F/grp_dsets/compressed Dataset {6/6} - Location: 1:105688 + Location: 1:101656 Links: 1 Chunks: {2} 8 bytes Storage:
Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /E/F/grp_dsets/named_vl Dataset {2/2} - Location: 1:109952 + Location: 1:105920 Links: 1 Storage:
- Type: shared-1:110080 variable length of + Type: shared-1:106048 variable length of 32-bit little-endian integer /E/F/grp_dsets/nested_vl Dataset {2/2} - Location: 1:110128 + Location: 1:106096 Links: 1 Storage:
Type: variable length of variable length of 32-bit little-endian integer /E/F/grp_dsets/simple Dataset {6/6} - Location: 1:110272 + Location: 1:106240 Links: 1 Storage:
Type: 32-bit little-endian integer /E/F/grp_dsets/vl Type - Location: 1:110080 + Location: 1:106048 Links: 2 - Type: shared-1:110080 variable length of + Type: shared-1:106048 variable length of 32-bit little-endian integer /G Group - Location: 1:126048 + Location: 1:122016 Links: 1 /G/H Group - Location: 1:126752 + Location: 1:122720 Links: 1 /G/H/grp_nested Group - Location: 1:113128 + Location: 1:109096 Links: 1 /G/H/grp_nested/grp_dsets Group - Location: 1:113920 + Location: 1:109888 Links: 1 /G/H/grp_nested/grp_dsets/chunk Dataset {6/6} - Location: 1:118104 + Location: 1:114072 Links: 1 Chunks: {2} 8 bytes Storage:
Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/compact Dataset {6/6} - Location: 1:118560 + Location: 1:114528 Links: 1 Storage:
Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/compound Dataset {2/2} - Location: 1:118696 + Location: 1:114664 Links: 1 Storage:
Type: struct { @@ -122,34 +122,34 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /G/H/grp_nested/grp_dsets/compressed Dataset {6/6} - Location: 1:121008 + Location: 1:116976 Links: 1 Chunks: {2} 8 bytes Storage:
Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/named_vl Dataset {2/2} - Location: 1:125272 + Location: 1:121240 Links: 1 Storage:
- Type: shared-1:125400 variable length of + Type: shared-1:121368 variable length of 32-bit little-endian integer /G/H/grp_nested/grp_dsets/nested_vl Dataset {2/2} - Location: 1:125448 + Location: 1:121416 Links: 1 Storage:
Type: variable length of variable length of 32-bit little-endian integer /G/H/grp_nested/grp_dsets/simple Dataset {6/6} - Location: 1:125592 + Location: 1:121560 Links: 1 Storage:
Type: 32-bit little-endian integer /G/H/grp_nested/grp_dsets/vl Type - Location: 1:125400 + Location: 1:121368 Links: 2 - Type: shared-1:125400 variable length of + Type: shared-1:121368 variable length of 32-bit little-endian integer /chunk Dataset {6/6} Location: 1:6312 @@ -178,21 +178,21 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_dsets Group - Location: 1:32160 + Location: 1:28128 Links: 1 /grp_dsets/chunk Dataset {6/6} - Location: 1:36344 + Location: 1:32312 Links: 1 Chunks: {2} 8 bytes Storage:
Type: 32-bit little-endian integer /grp_dsets/compact Dataset {6/6} - Location: 1:36800 + Location: 1:32768 Links: 1 Storage:
Type: 32-bit little-endian integer /grp_dsets/compound Dataset {2/2} - Location: 1:36936 + Location: 1:32904 Links: 1 Storage:
Type: struct { @@ -200,62 +200,62 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_dsets/compressed Dataset {6/6} - Location: 1:39248 + Location: 1:35216 Links: 1 Chunks: {2} 8 bytes Storage:
Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_dsets/named_vl Dataset {2/2} - Location: 1:43512 + Location: 1:39480 Links: 1 Storage:
- Type: shared-1:43640 variable length of + Type: shared-1:39608 variable length of 32-bit little-endian integer /grp_dsets/nested_vl Dataset {2/2} - Location: 1:43688 + Location: 1:39656 Links: 1 Storage:
Type: variable length of variable length of 32-bit little-endian integer /grp_dsets/simple Dataset {6/6} - Location: 1:43832 + Location: 1:39800 Links: 1 Storage:
Type: 32-bit little-endian integer /grp_dsets/simple_group Dataset {6/6} - Location: 1:59944 + Location: 1:55912 Links: 1 Storage:
Type: 32-bit little-endian integer /grp_dsets/vl Type - Location: 1:43640 + Location: 1:39608 Links: 2 - Type: shared-1:43640 variable length of + Type: shared-1:39608 variable length of 32-bit little-endian integer /grp_empty Group - Location: 1:31368 + Location: 1:27336 Links: 1 /grp_nested Group - Location: 1:44624 + Location: 1:40592 Links: 1 /grp_nested/grp_dsets Group - Location: 1:45416 + Location: 1:41384 Links: 1 /grp_nested/grp_dsets/chunk Dataset {6/6} - Location: 1:49600 + Location: 1:45568 Links: 1 Chunks: {2} 8 bytes Storage:
Type: 32-bit little-endian integer /grp_nested/grp_dsets/compact Dataset {6/6} - Location: 1:50056 + Location: 1:46024 Links: 1 Storage:
Type: 32-bit little-endian integer /grp_nested/grp_dsets/compound Dataset {2/2} - Location: 1:50192 + Location: 1:46160 Links: 1 Storage:
Type: struct { @@ -263,51 +263,51 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_nested/grp_dsets/compressed Dataset {6/6} - Location: 1:52504 + Location: 1:48472 Links: 1 Chunks: {2} 8 bytes Storage:
Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_nested/grp_dsets/named_vl Dataset {2/2} - Location: 1:56768 + Location: 1:52736 Links: 1 Storage:
- Type: shared-1:56896 variable length of + Type: shared-1:52864 variable length of 32-bit little-endian integer /grp_nested/grp_dsets/nested_vl Dataset {2/2} - Location: 1:56944 + Location: 1:52912 Links: 1 Storage:
Type: variable length of variable length of 32-bit little-endian integer /grp_nested/grp_dsets/simple Dataset {6/6} - Location: 1:57088 + Location: 1:53056 Links: 1 Storage:
Type: 32-bit little-endian integer /grp_nested/grp_dsets/vl Type - Location: 1:56896 + Location: 1:52864 Links: 2 - Type: shared-1:56896 variable length of + Type: shared-1:52864 variable length of 32-bit little-endian integer /grp_rename Group - Location: 1:61152 + Location: 1:57120 Links: 1 /grp_rename/chunk Dataset {6/6} - Location: 1:65336 + Location: 1:61304 Links: 1 Chunks: {2} 8 bytes Storage:
Type: 32-bit little-endian integer /grp_rename/compact Dataset {6/6} - Location: 1:65792 + Location: 1:61760 Links: 1 Storage:
Type: 32-bit little-endian integer /grp_rename/compound Dataset {2/2} - Location: 1:65928 + Location: 1:61896 Links: 1 Storage:
Type: struct { @@ -315,28 +315,28 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_rename/compressed Dataset {6/6} - Location: 1:68240 + Location: 1:64208 Links: 1 Chunks: {2} 8 bytes Storage:
Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_rename/grp_dsets Group - Location: 1:74032 + Location: 1:70000 Links: 1 /grp_rename/grp_dsets/chunk Dataset {6/6} - Location: 1:78216 + Location: 1:74184 Links: 1 Chunks: {2} 8 bytes Storage:
Type: 32-bit little-endian integer /grp_rename/grp_dsets/compact Dataset {6/6} - Location: 1:78672 + Location: 1:74640 Links: 1 Storage:
Type: 32-bit little-endian integer /grp_rename/grp_dsets/compound Dataset {2/2} - Location: 1:78808 + Location: 1:74776 Links: 1 Storage:
Type: struct { @@ -344,73 +344,73 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. "str2" +20 20-byte null-terminated ASCII string } 40 bytes /grp_rename/grp_dsets/compressed Dataset {6/6} - Location: 1:81120 + Location: 1:77088 Links: 1 Chunks: {2} 8 bytes Storage:
Filter-0: deflate-1 OPT {1} Type: 32-bit little-endian integer /grp_rename/grp_dsets/named_vl Dataset {2/2} - Location: 1:85384 + Location: 1:81352 Links: 1 Storage:
- Type: shared-1:85512 variable length of + Type: shared-1:81480 variable length of 32-bit little-endian integer /grp_rename/grp_dsets/nested_vl Dataset {2/2} - Location: 1:85560 + Location: 1:81528 Links: 1 Storage:
Type: variable length of variable length of 32-bit little-endian integer /grp_rename/grp_dsets/simple Dataset {6/6} - Location: 1:85704 + Location: 1:81672 Links: 1 Storage:
Type: 32-bit little-endian integer /grp_rename/grp_dsets/vl Type - Location: 1:85512 + Location: 1:81480 Links: 2 - Type: shared-1:85512 variable length of + Type: shared-1:81480 variable length of 32-bit little-endian integer /grp_rename/named_vl Dataset {2/2} - Location: 1:72504 + Location: 1:68472 Links: 1 Storage:
- Type: shared-1:72632 variable length of + Type: shared-1:68600 variable length of 32-bit little-endian integer /grp_rename/nested_vl Dataset {2/2} - Location: 1:72680 + Location: 1:68648 Links: 1 Storage:
Type: variable length of variable length of 32-bit little-endian integer /grp_rename/simple Dataset {6/6} - Location: 1:72824 + Location: 1:68792 Links: 1 Storage:
Type: 32-bit little-endian integer /grp_rename/vl Type - Location: 1:72632 + Location: 1:68600 Links: 2 - Type: shared-1:72632 variable length of + Type: shared-1:68600 variable length of 32-bit little-endian integer /named_vl Dataset {2/2} - Location: 1:19296 + Location: 1:17280 Links: 1 Storage:
- Type: shared-1:19424 variable length of + Type: shared-1:17408 variable length of 32-bit little-endian integer /nested_vl Dataset {2/2} - Location: 1:25792 + Location: 1:21760 Links: 1 Storage:
Type: variable length of variable length of 32-bit little-endian integer /rename Dataset {2/2} - Location: 1:30160 + Location: 1:26128 Links: 1 Storage:
Type: struct { @@ -423,7 +423,7 @@ Opened "./testfiles/h5copytst.out.h5" with sec2 driver. Storage:
Type: 32-bit little-endian integer /simple_top Dataset {6/6} - Location: 1:27984 + Location: 1:23952 Links: 1 Storage:
Type: 32-bit little-endian integer diff --git a/tools/h5stat/h5stat.c b/tools/h5stat/h5stat.c index 9e414a8..961e758 100644 --- a/tools/h5stat/h5stat.c +++ b/tools/h5stat/h5stat.c @@ -845,36 +845,9 @@ error: /*------------------------------------------------------------------------- - * Function: init_iter + * Function: iter_free * - * Purpose: Initialize iter structure - * - * Return: Success: 0 - * - * Failure: Never fails - * - * Programmer: Elena Pourmal - * Saturday, August 12, 2006 - * - *------------------------------------------------------------------------- - */ -static herr_t -iter_init(iter_t *iter, hid_t fid) -{ - /* Clear everything to zeros */ - HDmemset(iter, 0, sizeof(*iter)); - - /* Set the file ID for later use in callbacks */ - iter->fid = fid; - - return 0; -} /* iter_init() */ - - -/*------------------------------------------------------------------------- - * Function: free_iter - * - * Purpose: Freee iter structure + * Purpose: Free iter structure * * Return: Success: 0 * @@ -1452,7 +1425,7 @@ main(int argc, const char *argv[]) { iter_t iter; const char *fname = NULL; - hid_t fid; + hid_t fid = -1; hid_t fcpl; struct handler_t *hand = NULL; H5F_info_t finfo; @@ -1475,6 +1448,8 @@ main(int argc, const char *argv[]) printf("Filename: %s\n", fname); + HDmemset(&iter, 0, sizeof(iter)); + fid = H5Fopen(fname, H5F_ACC_RDONLY, H5P_DEFAULT); if(fid < 0) { error_msg("unable to open file \"%s\"\n", fname); @@ -1483,7 +1458,7 @@ main(int argc, const char *argv[]) } /* end if */ /* Initialize iter structure */ - iter_init(&iter, fid); + iter.fid = fid; if(H5Fget_filesize(fid, &iter.filesize) < 0) warn_msg("Unable to retrieve file size\n"); @@ -1539,7 +1514,7 @@ done: /* Free iter structure */ iter_free(&iter); - if(H5Fclose(fid) < 0) { + if(fid >= 0 && H5Fclose(fid) < 0) { error_msg("unable to close file \"%s\"\n", fname); h5tools_setstatus(EXIT_FAILURE); } diff --git a/tools/h5stat/testh5stat.sh.in b/tools/h5stat/testh5stat.sh.in index c5cb3e8..c594a97 100644 --- a/tools/h5stat/testh5stat.sh.in +++ b/tools/h5stat/testh5stat.sh.in @@ -77,6 +77,7 @@ $SRC_H5STAT_TESTFILES/h5stat_newgrat.h5 LIST_OTHER_TEST_FILES=" $SRC_H5STAT_TESTFILES/h5stat_help1.ddl $SRC_H5STAT_TESTFILES/h5stat_help2.ddl +$SRC_H5STAT_TESTFILES/h5stat_notexist.ddl $SRC_H5STAT_TESTFILES/h5stat_filters.ddl $SRC_H5STAT_TESTFILES/h5stat_filters-file.ddl $SRC_H5STAT_TESTFILES/h5stat_filters-F.ddl @@ -189,6 +190,8 @@ COPY_TESTFILES_TO_TESTDIR # Test for help flag TOOLTEST h5stat_help1.ddl -h TOOLTEST h5stat_help2.ddl --help +# Test when h5stat a file that does not exist +TOOLTEST h5stat_notexist.ddl notexist.h5 # Test file with groups, compressed datasets, user-applied fileters, etc. # h5stat_filters.h5 is a copy of ../../testfiles/tfilters.h5 as of release 1.8.0-alpha4 -- cgit v0.12