diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/H5B.c | 5 | ||||
| -rw-r--r-- | src/H5FD.c | 1 | ||||
| -rw-r--r-- | src/H5FDcore.c | 3 | ||||
| -rw-r--r-- | src/H5FDpublic.h | 1 | ||||
| -rw-r--r-- | src/H5HG.c | 8 | ||||
| -rw-r--r-- | src/H5HL.c | 27 | ||||
| -rw-r--r-- | src/H5HLpkg.h | 1 | ||||
| -rw-r--r-- | src/H5MF.c | 147 | ||||
| -rw-r--r-- | src/H5MFprivate.h | 3 | ||||
| -rw-r--r-- | src/H5O.c | 34 | ||||
| -rw-r--r-- | src/H5Tconv.c | 7 |
11 files changed, 216 insertions, 21 deletions
@@ -229,7 +229,7 @@ H5B_create(H5F_t *f, hid_t dxpl_id, const H5B_class_t *type, void *udata, bt->right = HADDR_UNDEF; bt->nchildren = 0; if((bt->rc_shared=(type->get_shared)(f, udata))==NULL) - HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "can't retrieve B-tree node buffer") + HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't retrieve B-tree node buffer") shared=H5RC_GET_OBJ(bt->rc_shared); HDassert(shared); if (NULL==(bt->native=H5FL_BLK_MALLOC(native_block,shared->sizeof_keys)) || @@ -536,9 +536,6 @@ H5B_dest(H5F_t UNUSED *f, H5B_t *bt) */ assert(bt); - /* Verify that node is clean */ - assert(bt->cache_info.is_dirty==0); - H5FL_SEQ_FREE(haddr_t,bt->child); H5FL_BLK_FREE(native_block,bt->native); H5RC_DEC(bt->rc_shared); @@ -1061,6 +1061,7 @@ H5FD_open(const char *name, unsigned flags, hid_t fapl_id, haddr_t maxaddr) HGOTO_ERROR(H5E_VFL, H5E_CANTINC, NULL, "unable to increment ref count on VFL driver") file->cls = driver; file->maxaddr = maxaddr; + file->reserved_alloc = 0; HDmemset(file->fl, 0, sizeof(file->fl)); if(H5P_get(plist, H5F_ACS_META_BLOCK_SIZE_NAME, &(meta_block_size)) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, NULL, "can't get meta data block size") diff --git a/src/H5FDcore.c b/src/H5FDcore.c index 95783aa..eecd336 100644 --- a/src/H5FDcore.c +++ b/src/H5FDcore.c @@ -382,7 +382,8 @@ done: */ static H5FD_t * H5FD_core_open(const char *name, unsigned UNUSED flags, hid_t fapl_id, - haddr_t maxaddr) + haddr_t + maxaddr) { H5FD_core_t *file=NULL; H5FD_core_fapl_t *fa=NULL; diff --git a/src/H5FDpublic.h b/src/H5FDpublic.h index 3454f28..596e0f1 100644 --- a/src/H5FDpublic.h +++ b/src/H5FDpublic.h @@ -187,6 +187,7 @@ struct H5FD_t { unsigned long feature_flags; /* VFL Driver feature Flags */ hsize_t threshold; /* Threshold for alignment */ hsize_t alignment; /* Allocation alignment */ + hsize_t reserved_alloc; /* Space reserved for later alloc calls */ /* Metadata aggregation fields */ hsize_t def_meta_block_size; /* Metadata allocation @@ -955,8 +955,14 @@ H5HG_insert (H5F_t *f, hid_t dxpl_id, size_t size, void *obj, H5HG_t *hobj/*out* * we can extend any of the collections to make enough room. */ if (!found) { + size_t new_need; + for (cwfsno=0; cwfsno<f->shared->ncwfs; cwfsno++) { - if((f->shared->cwfs[cwfsno]->size+need)<=H5HG_MAXSIZE && H5MF_can_extend(f,H5FD_MEM_GHEAP,f->shared->cwfs[cwfsno]->addr,(hsize_t)f->shared->cwfs[cwfsno]->size,(hsize_t)need)) { + new_need = need; + new_need -= f->shared->cwfs[cwfsno]->obj[0].size; + new_need = MAX(f->shared->cwfs[cwfsno]->size, new_need); + + if((f->shared->cwfs[cwfsno]->size+need)<=H5HG_MAXSIZE && H5MF_can_extend(f,H5FD_MEM_GHEAP,f->shared->cwfs[cwfsno]->addr,(hsize_t)f->shared->cwfs[cwfsno]->size,(hsize_t)new_need)) { if(H5HG_extend(f,f->shared->cwfs[cwfsno],size)<0) HGOTO_ERROR (H5E_HEAP, H5E_CANTINIT, FAIL, "unable to extend global heap collection"); addr = f->shared->cwfs[cwfsno]->addr; @@ -155,6 +155,7 @@ H5HL_create(H5F_t *f, hid_t dxpl_id, size_t size_hint, haddr_t *addr_p/*out*/) heap->addr = *addr_p + (hsize_t)sizeof_hdr; heap->disk_alloc = size_hint; heap->mem_alloc = size_hint; + heap->disk_resrv = 0; if (NULL==(heap->chunk = H5FL_BLK_CALLOC(heap_chunk,(sizeof_hdr + size_hint)))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed"); @@ -253,7 +254,7 @@ H5HL_load(H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * udata1, /* Allocate space in memory for the heap */ if (NULL==(heap = H5FL_CALLOC(H5HL_t))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); - + /* heap data size */ H5F_DECODE_LENGTH(f, p, heap->disk_alloc); heap->mem_alloc = heap->disk_alloc; @@ -337,6 +338,13 @@ H5HL_minimize_heap_space(H5F_t *f, hid_t dxpl_id, H5HL_t *heap) sizeof_hdr = H5HL_SIZEOF_HDR(f); /* cache H5HL header size for file */ /* + * When the heap is being flushed to disk, release the file space reserved + * for it. + */ + H5MF_free_reserved(f, heap->disk_resrv); + heap->disk_resrv = 0; + + /* * Check to see if we can reduce the size of the heap in memory by * eliminating free blocks at the tail of the buffer before flushing the * buffer out. @@ -406,8 +414,9 @@ H5HL_minimize_heap_space(H5F_t *f, hid_t dxpl_id, H5HL_t *heap) assert(last_fl->size == H5HL_ALIGN(last_fl->size)); } - /* Resize the memory buffer */ + /* Resize the memory buffer and reserved space in file */ if (new_mem_size != heap->mem_alloc) { + heap->mem_alloc = new_mem_size; heap->chunk = H5FL_BLK_REALLOC(heap_chunk, heap->chunk, (sizeof_hdr + new_mem_size)); @@ -939,6 +948,7 @@ H5HL_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, size_t buf_size, const void * size_t offset = 0; size_t need_size, old_size, need_more; hbool_t found; + size_t disk_resrv; /* Amount of additional space to reserve in file */ size_t sizeof_hdr; /* Cache H5HL header size for file */ size_t ret_value; /* Return value */ @@ -1004,6 +1014,19 @@ H5HL_insert(H5F_t *f, hid_t dxpl_id, haddr_t addr, size_t buf_size, const void * if (found==FALSE) { need_more = MAX3(need_size, heap->mem_alloc, H5HL_SIZEOF_FREE(f)); + /* Reserve space in the file to hold the increased heap size + */ + if( heap->disk_resrv == heap->mem_alloc) + disk_resrv = need_more; + else + disk_resrv = heap->mem_alloc + need_more - heap->disk_resrv; + + if( H5MF_reserve(f, disk_resrv) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, (size_t)(-1), "unable to reserve space in file"); + + /* Update heap's record of how much space it has reserved */ + heap->disk_resrv += disk_resrv; + if (max_fl && max_fl->offset + max_fl->size == heap->mem_alloc) { /* * Increase the size of the maximum free block. diff --git a/src/H5HLpkg.h b/src/H5HLpkg.h index 8b099cc..1a61866 100644 --- a/src/H5HLpkg.h +++ b/src/H5HLpkg.h @@ -67,6 +67,7 @@ struct H5HL_t { haddr_t addr; /*address of data */ size_t disk_alloc; /*data bytes allocated on disk */ size_t mem_alloc; /*data bytes allocated in mem */ + size_t disk_resrv; /*data bytes "reserved" on disk */ uint8_t *chunk; /*the chunk, including header */ H5HL_free_t *freelist; /*the free list */ }; @@ -78,10 +78,13 @@ H5MF_alloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, hsize_t size) /* Fail if we don't have write access */ if (0==(f->intent & H5F_ACC_RDWR)) HGOTO_ERROR(H5E_RESOURCE, H5E_CANTINIT, HADDR_UNDEF, "file is read-only"); + /* Check that the file can address the new space */ + if( H5MF_alloc_overflow(f, size) != 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "not enough address space in file"); /* Allocate space from the virtual file layer */ if (HADDR_UNDEF==(ret_value=H5FD_alloc(f->shared->lf, type, dxpl_id, size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, HADDR_UNDEF, "file allocation failed"); + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, HADDR_UNDEF, "file allocation failed"); /* Convert absolute file address to relative file address */ assert(ret_value>=f->shared->base_addr); @@ -192,6 +195,11 @@ H5MF_realloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t old_addr, hsize_t /* Convert old relative address to absolute address */ old_addr += f->shared->base_addr; + /* Check that the file can address the new space. */ + /* In the worst case, this means adding new_size bytes to the end of the file. */ + if( H5MF_alloc_overflow(f, new_size) != 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, HADDR_UNDEF, "not enough address space in file"); + /* Reallocate memory from the virtual file layer */ ret_value = H5FD_realloc(f->shared->lf, type, dxpl_id, old_addr, old_size, new_size); @@ -208,6 +216,133 @@ done: FUNC_LEAVE_NOAPI(ret_value); } + +/*------------------------------------------------------------------------- + * Function: H5MF_reserve + * + * Purpose: Sets aside file space that has not yet been allocated, but will + * be (or might be in the worst case). This number is used to + * ensure that there is room in the file when it is flushed to disk. + * + * Nothing changes (and no error is generated) if the file is opened + * as read-only. + * + * Return: Success: 0 + * + * Failure: negative + * + * Programmer: James Laird + * Nat Furrer + * Thursday, May 27, 2004 + * + * Modifications: + *------------------------------------------------------------------------- + */ +herr_t H5MF_reserve(H5F_t *f, hsize_t size) +{ + herr_t ret_value = SUCCEED; + FUNC_ENTER_NOAPI(H5MF_reserve, FAIL); + + /* Check arguments */ + assert(f); + + /* Check that there is room in the file to reserve this space */ + if( H5MF_alloc_overflow( f, size ) != 0) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "not enough address space in file"); + + f->shared->lf->reserved_alloc += size; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} + +/*------------------------------------------------------------------------- + * Function: H5MF_free_reserved + * + * Purpose: Releases the file space set aside by H5MF_reserve. This should + * be called immediately before allocating the file space for which + * the space was reserved. + * + * Return: None + * + * Programmer: James Laird + * Nat Furrer + * Thursday, May 27, 2004 + * + * Modifications: + *------------------------------------------------------------------------- + */ +void H5MF_free_reserved(H5F_t *f, hsize_t size) +{ + /* Check arguments */ + assert(f); + + /* If this assert breaks, it means that HDF5 is trying to free file space + * that was never reserved. + */ + assert(size <= f->shared->lf->reserved_alloc); + + f->shared->lf->reserved_alloc -= size; +} + +/*------------------------------------------------------------------------- + * Function: H5MF_alloc_overflow + * + * Purpose: Checks if an allocation of file space would cause an overflow. + * F is the file whose space is being allocated, SIZE is the amount + * of space needed. + * + * Return: 0 if no overflow would result + * 1 if overflow would result (the allocation should not be allowed) + * + * Programmer: James Laird + * Nat Furrer + * Tuesday, June 1, 2004 + * + * Modifications: + *------------------------------------------------------------------------- + */ +int H5MF_alloc_overflow(H5F_t *f, hsize_t size) +{ + unsigned long long space_needed; /* Accumulator variable */ + hsize_t c; + + /* Start with the current end of the file's address. */ + space_needed = f->shared->lf->cls->get_eoa(f->shared->lf); + + /* Subtract the file's base address to get the actual amount of + * space being used: + * (end of allocated space - beginning of allocated space) + */ + assert(f->shared->base_addr < space_needed); + space_needed -= f->shared->base_addr; + + /* Add the amount of space requested for this allocation */ + space_needed += size; + + /* Also add space that is "reserved" for data to be flushed + * to disk (e.g., for object headers and the heap). + * This is the total amount of file space that will be + * allocated. + */ + space_needed += f->shared->lf->reserved_alloc; + + /* Ensure that this final number is less than the file's + * address space. We do this by shifting in multiples + * of 16 bits because some systems will do nothing if + * we shift by the size of a long long (64 bits) all at + * once (<cough> Linux <cough>). Thus, we break one shift + * into several smaller shifts. + */ + for(c=0; c < H5F_SIZEOF_ADDR(f); c += 2) + space_needed = space_needed >> 16; + + if(space_needed != 0) + return 1; + else + return 0; +} + /*------------------------------------------------------------------------- * Function: H5MF_can_extend @@ -234,7 +369,6 @@ htri_t H5MF_can_extend(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hsize_t extra_requested) { htri_t ret_value; /* Return value */ - FUNC_ENTER_NOAPI(H5MF_can_extend, FAIL); /* Convert old relative address to absolute address */ @@ -244,6 +378,10 @@ H5MF_can_extend(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hsize_t e if((ret_value=H5FD_can_extend(f->shared->lf, type, addr, size, extra_requested))<0) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate new file memory"); + /* Make sure there is enough addressable space to satisfy the request */ + if (ret_value == TRUE) + ret_value = !H5MF_alloc_overflow(f, extra_requested); + done: FUNC_LEAVE_NOAPI(ret_value); } /* end H5MF_can_extend() */ @@ -269,12 +407,15 @@ htri_t H5MF_extend(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hsize_t extra_requested) { htri_t ret_value; /* Return value */ - FUNC_ENTER_NOAPI(H5MF_extend, FAIL); /* Convert old relative address to absolute address */ addr += f->shared->base_addr; + /* Make sure there is enough addressable space to satisfy the request */ + if ( H5MF_alloc_overflow(f, extra_requested) ) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate new file memory: out of address space"); + /* Pass the request down to the virtual file layer */ if((ret_value=H5FD_extend(f->shared->lf, type, addr, size, extra_requested))<0) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate new file memory"); diff --git a/src/H5MFprivate.h b/src/H5MFprivate.h index 14b2761..2a129ea 100644 --- a/src/H5MFprivate.h +++ b/src/H5MFprivate.h @@ -48,6 +48,9 @@ H5_DLL herr_t H5MF_xfree(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t addr, hsize_t size); H5_DLL haddr_t H5MF_realloc(H5F_t *f, H5FD_mem_t type, hid_t dxpl_id, haddr_t old_addr, hsize_t old_size, hsize_t new_size); +H5_DLL herr_t H5MF_reserve(H5F_t *f, hsize_t size); +H5_DLL void H5MF_free_reserved(H5F_t *f, hsize_t size); +H5_DLL int H5MF_alloc_overflow(H5F_t *f, hsize_t size); H5_DLL htri_t H5MF_can_extend(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, hsize_t extra_requested); H5_DLL htri_t H5MF_extend(H5F_t *f, H5FD_mem_t type, haddr_t addr, hsize_t size, @@ -75,7 +75,7 @@ static herr_t H5O_remove_real(H5G_entry_t *ent, const H5O_class_t *type, int sequence, hid_t dxpl_id); static unsigned H5O_alloc(H5F_t *f, H5O_t *oh, const H5O_class_t *type, size_t size); -static unsigned H5O_alloc_extend_chunk(H5O_t *oh, unsigned chunkno, size_t size); +static unsigned H5O_alloc_extend_chunk(H5F_t *f, H5O_t *oh, unsigned chunkno, size_t size); static unsigned H5O_alloc_new_chunk(H5F_t *f, H5O_t *oh, size_t size); static herr_t H5O_delete_oh(H5F_t *f, hid_t dxpl_id, H5O_t *oh); static herr_t H5O_delete_mesg(H5F_t *f, hid_t dxpl_id, H5O_mesg_t *mesg); @@ -715,7 +715,11 @@ H5O_flush(H5F_t *f, hid_t dxpl_id, hbool_t destroy, haddr_t addr, H5O_t *oh) assert(cont->chunkno < oh->nchunks); assert(!H5F_addr_defined(oh->chunk[cont->chunkno].addr)); cont->size = oh->chunk[cont->chunkno].size; - if (HADDR_UNDEF==(cont->addr=H5MF_alloc(f, + + /* Free the space we'd reserved in the file to hold this chunk */ + H5MF_free_reserved(f, cont->size); + + if (HADDR_UNDEF==(cont->addr=H5MF_alloc(f, H5FD_MEM_OHDR, dxpl_id, (hsize_t)cont->size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "unable to allocate space for object header data"); oh->chunk[cont->chunkno].addr = cont->addr; @@ -2592,6 +2596,10 @@ done: * that message will be extended with the chunk. Otherwise a * new null message is created. * + * F is the file in which the chunk will be written. It is + * included to ensure that there is enough space to extend + * this chunk. + * * Return: Success: Message index for null message which * is large enough to hold SIZE bytes. * @@ -2610,7 +2618,7 @@ done: *------------------------------------------------------------------------- */ static unsigned -H5O_alloc_extend_chunk(H5O_t *oh, unsigned chunkno, size_t size) +H5O_alloc_extend_chunk(H5F_t *f, H5O_t *oh, unsigned chunkno, size_t size) { unsigned u; unsigned idx; @@ -2638,6 +2646,11 @@ H5O_alloc_extend_chunk(H5O_t *oh, unsigned chunkno, size_t size) delta = MAX (H5O_MIN_SIZE, aligned_size - oh->mesg[idx].raw_size); assert (delta=H5O_ALIGN (delta)); + + /* Reserve space in the file to hold the increased chunk size */ + if( H5MF_reserve(f, delta) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, UFAIL, "unable to reserve space in file"); + oh->mesg[idx].dirty = TRUE; oh->mesg[idx].raw_size += delta; @@ -2665,6 +2678,13 @@ H5O_alloc_extend_chunk(H5O_t *oh, unsigned chunkno, size_t size) } /* end if */ } + /* Reserve space in the file */ + delta = MAX(H5O_MIN_SIZE, aligned_size+H5O_SIZEOF_MSGHDR(f)); + delta = H5O_ALIGN(delta); + + if( H5MF_reserve(f, delta) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, UFAIL, "unable to reserve space in file"); + /* create a new null message */ if (oh->nmesgs >= oh->alloc_nmesgs) { unsigned na = oh->alloc_nmesgs + H5O_NMESGS; @@ -2675,8 +2695,6 @@ H5O_alloc_extend_chunk(H5O_t *oh, unsigned chunkno, size_t size) oh->alloc_nmesgs = na; oh->mesg = x; } - delta = MAX(H5O_MIN_SIZE, aligned_size+H5O_SIZEOF_MSGHDR(f)); - delta = H5O_ALIGN(delta); idx = oh->nmesgs++; oh->mesg[idx].type = H5O_NULL; oh->mesg[idx].dirty = TRUE; @@ -2805,6 +2823,10 @@ H5O_alloc_new_chunk(H5F_t *f, H5O_t *oh, size_t size) size = MAX(H5O_MIN_SIZE, size + H5O_SIZEOF_MSGHDR(f)); assert (size == H5O_ALIGN (size)); + /* Reserve space in the file to hold the new chunk */ + if( H5MF_reserve(f, size) < 0 ) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, UFAIL, "unable to reserve space in file for new chunk"); + /* * Create the new chunk without giving it a file address. */ @@ -2969,7 +2991,7 @@ H5O_alloc(H5F_t *f, H5O_t *oh, const H5O_class_t *type, size_t size) * since we can just increase the size of that chunk. */ for (chunkno = 0; chunkno < oh->nchunks; chunkno++) { - if ((idx = H5O_alloc_extend_chunk(oh, chunkno, size)) != UFAIL) { + if ((idx = H5O_alloc_extend_chunk(f, oh, chunkno, size)) != UFAIL) { break; } H5E_clear(NULL); diff --git a/src/H5Tconv.c b/src/H5Tconv.c index 82c7084..698c99f 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -2685,10 +2685,6 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, nelmts-=safe; } /* end while */ - /* Reset the conversion buffer pointer, so it doesn't get freed */ - if(write_to_file && noop_conv) - conv_buf=NULL; - /* Release the temporary datatype IDs used */ if (tsrc_id >= 0) H5I_dec_ref(tsrc_id); @@ -2701,6 +2697,9 @@ H5T_conv_vlen(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, size_t nelmts, } /* end switch */ done: + /* If the conversion buffer doesn't need to be freed, reset its pointer */ + if(write_to_file && noop_conv) + conv_buf=NULL; /* Release the conversion buffer (always allocated, except on errors) */ if(conv_buf!=NULL) H5FL_BLK_FREE(vlen_seq,conv_buf); |