diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | src/H5Dcompact.c | 119 | ||||
| -rw-r--r-- | src/H5Dcontig.c | 193 | ||||
| -rw-r--r-- | src/H5Dfill.c | 667 | ||||
| -rw-r--r-- | src/H5Dio.c | 252 | ||||
| -rw-r--r-- | src/H5Distore.c | 376 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 41 | ||||
| -rwxr-xr-x | src/Makefile.am | 5 | ||||
| -rw-r--r-- | src/Makefile.in | 5 |
9 files changed, 873 insertions, 786 deletions
@@ -451,6 +451,7 @@ ./src/H5Dcontig.c ./src/H5Ddeprec.c ./src/H5Defl.c +./src/H5Dfill.c ./src/H5Dio.c ./src/H5Distore.c ./src/H5Dmpio.c diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c index a9b2007..38e6cad 100644 --- a/src/H5Dcompact.c +++ b/src/H5Dcompact.c @@ -80,6 +80,8 @@ H5FL_BLK_EXTERN(type_conv); herr_t H5D_compact_fill(H5D_t *dset, hid_t dxpl_id) { + H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ + hbool_t fb_info_init = FALSE; /* Whether the fill value buffer has been initialized */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5D_compact_fill, FAIL) @@ -91,106 +93,27 @@ H5D_compact_fill(H5D_t *dset, hid_t dxpl_id) HDassert(dset->shared->type); HDassert(dset->shared->space); - /* If the fill value is defined, initialize the data buffer with it */ - if(dset->shared->dcpl_cache.fill.buf) { - hssize_t snpoints; /* Number of points in space (for error checking) */ - size_t npoints; /* Number of points in space */ - - /* Get the number of elements in the dataset's dataspace */ - snpoints = H5S_GET_EXTENT_NPOINTS(dset->shared->space); - HDassert(snpoints >= 0); - H5_ASSIGN_OVERFLOW(npoints, snpoints, hssize_t, size_t); - - /* If necessary, convert fill value datatypes (which copies VL components, etc.) */ - if(H5T_detect_class(dset->shared->type, H5T_VLEN) > 0) { - H5T_path_t *tpath; /* Datatype conversion path */ - uint8_t *bkg_buf = NULL; /* Background conversion buffer */ - H5T_t *mem_type; /* Pointer to memory datatype */ - size_t mem_type_size, file_type_size; /* Size of datatype in memory and on disk */ - hid_t mem_tid; /* Memory version of disk datatype */ - - /* Create temporary datatype for conversion operation */ - if(NULL == (mem_type = H5T_copy(dset->shared->type, H5T_COPY_REOPEN))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy file datatype") - if((mem_tid = H5I_register(H5I_DATATYPE, mem_type)) < 0) { - H5T_close(mem_type); - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") - } /* end if */ - - /* Retrieve sizes of memory & file datatypes */ - mem_type_size = H5T_get_size(mem_type); - HDassert(mem_type_size > 0); - file_type_size = H5T_get_size(dset->shared->type); - HDassert((ssize_t)file_type_size == dset->shared->dcpl_cache.fill.size); - - /* Get the datatype conversion path for this operation */ - if(NULL == (tpath = H5T_path_find(dset->shared->type, mem_type, NULL, NULL, dxpl_id, FALSE))) { - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") - } /* end if */ - - /* Allocate a background buffer, if necessary */ - if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (npoints * MAX(mem_type_size, file_type_size))))) { - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - } /* end if */ - - /* Make a copy of the (disk-based) fill value into the compact buffer */ - HDmemcpy(dset->shared->layout.u.compact.buf, dset->shared->dcpl_cache.fill.buf, file_type_size); - - /* Type convert the compact dataset buffer, to copy any VL components */ - if(H5T_convert(tpath, dset->shared->type_id, mem_tid, (size_t)1, (size_t)0, (size_t)0, dset->shared->layout.u.compact.buf, bkg_buf, dxpl_id) < 0) { - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") - } /* end if */ - - /* Replicate the fill value into the cached buffer */ - H5V_array_fill(dset->shared->layout.u.compact.buf, dset->shared->layout.u.compact.buf, mem_type_size, npoints); - - /* Get the inverse datatype conversion path for this operation */ - if(NULL == (tpath = H5T_path_find(mem_type, dset->shared->type, NULL, NULL, dxpl_id, FALSE))) { - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") - } /* end if */ - - /* Allocate or reset the background buffer, if necessary */ - if(H5T_path_bkg(tpath)) { - if(bkg_buf) - HDmemset(bkg_buf, 0, MAX(mem_type_size, file_type_size)); - else { - if(NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (npoints * MAX(mem_type_size, file_type_size))))) { - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - } /* end if */ - } /* end else */ - } /* end if */ - - /* Type convert the compact dataset buffer, to copy any VL components */ - if(H5T_convert(tpath, mem_tid, dset->shared->type_id, npoints, (size_t)0, (size_t)0, dset->shared->layout.u.compact.buf, bkg_buf, dxpl_id) < 0) { - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") - } /* end if */ - - /* Release resources used */ - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - H5I_dec_ref(mem_tid); - } /* end if */ - else - /* Replicate the fill value into the cached buffer */ - H5V_array_fill(dset->shared->layout.u.compact.buf, dset->shared->dcpl_cache.fill.buf, (size_t)dset->shared->dcpl_cache.fill.size, npoints); - - } /* end if */ - else /* If the fill value is default, zero set data buf. */ - HDmemset(dset->shared->layout.u.compact.buf, 0, dset->shared->layout.u.compact.size); + /* Initialize the fill value buffer */ + /* (use the compact dataset storage buffer as the fill value buffer) */ + if(H5D_fill_init(&fb_info, dset->shared->layout.u.compact.buf, FALSE, + NULL, NULL, NULL, NULL, + &dset->shared->dcpl_cache.fill, dset->shared->type, + dset->shared->type_id, (size_t)0, dset->shared->layout.u.compact.size, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info") + fb_info_init = TRUE; + + /* Check for VL datatype & non-default fill value */ + if(fb_info.has_vlen_fill_type) + /* Fill the buffer with VL datatype fill values */ + if(H5D_fill_refill_vl(&fb_info, fb_info.elmts_per_buf, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't refill fill value buffer") done: + /* Release the fill buffer info, if it's been initialized */ + if(fb_info_init) + if(H5D_fill_term(&fb_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_compact_fill() */ diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index 2f41572..ac5cd50 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -70,12 +70,6 @@ static herr_t H5D_contig_write(H5D_t *dset, const H5D_dxpl_cache_t *dxpl_cache, /* Declare a PQ free list to manage the sieve buffer information */ H5FL_BLK_DEFINE(sieve_buf); -/* Declare the free list to manage blocks of non-zero fill-value data */ -H5FL_BLK_DEFINE_STATIC(non_zero_fill); - -/* Declare the free list to manage blocks of zero fill-value data */ -H5FL_BLK_DEFINE_STATIC(zero_fill); - /* Declare extern the free list to manage blocks of type conversion data */ H5FL_BLK_EXTERN(type_conv); @@ -132,11 +126,7 @@ H5D_contig_fill(H5D_t *dset, hid_t dxpl_id) H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ hssize_t snpoints; /* Number of points in space (for error checking) */ size_t npoints; /* Number of points in space */ - size_t ptsperbuf; /* Maximum # of points which fit in the buffer */ - size_t elmt_size; /* Size of each element */ - size_t bufsize = H5D_TEMP_BUF_SIZE; /* Size of buffer to write */ hsize_t offset; /* Offset of dataset */ - void *buf = NULL; /* Buffer for fill value writing */ #ifdef H5_HAVE_PARALLEL MPI_Comm mpi_comm = MPI_COMM_NULL; /* MPI communicator for file */ int mpi_rank = (-1); /* This process's rank */ @@ -144,15 +134,9 @@ H5D_contig_fill(H5D_t *dset, hid_t dxpl_id) hbool_t blocks_written = FALSE; /* Flag to indicate that chunk was actually written */ hbool_t using_mpi = FALSE; /* Flag to indicate that the file is being accessed with an MPI-capable file driver */ #endif /* H5_HAVE_PARALLEL */ - htri_t non_zero_fill_f = (-1); /* Indicate that a non-zero fill-value was used */ - H5T_path_t *fill_to_mem_tpath; /* Datatype conversion path for converting the fill value to the memory buffer */ - H5T_path_t *mem_to_dset_tpath; /* Datatype conversion path for converting the memory buffer to the dataset elements */ - uint8_t *bkg_buf = NULL; /* Background conversion buffer */ - H5T_t *mem_type = NULL; /* Pointer to memory datatype */ - size_t mem_type_size, file_type_size; /* Size of datatype in memory and on disk */ - hid_t mem_tid = (-1); /* Memory version of disk datatype */ - size_t bkg_buf_size; /* Size of background buffer */ - hbool_t has_vlen_fill_type = FALSE; /* Whether the datatype for the fill value has a variable-length component */ + H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ + hbool_t fb_info_init = FALSE; /* Whether the fill value buffer has been initialized */ + hid_t my_dxpl_id; /* DXPL ID to use for this operation */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5D_contig_fill, FAIL) @@ -179,19 +163,21 @@ H5D_contig_fill(H5D_t *dset, hid_t dxpl_id) /* Set the MPI-capable file driver flag */ using_mpi = TRUE; - /* Fill the DXPL cache values for later use */ - if (H5D_get_dxpl_cache(H5AC_ind_dxpl_id,&dxpl_cache)<0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") + /* Use the internal "independent" DXPL */ + my_dxpl_id = H5AC_ind_dxpl_id; } /* end if */ else { #endif /* H5_HAVE_PARALLEL */ - /* Fill the DXPL cache values for later use */ - if (H5D_get_dxpl_cache(dxpl_id,&dxpl_cache)<0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") + /* Use the DXPL we were given */ + my_dxpl_id = dxpl_id; #ifdef H5_HAVE_PARALLEL } /* end else */ #endif /* H5_HAVE_PARALLEL */ + /* Fill the DXPL cache values for later use */ + if(H5D_get_dxpl_cache(my_dxpl_id, &dxpl_cache) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") + /* Initialize storage info for this dataset */ store.contig.dset_addr = dset->shared->layout.u.contig.addr; store.contig.dset_size = dset->shared->layout.u.contig.size; @@ -201,99 +187,13 @@ H5D_contig_fill(H5D_t *dset, hid_t dxpl_id) HDassert(snpoints >= 0); H5_ASSIGN_OVERFLOW(npoints, snpoints, hssize_t, size_t); - /* Fill the buffer with the user's fill value */ - if(dset->shared->dcpl_cache.fill.buf) { - /* Indicate that a non-zero fill buffer will be used */ - non_zero_fill_f = TRUE; - - /* Detect whether the datatype has a VL component */ - has_vlen_fill_type = H5T_detect_class(dset->shared->type, H5T_VLEN); - - /* If necessary, convert fill value datatypes (which copies VL components, etc.) */ - if(has_vlen_fill_type) { - /* Create temporary datatype for conversion operation */ - if(NULL == (mem_type = H5T_copy(dset->shared->type, H5T_COPY_REOPEN))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy file datatype") - if((mem_tid = H5I_register(H5I_DATATYPE, mem_type)) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") - - /* Retrieve sizes of memory & file datatypes */ - mem_type_size = H5T_get_size(mem_type); - HDassert(mem_type_size > 0); - file_type_size = H5T_get_size(dset->shared->type); - HDassert(file_type_size == (size_t)dset->shared->dcpl_cache.fill.size); - - /* If fill value is not library default, use it to set the element size */ - elmt_size = MAX(mem_type_size, file_type_size); - - /* Compute the number of elements that fit within a buffer to write */ - ptsperbuf = MIN(npoints, MAX(1, bufsize / elmt_size)); - bufsize = MAX(bufsize, ptsperbuf * elmt_size); - - /* Allocate temporary buffer */ - if(NULL == (buf = H5FL_BLK_MALLOC(non_zero_fill, bufsize))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer") - - /* Get the datatype conversion path for this operation */ - if(NULL == (fill_to_mem_tpath = H5T_path_find(dset->shared->type, mem_type, NULL, NULL, dxpl_id, FALSE))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") - - /* Get the inverse datatype conversion path for this operation */ - if(NULL == (mem_to_dset_tpath = H5T_path_find(mem_type, dset->shared->type, NULL, NULL, dxpl_id, FALSE))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") - - /* Check if we need to allocate a background buffer */ - if(H5T_path_bkg(fill_to_mem_tpath) || H5T_path_bkg(mem_to_dset_tpath)) { - /* Check for inverse datatype conversion needing a background buffer */ - /* (do this first, since it needs a larger buffer) */ - if(H5T_path_bkg(mem_to_dset_tpath)) - bkg_buf_size = ptsperbuf * elmt_size; - else - bkg_buf_size = elmt_size; - - /* Allocate the background buffer */ - if(NULL == (bkg_buf = H5FL_BLK_MALLOC(type_conv, bkg_buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - } /* end if */ - } /* end if */ - else { - /* If fill value is not library default, use it to set the element size */ - elmt_size = dset->shared->dcpl_cache.fill.size; - - /* Compute the number of elements that fit within a buffer to write */ - ptsperbuf = MIN(npoints, MAX(1, bufsize / elmt_size)); - bufsize = MAX(bufsize, ptsperbuf * elmt_size); - - /* Allocate temporary buffer */ - if(NULL == (buf = H5FL_BLK_MALLOC(non_zero_fill, bufsize))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer") - - /* Replicate the fill value into the cached buffer */ - H5V_array_fill(buf, dset->shared->dcpl_cache.fill.buf, elmt_size, ptsperbuf); - } /* end else */ - } /* end if */ - else { /* Fill the buffer with the default fill value */ - htri_t buf_avail = H5FL_BLK_AVAIL(zero_fill, bufsize); /* Check if there is an already zeroed out buffer available */ - HDassert(buf_avail != FAIL); - - /* Allocate temporary buffer (zeroing it if no buffer is available) */ - if(!buf_avail) - buf = H5FL_BLK_CALLOC(zero_fill, bufsize); - else - buf = H5FL_BLK_MALLOC(zero_fill, bufsize); - if(buf == NULL) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer") - - /* Retrieve size of elements */ - elmt_size = H5T_get_size(dset->shared->type); - HDassert(elmt_size > 0); - - /* Compute the number of elements that fit within a buffer to write */ - ptsperbuf = MIN(npoints, MAX(1, bufsize / elmt_size)); - - /* Indicate that a zero fill buffer was used */ - non_zero_fill_f = FALSE; - } /* end else */ + /* Initialize the fill value buffer */ + if(H5D_fill_init(&fb_info, NULL, FALSE, NULL, NULL, NULL, NULL, + &dset->shared->dcpl_cache.fill, + dset->shared->type, dset->shared->type_id, npoints, + dxpl_cache->max_temp_buf, my_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info") + fb_info_init = TRUE; /* Start at the beginning of the dataset */ offset = 0; @@ -310,33 +210,14 @@ H5D_contig_fill(H5D_t *dset, hid_t dxpl_id) size_t size; /* Size of buffer to write */ /* Compute # of elements and buffer size to write for this iteration */ - curr_points = MIN(ptsperbuf, npoints); - size = curr_points * elmt_size; + curr_points = MIN(fb_info.elmts_per_buf, npoints); + size = curr_points * fb_info.file_elmt_size; /* Check for VL datatype & non-default fill value */ - if(has_vlen_fill_type) { - /* Make a copy of the (disk-based) fill value into the buffer */ - HDmemcpy(buf, dset->shared->dcpl_cache.fill.buf, file_type_size); - - /* Reset first element of background buffer, if necessary */ - if(H5T_path_bkg(fill_to_mem_tpath)) - HDmemset(bkg_buf, 0, elmt_size); - - /* Type convert the dataset buffer, to copy any VL components */ - if(H5T_convert(fill_to_mem_tpath, dset->shared->type_id, mem_tid, (size_t)1, (size_t)0, (size_t)0, buf, bkg_buf, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") - - /* Replicate the fill value into the cached buffer */ - H5V_array_fill(buf, buf, mem_type_size, curr_points); - - /* Reset the entire background buffer, if necessary */ - if(H5T_path_bkg(mem_to_dset_tpath)) - HDmemset(bkg_buf, 0, bkg_buf_size); - - /* Type convert the dataset buffer, to copy any VL components */ - if(H5T_convert(mem_to_dset_tpath, mem_tid, dset->shared->type_id, curr_points, (size_t)0, (size_t)0, buf, bkg_buf, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") - } /* end if */ + if(fb_info.has_vlen_fill_type) + /* Re-fill the buffer to use for this I/O operation */ + if(H5D_fill_refill_vl(&fb_info, curr_points, my_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't refill fill value buffer") #ifdef H5_HAVE_PARALLEL /* Check if this file is accessed with an MPI-capable file driver */ @@ -344,7 +225,7 @@ H5D_contig_fill(H5D_t *dset, hid_t dxpl_id) /* Write the chunks out from only one process */ /* !! Use the internal "independent" DXPL!! -QAK */ if(H5_PAR_META_WRITE == mpi_rank) - if(H5D_contig_write(dset, dxpl_cache, H5AC_ind_dxpl_id, &store, offset, size, buf) < 0) + if(H5D_contig_write(dset, dxpl_cache, my_dxpl_id, &store, offset, size, fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to write fill value to dataset") /* Indicate that blocks are being written */ @@ -353,7 +234,7 @@ H5D_contig_fill(H5D_t *dset, hid_t dxpl_id) else { #endif /* H5_HAVE_PARALLEL */ H5_CHECK_OVERFLOW(size, size_t, hsize_t); - if(H5D_contig_write(dset, dxpl_cache, dxpl_id, &store, offset, size, buf) < 0) + if(H5D_contig_write(dset, dxpl_cache, my_dxpl_id, &store, offset, size, fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to write fill value to dataset") #ifdef H5_HAVE_PARALLEL } /* end else */ @@ -377,24 +258,10 @@ H5D_contig_fill(H5D_t *dset, hid_t dxpl_id) #endif /* H5_HAVE_PARALLEL */ done: - /* Free the buffer for fill values */ - if(buf) { - HDassert(non_zero_fill_f >= 0); - if(non_zero_fill_f) - H5FL_BLK_FREE(non_zero_fill, buf); - else - H5FL_BLK_FREE(zero_fill, buf); - } /* end if */ - - /* Free other resources for vlen fill values */ - if(has_vlen_fill_type) { - if(mem_tid > 0) - H5I_dec_ref(mem_tid); - else if(mem_type) - H5T_close(mem_type); - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - } /* end if */ + /* Release the fill buffer info, if it's been initialized */ + if(fb_info_init) + if(H5D_fill_term(&fb_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_contig_fill() */ diff --git a/src/H5Dfill.c b/src/H5Dfill.c new file mode 100644 index 0000000..89e3a6b --- /dev/null +++ b/src/H5Dfill.c @@ -0,0 +1,667 @@ +/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * + * Copyright by The HDF Group. * + * Copyright by the Board of Trustees of the University of Illinois. * + * All rights reserved. * + * * + * This file is part of HDF5. The full HDF5 copyright notice, including * + * terms governing use, modification, and redistribution, is contained in * + * the files COPYING and Copyright.html. COPYING can be found at the root * + * of the source code distribution tree; Copyright.html can be found at the * + * root level of an installed copy of the electronic HDF5 document set and * + * is linked from the top-level documents page. It can also be found at * + * http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have * + * access to either file, you may request a copy from help@hdfgroup.org. * + * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */ + +/*------------------------------------------------------------------------- + * + * Created: H5Dfill.c + * Jun 19 2007 + * Quincey Koziol <koziol@hdfgroup.org> + * + * Purpose: Fill value operations for datasets + * + *------------------------------------------------------------------------- + */ + +/****************/ +/* Module Setup */ +/****************/ + +#define H5D_PACKAGE /*suppress error about including H5Dpkg */ + +/***********/ +/* Headers */ +/***********/ +#include "H5private.h" /* Generic Functions */ +#include "H5Dpkg.h" /* Dataset functions */ +#include "H5Eprivate.h" /* Error handling */ +#include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ +#include "H5Vprivate.h" /* Vector and array functions */ + + +/****************/ +/* Local Macros */ +/****************/ + + +/******************/ +/* Local Typedefs */ +/******************/ + + +/********************/ +/* Package Typedefs */ +/********************/ + + +/********************/ +/* Local Prototypes */ +/********************/ + + +/*********************/ +/* Package Variables */ +/*********************/ + +/* Declare a free list to manage blocks of single datatype element data */ +H5FL_BLK_DEFINE(type_elem); + +/* Declare extern the free list to manage blocks of type conversion data */ +H5FL_BLK_EXTERN(type_conv); + + +/*****************************/ +/* Library Private Variables */ +/*****************************/ + + +/*******************/ +/* Local Variables */ +/*******************/ + +/* Declare the free list to manage blocks of non-zero fill-value data */ +H5FL_BLK_DEFINE_STATIC(non_zero_fill); + +/* Declare the free list to manage blocks of zero fill-value data */ +H5FL_BLK_DEFINE_STATIC(zero_fill); + + +/*-------------------------------------------------------------------------- + NAME + H5Dfill + PURPOSE + Fill a selection in memory with a value + USAGE + herr_t H5Dfill(fill, fill_type, space, buf, buf_type) + const void *fill; IN: Pointer to fill value to use + hid_t fill_type_id; IN: Datatype of the fill value + void *buf; IN/OUT: Memory buffer to fill selection within + hid_t buf_type_id; IN: Datatype of the elements in buffer + hid_t space_id; IN: Dataspace describing memory buffer & + containing selection to use. + RETURNS + Non-negative on success/Negative on failure. + DESCRIPTION + Use the selection in the dataspace to fill elements in a memory buffer. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + If "fill" parameter is NULL, use all zeros as fill value + EXAMPLES + REVISION LOG +--------------------------------------------------------------------------*/ +herr_t +H5Dfill(const void *fill, hid_t fill_type_id, void *buf, hid_t buf_type_id, hid_t space_id) +{ + H5S_t *space; /* Dataspace */ + H5T_t *fill_type; /* Fill-value datatype */ + H5T_t *buf_type; /* Buffer datatype */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_API(H5Dfill, FAIL) + H5TRACE5("e", "*xi*xii", fill, fill_type_id, buf, buf_type_id, space_id); + + /* Check args */ + if(buf == NULL) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid buffer") + if(NULL == (space = H5I_object_verify(space_id, H5I_DATASPACE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "not a dataspace") + if(NULL == (fill_type = H5I_object_verify(fill_type_id, H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "not a datatype") + if(NULL == (buf_type = H5I_object_verify(buf_type_id, H5I_DATATYPE))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "not a datatype") + + /* Fill the selection in the memory buffer */ + if(H5D_fill(fill, fill_type, buf, buf_type, space, H5AC_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed") + +done: + FUNC_LEAVE_API(ret_value) +} /* H5Dfill() */ + + +/*-------------------------------------------------------------------------- + NAME + H5D_fill + PURPOSE + Fill a selection in memory with a value (internal version) + USAGE + herr_t H5D_fill(fill, fill_type, buf, buf_type, space) + const void *fill; IN: Pointer to fill value to use + H5T_t *fill_type; IN: Datatype of the fill value + void *buf; IN/OUT: Memory buffer to fill selection within + H5T_t *buf_type; IN: Datatype of the elements in buffer + H5S_t *space; IN: Dataspace describing memory buffer & + containing selection to use. + RETURNS + Non-negative on success/Negative on failure. + DESCRIPTION + Use the selection in the dataspace to fill elements in a memory buffer. + GLOBAL VARIABLES + COMMENTS, BUGS, ASSUMPTIONS + If "fill" parameter is NULL, use all zeros as fill value. If "fill_type" + parameter is NULL, use "buf_type" for the fill value datatype. + EXAMPLES + REVISION LOG + Raymond Lu - 20 March 2007 + If there's VL type of data, the address of the data is copied multiple + times into the buffer, causing some trouble when the data is released. + Instead, make multiple copies of fill value first, then do conversion + on each element so that each of them has a copy of the VL data. +--------------------------------------------------------------------------*/ +herr_t +H5D_fill(const void *fill, const H5T_t *fill_type, void *buf, + const H5T_t *buf_type, const H5S_t *space, hid_t dxpl_id) +{ + uint8_t *tconv_buf = NULL; /* Data type conv buffer */ + uint8_t *bkg_buf = NULL; /* Background conversion buffer */ + uint8_t *tmp_buf = NULL; /* Temp conversion buffer */ + hid_t src_id = -1, dst_id = -1; /* Temporary type IDs */ + size_t dst_type_size; /* Size of destination type*/ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_fill) + + /* Check args */ + HDassert(fill_type); + HDassert(buf); + HDassert(buf_type); + HDassert(space); + + /* Make sure the dataspace has an extent set (or is NULL) */ + if(!(H5S_has_extent(space))) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "dataspace extent has not been set") + + /* Get the memory datatype size */ + dst_type_size = H5T_get_size(buf_type); + + /* If there's no fill value, just use zeros */ + if(fill == NULL) { + /* Allocate space & initialize conversion buffer to zeros */ + if(NULL == (tconv_buf = H5FL_BLK_CALLOC(type_elem, dst_type_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Fill the selection in the memory buffer */ + if(H5S_select_fill(tconv_buf, dst_type_size, space, buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed") + } /* end if */ + else { + H5T_path_t *tpath; /* Conversion path information */ + size_t src_type_size; /* Size of source type */ + size_t buf_size; /* Desired buffer size */ + + /* Get the file datatype size */ + src_type_size = H5T_get_size(fill_type); + + /* Get the maximum buffer size needed and allocate it */ + buf_size = MAX(src_type_size, dst_type_size); + + /* Set up type conversion function */ + if(NULL == (tpath = H5T_path_find(fill_type, buf_type, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert between src and dest datatype") + + /* Construct source & destination datatype IDs, if we will need them */ + if(!H5T_path_noop(tpath)) { + if((src_id = H5I_register(H5I_DATATYPE, H5T_copy(fill_type, H5T_COPY_ALL))) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, "unable to register types for conversion") + + if((dst_id = H5I_register(H5I_DATATYPE, H5T_copy(buf_type, H5T_COPY_ALL))) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, "unable to register types for conversion") + } /* end if */ + + /* If there's VL type of data, make multiple copies of fill value first, + * then do conversion on each element so that each of them has a copy + * of the VL data. + */ + if(TRUE == H5T_detect_class(fill_type, H5T_VLEN)) { + H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ + H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ + H5S_sel_iter_t mem_iter; /* Memory selection iteration info */ + hssize_t nelmts; /* Number of data elements */ + + /* Get the number of elements in the selection */ + nelmts = H5S_GET_SELECT_NPOINTS(space); + HDassert(nelmts >= 0); + H5_CHECK_OVERFLOW(nelmts, hssize_t, size_t); + + /* Allocate a temporary buffer */ + if(NULL == (tmp_buf = H5FL_BLK_MALLOC(type_conv, (size_t)nelmts * buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Allocate a background buffer, if necessary */ + if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (size_t)nelmts * buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Replicate the file's fill value into the temporary buffer */ + H5V_array_fill(tmp_buf, fill, src_type_size, (size_t)nelmts); + + /* Convert from file's fill value into memory form */ + if(H5T_convert(tpath, src_id, dst_id, (size_t)nelmts, (size_t)0, (size_t)0, tmp_buf, bkg_buf, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") + + /* Fill the DXPL cache values for later use */ + if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") + + /* Create a selection iterator for scattering the elements to memory buffer */ + if(H5S_select_iter_init(&mem_iter, space, dst_type_size) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information") + + /* Scatter the data into memory */ + if(H5D_select_mscat(tmp_buf, space, &mem_iter, (size_t)nelmts, dxpl_cache, buf/*out*/) < 0) { + H5S_SELECT_ITER_RELEASE(&mem_iter); + HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "scatter failed") + } /* end if */ + + /* Release the selection iterator */ + if(H5S_SELECT_ITER_RELEASE(&mem_iter) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") + } /* end if */ + else { + const uint8_t *fill_buf; /* Buffer to use for writing fill values */ + + /* Convert disk buffer into memory buffer */ + if(!H5T_path_noop(tpath)) { + /* Allocate space for conversion buffer */ + if(NULL == (tconv_buf = H5FL_BLK_MALLOC(type_elem, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Copy the user's data into the buffer for conversion */ + HDmemcpy(tconv_buf, fill, src_type_size); + + /* If there's no VL type of data, do conversion first then fill the data into + * the memory buffer. */ + if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_elem, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + + /* Perform datatype conversion */ + if(H5T_convert(tpath, src_id, dst_id, (size_t)1, (size_t)0, (size_t)0, tconv_buf, bkg_buf, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") + + /* Point at temporary buffer */ + fill_buf = tconv_buf; + } /* end if */ + else + fill_buf = fill; + + /* Fill the selection in the memory buffer */ + if(H5S_select_fill(fill_buf, dst_type_size, space, buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed") + } /* end else */ + } /* end else */ + +done: + if(src_id != (-1) && H5I_dec_ref(src_id) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(dst_id != (-1) && H5I_dec_ref(dst_id) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(tmp_buf) + H5FL_BLK_FREE(type_conv, tmp_buf); + if(tconv_buf) + H5FL_BLK_FREE(type_elem, tconv_buf); + if(bkg_buf) { + if(TRUE == H5T_detect_class(fill_type, H5T_VLEN)) + H5FL_BLK_FREE(type_conv, bkg_buf); + else + H5FL_BLK_FREE(type_elem, bkg_buf); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* H5D_fill() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_fill_init + * + * Purpose: Initialize buffer filling operation + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * June 21, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf, + hbool_t alloc_vl_during_refill, + H5MM_allocate_t alloc_func, void *alloc_info, + H5MM_free_t free_func, void *free_info, + const H5O_fill_t *fill, const H5T_t *dset_type, hid_t dset_type_id, + size_t total_nelmts, size_t max_buf_size, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_fill_init, FAIL) + + /* Check args */ + HDassert(fb_info); + HDassert(fill); + HDassert(dset_type); + HDassert(dset_type_id > 0); + + /* Reset fill buffer information */ + HDmemset(fb_info, 0, sizeof(*fb_info)); + + /* Cache constant information from the dataset */ + fb_info->fill = fill; + fb_info->file_type = dset_type; + fb_info->file_tid = dset_type_id; + fb_info->alloc_vl_during_refill = alloc_vl_during_refill; + fb_info->fill_alloc_func = alloc_func; + fb_info->fill_alloc_info = alloc_info; + fb_info->fill_free_func = free_func; + fb_info->fill_free_info = free_info; + + /* Fill the buffer with the user's fill value */ + if(fill->buf) { + /* Detect whether the datatype has a VL component */ + fb_info->has_vlen_fill_type = H5T_detect_class(dset_type, H5T_VLEN); + + /* If necessary, convert fill value datatypes (which copies VL components, etc.) */ + if(fb_info->has_vlen_fill_type) { + /* Create temporary datatype for conversion operation */ + if(NULL == (fb_info->mem_type = H5T_copy(dset_type, H5T_COPY_REOPEN))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy file datatype") + if((fb_info->mem_tid = H5I_register(H5I_DATATYPE, fb_info->mem_type)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") + + /* Retrieve sizes of memory & file datatypes */ + fb_info->mem_elmt_size = H5T_get_size(fb_info->mem_type); + HDassert(fb_info->mem_elmt_size > 0); + fb_info->file_elmt_size = H5T_get_size(dset_type); + HDassert(fb_info->file_elmt_size == (size_t)fill->size); + + /* If fill value is not library default, use it to set the element size */ + fb_info->max_elmt_size = MAX(fb_info->mem_elmt_size, fb_info->file_elmt_size); + + /* Compute the number of elements that fit within a buffer to write */ + if(total_nelmts > 0) + fb_info->elmts_per_buf = MIN(total_nelmts, MAX(1, (max_buf_size / fb_info->max_elmt_size))); + else + fb_info->elmts_per_buf = max_buf_size / fb_info->max_elmt_size; + HDassert(fb_info->elmts_per_buf > 0); + + /* Compute the buffer size to use */ + fb_info->fill_buf_size = MIN(max_buf_size, (fb_info->elmts_per_buf * fb_info->max_elmt_size)); + + /* Allocate fill buffer */ + if(caller_fill_buf) { + fb_info->fill_buf = caller_fill_buf; + fb_info->use_caller_fill_buf = TRUE; + } /* end if */ + else { + if(alloc_vl_during_refill) + fb_info->fill_buf = NULL; + else { + if(alloc_func) + fb_info->fill_buf = alloc_func(fb_info->fill_buf_size, alloc_info); + else + fb_info->fill_buf = H5FL_BLK_MALLOC(non_zero_fill, fb_info->fill_buf_size); + if(NULL == fb_info->fill_buf) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer") + } /* end else */ + } /* end else */ + + /* Get the datatype conversion path for this operation */ + if(NULL == (fb_info->fill_to_mem_tpath = H5T_path_find(dset_type, fb_info->mem_type, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") + + /* Get the inverse datatype conversion path for this operation */ + if(NULL == (fb_info->mem_to_dset_tpath = H5T_path_find(fb_info->mem_type, dset_type, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") + + /* Check if we need to allocate a background buffer */ + if(H5T_path_bkg(fb_info->fill_to_mem_tpath) || H5T_path_bkg(fb_info->mem_to_dset_tpath)) { + /* Check for inverse datatype conversion needing a background buffer */ + /* (do this first, since it needs a larger buffer) */ + if(H5T_path_bkg(fb_info->mem_to_dset_tpath)) + fb_info->bkg_buf_size = fb_info->elmts_per_buf * fb_info->max_elmt_size; + else + fb_info->bkg_buf_size = fb_info->max_elmt_size; + + /* Allocate the background buffer */ + if(NULL == (fb_info->bkg_buf = H5FL_BLK_MALLOC(type_conv, fb_info->bkg_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + } /* end if */ + } /* end if */ + else { + /* If fill value is not library default, use it to set the element size */ + fb_info->max_elmt_size = fb_info->file_elmt_size = fb_info->mem_elmt_size = fill->size; + + /* Compute the number of elements that fit within a buffer to write */ + if(total_nelmts > 0) + fb_info->elmts_per_buf = MIN(total_nelmts, MAX(1, (max_buf_size / fb_info->max_elmt_size))); + else + fb_info->elmts_per_buf = max_buf_size / fb_info->max_elmt_size; + HDassert(fb_info->elmts_per_buf > 0); + + /* Compute the buffer size to use */ + fb_info->fill_buf_size = MIN(max_buf_size, fb_info->elmts_per_buf * fb_info->max_elmt_size); + + /* Allocate temporary buffer */ + if(caller_fill_buf) { + fb_info->fill_buf = caller_fill_buf; + fb_info->use_caller_fill_buf = TRUE; + } /* end if */ + else { + if(alloc_func) + fb_info->fill_buf = alloc_func(fb_info->fill_buf_size, alloc_info); + else + fb_info->fill_buf = H5FL_BLK_MALLOC(non_zero_fill, fb_info->fill_buf_size); + if(NULL == fb_info->fill_buf) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer") + } /* end else */ + + /* Replicate the fill value into the cached buffer */ + H5V_array_fill(fb_info->fill_buf, fill->buf, fb_info->max_elmt_size, fb_info->elmts_per_buf); + } /* end else */ + } /* end if */ + else { /* Fill the buffer with the default fill value */ + /* Retrieve size of elements */ + fb_info->max_elmt_size = fb_info->file_elmt_size = fb_info->mem_elmt_size = H5T_get_size(dset_type); + HDassert(fb_info->max_elmt_size > 0); + + /* Compute the number of elements that fit within a buffer to write */ + if(total_nelmts > 0) + fb_info->elmts_per_buf = MIN(total_nelmts, MAX(1, (max_buf_size / fb_info->max_elmt_size))); + else + fb_info->elmts_per_buf = max_buf_size / fb_info->max_elmt_size; + HDassert(fb_info->elmts_per_buf > 0); + + /* Compute the buffer size to use */ + fb_info->fill_buf_size = MIN(max_buf_size, (fb_info->elmts_per_buf * fb_info->max_elmt_size)); + + /* Use (and zero) caller's buffer, if provided */ + if(caller_fill_buf) { + fb_info->fill_buf = caller_fill_buf; + fb_info->use_caller_fill_buf = TRUE; + + HDmemset(fb_info->fill_buf, 0, fb_info->fill_buf_size); + } /* end if */ + else { + if(alloc_func) { + fb_info->fill_buf = alloc_func(fb_info->fill_buf_size, alloc_info); + + HDmemset(fb_info->fill_buf, 0, fb_info->fill_buf_size); + } /* end if */ + else { + htri_t buf_avail = H5FL_BLK_AVAIL(zero_fill, fb_info->fill_buf_size); /* Check if there is an already zeroed out buffer available */ + HDassert(buf_avail != FAIL); + + /* Allocate temporary buffer (zeroing it if no buffer is available) */ + if(!buf_avail) + fb_info->fill_buf = H5FL_BLK_CALLOC(zero_fill, fb_info->fill_buf_size); + else + fb_info->fill_buf = H5FL_BLK_MALLOC(zero_fill, fb_info->fill_buf_size); + } /* end else */ + if(fb_info->fill_buf == NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer") + } /* end else */ + } /* end else */ + +done: + /* Cleanup on error */ + if(ret_value < 0) + if(H5D_fill_term(fb_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_fill_init() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_fill_refill_vl + * + * Purpose: Refill fill value buffer that contains VL-datatype fill values + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * June 21, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_fill_refill_vl(H5D_fill_buf_info_t *fb_info, size_t nelmts, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_fill_refill_vl, FAIL) + + /* Check args */ + HDassert(fb_info); + HDassert(fb_info->has_vlen_fill_type); + + /* Check if we should allocate the fill buffer now */ + if(fb_info->alloc_vl_during_refill) { + if(fb_info->fill_alloc_func) + fb_info->fill_buf = fb_info->fill_alloc_func(fb_info->fill_buf_size, fb_info->fill_alloc_info); + else + fb_info->fill_buf = H5FL_BLK_MALLOC(non_zero_fill, fb_info->fill_buf_size); + if(NULL == fb_info->fill_buf) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for fill buffer") + } /* end if */ + + /* Make a copy of the (disk-based) fill value into the buffer */ + HDmemcpy(fb_info->fill_buf, fb_info->fill->buf, fb_info->file_elmt_size); + + /* Reset first element of background buffer, if necessary */ + if(H5T_path_bkg(fb_info->fill_to_mem_tpath)) + HDmemset(fb_info->bkg_buf, 0, fb_info->max_elmt_size); + + /* Type convert the dataset buffer, to copy any VL components */ + if(H5T_convert(fb_info->fill_to_mem_tpath, fb_info->file_tid, fb_info->mem_tid, (size_t)1, (size_t)0, (size_t)0, fb_info->fill_buf, fb_info->bkg_buf, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") + + /* Replicate the fill value into the cached buffer */ + H5V_array_fill(fb_info->fill_buf, fb_info->fill_buf, fb_info->mem_elmt_size, nelmts); + + /* Reset the entire background buffer, if necessary */ + if(H5T_path_bkg(fb_info->mem_to_dset_tpath)) + HDmemset(fb_info->bkg_buf, 0, fb_info->bkg_buf_size); + + /* Type convert the dataset buffer, to copy any VL components */ + if(H5T_convert(fb_info->mem_to_dset_tpath, fb_info->mem_tid, fb_info->file_tid, nelmts, (size_t)0, (size_t)0, fb_info->fill_buf, fb_info->bkg_buf, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_fill_refill_vl() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_fill_release + * + * Purpose: Release fill value buffer + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * June 22, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_fill_release(H5D_fill_buf_info_t *fb_info) +{ + FUNC_ENTER_NOAPI_NOFUNC(H5D_fill_release) + + /* Check args */ + HDassert(fb_info); + HDassert(fb_info->fill); + + /* Free the buffer for fill values */ + if(!fb_info->use_caller_fill_buf && fb_info->fill_buf) { + if(fb_info->fill_free_func) + fb_info->fill_free_func(fb_info->fill_buf, fb_info->fill_free_info); + else { + if(fb_info->fill->buf) + H5FL_BLK_FREE(non_zero_fill, fb_info->fill_buf); + else + H5FL_BLK_FREE(zero_fill, fb_info->fill_buf); + } /* end else */ + fb_info->fill_buf = NULL; + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_fill_release() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_fill_term + * + * Purpose: Release fill value buffer info + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * June 21, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_fill_term(H5D_fill_buf_info_t *fb_info) +{ + FUNC_ENTER_NOAPI_NOFUNC(H5D_fill_term) + + /* Check args */ + HDassert(fb_info); + + /* Free the buffer for fill values */ + H5D_fill_release(fb_info); + + /* Free other resources for vlen fill values */ + if(fb_info->has_vlen_fill_type) { + if(fb_info->mem_tid > 0) + H5I_dec_ref(fb_info->mem_tid); + else if(fb_info->mem_type) + H5T_close(fb_info->mem_type); + if(fb_info->bkg_buf) + H5FL_BLK_FREE(type_conv, fb_info->bkg_buf); + } /* end if */ + + FUNC_LEAVE_NOAPI(SUCCEED) +} /* end H5D_fill_term() */ + diff --git a/src/H5Dio.c b/src/H5Dio.c index 5fbe72a..1bf5b04 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -53,8 +53,6 @@ /* Local Prototypes */ /********************/ -static herr_t H5D_fill(const void *fill, const H5T_t *fill_type, void *buf, - const H5T_t *buf_type, const H5S_t *space, hid_t dxpl_id); static herr_t H5D_read(H5D_t *dataset, hid_t mem_type_id, const H5S_t *mem_space, const H5S_t *file_space, hid_t dset_xfer_plist, void *buf/*out*/); @@ -77,11 +75,11 @@ static herr_t H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, const H5T_t *mem_type, const H5S_t *mem_space, const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, const void *buf); -static herr_t H5D_compound_opt_read(hsize_t nelmts, const H5S_t *mem_space, +static herr_t H5D_compound_opt_read(size_t nelmts, const H5S_t *mem_space, H5S_sel_iter_t *iter, const H5D_dxpl_cache_t *dxpl_cache, hid_t src_id, hid_t dst_id, H5T_subset_t subset, void *data_buf, void *user_buf/*out*/); -static herr_t H5D_compound_opt_write(hsize_t nelmts, hid_t src_id, hid_t dst_id, +static herr_t H5D_compound_opt_write(size_t nelmts, hid_t src_id, hid_t dst_id, void *data_buf); #ifdef H5_HAVE_PARALLEL @@ -113,9 +111,6 @@ static herr_t H5D_chunk_mem_cb(void *elem, hid_t type_id, unsigned ndims, /* Local Variables */ /*******************/ -/* Declare a free list to manage blocks of single datatype element data */ -H5FL_BLK_DEFINE(type_elem); - /* Declare a free list to manage blocks of type conversion data */ H5FL_BLK_DEFINE(type_conv); @@ -132,245 +127,6 @@ H5FL_SEQ_DEFINE_STATIC(hsize_t); /*-------------------------------------------------------------------------- NAME - H5Dfill - PURPOSE - Fill a selection in memory with a value - USAGE - herr_t H5Dfill(fill, fill_type, space, buf, buf_type) - const void *fill; IN: Pointer to fill value to use - hid_t fill_type_id; IN: Datatype of the fill value - void *buf; IN/OUT: Memory buffer to fill selection within - hid_t buf_type_id; IN: Datatype of the elements in buffer - hid_t space_id; IN: Dataspace describing memory buffer & - containing selection to use. - RETURNS - Non-negative on success/Negative on failure. - DESCRIPTION - Use the selection in the dataspace to fill elements in a memory buffer. - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - If "fill" parameter is NULL, use all zeros as fill value - EXAMPLES - REVISION LOG ---------------------------------------------------------------------------*/ -herr_t -H5Dfill(const void *fill, hid_t fill_type_id, void *buf, hid_t buf_type_id, hid_t space_id) -{ - H5S_t *space; /* Dataspace */ - H5T_t *fill_type; /* Fill-value datatype */ - H5T_t *buf_type; /* Buffer datatype */ - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_API(H5Dfill, FAIL) - H5TRACE5("e", "*xi*xii", fill, fill_type_id, buf, buf_type_id, space_id); - - /* Check args */ - if (buf==NULL) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "invalid buffer") - if (NULL == (space=H5I_object_verify(space_id, H5I_DATASPACE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "not a dataspace") - if (NULL == (fill_type=H5I_object_verify(fill_type_id, H5I_DATATYPE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "not a datatype") - if (NULL == (buf_type=H5I_object_verify(buf_type_id, H5I_DATATYPE))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, 0, "not a datatype") - - /* Fill the selection in the memory buffer */ - if(H5D_fill(fill,fill_type,buf,buf_type,space, H5AC_dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed") - -done: - FUNC_LEAVE_API(ret_value) -} /* H5Dfill() */ - - -/*-------------------------------------------------------------------------- - NAME - H5D_fill - PURPOSE - Fill a selection in memory with a value (internal version) - USAGE - herr_t H5D_fill(fill, fill_type, buf, buf_type, space) - const void *fill; IN: Pointer to fill value to use - H5T_t *fill_type; IN: Datatype of the fill value - void *buf; IN/OUT: Memory buffer to fill selection within - H5T_t *buf_type; IN: Datatype of the elements in buffer - H5S_t *space; IN: Dataspace describing memory buffer & - containing selection to use. - RETURNS - Non-negative on success/Negative on failure. - DESCRIPTION - Use the selection in the dataspace to fill elements in a memory buffer. - GLOBAL VARIABLES - COMMENTS, BUGS, ASSUMPTIONS - If "fill" parameter is NULL, use all zeros as fill value. If "fill_type" - parameter is NULL, use "buf_type" for the fill value datatype. - EXAMPLES - REVISION LOG - Raymond Lu - 20 March 2007 - If there's VL type of data, the address of the data is copied multiple - times into the buffer, causing some trouble when the data is released. - Instead, make multiple copies of fill value first, then do conversion - on each element so that each of them has a copy of the VL data. ---------------------------------------------------------------------------*/ -static herr_t -H5D_fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_type, const H5S_t *space, hid_t dxpl_id) -{ - uint8_t *tconv_buf = NULL; /* Data type conv buffer */ - uint8_t *bkg_buf = NULL; /* Background conversion buffer */ - uint8_t *tmp_buf = NULL; /* Temp conversion buffer */ - hid_t src_id = -1, dst_id = -1; /* Temporary type IDs */ - size_t src_type_size; /* Size of source type */ - size_t dst_type_size; /* Size of destination type*/ - size_t buf_size; /* Desired buffer size */ - herr_t ret_value=SUCCEED; /* Return value */ - - FUNC_ENTER_NOAPI_NOINIT(H5D_fill) - - /* Check args */ - HDassert(fill_type); - HDassert(buf); - HDassert(buf_type); - HDassert(space); - - /* Make sure the dataspace has an extent set (or is NULL) */ - if(!(H5S_has_extent(space))) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "dataspace extent has not been set") - - /* Get the memory and file datatype sizes */ - src_type_size = H5T_get_size(fill_type); - dst_type_size = H5T_get_size(buf_type); - - /* Get the maximum buffer size needed and allocate it */ - buf_size = MAX(src_type_size, dst_type_size); - - /* If there's no fill value, just use zeros */ - if(fill == NULL) { - /* Allocate space & initialize conversion buffer to zeros */ - if(NULL == (tconv_buf = H5FL_BLK_CALLOC(type_elem, buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - - /* Fill the selection in the memory buffer */ - if(H5S_select_fill(tconv_buf, dst_type_size, space, buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed") - } /* end if */ - else { - H5T_path_t *tpath; /* Conversion path information */ - - /* Set up type conversion function */ - if(NULL == (tpath = H5T_path_find(fill_type, buf_type, NULL, NULL, dxpl_id, FALSE))) - HGOTO_ERROR(H5E_DATASET, H5E_UNSUPPORTED, FAIL, "unable to convert between src and dest datatype") - - /* Construct source & destination datatype IDs, if we will need them */ - if(!H5T_path_noop(tpath)) { - if((src_id = H5I_register(H5I_DATATYPE, H5T_copy(fill_type, H5T_COPY_ALL))) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, "unable to register types for conversion") - - if((dst_id = H5I_register(H5I_DATATYPE, H5T_copy(buf_type, H5T_COPY_ALL))) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTREGISTER, FAIL, "unable to register types for conversion") - } /* end if */ - - /* If there's VL type of data, make multiple copies of fill value first, - * then do conversion on each element so that each of them has a copy - * of the VL data. - */ - if(TRUE == H5T_detect_class(fill_type, H5T_VLEN)) { - H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ - H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ - H5S_sel_iter_t mem_iter; /* Memory selection iteration info */ - hssize_t nelmts; /* Number of data elements */ - - /* Get the number of elements in the selection */ - nelmts = H5S_GET_SELECT_NPOINTS(space); - HDassert(nelmts >= 0); - H5_CHECK_OVERFLOW(nelmts, hssize_t, size_t); - - /* Allocate a temporary buffer */ - if(NULL == (tmp_buf = H5FL_BLK_MALLOC(type_conv, (size_t)nelmts * buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - - /* Allocate a background buffer, if necessary */ - if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (size_t)nelmts * buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - - /* Replicate the file's fill value into the temporary buffer */ - H5V_array_fill(tmp_buf, fill, src_type_size, (size_t)nelmts); - - /* Convert from file's fill value into memory form */ - if(H5T_convert(tpath, src_id, dst_id, (size_t)nelmts, (size_t)0, (size_t)0, tmp_buf, bkg_buf, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") - - /* Fill the DXPL cache values for later use */ - if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") - - /* Create a selection iterator for scattering the elements to memory buffer */ - if(H5S_select_iter_init(&mem_iter, space, dst_type_size) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize memory selection information") - - /* Scatter the data into memory */ - if(H5D_select_mscat(tmp_buf, space, &mem_iter, (size_t)nelmts, dxpl_cache, buf/*out*/) < 0) { - H5S_SELECT_ITER_RELEASE(&mem_iter); - HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "scatter failed") - } /* end if */ - - /* Release the selection iterator */ - if(H5S_SELECT_ITER_RELEASE(&mem_iter) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release selection iterator") - } else { - const uint8_t *fill_buf; /* Buffer to use for writing fill values */ - - /* Convert disk buffer into memory buffer */ - if(!H5T_path_noop(tpath)) { - /* Allocate space for conversion buffer */ - if(NULL == (tconv_buf = H5FL_BLK_MALLOC(type_elem, buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - - /* Copy the user's data into the buffer for conversion */ - HDmemcpy(tconv_buf, fill, src_type_size); - - /* If there's no VL type of data, do conversion first then fill the data into - * the memory buffer. */ - if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_elem, buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - - /* Perform datatype conversion */ - if(H5T_convert(tpath, src_id, dst_id, (size_t)1, (size_t)0, (size_t)0, tconv_buf, bkg_buf, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") - - /* Point at temporary buffer */ - fill_buf = tconv_buf; - } /* end if */ - else - fill_buf = fill; - - /* Fill the selection in the memory buffer */ - if(H5S_select_fill(fill_buf, dst_type_size, space, buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTENCODE, FAIL, "filling selection failed") - } /* end else */ - } /* end else */ - -done: - if(src_id != (-1) && H5I_dec_ref(src_id) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") - if(dst_id != (-1) && H5I_dec_ref(dst_id) < 0) - HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") - if(tmp_buf) - H5FL_BLK_FREE(type_conv, tmp_buf); - if(tconv_buf) - H5FL_BLK_FREE(type_elem, tconv_buf); - if(bkg_buf) { - if(TRUE == H5T_detect_class(fill_type, H5T_VLEN)) - H5FL_BLK_FREE(type_conv, bkg_buf); - else - H5FL_BLK_FREE(type_elem, bkg_buf); - } /* end if */ - - FUNC_LEAVE_NOAPI(ret_value) -} /* H5D_fill() */ - - -/*-------------------------------------------------------------------------- - NAME H5D_get_dxpl_cache_real PURPOSE Get all the values for the DXPL cache. @@ -2238,7 +1994,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_compound_opt_read(hsize_t nelmts, const H5S_t *space, +H5D_compound_opt_read(size_t nelmts, const H5S_t *space, H5S_sel_iter_t *iter, const H5D_dxpl_cache_t *dxpl_cache, hid_t src_id, hid_t dst_id, H5T_subset_t subset, void *data_buf, void *user_buf/*out*/) @@ -2377,7 +2133,7 @@ done: *------------------------------------------------------------------------- */ static herr_t -H5D_compound_opt_write(hsize_t nelmts, hid_t src_id, hid_t dst_id, void *data_buf) +H5D_compound_opt_write(size_t nelmts, hid_t src_id, hid_t dst_id, void *data_buf) { uint8_t *dbuf = (uint8_t *)data_buf; /*cast for pointer arithmetic */ uint8_t *xsbuf, *xdbuf; diff --git a/src/H5Distore.c b/src/H5Distore.c index 3246467..778f9ed 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -339,9 +339,6 @@ H5FL_SEQ_DEFINE_STATIC(size_t); /* Declare a free list to manage the raw page information */ H5FL_BLK_DEFINE_STATIC(chunk_page); -/* Declare extern the free list to manage blocks of type conversion data */ -H5FL_BLK_EXTERN(type_conv); - /* Declare a free list to manage H5D_istore_sl_ck_t objects */ H5FL_DEFINE_STATIC(H5D_istore_sl_ck_t); @@ -1726,6 +1723,8 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info */ const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ const H5O_fill_t *fill = &(dset->shared->dcpl_cache.fill); /* Fill value info */ + H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ + hbool_t fb_info_init = FALSE; /* Whether the fill value buffer has been initialized */ H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache*/ H5D_rdcc_ent_t *ent = NULL; /*cache entry */ unsigned idx = 0; /*hash index number */ @@ -1752,14 +1751,12 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, idx = H5D_HASH(dset->shared,io_info->store->chunk.index); ent = rdcc->slot[idx]; - if(ent) { - for(u = 0, found = TRUE; u < layout->u.chunk.ndims; u++) { + if(ent) + for(u = 0, found = TRUE; u < layout->u.chunk.ndims; u++) if(io_info->store->chunk.offset[u] != ent->offset[u]) { found = FALSE; break; } /* end if */ - } /* end for */ - } /* end if */ } /* end if */ if(found) { @@ -1817,11 +1814,10 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, if(H5F_block_read(dset->oloc.file, H5FD_MEM_DRAW, chunk_addr, udata->common.key.nbytes, io_info->dxpl_id, chunk) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, NULL, "unable to read raw data chunk") - if(pline->nused) { + if(pline->nused) if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &(udata->common.key.filter_mask), io_info->dxpl_cache->err_detect, io_info->dxpl_cache->filter_cb, &(udata->common.key.nbytes), &chunk_alloc, &chunk) < 0) HGOTO_ERROR(H5E_PLINE, H5E_CANTFILTER, NULL, "data pipeline read failed") - } /* end if */ #ifdef H5D_ISTORE_DEBUG rdcc->nmisses++; #endif /* H5D_ISTORE_DEBUG */ @@ -1847,105 +1843,21 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, * The chunk doesn't exist in the file. Replicate the fill * value throughout the chunk, if the fill value is defined. */ - if(fill->buf) { - size_t elmts_per_chunk; /* # of elements per chunk */ - - /* Sanity check */ - HDassert(0 == (chunk_size % fill->size)); - elmts_per_chunk = chunk_size / fill->size; - - /* If necessary, convert fill value datatypes (which copies VL components, etc.) */ - if(H5T_detect_class(dset->shared->type, H5T_VLEN) > 0) { - H5T_path_t *tpath; /* Datatype conversion path */ - uint8_t *bkg_buf = NULL; /* Background conversion buffer */ - H5T_t *mem_type; /* Pointer to memory datatype */ - size_t mem_type_size, file_type_size; /* Size of datatype in memory and on disk */ - hid_t mem_tid; /* Memory version of disk datatype */ - - /* Create temporary datatype for conversion operation */ - if(NULL == (mem_type = H5T_copy(dset->shared->type, H5T_COPY_REOPEN))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, NULL, "unable to copy file datatype") - if((mem_tid = H5I_register(H5I_DATATYPE, mem_type)) < 0) { - H5T_close(mem_type); - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, NULL, "unable to register memory datatype") - } /* end if */ - - /* Retrieve sizes of memory & file datatypes */ - mem_type_size = H5T_get_size(mem_type); - HDassert(mem_type_size > 0); - file_type_size = H5T_get_size(dset->shared->type); - HDassert(file_type_size == (size_t)fill->size); - - /* Get the datatype conversion path for this operation */ - if(NULL == (tpath = H5T_path_find(dset->shared->type, mem_type, NULL, NULL, io_info->dxpl_id, FALSE))) { - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to convert between src and dst datatypes") - } /* end if */ - /* Allocate a background buffer, if necessary */ - if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (elmts_per_chunk * MAX(mem_type_size, file_type_size))))) { - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - } /* end if */ - - /* Make a copy of the (disk-based) fill value into the chunk buffer */ - HDmemcpy(chunk, fill->buf, file_type_size); - - /* Type convert the chunk buffer, to copy any VL components */ - if(H5T_convert(tpath, dset->shared->type_id, mem_tid, (size_t)1, (size_t)0, (size_t)0, chunk, bkg_buf, io_info->dxpl_id) < 0) { - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL, "data type conversion failed") - } /* end if */ - - /* Replicate the fill value into the cached buffer */ - H5V_array_fill(chunk, chunk, mem_type_size, elmts_per_chunk); - - /* Get the inverse datatype conversion path for this operation */ - if(NULL == (tpath = H5T_path_find(mem_type, dset->shared->type, NULL, NULL, io_info->dxpl_id, FALSE))) { - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to convert between src and dst datatypes") - } /* end if */ - - /* Allocate or reset the background buffer, if necessary */ - if(H5T_path_bkg(tpath)) { - if(bkg_buf) - HDmemset(bkg_buf, 0, MAX(mem_type_size, file_type_size)); - else { - if(NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (elmts_per_chunk * MAX(mem_type_size, file_type_size))))) { - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - } /* end if */ - } /* end else */ - } /* end if */ + /* Initialize the fill value buffer */ + /* (use the compact dataset storage buffer as the fill value buffer) */ + if(H5D_fill_init(&fb_info, chunk, FALSE, + NULL, NULL, NULL, NULL, + &dset->shared->dcpl_cache.fill, dset->shared->type, + dset->shared->type_id, (size_t)0, chunk_size, io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, NULL, "can't initialize fill buffer info") + fb_info_init = TRUE; - /* Type convert the chunk buffer, to copy any VL components */ - if(H5T_convert(tpath, mem_tid, dset->shared->type_id, elmts_per_chunk, (size_t)0, (size_t)0, chunk, bkg_buf, io_info->dxpl_id) < 0) { - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - H5I_dec_ref(mem_tid); - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL, "data type conversion failed") - } /* end if */ - - /* Release resources used */ - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - H5I_dec_ref(mem_tid); - } /* end if */ - else - /* Replicate the [non-VL] fill value into chunk */ - H5V_array_fill(chunk, fill->buf, (size_t)fill->size, elmts_per_chunk); - } /* end if */ - else { - /* - * The chunk doesn't exist in the file and no fill value was - * specified. Assume all zeros. - */ - HDmemset(chunk, 0, chunk_size); - } /* end else */ + /* Check for VL datatype & non-default fill value */ + if(fb_info.has_vlen_fill_type) + /* Fill the buffer with VL datatype fill values */ + if(H5D_fill_refill_vl(&fb_info, fb_info.elmts_per_buf, io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL, "can't refill fill value buffer") } /* end if */ #ifdef H5_CLEAR_MEMORY else @@ -2051,9 +1963,16 @@ H5D_istore_lock(const H5D_io_info_t *io_info, H5D_istore_ud1_t *udata, ret_value = chunk; done: - if (!ret_value) + /* Release the fill buffer info, if it's been initialized */ + if(fb_info_init) + if(H5D_fill_term(&fb_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't release fill buffer info") + + /* Release the chunk allocated, on error */ + if(!ret_value) if(chunk) - chunk=H5D_istore_chunk_xfree (chunk,pline); + chunk = H5D_istore_chunk_xfree(chunk, pline); + FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_lock() */ @@ -2764,13 +2683,13 @@ H5D_istore_chunk_xfree(void *chk, const H5O_pline_t *pline) { FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5D_istore_chunk_xfree) - assert(pline); + HDassert(pline); if(chk) { - if(pline->nused>0) + if(pline->nused > 0) H5MM_xfree(chk); else - H5FL_BLK_FREE(chunk,chk); + H5FL_BLK_FREE(chunk, chk); } /* end if */ FUNC_LEAVE_NOAPI(NULL) @@ -2803,14 +2722,13 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) H5D_io_info_t io_info; /* Dataset I/O info */ H5D_storage_t store; /* Dataset storage information */ hsize_t chunk_offset[H5O_LAYOUT_NDIMS]; /* Offset of current chunk */ - size_t elmts_per_chunk; /* # of elements which fit in a chunk */ size_t orig_chunk_size; /* Original size of chunk in bytes */ unsigned filter_mask = 0; /* Filter mask for chunks that have them */ + const H5O_layout_t *layout = &(dset->shared->layout); /* Dataset layout */ const H5O_pline_t *pline = &(dset->shared->dcpl_cache.pline); /* I/O pipeline info */ const H5O_fill_t *fill = &(dset->shared->dcpl_cache.fill); /* Fill value info */ H5D_fill_value_t fill_status; /* The fill value status */ hbool_t should_fill = FALSE; /* Whether fill values should be written */ - void *chunk = NULL; /* Chunk buffer for writing fill values */ H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ #ifdef H5_HAVE_PARALLEL @@ -2823,33 +2741,23 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) hbool_t carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ int space_ndims; /* Dataset's space rank */ hsize_t space_dim[H5O_LAYOUT_NDIMS]; /* Dataset's dataspace dimensions */ - H5T_path_t *fill_to_mem_tpath; /* Datatype conversion path for converting the fill value to the memory buffer */ - H5T_path_t *mem_to_dset_tpath; /* Datatype conversion path for converting the memory buffer to the dataset elements */ - uint8_t *bkg_buf = NULL; /* Background conversion buffer */ - H5T_t *mem_type = NULL; /* Pointer to memory datatype */ - size_t mem_type_size, file_type_size; /* Size of datatype in memory and on disk */ - size_t elmt_size; /* Size of each element */ - hid_t mem_tid = (-1); /* Memory version of disk datatype */ - size_t bkg_buf_size; /* Size of background buffer */ - hbool_t has_vlen_fill_type = FALSE; /* Whether the datatype for the fill value has a variable-length component */ + H5D_fill_buf_info_t fb_info; /* Dataset's fill buffer info */ + hbool_t fb_info_init = FALSE; /* Whether the fill value buffer has been initialized */ + hid_t data_dxpl_id; /* DXPL ID to use for raw data I/O operations */ herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5D_istore_allocate, FAIL) /* Check args */ - HDassert(dset && H5D_CHUNKED == dset->shared->layout.type); - HDassert(dset->shared->layout.u.chunk.ndims > 0 && dset->shared->layout.u.chunk.ndims <= H5O_LAYOUT_NDIMS); - HDassert(H5F_addr_defined(dset->shared->layout.u.chunk.addr)); + HDassert(dset && H5D_CHUNKED == layout->type); + HDassert(layout->u.chunk.ndims > 0 && layout->u.chunk.ndims <= H5O_LAYOUT_NDIMS); + HDassert(H5F_addr_defined(layout->u.chunk.addr)); HDassert(TRUE == H5P_isa_class(dxpl_id, H5P_DATASET_XFER)); /* Retrieve the dataset dimensions */ if((space_ndims = H5S_get_simple_extent_dims(dset->shared->space, space_dim, NULL)) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get simple dataspace info") - space_dim[space_ndims] = dset->shared->layout.u.chunk.dim[space_ndims]; - - /* Fill the DXPL cache values for later use */ - if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") + space_dim[space_ndims] = layout->u.chunk.dim[space_ndims]; #ifdef H5_HAVE_PARALLEL /* Retrieve MPI parameters */ @@ -2864,12 +2772,25 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) /* Set the MPI-capable file driver flag */ using_mpi = TRUE; + + /* Use the internal "independent" DXPL */ + data_dxpl_id = H5AC_ind_dxpl_id; } /* end if */ + else { +#endif /* H5_HAVE_PARALLEL */ + /* Use the DXPL we were given */ + data_dxpl_id = dxpl_id; +#ifdef H5_HAVE_PARALLEL + } /* end else */ #endif /* H5_HAVE_PARALLEL */ + /* Fill the DXPL cache values for later use */ + if(H5D_get_dxpl_cache(data_dxpl_id, &dxpl_cache) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") + /* Get original chunk size */ - H5_CHECK_OVERFLOW(dset->shared->layout.u.chunk.size, hsize_t, size_t); - orig_chunk_size = (size_t)dset->shared->layout.u.chunk.size; + H5_CHECK_OVERFLOW(layout->u.chunk.size, hsize_t, size_t); + orig_chunk_size = (size_t)layout->u.chunk.size; /* Check the dataset's fill-value status */ if(H5P_is_fill_value_defined(fill, &fill_status) < 0) @@ -2887,99 +2808,36 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) /* Check if fill values should be written to chunks */ if(should_fill) { - /* Fill the chunk with the proper values */ - if(fill->buf) { - /* Detect whether the datatype has a VL component */ - has_vlen_fill_type = H5T_detect_class(dset->shared->type, H5T_VLEN); - - /* If necessary, convert fill value datatypes (which copies VL components, etc.) */ - if(has_vlen_fill_type) { - /* Create temporary datatype for conversion operation */ - if(NULL == (mem_type = H5T_copy(dset->shared->type, H5T_COPY_REOPEN))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy file datatype") - if((mem_tid = H5I_register(H5I_DATATYPE, mem_type)) < 0) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") - - /* Retrieve sizes of memory & file datatypes */ - mem_type_size = H5T_get_size(mem_type); - HDassert(mem_type_size > 0); - file_type_size = H5T_get_size(dset->shared->type); - HDassert(file_type_size == (size_t)fill->size); - - /* Compute the base size for a chunk to operate on */ - elmt_size = MAX(mem_type_size, file_type_size); - elmts_per_chunk = dset->shared->layout.u.chunk.size / file_type_size; - orig_chunk_size = elmts_per_chunk * elmt_size; - - /* Allocate a chunk buffer now, if _no_ filters are used */ - if(pline->nused == 0) - if(NULL == (chunk = H5D_istore_chunk_alloc(orig_chunk_size, pline))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for chunk") - - /* Get the datatype conversion path for this operation */ - if(NULL == (fill_to_mem_tpath = H5T_path_find(dset->shared->type, mem_type, NULL, NULL, dxpl_id, FALSE))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") - - /* Get the inverse datatype conversion path for this operation */ - if(NULL == (mem_to_dset_tpath = H5T_path_find(mem_type, dset->shared->type, NULL, NULL, dxpl_id, FALSE))) - HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") - - /* Check if we need to allocate a background buffer */ - if(H5T_path_bkg(fill_to_mem_tpath) || H5T_path_bkg(mem_to_dset_tpath)) { - /* Check for inverse datatype conversion needing a background buffer */ - /* (do this first, since it needs a larger buffer) */ - if(H5T_path_bkg(mem_to_dset_tpath)) - bkg_buf_size = elmts_per_chunk * elmt_size; - else - bkg_buf_size = elmt_size; - - /* Allocate the background buffer */ - if(NULL == (bkg_buf = H5FL_BLK_MALLOC(type_conv, bkg_buf_size))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") - } /* end if */ - } /* end if */ - else { - /* Allocate chunk buffer for processes to use when writing fill values */ - if(NULL == (chunk = H5D_istore_chunk_alloc(orig_chunk_size, pline))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for chunk") - - /* - * Replicate the fill value throughout the chunk. - */ - HDassert(0 == (orig_chunk_size % fill->size)); - H5V_array_fill(chunk, fill->buf, (size_t)fill->size, (size_t)(orig_chunk_size / fill->size)); - } /* end else */ - } /* end if */ - else { - /* Allocate chunk buffer for processes to use when writing fill values */ - if(NULL == (chunk = H5D_istore_chunk_alloc(orig_chunk_size, pline))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for chunk") - - /* - * No fill value was specified, assume all zeros. - */ - HDmemset(chunk, 0, orig_chunk_size); - } /* end else */ - - /* Check if there are filters which need to be applied to the chunk */ - /* (only do this in advance when the chunk info can be re-used (i.e. - * it doesn't contain any non-default VL datatype fill values) - */ - if(!has_vlen_fill_type && pline->nused > 0) { - size_t buf_size = orig_chunk_size; - - /* Push the chunk through the filters */ - if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &orig_chunk_size, &buf_size, &chunk) < 0) - HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") - } /* end if */ + /* Initialize the fill value buffer */ + /* (delay allocating fill buffer for VL datatypes until refilling) */ + /* (casting away const OK - QAK) */ + if(H5D_fill_init(&fb_info, NULL, (hbool_t)(pline->nused > 0), + (H5MM_allocate_t)H5D_istore_chunk_alloc, (void *)pline, + (H5MM_free_t)H5D_istore_chunk_xfree, (void *)pline, + &dset->shared->dcpl_cache.fill, dset->shared->type, + dset->shared->type_id, (size_t)0, orig_chunk_size, data_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "can't initialize fill buffer info") + fb_info_init = TRUE; + + /* Check if there are filters which need to be applied to the chunk */ + /* (only do this in advance when the chunk info can be re-used (i.e. + * it doesn't contain any non-default VL datatype fill values) + */ + if(!fb_info.has_vlen_fill_type && pline->nused > 0) { + size_t buf_size = orig_chunk_size; + + /* Push the chunk through the filters */ + if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &orig_chunk_size, &buf_size, &fb_info.fill_buf) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") + } /* end if */ } /* end if */ /* Set up dataset I/O info */ store.chunk.offset = chunk_offset; - H5D_BUILD_IO_INFO(&io_info, dset, dxpl_cache, dxpl_id, &store); + H5D_BUILD_IO_INFO(&io_info, dset, dxpl_cache, data_dxpl_id, &store); /* Reset the chunk offset indices */ - HDmemset(chunk_offset, 0, (dset->shared->layout.u.chunk.ndims * sizeof(chunk_offset[0]))); + HDmemset(chunk_offset, 0, (layout->u.chunk.ndims * sizeof(chunk_offset[0]))); /* Loop over all chunks */ carry = FALSE; @@ -3000,7 +2858,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) for(ent = rdcc->head; ent && !chunk_exists; ent = ent->next) { /* Assume a match */ chunk_exists = TRUE; - for(u = 0; u < dset->shared->layout.u.chunk.ndims; u++) + for(u = 0; u < layout->u.chunk.ndims; u++) if(ent->offset[u] != chunk_offset[u]) { chunk_exists = FALSE; /* Reset if no match */ break; @@ -3013,73 +2871,56 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) size_t chunk_size; /* Size of chunk in bytes, possibly filtered */ /* Check for VL datatype & non-default fill value */ - if(has_vlen_fill_type) { - /* Allocate a new chunk buffer each time, if filters are used */ - if(pline->nused > 0) - if(NULL == (chunk = H5D_istore_chunk_alloc(orig_chunk_size, pline))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for chunk") - - /* Make a copy of the (disk-based) fill value into the buffer */ - HDmemcpy(chunk, fill->buf, file_type_size); - - /* Reset first element of background buffer, if necessary */ - if(H5T_path_bkg(fill_to_mem_tpath)) - HDmemset(bkg_buf, 0, elmt_size); - - /* Type convert the dataset buffer, to copy any VL components */ - if(H5T_convert(fill_to_mem_tpath, dset->shared->type_id, mem_tid, (size_t)1, (size_t)0, (size_t)0, chunk, bkg_buf, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") - - /* Replicate the fill value into the cached buffer */ - H5V_array_fill(chunk, chunk, mem_type_size, elmts_per_chunk); - - /* Reset the entire background buffer, if necessary */ - if(H5T_path_bkg(mem_to_dset_tpath)) - HDmemset(bkg_buf, 0, bkg_buf_size); + if(fb_info_init && fb_info.has_vlen_fill_type) { + /* Sanity check */ + HDassert(should_fill); - /* Type convert the dataset buffer, to copy any VL components */ - if(H5T_convert(mem_to_dset_tpath, mem_tid, dset->shared->type_id, elmts_per_chunk, (size_t)0, (size_t)0, chunk, bkg_buf, dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") + /* Fill the buffer with VL datatype fill values */ + if(H5D_fill_refill_vl(&fb_info, fb_info.elmts_per_buf, data_dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "can't refill fill value buffer") /* Check if there are filters which need to be applied to the chunk */ if(pline->nused > 0) { size_t buf_size = orig_chunk_size; - size_t nbytes = (size_t)dset->shared->layout.u.chunk.size; + size_t nbytes = fb_info.fill_buf_size; /* Push the chunk through the filters */ - if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &nbytes, &buf_size, &chunk) < 0) + if(H5Z_pipeline(pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &nbytes, &buf_size, &fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") /* Keep the number of bytes the chunk turned in to */ chunk_size = nbytes; } /* end if */ else - chunk_size = (size_t)dset->shared->layout.u.chunk.size; + chunk_size = (size_t)layout->u.chunk.size; } /* end if */ else chunk_size = orig_chunk_size; /* Initialize the chunk information */ - udata.common.mesg = &dset->shared->layout; + udata.common.mesg = layout; udata.common.key.filter_mask = filter_mask; udata.addr = HADDR_UNDEF; udata.common.key.nbytes = chunk_size; - for(u = 0; u < dset->shared->layout.u.chunk.ndims; u++) + for(u = 0; u < layout->u.chunk.ndims; u++) udata.common.key.offset[u] = chunk_offset[u]; /* Allocate the chunk with all processes */ - if(H5B_insert(dset->oloc.file, dxpl_id, H5B_ISTORE, dset->shared->layout.u.chunk.addr, &udata) < 0) + if(H5B_insert(dset->oloc.file, dxpl_id, H5B_ISTORE, layout->u.chunk.addr, &udata) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to allocate chunk") /* Check if fill values should be written to chunks */ if(should_fill) { + /* Sanity check */ + HDassert(fb_info_init); + #ifdef H5_HAVE_PARALLEL /* Check if this file is accessed with an MPI-capable file driver */ if(using_mpi) { /* Write the chunks out from only one process */ /* !! Use the internal "independent" DXPL!! -QAK */ if(H5_PAR_META_WRITE == mpi_rank) - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.common.key.nbytes, H5AC_ind_dxpl_id, chunk) < 0) + if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.common.key.nbytes, data_dxpl_id, fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") /* Indicate that blocks are being written */ @@ -3087,7 +2928,7 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) } /* end if */ else { #endif /* H5_HAVE_PARALLEL */ - if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.common.key.nbytes, dxpl_id, chunk) < 0) + if(H5F_block_write(dset->oloc.file, H5FD_MEM_DRAW, udata.addr, udata.common.key.nbytes, data_dxpl_id, fb_info.fill_buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") #ifdef H5_HAVE_PARALLEL } /* end else */ @@ -3095,15 +2936,15 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) } /* end if */ /* Release the chunk if we need to re-allocate it each time */ - if(has_vlen_fill_type && pline->nused > 0) - chunk = H5D_istore_chunk_xfree(chunk, pline); + if(fb_info_init && fb_info.has_vlen_fill_type && pline->nused > 0) + H5D_fill_release(&fb_info); } /* end if */ } /* end if */ /* Increment indices */ carry = TRUE; - for(i = (int)dset->shared->layout.u.chunk.ndims - 1; i >= 0; --i) { - chunk_offset[i] += dset->shared->layout.u.chunk.dim[i]; + for(i = (int)layout->u.chunk.ndims - 1; i >= 0; --i) { + chunk_offset[i] += layout->u.chunk.dim[i]; if(chunk_offset[i] >= space_dim[i]) chunk_offset[i] = 0; else { @@ -3127,19 +2968,10 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) #endif /* H5_HAVE_PARALLEL */ done: - /* Free the chunk for fill values */ - if(chunk) - chunk = H5D_istore_chunk_xfree(chunk, pline); - - /* Free other resources for vlen fill values */ - if(has_vlen_fill_type) { - if(mem_tid > 0) - H5I_dec_ref(mem_tid); - else if(mem_type) - H5T_close(mem_type); - if(bkg_buf) - H5FL_BLK_FREE(type_conv, bkg_buf); - } /* end if */ + /* Release the fill buffer info, if it's been initialized */ + if(fb_info_init) + if(H5D_fill_term(&fb_info) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't release fill buffer info") FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_allocate() */ @@ -3516,7 +3348,7 @@ H5D_istore_initialize_by_extent(H5D_io_info_t *io_info) { const H5O_layout_t *layout = &(io_info->dset->shared->layout); /* Dataset layout */ uint8_t *chunk = NULL; /*the file chunk */ - unsigned idx_hint = 0; /*input value for H5F_istore_lock */ + unsigned idx_hint = 0; /*input value for H5D_istore_lock */ hsize_t chunk_offset[H5O_LAYOUT_NDIMS]; /*logical location of the chunks */ hsize_t idx_cur[H5O_LAYOUT_NDIMS]; /*multi-dimensional counters */ hsize_t idx_max[H5O_LAYOUT_NDIMS]; diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index ef2c0ba..c4e4751 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -225,6 +225,31 @@ typedef struct { hid_t dcpl_id; /* Dataset creation property list */ } H5D_obj_create_t; +/* Typedef for filling a buffer with a fill value */ +typedef struct { + hbool_t alloc_vl_during_refill; /* Whether to allocate VL-datatype fill buffer during refill */ + H5MM_allocate_t fill_alloc_func; /* Routine to call for allocating fill buffer */ + void *fill_alloc_info; /* Extra info for allocation routine */ + H5MM_free_t fill_free_func; /* Routine to call for freeing fill buffer */ + void *fill_free_info; /* Extra info for free routine */ + H5T_path_t *fill_to_mem_tpath; /* Datatype conversion path for converting the fill value to the memory buffer */ + H5T_path_t *mem_to_dset_tpath; /* Datatype conversion path for converting the memory buffer to the dataset elements */ + const H5O_fill_t *fill; /* Pointer to fill value */ + void *fill_buf; /* Fill buffer */ + size_t fill_buf_size; /* Size of fill buffer */ + hbool_t use_caller_fill_buf; /* Whether the caller provided the fill buffer */ + void *bkg_buf; /* Background conversion buffer */ + size_t bkg_buf_size; /* Size of background buffer */ + H5T_t *mem_type; /* Pointer to memory datatype */ + const H5T_t *file_type; /* Pointer to file datatype */ + hid_t mem_tid; /* ID for memory version of disk datatype */ + hid_t file_tid; /* ID for disk datatype */ + size_t mem_elmt_size, file_elmt_size; /* Size of element in memory and on disk */ + size_t max_elmt_size; /* Max. size of memory or file datatype */ + size_t elmts_per_buf; /* # of elements that fit into a buffer */ + hbool_t has_vlen_fill_type; /* Whether the datatype for the fill value has a variable-length component */ +} H5D_fill_buf_info_t; + /*****************************/ /* Package Private Variables */ @@ -241,7 +266,7 @@ H5_DLL H5D_t *H5D_create(H5F_t *file, hid_t type_id, const H5S_t *space, H5_DLL H5D_t *H5D_create_named(const H5G_loc_t *loc, const char *name, hid_t type_id, const H5S_t *space, hid_t lcpl_id, hid_t dcpl_id, hid_t dapl_id, hid_t dxpl_id); -H5_DLL herr_t H5D_alloc_storage (H5F_t *f, hid_t dxpl_id, H5D_t *dset, H5D_time_alloc_t time_alloc, +H5_DLL herr_t H5D_alloc_storage(H5F_t *f, hid_t dxpl_id, H5D_t *dset, H5D_time_alloc_t time_alloc, hbool_t update_time, hbool_t full_overwrite); /* Functions that perform serial I/O operations */ @@ -338,6 +363,20 @@ H5_DLL ssize_t H5D_efl_writevv(const H5D_io_info_t *io_info, size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_len_arr[], hsize_t mem_offset_arr[], const void *buf); +/* Functions that perform fill value operations on datasets */ +H5_DLL herr_t H5D_fill(const void *fill, const H5T_t *fill_type, void *buf, + const H5T_t *buf_type, const H5S_t *space, hid_t dxpl_id); +H5_DLL herr_t H5D_fill_init(H5D_fill_buf_info_t *fb_info, void *caller_fill_buf, + hbool_t alloc_vl_during_refill, + H5MM_allocate_t alloc_func, void *alloc_info, + H5MM_free_t free_func, void *free_info, + const H5O_fill_t *fill, const H5T_t *dset_type, hid_t dset_type_id, + size_t nelmts, size_t min_buf_size, hid_t dxpl_id); +H5_DLL herr_t H5D_fill_refill_vl(H5D_fill_buf_info_t *fb_info, size_t nelmts, + hid_t dxpl_id); +H5_DLL herr_t H5D_fill_release(H5D_fill_buf_info_t *fb_info); +H5_DLL herr_t H5D_fill_term(H5D_fill_buf_info_t *fb_info); + #ifdef H5_HAVE_PARALLEL #ifdef H5S_DEBUG diff --git a/src/Makefile.am b/src/Makefile.am index 40974e8..7f4fae2 100755 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -45,8 +45,9 @@ libhdf5_la_SOURCES= H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5A.c H5Abtree2.c H5Adense.c H5Adeprec.c H5Aint.c H5Atest.c \ H5AC.c H5B.c H5Bcache.c \ H5B2.c H5B2cache.c H5B2dbg.c H5B2int.c H5B2stat.c H5B2test.c \ - H5C.c H5CS.c H5D.c H5Dcompact.c H5Dcontig.c H5Ddeprec.c \ - H5Defl.c H5Dio.c H5Distore.c H5Dmpio.c H5Doh.c H5Dselect.c H5Dtest.c \ + H5C.c H5CS.c \ + H5D.c H5Dcompact.c H5Dcontig.c H5Ddeprec.c H5Defl.c H5Dfill.c H5Dio.c \ + H5Distore.c H5Dmpio.c H5Doh.c H5Dselect.c H5Dtest.c \ H5E.c H5Edeprec.c H5Eint.c \ H5F.c H5Fdbg.c H5Ffake.c H5Fmount.c H5Fsfile.c H5Fsuper.c H5Ftest.c \ H5FD.c H5FDcore.c \ diff --git a/src/Makefile.in b/src/Makefile.in index c985af0..74704f7 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -83,7 +83,7 @@ am_libhdf5_la_OBJECTS = H5.lo H5checksum.lo H5dbg.lo H5system.lo \ H5Adeprec.lo H5Aint.lo H5Atest.lo H5AC.lo H5B.lo H5Bcache.lo \ H5B2.lo H5B2cache.lo H5B2dbg.lo H5B2int.lo H5B2stat.lo \ H5B2test.lo H5C.lo H5CS.lo H5D.lo H5Dcompact.lo H5Dcontig.lo \ - H5Ddeprec.lo H5Defl.lo H5Dio.lo H5Distore.lo H5Dmpio.lo \ + H5Ddeprec.lo H5Defl.lo H5Dfill.lo H5Dio.lo H5Distore.lo H5Dmpio.lo \ H5Doh.lo H5Dselect.lo H5Dtest.lo H5E.lo H5Edeprec.lo H5Eint.lo \ H5F.lo H5Fdbg.lo H5Ffake.lo H5Fmount.lo H5Fsfile.lo \ H5Fsuper.lo H5Ftest.lo H5FD.lo H5FDcore.lo H5FDdirect.lo \ @@ -399,7 +399,7 @@ libhdf5_la_SOURCES = H5.c H5checksum.c H5dbg.c H5system.c H5timer.c H5trace.c \ H5AC.c H5B.c H5Bcache.c \ H5B2.c H5B2cache.c H5B2dbg.c H5B2int.c H5B2stat.c H5B2test.c \ H5C.c H5CS.c H5D.c H5Dcompact.c H5Dcontig.c H5Ddeprec.c \ - H5Defl.c H5Dio.c H5Distore.c H5Dmpio.c H5Doh.c H5Dselect.c H5Dtest.c \ + H5Defl.c H5Dfill.c H5Dio.c H5Distore.c H5Dmpio.c H5Doh.c H5Dselect.c H5Dtest.c \ H5E.c H5Edeprec.c H5Eint.c \ H5F.c H5Fdbg.c H5Ffake.c H5Fmount.c H5Fsfile.c H5Fsuper.c H5Ftest.c \ H5FD.c H5FDcore.c \ @@ -592,6 +592,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dcontig.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Ddeprec.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Defl.Plo@am__quote@ +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dfill.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dio.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Distore.Plo@am__quote@ @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/H5Dmpio.Plo@am__quote@ |