diff options
| -rw-r--r-- | src/H5Distore.c | 484 | ||||
| -rw-r--r-- | test/tvltypes.c | 756 |
2 files changed, 675 insertions, 565 deletions
diff --git a/src/H5Distore.c b/src/H5Distore.c index 8d1d2c7..8f967bf 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -279,6 +279,9 @@ H5FL_SEQ_DEFINE_STATIC(size_t); /* Declare a free list to manage the raw page information */ H5FL_BLK_DEFINE_STATIC(chunk_page); +/* Declare a free list to manage blocks of type conversion data */ +H5FL_BLK_EXTERN(type_conv); + /*------------------------------------------------------------------------- * Function: H5D_istore_get_shared @@ -1470,7 +1473,7 @@ H5D_istore_lock(const H5D_io_info_t *io_info, unsigned idx=0; /*hash index number */ hbool_t found = FALSE; /*already in cache? */ unsigned u; /*counters */ - size_t chunk_size=0; /*size of a chunk */ + size_t chunk_size; /*size of a chunk */ void *chunk=NULL; /*the file chunk */ void *ret_value; /*return value */ @@ -1581,14 +1584,103 @@ H5D_istore_lock(const H5D_io_info_t *io_info, if(fill_time==H5D_FILL_TIME_ALLOC || (fill_time==H5D_FILL_TIME_IFSET && fill_status==H5D_FILL_VALUE_USER_DEFINED)) { - if (fill && fill->buf) { - /* - * The chunk doesn't exist in the file. Replicate the fill - * value throughout the chunk. - */ - assert(0==chunk_size % fill->size); - H5V_array_fill(chunk, fill->buf, fill->size, chunk_size/fill->size); - } else { + /* + * The chunk doesn't exist in the file. Replicate the fill + * value throughout the chunk, if the fill value is defined. + */ + if(fill->buf) { + size_t elmts_per_chunk; /* # of elements per chunk */ + + /* Sanity check */ + HDassert(0 == (chunk_size % fill->size)); + elmts_per_chunk = chunk_size / fill->size; + + /* If necessary, convert fill value datatypes (which copies VL components, etc.) */ + if(H5T_detect_class(dset->shared->type, H5T_VLEN) > 0) { + H5T_path_t *tpath; /* Datatype conversion path */ + uint8_t *bkg_buf = NULL; /* Background conversion buffer */ + H5T_t *mem_type; /* Pointer to memory datatype */ + size_t mem_type_size, file_type_size; /* Size of datatype in memory and on disk */ + hid_t mem_tid; /* Memory version of disk datatype */ + + /* Create temporary datatype for conversion operation */ + if(NULL == (mem_type = H5T_copy(dset->shared->type, H5T_COPY_REOPEN))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, NULL, "unable to copy file datatype") + if((mem_tid = H5I_register(H5I_DATATYPE, mem_type)) < 0) { + H5T_close(mem_type); + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, NULL, "unable to register memory datatype") + } /* end if */ + + /* Retrieve sizes of memory & file datatypes */ + mem_type_size = H5T_get_size(mem_type); + HDassert(mem_type_size > 0); + file_type_size = H5T_get_size(dset->shared->type); + HDassert(file_type_size == fill->size); + + /* Get the datatype conversion path for this operation */ + if(NULL == (tpath = H5T_path_find(dset->shared->type, mem_type, NULL, NULL, io_info->dxpl_id))) { + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to convert between src and dst datatypes") + } /* end if */ + + /* Allocate a background buffer, if necessary */ + if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (elmts_per_chunk * MAX(mem_type_size, file_type_size))))) { + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + } /* end if */ + + /* Make a copy of the (disk-based) fill value into the chunk buffer */ + HDmemcpy(chunk, fill->buf, file_type_size); + + /* Type convert the chunk buffer, to copy any VL components */ + if(H5T_convert(tpath, dset->shared->type_id, mem_tid, (size_t)1, (size_t)0, (size_t)0, chunk, bkg_buf, io_info->dxpl_id) < 0) { + if(bkg_buf) + H5FL_BLK_FREE(type_conv, bkg_buf); + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL, "data type conversion failed") + } /* end if */ + + /* Replicate the fill value into the cached buffer */ + H5V_array_fill(chunk, chunk, mem_type_size, elmts_per_chunk); + + /* Get the inverse datatype conversion path for this operation */ + if(NULL == (tpath = H5T_path_find(mem_type, dset->shared->type, NULL, NULL, io_info->dxpl_id))) { + if(bkg_buf) + H5FL_BLK_FREE(type_conv, bkg_buf); + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to convert between src and dst datatypes") + } /* end if */ + + /* Allocate or reset the background buffer, if necessary */ + if(H5T_path_bkg(tpath)) { + if(bkg_buf) + HDmemset(bkg_buf, 0, MAX(mem_type_size, file_type_size)); + else { + if(NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (elmts_per_chunk * MAX(mem_type_size, file_type_size))))) { + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") + } /* end if */ + } /* end else */ + } /* end if */ + + /* Type convert the chunk buffer, to copy any VL components */ + if(H5T_convert(tpath, mem_tid, dset->shared->type_id, elmts_per_chunk, (size_t)0, (size_t)0, chunk, bkg_buf, io_info->dxpl_id) < 0) { + if(bkg_buf) + H5FL_BLK_FREE(type_conv, bkg_buf); + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, NULL, "data type conversion failed") + } /* end if */ + + /* Release resources used */ + if(bkg_buf) + H5FL_BLK_FREE(type_conv, bkg_buf); + H5I_dec_ref(mem_tid); + } /* end if */ + else + /* Replicate the [non-VL] fill value into chunk */ + H5V_array_fill(chunk, fill->buf, fill->size, elmts_per_chunk); + } /* end if */ + else { /* * The chunk doesn't exist in the file and no fill value was * specified. Assume all zeros. @@ -2394,89 +2486,74 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) H5D_io_info_t io_info; /* Dataset I/O info */ H5D_storage_t store; /* Dataset storage information */ hsize_t chunk_offset[H5O_LAYOUT_NDIMS]; /* Offset of current chunk */ - hsize_t chunk_size; /* Size of chunk in bytes */ - unsigned filter_mask=0; /* Filter mask for chunks that have them */ - H5O_pline_t pline; /* I/O pipeline information */ - hbool_t pline_initialized=FALSE; /* Flag to indicate that pline has valid info */ - H5O_fill_t fill; /* Fill value information */ - H5D_fill_time_t fill_time; /* When to write fill values */ + size_t elmts_per_chunk; /* # of elements which fit in a chunk */ + size_t orig_chunk_size; /* Original size of chunk in bytes */ + unsigned filter_mask = 0; /* Filter mask for chunks that have them */ H5D_fill_value_t fill_status; /* The fill value status */ - unsigned should_fill=0; /* Whether fill values should be written */ - void *chunk=NULL; /* Chunk buffer for writing fill values */ + hbool_t should_fill = FALSE; /* Whether fill values should be written */ + void *chunk = NULL; /* Chunk buffer for writing fill values */ H5D_dxpl_cache_t _dxpl_cache; /* Data transfer property cache buffer */ - H5D_dxpl_cache_t *dxpl_cache=&_dxpl_cache; /* Data transfer property cache */ + H5D_dxpl_cache_t *dxpl_cache = &_dxpl_cache; /* Data transfer property cache */ #ifdef H5_HAVE_PARALLEL - MPI_Comm mpi_comm=MPI_COMM_NULL; /* MPI communicator for file */ - int mpi_rank=(-1); /* This process's rank */ + MPI_Comm mpi_comm = MPI_COMM_NULL; /* MPI communicator for file */ + int mpi_rank = (-1); /* This process's rank */ int mpi_code; /* MPI return code */ - unsigned blocks_written=0; /* Flag to indicate that chunk was actually written */ - unsigned using_mpi=0; /* Flag to indicate that the file is being accessed with an MPI-capable file driver */ + hbool_t blocks_written = FALSE; /* Flag to indicate that chunk was actually written */ + hbool_t using_mpi = FALSE; /* Flag to indicate that the file is being accessed with an MPI-capable file driver */ #endif /* H5_HAVE_PARALLEL */ - int carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ - unsigned chunk_exists; /* Flag to indicate whether a chunk exists already */ - int i; /* Local index variable */ - unsigned u; /* Local index variable */ - H5P_genplist_t *dc_plist; /* Property list */ + hbool_t carry; /* Flag to indicate that chunk increment carrys to higher dimension (sorta) */ int space_ndims; /* Dataset's space rank */ hsize_t space_dim[H5O_LAYOUT_NDIMS]; /* Dataset's dataspace dimensions */ - herr_t ret_value=SUCCEED; /* Return value */ + H5T_path_t *fill_to_mem_tpath; /* Datatype conversion path for converting the fill value to the memory buffer */ + H5T_path_t *mem_to_dset_tpath; /* Datatype conversion path for converting the memory buffer to the dataset elements */ + uint8_t *bkg_buf = NULL; /* Background conversion buffer */ + H5T_t *mem_type = NULL; /* Pointer to memory datatype */ + size_t mem_type_size, file_type_size; /* Size of datatype in memory and on disk */ + size_t elmt_size; /* Size of each element */ + hid_t mem_tid = (-1); /* Memory version of disk datatype */ + size_t bkg_buf_size; /* Size of background buffer */ + hbool_t has_vlen_fill_type = FALSE; /* Whether the datatype for the fill value has a variable-length component */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5D_istore_allocate, FAIL) /* Check args */ - assert(dset && H5D_CHUNKED==dset->shared->layout.type); - assert(dset->shared->layout.u.chunk.ndims>0 && dset->shared->layout.u.chunk.ndims<=H5O_LAYOUT_NDIMS); - assert(H5F_addr_defined(dset->shared->layout.u.chunk.addr)); - assert(TRUE==H5P_isa_class(dxpl_id,H5P_DATASET_XFER)); - - /* Get dataset's creation property list */ - if (NULL == (dc_plist = H5I_object(dset->shared->dcpl_id))) - HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a dataset creation property list") - - /* We only handle simple data spaces so far */ - if ((space_ndims=H5S_get_simple_extent_dims(dset->shared->space, space_dim, NULL))<0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get simple data space info") + HDassert(dset && H5D_CHUNKED == dset->shared->layout.type); + HDassert(dset->shared->layout.u.chunk.ndims > 0 && dset->shared->layout.u.chunk.ndims <= H5O_LAYOUT_NDIMS); + HDassert(H5F_addr_defined(dset->shared->layout.u.chunk.addr)); + HDassert(TRUE == H5P_isa_class(dxpl_id, H5P_DATASET_XFER)); + + /* Retrieve the dataset dimensions */ + if((space_ndims = H5S_get_simple_extent_dims(dset->shared->space, space_dim, NULL)) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to get simple dataspace info") space_dim[space_ndims] = dset->shared->layout.u.chunk.dim[space_ndims]; - /* Get necessary properties from dataset creation property list */ - if(H5P_get(dc_plist, H5D_CRT_FILL_VALUE_NAME, &fill) < 0) - HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "can't get fill value") - if(H5P_get(dc_plist, H5D_CRT_DATA_PIPELINE_NAME, &pline) < 0) - HGOTO_ERROR(H5E_STORAGE, H5E_CANTGET, FAIL, "can't get data pipeline") - pline_initialized=TRUE; - if(H5P_get(dc_plist, H5D_CRT_FILL_TIME_NAME, &fill_time) < 0) - HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't retrieve fill time") - /* Fill the DXPL cache values for later use */ - if (H5D_get_dxpl_cache(dxpl_id,&dxpl_cache)<0) + if(H5D_get_dxpl_cache(dxpl_id, &dxpl_cache) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTGET, FAIL, "can't fill dxpl cache") #ifdef H5_HAVE_PARALLEL /* Retrieve MPI parameters */ if(IS_H5FD_MPI(dset->ent.file)) { /* Get the MPI communicator */ - if (MPI_COMM_NULL == (mpi_comm=H5F_mpi_get_comm(dset->ent.file))) + if(MPI_COMM_NULL == (mpi_comm = H5F_mpi_get_comm(dset->ent.file))) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI communicator") /* Get the MPI rank */ - if ((mpi_rank=H5F_mpi_get_rank(dset->ent.file))<0) + if((mpi_rank = H5F_mpi_get_rank(dset->ent.file)) < 0) HGOTO_ERROR(H5E_INTERNAL, H5E_MPI, FAIL, "Can't retrieve MPI rank") /* Set the MPI-capable file driver flag */ - using_mpi=1; + using_mpi = TRUE; } /* end if */ #endif /* H5_HAVE_PARALLEL */ - /* - * Setup indice to go through all chunks. (Future improvement - * should allocate only chunks that have no file space assigned yet. - */ - for (u=0; u<dset->shared->layout.u.chunk.ndims; u++) - chunk_offset[u] = 0; - chunk_size = dset->shared->layout.u.chunk.size; + /* Get original chunk size */ + H5_CHECK_OVERFLOW(dset->shared->layout.u.chunk.size, hsize_t, size_t); + orig_chunk_size = (size_t)dset->shared->layout.u.chunk.size; /* Check the dataset's fill-value status */ - if (H5P_is_fill_value_defined(&fill, &fill_status) < 0) + if(H5P_is_fill_value_defined(&dset->shared->dcpl_cache.fill, &fill_status) < 0) HGOTO_ERROR(H5E_PLIST, H5E_CANTGET, FAIL, "can't tell if fill value defined") /* If we are filling the dataset on allocation or "if set" and @@ -2484,141 +2561,266 @@ H5D_istore_allocate(H5D_t *dset, hid_t dxpl_id, hbool_t full_overwrite) * or if there are any pipeline filters defined, * set the "should fill" flag */ - if((!full_overwrite && (fill_time==H5D_FILL_TIME_ALLOC || - (fill_time==H5D_FILL_TIME_IFSET && fill_status==H5D_FILL_VALUE_USER_DEFINED))) - || pline.nused>0) - should_fill=1; + if((!full_overwrite && (dset->shared->dcpl_cache.fill_time == H5D_FILL_TIME_ALLOC || + (dset->shared->dcpl_cache.fill_time == H5D_FILL_TIME_IFSET && fill_status == H5D_FILL_VALUE_USER_DEFINED))) + || dset->shared->dcpl_cache.pline.nused > 0) + should_fill = TRUE; - /* Check if fill values should be written to blocks */ + /* Check if fill values should be written to chunks */ if(should_fill) { - /* Allocate chunk buffer for processes to use when writing fill values */ - H5_CHECK_OVERFLOW(chunk_size,hsize_t,size_t); - if (NULL==(chunk = H5D_istore_chunk_alloc((size_t)chunk_size,&pline))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for chunk") - /* Fill the chunk with the proper values */ - if(fill.buf) { - /* - * Replicate the fill value throughout the chunk. - */ - assert(0==chunk_size % fill.size); - H5V_array_fill(chunk, fill.buf, fill.size, (size_t)chunk_size/fill.size); - } else { + if(dset->shared->dcpl_cache.fill.buf) { + /* Detect whether the datatype has a VL component */ + has_vlen_fill_type = H5T_detect_class(dset->shared->type, H5T_VLEN); + + /* If necessary, convert fill value datatypes (which copies VL components, etc.) */ + if(has_vlen_fill_type) { + /* Create temporary datatype for conversion operation */ + if(NULL == (mem_type = H5T_copy(dset->shared->type, H5T_COPY_REOPEN))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy file datatype") + if((mem_tid = H5I_register(H5I_DATATYPE, mem_type)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") + + /* Retrieve sizes of memory & file datatypes */ + mem_type_size = H5T_get_size(mem_type); + HDassert(mem_type_size > 0); + file_type_size = H5T_get_size(dset->shared->type); + HDassert(file_type_size == dset->shared->dcpl_cache.fill.size); + + /* Compute the base size for a chunk to operate on */ + elmt_size = MAX(mem_type_size, file_type_size); + elmts_per_chunk = dset->shared->layout.u.chunk.size / file_type_size; + orig_chunk_size = elmts_per_chunk * elmt_size; + + /* Allocate a chunk buffer now, if _no_ filters are used */ + if(dset->shared->dcpl_cache.pline.nused == 0) + if(NULL == (chunk = H5D_istore_chunk_alloc(orig_chunk_size, &dset->shared->dcpl_cache.pline))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for chunk") + + /* Get the datatype conversion path for this operation */ + if(NULL == (fill_to_mem_tpath = H5T_path_find(dset->shared->type, mem_type, NULL, NULL, dxpl_id))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") + + /* Get the inverse datatype conversion path for this operation */ + if(NULL == (mem_to_dset_tpath = H5T_path_find(mem_type, dset->shared->type, NULL, NULL, dxpl_id))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") + + /* Check if we need to allocate a background buffer */ + if(H5T_path_bkg(fill_to_mem_tpath) || H5T_path_bkg(mem_to_dset_tpath)) { + /* Check for inverse datatype conversion needing a background buffer */ + /* (do this first, since it needs a larger buffer) */ + if(H5T_path_bkg(mem_to_dset_tpath)) + bkg_buf_size = elmts_per_chunk * elmt_size; + else + bkg_buf_size = elmt_size; + + /* Allocate the background buffer */ + if(NULL == (bkg_buf = H5FL_BLK_MALLOC(type_conv, bkg_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + } /* end if */ + } /* end if */ + else { + /* Allocate chunk buffer for processes to use when writing fill values */ + if(NULL == (chunk = H5D_istore_chunk_alloc(orig_chunk_size, &dset->shared->dcpl_cache.pline))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for chunk") + + /* + * Replicate the fill value throughout the chunk. + */ + HDassert(0 == (orig_chunk_size % dset->shared->dcpl_cache.fill.size)); + H5V_array_fill(chunk, dset->shared->dcpl_cache.fill.buf, dset->shared->dcpl_cache.fill.size, (size_t)(orig_chunk_size / dset->shared->dcpl_cache.fill.size)); + } /* end else */ + } /* end if */ + else { + /* Allocate chunk buffer for processes to use when writing fill values */ + if(NULL == (chunk = H5D_istore_chunk_alloc(orig_chunk_size, &dset->shared->dcpl_cache.pline))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for chunk") + /* * No fill value was specified, assume all zeros. */ - HDmemset (chunk, 0, (size_t)chunk_size); + HDmemset(chunk, 0, orig_chunk_size); } /* end else */ /* Check if there are filters which need to be applied to the chunk */ - if (pline.nused>0) { - size_t buf_size=(size_t)chunk_size; - size_t nbytes=(size_t)chunk_size; + /* (only do this in advance when the chunk info can be re-used (i.e. + * it doesn't contain any non-default VL datatype fill values) + */ + if(!has_vlen_fill_type && dset->shared->dcpl_cache.pline.nused > 0) { + size_t buf_size = orig_chunk_size; /* Push the chunk through the filters */ - if (H5Z_pipeline(&pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &nbytes, &buf_size, &chunk)<0) + if(H5Z_pipeline(&dset->shared->dcpl_cache.pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &orig_chunk_size, &buf_size, &chunk) < 0) HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") - - /* Keep the number of bytes the chunk turned in to */ - chunk_size=nbytes; } /* end if */ } /* end if */ /* Set up dataset I/O info */ - store.chunk.offset=chunk_offset; - H5D_BUILD_IO_INFO(&io_info,dset,dxpl_cache,dxpl_id,&store); + store.chunk.offset = chunk_offset; + H5D_BUILD_IO_INFO(&io_info, dset, dxpl_cache, dxpl_id, &store); + + /* Reset the chunk offset indices */ + HDmemset(chunk_offset, 0, (dset->shared->layout.u.chunk.ndims * sizeof(chunk_offset[0]))); /* Loop over all chunks */ - carry=0; - while (carry==0) { + carry = FALSE; + while(!carry) { + int i; /* Local index variable */ + /* Check if the chunk exists yet on disk */ - chunk_exists=1; - if(H5D_istore_get_addr(&io_info,NULL)==HADDR_UNDEF) { - const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /*raw data chunk cache */ - H5D_rdcc_ent_t *ent = NULL; /*cache entry */ + if(!H5F_addr_defined(H5D_istore_get_addr(&io_info, NULL))) { + const H5D_rdcc_t *rdcc = &(dset->shared->cache.chunk); /* Raw data chunk cache */ + H5D_rdcc_ent_t *ent; /* Cache entry */ + hbool_t chunk_exists; /* Flag to indicate whether a chunk exists already */ + unsigned u; /* Local index variable */ /* Didn't find the chunk on disk */ - chunk_exists = 0; + chunk_exists = FALSE; /* Look for chunk in cache */ for(ent = rdcc->head; ent && !chunk_exists; ent = ent->next) { /* Assume a match */ - chunk_exists = 1; - for(u = 0; u < dset->shared->layout.u.chunk.ndims && chunk_exists; u++) { - if(ent->offset[u] != chunk_offset[u]) - chunk_exists = 0; /* Reset if no match */ - } /* end for */ + chunk_exists = TRUE; + for(u = 0; u < dset->shared->layout.u.chunk.ndims; u++) + if(ent->offset[u] != chunk_offset[u]) { + chunk_exists = FALSE; /* Reset if no match */ + break; + } /* end if */ } /* end for */ - } /* end if */ - if(!chunk_exists) { - H5D_istore_ud1_t udata; /* B-tree pass-through for creating chunk */ + /* Chunk wasn't in cache either, create it now */ + if(!chunk_exists) { + H5D_istore_ud1_t udata; /* B-tree pass-through for creating chunk */ + size_t chunk_size; /* Size of chunk in bytes, possibly filtered */ - /* Initialize the chunk information */ - udata.common.mesg = &dset->shared->layout; - udata.common.key.filter_mask = filter_mask; - udata.addr = HADDR_UNDEF; - H5_CHECK_OVERFLOW(chunk_size,hsize_t,size_t); - udata.common.key.nbytes = (size_t)chunk_size; - for(u = 0; u < dset->shared->layout.u.chunk.ndims; u++) - udata.common.key.offset[u] = chunk_offset[u]; + /* Check for VL datatype & non-default fill value */ + if(has_vlen_fill_type) { + /* Allocate a new chunk buffer each time, if filters are used */ + if(dset->shared->dcpl_cache.pline.nused > 0) + if(NULL == (chunk = H5D_istore_chunk_alloc(orig_chunk_size, &dset->shared->dcpl_cache.pline))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for chunk") - /* Allocate the chunk with all processes */ - if (H5B_insert(dset->ent.file, dxpl_id, H5B_ISTORE, dset->shared->layout.u.chunk.addr, &udata)<0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to allocate chunk") + /* Make a copy of the (disk-based) fill value into the buffer */ + HDmemcpy(chunk, dset->shared->dcpl_cache.fill.buf, file_type_size); - /* Check if fill values should be written to blocks */ - if(should_fill) { -#ifdef H5_HAVE_PARALLEL - /* Check if this file is accessed with an MPI-capable file driver */ - if(using_mpi) { - /* Write the chunks out from only one process */ - /* !! Use the internal "independent" DXPL!! -QAK */ - if(H5_PAR_META_WRITE==mpi_rank) { - if (H5F_block_write(dset->ent.file, H5FD_MEM_DRAW, udata.addr, udata.common.key.nbytes, H5AC_ind_dxpl_id, chunk)<0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") - } /* end if */ + /* Reset first element of background buffer, if necessary */ + if(H5T_path_bkg(fill_to_mem_tpath)) + HDmemset(bkg_buf, 0, elmt_size); + + /* Type convert the dataset buffer, to copy any VL components */ + if(H5T_convert(fill_to_mem_tpath, dset->shared->type_id, mem_tid, (size_t)1, (size_t)0, (size_t)0, chunk, bkg_buf, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") + + /* Replicate the fill value into the cached buffer */ + H5V_array_fill(chunk, chunk, mem_type_size, elmts_per_chunk); + + /* Reset the entire background buffer, if necessary */ + if(H5T_path_bkg(mem_to_dset_tpath)) + HDmemset(bkg_buf, 0, bkg_buf_size); - /* Indicate that blocks are being written */ - blocks_written=1; + /* Type convert the dataset buffer, to copy any VL components */ + if(H5T_convert(mem_to_dset_tpath, mem_tid, dset->shared->type_id, elmts_per_chunk, (size_t)0, (size_t)0, chunk, bkg_buf, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") + + /* Check if there are filters which need to be applied to the chunk */ + if(dset->shared->dcpl_cache.pline.nused > 0) { + size_t buf_size = orig_chunk_size; + size_t nbytes = (size_t)dset->shared->layout.u.chunk.size; + + /* Push the chunk through the filters */ + if(H5Z_pipeline(&dset->shared->dcpl_cache.pline, 0, &filter_mask, dxpl_cache->err_detect, dxpl_cache->filter_cb, &nbytes, &buf_size, &chunk) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_WRITEERROR, FAIL, "output pipeline failed") + + /* Keep the number of bytes the chunk turned in to */ + chunk_size = nbytes; + } /* end if */ + else + chunk_size = (size_t)dset->shared->layout.u.chunk.size; } /* end if */ - else { + else + chunk_size = orig_chunk_size; + + /* Initialize the chunk information */ + udata.common.mesg = &dset->shared->layout; + udata.common.key.filter_mask = filter_mask; + udata.addr = HADDR_UNDEF; + udata.common.key.nbytes = chunk_size; + for(u = 0; u < dset->shared->layout.u.chunk.ndims; u++) + udata.common.key.offset[u] = chunk_offset[u]; + + /* Allocate the chunk with all processes */ + if(H5B_insert(dset->ent.file, dxpl_id, H5B_ISTORE, dset->shared->layout.u.chunk.addr, &udata) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to allocate chunk") + + /* Check if fill values should be written to chunks */ + if(should_fill) { +#ifdef H5_HAVE_PARALLEL + /* Check if this file is accessed with an MPI-capable file driver */ + if(using_mpi) { + /* Write the chunks out from only one process */ + /* !! Use the internal "independent" DXPL!! -QAK */ + if(H5_PAR_META_WRITE == mpi_rank) + if(H5F_block_write(dset->ent.file, H5FD_MEM_DRAW, udata.addr, udata.common.key.nbytes, H5AC_ind_dxpl_id, chunk) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") + + /* Indicate that blocks are being written */ + blocks_written = TRUE; + } /* end if */ + else { #endif /* H5_HAVE_PARALLEL */ - if (H5F_block_write(dset->ent.file, H5FD_MEM_DRAW, udata.addr, udata.common.key.nbytes, dxpl_id, chunk)<0) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") + if(H5F_block_write(dset->ent.file, H5FD_MEM_DRAW, udata.addr, udata.common.key.nbytes, dxpl_id, chunk) < 0) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "unable to write raw data to file") #ifdef H5_HAVE_PARALLEL - } /* end else */ + } /* end else */ #endif /* H5_HAVE_PARALLEL */ + } /* end if */ + + /* Release the chunk if we need to re-allocate it each time */ + if(has_vlen_fill_type && dset->shared->dcpl_cache.pline.nused > 0) + chunk = H5D_istore_chunk_xfree(chunk, &dset->shared->dcpl_cache.pline); } /* end if */ } /* end if */ - /* Increment indices */ - for (i=(int)dset->shared->layout.u.chunk.ndims-1, carry=1; i>=0 && carry; --i) { + /* Increment chunk offset indices */ + carry = TRUE; + for(i = (int)dset->shared->layout.u.chunk.ndims - 1; i >= 0; --i) { chunk_offset[i] += dset->shared->layout.u.chunk.dim[i]; - if (chunk_offset[i] >= space_dim[i]) + if(chunk_offset[i] >= space_dim[i]) chunk_offset[i] = 0; - else - carry = 0; + else { + carry = FALSE; + break; + } /* end else */ } /* end for */ } /* end while */ #ifdef H5_HAVE_PARALLEL - /* Only need to block at the barrier if we actually allocated a chunk */ - /* And if we are using an MPI-capable file driver */ + /* Only need to block at the barrier if we actually initialized a chunk */ + /* using an MPI-capable file driver */ if(using_mpi && blocks_written) { /* Wait at barrier to avoid race conditions where some processes are * still writing out chunks and other processes race ahead to read * them in, getting bogus data. */ - if (MPI_SUCCESS != (mpi_code=MPI_Barrier(mpi_comm))) + if(MPI_SUCCESS != (mpi_code = MPI_Barrier(mpi_comm))) HMPI_GOTO_ERROR(FAIL, "MPI_Barrier failed", mpi_code); } /* end if */ #endif /* H5_HAVE_PARALLEL */ done: /* Free the chunk for fill values */ - if(chunk!=NULL && pline_initialized) - chunk=H5D_istore_chunk_xfree(chunk,&pline); + if(chunk) + chunk = H5D_istore_chunk_xfree(chunk, &dset->shared->dcpl_cache.pline); + + /* Free other resources for vlen fill values */ + if(has_vlen_fill_type) { + if(mem_tid > 0) + H5I_dec_ref(mem_tid); + else if(mem_type) + H5T_close(mem_type); + if(bkg_buf) + H5FL_BLK_FREE(type_conv, bkg_buf); + } /* end if */ FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_allocate() */ diff --git a/test/tvltypes.c b/test/tvltypes.c index 48ed402..15af456 100644 --- a/test/tvltypes.c +++ b/test/tvltypes.c @@ -2538,99 +2538,91 @@ test_vltypes_fill_value(void) CHECK(file_id, FAIL, "H5Fcreate"); /* Create datasets with different storage layouts */ -#ifdef QAK -HDfprintf(stderr, "Before creating datasets\n"); -#endif /* QAK */ -HDfprintf(stderr, "Uncomment loop!\n"); -/* for(layout = H5D_COMPACT; layout <= H5D_CHUNKED; layout++) { */ - for(layout = H5D_COMPACT; layout <= H5D_CONTIGUOUS; layout++) { - hid_t tmp_dcpl_id; /* Temporary copy of the dataset creation property list */ - - /* Make a copy of the dataset creation property list */ - tmp_dcpl_id = H5Pcopy(dcpl_id); - CHECK(tmp_dcpl_id, FAIL, "H5Pcopy"); - - /* Layout specific actions */ - switch(layout) { - case H5D_COMPACT: - HDstrcpy(dset_name1, "dataset1-compact"); - HDstrcpy(dset_name2, "dataset2-compact"); - dset_dspace_id = small_dspace_id; - ret = H5Pset_layout(tmp_dcpl_id, H5D_COMPACT); - CHECK(ret, FAIL, "H5Pset_layout"); - break; - - case H5D_CONTIGUOUS: - HDstrcpy(dset_name1, "dataset1-contig"); - HDstrcpy(dset_name2, "dataset2-contig"); - dset_dspace_id = large_dspace_id; - break; - - case H5D_CHUNKED: - { - hsize_t chunk_dims[1] = {SPACE4_DIM_LARGE / 4}; - - HDstrcpy(dset_name1, "dataset1-chunked"); - HDstrcpy(dset_name2, "dataset2-chunked"); + for(layout = H5D_COMPACT; layout <= H5D_CHUNKED; layout++) { + unsigned compress_loop; /* # of times to run loop, for testing compressed chunked dataset */ + unsigned test_loop; /* Loop over datasets */ + +#ifdef H5_HAVE_FILTER_DEFLATE + if(layout == H5D_CHUNKED) + compress_loop = 2; + else +#endif /* H5_HAVE_FILTER_DEFLATE */ + compress_loop = 1; + + /* Loop over dataset operations */ + for(test_loop = 0; test_loop < compress_loop; test_loop++) { + hid_t tmp_dcpl_id; /* Temporary copy of the dataset creation property list */ + + /* Make a copy of the dataset creation property list */ + tmp_dcpl_id = H5Pcopy(dcpl_id); + CHECK(tmp_dcpl_id, FAIL, "H5Pcopy"); + + /* Layout specific actions */ + switch(layout) { + case H5D_COMPACT: + HDstrcpy(dset_name1, "dataset1-compact"); + HDstrcpy(dset_name2, "dataset2-compact"); + dset_dspace_id = small_dspace_id; + ret = H5Pset_layout(tmp_dcpl_id, H5D_COMPACT); + CHECK(ret, FAIL, "H5Pset_layout"); + break; + + case H5D_CONTIGUOUS: + HDstrcpy(dset_name1, "dataset1-contig"); + HDstrcpy(dset_name2, "dataset2-contig"); dset_dspace_id = large_dspace_id; - ret = H5Pset_chunk(tmp_dcpl_id, 1, chunk_dims); - CHECK(ret, FAIL, "H5Pset_chunk"); - } - break; - } /* end switch */ - - /* Create first data set with default setting - no space is allocated */ -#ifdef QAK -HDfprintf(stderr, "Before creating first dataset: '%s'\n", dset_name1); -#endif /* QAK */ - dset_id = H5Dcreate(file_id, dset_name1, dtype1_id, dset_dspace_id, tmp_dcpl_id); -#ifdef QAK -HDfprintf(stderr, "After creating first dataset\n"); -#endif /* QAK */ - CHECK(dset_id, FAIL, "H5Dcreate"); - -#ifdef QAK -HDfprintf(stderr, "Before closing first dataset\n"); -#endif /* QAK */ - ret = H5Dclose(dset_id); -#ifdef QAK -HDfprintf(stderr, "After closing first dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dclose"); - - -#ifndef QAK2 - /* Create a second data set with space allocated and fill value written */ - ret = H5Pset_fill_time(tmp_dcpl_id, H5D_FILL_TIME_IFSET); - CHECK(ret, FAIL, "H5Pset_fill_time"); - - ret = H5Pset_alloc_time(tmp_dcpl_id, H5D_ALLOC_TIME_EARLY); - CHECK(ret, FAIL, "H5Pset_alloc_time"); - -#ifdef QAK -HDfprintf(stderr, "Before creating second dataset: '%s'\n", dset_name2); -#endif /* QAK */ - dset_id = H5Dcreate(file_id, dset_name2, dtype1_id, dset_dspace_id, tmp_dcpl_id); -#ifdef QAK -HDfprintf(stderr, "After creating second dataset\n"); -#endif /* QAK */ - CHECK(dset_id, FAIL, "H5Dcreate"); - -#ifdef QAK -HDfprintf(stderr, "Before closing second dataset\n"); -#endif /* QAK */ - ret = H5Dclose(dset_id); -#ifdef QAK -HDfprintf(stderr, "After closing second dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dclose"); -#else /* QAK2 */ -HDfprintf(stderr, "Uncomment test for second dataset!\n"); -#endif /* QAK2 */ - - /* Close temporary DCPL */ - ret = H5Pclose(tmp_dcpl_id); - CHECK(ret, FAIL, "H5Pclose"); + break; + + case H5D_CHUNKED: + { + hsize_t chunk_dims[1] = {SPACE4_DIM_LARGE / 4}; + + dset_dspace_id = large_dspace_id; + ret = H5Pset_chunk(tmp_dcpl_id, 1, chunk_dims); + CHECK(ret, FAIL, "H5Pset_chunk"); +#ifdef H5_HAVE_FILTER_DEFLATE + if(test_loop == 1) { + HDstrcpy(dset_name1, "dataset1-chunked-compressed"); + HDstrcpy(dset_name2, "dataset2-chunked-compressed"); + ret = H5Pset_deflate(tmp_dcpl_id, 3); + CHECK(ret, FAIL, "H5Pset_deflate"); + } /* end if */ + else { +#endif /* H5_HAVE_FILTER_DEFLATE */ + HDstrcpy(dset_name1, "dataset1-chunked"); + HDstrcpy(dset_name2, "dataset2-chunked"); +#ifdef H5_HAVE_FILTER_DEFLATE + } /* end else */ +#endif /* H5_HAVE_FILTER_DEFLATE */ + } + break; + } /* end switch */ + + /* Create first data set with default setting - no space is allocated */ + dset_id = H5Dcreate(file_id, dset_name1, dtype1_id, dset_dspace_id, tmp_dcpl_id); + CHECK(dset_id, FAIL, "H5Dcreate"); + + ret = H5Dclose(dset_id); + CHECK(ret, FAIL, "H5Dclose"); + + + /* Create a second data set with space allocated and fill value written */ + ret = H5Pset_fill_time(tmp_dcpl_id, H5D_FILL_TIME_IFSET); + CHECK(ret, FAIL, "H5Pset_fill_time"); + + ret = H5Pset_alloc_time(tmp_dcpl_id, H5D_ALLOC_TIME_EARLY); + CHECK(ret, FAIL, "H5Pset_alloc_time"); + + dset_id = H5Dcreate(file_id, dset_name2, dtype1_id, dset_dspace_id, tmp_dcpl_id); + CHECK(dset_id, FAIL, "H5Dcreate"); + + ret = H5Dclose(dset_id); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close temporary DCPL */ + ret = H5Pclose(tmp_dcpl_id); + CHECK(ret, FAIL, "H5Pclose"); + } /* end for */ } /* end for */ ret = H5Fclose(file_id); @@ -2652,358 +2644,278 @@ HDfprintf(stderr, "Uncomment test for second dataset!\n"); CHECK(file_id, FAIL, "H5Fopen"); /* Read datasets with different storage layouts */ -#ifdef QAK -HDfprintf(stderr, "Before testing empty reads\n"); -#endif /* QAK */ -HDfprintf(stderr, "Uncomment loop!\n"); -/* for(layout = H5D_COMPACT; layout <= H5D_CHUNKED; layout++) { */ - for(layout = H5D_COMPACT; layout <= H5D_CONTIGUOUS; layout++) { - - /* Layout specific actions */ - switch(layout) { - case H5D_COMPACT: - HDstrcpy(dset_name1, "dataset1-compact"); - HDstrcpy(dset_name2, "dataset2-compact"); - dset_dspace_id = small_dspace_id; - dset_elmts = SPACE4_DIM_SMALL; - break; - - case H5D_CONTIGUOUS: - HDstrcpy(dset_name1, "dataset1-contig"); - HDstrcpy(dset_name2, "dataset2-contig"); - dset_dspace_id = large_dspace_id; - dset_elmts = SPACE4_DIM_LARGE; - break; - - case H5D_CHUNKED: - HDstrcpy(dset_name1, "dataset1-chunked"); - HDstrcpy(dset_name2, "dataset2-chunked"); - dset_dspace_id = large_dspace_id; - dset_elmts = SPACE4_DIM_LARGE; - break; - } /* end switch */ - - /* Open first data set */ -#ifdef QAK -HDfprintf(stderr, "Before opening first dataset: '%s'\n", dset_name1); -#endif /* QAK */ - dset_id = H5Dopen(file_id, dset_name1); -#ifdef QAK -HDfprintf(stderr, "After opening first dataset\n"); -#endif /* QAK */ - CHECK(dset_id, FAIL, "H5Dopen"); - - /* Read in the data of fill value */ -#ifdef QAK -HDfprintf(stderr, "Before reading from first dataset\n"); -#endif /* QAK */ - ret = H5Dread(dset_id, dtype1_id, dset_dspace_id, dset_dspace_id, xfer_pid, rbuf); -#ifdef QAK -HDfprintf(stderr, "After reading from first dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dread"); - - /* Compare data read in */ - for(i = 0; i < dset_elmts; i++) { - if(HDstrcmp(rbuf[i].str_id, "foobar") - || HDstrcmp(rbuf[i].str_name, "") - || rbuf[i].str_desc - || HDstrcmp(rbuf[i].str_orig, "\0") - || HDstrcmp(rbuf[i].str_stat, "dead") - || HDstrcmp(rbuf[i].str_form, "liquid") - || HDstrcmp(rbuf[i].str_unit, "meter")) { - TestErrPrintf("%d: VL data doesn't match!, index(i) = %d\n", __LINE__, (int)i); - continue; - } /* end if */ - } /* end for */ - -#ifdef QAK -HDfprintf(stderr, "Before closing first dataset\n"); -#endif /* QAK */ - ret = H5Dclose(dset_id); -#ifdef QAK -HDfprintf(stderr, "After closing first dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dclose"); - - /* Release the space */ -#ifdef QAK -HDfprintf(stderr, "Before reclaiming space\n"); -#endif /* QAK */ - ret = H5Dvlen_reclaim(dtype1_id, dset_dspace_id, xfer_pid, rbuf); -#ifdef QAK -HDfprintf(stderr, "After reclaiming space\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dvlen_reclaim"); - - -#ifndef QAK2 - /* Open the second data set to check the value of data */ -#ifdef QAK -HDfprintf(stderr, "Before opening second dataset: '%s'\n", dset_name2); -#endif /* QAK */ - dset_id = H5Dopen(file_id, dset_name2); -#ifdef QAK -HDfprintf(stderr, "After opening second dataset\n"); -#endif /* QAK */ - CHECK(dset_id, FAIL, "H5Dopen"); - -#ifdef QAK -HDfprintf(stderr, "Before reading from second dataset\n"); -#endif /* QAK */ - ret = H5Dread(dset_id, dtype1_id, dset_dspace_id, dset_dspace_id, xfer_pid, rbuf); -#ifdef QAK -HDfprintf(stderr, "After reading from second dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dread"); - - /* Compare data read in */ - for(i = 0; i < dset_elmts; i++) { - if(HDstrcmp(rbuf[i].str_id, "foobar") - || HDstrcmp(rbuf[i].str_name, "") - || rbuf[i].str_desc - || HDstrcmp(rbuf[i].str_orig, "\0") - || HDstrcmp(rbuf[i].str_stat, "dead") - || HDstrcmp(rbuf[i].str_form, "liquid") - || HDstrcmp(rbuf[i].str_unit, "meter")) { - TestErrPrintf("%d: VL data doesn't match!, index(i)=%d\n",__LINE__,(int)i); - continue; - } /* end if */ - } /* end for */ - -#ifdef QAK -HDfprintf(stderr, "Before closing second dataset\n"); -#endif /* QAK */ - ret = H5Dclose(dset_id); -#ifdef QAK -HDfprintf(stderr, "After closing second dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dclose"); - - /* Release the space */ -#ifdef QAK -HDfprintf(stderr, "Before reclaiming space\n"); -#endif /* QAK */ - ret = H5Dvlen_reclaim(dtype1_id, dset_dspace_id, xfer_pid, rbuf); -#ifdef QAK -HDfprintf(stderr, "After reclaiming space\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dvlen_reclaim"); -#else /* QAK2 */ -HDfprintf(stderr, "Uncomment test for second dataset!\n"); -#endif /* QAK2 */ - } /* end for */ - - ret = H5Fclose(file_id); - CHECK(ret, FAIL, "H5Fclose"); + for(layout = H5D_COMPACT; layout <= H5D_CHUNKED; layout++) { + unsigned compress_loop; /* # of times to run loop, for testing compressed chunked dataset */ + unsigned test_loop; /* Loop over datasets */ + +#ifdef H5_HAVE_FILTER_DEFLATE + if(layout == H5D_CHUNKED) + compress_loop = 2; + else +#endif /* H5_HAVE_FILTER_DEFLATE */ + compress_loop = 1; + + /* Loop over dataset operations */ + for(test_loop = 0; test_loop < compress_loop; test_loop++) { + + /* Layout specific actions */ + switch(layout) { + case H5D_COMPACT: + HDstrcpy(dset_name1, "dataset1-compact"); + HDstrcpy(dset_name2, "dataset2-compact"); + dset_dspace_id = small_dspace_id; + dset_elmts = SPACE4_DIM_SMALL; + break; + + case H5D_CONTIGUOUS: + HDstrcpy(dset_name1, "dataset1-contig"); + HDstrcpy(dset_name2, "dataset2-contig"); + dset_dspace_id = large_dspace_id; + dset_elmts = SPACE4_DIM_LARGE; + break; + + case H5D_CHUNKED: +#ifdef H5_HAVE_FILTER_DEFLATE + if(test_loop == 1) { + HDstrcpy(dset_name1, "dataset1-chunked-compressed"); + HDstrcpy(dset_name2, "dataset2-chunked-compressed"); + } /* end if */ + else { +#endif /* H5_HAVE_FILTER_DEFLATE */ + HDstrcpy(dset_name1, "dataset1-chunked"); + HDstrcpy(dset_name2, "dataset2-chunked"); +#ifdef H5_HAVE_FILTER_DEFLATE + } /* end else */ +#endif /* H5_HAVE_FILTER_DEFLATE */ + dset_dspace_id = large_dspace_id; + dset_elmts = SPACE4_DIM_LARGE; + break; + } /* end switch */ + /* Open first data set */ + dset_id = H5Dopen(file_id, dset_name1); + CHECK(dset_id, FAIL, "H5Dopen"); - /* Open the file to check data set value */ - file_id = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); - CHECK(file_id, FAIL, "H5Fopen"); + /* Read in the data of fill value */ + ret = H5Dread(dset_id, dtype1_id, dset_dspace_id, dset_dspace_id, xfer_pid, rbuf); + CHECK(ret, FAIL, "H5Dread"); - /* Write one element & fill values to datasets with different storage layouts */ -#ifdef QAK -HDfprintf(stderr, "Before testing single element writes\n"); -#endif /* QAK */ -HDfprintf(stderr, "Uncomment loop!\n"); -/* for(layout = H5D_COMPACT; layout <= H5D_CHUNKED; layout++) { */ - for(layout = H5D_COMPACT; layout <= H5D_CONTIGUOUS; layout++) { - - /* Layout specific actions */ - switch(layout) { - case H5D_COMPACT: - HDstrcpy(dset_name1, "dataset1-compact"); - HDstrcpy(dset_name2, "dataset2-compact"); - dset_dspace_id = small_dspace_id; - dset_elmts = SPACE4_DIM_SMALL; - break; - - case H5D_CONTIGUOUS: - HDstrcpy(dset_name1, "dataset1-contig"); - HDstrcpy(dset_name2, "dataset2-contig"); - dset_dspace_id = large_dspace_id; - dset_elmts = SPACE4_DIM_LARGE; - break; - - case H5D_CHUNKED: - HDstrcpy(dset_name1, "dataset1-chunked"); - HDstrcpy(dset_name2, "dataset2-chunked"); - dset_dspace_id = large_dspace_id; - dset_elmts = SPACE4_DIM_LARGE; - break; - } /* end switch */ - - /* Copy the dataset's dataspace */ - single_dspace_id = H5Scopy(dset_dspace_id); - CHECK(single_dspace_id, FAIL, "H5Scopy"); - - /* Set a single element in the dataspace */ - ret = H5Sselect_hyperslab(single_dspace_id, H5S_SELECT_SET, single_offset, - NULL, single_block, NULL); - CHECK(ret, FAIL, "H5Sselect_hyperslab"); - - /* Open first data set */ -#ifdef QAK -HDfprintf(stderr, "Before opening first dataset: '%s'\n", dset_name1); -#endif /* QAK */ - dset_id = H5Dopen(file_id, dset_name1); -#ifdef QAK -HDfprintf(stderr, "After opening first dataset\n"); -#endif /* QAK */ - CHECK(dset_id, FAIL, "H5Dopen"); - - /* Write one element in the dataset */ -#ifdef QAK -HDfprintf(stderr, "Before writing to first dataset\n"); -#endif /* QAK */ - ret = H5Dwrite(dset_id, dtype1_id, scalar_dspace_id, single_dspace_id, xfer_pid, &wdata); -#ifdef QAK -HDfprintf(stderr, "After writing to first dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dwrite"); - -#ifdef QAK -HDfprintf(stderr, "Before reading from first dataset\n"); -#endif /* QAK */ - ret = H5Dread(dset_id, dtype1_id, dset_dspace_id, dset_dspace_id, xfer_pid, rbuf); -#ifdef QAK -HDfprintf(stderr, "After reading from first dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dread"); - - /* Compare data read in */ - for(i = 0; i < dset_elmts; i++) { - if(i == single_offset[0]) { - if(HDstrcmp(rbuf[i].str_id, wdata.str_id) - || rbuf[i].str_name - || HDstrcmp(rbuf[i].str_desc, wdata.str_desc) - || HDstrcmp(rbuf[i].str_orig, wdata.str_orig) - || HDstrcmp(rbuf[i].str_stat, wdata.str_stat) - || HDstrcmp(rbuf[i].str_form, wdata.str_form) - || HDstrcmp(rbuf[i].str_unit, wdata.str_unit)) { - TestErrPrintf("%d: VL data doesn't match!, index(i)=%d\n",__LINE__,(int)i); - continue; - } /* end if */ - } /* end if */ - else { + /* Compare data read in */ + for(i = 0; i < dset_elmts; i++) { if(HDstrcmp(rbuf[i].str_id, "foobar") || HDstrcmp(rbuf[i].str_name, "") || rbuf[i].str_desc - || HDstrcmp(rbuf[i].str_orig,"\0") + || HDstrcmp(rbuf[i].str_orig, "\0") || HDstrcmp(rbuf[i].str_stat, "dead") || HDstrcmp(rbuf[i].str_form, "liquid") || HDstrcmp(rbuf[i].str_unit, "meter")) { - TestErrPrintf("%d: VL data doesn't match!, index(i)=%d\n",__LINE__,(int)i); + TestErrPrintf("%d: VL data doesn't match!, index(i) = %d\n", __LINE__, (int)i); continue; } /* end if */ - } /* end if */ - } /* end for */ + } /* end for */ + ret = H5Dclose(dset_id); + CHECK(ret, FAIL, "H5Dclose"); -#ifdef QAK -HDfprintf(stderr, "Before closing first dataset\n"); -#endif /* QAK */ - ret = H5Dclose(dset_id); -#ifdef QAK -HDfprintf(stderr, "After closing first dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dclose"); - - /* Release the space */ -#ifdef QAK -HDfprintf(stderr, "Before reclaiming space\n"); -#endif /* QAK */ - ret = H5Dvlen_reclaim(dtype1_id, dset_dspace_id, xfer_pid, rbuf); -#ifdef QAK -HDfprintf(stderr, "After reclaiming space\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dvlen_reclaim"); - - -#ifndef QAK2 - /* Open the second data set to check the value of data */ -#ifdef QAK -HDfprintf(stderr, "Before opening second dataset: '%s'\n", dset_name2); -#endif /* QAK */ - dset_id = H5Dopen(file_id, dset_name2); -#ifdef QAK -HDfprintf(stderr, "After opening second dataset\n"); -#endif /* QAK */ - CHECK(dset_id, FAIL, "H5Dopen"); - - /* Write one element in the dataset */ -#ifdef QAK -HDfprintf(stderr, "Before writing to second dataset\n"); -#endif /* QAK */ - ret = H5Dwrite(dset_id, dtype1_id, scalar_dspace_id, single_dspace_id, xfer_pid, &wdata); -#ifdef QAK -HDfprintf(stderr, "After writing to second dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dwrite"); - -#ifdef QAK -HDfprintf(stderr, "Before reading from second dataset\n"); -#endif /* QAK */ - ret = H5Dread(dset_id, dtype1_id, dset_dspace_id, dset_dspace_id, xfer_pid, rbuf); -#ifdef QAK -HDfprintf(stderr, "After reading from second dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dread"); - - /* Compare data read in */ - for(i = 0; i < dset_elmts; i++) { - if(i == single_offset[0]) { - if(HDstrcmp(rbuf[i].str_id, wdata.str_id) - || rbuf[i].str_name - || HDstrcmp(rbuf[i].str_desc, wdata.str_desc) - || HDstrcmp(rbuf[i].str_orig, wdata.str_orig) - || HDstrcmp(rbuf[i].str_stat, wdata.str_stat) - || HDstrcmp(rbuf[i].str_form, wdata.str_form) - || HDstrcmp(rbuf[i].str_unit, wdata.str_unit)) { - TestErrPrintf("%d: VL data doesn't match!, index(i)=%d\n",__LINE__,(int)i); - continue; - } /* end if */ - } /* end if */ - else { + /* Release the space */ + ret = H5Dvlen_reclaim(dtype1_id, dset_dspace_id, xfer_pid, rbuf); + CHECK(ret, FAIL, "H5Dvlen_reclaim"); + + + /* Open the second data set to check the value of data */ + dset_id = H5Dopen(file_id, dset_name2); + CHECK(dset_id, FAIL, "H5Dopen"); + + ret = H5Dread(dset_id, dtype1_id, dset_dspace_id, dset_dspace_id, xfer_pid, rbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Compare data read in */ + for(i = 0; i < dset_elmts; i++) { if(HDstrcmp(rbuf[i].str_id, "foobar") || HDstrcmp(rbuf[i].str_name, "") || rbuf[i].str_desc - || HDstrcmp(rbuf[i].str_orig,"\0") + || HDstrcmp(rbuf[i].str_orig, "\0") || HDstrcmp(rbuf[i].str_stat, "dead") || HDstrcmp(rbuf[i].str_form, "liquid") || HDstrcmp(rbuf[i].str_unit, "meter")) { TestErrPrintf("%d: VL data doesn't match!, index(i)=%d\n",__LINE__,(int)i); continue; } /* end if */ - } /* end if */ + } /* end for */ + + ret = H5Dclose(dset_id); + CHECK(ret, FAIL, "H5Dclose"); + + /* Release the space */ + ret = H5Dvlen_reclaim(dtype1_id, dset_dspace_id, xfer_pid, rbuf); + CHECK(ret, FAIL, "H5Dvlen_reclaim"); } /* end for */ + } /* end for */ + + ret = H5Fclose(file_id); + CHECK(ret, FAIL, "H5Fclose"); + + + /* Open the file to check data set value */ + file_id = H5Fopen(FILENAME, H5F_ACC_RDWR, H5P_DEFAULT); + CHECK(file_id, FAIL, "H5Fopen"); + + /* Write one element & fill values to datasets with different storage layouts */ + for(layout = H5D_COMPACT; layout <= H5D_CHUNKED; layout++) { + unsigned compress_loop; /* # of times to run loop, for testing compressed chunked dataset */ + unsigned test_loop; /* Loop over datasets */ + +#ifdef H5_HAVE_FILTER_DEFLATE + if(layout == H5D_CHUNKED) + compress_loop = 2; + else +#endif /* H5_HAVE_FILTER_DEFLATE */ + compress_loop = 1; + + /* Loop over dataset operations */ + for(test_loop = 0; test_loop < compress_loop; test_loop++) { + + /* Layout specific actions */ + switch(layout) { + case H5D_COMPACT: + HDstrcpy(dset_name1, "dataset1-compact"); + HDstrcpy(dset_name2, "dataset2-compact"); + dset_dspace_id = small_dspace_id; + dset_elmts = SPACE4_DIM_SMALL; + break; + + case H5D_CONTIGUOUS: + HDstrcpy(dset_name1, "dataset1-contig"); + HDstrcpy(dset_name2, "dataset2-contig"); + dset_dspace_id = large_dspace_id; + dset_elmts = SPACE4_DIM_LARGE; + break; + + case H5D_CHUNKED: +#ifdef H5_HAVE_FILTER_DEFLATE + if(test_loop == 1) { + HDstrcpy(dset_name1, "dataset1-chunked-compressed"); + HDstrcpy(dset_name2, "dataset2-chunked-compressed"); + } /* end if */ + else { +#endif /* H5_HAVE_FILTER_DEFLATE */ + HDstrcpy(dset_name1, "dataset1-chunked"); + HDstrcpy(dset_name2, "dataset2-chunked"); +#ifdef H5_HAVE_FILTER_DEFLATE + } /* end else */ +#endif /* H5_HAVE_FILTER_DEFLATE */ + dset_dspace_id = large_dspace_id; + dset_elmts = SPACE4_DIM_LARGE; + break; + } /* end switch */ + + /* Copy the dataset's dataspace */ + single_dspace_id = H5Scopy(dset_dspace_id); + CHECK(single_dspace_id, FAIL, "H5Scopy"); + + /* Set a single element in the dataspace */ + ret = H5Sselect_hyperslab(single_dspace_id, H5S_SELECT_SET, single_offset, + NULL, single_block, NULL); + CHECK(ret, FAIL, "H5Sselect_hyperslab"); + + /* Open first data set */ + dset_id = H5Dopen(file_id, dset_name1); + CHECK(dset_id, FAIL, "H5Dopen"); + + /* Write one element in the dataset */ + ret = H5Dwrite(dset_id, dtype1_id, scalar_dspace_id, single_dspace_id, xfer_pid, &wdata); + CHECK(ret, FAIL, "H5Dwrite"); + + ret = H5Dread(dset_id, dtype1_id, dset_dspace_id, dset_dspace_id, xfer_pid, rbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Compare data read in */ + for(i = 0; i < dset_elmts; i++) { + if(i == single_offset[0]) { + if(HDstrcmp(rbuf[i].str_id, wdata.str_id) + || rbuf[i].str_name + || HDstrcmp(rbuf[i].str_desc, wdata.str_desc) + || HDstrcmp(rbuf[i].str_orig, wdata.str_orig) + || HDstrcmp(rbuf[i].str_stat, wdata.str_stat) + || HDstrcmp(rbuf[i].str_form, wdata.str_form) + || HDstrcmp(rbuf[i].str_unit, wdata.str_unit)) { + TestErrPrintf("%d: VL data doesn't match!, index(i)=%d\n",__LINE__,(int)i); + continue; + } /* end if */ + } /* end if */ + else { + if(HDstrcmp(rbuf[i].str_id, "foobar") + || HDstrcmp(rbuf[i].str_name, "") + || rbuf[i].str_desc + || HDstrcmp(rbuf[i].str_orig,"\0") + || HDstrcmp(rbuf[i].str_stat, "dead") + || HDstrcmp(rbuf[i].str_form, "liquid") + || HDstrcmp(rbuf[i].str_unit, "meter")) { + TestErrPrintf("%d: VL data doesn't match!, index(i)=%d\n",__LINE__,(int)i); + continue; + } /* end if */ + } /* end if */ + } /* end for */ + + + ret = H5Dclose(dset_id); + CHECK(ret, FAIL, "H5Dclose"); + + /* Release the space */ + ret = H5Dvlen_reclaim(dtype1_id, dset_dspace_id, xfer_pid, rbuf); + CHECK(ret, FAIL, "H5Dvlen_reclaim"); + + + /* Open the second data set to check the value of data */ + dset_id = H5Dopen(file_id, dset_name2); + CHECK(dset_id, FAIL, "H5Dopen"); + + /* Write one element in the dataset */ + ret = H5Dwrite(dset_id, dtype1_id, scalar_dspace_id, single_dspace_id, xfer_pid, &wdata); + CHECK(ret, FAIL, "H5Dwrite"); -#ifdef QAK -HDfprintf(stderr, "Before closing second dataset\n"); -#endif /* QAK */ - ret = H5Dclose(dset_id); -#ifdef QAK -HDfprintf(stderr, "After closing second dataset\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dclose"); - - /* Release the space */ -#ifdef QAK -HDfprintf(stderr, "Before reclaiming space\n"); -#endif /* QAK */ - ret = H5Dvlen_reclaim(dtype1_id, dset_dspace_id, xfer_pid, rbuf); -#ifdef QAK -HDfprintf(stderr, "After reclaiming space\n"); -#endif /* QAK */ - CHECK(ret, FAIL, "H5Dvlen_reclaim"); -#else /* QAK2 */ -HDfprintf(stderr, "Uncomment test for second dataset!\n"); -#endif /* QAK2 */ - - /* Close the dataspace for the writes */ - ret = H5Sclose(single_dspace_id); - CHECK(ret, FAIL, "H5Sclose"); + ret = H5Dread(dset_id, dtype1_id, dset_dspace_id, dset_dspace_id, xfer_pid, rbuf); + CHECK(ret, FAIL, "H5Dread"); + + /* Compare data read in */ + for(i = 0; i < dset_elmts; i++) { + if(i == single_offset[0]) { + if(HDstrcmp(rbuf[i].str_id, wdata.str_id) + || rbuf[i].str_name + || HDstrcmp(rbuf[i].str_desc, wdata.str_desc) + || HDstrcmp(rbuf[i].str_orig, wdata.str_orig) + || HDstrcmp(rbuf[i].str_stat, wdata.str_stat) + || HDstrcmp(rbuf[i].str_form, wdata.str_form) + || HDstrcmp(rbuf[i].str_unit, wdata.str_unit)) { + TestErrPrintf("%d: VL data doesn't match!, index(i)=%d\n",__LINE__,(int)i); + continue; + } /* end if */ + } /* end if */ + else { + if(HDstrcmp(rbuf[i].str_id, "foobar") + || HDstrcmp(rbuf[i].str_name, "") + || rbuf[i].str_desc + || HDstrcmp(rbuf[i].str_orig,"\0") + || HDstrcmp(rbuf[i].str_stat, "dead") + || HDstrcmp(rbuf[i].str_form, "liquid") + || HDstrcmp(rbuf[i].str_unit, "meter")) { + TestErrPrintf("%d: VL data doesn't match!, index(i)=%d\n",__LINE__,(int)i); + continue; + } /* end if */ + } /* end if */ + } /* end for */ + + ret = H5Dclose(dset_id); + CHECK(ret, FAIL, "H5Dclose"); + + /* Release the space */ + ret = H5Dvlen_reclaim(dtype1_id, dset_dspace_id, xfer_pid, rbuf); + CHECK(ret, FAIL, "H5Dvlen_reclaim"); + + /* Close the dataspace for the writes */ + ret = H5Sclose(single_dspace_id); + CHECK(ret, FAIL, "H5Sclose"); + } /* end for */ } /* end for */ ret = H5Fclose(file_id); @@ -3043,7 +2955,6 @@ test_vltypes(void) MESSAGE(5, ("Testing Variable-Length Datatypes\n")); /* These next tests use the same file */ -#ifndef QAK test_vltypes_dataset_create(); /* Check dataset of VL when fill value * won't be rewritten to it.*/ test_vltypes_vlen_atomic(); /* Test VL atomic datatypes */ @@ -3057,9 +2968,6 @@ test_vltypes(void) rewrite_shorter_vltypes_vlen_vlen_atomic(); /*overwrite with VL data of shorted sequence*/ test_vltypes_compound_vlen_vlen(); /* Test compound datatypes with VL atomic components */ test_vltypes_compound_vlstr(); /* Test data rewritten of nested VL data */ -#else /* QAK */ -HDfprintf(stderr, "Uncomment tests!\n"); -#endif /* QAK */ test_vltypes_fill_value(); /* Test fill value for VL data */ } /* test_vltypes() */ |