From 15dbc76104b87ff2aa748e73633cfe87270e3807 Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Thu, 7 Jun 2007 22:06:56 -0500 Subject: [svn-r13842] Description: Fix compact dataset storage to initialize VL datatype fill values correctly. Also, fix bug in global heap code when the fix action on a global heap is to delete an object in a heap with no free space. Tested on: Mac OS X/32 10.4.9 (amazon) FreeBSD/32 6.2 (duty) --- src/H5D.c | 18 +--- src/H5Dcompact.c | 130 +++++++++++++++++++++++ src/H5Dio.c | 5 +- src/H5Dpkg.h | 1 + src/H5HG.c | 38 ++++--- test/tvltypes.c | 319 +++++++++++++++++++++++++++++++++++++++++++------------ 6 files changed, 405 insertions(+), 106 deletions(-) diff --git a/src/H5D.c b/src/H5D.c index dbd2315..8aa5996 100644 --- a/src/H5D.c +++ b/src/H5D.c @@ -2497,21 +2497,9 @@ H5D_init_storage(H5D_t *dset, hbool_t full_overwrite, hid_t dxpl_id) case H5D_COMPACT: /* If we will be immediately overwriting the values, don't bother to clear them */ if(!full_overwrite) { - /* If the fill value is defined, initialize the data buffer with it */ - if(dset->shared->dcpl_cache.fill.buf) { - hssize_t snpoints; /* Number of points in space (for error checking) */ - size_t npoints; /* Number of points in space */ - - /* Get the number of elements in the dataset's dataspace */ - snpoints = H5S_GET_EXTENT_NPOINTS(dset->shared->space); - HDassert(snpoints >= 0); - H5_ASSIGN_OVERFLOW(npoints, snpoints, hssize_t, size_t); - - /* Initialize the cached data buffer with the fill value */ - H5V_array_fill(dset->shared->layout.u.compact.buf, dset->shared->dcpl_cache.fill.buf, (size_t)dset->shared->dcpl_cache.fill.size, npoints); - } /* end if */ - else /* If the fill value is default, zero set data buf. */ - HDmemset(dset->shared->layout.u.compact.buf, 0, dset->shared->layout.u.compact.size); + /* Fill the compact dataset storage */ + if(H5D_compact_fill(dset, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "unable to initialize compact dataset storage") } /* end if */ break; diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c index 36b5357..a9b2007 100644 --- a/src/H5Dcompact.c +++ b/src/H5Dcompact.c @@ -66,6 +66,136 @@ H5FL_BLK_EXTERN(type_conv); /*------------------------------------------------------------------------- + * Function: H5D_compact_fill + * + * Purpose: Write fill values to a compactly stored dataset. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Quincey Koziol + * May 6, 2007 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_compact_fill(H5D_t *dset, hid_t dxpl_id) +{ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_compact_fill, FAIL) + + /* Check args */ + HDassert(TRUE == H5P_isa_class(dxpl_id, H5P_DATASET_XFER)); + HDassert(dset && H5D_COMPACT == dset->shared->layout.type); + HDassert(dset->shared->layout.u.compact.buf); + HDassert(dset->shared->type); + HDassert(dset->shared->space); + + /* If the fill value is defined, initialize the data buffer with it */ + if(dset->shared->dcpl_cache.fill.buf) { + hssize_t snpoints; /* Number of points in space (for error checking) */ + size_t npoints; /* Number of points in space */ + + /* Get the number of elements in the dataset's dataspace */ + snpoints = H5S_GET_EXTENT_NPOINTS(dset->shared->space); + HDassert(snpoints >= 0); + H5_ASSIGN_OVERFLOW(npoints, snpoints, hssize_t, size_t); + + /* If necessary, convert fill value datatypes (which copies VL components, etc.) */ + if(H5T_detect_class(dset->shared->type, H5T_VLEN) > 0) { + H5T_path_t *tpath; /* Datatype conversion path */ + uint8_t *bkg_buf = NULL; /* Background conversion buffer */ + H5T_t *mem_type; /* Pointer to memory datatype */ + size_t mem_type_size, file_type_size; /* Size of datatype in memory and on disk */ + hid_t mem_tid; /* Memory version of disk datatype */ + + /* Create temporary datatype for conversion operation */ + if(NULL == (mem_type = H5T_copy(dset->shared->type, H5T_COPY_REOPEN))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTCOPY, FAIL, "unable to copy file datatype") + if((mem_tid = H5I_register(H5I_DATATYPE, mem_type)) < 0) { + H5T_close(mem_type); + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") + } /* end if */ + + /* Retrieve sizes of memory & file datatypes */ + mem_type_size = H5T_get_size(mem_type); + HDassert(mem_type_size > 0); + file_type_size = H5T_get_size(dset->shared->type); + HDassert((ssize_t)file_type_size == dset->shared->dcpl_cache.fill.size); + + /* Get the datatype conversion path for this operation */ + if(NULL == (tpath = H5T_path_find(dset->shared->type, mem_type, NULL, NULL, dxpl_id, FALSE))) { + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") + } /* end if */ + + /* Allocate a background buffer, if necessary */ + if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (npoints * MAX(mem_type_size, file_type_size))))) { + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + } /* end if */ + + /* Make a copy of the (disk-based) fill value into the compact buffer */ + HDmemcpy(dset->shared->layout.u.compact.buf, dset->shared->dcpl_cache.fill.buf, file_type_size); + + /* Type convert the compact dataset buffer, to copy any VL components */ + if(H5T_convert(tpath, dset->shared->type_id, mem_tid, (size_t)1, (size_t)0, (size_t)0, dset->shared->layout.u.compact.buf, bkg_buf, dxpl_id) < 0) { + if(bkg_buf) + H5FL_BLK_FREE(type_conv, bkg_buf); + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") + } /* end if */ + + /* Replicate the fill value into the cached buffer */ + H5V_array_fill(dset->shared->layout.u.compact.buf, dset->shared->layout.u.compact.buf, mem_type_size, npoints); + + /* Get the inverse datatype conversion path for this operation */ + if(NULL == (tpath = H5T_path_find(mem_type, dset->shared->type, NULL, NULL, dxpl_id, FALSE))) { + if(bkg_buf) + H5FL_BLK_FREE(type_conv, bkg_buf); + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and dst datatypes") + } /* end if */ + + /* Allocate or reset the background buffer, if necessary */ + if(H5T_path_bkg(tpath)) { + if(bkg_buf) + HDmemset(bkg_buf, 0, MAX(mem_type_size, file_type_size)); + else { + if(NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (npoints * MAX(mem_type_size, file_type_size))))) { + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + } /* end if */ + } /* end else */ + } /* end if */ + + /* Type convert the compact dataset buffer, to copy any VL components */ + if(H5T_convert(tpath, mem_tid, dset->shared->type_id, npoints, (size_t)0, (size_t)0, dset->shared->layout.u.compact.buf, bkg_buf, dxpl_id) < 0) { + if(bkg_buf) + H5FL_BLK_FREE(type_conv, bkg_buf); + H5I_dec_ref(mem_tid); + HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "data type conversion failed") + } /* end if */ + + /* Release resources used */ + if(bkg_buf) + H5FL_BLK_FREE(type_conv, bkg_buf); + H5I_dec_ref(mem_tid); + } /* end if */ + else + /* Replicate the fill value into the cached buffer */ + H5V_array_fill(dset->shared->layout.u.compact.buf, dset->shared->dcpl_cache.fill.buf, (size_t)dset->shared->dcpl_cache.fill.size, npoints); + + } /* end if */ + else /* If the fill value is default, zero set data buf. */ + HDmemset(dset->shared->layout.u.compact.buf, 0, dset->shared->layout.u.compact.size); + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_compact_fill() */ + + +/*------------------------------------------------------------------------- * Function: H5D_compact_readvv * * Purpose: Reads some data vectors from a dataset into a buffer. diff --git a/src/H5Dio.c b/src/H5Dio.c index 9fd24d3..3c36953 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -284,9 +284,10 @@ H5D_fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_t /* Convert disk buffer into memory buffer */ if(!H5T_path_noop(tpath)) { /* Allocate a background buffer */ - if(NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (size_t)nelmts * buf_size))) + if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_conv, (size_t)nelmts * buf_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") + /* Perform datatype conversion */ if(H5T_convert(tpath, src_id, dst_id, (size_t)nelmts, (size_t)0, (size_t)0, tmp_buf, bkg_buf, dxpl_id) < 0) HGOTO_ERROR(H5E_DATASET, H5E_CANTCONVERT, FAIL, "datatype conversion failed") } /* end if */ @@ -298,7 +299,7 @@ H5D_fill(const void *fill, const H5T_t *fill_type, void *buf, const H5T_t *buf_t if(!H5T_path_noop(tpath)) { /* If there's no VL type of data, do conversion first then fill the data into * the memory buffer. */ - if(NULL == (bkg_buf = H5FL_BLK_CALLOC(type_elem, buf_size))) + if(H5T_path_bkg(tpath) && NULL == (bkg_buf = H5FL_BLK_CALLOC(type_elem, buf_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed") /* Perform datatype conversion */ diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 4bbaeff..ef2c0ba 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -282,6 +282,7 @@ H5_DLL herr_t H5D_contig_copy(H5F_t *f_src, const H5O_layout_t *layout_src, H5F_ H5O_layout_t *layout_dst, H5T_t *src_dtype, H5O_copy_t *cpy_info, hid_t dxpl_id); /* Functions that operate on compact dataset storage */ +H5_DLL herr_t H5D_compact_fill(H5D_t *dset, hid_t dxpl_id); H5_DLL ssize_t H5D_compact_readvv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_size_arr[], hsize_t dset_offset_arr[], size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_size_arr[], hsize_t mem_offset_arr[], diff --git a/src/H5HG.c b/src/H5HG.c index a4cb233..4ee63d1 100644 --- a/src/H5HG.c +++ b/src/H5HG.c @@ -456,26 +456,24 @@ H5HG_load (H5F_t *f, hid_t dxpl_id, haddr_t addr, const void UNUSED * udata1, * necessary to make room. We remove the right-most entry that has less * free space than this heap. */ - if (heap->obj[0].size>0) { - if (!f->shared->cwfs) { - f->shared->cwfs = H5MM_malloc (H5HG_NCWFS*sizeof(H5HG_heap_t*)); - if (NULL==f->shared->cwfs) - HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); - f->shared->ncwfs = 1; - f->shared->cwfs[0] = heap; - } else if (H5HG_NCWFS==f->shared->ncwfs) { - for (i=H5HG_NCWFS-1; i>=0; --i) { - if (f->shared->cwfs[i]->obj[0].size < heap->obj[0].size) { - HDmemmove (f->shared->cwfs+1, f->shared->cwfs, i * sizeof(H5HG_heap_t*)); - f->shared->cwfs[0] = heap; - break; - } - } - } else { - HDmemmove (f->shared->cwfs+1, f->shared->cwfs, f->shared->ncwfs*sizeof(H5HG_heap_t*)); - f->shared->ncwfs += 1; - f->shared->cwfs[0] = heap; - } + if (!f->shared->cwfs) { + f->shared->cwfs = H5MM_malloc (H5HG_NCWFS*sizeof(H5HG_heap_t*)); + if (NULL==f->shared->cwfs) + HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed"); + f->shared->ncwfs = 1; + f->shared->cwfs[0] = heap; + } else if (H5HG_NCWFS==f->shared->ncwfs) { + for (i=H5HG_NCWFS-1; i>=0; --i) { + if (f->shared->cwfs[i]->obj[0].size < heap->obj[0].size) { + HDmemmove (f->shared->cwfs+1, f->shared->cwfs, i * sizeof(H5HG_heap_t*)); + f->shared->cwfs[0] = heap; + break; + } + } + } else { + HDmemmove (f->shared->cwfs+1, f->shared->cwfs, f->shared->ncwfs*sizeof(H5HG_heap_t*)); + f->shared->ncwfs += 1; + f->shared->cwfs[0] = heap; } ret_value = heap; diff --git a/test/tvltypes.c b/test/tvltypes.c index b6a5509..c41bf0e 100644 --- a/test/tvltypes.c +++ b/test/tvltypes.c @@ -2417,17 +2417,30 @@ test_vltypes_fill_value(void) hid_t dtype1_id = -1; hid_t str_id = -1; hid_t dspace_id; + hid_t scalar_dspace_id; /* Dataspace ID for scalar dataspace */ + hid_t single_dspace_id; /* Dataspace ID for single element selection */ + hsize_t single_offset[] = {2}; /* Offset of single element selection */ + hsize_t single_block[] = {1}; /* Block size of single element selection */ hid_t dcpl_id, xfer_pid; hid_t dset_id; - hsize_t dim1[] = {SPACE1_DIM1}; + hsize_t dim1[] = {SPACE3_DIM1}; const dtype1_struct fill1 = {1, 2, "foobar", "", NULL, "\0", "dead", 3, 4.0, 100.0, 1.0, "liquid", "meter"}; - dtype1_struct buf[SPACE1_DIM1]; + const dtype1_struct wdata = {3, 4, "", NULL, "\0", "foo", "two", 6, 8.0, 200.0, 2.0, "solid", "yard"}; + dtype1_struct buf[SPACE3_DIM1]; size_t mem_used=0; /* Memory used during allocation */ + H5D_layout_t layout; /* Dataset storage layout */ + char dset_name1[64], dset_name2[64]; /* Dataset names */ int i; /* Output message about test being performed */ MESSAGE(5, ("Check fill value for VL data\n")); + /* Create a string datatype */ + str_id = H5Tcopy(H5T_C_S1); + CHECK(str_id, FAIL, "H5Tcopy"); + ret = H5Tset_size(str_id,H5T_VARIABLE); + CHECK(ret, FAIL, "H5Tset_size"); + /* Create a compound data type */ dtype1_id = H5Tcreate(H5T_COMPOUND, sizeof(struct dtype1_struct)); CHECK(dtype1_id, FAIL, "H5Tcreate"); @@ -2438,12 +2451,6 @@ test_vltypes_fill_value(void) ret = H5Tinsert(dtype1_id,"pguid",HOFFSET(struct dtype1_struct,pgui),H5T_NATIVE_UINT); CHECK(ret, FAIL, "H5Tinsert"); - - str_id = H5Tcopy(H5T_C_S1); - CHECK(str_id, FAIL, "H5Tcopy"); - ret = H5Tset_size(str_id,H5T_VARIABLE); - CHECK(ret, FAIL, "H5Tset_size"); - ret = H5Tinsert(dtype1_id,"str_id",HOFFSET(dtype1_struct,str_id),str_id); CHECK(ret, FAIL, "H5Tinsert"); @@ -2459,7 +2466,6 @@ test_vltypes_fill_value(void) ret = H5Tinsert(dtype1_id,"str_stat",HOFFSET(dtype1_struct,str_stat),str_id); CHECK(ret, FAIL, "H5Tinsert"); - ret = H5Tinsert(dtype1_id,"ver",HOFFSET(struct dtype1_struct,ver),H5T_NATIVE_UINT); CHECK(ret, FAIL, "H5Tinsert"); @@ -2472,17 +2478,25 @@ test_vltypes_fill_value(void) ret = H5Tinsert(dtype1_id,"mi",HOFFSET(struct dtype1_struct,mi),H5T_NATIVE_DOUBLE); CHECK(ret, FAIL, "H5Tinsert"); - ret = H5Tinsert(dtype1_id,"str_form",HOFFSET(dtype1_struct,str_form),str_id); CHECK(ret, FAIL, "H5Tinsert"); ret = H5Tinsert(dtype1_id,"str_unit",HOFFSET(dtype1_struct,str_unit),str_id); CHECK(ret, FAIL, "H5Tinsert"); - + /* Close string datatype */ ret = H5Tclose(str_id); CHECK(ret, FAIL, "H5Tclose"); + + /* Create the main dataspace to use */ + dspace_id = H5Screate_simple(SPACE3_RANK, dim1, NULL); + CHECK(dspace_id, FAIL, "H5Screate_simple"); + + /* Create a scalar dataspace */ + scalar_dspace_id = H5Screate(H5S_SCALAR); + CHECK(scalar_dspace_id, FAIL, "H5Screate"); + /* Create dataset create property list and set the fill value */ dcpl_id = H5Pcreate(H5P_DATASET_CREATE); CHECK(dcpl_id, FAIL, "H5Pcreate"); @@ -2494,40 +2508,72 @@ test_vltypes_fill_value(void) file_id = H5Fcreate(FILENAME, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT); CHECK(file_id, FAIL, "H5Fcreate"); - dspace_id=H5Screate_simple(SPACE1_RANK,dim1,NULL); - CHECK(dspace_id, FAIL, "H5Screate_simple"); + /* Create datasets with different storage layouts */ + for(layout = H5D_COMPACT; layout <= H5D_CHUNKED; layout++) { + hid_t tmp_dcpl_id; /* Temporary copy of the dataset creation property list */ - /* Create first data set with default setting - no space is allocated */ - dset_id = H5Dcreate(file_id, "dataset1", dtype1_id, dspace_id, dcpl_id); - CHECK(dset_id, FAIL, "H5Dcreate"); + /* Make a copy of the dataset creation property list */ + tmp_dcpl_id = H5Pcopy(dcpl_id); + CHECK(tmp_dcpl_id, FAIL, "H5Pcopy"); - ret = H5Dclose(dset_id); - CHECK(ret, FAIL, "H5Dclose"); + /* Layout specific actions */ + switch(layout) { + case H5D_COMPACT: + HDstrcpy(dset_name1, "dataset1-compact"); + HDstrcpy(dset_name2, "dataset2-compact"); + ret = H5Pset_layout(tmp_dcpl_id, H5D_COMPACT); + CHECK(ret, FAIL, "H5Pset_layout"); + break; - /* Create a second data set with space allocated and fill value written */ - ret = H5Pset_fill_time(dcpl_id, H5D_FILL_TIME_IFSET); - CHECK(ret, FAIL, "H5Pset_fill_time"); + case H5D_CONTIGUOUS: + HDstrcpy(dset_name1, "dataset1-contig"); + HDstrcpy(dset_name2, "dataset2-contig"); + break; - ret = H5Pset_alloc_time(dcpl_id, H5D_ALLOC_TIME_EARLY); - CHECK(ret, FAIL, "H5Pset_alloc_time"); + case H5D_CHUNKED: + { + hsize_t chunk_dims[1] = {SPACE3_DIM1 / 4}; - dset_id = H5Dcreate(file_id, "dataset2", dtype1_id, dspace_id, dcpl_id); - CHECK(dset_id, FAIL, "H5Dcreate"); + HDstrcpy(dset_name1, "dataset1-chunked"); + HDstrcpy(dset_name2, "dataset2-chunked"); + ret = H5Pset_chunk(tmp_dcpl_id, 1, chunk_dims); + CHECK(ret, FAIL, "H5Pset_chunk"); + } + break; + } /* end switch */ - ret = H5Dclose(dset_id); - CHECK(ret, FAIL, "H5Dclose"); + /* Create first data set with default setting - no space is allocated */ + dset_id = H5Dcreate(file_id, dset_name1, dtype1_id, dspace_id, tmp_dcpl_id); + CHECK(dset_id, FAIL, "H5Dcreate"); + + ret = H5Dclose(dset_id); + CHECK(ret, FAIL, "H5Dclose"); + + + /* Create a second data set with space allocated and fill value written */ + ret = H5Pset_fill_time(tmp_dcpl_id, H5D_FILL_TIME_IFSET); + CHECK(ret, FAIL, "H5Pset_fill_time"); + + ret = H5Pset_alloc_time(tmp_dcpl_id, H5D_ALLOC_TIME_EARLY); + CHECK(ret, FAIL, "H5Pset_alloc_time"); + + dset_id = H5Dcreate(file_id, dset_name2, dtype1_id, dspace_id, tmp_dcpl_id); + CHECK(dset_id, FAIL, "H5Dcreate"); + + ret = H5Dclose(dset_id); + CHECK(ret, FAIL, "H5Dclose"); + + /* Close temporary DCPL */ + ret = H5Pclose(tmp_dcpl_id); + CHECK(ret, FAIL, "H5Pclose"); + } /* end for */ ret = H5Fclose(file_id); CHECK(ret, FAIL, "H5Fclose"); + ret = H5Pclose(dcpl_id); + CHECK(ret, FAIL, "H5Pclose"); - /* Open the file to check data set value */ - file_id = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); - CHECK(file_id, FAIL, "H5Fopen"); - - /* Open first data set */ - dset_id = H5Dopen(file_id, "dataset1"); - CHECK(dset_id, FAIL, "H5Dopen"); /* Change to the custom memory allocation routines for reading VL data */ xfer_pid=H5Pcreate(H5P_DATASET_XFER); @@ -2536,58 +2582,193 @@ test_vltypes_fill_value(void) ret=H5Pset_vlen_mem_manager(xfer_pid,test_vltypes_alloc_custom,&mem_used,test_vltypes_free_custom,&mem_used); CHECK(ret, FAIL, "H5Pset_vlen_mem_manager"); - /* Read in the data of fill value */ - ret = H5Dread(dset_id, dtype1_id, dspace_id, dspace_id,xfer_pid, buf); - CHECK(ret, FAIL, "H5Dread"); + /* Open the file to check data set value */ + file_id = H5Fopen(FILENAME, H5F_ACC_RDONLY, H5P_DEFAULT); + CHECK(file_id, FAIL, "H5Fopen"); - /* Compare data read in */ - for(i=0; i