From d579d6aa5eff6674ae523da1dc35b1bd0d63deee Mon Sep 17 00:00:00 2001 From: Quincey Koziol Date: Sat, 28 Jan 2006 13:31:22 -0500 Subject: [svn-r11899] Purpose: Bug fix & new feature Description: Support variable-length datatypes in compact data storage and chunked data storage, along with attributes. Bug fix on the H5T_vlen_set_loc to allow for changing the file on a variable-length datatype on disk. Platforms tested: FreeBSD 4.11 (sleipnir) Linux 2.4 Can't h5committest right now, due to missing cache files. --- src/H5A.c | 1 + src/H5Dcompact.c | 155 ++++++++++++ src/H5Dcontig.c | 88 ++++--- src/H5Distore.c | 240 ++++++++++++++++-- src/H5Doh.c | 8 + src/H5Dpkg.h | 8 +- src/H5Dprivate.h | 5 + src/H5O.c | 6 +- src/H5Oattr.c | 164 ++++++++++++- src/H5Olayout.c | 23 +- src/H5Opkg.h | 1 + src/H5Opline.c | 47 +++- src/H5R.c | 1 + src/H5Sprivate.h | 5 +- src/H5Sselect.c | 1 + src/H5Tvlen.c | 2 +- test/objcopy.c | 728 +++++++++++++++++++++++++++++++++++++++++++++++-------- 17 files changed, 1313 insertions(+), 170 deletions(-) diff --git a/src/H5A.c b/src/H5A.c index d604618..9c8da81 100644 --- a/src/H5A.c +++ b/src/H5A.c @@ -21,6 +21,7 @@ /* Private header files */ #include "H5private.h" /* Generic Functions */ #include "H5Apkg.h" /* Attributes */ +#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c index 83f9b8c..0ad8e05 100644 --- a/src/H5Dcompact.c +++ b/src/H5Dcompact.c @@ -36,6 +36,7 @@ #include "H5Fprivate.h" /* Files */ #include "H5FDprivate.h" /* File drivers */ #include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ #include "H5Oprivate.h" /* Object headers */ #include "H5Vprivate.h" /* Vector and array functions */ @@ -59,6 +60,9 @@ /* Local Variables */ /*******************/ +/* Declare extern the free list to manage blocks of type conversion data */ +H5FL_BLK_EXTERN(type_conv); + /*------------------------------------------------------------------------- * Function: H5D_compact_readvv @@ -141,3 +145,154 @@ H5D_compact_writevv(const H5D_io_info_t *io_info, done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_compact_writevv() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_compact_copy + * + * Purpose: Copy compact storage raw data from SRC file to DST file. + * + * Return: Non-negative on success, negative on failure. + * + * Programmer: Peter Cao + * December 11, 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_compact_copy(H5F_t *f_src, H5O_layout_t *layout_src, + H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *dt_src, hid_t dxpl_id) +{ + hid_t tid_src = -1; /* Datatype ID for source datatype */ + hid_t tid_dst = -1; /* Datatype ID for destination datatype */ + hid_t tid_mem = -1; /* Datatype ID for memory datatype */ + void *buf = NULL; /* Buffer for copying data */ + void *reclaim_buf = NULL; /* Buffer for reclaiming data */ + hid_t buf_sid = -1; /* ID for buffer dataspace */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_compact_copy, FAIL) + + /* Check args */ + HDassert(f_src); + HDassert(f_dst); + HDassert(layout_src && H5D_COMPACT == layout_src->type); + HDassert(layout_dst && H5D_COMPACT == layout_dst->type); + + /* If there's a source datatype, set up type conversion information */ + if (!dt_src) + /* Type conversion not necessary */ + HDmemcpy(layout_dst->u.compact.buf, layout_src->u.compact.buf, layout_src->u.compact.size); + else { + H5T_path_t *tpath_src_mem, *tpath_mem_dst; /* Datatype conversion paths */ + H5T_t *dt_dst; /* Destination datatype */ + H5T_t *dt_mem; /* Memory datatype */ + H5S_t *buf_space; /* Dataspace describing buffer */ + size_t buf_size; /* Size of copy buffer */ + size_t nelmts; /* Number of elements in buffer */ + size_t src_dt_size; /* Source datatype size */ + size_t tmp_dt_size; /* Temporary datatype size */ + size_t max_dt_size; /* Max atatype size */ + hsize_t buf_dim; /* Dimension for buffer */ + + /* Create datatype ID for src datatype */ + if((tid_src = H5I_register(H5I_DATATYPE, dt_src)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register source file datatype") + + /* create a memory copy of the variable-length datatype */ + if(NULL == (dt_mem = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") + if((tid_mem = H5I_register(H5I_DATATYPE, dt_mem)) < 0) + HGOTO_ERROR (H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") + + /* create variable-length datatype at the destinaton file */ + if(NULL == (dt_dst = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") + if(H5T_set_loc(dt_dst, f_dst, H5T_LOC_DISK) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "cannot mark datatype on disk") + if((tid_dst = H5I_register(H5I_DATATYPE, dt_dst)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register destination file datatype") + + /* Set up the conversion functions */ + if(NULL == (tpath_src_mem = H5T_path_find(dt_src, dt_mem, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and mem datatypes") + if(NULL == (tpath_mem_dst = H5T_path_find(dt_mem, dt_dst, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between mem and dst datatypes") + + /* Determine largest datatype size */ + if(0 == (src_dt_size = H5T_get_size(dt_src))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + if(0 == (tmp_dt_size = H5T_get_size(dt_mem))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + max_dt_size = MAX(src_dt_size, tmp_dt_size); + if(0 == (tmp_dt_size = H5T_get_size(dt_dst))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + max_dt_size = MAX(max_dt_size, tmp_dt_size); + + /* Set number of whole elements that fit in buffer */ + if(0 == (nelmts = layout_src->u.compact.size / src_dt_size)) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "element size too large") + + /* Set up number of bytes to copy, and initial buffer size */ + buf_size = nelmts * max_dt_size; + + /* Create dataspace for number of elements in buffer */ + buf_dim = nelmts; + + /* Create the space and set the initial extent */ + if(NULL == (buf_space = H5S_create_simple((unsigned)1, &buf_dim, NULL))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "can't create simple dataspace") + + /* Atomize */ + if((buf_sid = H5I_register(H5I_DATASPACE, buf_space)) < 0) { + H5S_close(buf_space); + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace ID") + } /* end if */ + + /* Allocate memory for recclaim buf */ + if(NULL == (reclaim_buf = H5FL_BLK_MALLOC(type_conv, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + + /* Allocate memory for copying the chunk */ + if(NULL == (buf = H5FL_BLK_MALLOC(type_conv, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + + HDmemcpy(buf, layout_src->u.compact.buf, layout_src->u.compact.size); + + /* Convert from source file to memory */ + if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed") + + HDmemcpy(reclaim_buf, buf, buf_size); + + /* Convert from memory to destination file */ + if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed") + + HDmemcpy(layout_dst->u.compact.buf, buf, layout_dst->u.compact.size); + + if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to reclaim variable-length data") + } /* end if */ + +done: + if(buf_sid > 0) + if(H5I_dec_ref(buf_sid) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary dataspace ID") + if(tid_src > 0) + if(H5I_dec_ref(tid_src) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(tid_dst > 0) + if(H5I_dec_ref(tid_dst) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(tid_mem > 0) + if(H5I_dec_ref(tid_mem) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(buf) + H5FL_BLK_FREE(type_conv, buf); + if(reclaim_buf) + H5FL_BLK_FREE(type_conv, reclaim_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_compact_copy() */ + diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index 21b2e56..f6d1648 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -42,7 +42,6 @@ #include "H5MFprivate.h" /* File memory management */ #include "H5Oprivate.h" /* Object headers */ #include "H5Pprivate.h" /* Property lists */ -#include "H5Sprivate.h" /* Dataspace functions */ #include "H5Vprivate.h" /* Vector and array functions */ /****************/ @@ -1005,9 +1004,15 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, hid_t tid_src = -1; /* Datatype ID for source datatype */ hid_t tid_dst = -1; /* Datatype ID for destination datatype */ hid_t tid_mem = -1; /* Datatype ID for memory datatype */ + size_t src_dt_size; /* Source datatype size */ + size_t mem_dt_size; /* Memory datatype size */ + size_t dst_dt_size; /* Destination datatype size */ size_t max_dt_size; /* Max. datatype size */ size_t nelmts = 0; /* Number of elements in buffer */ - hsize_t total_nbytes; /* Total number of bytes to copy */ + size_t src_nbytes; /* Number of bytes to read from source */ + size_t mem_nbytes; /* Number of bytes to convert in memory */ + size_t dst_nbytes; /* Number of bytes to write to destination */ + hsize_t total_src_nbytes; /* Total number of bytes to copy */ size_t buf_size; /* Size of copy buffer */ void *buf = NULL; /* Buffer for copying data */ void *reclaim_buf = NULL; /* Buffer for reclaiming data */ @@ -1030,14 +1035,12 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to allocate contiguous storage") /* Set up number of bytes to copy, and initial buffer size */ - total_nbytes = layout_src->u.contig.size; - H5_CHECK_OVERFLOW(total_nbytes,hsize_t,size_t); - buf_size = MIN(H5D_XFER_MAX_TEMP_BUF_DEF, (size_t)total_nbytes); + total_src_nbytes = layout_src->u.contig.size; + H5_CHECK_OVERFLOW(total_src_nbytes,hsize_t,size_t); + buf_size = MIN(H5D_XFER_MAX_TEMP_BUF_DEF, (size_t)total_src_nbytes); /* If there's a source datatype, set up type conversion information */ if(dt_src) { - size_t tmp_dt_size; /* Temp. atatype size */ - /* Create datatype ID for src datatype */ if((tid_src = H5I_register(H5I_DATATYPE, dt_src)) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register source file datatype") @@ -1063,19 +1066,24 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between mem and dst datatypes") /* Determine largest datatype size */ - if(0 == (max_dt_size = H5T_get_size(dt_src))) + if(0 == (src_dt_size = H5T_get_size(dt_src))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") - if(0 == (tmp_dt_size = H5T_get_size(dt_mem))) + if(0 == (mem_dt_size = H5T_get_size(dt_mem))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") - max_dt_size = MAX(max_dt_size, tmp_dt_size); - if(0 == (tmp_dt_size = H5T_get_size(dt_dst))) + max_dt_size = MAX(src_dt_size, mem_dt_size); + if(0 == (dst_dt_size = H5T_get_size(dt_dst))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") - max_dt_size = MAX(max_dt_size, tmp_dt_size); + max_dt_size = MAX(max_dt_size, dst_dt_size); - /* Set number of whole elements that fit in buffer */ + /* Set maximum number of whole elements that fit in buffer */ if(0 == (nelmts = buf_size / max_dt_size)) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "element size too large") + /* Set the number of bytes to transfer */ + src_nbytes = nelmts * src_dt_size; + dst_nbytes = nelmts * dst_dt_size; + mem_nbytes = nelmts * mem_dt_size; + /* Adjust buffer size to be multiple of elements */ buf_size = nelmts * max_dt_size; @@ -1095,10 +1103,14 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, /* Set flag to do type conversion */ do_conv = TRUE; } /* end if */ - else + else { /* Type conversion not necessary */ do_conv = FALSE; + /* Set the number of bytes to read & write to the buffer size */ + src_nbytes = dst_nbytes = mem_nbytes = buf_size; + } /* end else */ + /* Allocate space for copy buffer */ HDassert(buf_size); if(NULL == (buf = H5FL_BLK_MALLOC(type_conv, buf_size))) @@ -1113,28 +1125,34 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, /* Loop over copying data */ addr_src = layout_src->u.contig.addr; addr_dst = layout_dst->u.contig.addr; - while(total_nbytes > 0) { - size_t nbytes; /* Number of bytes to copy each time */ - - /* Compute number of bytes to copy for this pass */ - if(total_nbytes >= buf_size) - nbytes = buf_size; - else { - nbytes = (size_t)total_nbytes; + while(total_src_nbytes > 0) { + /* Check if we should reduce the number of bytes to transfer */ + if(total_src_nbytes < src_nbytes) { + /* Adjust bytes to transfer */ + src_nbytes = (size_t)total_src_nbytes; /* Adjust dataspace describing buffer */ if(do_conv) { + /* Adjust destination & memory bytes to transfer */ + nelmts = src_nbytes / src_dt_size; + dst_nbytes = nelmts * dst_dt_size; + mem_nbytes = nelmts * mem_dt_size; + /* Adjust size of buffer's dataspace dimension */ - buf_dim = nelmts = nbytes / max_dt_size; + buf_dim = nelmts; /* Adjust size of buffer's dataspace */ if(H5S_set_extent_real(buf_space, &buf_dim) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSET, FAIL, "unable to change buffer dataspace size") } /* end if */ - } /* end else */ + else { + /* Adjust destination & memory bytes to transfer */ + dst_nbytes = mem_nbytes = src_nbytes; + } /* end else */ + } /* end if */ /* Read raw data from source file */ - if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0) + if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, src_nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read raw data") /* Perform datatype conversion, if necessary */ @@ -1144,27 +1162,25 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed") /* Copy into another buffer, to reclaim memory later */ - HDmemcpy(reclaim_buf, buf, nbytes); + HDmemcpy(reclaim_buf, buf, mem_nbytes); /* Convert from memory to destination file */ if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed") + + /* Reclaim space from variable length data */ + if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to reclaim variable-length data") } /* end if */ /* Write raw data to destination file */ - if(H5F_block_write(f_dst, H5FD_MEM_DRAW, addr_dst, nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0) + if(H5F_block_write(f_dst, H5FD_MEM_DRAW, addr_dst, dst_nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data") - /* Reclaim any space from variable length data */ - if(do_conv) { - if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to reclaim variable-length data") - } /* end if */ - /* Adjust loop variables */ - addr_src += nbytes; - addr_dst += nbytes; - total_nbytes -= nbytes; + addr_src += src_nbytes; + addr_dst += dst_nbytes; + total_src_nbytes -= src_nbytes; } /* end while */ done: diff --git a/src/H5Distore.c b/src/H5Distore.c index c450293..570cf81 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -194,6 +194,22 @@ typedef struct H5D_istore_it_ud4_t { H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ H5F_t *file_dst; /* Destination file for copy */ haddr_t addr_dst; /* Address of dest. B-tree */ + void *buf; /* Buffer to hold chunk data for read/write */ + size_t buf_size; /* Buffer size */ + + /* needed for converting variable-length data */ + hid_t tid_src; /* Datatype ID for source datatype */ + hid_t tid_dst; /* Datatype ID for destination datatype */ + hid_t tid_mem; /* Datatype ID for memory datatype */ + H5T_path_t *tpath_src_mem; /* Datatype conversion path from source file to memory */ + H5T_path_t *tpath_mem_dst; /* Datatype conversion path from memory to dest. file */ + void *reclaim_buf; /* Buffer for reclaiming data */ + size_t reclaim_buf_size; /* Reclaim buffer size */ + size_t nelmts; /* Number of elements in buffer */ + H5S_t *buf_space; /* Dataspace describing buffer */ + + /* needed for compressed variable-length data */ + H5O_pline_t *pline; /* Filter pipeline */ } H5D_istore_it_ud4_t; /* B-tree callback info for iteration to obtain chunk address and the index of the chunk for all chunks in the B-tree. */ @@ -975,40 +991,111 @@ static int H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, haddr_t addr_src, const void UNUSED *_rt_key, void *_udata) { - H5D_istore_it_ud4_t *udata = (H5D_istore_it_ud4_t *)_udata; - const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; - void *chunk = NULL; /*the chunk data */ - H5D_istore_ud1_t udata_dst; - int ret_value = H5B_ITER_CONT; /* Return value */ + H5D_istore_it_ud4_t *udata = (H5D_istore_it_ud4_t *)_udata; + const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; + H5D_istore_ud1_t udata_dst; /* User data about new destination chunk */ + hbool_t is_vlen = FALSE; + + /* General information about chunk copy */ + void *buf = udata->buf; + size_t buf_size = udata->buf_size; + H5O_pline_t *pline = udata->pline; + + /* needed for commpressed variable length data */ + hbool_t is_compressed = FALSE; + H5Z_EDC_t edc_read = H5Z_NO_EDC; + size_t nbytes = lt_key->nbytes; + H5Z_cb_t cb_struct; + + int ret_value = H5B_ITER_CONT; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_copy) - /* Allocate memory for copying the chunk */ - if(NULL == (chunk = H5MM_malloc(lt_key->nbytes))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5B_ITER_ERROR, "memory allocation failed for raw data chunk") + /* Check parameter for type conversion */ + if (udata->tid_src > 0) + is_vlen = TRUE; + + /* Check for filtered chunks */ + if (pline && pline->nused) { + is_compressed = TRUE; + cb_struct.func = NULL; /* no callback function when failed */ + } /* end if */ + + /* Resize the buf if it is too small to hold the data */ + if ( nbytes > buf_size) { + /* Re-allocate memory for copying the chunk */ + if(NULL == (udata->buf = H5MM_realloc(udata->buf, nbytes))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5B_ITER_ERROR, "memory allocation failed for raw data chunk") + + buf = udata->buf; + udata->buf_size = buf_size = nbytes; + } /* end if */ /* read chunk data from the source file */ - if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, lt_key->nbytes, dxpl_id, chunk) < 0) + if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, nbytes, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, H5B_ITER_ERROR, "unable to read raw data chunk") + /* need to uncompress variable-length data */ + if (is_compressed && is_vlen) { + unsigned filter_mask = lt_key->filter_mask; + + if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, cb_struct, &nbytes, &buf_size, &buf) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_READERROR, H5B_ITER_ERROR, "data pipeline read failed") + } /* end if */ + + /* Perform datatype conversion, if necessary */ + if(is_vlen) { + H5T_path_t *tpath_src_mem = udata->tpath_src_mem; + H5T_path_t *tpath_mem_dst = udata->tpath_mem_dst; + H5S_t *buf_space = udata->buf_space; + hid_t tid_src = udata->tid_src; + hid_t tid_dst = udata->tid_dst; + hid_t tid_mem = udata->tid_mem; + size_t nelmts = udata->nelmts; + void *reclaim_buf = udata->reclaim_buf; + size_t reclaim_buf_size = udata->reclaim_buf_size; + + /* Convert from source file to memory */ + if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5B_ITER_ERROR, "datatype conversion failed") + + /* Copy into another buffer, to reclaim memory later */ + HDmemcpy(reclaim_buf, buf, reclaim_buf_size); + + /* Convert from memory to destination file */ + if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5B_ITER_ERROR, "datatype conversion failed") + + /* Reclaim space from variable length data */ + if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADITER, H5B_ITER_ERROR, "unable to reclaim variable-length data") + } /* end if */ + /* Copy source chunk callback information for insertion */ HDmemset(&udata_dst, 0, sizeof(udata_dst)); HDmemcpy(&(udata_dst.common.key), lt_key, sizeof(H5D_istore_key_t)); udata_dst.common.mesg = udata->common.mesg; /* Share this pointer for a short while */ + /* need to compress variable-length data before writing to file*/ + if (is_compressed && is_vlen) { + if(H5Z_pipeline(pline, 0, &(udata_dst.common.key.filter_mask), edc_read, + cb_struct, &nbytes, &buf_size, &buf) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_READERROR, H5B_ITER_ERROR, "output pipeline failed") + udata_dst.common.key.nbytes = nbytes; + udata->buf = buf; + udata->buf_size = buf_size; + } /* end if */ + /* Insert chunk into the destination Btree */ if(H5B_insert(udata->file_dst, dxpl_id, H5B_ISTORE, udata->addr_dst, &udata_dst) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, H5B_ITER_ERROR, "unable to allocate chunk") /* Write chunk data to destination file */ HDassert(H5F_addr_defined(udata_dst.addr)); - if(H5F_block_write(udata->file_dst, H5FD_MEM_DRAW, udata_dst.addr, udata_dst.common.key.nbytes, dxpl_id, chunk) < 0) + if(H5F_block_write(udata->file_dst, H5FD_MEM_DRAW, udata_dst.addr, nbytes, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, H5B_ITER_ERROR, "unable to write raw data to file") done: - if(chunk) - H5MM_xfree(chunk); - FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_iter_copy() */ @@ -3438,11 +3525,22 @@ done: *------------------------------------------------------------------------- */ herr_t -H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, - H5F_t *f_dst, H5O_layout_t *layout_dst, hid_t dxpl_id) +H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, + H5O_layout_t *layout_dst, H5T_t *dt_src, H5O_pline_t *pline, hid_t dxpl_id) { H5D_istore_it_ud4_t udata; - herr_t ret_value = SUCCEED; /* Return value */ + H5T_path_t *tpath_src_mem = NULL, *tpath_mem_dst = NULL; /* Datatype conversion paths */ + hid_t tid_src = -1; /* Datatype ID for source datatype */ + hid_t tid_dst = -1; /* Datatype ID for destination datatype */ + hid_t tid_mem = -1; /* Datatype ID for memory datatype */ + size_t buf_size; /* Size of copy buffer */ + size_t reclaim_buf_size; /* Size of reclaim buffer */ + void *buf = NULL; /* Buffer for copying data */ + void *reclaim_buf = NULL; /* Buffer for reclaiming data */ + H5S_t *buf_space = NULL; /* Dataspace describing buffer */ + hid_t sid_buf = -1; /* ID for buffer dataspace */ + size_t nelmts = 0; /* Number of elements in buffer */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5D_istore_copy, FAIL) @@ -3465,19 +3563,129 @@ H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to initialize chunked storage") } /* end if */ + /* If there's a source datatype, set up type conversion information */ + if(dt_src) { + H5T_t *dt_dst; /* Destination datatype */ + H5T_t *dt_mem; /* Memory datatype */ + size_t mem_dt_size; /* Memory datatype size */ + size_t tmp_dt_size; /* Temp. datatype size */ + size_t max_dt_size; /* Max atatype size */ + hsize_t buf_dim; /* Dimension for buffer */ + unsigned u; + + /* Create datatype ID for src datatype */ + if((tid_src = H5I_register(H5I_DATATYPE, dt_src)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register source file datatype") + + /* create a memory copy of the variable-length datatype */ + if(NULL == (dt_mem = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") + if((tid_mem = H5I_register(H5I_DATATYPE, dt_mem)) < 0) + HGOTO_ERROR (H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") + + /* create variable-length datatype at the destinaton file */ + if(NULL == (dt_dst = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") + if(H5T_set_loc(dt_dst, f_dst, H5T_LOC_DISK) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "cannot mark datatype on disk") + if((tid_dst = H5I_register(H5I_DATATYPE, dt_dst)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register destination file datatype") + + /* Set up the conversion functions */ + if(NULL == (tpath_src_mem = H5T_path_find(dt_src, dt_mem, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and mem datatypes") + if(NULL == (tpath_mem_dst = H5T_path_find(dt_mem, dt_dst, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between mem and dst datatypes") + + /* Determine largest datatype size */ + if(0 == (max_dt_size = H5T_get_size(dt_src))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + if(0 == (mem_dt_size = H5T_get_size(dt_mem))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + max_dt_size = MAX(max_dt_size, mem_dt_size); + if(0 == (tmp_dt_size = H5T_get_size(dt_dst))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + max_dt_size = MAX(max_dt_size, tmp_dt_size); + + /* Compute the number of elements per chunk */ + nelmts = 1; + for(u = 0; u < (layout_src->u.chunk.ndims - 1); u++) + nelmts *= layout_src->u.chunk.dim[u]; + + /* Create the space and set the initial extent */ + buf_dim = nelmts; + if(NULL == (buf_space = H5S_create_simple((unsigned)1, &buf_dim, NULL))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "can't create simple dataspace") + + /* Atomize */ + if((sid_buf = H5I_register(H5I_DATASPACE, buf_space)) < 0) { + H5S_close(buf_space); + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace ID") + } /* end if */ + + /* Set initial buffer sizes */ + buf_size = nelmts * max_dt_size; + reclaim_buf_size = nelmts * mem_dt_size; + + /* Allocate memory for reclaim buf */ + if(NULL == (reclaim_buf = H5MM_malloc(reclaim_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + } /* end if */ + else { + buf_size = layout_src->u.chunk.size; + reclaim_buf_size = 0; + } /* end else */ + + /* Allocate memory for copying the chunk */ + if(NULL == (buf = H5MM_malloc(buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + /* Initialize the callback structure for the source */ HDmemset(&udata, 0, sizeof udata); udata.common.mesg = layout_src; udata.file_dst = f_dst; udata.addr_dst = layout_dst->u.chunk.addr; + udata.buf = buf; + udata.buf_size = buf_size; + udata.tid_src = tid_src; + udata.tid_mem = tid_mem; + udata.tid_dst = tid_dst; + udata.tpath_src_mem = tpath_src_mem; + udata.tpath_mem_dst = tpath_mem_dst; + udata.reclaim_buf = reclaim_buf; + udata.reclaim_buf_size = reclaim_buf_size; + udata.buf_space = buf_space; + udata.nelmts = nelmts; + udata.pline = pline; /* copy the chunked data by iteration */ if(H5B_iterate(f_src, dxpl_id, H5B_ISTORE, H5D_istore_iter_copy, layout_src->u.chunk.addr, &udata) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to iterate over chunk B-tree") + /* I/O buffer may have been re-allocated */ + buf = udata.buf; + done: + if(sid_buf > 0) + if(H5I_dec_ref(sid_buf) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary dataspace ID") + if(tid_src > 0) + if(H5I_dec_ref(tid_src) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(tid_dst > 0) + if(H5I_dec_ref(tid_dst) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(tid_mem > 0) + if(H5I_dec_ref(tid_mem) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(buf) + H5MM_xfree (buf); + if(reclaim_buf) + H5MM_xfree (reclaim_buf); + if(H5RC_DEC(layout_src->u.chunk.btree_shared) < 0) HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") + if(H5RC_DEC(layout_dst->u.chunk.btree_shared) < 0) HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") diff --git a/src/H5Doh.c b/src/H5Doh.c index f9f9249..4da2381 100644 --- a/src/H5Doh.c +++ b/src/H5Doh.c @@ -152,6 +152,10 @@ done: * Programmer: Quincey Koziol * Monday, November 21, 2005 * + * Modifications: Peter Cao + * Tuesday, December 27, 2005 + * Free filter pipeline for copying a dataset + * *------------------------------------------------------------------------- */ static void @@ -168,6 +172,10 @@ H5O_dset_free_copy_file_udata(void *_udata) if(udata->src_dtype) H5T_close(udata->src_dtype); + /* Release copy of dataset's filter pipeline, if it was set */ + if (udata->src_pline) + H5O_free(H5O_PLINE_ID, udata->src_pline); + /* Release space for 'copy file' user data */ H5FL_FREE(H5D_copy_file_ud_t, udata); diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 9b73d86..97e7763 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -34,7 +34,7 @@ #include "H5Gprivate.h" /* Groups */ #include "H5Oprivate.h" /* Object headers */ #include "H5Sprivate.h" /* Dataspaces */ -#include "H5Tprivate.h" /* Datatype functions */ +#include "H5Tprivate.h" /* Datatypes */ /**************************/ /* Package Private Macros */ @@ -198,8 +198,6 @@ extern H5D_dxpl_cache_t H5D_def_dxpl_cache; H5_DLL herr_t H5D_alloc_storage (H5F_t *f, hid_t dxpl_id, H5D_t *dset, H5D_time_alloc_t time_alloc, hbool_t update_time, hbool_t full_overwrite); -H5_DLL herr_t H5D_vlen_reclaim(hid_t type_id, H5S_t *space, hid_t plist_id, - void *buf); /* Functions that perform serial I/O operations */ H5_DLL herr_t H5D_select_fscat (H5D_io_info_t *io_info, @@ -247,6 +245,8 @@ H5_DLL ssize_t H5D_compact_writevv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_size_arr[], hsize_t dset_offset_arr[], size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_size_arr[], hsize_t mem_offset_arr[], const void *buf); +H5_DLL herr_t H5D_compact_copy(H5F_t *f_src, H5O_layout_t *layout_src, + H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *src_dtype, hid_t dxpl_id); /* Functions that operate on indexed storage */ /* forward reference for collective-chunk IO use */ @@ -278,7 +278,7 @@ H5_DLL ssize_t H5D_istore_writevv(const H5D_io_info_t *io_info, H5_DLL haddr_t H5D_istore_get_addr(const H5D_io_info_t *io_info, struct H5D_istore_ud1_t *_udata); H5_DLL herr_t H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, - H5F_t *f_dst, H5O_layout_t *layout_dst, hid_t dxpl_id); + H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *src_dtype, H5O_pline_t *pline, hid_t dxpl_id); /* Functions that operate on external file list (efl) storage */ H5_DLL ssize_t H5D_efl_readvv(const H5D_io_info_t *io_info, diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index d762abc..2117d30 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -24,6 +24,7 @@ /* Private headers needed by this file */ #include "H5FDprivate.h" /* File drivers */ #include "H5Oprivate.h" /* Object headers */ +#include "H5Sprivate.h" /* Dataspaces */ #include "H5Zprivate.h" /* Data filters */ /**************************/ @@ -237,6 +238,10 @@ H5_DLL herr_t H5D_flush(const H5F_t *f, hid_t dxpl_id, unsigned flags); H5_DLL herr_t H5D_get_dxpl_cache(hid_t dxpl_id, H5D_dxpl_cache_t **cache); H5_DLL herr_t H5D_get_dxpl_cache_real(hid_t dxpl_id, H5D_dxpl_cache_t *cache); +/* Functions that operate on vlen data */ +H5_DLL herr_t H5D_vlen_reclaim(hid_t type_id, H5S_t *space, hid_t plist_id, + void *buf); + /* Functions that operate on contiguous storage */ H5_DLL herr_t H5D_contig_delete(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout); diff --git a/src/H5O.c b/src/H5O.c index a05a378..1d54077 100644 --- a/src/H5O.c +++ b/src/H5O.c @@ -4120,7 +4120,7 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, continuation block. */ for(chunkno = 0; chunkno < oh_src->nchunks; chunkno++) { - size_t chunk_size = oh_src->chunk[chunkno].size; + size_t chunk_size = oh_src->chunk[chunkno].size; /* '0th' chunk is preceded by object header prefix */ if(0 == chunkno) { @@ -4267,7 +4267,9 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, if(H5SL_insert(map_list, addr_map, &(addr_map->src_addr)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "can't insert object into skip list") - /* "post copy" loop over messages, to fix up any messages which require a complete object header for destination object */ + /* "post copy" loop over messages, to fix up any messages which require a complete + * object header for destination object + */ for(mesgno = 0; mesgno < oh_src->nmesgs; mesgno++) { /* Set up convenience variables */ mesg_src = &(oh_src->mesg[mesgno]); diff --git a/src/H5Oattr.c b/src/H5Oattr.c index 5d05a7c..6ee9c40 100644 --- a/src/H5Oattr.c +++ b/src/H5Oattr.c @@ -19,9 +19,11 @@ #include "H5private.h" /* Generic Functions */ #include "H5Apkg.h" /* Attributes */ +#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Gprivate.h" /* Groups */ +#include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ #include "H5Spkg.h" /* Dataspaces */ @@ -644,6 +646,10 @@ done: * Programmer: Quincey Koziol * November 1, 2005 * + * Modifications: Peter Cao + * December 17, 2005 + * Datatype conversion for variable length datatype + * *------------------------------------------------------------------------- */ static void * @@ -652,7 +658,16 @@ H5O_attr_copy_file(H5F_t UNUSED *file_src, void *native_src, { H5A_t *attr_src = (H5A_t *)native_src; H5A_t *attr_dst = NULL; - void *ret_value; /* Return value */ + + /* for dataype conversion */ + hid_t tid_src = -1; /* Datatype ID for source datatype */ + hid_t tid_dst = -1; /* Datatype ID for destination datatype */ + hid_t tid_mem = -1; /* Datatype ID for memory datatype */ + void *buf = NULL; /* Buffer for copying data */ + void *reclaim_buf = NULL; /* Buffer for reclaiming data */ + hid_t buf_sid = -1; /* ID for buffer dataspace */ + + void *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5O_attr_copy_file) @@ -675,15 +690,22 @@ H5O_attr_copy_file(H5F_t UNUSED *file_src, void *native_src, /* Copy attribute's name */ attr_dst->name = H5MM_strdup(attr_src->name); + HDassert(attr_dst->name); /* Copy attribute's datatype */ /* (Start destination datatype as transient, even if source is named) */ attr_dst->dt = H5T_copy(attr_src->dt, H5T_COPY_ALL); + HDassert(attr_dst->dt); + + /* Set the location of the destination datatype */ + if(H5T_set_loc(attr_dst->dt, file_dst, H5T_LOC_DISK) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "cannot mark datatype on disk") /* Check for named datatype being copied */ if(H5T_committed(attr_src->dt)) { H5O_loc_t *src_oloc; /* Pointer to source datatype's object location */ H5O_loc_t *dst_oloc; /* Pointer to dest. datatype's object location */ + H5O_shared_t sh_mesg; /* Get group entries for source & destination */ src_oloc = H5T_oloc(attr_src->dt); @@ -698,22 +720,156 @@ H5O_attr_copy_file(H5F_t UNUSED *file_src, void *native_src, /* Copy the shared object from source to destination */ if(H5O_copy_header_map(src_oloc, dst_oloc, dxpl_id, map_list) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy object") + + /* Reset shared message information */ + HDmemset(&sh_mesg, 0, sizeof(H5O_shared_t)); + + /* Get shared message information for datatype */ + if(H5O_get_share(H5O_DTYPE_ID, file_dst, attr_src->dt, &sh_mesg/*out*/) < 0) + HGOTO_ERROR(H5E_ATTR, H5E_CANTOPENOBJ, NULL, "unable to get shared message") + + /* Compute shared message size for datatype */ + attr_dst->dt_size = H5O_raw_size(H5O_SHARED_ID, file_dst, &sh_mesg); } /* end if */ + else + attr_dst->dt_size = H5O_raw_size(H5O_DTYPE_ID, file_dst, attr_src->dt); + HDassert(attr_dst->dt_size > 0); + attr_dst->ds_size = H5S_raw_size(file_dst, attr_src->ds); + HDassert(attr_dst->ds_size > 0); - /* Copy the guts of the attribute */ + /* Copy the dataspace for the attribute */ attr_dst->ds = H5S_copy(attr_src->ds, FALSE); + HDassert(attr_dst->ds); + + /* Compute the size of the data */ + H5_ASSIGN_OVERFLOW(attr_dst->data_size, H5S_GET_EXTENT_NPOINTS(attr_dst->ds) * H5T_get_size(attr_dst->dt), hsize_t, size_t); + /* Copy (& convert) the data, if necessary */ if(attr_src->data) { - if(NULL == (attr_dst->data = H5FL_BLK_MALLOC(attr_buf, attr_src->data_size))) + if(NULL == (attr_dst->data = H5FL_BLK_MALLOC(attr_buf, attr_dst->data_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - HDmemcpy(attr_dst->data, attr_src->data, attr_src->data_size); + /* Check if we need to convert data */ + if(H5T_detect_class(attr_src->dt, H5T_VLEN) > 0) { + H5T_path_t *tpath_src_mem, *tpath_mem_dst; /* Datatype conversion paths */ + H5T_t *dt_mem; /* Memory datatype */ + size_t src_dt_size; /* Source datatype size */ + size_t tmp_dt_size; /* Temp. datatype size */ + size_t max_dt_size; /* Max atatype size */ + H5S_t *buf_space; /* Dataspace describing buffer */ + hsize_t buf_dim; /* Dimension for buffer */ + size_t nelmts; /* Number of elements in buffer */ + size_t buf_size; /* Size of copy buffer */ + + /* Create datatype ID for src datatype */ + if((tid_src = H5I_register(H5I_DATATYPE, attr_src->dt)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, NULL, "unable to register source file datatype") + + /* create a memory copy of the variable-length datatype */ + if(NULL == (dt_mem = H5T_copy(attr_src->dt, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to copy") + if((tid_mem = H5I_register(H5I_DATATYPE, dt_mem)) < 0) + HGOTO_ERROR (H5E_DATATYPE, H5E_CANTREGISTER, NULL, "unable to register memory datatype") + + /* create variable-length datatype at the destinaton file */ + if((tid_dst = H5I_register(H5I_DATATYPE, attr_dst->dt)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, NULL, "unable to register destination file datatype") + + /* Set up the conversion functions */ + if(NULL == (tpath_src_mem = H5T_path_find(attr_src->dt, dt_mem, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to convert between src and mem datatypes") + if(NULL == (tpath_mem_dst = H5T_path_find(dt_mem, attr_dst->dt, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to convert between mem and dst datatypes") + + /* Determine largest datatype size */ + if(0 == (src_dt_size = H5T_get_size(attr_src->dt))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to determine datatype size") + if(0 == (tmp_dt_size = H5T_get_size(dt_mem))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to determine datatype size") + max_dt_size = MAX(src_dt_size, tmp_dt_size); + if(0 == (tmp_dt_size = H5T_get_size(attr_dst->dt))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to determine datatype size") + max_dt_size = MAX(max_dt_size, tmp_dt_size); + + /* Set number of whole elements that fit in buffer */ + if(0 == (nelmts = attr_src->data_size / src_dt_size)) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "element size too large") + + /* Set up number of bytes to copy, and initial buffer size */ + buf_size = nelmts * max_dt_size; + + /* Create dataspace for number of elements in buffer */ + buf_dim = nelmts; + + /* Create the space and set the initial extent */ + if(NULL == (buf_space = H5S_create_simple((unsigned)1, &buf_dim, NULL))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, NULL, "can't create simple dataspace") + + /* Atomize */ + if((buf_sid = H5I_register(H5I_DATASPACE, buf_space)) < 0) { + H5S_close(buf_space); + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, NULL, "unable to register dataspace ID") + } /* end if */ + + /* Allocate memory for recclaim buf */ + if(NULL == (reclaim_buf = H5FL_BLK_MALLOC(attr_buf, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation NULLed for raw data chunk") + + /* Allocate memory for copying the chunk */ + if(NULL == (buf = H5FL_BLK_MALLOC(attr_buf, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation NULLed for raw data chunk") + + HDmemcpy(buf, attr_src->data, attr_src->data_size); + + /* Convert from source file to memory */ + if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "datatype conversion NULLed") + + HDmemcpy(reclaim_buf, buf, buf_size); + + /* Convert from memory to destination file */ + if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "datatype conversion NULLed") + + HDmemcpy(attr_dst->data, buf, attr_dst->data_size); + + if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADITER, NULL, "unable to reclaim variable-length data") + } /* type conversion */ + else { + HDassert(attr_dst->data_size == attr_src->data_size); + HDmemcpy(attr_dst->data, attr_src->data, attr_src->data_size); + } /* end else */ } /* end if */ + /* Indicate that the fill values aren't to be written out */ + attr_dst->initialized = TRUE; + /* Set return value */ ret_value = attr_dst; done: + if(buf_sid > 0) + if(H5I_dec_ref(buf_sid) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't decrement temporary dataspace ID") + if(tid_src > 0) + /* Don't decrement ID, we want to keep underlying datatype */ + if(H5I_remove(tid_src) == NULL) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't decrement temporary datatype ID") + if(tid_dst > 0) + /* Don't decrement ID, we want to keep underlying datatype */ + if(H5I_remove(tid_dst) == NULL) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't decrement temporary datatype ID") + if(tid_mem > 0) + /* Decrement the memory datatype ID, it's transient */ + if(H5I_dec_ref(tid_mem) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't decrement temporary datatype ID") + if(buf) + H5FL_BLK_FREE(attr_buf, buf); + if(reclaim_buf) + H5FL_BLK_FREE(attr_buf, reclaim_buf); + + /* Release destination attribute information on failure */ if(!ret_value) if(attr_dst) (void)H5A_free(attr_dst); diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 3665e13..7e6e4f5 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -625,9 +625,9 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst, hid_t dxpl_id, H5SL_t UNUSED *map_list, void *_udata) { H5D_copy_file_ud_t *udata = (H5D_copy_file_ud_t *)_udata; /* Dataset copying user data */ - H5O_layout_t *layout_src = (H5O_layout_t *) mesg_src; - H5O_layout_t *layout_dst = NULL; - void *ret_value; /* Return value */ + H5O_layout_t *layout_src = (H5O_layout_t *) mesg_src; + H5O_layout_t *layout_dst = NULL; + void *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5O_layout_copy_file) @@ -648,9 +648,15 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, if(layout_src->u.compact.buf) { if(NULL == (layout_dst->u.compact.buf = H5MM_malloc(layout_src->u.compact.size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate memory for compact dataset") -HDassert(!udata->src_dtype); - HDmemcpy(layout_dst->u.compact.buf, layout_src->u.compact.buf, layout_src->u.compact.size); + + /* copy compact raw data */ + if(H5D_compact_copy(file_src, layout_src, file_dst, layout_dst, udata->src_dtype, dxpl_id) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to copy chunked storage") + layout_dst->u.compact.dirty = TRUE; + + /* Freed by copy routine */ + udata->src_dtype = NULL; } break; @@ -667,13 +673,16 @@ HDassert(!udata->src_dtype); case H5D_CHUNKED: if(H5F_addr_defined(layout_src->u.chunk.addr)) { -HDassert(!udata->src_dtype); /* layout is not created in the destination file, undef btree address */ layout_dst->u.chunk.addr = HADDR_UNDEF; /* create chunked layout */ - if(H5D_istore_copy(file_src, layout_src, file_dst, layout_dst, dxpl_id) < 0) + if(H5D_istore_copy(file_src, layout_src, file_dst, layout_dst, + udata->src_dtype, udata->src_pline, dxpl_id) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to copy chunked storage") + + /* Freed by copy routine */ + udata->src_dtype = NULL; } /* if ( H5F_addr_defined(layout_srct->u.chunk.addr)) */ break; diff --git a/src/H5Opkg.h b/src/H5Opkg.h index ed70bb7..0ce644b 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -112,6 +112,7 @@ struct H5O_t { /* Callback information for copying dataset */ typedef struct { H5T_t *src_dtype; /* Copy of datatype for dataset */ + H5O_pline_t *src_pline; /* Copy of filter pipeline for dataet */ } H5D_copy_file_ud_t; /* Class for types of objects in file */ diff --git a/src/H5Opline.c b/src/H5Opline.c index 9b6f147..d6e9870 100644 --- a/src/H5Opline.c +++ b/src/H5Opline.c @@ -36,6 +36,8 @@ static void *H5O_pline_copy (const void *_mesg, void *_dest, unsigned update_fla static size_t H5O_pline_size (const H5F_t *f, const void *_mesg); static herr_t H5O_pline_reset (void *_mesg); static herr_t H5O_pline_free (void *_mesg); +static herr_t H5O_pline_pre_copy_file(H5F_t *file_src, void *mesg_src, + void *_udata); static herr_t H5O_pline_debug (H5F_t *f, hid_t dxpl_id, const void *_mesg, FILE * stream, int indent, int fwidth); @@ -54,7 +56,7 @@ const H5O_msg_class_t H5O_MSG_PLINE[1] = {{ NULL, /* link method */ NULL, /* get share method */ NULL, /* set share method */ - NULL, /* pre copy native value to file */ + H5O_pline_pre_copy_file, /* pre copy native value to file */ NULL, /* copy native value to file */ NULL, /* post copy native value to file */ H5O_pline_debug /* debug the message */ @@ -424,6 +426,49 @@ H5O_pline_free (void *mesg) /*------------------------------------------------------------------------- + * Function: H5O_pline_pre_copy_file + * + * Purpose: Perform any necessary actions before copying message between + * files + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Peter Cao + * December 27, 2005 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_pline_pre_copy_file(H5F_t *file_src, void *mesg_src, void *_udata) +{ + H5O_pline_t *pline_src = (H5O_pline_t *)mesg_src; /* Source datatype */ + H5D_copy_file_ud_t *udata = (H5D_copy_file_ud_t *)_udata; /* Dataset copying user data */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5O_pline_pre_copy_file) + + /* check args */ + HDassert(file_src); + HDassert(pline_src); + + /* If the user data is non-NULL, assume we are copying a dataset + * and make a copy of the filter pipeline for later in + * the object copying process. + */ + if(udata) { + if(NULL == (udata->src_pline = H5O_copy(H5O_PLINE_ID, pline_src, udata->src_pline))) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to copy") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_pline_pre_copy_file() */ + + + +/*------------------------------------------------------------------------- * Function: H5O_pline_debug * * Purpose: Prints debugging information for filter pipeline message MESG diff --git a/src/H5R.c b/src/H5R.c index fa5b97d..35f1313 100644 --- a/src/H5R.c +++ b/src/H5R.c @@ -17,6 +17,7 @@ #include "H5private.h" /* Generic Functions */ +#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ #include "H5Gprivate.h" /* Groups */ diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index bf9ff84..9ea37bd 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -18,11 +18,14 @@ #ifndef _H5Sprivate_H #define _H5Sprivate_H +/* Include package's public header */ #include "H5Spublic.h" +/* Public headers needed by this file */ +#include "H5Dpublic.h" /* Datasets */ + /* Private headers needed by this file */ #include "H5private.h" /* Generic Functions */ -#include "H5Dprivate.h" /* Dataset functions */ #include "H5Fprivate.h" /* Files */ #include "H5Gprivate.h" /* Groups */ #include "H5Oprivate.h" /* Object headers */ diff --git a/src/H5Sselect.c b/src/H5Sselect.c index f7a58d3..c351a40 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -22,6 +22,7 @@ #include "H5private.h" /* Generic Functions */ +#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c index 6ae20a7..29e4ec8 100644 --- a/src/H5Tvlen.c +++ b/src/H5Tvlen.c @@ -215,7 +215,7 @@ H5T_vlen_set_loc(const H5T_t *dt, H5F_t *f, H5T_loc_t loc) assert(loc>H5T_LOC_BADLOC && locshared->u.vlen.loc) { + if(loc != dt->shared->u.vlen.loc || f != dt->shared->u.vlen.f) { /* Indicate that the location changed */ ret_value=TRUE; diff --git a/test/objcopy.c b/test/objcopy.c index 9a74cef..f64f8c8 100755 --- a/test/objcopy.c +++ b/test/objcopy.c @@ -67,6 +67,8 @@ const char *FILENAME[] = { #define ATTR_NAME_LEN 40 #define DIM_SIZE_1 12 #define DIM_SIZE_2 6 +#define CHUNK_SIZE_1 5 /* Not an even fraction of dimension sizes, so we test copying partial chunks */ +#define CHUNK_SIZE_2 5 #define NUM_SUB_GROUPS 20 #define NUM_WIDE_LOOP_GROUPS 10 #define NUM_DATASETS 10 @@ -79,6 +81,10 @@ static struct { haddr_t *obj; /* Addresses of objects seen */ } idtab_g; +/* Local function prototypes */ +static int +compare_data(hid_t tid, size_t nelmts, const void *buf1, const void *buf2); + /*------------------------------------------------------------------------- * Function: addr_insert @@ -165,6 +171,63 @@ addr_reset(void) /*------------------------------------------------------------------------- + * Function: attach_attribute_vl + * + * Purpose: Attach an vlen attribute to the object to be copied + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Peter Cao + * Saturday, December 17, 2005 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +static int +test_copy_attach_attribute_vl(hid_t loc_id) +{ + hid_t aid = -1, sid = -1, tid=-1; + hvl_t buf[4]; + hsize_t dim1=4; + unsigned int i, j; + int ret_value = -1; + + if ( (sid = H5Screate_simple(1, &dim1, NULL)) < 0 ) + goto done; + + if ( (tid = H5Tvlen_create(H5T_NATIVE_INT)) < 0) + goto done; + + for(i = 0; i < 4; i++) { + buf[i].len = i*3+1; + buf[i].p = (int *)HDmalloc(buf[i].len * sizeof(int)); + for(j = 0; j < buf[i].len; j++) + ((int *)buf[i].p)[j] = j+1; + } /* end for */ + + if ( (aid = H5Acreate(loc_id, "vlen attribute", tid, sid, H5P_DEFAULT)) < 0) + goto done; + + if ( H5Awrite(aid, tid, buf) < 0) + goto done; + + ret_value = 0; + +done: + if (tid >0 && sid > 0) + H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, buf); + if (sid > 0) + H5Sclose(sid); + if (tid > 0) + H5Tclose(tid); + if (aid > 0) + H5Aclose(aid); + return ret_value; +} /* end of attach_attribute_vl */ + + +/*------------------------------------------------------------------------- * Function: test_copy_attach_attributes * * Purpose: Attach NUM_ATTRIBUTES attributes to the object to be copied @@ -276,9 +339,126 @@ done: /*------------------------------------------------------------------------- - * Function: compare_attributes + * Function: compare_attribute + * + * Purpose: Compare two attributes to check that they are equal + * + * Return: TRUE if attributes are equal/FALSE if they are different + * + * Programmer: Peter Cao + * Saturday, December 17, 2005 + * + *------------------------------------------------------------------------- + */ +static int +compare_attribute(hid_t aid, hid_t aid2, const void *wbuf) +{ + hid_t sid = -1, sid2 = -1; /* Dataspace IDs */ + hid_t tid = -1, tid2 = -1; /* Datatype IDs */ + size_t elmt_size; /* Size of datatype */ + htri_t is_committed; /* If the datatype is committed */ + htri_t is_committed2; /* If the datatype is committed */ + hssize_t nelmts; /* # of elements in dataspace */ + void *rbuf = NULL; /* Buffer for reading raw data */ + void *rbuf2 = NULL; /* Buffer for reading raw data */ + + /* Check the datatypes are equal */ + + /* Open the datatype for the source attribute */ + if ( (tid = H5Aget_type(aid)) < 0) TEST_ERROR; + + /* Open the datatype for the destination attribute */ + if ( (tid2 = H5Aget_type(aid2)) < 0) TEST_ERROR; + + /* Check that both datatypes are committed/not committed */ + if ( (is_committed = H5Tcommitted(tid)) < 0) TEST_ERROR; + if ( (is_committed2 = H5Tcommitted(tid2)) < 0) TEST_ERROR; + if ( is_committed != is_committed2) TEST_ERROR; + + /* Compare the datatypes */ + if ( H5Tequal(tid, tid2) != TRUE) TEST_ERROR; + + /* Determine the size of datatype (for later) */ + if ( (elmt_size = H5Tget_size(tid)) == 0) TEST_ERROR + + /* Check the dataspaces are equal */ + + /* Open the dataspace for the source attribute */ + if ( (sid = H5Aget_space(aid)) < 0) TEST_ERROR; + + /* Open the dataspace for the destination attribute */ + if ( (sid2 = H5Aget_space(aid2)) < 0) TEST_ERROR; + + /* Compare the dataspaces */ + if ( H5Sextent_equal(sid, sid2) != TRUE) TEST_ERROR; + + /* Determine the number of elements in dataspace (for later) */ + if ( (nelmts = H5Sget_simple_extent_npoints(sid2)) < 0) TEST_ERROR + + /* Check the raw data is equal */ + + /* Allocate & initialize space for the raw data buffers */ + if ( (rbuf = HDcalloc( elmt_size, (size_t)nelmts)) == NULL) TEST_ERROR; + if ( (rbuf2 = HDcalloc( elmt_size, (size_t)nelmts)) == NULL) TEST_ERROR; + + /* Read data from the source attribute */ + if ( H5Aread(aid, tid, rbuf) < 0) TEST_ERROR; + + /* Read data from the destination attribute */ + if ( H5Aread(aid2, tid2, rbuf2) < 0) TEST_ERROR; + + /* Check raw data read in against data written out */ + if(wbuf) { + if ( !compare_data(tid, (size_t)nelmts, wbuf, rbuf)) TEST_ERROR + if ( !compare_data(tid2, (size_t)nelmts, wbuf, rbuf2)) TEST_ERROR + } /* end if */ + /* Don't have written data, just compare data between the two attributes */ + else + if ( !compare_data(tid, (size_t)nelmts, rbuf, rbuf2)) TEST_ERROR + + /* Reclaim vlen data, if necessary */ + if(H5Tdetect_class(tid, H5T_VLEN) == TRUE) + if(H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, rbuf) < 0) TEST_ERROR + if(H5Tdetect_class(tid2, H5T_VLEN) == TRUE) + if(H5Dvlen_reclaim(tid2, sid2, H5P_DEFAULT, rbuf2) < 0) TEST_ERROR + + /* Release raw data buffers */ + HDfree(rbuf); + HDfree(rbuf2); + + /* close the source dataspace */ + if ( H5Sclose(sid) < 0) TEST_ERROR; + + /* close the destination dataspace */ + if ( H5Sclose(sid2) < 0) TEST_ERROR; + + /* close the source datatype */ + if ( H5Tclose(tid) < 0) TEST_ERROR; + + /* close the destination datatype */ + if ( H5Tclose(tid2) < 0) TEST_ERROR; + + return TRUE; + +error: + H5E_BEGIN_TRY { + if(rbuf) + HDfree(rbuf); + if(rbuf2) + HDfree(rbuf2); + H5Sclose(sid2); + H5Sclose(sid); + H5Tclose(tid2); + H5Tclose(tid); + } H5E_END_TRY; + return FALSE; +} /* end compare_attribute() */ + + +/*------------------------------------------------------------------------- + * Function: compare_std_attributes * - * Purpose: Compare attributes on two objects to check that they are equal + * Purpose: Compare "standard" attributes on two objects to check that they are equal * * Return: TRUE if objects have same attributes/FALSE if they are different * @@ -291,18 +471,13 @@ done: *------------------------------------------------------------------------- */ static int -compare_attributes(hid_t oid, hid_t oid2) +compare_std_attributes(hid_t oid, hid_t oid2) { - hid_t sid = -1, sid2 = -1; /* Dataspace IDs */ - hid_t tid = -1, tid2 = -1; /* Datatype IDs */ hid_t aid = -1, aid2 = -1; /* Attribute IDs */ int num_attrs; /* Number of attributes */ int num_attrs2; /* Number of attributes */ char attr_name[ATTR_NAME_LEN]; /* Attribute name */ int wattr_data[2]; /* Attribute buffer for writing */ - int attr_data[2], attr_data2[2]; /* Attribute buffers for reading */ - htri_t is_committed; /* If the datatype is committed */ - htri_t is_committed2; /* If the datatype is committed */ unsigned i; /* Local index variable */ /* Check the number of attributes on source dataset */ @@ -326,56 +501,8 @@ compare_attributes(hid_t oid, hid_t oid2) if ( (aid = H5Aopen_name(oid, attr_name)) < 0) TEST_ERROR if ( (aid2 = H5Aopen_name(oid2, attr_name)) < 0) TEST_ERROR - /* Check the datatypes are equal */ - - /* Open the datatype for the source attribute */ - if ( (tid = H5Aget_type(aid)) < 0) TEST_ERROR; - - /* Open the datatype for the destination attribute */ - if ( (tid2 = H5Aget_type(aid2)) < 0) TEST_ERROR; - - /* Check that both datatypes are committed/not committed */ - if ( (is_committed = H5Tcommitted(tid)) < 0) TEST_ERROR; - if ( (is_committed2 = H5Tcommitted(tid2)) < 0) TEST_ERROR; - if ( is_committed != is_committed2) TEST_ERROR; - - /* Compare the datatypes */ - if ( H5Tequal(tid, tid2) != TRUE) TEST_ERROR; - - /* close the source datatype */ - if ( H5Tclose(tid) < 0) TEST_ERROR; - - /* close the destination datatype */ - if ( H5Tclose(tid2) < 0) TEST_ERROR; - - - /* Check the dataspaces are equal */ - - /* Open the dataspace for the source attribute */ - if ( (sid = H5Aget_space(aid)) < 0) TEST_ERROR; - - /* Open the dataspace for the destination attribute */ - if ( (sid2 = H5Aget_space(aid2)) < 0) TEST_ERROR; - - /* Compare the dataspaces */ - if ( H5Sextent_equal(sid, sid2) != TRUE) TEST_ERROR; - - /* close the source dataspace */ - if ( H5Sclose(sid) < 0) TEST_ERROR; - - /* close the destination dataspace */ - if ( H5Sclose(sid2) < 0) TEST_ERROR; - - - /* Check the attribute data is equal */ - - /* Read the attribute data */ - if ( (H5Aread(aid, H5T_NATIVE_INT, attr_data)) < 0) TEST_ERROR - if ( (H5Aread(aid2, H5T_NATIVE_INT, attr_data2)) < 0) TEST_ERROR - - /* Check attribute data read in against data written out */ - if(wattr_data[0] != attr_data[0] || wattr_data[0] != attr_data2[0]) TEST_ERROR - if(wattr_data[1] != attr_data[1] || wattr_data[1] != attr_data2[1]) TEST_ERROR + /* Check the attributes are equal */ + if ( !compare_attribute(aid, aid2, wattr_data)) TEST_ERROR /* Close the attributes */ if ( H5Aclose(aid) < 0) TEST_ERROR @@ -389,13 +516,9 @@ error: H5E_BEGIN_TRY { H5Aclose(aid2); H5Aclose(aid); - H5Sclose(sid2); - H5Sclose(sid); - H5Tclose(tid2); - H5Tclose(tid); } H5E_END_TRY; return FALSE; -} /* end compare_attributes() */ +} /* end compare_std_attributes() */ /*------------------------------------------------------------------------- @@ -411,24 +534,27 @@ error: *------------------------------------------------------------------------- */ static int -compare_data(hid_t tid, size_t elmt_size, size_t nelmts, void *buf1, void *buf2) +compare_data(hid_t tid, size_t nelmts, const void *buf1, const void *buf2) { + size_t elmt_size; /* Size of an element */ + + /* Check size of each element */ + if((elmt_size = H5Tget_size(tid)) == 0) TEST_ERROR + /* Check for references, which aren't handled */ if(H5Tdetect_class(tid, H5T_REFERENCE) == TRUE) TEST_ERROR /* Check for vlen datatype */ if(H5Tdetect_class(tid, H5T_VLEN) == TRUE) { - hvl_t *vl_buf1, *vl_buf2; /* Aliases for buffers to compare */ + const hvl_t *vl_buf1, *vl_buf2; /* Aliases for buffers to compare */ hid_t base_tid; /* Base type of vlen datatype */ - size_t base_size; /* Size of base type */ size_t u; /* Local index variable */ /* Check for "simple" vlen datatype */ - if(H5Tget_class(tid) != H5T_VLEN) TEST_ERROR; + if(H5Tget_class(tid) != H5T_VLEN) TEST_ERROR /* Get base type of vlen datatype */ - if ( (base_tid = H5Tget_super(tid)) < 0) TEST_ERROR - if ( (base_size = H5Tget_size(base_tid)) == 0) TEST_ERROR + if((base_tid = H5Tget_super(tid)) < 0) TEST_ERROR /* Loop over elements in buffers */ vl_buf1 = buf1; @@ -438,7 +564,7 @@ compare_data(hid_t tid, size_t elmt_size, size_t nelmts, void *buf1, void *buf2) if(vl_buf1->len != vl_buf2->len) TEST_ERROR /* Check vlen data */ - if(!compare_data(base_tid, base_size, vl_buf1->len, vl_buf1->p, vl_buf2->p)) TEST_ERROR + if(!compare_data(base_tid, vl_buf1->len, vl_buf1->p, vl_buf2->p)) TEST_ERROR } /* end for */ if(H5Tclose(base_tid) < 0) TEST_ERROR @@ -467,7 +593,7 @@ error: *------------------------------------------------------------------------- */ static int -compare_datasets(hid_t did, hid_t did2, void *wbuf) +compare_datasets(hid_t did, hid_t did2, const void *wbuf) { hid_t sid = -1, sid2 = -1; /* Dataspace IDs */ hid_t tid = -1, tid2 = -1; /* Datatype IDs */ @@ -475,13 +601,12 @@ compare_datasets(hid_t did, hid_t did2, void *wbuf) size_t elmt_size; /* Size of datatype */ htri_t is_committed; /* If the datatype is committed */ htri_t is_committed2; /* If the datatype is committed */ + int nfilters; /* Number of filters applied to dataset */ hssize_t nelmts; /* # of elements in dataspace */ void *rbuf = NULL; /* Buffer for reading raw data */ void *rbuf2 = NULL; /* Buffer for reading raw data */ H5D_space_status_t space_status; /* Dataset's raw data space status */ H5D_space_status_t space_status2; /* Dataset's raw data space status */ - hsize_t storage_size; /* Dataset's raw data storage size */ - hsize_t storage_size2; /* Dataset's raw data storage size */ /* Check the datatypes are equal */ @@ -529,6 +654,9 @@ compare_datasets(hid_t did, hid_t did2, void *wbuf) /* Compare the dataset creation property lists */ if ( H5Pequal(dcpl, dcpl2) != TRUE) TEST_ERROR; + /* Get the number of filters on dataset */ + if ( (nfilters = H5Pget_nfilters(dcpl)) < 0) TEST_ERROR; + /* close the source dataset creation property list */ if ( H5Pclose(dcpl) < 0) TEST_ERROR; @@ -544,10 +672,17 @@ compare_datasets(hid_t did, hid_t did2, void *wbuf) if(space_status != space_status2) TEST_ERROR; /* Check that the space used is the same */ - storage_size = H5Dget_storage_size(did); - storage_size2 = H5Dget_storage_size(did2); - if(storage_size != storage_size2) TEST_ERROR; - + /* (Don't check if the dataset is filtered (i.e. compressed, etc.) and + * the datatype is variable-length, since the addresses for the vlen + * data in each dataset will (probably) be different and the storage + * size will thus vary) + */ + if (!(nfilters > 0 && H5Tdetect_class(tid, H5T_VLEN))) { + hsize_t storage_size = H5Dget_storage_size(did); /* Dataset's raw data storage size */ + hsize_t storage_size2 = H5Dget_storage_size(did2); /* 2nd Dataset's raw data storage size */ + + if(storage_size != storage_size2) TEST_ERROR; + } /* end if */ /* Check the raw data is equal */ @@ -561,12 +696,12 @@ compare_datasets(hid_t did, hid_t did2, void *wbuf) /* Check raw data read in against data written out */ if(wbuf) { - if ( !compare_data(tid, elmt_size, (size_t)nelmts, wbuf, rbuf)) TEST_ERROR - if ( !compare_data(tid2, elmt_size, (size_t)nelmts, wbuf, rbuf2)) TEST_ERROR + if ( !compare_data(tid, (size_t)nelmts, wbuf, rbuf)) TEST_ERROR + if ( !compare_data(tid2, (size_t)nelmts, wbuf, rbuf2)) TEST_ERROR } /* end if */ /* Don't have written data, just compare data between the two datasets */ else - if ( !compare_data(tid, elmt_size, (size_t)nelmts, rbuf, rbuf2)) TEST_ERROR + if ( !compare_data(tid, (size_t)nelmts, rbuf, rbuf2)) TEST_ERROR /* Reclaim vlen data, if necessary */ if(H5Tdetect_class(tid, H5T_VLEN) == TRUE) @@ -592,7 +727,7 @@ compare_datasets(hid_t did, hid_t did2, void *wbuf) /* Check if the attributes are equal */ - if ( compare_attributes(did, did2) != TRUE) TEST_ERROR; + if ( compare_std_attributes(did, did2) != TRUE) TEST_ERROR; /* Datasets should be the same. :-) */ @@ -738,7 +873,7 @@ HDassert(0 && "Unknown type of object"); } /* end if */ /* Check if the attributes are equal */ - if ( compare_attributes(gid, gid2) != TRUE) TEST_ERROR; + if ( compare_std_attributes(gid, gid2) != TRUE) TEST_ERROR; /* Groups should be the same. :-) */ return TRUE; @@ -1416,7 +1551,7 @@ test_copy_dataset_chunked(hid_t fapl) hid_t pid = -1; /* Dataset creation property list ID */ hid_t did = -1, did2 = -1; /* Dataset IDs */ hsize_t dim2d[2]; /* Dataset dimensions */ - hsize_t chunk_dim2d[2] ={2, 3}; /* Chunk dimensions */ + hsize_t chunk_dim2d[2] ={CHUNK_SIZE_1, CHUNK_SIZE_2}; /* Chunk dimensions */ float buf[DIM_SIZE_1][DIM_SIZE_2]; /* Buffer for writing data */ int i, j; /* Local index variables */ char src_filename[NAME_BUF_SIZE]; @@ -1543,7 +1678,7 @@ test_copy_dataset_chunked_empty(hid_t fapl) hid_t pid = -1; /* Dataset creation property list ID */ hid_t did = -1, did2 = -1; /* Dataset IDs */ hsize_t dim2d[2]; /* Dataset dimensions */ - hsize_t chunk_dim2d[2] ={2, 3}; /* Chunk dimensions */ + hsize_t chunk_dim2d[2] ={CHUNK_SIZE_1, CHUNK_SIZE_2}; /* Chunk dimensions */ char src_filename[NAME_BUF_SIZE]; char dst_filename[NAME_BUF_SIZE]; @@ -1662,7 +1797,7 @@ test_copy_dataset_chunked_sparse(hid_t fapl) hsize_t dim2d[2]; /* Dataset dimensions */ hsize_t new_dim2d[2]; /* Dataset dimensions */ hsize_t max_dim2d[2]; /* Dataset max. dimensions */ - hsize_t chunk_dim2d[2] ={2, 3}; /* Chunk dimensions */ + hsize_t chunk_dim2d[2] ={CHUNK_SIZE_1, CHUNK_SIZE_2}; /* Chunk dimensions */ float buf[DIM_SIZE_1][DIM_SIZE_2]; /* Buffer for writing data */ int i, j; /* Local index variables */ char src_filename[NAME_BUF_SIZE]; @@ -1798,7 +1933,7 @@ test_copy_dataset_compressed(hid_t fapl) hid_t pid = -1; /* Dataset creation property list ID */ hid_t did = -1, did2 = -1; /* Dataset IDs */ hsize_t dim2d[2]; /* Dataset dimensions */ - hsize_t chunk_dim2d[2] ={2, 3}; /* Chunk dimensions */ + hsize_t chunk_dim2d[2] ={CHUNK_SIZE_1, CHUNK_SIZE_2}; /* Chunk dimensions */ float buf[DIM_SIZE_1][DIM_SIZE_2]; /* Buffer for writing data */ int i, j; /* Local index variables */ char src_filename[NAME_BUF_SIZE]; @@ -2984,14 +3119,13 @@ error: * Failure: number of errors * * Programmer: Peter Cao - * Friday, September 30, 2005 + * Saturday, December 10, 2005 * *------------------------------------------------------------------------- */ static int test_copy_dataset_chunked_vl(hid_t fapl) { -#ifdef NOT_YET hid_t fid_src = -1, fid_dst = -1; /* File IDs */ hid_t tid = -1; /* Datatype ID */ hid_t sid = -1; /* Dataspace ID */ @@ -2999,18 +3133,13 @@ test_copy_dataset_chunked_vl(hid_t fapl) hid_t did = -1, did2 = -1; /* Dataset IDs */ unsigned int i, j; /* Local index variables */ hsize_t dim1d[1]; /* Dataset dimensions */ - hsize_t chunk_dim1d[1] = {3}; /* Chunk dimensions */ + hsize_t chunk_dim1d[1] = {CHUNK_SIZE_1}; /* Chunk dimensions */ hvl_t buf[DIM_SIZE_1]; /* Buffer for writing data */ char src_filename[NAME_BUF_SIZE]; char dst_filename[NAME_BUF_SIZE]; -#endif /* NOT_YET */ TESTING("H5Gcopy(): chunked dataset with variable-length datatype"); -#ifndef NOT_YET - SKIPPED(); - puts(" Not supported yet!!"); -#else /* NOT_YET */ /* set initial data values */ for(i = 0; i < DIM_SIZE_1; i++) { buf[i].len = i+1; @@ -3102,10 +3231,8 @@ test_copy_dataset_chunked_vl(hid_t fapl) if ( H5Sclose(sid) < 0) TEST_ERROR; PASSED(); -#endif /* NOT_YET */ return 0; -#ifdef NOT_YET error: H5E_BEGIN_TRY { H5Dclose(did2); @@ -3117,11 +3244,413 @@ error: H5Fclose(fid_src); } H5E_END_TRY; return 1; -#endif /* NOT_YET */ } /* end test_copy_dataset_chunked_vl */ /*------------------------------------------------------------------------- + * Function: test_copy_dataset_compact_vl + * + * Purpose: Create a compact dataset w/variable-length datatype in SRC + * file and copy it to DST file + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Peter Cao + * Sunday, December 11, 2005 + * + *------------------------------------------------------------------------- + */ +static int +test_copy_dataset_compact_vl(hid_t fapl) +{ + hid_t fid_src = -1, fid_dst = -1; /* File IDs */ + hid_t tid = -1; /* Datatype ID */ + hid_t sid = -1; /* Dataspace ID */ + hid_t pid = -1; /* Dataset creation property list ID */ + hid_t did = -1, did2 = -1; /* Dataset IDs */ + unsigned int i, j; /* Local index variables */ + hsize_t dim1d[1]; /* Dataset dimensions */ + hvl_t buf[DIM_SIZE_1]; /* Buffer for writing data */ + char src_filename[NAME_BUF_SIZE]; + char dst_filename[NAME_BUF_SIZE]; + + TESTING("H5Gcopy(): compact dataset with variable-length datatype"); + + /* set initial data values */ + for(i = 0; i < DIM_SIZE_1; i++) { + buf[i].len = i+1; + buf[i].p = (int *)HDmalloc(buf[i].len * sizeof(int)); + for(j = 0; j < buf[i].len; j++) + ((int *)buf[i].p)[j] = i*10+j; + } /* end for */ + + /* Initialize the filenames */ + h5_fixname(FILENAME[0], fapl, src_filename, sizeof src_filename); + h5_fixname(FILENAME[1], fapl, dst_filename, sizeof dst_filename); + + /* Reset file address checking info */ + addr_reset(); + + /* create source file */ + if ( (fid_src = H5Fcreate(src_filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; + + /* Set dataspace dimensions */ + dim1d[0]=DIM_SIZE_1; + + /* create dataspace */ + if ( (sid = H5Screate_simple(1, dim1d, NULL)) < 0) TEST_ERROR; + + /* create datatype */ + if ( (tid = H5Tvlen_create(H5T_NATIVE_INT)) < 0) TEST_ERROR; + + /* create and set compact plist */ + if ( (pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR; + if ( H5Pset_layout(pid, H5D_COMPACT) < 0) TEST_ERROR; + + /* create dataset at SRC file */ + if ( (did = H5Dcreate(fid_src, NAME_DATASET_VL, tid, sid, pid)) < 0) TEST_ERROR; + + /* write data into file */ + if ( H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) TEST_ERROR; + + /* close compact plist */ + if ( H5Pclose(pid) < 0) TEST_ERROR; + + /* close the dataset */ + if ( H5Dclose(did) < 0) TEST_ERROR; + + /* close the SRC file */ + if ( H5Fclose(fid_src) < 0) TEST_ERROR; + + + /* open the source file with read-only */ + if ( (fid_src = H5Fopen(src_filename, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR; + + /* create destination file */ + if ( (fid_dst = H5Fcreate(dst_filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; + + /* Create an uncopied object in destination file so that addresses in source and destination files aren't the same */ + if ( H5Gclose(H5Gcreate(fid_dst, NAME_GROUP_UNCOPIED, (size_t)0)) < 0) TEST_ERROR; + + /* open the dataset for copy */ + if ( (did = H5Dopen(fid_src, NAME_DATASET_VL)) < 0) TEST_ERROR; + + /* copy the dataset from SRC to DST */ + if ( H5Gcopy(did, fid_dst, NAME_DATASET_VL, H5P_DEFAULT) < 0) TEST_ERROR; + + /* open the destination dataset */ + if ( (did2 = H5Dopen(fid_dst, NAME_DATASET_VL)) < 0) TEST_ERROR; + + /* Check if the datasets are equal */ + if ( compare_datasets(did, did2, buf) != TRUE) TEST_ERROR; + + /* close the destination dataset */ + if ( H5Dclose(did2) < 0) TEST_ERROR; + + /* close the source dataset */ + if ( H5Dclose(did) < 0) TEST_ERROR; + + /* close the SRC file */ + if ( H5Fclose(fid_src) < 0) TEST_ERROR; + + /* close the DST file */ + if ( H5Fclose(fid_dst) < 0) TEST_ERROR; + + + /* Reclaim vlen buffer */ + if ( H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, buf) < 0) TEST_ERROR; + + /* close datatype */ + if ( H5Tclose(tid) < 0) TEST_ERROR; + + /* close dataspace */ + if ( H5Sclose(sid) < 0) TEST_ERROR; + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Dclose(did2); + H5Dclose(did); + H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, buf); + H5Tclose(tid); + H5Sclose(sid); + H5Fclose(fid_dst); + H5Fclose(fid_src); + } H5E_END_TRY; + return 1; +} /* end test_copy_dataset_compact_vl */ + + +/*------------------------------------------------------------------------- + * Function: test_copy_attribute_vl + * + * Purpose: Create a simple dataset with vlen attributes in SRC file + * and copy it to DST file (Note: dataset has no data) + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Peter Cao + * Saturday, December , 2005 + * + *------------------------------------------------------------------------- + */ +static int +test_copy_attribute_vl(hid_t fapl) +{ + hid_t fid_src = -1, fid_dst = -1; /* File IDs */ + hid_t sid = -1; /* Dataspace ID */ + hid_t did = -1, did2 = -1; /* Dataset IDs */ + hid_t aid = -1, aid2 = -1; /* Attribute IDs */ + hsize_t dim2d[2]; /* Dataset dimensions */ + char src_filename[NAME_BUF_SIZE]; + char dst_filename[NAME_BUF_SIZE]; + + TESTING("H5Gcopy(): variable length attribute"); + + /* Initialize the filenames */ + h5_fixname(FILENAME[0], fapl, src_filename, sizeof src_filename); + h5_fixname(FILENAME[1], fapl, dst_filename, sizeof dst_filename); + + /* Reset file address checking info */ + addr_reset(); + + /* create source file */ + if ( (fid_src = H5Fcreate(src_filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; + + /* Set dataspace dimensions */ + dim2d[0] = DIM_SIZE_1; + dim2d[1] = DIM_SIZE_2; + + /* create 2D dataspace */ + if ( (sid = H5Screate_simple(2, dim2d, NULL)) < 0) TEST_ERROR; + + /* create 2D int dataset at SRC file */ + if ( (did = H5Dcreate(fid_src, NAME_DATASET_SIMPLE, H5T_NATIVE_INT, sid, H5P_DEFAULT)) < 0) TEST_ERROR; + + /* close dataspace */ + if ( H5Sclose(sid) < 0) TEST_ERROR; + + /* attach VL attribute to the dataset */ + if ( test_copy_attach_attribute_vl(did) < 0) TEST_ERROR; + + /* close the dataset */ + if ( H5Dclose(did) < 0) TEST_ERROR; + + /* close the SRC file */ + if ( H5Fclose(fid_src) < 0) TEST_ERROR; + + + /* open the source file with read-only */ + if ( (fid_src = H5Fopen(src_filename, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR; + + /* create destination file */ + if ( (fid_dst = H5Fcreate(dst_filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; + + /* Create an uncopied object in destination file so that addresses in source and destination files aren't the same */ + if ( H5Gclose(H5Gcreate(fid_dst, NAME_GROUP_UNCOPIED, (size_t)0)) < 0) TEST_ERROR; + + /* open the dataset for copy */ + if ( (did = H5Dopen(fid_src, NAME_DATASET_SIMPLE)) < 0) TEST_ERROR; + + /* copy the dataset from SRC to DST */ + if ( H5Gcopy(did, fid_dst, NAME_DATASET_SIMPLE, H5P_DEFAULT) < 0) TEST_ERROR; + + /* open the destination dataset */ + if ( (did2 = H5Dopen(fid_dst, NAME_DATASET_SIMPLE)) < 0) TEST_ERROR; + + /* Check if the attributes are equal */ + + if ( (aid = H5Aopen_idx(did, 0)) < 0) TEST_ERROR; + if ( (aid2 = H5Aopen_idx(did2, 0)) < 0) TEST_ERROR; + if ( compare_attribute(aid, aid2, NULL) != TRUE) TEST_ERROR; + if ( H5Aclose(aid) < 0) TEST_ERROR; + if ( H5Aclose(aid2) < 0) TEST_ERROR; + + /* close the destination dataset */ + if ( H5Dclose(did2) < 0) TEST_ERROR; + + /* close the source dataset */ + if ( H5Dclose(did) < 0) TEST_ERROR; + + /* close the SRC file */ + if ( H5Fclose(fid_src) < 0) TEST_ERROR; + + /* close the DST file */ + if ( H5Fclose(fid_dst) < 0) TEST_ERROR; + + PASSED(); + return 0; + +error: + H5E_BEGIN_TRY { + H5Aclose(aid2); + H5Aclose(aid); + H5Dclose(did2); + H5Dclose(did); + H5Sclose(sid); + H5Fclose(fid_dst); + H5Fclose(fid_src); + } H5E_END_TRY; + return 1; +} /* end test_copy_dataset_simple_empty */ + + +/*------------------------------------------------------------------------- + * Function: test_copy_dataset_compressed_vl + * + * Purpose: Create a compressed, chunked, variable-length dataset in SRC + * file and copy it to DST file + * + * Return: Success: 0 + * Failure: number of errors + * + * Programmer: Peter Cao + * Tuesday, December 27, 2005 + * + *------------------------------------------------------------------------- + */ +static int +test_copy_dataset_compressed_vl(hid_t fapl) +{ +#ifdef H5_HAVE_FILTER_DEFLATE + hid_t fid_src = -1, fid_dst = -1; /* File IDs */ + hid_t sid = -1; /* Dataspace ID */ + hid_t tid = -1; /* Datatype ID */ + hid_t pid = -1; /* Dataset creation property list ID */ + hid_t did = -1, did2 = -1; /* Dataset IDs */ + hsize_t dim2d[2]; /* Dataset dimensions */ + hsize_t chunk_dim2d[2] ={CHUNK_SIZE_1, CHUNK_SIZE_2}; /* Chunk dimensions */ + hvl_t buf[DIM_SIZE_1][DIM_SIZE_2]; /* Buffer for writing data */ + int i, j, k; /* Local index variables */ + char src_filename[NAME_BUF_SIZE]; + char dst_filename[NAME_BUF_SIZE]; +#endif /* H5_HAVE_FILTER_DEFLATE */ + + TESTING("H5Gcopy(): compressed dataset with variable-length datatype"); + +#ifndef H5_HAVE_FILTER_DEFLATE + SKIPPED(); + puts(" Deflation filter not available"); +#else /* H5_HAVE_FILTER_DEFLATE */ + /* set initial data values */ + for (i = 0; i < DIM_SIZE_1; i++) { + for (j = 0; j < DIM_SIZE_2; j++) { + buf[i][j].len = j + 1; + buf[i][j].p = (int *)HDmalloc(buf[i][j].len * sizeof(int)); + for (k = 0; k < (int)buf[i][j].len; k++) + ((int *)buf[i][j].p)[k] = i * 10000 + j * 100 + k; + } + } + + /* Initialize the filenames */ + h5_fixname(FILENAME[0], fapl, src_filename, sizeof src_filename); + h5_fixname(FILENAME[1], fapl, dst_filename, sizeof dst_filename); + + /* Reset file address checking info */ + addr_reset(); + + /* create source file */ + if ( (fid_src = H5Fcreate(src_filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; + + /* Set dataspace dimensions */ + dim2d[0]=DIM_SIZE_1; + dim2d[1]=DIM_SIZE_2; + + /* create dataspace */ + if ( (sid = H5Screate_simple(2, dim2d, NULL)) < 0) TEST_ERROR; + + /* create datatype */ + if ( (tid = H5Tvlen_create(H5T_NATIVE_INT)) < 0) TEST_ERROR; + + /* create and set comp & chunk plist */ + if ( (pid = H5Pcreate(H5P_DATASET_CREATE)) < 0) TEST_ERROR; + if ( H5Pset_chunk(pid, 2, chunk_dim2d) < 0) TEST_ERROR; + if ( H5Pset_deflate(pid, 9) < 0) TEST_ERROR; + + /* create dataset */ + if ( (did = H5Dcreate(fid_src, NAME_DATASET_CHUNKED, tid, sid, pid)) < 0) TEST_ERROR; + + /* close chunk plist */ + if ( H5Pclose(pid) < 0) TEST_ERROR; + + /* write data into file */ + if ( H5Dwrite(did, tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, buf) < 0) TEST_ERROR; + + /* close the dataset */ + if ( H5Dclose(did) < 0) TEST_ERROR; + + /* close the SRC file */ + if ( H5Fclose(fid_src) < 0) TEST_ERROR; + + + /* open the source file with read-only */ + if ( (fid_src = H5Fopen(src_filename, H5F_ACC_RDONLY, fapl)) < 0) TEST_ERROR; + + /* create destination file */ + if ( (fid_dst = H5Fcreate(dst_filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) TEST_ERROR; + + /* Create an uncopied object in destination file so that addresses in source and destination files aren't the same */ + if ( H5Gclose(H5Gcreate(fid_dst, NAME_GROUP_UNCOPIED, (size_t)0)) < 0) TEST_ERROR; + + /* open the dataset for copy */ + if ( (did = H5Dopen(fid_src, NAME_DATASET_CHUNKED)) < 0) TEST_ERROR; + + /* copy the dataset from SRC to DST */ + if ( H5Gcopy(did, fid_dst, NAME_DATASET_CHUNKED, H5P_DEFAULT) < 0) TEST_ERROR; + + /* open the destination dataset */ + if ( (did2 = H5Dopen(fid_dst, NAME_DATASET_CHUNKED)) < 0) TEST_ERROR; + + /* Check if the datasets are equal */ + if ( compare_datasets(did, did2, NULL) != TRUE) TEST_ERROR; + + /* close the destination dataset */ + if ( H5Dclose(did2) < 0) TEST_ERROR; + + /* close the source dataset */ + if ( H5Dclose(did) < 0) TEST_ERROR; + + /* close the SRC file */ + if ( H5Fclose(fid_src) < 0) TEST_ERROR; + + /* close the DST file */ + if ( H5Fclose(fid_dst) < 0) TEST_ERROR; + + /* Reclaim vlen buffer */ + if ( H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, buf) < 0) TEST_ERROR; + + /* close datatype */ + if ( H5Tclose(tid) < 0) TEST_ERROR; + + /* close dataspace */ + if ( H5Sclose(sid) < 0) TEST_ERROR; + + PASSED(); +#endif /* H5_HAVE_FILTER_DEFLATE */ + return 0; + +#ifdef H5_HAVE_FILTER_DEFLATE +error: + H5E_BEGIN_TRY { + H5Dclose(did2); + H5Dclose(did); + H5Pclose(pid); + H5Dvlen_reclaim(tid, sid, H5P_DEFAULT, buf); + H5Tclose(tid); + H5Sclose(sid); + H5Fclose(fid_dst); + H5Fclose(fid_src); + } H5E_END_TRY; + return 1; +#endif /* H5_HAVE_FILTER_DEFLATE */ +} /* end test_copy_dataset_compressed_vl */ + + +/*------------------------------------------------------------------------- * Function: test_copy_group_empty * * Purpose: Create an empty group in SRC file and copy it to DST file @@ -4375,7 +4904,10 @@ main(void) nerrors += test_copy_dataset_multi_ohdr_chunks(fapl); nerrors += test_copy_dataset_attr_named_dtype(fapl); nerrors += test_copy_dataset_contig_vl(fapl); - nerrors += test_copy_dataset_chunked_vl(fapl); /* TODO */ + nerrors += test_copy_dataset_chunked_vl(fapl); + nerrors += test_copy_dataset_compact_vl(fapl); + nerrors += test_copy_dataset_compressed_vl(fapl); + nerrors += test_copy_attribute_vl(fapl); /* TODO: Add more tests for copying vlen data */ nerrors += test_copy_group_empty(fapl); nerrors += test_copy_group(fapl); -- cgit v0.12