diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/H5A.c | 1 | ||||
| -rw-r--r-- | src/H5Dcompact.c | 155 | ||||
| -rw-r--r-- | src/H5Dcontig.c | 88 | ||||
| -rw-r--r-- | src/H5Distore.c | 240 | ||||
| -rw-r--r-- | src/H5Doh.c | 8 | ||||
| -rw-r--r-- | src/H5Dpkg.h | 8 | ||||
| -rw-r--r-- | src/H5Dprivate.h | 5 | ||||
| -rw-r--r-- | src/H5O.c | 6 | ||||
| -rw-r--r-- | src/H5Oattr.c | 164 | ||||
| -rw-r--r-- | src/H5Olayout.c | 23 | ||||
| -rw-r--r-- | src/H5Opkg.h | 1 | ||||
| -rw-r--r-- | src/H5Opline.c | 47 | ||||
| -rw-r--r-- | src/H5R.c | 1 | ||||
| -rw-r--r-- | src/H5Sprivate.h | 5 | ||||
| -rw-r--r-- | src/H5Sselect.c | 1 | ||||
| -rw-r--r-- | src/H5Tvlen.c | 2 |
16 files changed, 683 insertions, 72 deletions
@@ -21,6 +21,7 @@ /* Private header files */ #include "H5private.h" /* Generic Functions */ #include "H5Apkg.h" /* Attributes */ +#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ diff --git a/src/H5Dcompact.c b/src/H5Dcompact.c index 83f9b8c..0ad8e05 100644 --- a/src/H5Dcompact.c +++ b/src/H5Dcompact.c @@ -36,6 +36,7 @@ #include "H5Fprivate.h" /* Files */ #include "H5FDprivate.h" /* File drivers */ #include "H5FLprivate.h" /* Free Lists */ +#include "H5Iprivate.h" /* IDs */ #include "H5Oprivate.h" /* Object headers */ #include "H5Vprivate.h" /* Vector and array functions */ @@ -59,6 +60,9 @@ /* Local Variables */ /*******************/ +/* Declare extern the free list to manage blocks of type conversion data */ +H5FL_BLK_EXTERN(type_conv); + /*------------------------------------------------------------------------- * Function: H5D_compact_readvv @@ -141,3 +145,154 @@ H5D_compact_writevv(const H5D_io_info_t *io_info, done: FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_compact_writevv() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_compact_copy + * + * Purpose: Copy compact storage raw data from SRC file to DST file. + * + * Return: Non-negative on success, negative on failure. + * + * Programmer: Peter Cao + * December 11, 2005 + * + *------------------------------------------------------------------------- + */ +herr_t +H5D_compact_copy(H5F_t *f_src, H5O_layout_t *layout_src, + H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *dt_src, hid_t dxpl_id) +{ + hid_t tid_src = -1; /* Datatype ID for source datatype */ + hid_t tid_dst = -1; /* Datatype ID for destination datatype */ + hid_t tid_mem = -1; /* Datatype ID for memory datatype */ + void *buf = NULL; /* Buffer for copying data */ + void *reclaim_buf = NULL; /* Buffer for reclaiming data */ + hid_t buf_sid = -1; /* ID for buffer dataspace */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI(H5D_compact_copy, FAIL) + + /* Check args */ + HDassert(f_src); + HDassert(f_dst); + HDassert(layout_src && H5D_COMPACT == layout_src->type); + HDassert(layout_dst && H5D_COMPACT == layout_dst->type); + + /* If there's a source datatype, set up type conversion information */ + if (!dt_src) + /* Type conversion not necessary */ + HDmemcpy(layout_dst->u.compact.buf, layout_src->u.compact.buf, layout_src->u.compact.size); + else { + H5T_path_t *tpath_src_mem, *tpath_mem_dst; /* Datatype conversion paths */ + H5T_t *dt_dst; /* Destination datatype */ + H5T_t *dt_mem; /* Memory datatype */ + H5S_t *buf_space; /* Dataspace describing buffer */ + size_t buf_size; /* Size of copy buffer */ + size_t nelmts; /* Number of elements in buffer */ + size_t src_dt_size; /* Source datatype size */ + size_t tmp_dt_size; /* Temporary datatype size */ + size_t max_dt_size; /* Max atatype size */ + hsize_t buf_dim; /* Dimension for buffer */ + + /* Create datatype ID for src datatype */ + if((tid_src = H5I_register(H5I_DATATYPE, dt_src)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register source file datatype") + + /* create a memory copy of the variable-length datatype */ + if(NULL == (dt_mem = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") + if((tid_mem = H5I_register(H5I_DATATYPE, dt_mem)) < 0) + HGOTO_ERROR (H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") + + /* create variable-length datatype at the destinaton file */ + if(NULL == (dt_dst = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") + if(H5T_set_loc(dt_dst, f_dst, H5T_LOC_DISK) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "cannot mark datatype on disk") + if((tid_dst = H5I_register(H5I_DATATYPE, dt_dst)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register destination file datatype") + + /* Set up the conversion functions */ + if(NULL == (tpath_src_mem = H5T_path_find(dt_src, dt_mem, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and mem datatypes") + if(NULL == (tpath_mem_dst = H5T_path_find(dt_mem, dt_dst, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between mem and dst datatypes") + + /* Determine largest datatype size */ + if(0 == (src_dt_size = H5T_get_size(dt_src))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + if(0 == (tmp_dt_size = H5T_get_size(dt_mem))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + max_dt_size = MAX(src_dt_size, tmp_dt_size); + if(0 == (tmp_dt_size = H5T_get_size(dt_dst))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + max_dt_size = MAX(max_dt_size, tmp_dt_size); + + /* Set number of whole elements that fit in buffer */ + if(0 == (nelmts = layout_src->u.compact.size / src_dt_size)) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "element size too large") + + /* Set up number of bytes to copy, and initial buffer size */ + buf_size = nelmts * max_dt_size; + + /* Create dataspace for number of elements in buffer */ + buf_dim = nelmts; + + /* Create the space and set the initial extent */ + if(NULL == (buf_space = H5S_create_simple((unsigned)1, &buf_dim, NULL))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "can't create simple dataspace") + + /* Atomize */ + if((buf_sid = H5I_register(H5I_DATASPACE, buf_space)) < 0) { + H5S_close(buf_space); + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace ID") + } /* end if */ + + /* Allocate memory for recclaim buf */ + if(NULL == (reclaim_buf = H5FL_BLK_MALLOC(type_conv, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + + /* Allocate memory for copying the chunk */ + if(NULL == (buf = H5FL_BLK_MALLOC(type_conv, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + + HDmemcpy(buf, layout_src->u.compact.buf, layout_src->u.compact.size); + + /* Convert from source file to memory */ + if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed") + + HDmemcpy(reclaim_buf, buf, buf_size); + + /* Convert from memory to destination file */ + if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed") + + HDmemcpy(layout_dst->u.compact.buf, buf, layout_dst->u.compact.size); + + if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to reclaim variable-length data") + } /* end if */ + +done: + if(buf_sid > 0) + if(H5I_dec_ref(buf_sid) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary dataspace ID") + if(tid_src > 0) + if(H5I_dec_ref(tid_src) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(tid_dst > 0) + if(H5I_dec_ref(tid_dst) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(tid_mem > 0) + if(H5I_dec_ref(tid_mem) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(buf) + H5FL_BLK_FREE(type_conv, buf); + if(reclaim_buf) + H5FL_BLK_FREE(type_conv, reclaim_buf); + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_compact_copy() */ + diff --git a/src/H5Dcontig.c b/src/H5Dcontig.c index 21b2e56..f6d1648 100644 --- a/src/H5Dcontig.c +++ b/src/H5Dcontig.c @@ -42,7 +42,6 @@ #include "H5MFprivate.h" /* File memory management */ #include "H5Oprivate.h" /* Object headers */ #include "H5Pprivate.h" /* Property lists */ -#include "H5Sprivate.h" /* Dataspace functions */ #include "H5Vprivate.h" /* Vector and array functions */ /****************/ @@ -1005,9 +1004,15 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, hid_t tid_src = -1; /* Datatype ID for source datatype */ hid_t tid_dst = -1; /* Datatype ID for destination datatype */ hid_t tid_mem = -1; /* Datatype ID for memory datatype */ + size_t src_dt_size; /* Source datatype size */ + size_t mem_dt_size; /* Memory datatype size */ + size_t dst_dt_size; /* Destination datatype size */ size_t max_dt_size; /* Max. datatype size */ size_t nelmts = 0; /* Number of elements in buffer */ - hsize_t total_nbytes; /* Total number of bytes to copy */ + size_t src_nbytes; /* Number of bytes to read from source */ + size_t mem_nbytes; /* Number of bytes to convert in memory */ + size_t dst_nbytes; /* Number of bytes to write to destination */ + hsize_t total_src_nbytes; /* Total number of bytes to copy */ size_t buf_size; /* Size of copy buffer */ void *buf = NULL; /* Buffer for copying data */ void *reclaim_buf = NULL; /* Buffer for reclaiming data */ @@ -1030,14 +1035,12 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to allocate contiguous storage") /* Set up number of bytes to copy, and initial buffer size */ - total_nbytes = layout_src->u.contig.size; - H5_CHECK_OVERFLOW(total_nbytes,hsize_t,size_t); - buf_size = MIN(H5D_XFER_MAX_TEMP_BUF_DEF, (size_t)total_nbytes); + total_src_nbytes = layout_src->u.contig.size; + H5_CHECK_OVERFLOW(total_src_nbytes,hsize_t,size_t); + buf_size = MIN(H5D_XFER_MAX_TEMP_BUF_DEF, (size_t)total_src_nbytes); /* If there's a source datatype, set up type conversion information */ if(dt_src) { - size_t tmp_dt_size; /* Temp. atatype size */ - /* Create datatype ID for src datatype */ if((tid_src = H5I_register(H5I_DATATYPE, dt_src)) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register source file datatype") @@ -1063,19 +1066,24 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between mem and dst datatypes") /* Determine largest datatype size */ - if(0 == (max_dt_size = H5T_get_size(dt_src))) + if(0 == (src_dt_size = H5T_get_size(dt_src))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") - if(0 == (tmp_dt_size = H5T_get_size(dt_mem))) + if(0 == (mem_dt_size = H5T_get_size(dt_mem))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") - max_dt_size = MAX(max_dt_size, tmp_dt_size); - if(0 == (tmp_dt_size = H5T_get_size(dt_dst))) + max_dt_size = MAX(src_dt_size, mem_dt_size); + if(0 == (dst_dt_size = H5T_get_size(dt_dst))) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") - max_dt_size = MAX(max_dt_size, tmp_dt_size); + max_dt_size = MAX(max_dt_size, dst_dt_size); - /* Set number of whole elements that fit in buffer */ + /* Set maximum number of whole elements that fit in buffer */ if(0 == (nelmts = buf_size / max_dt_size)) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "element size too large") + /* Set the number of bytes to transfer */ + src_nbytes = nelmts * src_dt_size; + dst_nbytes = nelmts * dst_dt_size; + mem_nbytes = nelmts * mem_dt_size; + /* Adjust buffer size to be multiple of elements */ buf_size = nelmts * max_dt_size; @@ -1095,10 +1103,14 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, /* Set flag to do type conversion */ do_conv = TRUE; } /* end if */ - else + else { /* Type conversion not necessary */ do_conv = FALSE; + /* Set the number of bytes to read & write to the buffer size */ + src_nbytes = dst_nbytes = mem_nbytes = buf_size; + } /* end else */ + /* Allocate space for copy buffer */ HDassert(buf_size); if(NULL == (buf = H5FL_BLK_MALLOC(type_conv, buf_size))) @@ -1113,28 +1125,34 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, /* Loop over copying data */ addr_src = layout_src->u.contig.addr; addr_dst = layout_dst->u.contig.addr; - while(total_nbytes > 0) { - size_t nbytes; /* Number of bytes to copy each time */ - - /* Compute number of bytes to copy for this pass */ - if(total_nbytes >= buf_size) - nbytes = buf_size; - else { - nbytes = (size_t)total_nbytes; + while(total_src_nbytes > 0) { + /* Check if we should reduce the number of bytes to transfer */ + if(total_src_nbytes < src_nbytes) { + /* Adjust bytes to transfer */ + src_nbytes = (size_t)total_src_nbytes; /* Adjust dataspace describing buffer */ if(do_conv) { + /* Adjust destination & memory bytes to transfer */ + nelmts = src_nbytes / src_dt_size; + dst_nbytes = nelmts * dst_dt_size; + mem_nbytes = nelmts * mem_dt_size; + /* Adjust size of buffer's dataspace dimension */ - buf_dim = nelmts = nbytes / max_dt_size; + buf_dim = nelmts; /* Adjust size of buffer's dataspace */ if(H5S_set_extent_real(buf_space, &buf_dim) < 0) HGOTO_ERROR(H5E_DATASPACE, H5E_CANTSET, FAIL, "unable to change buffer dataspace size") } /* end if */ - } /* end else */ + else { + /* Adjust destination & memory bytes to transfer */ + dst_nbytes = mem_nbytes = src_nbytes; + } /* end else */ + } /* end if */ /* Read raw data from source file */ - if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0) + if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, src_nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_READERROR, FAIL, "unable to read raw data") /* Perform datatype conversion, if necessary */ @@ -1144,27 +1162,25 @@ H5D_contig_copy(H5F_t *f_src, H5O_layout_t *layout_src, HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed") /* Copy into another buffer, to reclaim memory later */ - HDmemcpy(reclaim_buf, buf, nbytes); + HDmemcpy(reclaim_buf, buf, mem_nbytes); /* Convert from memory to destination file */ if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "datatype conversion failed") + + /* Reclaim space from variable length data */ + if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to reclaim variable-length data") } /* end if */ /* Write raw data to destination file */ - if(H5F_block_write(f_dst, H5FD_MEM_DRAW, addr_dst, nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0) + if(H5F_block_write(f_dst, H5FD_MEM_DRAW, addr_dst, dst_nbytes, H5P_DATASET_XFER_DEFAULT, buf) < 0) HGOTO_ERROR(H5E_DATASET, H5E_WRITEERROR, FAIL, "unable to write raw data") - /* Reclaim any space from variable length data */ - if(do_conv) { - if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_BADITER, FAIL, "unable to reclaim variable-length data") - } /* end if */ - /* Adjust loop variables */ - addr_src += nbytes; - addr_dst += nbytes; - total_nbytes -= nbytes; + addr_src += src_nbytes; + addr_dst += dst_nbytes; + total_src_nbytes -= src_nbytes; } /* end while */ done: diff --git a/src/H5Distore.c b/src/H5Distore.c index c450293..570cf81 100644 --- a/src/H5Distore.c +++ b/src/H5Distore.c @@ -194,6 +194,22 @@ typedef struct H5D_istore_it_ud4_t { H5D_istore_bt_ud_common_t common; /* Common info for B-tree user data (must be first) */ H5F_t *file_dst; /* Destination file for copy */ haddr_t addr_dst; /* Address of dest. B-tree */ + void *buf; /* Buffer to hold chunk data for read/write */ + size_t buf_size; /* Buffer size */ + + /* needed for converting variable-length data */ + hid_t tid_src; /* Datatype ID for source datatype */ + hid_t tid_dst; /* Datatype ID for destination datatype */ + hid_t tid_mem; /* Datatype ID for memory datatype */ + H5T_path_t *tpath_src_mem; /* Datatype conversion path from source file to memory */ + H5T_path_t *tpath_mem_dst; /* Datatype conversion path from memory to dest. file */ + void *reclaim_buf; /* Buffer for reclaiming data */ + size_t reclaim_buf_size; /* Reclaim buffer size */ + size_t nelmts; /* Number of elements in buffer */ + H5S_t *buf_space; /* Dataspace describing buffer */ + + /* needed for compressed variable-length data */ + H5O_pline_t *pline; /* Filter pipeline */ } H5D_istore_it_ud4_t; /* B-tree callback info for iteration to obtain chunk address and the index of the chunk for all chunks in the B-tree. */ @@ -975,40 +991,111 @@ static int H5D_istore_iter_copy(H5F_t *f_src, hid_t dxpl_id, const void *_lt_key, haddr_t addr_src, const void UNUSED *_rt_key, void *_udata) { - H5D_istore_it_ud4_t *udata = (H5D_istore_it_ud4_t *)_udata; - const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; - void *chunk = NULL; /*the chunk data */ - H5D_istore_ud1_t udata_dst; - int ret_value = H5B_ITER_CONT; /* Return value */ + H5D_istore_it_ud4_t *udata = (H5D_istore_it_ud4_t *)_udata; + const H5D_istore_key_t *lt_key = (const H5D_istore_key_t *)_lt_key; + H5D_istore_ud1_t udata_dst; /* User data about new destination chunk */ + hbool_t is_vlen = FALSE; + + /* General information about chunk copy */ + void *buf = udata->buf; + size_t buf_size = udata->buf_size; + H5O_pline_t *pline = udata->pline; + + /* needed for commpressed variable length data */ + hbool_t is_compressed = FALSE; + H5Z_EDC_t edc_read = H5Z_NO_EDC; + size_t nbytes = lt_key->nbytes; + H5Z_cb_t cb_struct; + + int ret_value = H5B_ITER_CONT; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5D_istore_iter_copy) - /* Allocate memory for copying the chunk */ - if(NULL == (chunk = H5MM_malloc(lt_key->nbytes))) - HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5B_ITER_ERROR, "memory allocation failed for raw data chunk") + /* Check parameter for type conversion */ + if (udata->tid_src > 0) + is_vlen = TRUE; + + /* Check for filtered chunks */ + if (pline && pline->nused) { + is_compressed = TRUE; + cb_struct.func = NULL; /* no callback function when failed */ + } /* end if */ + + /* Resize the buf if it is too small to hold the data */ + if ( nbytes > buf_size) { + /* Re-allocate memory for copying the chunk */ + if(NULL == (udata->buf = H5MM_realloc(udata->buf, nbytes))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, H5B_ITER_ERROR, "memory allocation failed for raw data chunk") + + buf = udata->buf; + udata->buf_size = buf_size = nbytes; + } /* end if */ /* read chunk data from the source file */ - if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, lt_key->nbytes, dxpl_id, chunk) < 0) + if(H5F_block_read(f_src, H5FD_MEM_DRAW, addr_src, nbytes, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_READERROR, H5B_ITER_ERROR, "unable to read raw data chunk") + /* need to uncompress variable-length data */ + if (is_compressed && is_vlen) { + unsigned filter_mask = lt_key->filter_mask; + + if(H5Z_pipeline(pline, H5Z_FLAG_REVERSE, &filter_mask, edc_read, cb_struct, &nbytes, &buf_size, &buf) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_READERROR, H5B_ITER_ERROR, "data pipeline read failed") + } /* end if */ + + /* Perform datatype conversion, if necessary */ + if(is_vlen) { + H5T_path_t *tpath_src_mem = udata->tpath_src_mem; + H5T_path_t *tpath_mem_dst = udata->tpath_mem_dst; + H5S_t *buf_space = udata->buf_space; + hid_t tid_src = udata->tid_src; + hid_t tid_dst = udata->tid_dst; + hid_t tid_mem = udata->tid_mem; + size_t nelmts = udata->nelmts; + void *reclaim_buf = udata->reclaim_buf; + size_t reclaim_buf_size = udata->reclaim_buf_size; + + /* Convert from source file to memory */ + if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5B_ITER_ERROR, "datatype conversion failed") + + /* Copy into another buffer, to reclaim memory later */ + HDmemcpy(reclaim_buf, buf, reclaim_buf_size); + + /* Convert from memory to destination file */ + if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, H5B_ITER_ERROR, "datatype conversion failed") + + /* Reclaim space from variable length data */ + if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADITER, H5B_ITER_ERROR, "unable to reclaim variable-length data") + } /* end if */ + /* Copy source chunk callback information for insertion */ HDmemset(&udata_dst, 0, sizeof(udata_dst)); HDmemcpy(&(udata_dst.common.key), lt_key, sizeof(H5D_istore_key_t)); udata_dst.common.mesg = udata->common.mesg; /* Share this pointer for a short while */ + /* need to compress variable-length data before writing to file*/ + if (is_compressed && is_vlen) { + if(H5Z_pipeline(pline, 0, &(udata_dst.common.key.filter_mask), edc_read, + cb_struct, &nbytes, &buf_size, &buf) < 0) + HGOTO_ERROR(H5E_PLINE, H5E_READERROR, H5B_ITER_ERROR, "output pipeline failed") + udata_dst.common.key.nbytes = nbytes; + udata->buf = buf; + udata->buf_size = buf_size; + } /* end if */ + /* Insert chunk into the destination Btree */ if(H5B_insert(udata->file_dst, dxpl_id, H5B_ISTORE, udata->addr_dst, &udata_dst) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, H5B_ITER_ERROR, "unable to allocate chunk") /* Write chunk data to destination file */ HDassert(H5F_addr_defined(udata_dst.addr)); - if(H5F_block_write(udata->file_dst, H5FD_MEM_DRAW, udata_dst.addr, udata_dst.common.key.nbytes, dxpl_id, chunk) < 0) + if(H5F_block_write(udata->file_dst, H5FD_MEM_DRAW, udata_dst.addr, nbytes, dxpl_id, buf) < 0) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, H5B_ITER_ERROR, "unable to write raw data to file") done: - if(chunk) - H5MM_xfree(chunk); - FUNC_LEAVE_NOAPI(ret_value) } /* end H5D_istore_iter_copy() */ @@ -3438,11 +3525,22 @@ done: *------------------------------------------------------------------------- */ herr_t -H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, - H5F_t *f_dst, H5O_layout_t *layout_dst, hid_t dxpl_id) +H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, H5F_t *f_dst, + H5O_layout_t *layout_dst, H5T_t *dt_src, H5O_pline_t *pline, hid_t dxpl_id) { H5D_istore_it_ud4_t udata; - herr_t ret_value = SUCCEED; /* Return value */ + H5T_path_t *tpath_src_mem = NULL, *tpath_mem_dst = NULL; /* Datatype conversion paths */ + hid_t tid_src = -1; /* Datatype ID for source datatype */ + hid_t tid_dst = -1; /* Datatype ID for destination datatype */ + hid_t tid_mem = -1; /* Datatype ID for memory datatype */ + size_t buf_size; /* Size of copy buffer */ + size_t reclaim_buf_size; /* Size of reclaim buffer */ + void *buf = NULL; /* Buffer for copying data */ + void *reclaim_buf = NULL; /* Buffer for reclaiming data */ + H5S_t *buf_space = NULL; /* Dataspace describing buffer */ + hid_t sid_buf = -1; /* ID for buffer dataspace */ + size_t nelmts = 0; /* Number of elements in buffer */ + herr_t ret_value = SUCCEED; /* Return value */ FUNC_ENTER_NOAPI(H5D_istore_copy, FAIL) @@ -3465,19 +3563,129 @@ H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to initialize chunked storage") } /* end if */ + /* If there's a source datatype, set up type conversion information */ + if(dt_src) { + H5T_t *dt_dst; /* Destination datatype */ + H5T_t *dt_mem; /* Memory datatype */ + size_t mem_dt_size; /* Memory datatype size */ + size_t tmp_dt_size; /* Temp. datatype size */ + size_t max_dt_size; /* Max atatype size */ + hsize_t buf_dim; /* Dimension for buffer */ + unsigned u; + + /* Create datatype ID for src datatype */ + if((tid_src = H5I_register(H5I_DATATYPE, dt_src)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register source file datatype") + + /* create a memory copy of the variable-length datatype */ + if(NULL == (dt_mem = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") + if((tid_mem = H5I_register(H5I_DATATYPE, dt_mem)) < 0) + HGOTO_ERROR (H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register memory datatype") + + /* create variable-length datatype at the destinaton file */ + if(NULL == (dt_dst = H5T_copy(dt_src, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to copy") + if(H5T_set_loc(dt_dst, f_dst, H5T_LOC_DISK) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "cannot mark datatype on disk") + if((tid_dst = H5I_register(H5I_DATATYPE, dt_dst)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, FAIL, "unable to register destination file datatype") + + /* Set up the conversion functions */ + if(NULL == (tpath_src_mem = H5T_path_find(dt_src, dt_mem, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between src and mem datatypes") + if(NULL == (tpath_mem_dst = H5T_path_find(dt_mem, dt_dst, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to convert between mem and dst datatypes") + + /* Determine largest datatype size */ + if(0 == (max_dt_size = H5T_get_size(dt_src))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + if(0 == (mem_dt_size = H5T_get_size(dt_mem))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + max_dt_size = MAX(max_dt_size, mem_dt_size); + if(0 == (tmp_dt_size = H5T_get_size(dt_dst))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, FAIL, "unable to determine datatype size") + max_dt_size = MAX(max_dt_size, tmp_dt_size); + + /* Compute the number of elements per chunk */ + nelmts = 1; + for(u = 0; u < (layout_src->u.chunk.ndims - 1); u++) + nelmts *= layout_src->u.chunk.dim[u]; + + /* Create the space and set the initial extent */ + buf_dim = nelmts; + if(NULL == (buf_space = H5S_create_simple((unsigned)1, &buf_dim, NULL))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, FAIL, "can't create simple dataspace") + + /* Atomize */ + if((sid_buf = H5I_register(H5I_DATASPACE, buf_space)) < 0) { + H5S_close(buf_space); + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, FAIL, "unable to register dataspace ID") + } /* end if */ + + /* Set initial buffer sizes */ + buf_size = nelmts * max_dt_size; + reclaim_buf_size = nelmts * mem_dt_size; + + /* Allocate memory for reclaim buf */ + if(NULL == (reclaim_buf = H5MM_malloc(reclaim_buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + } /* end if */ + else { + buf_size = layout_src->u.chunk.size; + reclaim_buf_size = 0; + } /* end else */ + + /* Allocate memory for copying the chunk */ + if(NULL == (buf = H5MM_malloc(buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed for raw data chunk") + /* Initialize the callback structure for the source */ HDmemset(&udata, 0, sizeof udata); udata.common.mesg = layout_src; udata.file_dst = f_dst; udata.addr_dst = layout_dst->u.chunk.addr; + udata.buf = buf; + udata.buf_size = buf_size; + udata.tid_src = tid_src; + udata.tid_mem = tid_mem; + udata.tid_dst = tid_dst; + udata.tpath_src_mem = tpath_src_mem; + udata.tpath_mem_dst = tpath_mem_dst; + udata.reclaim_buf = reclaim_buf; + udata.reclaim_buf_size = reclaim_buf_size; + udata.buf_space = buf_space; + udata.nelmts = nelmts; + udata.pline = pline; /* copy the chunked data by iteration */ if(H5B_iterate(f_src, dxpl_id, H5B_ISTORE, H5D_istore_iter_copy, layout_src->u.chunk.addr, &udata) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, FAIL, "unable to iterate over chunk B-tree") + /* I/O buffer may have been re-allocated */ + buf = udata.buf; + done: + if(sid_buf > 0) + if(H5I_dec_ref(sid_buf) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary dataspace ID") + if(tid_src > 0) + if(H5I_dec_ref(tid_src) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(tid_dst > 0) + if(H5I_dec_ref(tid_dst) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(tid_mem > 0) + if(H5I_dec_ref(tid_mem) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, FAIL, "Can't decrement temporary datatype ID") + if(buf) + H5MM_xfree (buf); + if(reclaim_buf) + H5MM_xfree (reclaim_buf); + if(H5RC_DEC(layout_src->u.chunk.btree_shared) < 0) HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") + if(H5RC_DEC(layout_dst->u.chunk.btree_shared) < 0) HDONE_ERROR(H5E_IO, H5E_CANTFREE, FAIL, "unable to decrement ref-counted page") diff --git a/src/H5Doh.c b/src/H5Doh.c index f9f9249..4da2381 100644 --- a/src/H5Doh.c +++ b/src/H5Doh.c @@ -152,6 +152,10 @@ done: * Programmer: Quincey Koziol * Monday, November 21, 2005 * + * Modifications: Peter Cao + * Tuesday, December 27, 2005 + * Free filter pipeline for copying a dataset + * *------------------------------------------------------------------------- */ static void @@ -168,6 +172,10 @@ H5O_dset_free_copy_file_udata(void *_udata) if(udata->src_dtype) H5T_close(udata->src_dtype); + /* Release copy of dataset's filter pipeline, if it was set */ + if (udata->src_pline) + H5O_free(H5O_PLINE_ID, udata->src_pline); + /* Release space for 'copy file' user data */ H5FL_FREE(H5D_copy_file_ud_t, udata); diff --git a/src/H5Dpkg.h b/src/H5Dpkg.h index 9b73d86..97e7763 100644 --- a/src/H5Dpkg.h +++ b/src/H5Dpkg.h @@ -34,7 +34,7 @@ #include "H5Gprivate.h" /* Groups */ #include "H5Oprivate.h" /* Object headers */ #include "H5Sprivate.h" /* Dataspaces */ -#include "H5Tprivate.h" /* Datatype functions */ +#include "H5Tprivate.h" /* Datatypes */ /**************************/ /* Package Private Macros */ @@ -198,8 +198,6 @@ extern H5D_dxpl_cache_t H5D_def_dxpl_cache; H5_DLL herr_t H5D_alloc_storage (H5F_t *f, hid_t dxpl_id, H5D_t *dset, H5D_time_alloc_t time_alloc, hbool_t update_time, hbool_t full_overwrite); -H5_DLL herr_t H5D_vlen_reclaim(hid_t type_id, H5S_t *space, hid_t plist_id, - void *buf); /* Functions that perform serial I/O operations */ H5_DLL herr_t H5D_select_fscat (H5D_io_info_t *io_info, @@ -247,6 +245,8 @@ H5_DLL ssize_t H5D_compact_writevv(const H5D_io_info_t *io_info, size_t dset_max_nseq, size_t *dset_curr_seq, size_t dset_size_arr[], hsize_t dset_offset_arr[], size_t mem_max_nseq, size_t *mem_curr_seq, size_t mem_size_arr[], hsize_t mem_offset_arr[], const void *buf); +H5_DLL herr_t H5D_compact_copy(H5F_t *f_src, H5O_layout_t *layout_src, + H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *src_dtype, hid_t dxpl_id); /* Functions that operate on indexed storage */ /* forward reference for collective-chunk IO use */ @@ -278,7 +278,7 @@ H5_DLL ssize_t H5D_istore_writevv(const H5D_io_info_t *io_info, H5_DLL haddr_t H5D_istore_get_addr(const H5D_io_info_t *io_info, struct H5D_istore_ud1_t *_udata); H5_DLL herr_t H5D_istore_copy(H5F_t *f_src, H5O_layout_t *layout_src, - H5F_t *f_dst, H5O_layout_t *layout_dst, hid_t dxpl_id); + H5F_t *f_dst, H5O_layout_t *layout_dst, H5T_t *src_dtype, H5O_pline_t *pline, hid_t dxpl_id); /* Functions that operate on external file list (efl) storage */ H5_DLL ssize_t H5D_efl_readvv(const H5D_io_info_t *io_info, diff --git a/src/H5Dprivate.h b/src/H5Dprivate.h index d762abc..2117d30 100644 --- a/src/H5Dprivate.h +++ b/src/H5Dprivate.h @@ -24,6 +24,7 @@ /* Private headers needed by this file */ #include "H5FDprivate.h" /* File drivers */ #include "H5Oprivate.h" /* Object headers */ +#include "H5Sprivate.h" /* Dataspaces */ #include "H5Zprivate.h" /* Data filters */ /**************************/ @@ -237,6 +238,10 @@ H5_DLL herr_t H5D_flush(const H5F_t *f, hid_t dxpl_id, unsigned flags); H5_DLL herr_t H5D_get_dxpl_cache(hid_t dxpl_id, H5D_dxpl_cache_t **cache); H5_DLL herr_t H5D_get_dxpl_cache_real(hid_t dxpl_id, H5D_dxpl_cache_t *cache); +/* Functions that operate on vlen data */ +H5_DLL herr_t H5D_vlen_reclaim(hid_t type_id, H5S_t *space, hid_t plist_id, + void *buf); + /* Functions that operate on contiguous storage */ H5_DLL herr_t H5D_contig_delete(H5F_t *f, hid_t dxpl_id, const H5O_layout_t *layout); @@ -4120,7 +4120,7 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, continuation block. */ for(chunkno = 0; chunkno < oh_src->nchunks; chunkno++) { - size_t chunk_size = oh_src->chunk[chunkno].size; + size_t chunk_size = oh_src->chunk[chunkno].size; /* '0th' chunk is preceded by object header prefix */ if(0 == chunkno) { @@ -4267,7 +4267,9 @@ H5O_copy_header_real(const H5O_loc_t *oloc_src, if(H5SL_insert(map_list, addr_map, &(addr_map->src_addr)) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTINSERT, FAIL, "can't insert object into skip list") - /* "post copy" loop over messages, to fix up any messages which require a complete object header for destination object */ + /* "post copy" loop over messages, to fix up any messages which require a complete + * object header for destination object + */ for(mesgno = 0; mesgno < oh_src->nmesgs; mesgno++) { /* Set up convenience variables */ mesg_src = &(oh_src->mesg[mesgno]); diff --git a/src/H5Oattr.c b/src/H5Oattr.c index 5d05a7c..6ee9c40 100644 --- a/src/H5Oattr.c +++ b/src/H5Oattr.c @@ -19,9 +19,11 @@ #include "H5private.h" /* Generic Functions */ #include "H5Apkg.h" /* Attributes */ +#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Gprivate.h" /* Groups */ +#include "H5Iprivate.h" /* IDs */ #include "H5MMprivate.h" /* Memory management */ #include "H5Opkg.h" /* Object headers */ #include "H5Spkg.h" /* Dataspaces */ @@ -644,6 +646,10 @@ done: * Programmer: Quincey Koziol * November 1, 2005 * + * Modifications: Peter Cao + * December 17, 2005 + * Datatype conversion for variable length datatype + * *------------------------------------------------------------------------- */ static void * @@ -652,7 +658,16 @@ H5O_attr_copy_file(H5F_t UNUSED *file_src, void *native_src, { H5A_t *attr_src = (H5A_t *)native_src; H5A_t *attr_dst = NULL; - void *ret_value; /* Return value */ + + /* for dataype conversion */ + hid_t tid_src = -1; /* Datatype ID for source datatype */ + hid_t tid_dst = -1; /* Datatype ID for destination datatype */ + hid_t tid_mem = -1; /* Datatype ID for memory datatype */ + void *buf = NULL; /* Buffer for copying data */ + void *reclaim_buf = NULL; /* Buffer for reclaiming data */ + hid_t buf_sid = -1; /* ID for buffer dataspace */ + + void *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5O_attr_copy_file) @@ -675,15 +690,22 @@ H5O_attr_copy_file(H5F_t UNUSED *file_src, void *native_src, /* Copy attribute's name */ attr_dst->name = H5MM_strdup(attr_src->name); + HDassert(attr_dst->name); /* Copy attribute's datatype */ /* (Start destination datatype as transient, even if source is named) */ attr_dst->dt = H5T_copy(attr_src->dt, H5T_COPY_ALL); + HDassert(attr_dst->dt); + + /* Set the location of the destination datatype */ + if(H5T_set_loc(attr_dst->dt, file_dst, H5T_LOC_DISK) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "cannot mark datatype on disk") /* Check for named datatype being copied */ if(H5T_committed(attr_src->dt)) { H5O_loc_t *src_oloc; /* Pointer to source datatype's object location */ H5O_loc_t *dst_oloc; /* Pointer to dest. datatype's object location */ + H5O_shared_t sh_mesg; /* Get group entries for source & destination */ src_oloc = H5T_oloc(attr_src->dt); @@ -698,22 +720,156 @@ H5O_attr_copy_file(H5F_t UNUSED *file_src, void *native_src, /* Copy the shared object from source to destination */ if(H5O_copy_header_map(src_oloc, dst_oloc, dxpl_id, map_list) < 0) HGOTO_ERROR(H5E_OHDR, H5E_CANTCOPY, NULL, "unable to copy object") + + /* Reset shared message information */ + HDmemset(&sh_mesg, 0, sizeof(H5O_shared_t)); + + /* Get shared message information for datatype */ + if(H5O_get_share(H5O_DTYPE_ID, file_dst, attr_src->dt, &sh_mesg/*out*/) < 0) + HGOTO_ERROR(H5E_ATTR, H5E_CANTOPENOBJ, NULL, "unable to get shared message") + + /* Compute shared message size for datatype */ + attr_dst->dt_size = H5O_raw_size(H5O_SHARED_ID, file_dst, &sh_mesg); } /* end if */ + else + attr_dst->dt_size = H5O_raw_size(H5O_DTYPE_ID, file_dst, attr_src->dt); + HDassert(attr_dst->dt_size > 0); + attr_dst->ds_size = H5S_raw_size(file_dst, attr_src->ds); + HDassert(attr_dst->ds_size > 0); - /* Copy the guts of the attribute */ + /* Copy the dataspace for the attribute */ attr_dst->ds = H5S_copy(attr_src->ds, FALSE); + HDassert(attr_dst->ds); + + /* Compute the size of the data */ + H5_ASSIGN_OVERFLOW(attr_dst->data_size, H5S_GET_EXTENT_NPOINTS(attr_dst->ds) * H5T_get_size(attr_dst->dt), hsize_t, size_t); + /* Copy (& convert) the data, if necessary */ if(attr_src->data) { - if(NULL == (attr_dst->data = H5FL_BLK_MALLOC(attr_buf, attr_src->data_size))) + if(NULL == (attr_dst->data = H5FL_BLK_MALLOC(attr_buf, attr_dst->data_size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation failed") - HDmemcpy(attr_dst->data, attr_src->data, attr_src->data_size); + /* Check if we need to convert data */ + if(H5T_detect_class(attr_src->dt, H5T_VLEN) > 0) { + H5T_path_t *tpath_src_mem, *tpath_mem_dst; /* Datatype conversion paths */ + H5T_t *dt_mem; /* Memory datatype */ + size_t src_dt_size; /* Source datatype size */ + size_t tmp_dt_size; /* Temp. datatype size */ + size_t max_dt_size; /* Max atatype size */ + H5S_t *buf_space; /* Dataspace describing buffer */ + hsize_t buf_dim; /* Dimension for buffer */ + size_t nelmts; /* Number of elements in buffer */ + size_t buf_size; /* Size of copy buffer */ + + /* Create datatype ID for src datatype */ + if((tid_src = H5I_register(H5I_DATATYPE, attr_src->dt)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, NULL, "unable to register source file datatype") + + /* create a memory copy of the variable-length datatype */ + if(NULL == (dt_mem = H5T_copy(attr_src->dt, H5T_COPY_TRANSIENT))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to copy") + if((tid_mem = H5I_register(H5I_DATATYPE, dt_mem)) < 0) + HGOTO_ERROR (H5E_DATATYPE, H5E_CANTREGISTER, NULL, "unable to register memory datatype") + + /* create variable-length datatype at the destinaton file */ + if((tid_dst = H5I_register(H5I_DATATYPE, attr_dst->dt)) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTREGISTER, NULL, "unable to register destination file datatype") + + /* Set up the conversion functions */ + if(NULL == (tpath_src_mem = H5T_path_find(attr_src->dt, dt_mem, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to convert between src and mem datatypes") + if(NULL == (tpath_mem_dst = H5T_path_find(dt_mem, attr_dst->dt, NULL, NULL, dxpl_id, FALSE))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to convert between mem and dst datatypes") + + /* Determine largest datatype size */ + if(0 == (src_dt_size = H5T_get_size(attr_src->dt))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to determine datatype size") + if(0 == (tmp_dt_size = H5T_get_size(dt_mem))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to determine datatype size") + max_dt_size = MAX(src_dt_size, tmp_dt_size); + if(0 == (tmp_dt_size = H5T_get_size(attr_dst->dt))) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "unable to determine datatype size") + max_dt_size = MAX(max_dt_size, tmp_dt_size); + + /* Set number of whole elements that fit in buffer */ + if(0 == (nelmts = attr_src->data_size / src_dt_size)) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "element size too large") + + /* Set up number of bytes to copy, and initial buffer size */ + buf_size = nelmts * max_dt_size; + + /* Create dataspace for number of elements in buffer */ + buf_dim = nelmts; + + /* Create the space and set the initial extent */ + if(NULL == (buf_space = H5S_create_simple((unsigned)1, &buf_dim, NULL))) + HGOTO_ERROR(H5E_DATASPACE, H5E_CANTCREATE, NULL, "can't create simple dataspace") + + /* Atomize */ + if((buf_sid = H5I_register(H5I_DATASPACE, buf_space)) < 0) { + H5S_close(buf_space); + HGOTO_ERROR(H5E_ATOM, H5E_CANTREGISTER, NULL, "unable to register dataspace ID") + } /* end if */ + + /* Allocate memory for recclaim buf */ + if(NULL == (reclaim_buf = H5FL_BLK_MALLOC(attr_buf, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation NULLed for raw data chunk") + + /* Allocate memory for copying the chunk */ + if(NULL == (buf = H5FL_BLK_MALLOC(attr_buf, buf_size))) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "memory allocation NULLed for raw data chunk") + + HDmemcpy(buf, attr_src->data, attr_src->data_size); + + /* Convert from source file to memory */ + if(H5T_convert(tpath_src_mem, tid_src, tid_mem, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "datatype conversion NULLed") + + HDmemcpy(reclaim_buf, buf, buf_size); + + /* Convert from memory to destination file */ + if(H5T_convert(tpath_mem_dst, tid_mem, tid_dst, nelmts, (size_t)0, (size_t)0, buf, NULL, dxpl_id) < 0) + HGOTO_ERROR(H5E_DATATYPE, H5E_CANTINIT, NULL, "datatype conversion NULLed") + + HDmemcpy(attr_dst->data, buf, attr_dst->data_size); + + if(H5D_vlen_reclaim(tid_mem, buf_space, H5P_DATASET_XFER_DEFAULT, reclaim_buf) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_BADITER, NULL, "unable to reclaim variable-length data") + } /* type conversion */ + else { + HDassert(attr_dst->data_size == attr_src->data_size); + HDmemcpy(attr_dst->data, attr_src->data, attr_src->data_size); + } /* end else */ } /* end if */ + /* Indicate that the fill values aren't to be written out */ + attr_dst->initialized = TRUE; + /* Set return value */ ret_value = attr_dst; done: + if(buf_sid > 0) + if(H5I_dec_ref(buf_sid) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't decrement temporary dataspace ID") + if(tid_src > 0) + /* Don't decrement ID, we want to keep underlying datatype */ + if(H5I_remove(tid_src) == NULL) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't decrement temporary datatype ID") + if(tid_dst > 0) + /* Don't decrement ID, we want to keep underlying datatype */ + if(H5I_remove(tid_dst) == NULL) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't decrement temporary datatype ID") + if(tid_mem > 0) + /* Decrement the memory datatype ID, it's transient */ + if(H5I_dec_ref(tid_mem) < 0) + HDONE_ERROR(H5E_DATASET, H5E_CANTFREE, NULL, "Can't decrement temporary datatype ID") + if(buf) + H5FL_BLK_FREE(attr_buf, buf); + if(reclaim_buf) + H5FL_BLK_FREE(attr_buf, reclaim_buf); + + /* Release destination attribute information on failure */ if(!ret_value) if(attr_dst) (void)H5A_free(attr_dst); diff --git a/src/H5Olayout.c b/src/H5Olayout.c index 3665e13..7e6e4f5 100644 --- a/src/H5Olayout.c +++ b/src/H5Olayout.c @@ -625,9 +625,9 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, H5F_t *file_dst, hid_t dxpl_id, H5SL_t UNUSED *map_list, void *_udata) { H5D_copy_file_ud_t *udata = (H5D_copy_file_ud_t *)_udata; /* Dataset copying user data */ - H5O_layout_t *layout_src = (H5O_layout_t *) mesg_src; - H5O_layout_t *layout_dst = NULL; - void *ret_value; /* Return value */ + H5O_layout_t *layout_src = (H5O_layout_t *) mesg_src; + H5O_layout_t *layout_dst = NULL; + void *ret_value; /* Return value */ FUNC_ENTER_NOAPI_NOINIT(H5O_layout_copy_file) @@ -648,9 +648,15 @@ H5O_layout_copy_file(H5F_t *file_src, void *mesg_src, if(layout_src->u.compact.buf) { if(NULL == (layout_dst->u.compact.buf = H5MM_malloc(layout_src->u.compact.size))) HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, NULL, "unable to allocate memory for compact dataset") -HDassert(!udata->src_dtype); - HDmemcpy(layout_dst->u.compact.buf, layout_src->u.compact.buf, layout_src->u.compact.size); + + /* copy compact raw data */ + if(H5D_compact_copy(file_src, layout_src, file_dst, layout_dst, udata->src_dtype, dxpl_id) < 0) + HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to copy chunked storage") + layout_dst->u.compact.dirty = TRUE; + + /* Freed by copy routine */ + udata->src_dtype = NULL; } break; @@ -667,13 +673,16 @@ HDassert(!udata->src_dtype); case H5D_CHUNKED: if(H5F_addr_defined(layout_src->u.chunk.addr)) { -HDassert(!udata->src_dtype); /* layout is not created in the destination file, undef btree address */ layout_dst->u.chunk.addr = HADDR_UNDEF; /* create chunked layout */ - if(H5D_istore_copy(file_src, layout_src, file_dst, layout_dst, dxpl_id) < 0) + if(H5D_istore_copy(file_src, layout_src, file_dst, layout_dst, + udata->src_dtype, udata->src_pline, dxpl_id) < 0) HGOTO_ERROR(H5E_IO, H5E_CANTINIT, NULL, "unable to copy chunked storage") + + /* Freed by copy routine */ + udata->src_dtype = NULL; } /* if ( H5F_addr_defined(layout_srct->u.chunk.addr)) */ break; diff --git a/src/H5Opkg.h b/src/H5Opkg.h index ed70bb7..0ce644b 100644 --- a/src/H5Opkg.h +++ b/src/H5Opkg.h @@ -112,6 +112,7 @@ struct H5O_t { /* Callback information for copying dataset */ typedef struct { H5T_t *src_dtype; /* Copy of datatype for dataset */ + H5O_pline_t *src_pline; /* Copy of filter pipeline for dataet */ } H5D_copy_file_ud_t; /* Class for types of objects in file */ diff --git a/src/H5Opline.c b/src/H5Opline.c index 9b6f147..d6e9870 100644 --- a/src/H5Opline.c +++ b/src/H5Opline.c @@ -36,6 +36,8 @@ static void *H5O_pline_copy (const void *_mesg, void *_dest, unsigned update_fla static size_t H5O_pline_size (const H5F_t *f, const void *_mesg); static herr_t H5O_pline_reset (void *_mesg); static herr_t H5O_pline_free (void *_mesg); +static herr_t H5O_pline_pre_copy_file(H5F_t *file_src, void *mesg_src, + void *_udata); static herr_t H5O_pline_debug (H5F_t *f, hid_t dxpl_id, const void *_mesg, FILE * stream, int indent, int fwidth); @@ -54,7 +56,7 @@ const H5O_msg_class_t H5O_MSG_PLINE[1] = {{ NULL, /* link method */ NULL, /* get share method */ NULL, /* set share method */ - NULL, /* pre copy native value to file */ + H5O_pline_pre_copy_file, /* pre copy native value to file */ NULL, /* copy native value to file */ NULL, /* post copy native value to file */ H5O_pline_debug /* debug the message */ @@ -424,6 +426,49 @@ H5O_pline_free (void *mesg) /*------------------------------------------------------------------------- + * Function: H5O_pline_pre_copy_file + * + * Purpose: Perform any necessary actions before copying message between + * files + * + * Return: Success: Non-negative + * + * Failure: Negative + * + * Programmer: Peter Cao + * December 27, 2005 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5O_pline_pre_copy_file(H5F_t *file_src, void *mesg_src, void *_udata) +{ + H5O_pline_t *pline_src = (H5O_pline_t *)mesg_src; /* Source datatype */ + H5D_copy_file_ud_t *udata = (H5D_copy_file_ud_t *)_udata; /* Dataset copying user data */ + herr_t ret_value = SUCCEED; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5O_pline_pre_copy_file) + + /* check args */ + HDassert(file_src); + HDassert(pline_src); + + /* If the user data is non-NULL, assume we are copying a dataset + * and make a copy of the filter pipeline for later in + * the object copying process. + */ + if(udata) { + if(NULL == (udata->src_pline = H5O_copy(H5O_PLINE_ID, pline_src, udata->src_pline))) + HGOTO_ERROR(H5E_PLINE, H5E_CANTINIT, FAIL, "unable to copy") + } /* end if */ + +done: + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5O_pline_pre_copy_file() */ + + + +/*------------------------------------------------------------------------- * Function: H5O_pline_debug * * Purpose: Prints debugging information for filter pipeline message MESG @@ -17,6 +17,7 @@ #include "H5private.h" /* Generic Functions */ +#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5Fprivate.h" /* File access */ #include "H5Gprivate.h" /* Groups */ diff --git a/src/H5Sprivate.h b/src/H5Sprivate.h index bf9ff84..9ea37bd 100644 --- a/src/H5Sprivate.h +++ b/src/H5Sprivate.h @@ -18,11 +18,14 @@ #ifndef _H5Sprivate_H #define _H5Sprivate_H +/* Include package's public header */ #include "H5Spublic.h" +/* Public headers needed by this file */ +#include "H5Dpublic.h" /* Datasets */ + /* Private headers needed by this file */ #include "H5private.h" /* Generic Functions */ -#include "H5Dprivate.h" /* Dataset functions */ #include "H5Fprivate.h" /* Files */ #include "H5Gprivate.h" /* Groups */ #include "H5Oprivate.h" /* Object headers */ diff --git a/src/H5Sselect.c b/src/H5Sselect.c index f7a58d3..c351a40 100644 --- a/src/H5Sselect.c +++ b/src/H5Sselect.c @@ -22,6 +22,7 @@ #include "H5private.h" /* Generic Functions */ +#include "H5Dprivate.h" /* Datasets */ #include "H5Eprivate.h" /* Error handling */ #include "H5FLprivate.h" /* Free Lists */ #include "H5Iprivate.h" /* IDs */ diff --git a/src/H5Tvlen.c b/src/H5Tvlen.c index 6ae20a7..29e4ec8 100644 --- a/src/H5Tvlen.c +++ b/src/H5Tvlen.c @@ -215,7 +215,7 @@ H5T_vlen_set_loc(const H5T_t *dt, H5F_t *f, H5T_loc_t loc) assert(loc>H5T_LOC_BADLOC && loc<H5T_LOC_MAXLOC); /* Only change the location if it's different */ - if(loc!=dt->shared->u.vlen.loc) { + if(loc != dt->shared->u.vlen.loc || f != dt->shared->u.vlen.f) { /* Indicate that the location changed */ ret_value=TRUE; |