From 6017d0c70fda2a36cf963193e979f0548cb0c954 Mon Sep 17 00:00:00 2001 From: Raymond Lu Date: Tue, 19 Jun 2007 15:05:24 -0500 Subject: [svn-r13884] The second step of optimization for compound data for the Chicago company. The I/O is optimized when the source and destination members are a subset of each other one way or another, and the order is the same, and no conversion is needed. For example: struct source { struct destination { TYPE1 A; --> TYPE1 A; TYPE2 B; --> TYPE2 B; TYPE3 C; --> TYPE3 C; }; TYPE4 D; TYPE5 E; }; or struct destination { struct source { TYPE1 A; --> TYPE1 A; TYPE2 B; --> TYPE2 B; TYPE3 C; --> TYPE3 C; }; TYPE4 D; TYPE5 E; }; The optimization is simply moving data from the source to the appropriate places in the buffer and bypass the reading of the background data and data conversion. Tested on smirom, liberty, sol, and copper. --- src/H5Dio.c | 363 +++++++++++++++++--- src/H5T.c | 45 +++ src/H5Tconv.c | 158 +++++++-- src/H5Tpkg.h | 3 + src/H5Tprivate.h | 12 + test/cmpd_dset.c | 1003 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 6 files changed, 1488 insertions(+), 96 deletions(-) diff --git a/src/H5Dio.c b/src/H5Dio.c index b494596..9b0102f 100644 --- a/src/H5Dio.c +++ b/src/H5Dio.c @@ -77,6 +77,12 @@ static herr_t H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, const H5T_t *mem_type, const H5S_t *mem_space, const H5S_t *file_space, H5T_path_t *tpath, hid_t src_id, hid_t dst_id, const void *buf); +static herr_t H5D_compound_opt_read(hsize_t nelmts, const H5S_t *mem_space, + H5S_sel_iter_t *iter, const H5D_dxpl_cache_t *dxpl_cache, + hid_t src_id, hid_t dst_id, H5T_subset_t subset, void *data_buf, + void *user_buf/*out*/); +static herr_t H5D_compound_opt_write(hsize_t nelmts, hid_t src_id, hid_t dst_id, + void *data_buf); #ifdef H5_HAVE_PARALLEL static herr_t H5D_ioinfo_term(H5D_io_info_t *io_info); @@ -116,6 +122,13 @@ H5FL_BLK_DEFINE(type_conv); /* Declare a free list to manage the H5D_chunk_info_t struct */ H5FL_DEFINE_STATIC(H5D_chunk_info_t); +/* Declare a free list to manage sequences of size_t */ +H5FL_SEQ_DEFINE_STATIC(size_t); + +/* Declare a free list to manage sequences of hsize_t */ +H5FL_SEQ_DEFINE_STATIC(hsize_t); + + /*-------------------------------------------------------------------------- NAME @@ -1167,6 +1180,23 @@ H5D_contig_read(H5D_io_info_t *io_info, hsize_t nelmts, if (n!=smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed") + /* If the source and destination are compound types and subset of each other + * and no conversion is needed, copy the data directly into user's buffer and + * bypass the rest of steps. This optimization is for Chicago company */ + if(H5T_SUBSET_SRC==H5T_path_compound_subset(tpath)) { + if(H5D_compound_opt_read(smine_nelmts, mem_space, &mem_iter, dxpl_cache, + src_id, dst_id, H5T_SUBSET_SRC, tconv_buf, buf /*out*/)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + + continue; + } else if(H5T_SUBSET_DST==H5T_path_compound_subset(tpath)) { + if(H5D_compound_opt_read(smine_nelmts, mem_space, &mem_iter, dxpl_cache, + src_id, dst_id, H5T_SUBSET_DST, tconv_buf, buf /*out*/)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + + continue; + } + if (H5T_BKG_YES==need_bkg) { #ifdef H5S_DEBUG H5_timer_begin(&timer); @@ -1421,33 +1451,44 @@ H5D_contig_write(H5D_io_info_t *io_info, hsize_t nelmts, if (n!=smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "mem gather failed") - if (H5T_BKG_YES==need_bkg) { + /* If the source and destination are compound types and the destination is + * is a subset of the source and no conversion is needed, copy the data + * directly into user's buffer and bypass the rest of steps. If the source + * is a subset of the destination, the optimization is done in conversion + * function H5T_conv_struct_opt to protect the background data. This + * optimization is for Chicago company */ + if(H5T_SUBSET_DST==H5T_path_compound_subset(tpath)) { + if(H5D_compound_opt_write(smine_nelmts, src_id, dst_id, tconv_buf)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + } else { + if (H5T_BKG_YES==need_bkg) { #ifdef H5S_DEBUG - H5_timer_begin(&timer); + H5_timer_begin(&timer); #endif - n = H5D_select_fgath(io_info, - file_space, &bkg_iter, smine_nelmts, - bkg_buf/*out*/); + n = H5D_select_fgath(io_info, + file_space, &bkg_iter, smine_nelmts, + bkg_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(io_info->stats->stats[0].bkg_timer), &timer); - io_info->stats->stats[0].bkg_nbytes += n * dst_type_size; - io_info->stats->stats[0].bkg_ncalls++; + H5_timer_end(&(io_info->stats->stats[0].bkg_timer), &timer); + io_info->stats->stats[0].bkg_nbytes += n * dst_type_size; + io_info->stats->stats[0].bkg_ncalls++; #endif - if (n!=smine_nelmts) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed") - } /* end if */ + if (n!=smine_nelmts) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed") + } /* end if */ - /* - * Perform datatype conversion. - */ - if(H5T_convert(tpath, src_id, dst_id, smine_nelmts, (size_t)0, (size_t)0, tconv_buf, bkg_buf, io_info->dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + /* + * Perform datatype conversion. + */ + if(H5T_convert(tpath, src_id, dst_id, smine_nelmts, (size_t)0, (size_t)0, tconv_buf, bkg_buf, io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") - /* Do the data transform after the type conversion (since we're using dataset->shared->type). */ - if(!H5Z_xform_noop(dxpl_cache->data_xform_prop)) - if( H5Z_xform_eval(dxpl_cache->data_xform_prop, tconv_buf, smine_nelmts, dataset->shared->type) < 0) + /* Do the data transform after the type conversion (since we're using dataset->shared->type). */ + if(!H5Z_xform_noop(dxpl_cache->data_xform_prop)) + if( H5Z_xform_eval(dxpl_cache->data_xform_prop, tconv_buf, smine_nelmts, dataset->shared->type) < 0) HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Error performing data transform") + } /* * Scatter the data out to the file. @@ -1455,9 +1496,8 @@ H5D_contig_write(H5D_io_info_t *io_info, hsize_t nelmts, #ifdef H5S_DEBUG H5_timer_begin(&timer); #endif - status = H5D_select_fscat(io_info, - file_space, &file_iter, smine_nelmts, - tconv_buf); + status = H5D_select_fscat(io_info, file_space, &file_iter, smine_nelmts, + tconv_buf); #ifdef H5S_DEBUG H5_timer_end(&(io_info->stats->stats[0].scat_timer), &timer); io_info->stats->stats[0].scat_nbytes += smine_nelmts * dst_type_size; @@ -1723,6 +1763,23 @@ H5D_chunk_read(H5D_io_info_t *io_info, hsize_t nelmts, if(n != smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed") + /* If the source and destination are compound types and subset of each other + * and no conversion is needed, copy the data directly into user's buffer and + * bypass the rest of steps. This optimization is for Chicago company */ + if(H5T_SUBSET_SRC==H5T_path_compound_subset(tpath)) { + if(H5D_compound_opt_read(smine_nelmts, chunk_info->mspace, &mem_iter, dxpl_cache, + src_id, dst_id, H5T_SUBSET_SRC, tconv_buf, buf /*out*/)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + + continue; + } else if(H5T_SUBSET_DST==H5T_path_compound_subset(tpath)) { + if(H5D_compound_opt_read(smine_nelmts, chunk_info->mspace, &mem_iter, dxpl_cache, + src_id, dst_id, H5T_SUBSET_DST, tconv_buf, buf /*out*/)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + + continue; + } + if(H5T_BKG_YES == need_bkg) { #ifdef H5S_DEBUG H5_timer_begin(&timer); @@ -2041,33 +2098,43 @@ H5D_chunk_write(H5D_io_info_t *io_info, hsize_t nelmts, if (n!=smine_nelmts) HGOTO_ERROR(H5E_IO, H5E_READERROR, FAIL, "file gather failed") - if (H5T_BKG_YES==need_bkg) { + /* If the source and destination are compound types and the destination is + * is a subset of the source and no conversion is needed, copy the data + * directly into user's buffer and bypass the rest of steps. If the source + * is a subset of the destination, the optimization is done in conversion + * function H5T_conv_struct_opt to protect the background data. This + * optimization is for Chicago company */ + if(H5T_SUBSET_DST==H5T_path_compound_subset(tpath)) { + if(H5D_compound_opt_write(smine_nelmts, src_id, dst_id, tconv_buf)<0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + } else { + if (H5T_BKG_YES==need_bkg) { #ifdef H5S_DEBUG - H5_timer_begin(&timer); + H5_timer_begin(&timer); #endif - n = H5D_select_fgath(io_info, - chunk_info->fspace, &bkg_iter, smine_nelmts, - bkg_buf/*out*/); + n = H5D_select_fgath(io_info, chunk_info->fspace, &bkg_iter, smine_nelmts, + bkg_buf/*out*/); #ifdef H5S_DEBUG - H5_timer_end(&(io_info->stats->stats[0].bkg_timer), &timer); - io_info->stats->stats[0].bkg_nbytes += n * dst_type_size; - io_info->stats->stats[0].bkg_ncalls++; + H5_timer_end(&(io_info->stats->stats[0].bkg_timer), &timer); + io_info->stats->stats[0].bkg_nbytes += n * dst_type_size; + io_info->stats->stats[0].bkg_ncalls++; #endif - if (n!=smine_nelmts) - HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed") - } /* end if */ + if (n!=smine_nelmts) + HGOTO_ERROR(H5E_IO, H5E_WRITEERROR, FAIL, "file gather failed") + } /* end if */ - /* - * Perform datatype conversion. - */ - if(H5T_convert(tpath, src_id, dst_id, smine_nelmts, (size_t)0, (size_t)0, tconv_buf, bkg_buf, io_info->dxpl_id) < 0) - HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") + /* + * Perform datatype conversion. + */ + if(H5T_convert(tpath, src_id, dst_id, smine_nelmts, (size_t)0, (size_t)0, tconv_buf, bkg_buf, io_info->dxpl_id) < 0) + HGOTO_ERROR(H5E_DATASET, H5E_CANTINIT, FAIL, "datatype conversion failed") - /* Do the data transform after the type conversion (since we're using dataset->shared->type) */ - if(!H5Z_xform_noop(dxpl_cache->data_xform_prop)) - if( H5Z_xform_eval(dxpl_cache->data_xform_prop, tconv_buf, smine_nelmts, dataset->shared->type) < 0) - HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Error performing data transform") + /* Do the data transform after the type conversion (since we're using dataset->shared->type) */ + if(!H5Z_xform_noop(dxpl_cache->data_xform_prop)) + if( H5Z_xform_eval(dxpl_cache->data_xform_prop, tconv_buf, smine_nelmts, dataset->shared->type) < 0) + HGOTO_ERROR(H5E_ARGS, H5E_BADVALUE, FAIL, "Error performing data transform") + } /* * Scatter the data out to the file. @@ -2138,6 +2205,218 @@ done: /*------------------------------------------------------------------------- + * Function: H5D_compound_opt_read + * + * Purpose: A shortcut optimization for the Chicago company for + * a special optimization case when the source and + * destination members are a subset of each other, and + * the order is the same, and no conversion is needed. + * For example: + * struct source { struct destination { + * TYPE1 A; --> TYPE1 A; + * TYPE2 B; --> TYPE2 B; + * TYPE3 C; --> TYPE3 C; + * }; TYPE4 D; + * TYPE5 E; + * }; + * or + * struct destination { struct source { + * TYPE1 A; --> TYPE1 A; + * TYPE2 B; --> TYPE2 B; + * TYPE3 C; --> TYPE3 C; + * }; TYPE4 D; + * TYPE5 E; + * }; + * The optimization is simply moving data to the appropriate + * places in the buffer. + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 11 June 2007 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_compound_opt_read(hsize_t nelmts, const H5S_t *space, + H5S_sel_iter_t *iter, const H5D_dxpl_cache_t *dxpl_cache, + hid_t src_id, hid_t dst_id, H5T_subset_t subset, + void *data_buf, void *user_buf/*out*/) +{ + uint8_t *dbuf = (uint8_t *)data_buf; /*cast for pointer arithmetic */ + uint8_t *ubuf = (uint8_t *)user_buf; /*cast for pointer arithmetic */ + uint8_t *xdbuf; + uint8_t *xubuf; + + hsize_t _off[H5D_IO_VECTOR_SIZE]; /* Array to store sequence offsets */ + hsize_t *off=NULL; /* Pointer to sequence offsets */ + size_t _len[H5D_IO_VECTOR_SIZE]; /* Array to store sequence lengths */ + size_t *len=NULL; /* Pointer to sequence lengths */ + size_t nseq; /* Number of sequences generated */ + size_t curr_off; /* offset of bytes left to process in sequence */ + size_t curr_seq; /* Current sequence being processed */ + size_t curr_len; /* Length of bytes left to process in sequence */ + size_t curr_nelmts; /* number of elements to process in sequence */ + size_t i; + + H5T_t *src, *dst; + size_t src_stride, dst_stride, type_size; + size_t elmtno; /*element counter */ + + herr_t ret_value = SUCCEED; /*return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_compound_opt_read) + + /* Check args */ + assert (data_buf); + assert (user_buf); + assert (space); + assert (iter); + assert (nelmts>0); + + /* Allocate the vector I/O arrays */ + if(dxpl_cache->vec_size != H5D_IO_VECTOR_SIZE) { + if((len = H5FL_SEQ_MALLOC(size_t,dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O length vector array"); + if((off = H5FL_SEQ_MALLOC(hsize_t,dxpl_cache->vec_size))==NULL) + HGOTO_ERROR(H5E_RESOURCE, H5E_NOSPACE, FAIL, "can't allocate I/O offset vector array"); + } /* end if */ + else { + len=_len; + off=_off; + } /* end else */ + + if (NULL == (src = H5I_object(src_id)) || NULL == (dst = H5I_object(dst_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data type"); + + src_stride = H5T_get_size(src); + dst_stride = H5T_get_size(dst); + + if(H5T_SUBSET_SRC == subset) + type_size = src_stride; + else if(H5T_SUBSET_DST == subset) + type_size = dst_stride; + + xdbuf = dbuf; + + /* Loop until all elements are written */ + while(nelmts>0) { + /* Get list of sequences for selection to write */ + if(H5S_SELECT_GET_SEQ_LIST(space,0,iter,dxpl_cache->vec_size,nelmts,&nseq,&elmtno,off,len)<0) + HGOTO_ERROR (H5E_INTERNAL, H5E_UNSUPPORTED, 0, "sequence length generation failed"); + + /* Loop, while sequences left to process */ + for(curr_seq=0; curr_seqvec_size != H5D_IO_VECTOR_SIZE) { + if(len!=NULL) + H5FL_SEQ_FREE(size_t,len); + if(off!=NULL) + H5FL_SEQ_FREE(hsize_t,off); + } /* end if */ + + FUNC_LEAVE_NOAPI(ret_value) +} /* end H5D_compound_opt_read() */ + + +/*------------------------------------------------------------------------- + * Function: H5D_compound_opt_write + * + * Purpose: A shortcut optimization for the Chicago company for + * a special optimization case when the source and + * destination members are a subset of each other, and + * the order is the same, and no conversion is needed. + * For example: + * struct source { struct destination { + * TYPE1 A; --> TYPE1 A; + * TYPE2 B; --> TYPE2 B; + * TYPE3 C; --> TYPE3 C; + * }; TYPE4 D; + * TYPE5 E; + * }; + * or + * struct destination { struct source { + * TYPE1 A; --> TYPE1 A; + * TYPE2 B; --> TYPE2 B; + * TYPE3 C; --> TYPE3 C; + * }; TYPE4 D; + * TYPE5 E; + * }; + * The optimization is simply moving data to the appropriate + * places in the buffer. + * + * + * Return: Non-negative on success/Negative on failure + * + * Programmer: Raymond Lu + * 11 June 2007 + * + *------------------------------------------------------------------------- + */ +static herr_t +H5D_compound_opt_write(hsize_t nelmts, hid_t src_id, hid_t dst_id, void *data_buf) +{ + uint8_t *dbuf = (uint8_t *)data_buf; /*cast for pointer arithmetic */ + uint8_t *xsbuf, *xdbuf; + size_t i; + H5T_t *src, *dst; + size_t src_stride, dst_stride; + herr_t ret_value = SUCCEED; /*return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5D_compound_opt_write) + + /* Check args */ + assert (data_buf); + assert (nelmts>0); + + if (NULL == (src = H5I_object(src_id)) || NULL == (dst = H5I_object(dst_id))) + HGOTO_ERROR(H5E_ARGS, H5E_BADTYPE, FAIL, "not a data type"); + + src_stride = H5T_get_size(src); + dst_stride = H5T_get_size(dst); + + xsbuf = dbuf; + xdbuf = dbuf; + + /* Loop until all elements are written */ + for(i=0; iare_compounds = TRUE; + /* Set return value */ ret_value = path; @@ -4496,6 +4501,46 @@ H5T_path_noop(const H5T_path_t *p) /*------------------------------------------------------------------------- + * Function: H5T_path_compound_subset + * + * Purpose: Checks if the source and destination types are both compound. + * Tells whether whether the source members are a subset of + * destination, and the order is the same, and no conversion + * is needed. For example: + * struct source { struct destination { + * TYPE1 A; --> TYPE1 A; + * TYPE2 B; --> TYPE2 B; + * TYPE3 C; --> TYPE3 C; + * }; TYPE4 D; + * TYPE5 E; + * }; + * + * Return: One of the values of H5T_subset_t (can't fail). + * + * Programmer: Raymond Lu + * 8 June 2007 + * + * Modifications: + * + *------------------------------------------------------------------------- + */ +H5T_subset_t +H5T_path_compound_subset(const H5T_path_t *p) +{ + H5T_subset_t ret_value = FALSE; + + FUNC_ENTER_NOAPI_NOINIT_NOFUNC(H5T_path_compound_subset); + + assert(p); + + if(p->are_compounds) + ret_value = H5T_conv_struct_subset(&(p->cdata)); + + FUNC_LEAVE_NOAPI(ret_value); +} /* end H5T_path_compound_subset */ + + +/*------------------------------------------------------------------------- * Function: H5T_path_bkg * * Purpose: Get the "background" flag for the conversion path. diff --git a/src/H5Tconv.c b/src/H5Tconv.c index a0ca3da..9dd7b30 100644 --- a/src/H5Tconv.c +++ b/src/H5Tconv.c @@ -35,7 +35,7 @@ typedef struct H5T_conv_struct_t { hid_t *src_memb_id; /*source member type ID's */ hid_t *dst_memb_id; /*destination member type ID's */ H5T_path_t **memb_path; /*conversion path for each member */ - hbool_t smembs_subset; /*are source members a subset and in the top of dest? */ + H5T_subset_t smembs_subset; /*are source and dest members a subset of each other? */ } H5T_conv_struct_t; /* Conversion data for H5T_conv_enum() */ @@ -1704,8 +1704,8 @@ done: * Modifications: * Raymond Lu, 3 May 2007 * Added the detection for a special optimization case when the - * source members are a subset of destination, and the order is - * the same, and no conversion is needed. For example: + * source and destination members are a subset of each other, and + * the order is the same, and no conversion is needed. For example: * struct source { struct destination { * TYPE1 A; --> TYPE1 A; * TYPE2 B; --> TYPE2 B; @@ -1713,6 +1713,14 @@ done: * }; TYPE4 D; * TYPE5 E; * }; + * or + * struct destination { struct source { + * TYPE1 A; --> TYPE1 A; + * TYPE2 B; --> TYPE2 B; + * TYPE3 C; --> TYPE3 C; + * }; TYPE4 D; + * TYPE5 E; + * }; * The optimization is simply moving data to the appropriate * places in the buffer. * @@ -1722,7 +1730,8 @@ static herr_t H5T_conv_struct_init (H5T_t *src, H5T_t *dst, H5T_cdata_t *cdata, hid_t dxpl_id) { H5T_conv_struct_t *priv = (H5T_conv_struct_t*)(cdata->priv); - int *src2dst = NULL; + int *src2dst = NULL; + unsigned src_nmembs, dst_nmembs; unsigned i, j; H5T_t *type = NULL; hid_t tid; @@ -1730,23 +1739,23 @@ H5T_conv_struct_init (H5T_t *src, H5T_t *dst, H5T_cdata_t *cdata, hid_t dxpl_id) FUNC_ENTER_NOAPI_NOINIT(H5T_conv_struct_init); + src_nmembs = src->shared->u.compnd.nmembs; + dst_nmembs = dst->shared->u.compnd.nmembs; + if (!priv) { /* * Allocate private data structure and arrays. */ if (NULL==(priv=cdata->priv=H5MM_calloc(sizeof(H5T_conv_struct_t))) || - NULL==(priv->src2dst=H5MM_malloc(src->shared->u.compnd.nmembs * - sizeof(int))) || - NULL==(priv->src_memb_id=H5MM_malloc(src->shared->u.compnd.nmembs * - sizeof(hid_t))) || - NULL==(priv->dst_memb_id=H5MM_malloc(dst->shared->u.compnd.nmembs * - sizeof(hid_t)))) + NULL==(priv->src2dst=H5MM_malloc(src_nmembs * sizeof(int))) || + NULL==(priv->src_memb_id=H5MM_malloc(src_nmembs * sizeof(hid_t))) || + NULL==(priv->dst_memb_id=H5MM_malloc(dst_nmembs * sizeof(hid_t)))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed"); src2dst = priv->src2dst; - /* The flag of special optimization to indicate if source members are a subset - * and in the top of the destination. Initialize it to TRUE */ - priv->smembs_subset = TRUE; + /* The flag of special optimization to indicate if source members and destination + * members are a subset of each other. Initialize it to FALSE */ + priv->smembs_subset = H5T_SUBSET_FALSE; /* * Insure that members are sorted. @@ -1761,9 +1770,9 @@ H5T_conv_struct_init (H5T_t *src, H5T_t *dst, H5T_cdata_t *cdata, hid_t dxpl_id) * source and destination member data type so we can look up the * member data type conversion functions later. */ - for (i=0; ishared->u.compnd.nmembs; i++) { + for (i=0; ishared->u.compnd.nmembs; j++) { + for (j=0; jshared->u.compnd.memb[i].name, dst->shared->u.compnd.memb[j].name)) { src2dst[i] = j; @@ -1782,11 +1791,6 @@ H5T_conv_struct_init (H5T_t *src, H5T_t *dst, H5T_cdata_t *cdata, hid_t dxpl_id) assert (tid>=0); priv->dst_memb_id[src2dst[i]] = tid; } - - /* If any of source member doesn't have counterpart in the same order, - * don't do the special optimization. */ - if(src2dst[i] != i || (src->shared->u.compnd.memb[i].offset != dst->shared->u.compnd.memb[i].offset)) - priv->smembs_subset = FALSE; } } else { @@ -1806,7 +1810,7 @@ H5T_conv_struct_init (H5T_t *src, H5T_t *dst, H5T_cdata_t *cdata, hid_t dxpl_id) sizeof(H5T_path_t*)))) HGOTO_ERROR (H5E_RESOURCE, H5E_NOSPACE, FAIL, "memory allocation failed"); - for (i=0; ishared->u.compnd.nmembs; i++) { + for (i=0; i=0) { H5T_path_t *tpath = H5T_path_find(src->shared->u.compnd.memb[i].type, dst->shared->u.compnd.memb[src2dst[i]].type, NULL, NULL, dxpl_id, FALSE); @@ -1819,18 +1823,39 @@ H5T_conv_struct_init (H5T_t *src, H5T_t *dst, H5T_cdata_t *cdata, hid_t dxpl_id) cdata->priv = priv = H5MM_xfree (priv); HGOTO_ERROR (H5E_DATATYPE, H5E_UNSUPPORTED, FAIL, "unable to convert member data type"); } - - /* If any of source member needs conversion, don't do the special optimization. */ - if(priv->smembs_subset && ((priv->memb_path[i])->is_noop == FALSE)) - priv->smembs_subset = FALSE; - } } /* Check if we need a background buffer */ - if (H5T_detect_class(src,H5T_COMPOUND)==TRUE || H5T_detect_class(dst,H5T_COMPOUND)==TRUE) + if (H5T_detect_class(src,H5T_COMPOUND)==TRUE || H5T_detect_class(dst,H5T_COMPOUND)==TRUE) { cdata->need_bkg = H5T_BKG_YES; + if(src_nmembs < dst_nmembs) { + priv->smembs_subset = H5T_SUBSET_SRC; + for (i=0; ishared->u.compnd.memb[i].offset != + dst->shared->u.compnd.memb[i].offset) || (priv->memb_path[i])->is_noop == + FALSE) + priv->smembs_subset = H5T_SUBSET_FALSE; + } + } else if(dst_nmembs < src_nmembs) { + priv->smembs_subset = H5T_SUBSET_DST; + for (i=0; ishared->u.compnd.memb[i].offset != + dst->shared->u.compnd.memb[i].offset) || (priv->memb_path[i])->is_noop == + FALSE) + priv->smembs_subset = H5T_SUBSET_FALSE; + } + } else /* If the numbers of source and dest members are equal and no conversion is needed, + * the case should have been handled as noop earlier in H5Dio.c. */ + ; + + } + cdata->recalc = FALSE; done: @@ -1839,6 +1864,49 @@ done: /*------------------------------------------------------------------------- + * Function: H5T_conv_struct_subset + * + * Purpose: A quick way to return a field in a struct private in this + * file. The flag SMEMBS_SUBSET indicates whether the source + * members are a subset of destination or the destination + * members are a subset of the source, and the order is the + * same, and no conversion is needed. For example: + * struct source { struct destination { + * TYPE1 A; --> TYPE1 A; + * TYPE2 B; --> TYPE2 B; + * TYPE3 C; --> TYPE3 C; + * }; TYPE4 D; + * TYPE5 E; + * }; + * + * Return: One of the value from H5T_subset_t. + * + * Programmer: Raymond Lu + * 8 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +H5T_subset_t +H5T_conv_struct_subset(const H5T_cdata_t *cdata) +{ + H5T_conv_struct_t *priv; + H5T_subset_t ret_value=FALSE; /* Return value */ + + FUNC_ENTER_NOAPI_NOINIT(H5T_conv_struct_subset); + + assert(cdata); + assert(cdata->priv); + + priv = (H5T_conv_struct_t*)(cdata->priv); + ret_value = priv->smembs_subset; + +done: + FUNC_LEAVE_NOAPI(ret_value); +} + + +/*------------------------------------------------------------------------- * Function: H5T_conv_struct * * Purpose: Converts between compound data types. This is a soft @@ -2109,9 +2177,9 @@ done: * datatype. * * Raymond Lu, 3 May 2007 - * Optimize a special case when the source members are a subset of - * destination, and the order is the same, and no conversion is needed. - * For example: + * Optimize a special case when the source and destination members + * are a subset of each other, and the order is the same, and no + * conversion is needed. For example: * struct source { struct destination { * TYPE1 A; --> TYPE1 A; * TYPE2 B; --> TYPE2 B; @@ -2257,7 +2325,7 @@ H5T_conv_struct_opt(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, buf_stride = src->shared->size; } - if(priv->smembs_subset == TRUE) { + if(priv->smembs_subset == H5T_SUBSET_SRC) { /* If the optimization flag is set to indicate source members are a subset and * in the top of the destination, simply copy the source members to background buffer. */ xbuf = buf; @@ -2286,6 +2354,34 @@ H5T_conv_struct_opt(hid_t src_id, hid_t dst_id, H5T_cdata_t *cdata, xbkg += bkg_stride; } } + } else if(priv->smembs_subset == H5T_SUBSET_DST) { + /* If the optimization flag is set to indicate destination members are a subset + * and in the top of the source, simply copy the source members to background + * buffer. */ + xbuf = buf; + xbkg = bkg; + + if(src->shared->size <= dst->shared->size) { + /* This is to deal with a very special situation when the fields and their + * offset for both source and destination are identical but the datatype + * sizes of source and destination are different. + */ + for (elmtno=0; elmtnoshared->size); + + /* Update pointers */ + xbuf += buf_stride; + xbkg += bkg_stride; + } + } else { + for (elmtno=0; elmtnoshared->size); + + /* Update pointers */ + xbuf += buf_stride; + xbkg += bkg_stride; + } + } } else { /* * For each member where the destination is not larger than the diff --git a/src/H5Tpkg.h b/src/H5Tpkg.h index c9fba39..fd3bc39 100644 --- a/src/H5Tpkg.h +++ b/src/H5Tpkg.h @@ -192,6 +192,7 @@ struct H5T_path_t { H5T_conv_t func; /*data conversion function */ hbool_t is_hard; /*is it a hard function? */ hbool_t is_noop; /*is it the noop conversion? */ + hbool_t are_compounds; /*are source and dest both compounds?*/ H5T_stats_t stats; /*statistics for the conversion */ H5T_cdata_t cdata; /*data for this function */ }; @@ -1338,6 +1339,8 @@ H5_DLL H5T_t *H5T_get_member_type(const H5T_t *dt, unsigned membno); H5_DLL size_t H5T_get_member_offset(const H5T_t *dt, unsigned membno); H5_DLL size_t H5T_get_member_size(const H5T_t *dt, unsigned membno); H5_DLL htri_t H5T_is_packed(const H5T_t *dt); +H5_DLL H5T_subset_t H5T_conv_struct_subset(const H5T_cdata_t *cdata); +H5_DLL H5T_subset_t H5T_path_compound_subset(const H5T_path_t *p); #endif /* _H5Tpkg_H */ diff --git a/src/H5Tprivate.h b/src/H5Tprivate.h index a566a0a..0ac6047 100644 --- a/src/H5Tprivate.h +++ b/src/H5Tprivate.h @@ -64,6 +64,18 @@ typedef struct H5T_conv_cb_t { void* user_data; } H5T_conv_cb_t; +/* Values for the optimization of compound data reading and writing. They indicate + * whether the fields of the source and destination are subset of each other and + * there is no conversion needed. It's for the Chicago company. + */ +typedef enum { + H5T_SUBSET_BADVALUE = -1, /* Invalid value */ + H5T_SUBSET_FALSE = 0, /* Source and destination aren't subset of each other */ + H5T_SUBSET_SRC, /* Source is the subset of dest and no conversion is needed */ + H5T_SUBSET_DST, /* Dest is the subset of source and no conversion is needed */ + H5T_SUBSET_CAP /* Must be the last value */ +} H5T_subset_t; + /* Forward declarations for prototype arguments */ struct H5O_t; diff --git a/test/cmpd_dset.c b/test/cmpd_dset.c index 08f1fd1..0902cd3 100644 --- a/test/cmpd_dset.c +++ b/test/cmpd_dset.c @@ -28,6 +28,16 @@ const char *FILENAME[] = { "cmpd_dset", + "src_subset", + "dst_subset", + NULL +}; + +const char *DSET_NAME[] = { + "contig_src_subset", + "chunk_src_subset", + "contig_dst_subset", + "chunk_dst_subset", NULL }; @@ -85,6 +95,28 @@ typedef struct s6_t { unsigned int post; } s6_t; +/* Structures for testing the optimization for the Chicago company. */ +typedef struct { + int a, b, c[8], d, e; + float f, g, h[16], i, j; + double k, l, m, n; +} stype1; +typedef struct { + int a, b, c[8], d, e; + float f, g, h[16], i, j; + double k, l, m, n; + long o, p, q; +} stype2; +typedef struct { + int a, b, c[8], d, e; +} stype3; +typedef struct { + int a, b, c[8], d, e; + float f, g, h[16], i, j; + double k, l, m, n; + long o, p, q; + long_long r, s, t; +} stype4; #if 1 # define NX 100u @@ -96,7 +128,7 @@ typedef struct s6_t { /*------------------------------------------------------------------------- - * Function: main + * Function: test_compound * * Purpose: Creates a simple dataset of a compound type and then reads * it back. The dataset is read back in various ways to @@ -113,10 +145,13 @@ typedef struct s6_t { * Robb Matzke, 1999-06-23 * If the command line switch `--noopt' is present then the fast * compound datatype conversion is turned off. + * + * Raymond Lu, 15 June 2007 + * Moved this part of code from MAIN to TEST_COMPOUND function. *------------------------------------------------------------------------- */ -int -main (int argc, char *argv[]) +static int +test_compound (char *filename, hid_t fapl) { /* First dataset */ static s1_t s1[NX*NY]; @@ -161,28 +196,14 @@ main (int argc, char *argv[]) /* Other variables */ unsigned int i, j; - hid_t file, dataset, space, PRESERVE, fapl; + hid_t file, dataset, space, PRESERVE; hid_t array_dt; static hsize_t dim[] = {NX, NY}; hsize_t f_offset[2]; /*offset of hyperslab in file */ hsize_t h_size[2]; /*size of hyperslab */ hsize_t memb_size[1] = {4}; - char filename[256]; - - h5_reset(); - - /* Turn off optimized compound converter? */ - if (argc>1) { - if (argc>2 || strcmp("--noopt", argv[1])) { - fprintf(stderr, "usage: %s [--noopt]\n", argv[0]); - exit(1); - } - H5Tunregister(H5T_PERS_DONTCARE, NULL, -1, -1, H5T_conv_struct_opt); - } /* Create the file */ - fapl = h5_fileaccess(); - h5_fixname(FILENAME[0], fapl, filename, sizeof(filename)); if ((file = H5Fcreate (filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl))<0) { goto error; } @@ -808,8 +829,6 @@ main (int argc, char *argv[]) } } } - PASSED(); - /* * Release resources. @@ -818,12 +837,950 @@ main (int argc, char *argv[]) H5Dclose (dataset); H5Fclose (file); - h5_cleanup(FILENAME, fapl); - puts("All compound dataset tests passed."); + PASSED(); + return 0; + +error: + puts("*** DATASET TESTS FAILED ***"); + return 1; +} + + +/*------------------------------------------------------------------------- + * Function: initialize_stype1 + * + * Purpose: Initialize data buffer. + * + * Return: void + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static void +initialize_stype1(unsigned char *buf, const size_t num) +{ + int i, j; + stype1 *s_ptr; + + for (i=0; i<(int)num; i++) { + s_ptr = (stype1*)buf + i; + s_ptr->a = i*8+0; + s_ptr->b = i*8+1; + for(j=0; j<8; j++) + s_ptr->c[j] = i*8+j; + s_ptr->d = i*8+6; + s_ptr->e = i*8+7; + + s_ptr->f = i*2/3; + s_ptr->g = i*2/3+1; + for(j=0; j<16; j++) + s_ptr->h[j] = i*j/5+j; + s_ptr->i = i*2/3+2; + s_ptr->j = i*2/3+3; + + s_ptr->k = i/7+1; + s_ptr->l = i/7+2; + s_ptr->m = i/7+3; + s_ptr->n = i/7+4; + } +} + + +/*------------------------------------------------------------------------- + * Function: initialize_stype2 + * + * Purpose: Initialize data buffer. + * + * Return: void + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static void +initialize_stype2(unsigned char *buf, const size_t num) +{ + size_t i, j; + stype2 *s_ptr; + + for (i=0; ia = i*8+0; + s_ptr->b = i*8+1; + for(j=0; j<8; j++) + s_ptr->c[j] = i*8+j; + s_ptr->d = i*8+6; + s_ptr->e = i*8+7; + + s_ptr->f = i*2/3; + s_ptr->g = i*2/3+1; + for(j=0; j<16; j++) + s_ptr->h[j] = i*j/5+j; + s_ptr->i = i*2/3+2; + s_ptr->j = i*2/3+3; + + s_ptr->k = i/7+1; + s_ptr->l = i/7+2; + s_ptr->m = i/7+3; + s_ptr->n = i/7+4; + + s_ptr->o = i*3+0; + s_ptr->p = i*3+1; + s_ptr->q = i*3+2; + } +} + + +/*------------------------------------------------------------------------- + * Function: initialize_stype3 + * + * Purpose: Initialize data buffer. + * + * Return: Success: + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static void +initialize_stype3(unsigned char *buf, const size_t num) +{ + int i, j; + stype3 *s_ptr; + + for (i=0; i<(int)num; i++) { + s_ptr = (stype3*)buf + i; + s_ptr->a = i*8+0; + s_ptr->b = i*8+1; + for(j=0; j<8; j++) + s_ptr->c[j] = i*8+j; + s_ptr->d = i*8+6; + s_ptr->e = i*8+7; + } +} + + +/*------------------------------------------------------------------------- + * Function: initialize_stype4 + * + * Purpose: Initialize data buffer. + * + * Return: void + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static void +initialize_stype4(unsigned char *buf, const size_t num) +{ + size_t i, j; + stype4 *s_ptr; + + for (i=0; ia = i*8+0; + s_ptr->b = i*8+1; + for(j=0; j<8; j++) + s_ptr->c[j] = i*8+j; + s_ptr->d = i*8+6; + s_ptr->e = i*8+7; + + s_ptr->f = i*2/3; + s_ptr->g = i*2/3+1; + for(j=0; j<16; j++) + s_ptr->h[j] = i*j/5+j; + s_ptr->i = i*2/3+2; + s_ptr->j = i*2/3+3; + + s_ptr->k = i/7+1; + s_ptr->l = i/7+2; + s_ptr->m = i/7+3; + s_ptr->n = i/7+4; + + s_ptr->o = i*3+0; + s_ptr->p = i*3+1; + s_ptr->q = i*3+2; + + s_ptr->r = i*5+1; + s_ptr->s = i*5+2; + s_ptr->t = i*5+3; + } +} + + +/*------------------------------------------------------------------------- + * Function: create_stype1 + * + * Purpose: Create HDF5 compound datatype for stype1. + * + * Return: Success: datatype ID + * + * Failure: negative + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static hid_t +create_stype1(void) +{ + hid_t array_dt1, array_dt2, tid; + const hsize_t eight = 8, sixteen = 16; + + /* Build hdf5 datatypes */ + if((array_dt1 = H5Tarray_create(H5T_NATIVE_INT,1, &eight, NULL))<0) + goto error; + if((array_dt2 = H5Tarray_create(H5T_NATIVE_FLOAT,1, &sixteen, NULL))<0) + goto error; + + if ((tid=H5Tcreate(H5T_COMPOUND, sizeof(stype1)))<0 || + H5Tinsert(tid, "a", HOFFSET(stype1, a), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "b", HOFFSET(stype1, b), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "c", HOFFSET(stype1, c), array_dt1)<0 || + H5Tinsert(tid, "d", HOFFSET(stype1, d), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "e", HOFFSET(stype1, e), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "f", HOFFSET(stype1, f), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "g", HOFFSET(stype1, g), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "h", HOFFSET(stype1, h), array_dt2)<0 || + H5Tinsert(tid, "i", HOFFSET(stype1, i), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "j", HOFFSET(stype1, j), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "k", HOFFSET(stype1, k), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "l", HOFFSET(stype1, l), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "m", HOFFSET(stype1, m), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "n", HOFFSET(stype1, n), H5T_NATIVE_DOUBLE)<0) + goto error; + + if(H5Tclose(array_dt1)<0) + goto error; + if(H5Tclose(array_dt2)<0) + goto error; + + return tid; + +error: + return FAIL; +} + + +/*------------------------------------------------------------------------- + * Function: create_stype2 + * + * Purpose: Create HDF5 compound datatype for stype2. + * + * Return: Success: datatype ID + * + * Failure: negative + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static hid_t +create_stype2(void) +{ + hid_t array_dt1, array_dt2, tid; + const hsize_t eight = 8, sixteen = 16; + + /* Build hdf5 datatypes */ + if((array_dt1 = H5Tarray_create(H5T_NATIVE_INT,1, &eight, NULL))<0) + goto error; + if((array_dt2 = H5Tarray_create(H5T_NATIVE_FLOAT,1, &sixteen, NULL))<0) + goto error; + + if ((tid=H5Tcreate(H5T_COMPOUND, sizeof(stype2)))<0 || + H5Tinsert(tid, "a", HOFFSET(stype2, a), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "b", HOFFSET(stype2, b), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "c", HOFFSET(stype2, c), array_dt1)<0 || + H5Tinsert(tid, "d", HOFFSET(stype2, d), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "e", HOFFSET(stype2, e), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "f", HOFFSET(stype2, f), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "g", HOFFSET(stype2, g), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "h", HOFFSET(stype2, h), array_dt2)<0 || + H5Tinsert(tid, "i", HOFFSET(stype2, i), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "j", HOFFSET(stype2, j), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "k", HOFFSET(stype2, k), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "l", HOFFSET(stype2, l), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "m", HOFFSET(stype2, m), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "n", HOFFSET(stype2, n), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "o", HOFFSET(stype2, o), H5T_NATIVE_LONG)<0 || + H5Tinsert(tid, "p", HOFFSET(stype2, p), H5T_NATIVE_LONG)<0 || + H5Tinsert(tid, "q", HOFFSET(stype2, q), H5T_NATIVE_LONG)<0) + goto error; + + if(H5Tclose(array_dt1)<0) + goto error; + if(H5Tclose(array_dt2)<0) + goto error; + + return tid; + +error: + return FAIL; +} + + +/*------------------------------------------------------------------------- + * Function: create_stype3 + * + * Purpose: Create HDF5 compound datatype for stype3. + * + * Return: Success: datatype ID + * + * Failure: negative + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static hid_t +create_stype3(void) +{ + hid_t array_dt1, tid; + const hsize_t eight = 8; + + /* Build hdf5 datatypes */ + if((array_dt1 = H5Tarray_create(H5T_NATIVE_INT,1, &eight, NULL))<0) + goto error; + + if ((tid=H5Tcreate(H5T_COMPOUND, sizeof(stype3)))<0 || + H5Tinsert(tid, "a", HOFFSET(stype3, a), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "b", HOFFSET(stype3, b), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "c", HOFFSET(stype3, c), array_dt1)<0 || + H5Tinsert(tid, "d", HOFFSET(stype3, d), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "e", HOFFSET(stype3, e), H5T_NATIVE_INT)<0) + goto error; + + if(H5Tclose(array_dt1)<0) + goto error; + + return tid; + +error: + return FAIL; +} + + +/*------------------------------------------------------------------------- + * Function: create_stype4 + * + * Purpose: Create HDF5 compound datatype for stype4. + * + * Return: Success: datatype ID + * + * Failure: negative + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static hid_t +create_stype4(void) +{ + hid_t array_dt1, array_dt2, tid; + const hsize_t eight = 8, sixteen = 16; + + /* Build hdf5 datatypes */ + if((array_dt1 = H5Tarray_create(H5T_NATIVE_INT,1, &eight, NULL))<0) + goto error; + if((array_dt2 = H5Tarray_create(H5T_NATIVE_FLOAT,1, &sixteen, NULL))<0) + goto error; + + if ((tid=H5Tcreate(H5T_COMPOUND, sizeof(stype4)))<0 || + H5Tinsert(tid, "a", HOFFSET(stype4, a), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "b", HOFFSET(stype4, b), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "c", HOFFSET(stype4, c), array_dt1)<0 || + H5Tinsert(tid, "d", HOFFSET(stype4, d), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "e", HOFFSET(stype4, e), H5T_NATIVE_INT)<0 || + H5Tinsert(tid, "f", HOFFSET(stype4, f), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "g", HOFFSET(stype4, g), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "h", HOFFSET(stype4, h), array_dt2)<0 || + H5Tinsert(tid, "i", HOFFSET(stype4, i), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "j", HOFFSET(stype4, j), H5T_NATIVE_FLOAT)<0 || + H5Tinsert(tid, "k", HOFFSET(stype4, k), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "l", HOFFSET(stype4, l), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "m", HOFFSET(stype4, m), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "n", HOFFSET(stype4, n), H5T_NATIVE_DOUBLE)<0 || + H5Tinsert(tid, "o", HOFFSET(stype4, o), H5T_NATIVE_LONG)<0 || + H5Tinsert(tid, "p", HOFFSET(stype4, p), H5T_NATIVE_LONG)<0 || + H5Tinsert(tid, "q", HOFFSET(stype4, q), H5T_NATIVE_LONG)<0 || + H5Tinsert(tid, "r", HOFFSET(stype4, r), H5T_NATIVE_LLONG)<0 || + H5Tinsert(tid, "s", HOFFSET(stype4, s), H5T_NATIVE_LLONG)<0 || + H5Tinsert(tid, "t", HOFFSET(stype4, t), H5T_NATIVE_LLONG)<0) + goto error; + + if(H5Tclose(array_dt1)<0) + goto error; + if(H5Tclose(array_dt2)<0) + goto error; + + return tid; + +error: + return FAIL; +} + + +/*------------------------------------------------------------------------- + * Function: compare_data + * + * Purpose: Compare data of stype1 and stype2. + * + * Return: Success: 0 + * + * Failure: negative + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static int +compare_data(unsigned char *src_data, unsigned char *dst_data, hbool_t src_subset) +{ + stype1 *s_ptr; + stype2 *d_ptr; + int i; + + for (i=0; i<(int)NX*NY; i++) { + if(src_subset) { + s_ptr = ((stype1*)src_data) + i; + d_ptr = ((stype2*)dst_data) + i; + } else { + s_ptr = ((stype2*)src_data) + i; + d_ptr = ((stype1*)dst_data) + i; + } + + if (s_ptr->a != d_ptr->a || + s_ptr->b != d_ptr->b || + s_ptr->c[0] != d_ptr->c[0] || + s_ptr->c[1] != d_ptr->c[1] || + s_ptr->c[2] != d_ptr->c[2] || + s_ptr->c[3] != d_ptr->c[3] || + s_ptr->d != d_ptr->d || + s_ptr->e != d_ptr->e || + !FLT_ABS_EQUAL(s_ptr->f, d_ptr->f) || + !FLT_ABS_EQUAL(s_ptr->g, d_ptr->g) || + !FLT_ABS_EQUAL(s_ptr->h[0], d_ptr->h[0]) || + !FLT_ABS_EQUAL(s_ptr->h[1], d_ptr->h[1]) || + !FLT_ABS_EQUAL(s_ptr->i, d_ptr->i) || + !FLT_ABS_EQUAL(s_ptr->j, d_ptr->j) || + !DBL_ABS_EQUAL(s_ptr->k, d_ptr->k) || + !DBL_ABS_EQUAL(s_ptr->l, d_ptr->l) || + !DBL_ABS_EQUAL(s_ptr->m, d_ptr->m) || + !DBL_ABS_EQUAL(s_ptr->n, d_ptr->n) ) { + + H5_FAILED(); + printf(" i=%d\n", i); + printf(" src={a=%d, b=%d, c=[%d,%d,%d,%d,%d,%d,%d,%d], d=%d, e=%d, f=%f, g=%f, h=[%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f], i=%f, j=%f, k=%f, l=%f, m=%f, n=%f}\n", + s_ptr->a, s_ptr->b, s_ptr->c[0], s_ptr->c[1], s_ptr->c[2], + s_ptr->c[3], s_ptr->c[4], s_ptr->c[5], s_ptr->c[6], s_ptr->c[7], + s_ptr->d, s_ptr->e, s_ptr->f, s_ptr->g,s_ptr->h[0],s_ptr->h[1],s_ptr->h[2], + s_ptr->h[3],s_ptr->h[4],s_ptr->h[5],s_ptr->h[6],s_ptr->h[7],s_ptr->h[8], + s_ptr->h[9],s_ptr->h[10],s_ptr->h[11],s_ptr->h[12],s_ptr->h[13],s_ptr->h[14], + s_ptr->h[15], s_ptr->i,s_ptr->j,s_ptr->k,s_ptr->l,s_ptr->m,s_ptr->n); + printf(" dst={a=%d, b=%d, c=[%d,%d,%d,%d,%d,%d,%d,%d], d=%d, e=%d, f=%f, g=%f, h=[%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f,%f], i=%f, j=%f, k=%f, l=%f, m=%f, n=%f}\n", + d_ptr->a, d_ptr->b, d_ptr->c[0], d_ptr->c[1], d_ptr->c[2], + d_ptr->c[3], d_ptr->c[4], d_ptr->c[5], d_ptr->c[6], d_ptr->c[7], + d_ptr->d, d_ptr->e, d_ptr->f, d_ptr->g,d_ptr->h[0],d_ptr->h[1],d_ptr->h[2], + d_ptr->h[3],d_ptr->h[4],d_ptr->h[5],d_ptr->h[6],d_ptr->h[7],d_ptr->h[8], + d_ptr->h[9],d_ptr->h[10],d_ptr->h[11],d_ptr->h[12],d_ptr->h[13], + d_ptr->h[14], d_ptr->h[15], d_ptr->i,d_ptr->j,d_ptr->k,d_ptr->l, + d_ptr->m,d_ptr->n); + goto error; + } + } + + return SUCCEED; + +error: + return FAIL; +} + + +/*------------------------------------------------------------------------- + * Function: test_hdf5_src_subset + * + * Purpose: Test the optimization of compound data writing, rewriting, + * and reading when the source type is a subset of destination + * type. For example: + * struct source { struct destination { + * TYPE1 A; --> TYPE1 A; + * TYPE2 B; --> TYPE2 B; + * TYPE3 C; --> TYPE3 C; + * }; TYPE4 D; + * TYPE5 E; + * }; + * This optimization is for the Chicago company. + * + * Return: Success: 0 + * + * Failure: 1 + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static int +test_hdf5_src_subset(char *filename, hid_t fapl) +{ + hid_t file; + hid_t rew_tid, src_tid, dst_tid; + hid_t dataset; + hid_t space; + hid_t dcpl, dxpl; + hsize_t dims[2] = {NX, NY}; + hsize_t chunk_dims[2] = {NX/10, NY/10}; + unsigned char *orig=NULL, *rew_buf=NULL, *rbuf=NULL; + + /* Create the file for this test */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + goto error; + + /* Build hdf5 datatypes */ + if ((src_tid=create_stype1())<0) + goto error; + + if ((dst_tid=create_stype2())<0) + goto error; + + if ((rew_tid=create_stype3())<0) + goto error; + + /* Create the data space */ + if((space = H5Screate_simple(2, dims, NULL))<0) + goto error; + + /* Allocate space and initialize data */ + orig = (unsigned char*)malloc(NX * NY * sizeof(stype1)); + initialize_stype1(orig, (size_t)NX*NY); + + rbuf = (unsigned char*)malloc(NX * NY * sizeof(stype2)); + + rew_buf = (unsigned char*)malloc(NX * NY * sizeof(stype3)); + initialize_stype3(rew_buf, (size_t)NX*NY); + + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + goto error; + + /* + *###################################################################### + * STEP 1. Write data to contiguous and chunked datasets. + */ + TESTING("writing data to contiguous and chunked datasets"); + + /* Create contiguous data set */ + if((dataset = H5Dcreate(file, DSET_NAME[0], src_tid, space, dcpl))<0) + goto error; + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, src_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, orig)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + /* Set chunking */ + if(H5Pset_chunk(dcpl, 2, chunk_dims)<0) + goto error; + + /* Create chunked data set */ + if((dataset = H5Dcreate(file, DSET_NAME[1], src_tid, space, dcpl))<0) + goto error; + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, src_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, orig)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + PASSED(); + + /* + *###################################################################### + * STEP 2. Rewrite the data with a subset of original data type. + */ + TESTING("rewriting data with a subset of original data type"); + + /* Create xfer properties to preserve initialized data */ + if ((dxpl = H5Pcreate (H5P_DATASET_XFER))<0) + goto error; + + if (H5Pset_preserve (dxpl, TRUE)<0) + goto error; + + /* Rewrite contiguous data set */ + if((dataset = H5Dopen(file, DSET_NAME[0]))<0) + goto error; + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, rew_tid, H5S_ALL, H5S_ALL, dxpl, rew_buf)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + /* Rewrite chunked data set */ + if((dataset = H5Dopen(file, DSET_NAME[1]))<0) + goto error; + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, rew_tid, H5S_ALL, H5S_ALL, dxpl, rew_buf)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + PASSED(); + + /* + *###################################################################### + * STEP 3. Read the data into a subset of the original compound type. + */ + TESTING("reading data with a subset of original data type"); + + /* Check contiguous data set */ + if((dataset = H5Dopen(file, DSET_NAME[0]))<0) + goto error; + + if(H5Dread(dataset, dst_tid, H5S_ALL, H5S_ALL, dxpl, rbuf)<0) + goto error; + + if(compare_data(orig, rbuf, TRUE)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + /* Check chunked data set */ + if((dataset = H5Dopen(file, DSET_NAME[1]))<0) + goto error; + + if(H5Dread(dataset, dst_tid, H5S_ALL, H5S_ALL, dxpl, rbuf)<0) + goto error; + + if(compare_data(orig, rbuf, TRUE)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + /* Finishing test and release resources */ + if(H5Sclose(space) < 0) + goto error; + + if(H5Pclose(dcpl) < 0) + goto error; + + if(H5Pclose(dxpl) < 0) + goto error; + + if(H5Tclose(src_tid)<0) + goto error; + if(H5Tclose(dst_tid)<0) + goto error; + if(H5Tclose(rew_tid)<0) + goto error; + if(H5Fclose(file) < 0) + goto error; + + free(orig); + free(rbuf); + free(rew_buf); + + PASSED(); + return 0; + +error: + puts("*** DATASET TESTS FAILED ***"); + return 1; +} + + +/*------------------------------------------------------------------------- + * Function: test_hdf5_dst_subset + * + * Purpose: Test the optimization of compound data writing, rewriting, + * and reading when the destination type is a subset of the + * source type. For example: + * struct source { struct destination { + * TYPE1 A; --> TYPE1 A; + * TYPE2 B; --> TYPE2 B; + * TYPE3 C; --> TYPE3 C; + * TYPE4 D; } + * TYPE5 E; + * }; + * This optimization is for the Chicago company. This test + * is in opposite of test_hdf5_src_subset. + * + * Return: Success: 0 + * + * Failure: 1 + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +static int +test_hdf5_dst_subset(char *filename, hid_t fapl) +{ + hid_t file; + hid_t rew_tid, src_tid, dst_tid; + hid_t dataset; + hid_t space; + hid_t dcpl, dxpl; + hsize_t dims[2] = {NX, NY}; + hsize_t chunk_dims[2] = {NX/10, NY/10}; + unsigned char *orig=NULL, *rew_buf=NULL, *rbuf=NULL; + + /* Create the file for this test */ + if((file = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, fapl)) < 0) + goto error; + + /* Build hdf5 datatypes */ + if ((src_tid=create_stype2())<0) + goto error; + + if ((dst_tid=create_stype1())<0) + goto error; + + if ((rew_tid=create_stype4())<0) + goto error; + + /* Create the data space */ + if((space = H5Screate_simple(2, dims, NULL))<0) + goto error; + + /* Allocate space and initialize data */ + orig = (unsigned char*)malloc(NX * NY * sizeof(stype2)); + initialize_stype2(orig, (size_t)NX*NY); + + rbuf = (unsigned char*)malloc(NX * NY * sizeof(stype1)); + + rew_buf = (unsigned char*)malloc(NX * NY * sizeof(stype4)); + initialize_stype4(rew_buf, (size_t)NX*NY); + + /* Create dataset creation property list */ + if((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + goto error; + + /* + *###################################################################### + * STEP 1. Write data to contiguous and chunked datasets. + */ + TESTING("writing data to contiguous and chunked datasets"); + + /* Create contiguous data set */ + if((dataset = H5Dcreate(file, DSET_NAME[2], src_tid, space, dcpl))<0) + goto error; + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, src_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, orig)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + /* Set chunking */ + if(H5Pset_chunk(dcpl, 2, chunk_dims)<0) + goto error; + + /* Create chunked data set */ + if((dataset = H5Dcreate(file, DSET_NAME[3], src_tid, space, dcpl))<0) + goto error; + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, src_tid, H5S_ALL, H5S_ALL, H5P_DEFAULT, orig)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + PASSED(); + + /* + *###################################################################### + * STEP 2. Rewrite the data with a subset of original data type. + */ + TESTING("rewriting data with a subset of original data type"); + + /* Create xfer properties to preserve initialized data */ + if ((dxpl = H5Pcreate (H5P_DATASET_XFER))<0) + goto error; + + if (H5Pset_preserve (dxpl, TRUE)<0) + goto error; + + /* Rewrite contiguous data set */ + if((dataset = H5Dopen(file, DSET_NAME[2]))<0) + goto error; + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, rew_tid, H5S_ALL, H5S_ALL, dxpl, rew_buf)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + /* Rewrite chunked data set */ + if((dataset = H5Dopen(file, DSET_NAME[3]))<0) + goto error; + + /* Write the data to the dataset */ + if(H5Dwrite(dataset, rew_tid, H5S_ALL, H5S_ALL, dxpl, rew_buf)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + PASSED(); + + /* + *###################################################################### + * STEP 3. Read the data into a subset of the original compound type. + */ + TESTING("reading data with a subset of original data type"); + + /* Check contiguous data set */ + if((dataset = H5Dopen(file, DSET_NAME[2]))<0) + goto error; + + if(H5Dread(dataset, dst_tid, H5S_ALL, H5S_ALL, dxpl, rbuf)<0) + goto error; + + if(compare_data(orig, rbuf, FALSE)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + /* Check chunked data set */ + if((dataset = H5Dopen(file, DSET_NAME[3]))<0) + goto error; + + if(H5Dread(dataset, dst_tid, H5S_ALL, H5S_ALL, dxpl, rbuf)<0) + goto error; + + if(compare_data(orig, rbuf, FALSE)<0) + goto error; + + if(H5Dclose(dataset) < 0) + goto error; + + /* Finishing test and release resources */ + if(H5Sclose(space) < 0) + goto error; + + if(H5Pclose(dcpl) < 0) + goto error; + + if(H5Pclose(dxpl) < 0) + goto error; + + if(H5Tclose(src_tid)<0) + goto error; + if(H5Tclose(dst_tid)<0) + goto error; + if(H5Tclose(rew_tid)<0) + goto error; + if(H5Fclose(file) < 0) + goto error; + + free(orig); + free(rbuf); + free(rew_buf); + + PASSED(); return 0; error: - puts("Remaining tests have been skipped."); puts("*** DATASET TESTS FAILED ***"); return 1; } + + +/*------------------------------------------------------------------------- + * Function: main + * + * Purpose: Test different cases of I/O for compound data and the + * compound optimization for the Chicago company. + * + * Return: Success: 0 + * + * Failure: 1 + * + * Programmer: Raymond Lu + * Friday, 15 June 2007 + * + * Modifications: + *------------------------------------------------------------------------- + */ +int +main (int argc, char *argv[]) +{ + hid_t fapl_id; + char fname[256]; + unsigned nerrors = 0; + + h5_reset(); + + /* Turn off optimized compound converter? */ + if (argc>1) { + if (argc>2 || strcmp("--noopt", argv[1])) { + fprintf(stderr, "usage: %s [--noopt]\n", argv[0]); + exit(1); + } + H5Tunregister(H5T_PERS_DONTCARE, NULL, -1, -1, H5T_conv_struct_opt); + } + + /* Create the file */ + fapl_id = h5_fileaccess(); + + h5_fixname(FILENAME[0], fapl_id, fname, sizeof(fname)); + + puts("Testing compound dataset:"); + nerrors += test_compound(fname, fapl_id); + + puts("Testing the optimization of when the source type is a subset of the dest:"); + h5_fixname(FILENAME[1], fapl_id, fname, sizeof(fname)); + nerrors += test_hdf5_src_subset(fname, fapl_id); + + puts("Testing the optimization of when the dest type is a subset of the source:"); + h5_fixname(FILENAME[2], fapl_id, fname, sizeof(fname)); + nerrors += test_hdf5_dst_subset(fname, fapl_id); + + if (nerrors) { + printf("***** %u FAILURE%s! *****\n", + nerrors, 1==nerrors?"":"S"); + HDexit(1); + } + + h5_cleanup(FILENAME, fapl_id); + puts("All compound dataset tests passed."); + return 0; +} -- cgit v0.12