diff options
Diffstat (limited to 'tools/h5repack')
-rw-r--r-- | tools/h5repack/h5repack.h | 5 | ||||
-rwxr-xr-x | tools/h5repack/h5repack.sh.in | 4 | ||||
-rw-r--r-- | tools/h5repack/h5repack_copy.c | 352 | ||||
-rw-r--r-- | tools/h5repack/h5repack_main.c | 10 | ||||
-rw-r--r-- | tools/h5repack/h5repack_refs.c | 74 | ||||
-rw-r--r-- | tools/h5repack/testh5repack.h | 2 | ||||
-rw-r--r-- | tools/h5repack/testh5repack_main.c | 19 | ||||
-rw-r--r-- | tools/h5repack/testh5repack_make.c | 170 |
8 files changed, 448 insertions, 188 deletions
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index 29909b7..da82a05 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -99,8 +99,9 @@ typedef struct { filter_info_t filter_g; /*global filter INFO for the ALL case */ chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ H5D_layout_t layout_g; /*global layout information for the ALL case */ - int verbose; /*verbose mode */ - hsize_t threshold; /*minimum size to compress, in bytes */ + int verbose; /*verbose mode */ + hsize_t threshold; /*minimum size to compress, in bytes */ + int use_native; /*use a native type in write */ } pack_opt_t; diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index e2b770b..5769783 100755 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -409,6 +409,10 @@ TOOLTEST $arg7 TOOLTEST $arg8 TOOLTEST $arg9 +#native option +arg="test1.h5 -n" +TOOLTEST $arg + if test $nerrors -eq 0 ; then echo "All $H5REPACK tests passed." diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index b0677ef..7c530ab 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -16,13 +16,15 @@ #include <string.h> #include <stdio.h> #include "H5private.h" +#include "h5tools.h" #include "h5tools_utils.h" #include "h5repack.h" + extern char *progname; -#define PER(A,B) { per = 0; \ +#define PER(A,B) { per = 0; \ if (A!=0) \ per = (float)fabs(1-( (float)B / (float)A )); \ } @@ -254,8 +256,16 @@ out: * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: October, 23, 2003 - * Modified: December, 03, 2004 - added a check for H5Dcreate; if the dataset - * cannot be created with the requested filter, use the input one + * + * Modifications: + * + * July 2004: Introduced the extra EC or NN option for SZIP + * + * December 2004: Added a check for H5Dcreate; if the dataset cannot be created + * with the requested filter, use the input one + * + * October 2006: Read/write using the file type by default. + * Read/write by hyperslabs for big datasets. * *------------------------------------------------------------------------- */ @@ -265,31 +275,30 @@ int do_copy_objects(hid_t fidin, trav_table_t *travt, pack_opt_t *options) /* repack options */ { - hid_t grp_in=-1; /* group ID */ - hid_t grp_out=-1; /* group ID */ - hid_t dset_in=-1; /* read dataset ID */ - hid_t dset_out=-1; /* write dataset ID */ - hid_t type_in=-1; /* named type ID */ - hid_t type_out=-1; /* named type ID */ - hid_t dcpl_id=-1; /* dataset creation property list ID */ - hid_t dcpl_out=-1; /* dataset creation property list ID */ - hid_t space_id=-1; /* space ID */ - hid_t ftype_id=-1; /* file data type ID */ - hid_t mtype_id=-1; /* memory data type ID */ - size_t msize; /* memory size of memory type */ - void *buf=NULL; /* data buffer */ - hsize_t nelmts; /* number of elements in dataset */ - int rank; /* rank of dataset */ - hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ - hsize_t dsize_in; /* input dataset size before filter */ - hsize_t dsize_out; /* output dataset size after filter */ - int next; /* external files */ - int apply_s; /* flag for apply filter to small dataset sizes */ - int apply_f; /* flag for apply filter to return error on H5Dcreate with filter */ - float per; /* percent utilization of storage */ - int i, j; - - + hid_t grp_in; /* group ID */ + hid_t grp_out; /* group ID */ + hid_t dset_in; /* read dataset ID */ + hid_t dset_out; /* write dataset ID */ + hid_t type_in; /* named type ID */ + hid_t type_out; /* named type ID */ + hid_t dcpl_id; /* dataset creation property list ID */ + hid_t dcpl_out; /* dataset creation property list ID */ + hid_t f_space_id; /* file space ID */ + hid_t ftype_id; /* file type ID */ + hid_t wtype_id; /* read/write type ID */ + size_t msize; /* size of type */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + hsize_t dsize_in; /* input dataset size before filter */ + hsize_t dsize_out; /* output dataset size after filter */ + int next; /* external files */ + int apply_s; /* flag for apply filter to small dataset sizes */ + int apply_f; /* flag for apply filter to return error on H5Dcreate */ + float per; /* percent utilization of storage */ + void *buf=NULL; /* buffer for raw data */ + void *sm_buf=NULL; /* buffer for raw data */ + int i, j; /*------------------------------------------------------------------------- * copy the suppplied object list @@ -305,7 +314,7 @@ int do_copy_objects(hid_t fidin, for ( i = 0; i < travt->nobjs; i++) { - buf=NULL; + buf = NULL; switch ( travt->objs[i].type ) { /*------------------------------------------------------------------------- @@ -345,27 +354,29 @@ int do_copy_objects(hid_t fidin, if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) goto error; - if ((space_id=H5Dget_space(dset_in))<0) + if ((f_space_id=H5Dget_space(dset_in))<0) goto error; if ((ftype_id=H5Dget_type (dset_in))<0) goto error; if ((dcpl_id=H5Dget_create_plist(dset_in))<0) goto error; - if ((dcpl_out = H5Pcopy (dcpl_id))<0) - goto error; - if ( (rank=H5Sget_simple_extent_ndims(space_id))<0) + if ((dcpl_out = H5Pcopy (dcpl_id))<0) + goto error; + if ( (rank=H5Sget_simple_extent_ndims(f_space_id))<0) goto error; HDmemset(dims, 0, sizeof dims); - if ( H5Sget_simple_extent_dims(space_id,dims,NULL)<0) + if ( H5Sget_simple_extent_dims(f_space_id,dims,NULL)<0) goto error; nelmts=1; for (j=0; j<rank; j++) nelmts*=dims[j]; - if ((mtype_id=h5tools_get_native_type(ftype_id))<0) - goto error; + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - if ((msize=H5Tget_size(mtype_id))==0) + if ((msize=H5Tget_size(wtype_id))==0) goto error; /*------------------------------------------------------------------------- @@ -397,79 +408,160 @@ int do_copy_objects(hid_t fidin, * in a second traversal of the output file *------------------------------------------------------------------------- */ - if ( (H5T_REFERENCE!=H5Tget_class(mtype_id))) + if ( (H5T_REFERENCE!=H5Tget_class(wtype_id))) { /* get the storage size of the input dataset */ dsize_in=H5Dget_storage_size(dset_in); + /* check for datasets too small */ + if (nelmts*msize < options->threshold ) + apply_s=0; + + /* apply the filter */ + if (apply_s){ + if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0) + goto error; + } + + /*------------------------------------------------------------------------- + * create the output dataset; + * disable error checking in case the dataset cannot be created with the + * modified dcpl; in that case use the original instead + *------------------------------------------------------------------------- + */ + H5E_BEGIN_TRY { + dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_out); + } H5E_END_TRY; + if (dset_out==FAIL) + { + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_id))<0) + goto error; + apply_f=0; + } + /*------------------------------------------------------------------------- - * read + * read/write *------------------------------------------------------------------------- */ if (nelmts) { - buf=(void *) HDmalloc((unsigned)(nelmts*msize)); - if ( buf==NULL){ - error_msg(progname, "cannot read into memory\n" ); - goto error; - } - if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - - /*------------------------------------------------------------------------- - * check for datasets too small - *------------------------------------------------------------------------- - */ + size_t need = (size_t)(nelmts*msize); /* bytes needed */ + if ( need < H5TOOLS_MALLOCSIZE ) + buf = HDmalloc(need); - if (nelmts*msize < options->threshold ) - apply_s=0; - - /*------------------------------------------------------------------------- - * apply the filter - *------------------------------------------------------------------------- - */ - if (apply_s){ - if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0) + if (buf != NULL ) + { + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; - } - + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto error; + } + + else /* possibly not enough memory, read/write by hyperslabs */ + + { + size_t p_type_nbytes = msize; /*size of memory type */ + hsize_t p_nelmts = nelmts; /*total selected elmts */ + hsize_t elmtno; /*counter */ + int carry; /*counter carry value */ + unsigned int vl_data = 0; /*contains VL datatypes */ + + /* stripmine info */ + hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ + hsize_t sm_nbytes; /*bytes per stripmine */ + hsize_t sm_nelmts; /*elements per stripmine*/ + hid_t sm_space; /*stripmine data space */ + + /* hyperslab info */ + hsize_t hs_offset[H5S_MAX_RANK];/*starting offset */ + hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ + hsize_t hs_nelmts; /*elements in request */ + hsize_t zero[8]; /*vector of zeros */ + + /* check if we have VL data in the dataset's datatype */ + if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE) + vl_data = TRUE; + + /* + * determine the strip mine size and allocate a buffer. The strip mine is + * a hyperslab whose size is manageable. + */ + sm_nbytes = p_type_nbytes; + + for (i = rank; i > 0; --i) { + sm_size[i - 1] = MIN(dims[i - 1], H5TOOLS_BUFSIZE / sm_nbytes); + sm_nbytes *= sm_size[i - 1]; + assert(sm_nbytes > 0); + } + sm_buf = HDmalloc((size_t)sm_nbytes); + + sm_nelmts = sm_nbytes / p_type_nbytes; + sm_space = H5Screate_simple(1, &sm_nelmts, NULL); + + /* the stripmine loop */ + memset(hs_offset, 0, sizeof hs_offset); + memset(zero, 0, sizeof zero); + + for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) + { + /* calculate the hyperslab size */ + if (rank > 0) + { + for (i = 0, hs_nelmts = 1; i < rank; i++) + { + hs_size[i] = MIN(dims[i] - hs_offset[i], sm_size[i]); + hs_nelmts *= hs_size[i]; + } + + if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0) + goto error; + if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0) + goto error; + } + else + { + H5Sselect_all(f_space_id); + H5Sselect_all(sm_space); + hs_nelmts = 1; + } /* rank */ + + /* read/write */ + if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) + goto error; + if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) + goto error; + + /* reclaim any VL memory, if necessary */ + if(vl_data) + H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf); + + /* calculate the next hyperslab offset */ + for (i = rank, carry = 1; i > 0 && carry; --i) + { + hs_offset[i - 1] += hs_size[i - 1]; + if (hs_offset[i - 1] == dims[i - 1]) + hs_offset[i - 1] = 0; + else + carry = 0; + } /* i */ + } /* elmtno */ + + H5Sclose(sm_space); + /* free */ + if (sm_buf!=NULL) + { + HDfree(sm_buf); + sm_buf=NULL; + } + } /* hyperslab read */ }/*nelmts*/ - /*------------------------------------------------------------------------- - * create; - * disable error checking in case the dataset cannot be created with the - * modified dcpl; in that case use the original instead - *------------------------------------------------------------------------- - */ - - H5E_BEGIN_TRY { - dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_out); - } H5E_END_TRY; - - - if (dset_out==FAIL) - { - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) - goto error; - - apply_f=0; - } - - /*------------------------------------------------------------------------- - * write dataset - *------------------------------------------------------------------------- - */ - - if (dsize_in && nelmts) { - if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - } - /*------------------------------------------------------------------------- * amount of compression used *------------------------------------------------------------------------- */ - if (options->verbose) { + if (options->verbose) + { if (apply_s && apply_f) { /* get the storage size of the input dataset */ @@ -487,7 +579,7 @@ int do_copy_objects(hid_t fidin, (int)options->threshold); if (apply_f==0 && options->verbose) - printf(" <warning: could not apply the filter to %s>\n", + printf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name); /*------------------------------------------------------------------------- @@ -500,11 +592,8 @@ int do_copy_objects(hid_t fidin, /*close */ if (H5Dclose(dset_out)<0) goto error; - - if (buf) - free(buf); - - }/*H5T_STD_REF_OBJ*/ + + }/*H5T_STD_REF_OBJ*/ }/*can_read*/ @@ -514,13 +603,13 @@ int do_copy_objects(hid_t fidin, */ if (H5Tclose(ftype_id)<0) goto error; - if (H5Tclose(mtype_id)<0) + if (H5Tclose(wtype_id)<0) goto error; if (H5Pclose(dcpl_id)<0) goto error; - if (H5Pclose(dcpl_out)<0) + if (H5Pclose(dcpl_out)<0) goto error; - if (H5Sclose(space_id)<0) + if (H5Sclose(f_space_id)<0) goto error; if (H5Dclose(dset_in)<0) goto error; @@ -581,18 +670,18 @@ int do_copy_objects(hid_t fidin, } break; -/*------------------------------------------------------------------------- - *------------------------------------------------------------------------- - */ - { - - } - break; - default: goto error; + } /* switch */ + + /* free */ + if (buf!=NULL) + { + HDfree(buf); + buf=NULL; } - } + + } /* i */ /*------------------------------------------------------------------------- * the root is a special case, we get an ID for the root group @@ -623,15 +712,24 @@ error: H5Gclose(grp_in); H5Gclose(grp_out); H5Pclose(dcpl_id); - H5Sclose(space_id); + H5Sclose(f_space_id); H5Dclose(dset_in); H5Dclose(dset_out); H5Tclose(ftype_id); - H5Tclose(mtype_id); + H5Tclose(wtype_id); H5Tclose(type_in); H5Tclose(type_out); - if (buf) - free(buf); + /* free */ + if (buf!=NULL) + { + HDfree(buf); + buf=NULL; + } + if (sm_buf!=NULL) + { + HDfree(sm_buf); + sm_buf=NULL; + } } H5E_END_TRY; return -1; @@ -663,9 +761,9 @@ int copy_attr(hid_t loc_in, hid_t attr_id; /* attr ID */ hid_t attr_out; /* attr ID */ hid_t space_id; /* space ID */ - hid_t ftype_id; /* file data type ID */ - hid_t mtype_id; /* memory data type ID */ - size_t msize; /* memory size of type */ + hid_t ftype_id; /* file type ID */ + hid_t wtype_id; /* read/write type ID */ + size_t msize; /* size of type */ void *buf=NULL; /* data buffer */ hsize_t nelmts; /* number of elements in dataset */ int rank; /* rank of dataset */ @@ -712,10 +810,12 @@ int copy_attr(hid_t loc_in, for (j=0; j<rank; j++) nelmts*=dims[j]; - if ((mtype_id=h5tools_get_native_type(ftype_id))<0) - goto error; + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - if ((msize=H5Tget_size(mtype_id))==0) + if ((msize=H5Tget_size(wtype_id))==0) goto error; /*------------------------------------------------------------------------- @@ -725,7 +825,7 @@ int copy_attr(hid_t loc_in, * the referenced objects *------------------------------------------------------------------------- */ - if ( ! H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + if ( ! H5Tequal(wtype_id, H5T_STD_REF_OBJ)) { @@ -739,7 +839,7 @@ int copy_attr(hid_t loc_in, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Aread(attr_id,mtype_id,buf)<0) + if (H5Aread(attr_id,wtype_id,buf)<0) goto error; /*------------------------------------------------------------------------- @@ -749,7 +849,7 @@ int copy_attr(hid_t loc_in, if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) goto error; - if(H5Awrite(attr_out,mtype_id,buf)<0) + if(H5Awrite(attr_out,wtype_id,buf)<0) goto error; /*close*/ @@ -773,7 +873,7 @@ int copy_attr(hid_t loc_in, */ if (H5Tclose(ftype_id)<0) goto error; - if (H5Tclose(mtype_id)<0) goto error; + if (H5Tclose(wtype_id)<0) goto error; if (H5Sclose(space_id)<0) goto error; if (H5Aclose(attr_id)<0) goto error; @@ -784,7 +884,7 @@ int copy_attr(hid_t loc_in, error: H5E_BEGIN_TRY { H5Tclose(ftype_id); - H5Tclose(mtype_id); + H5Tclose(wtype_id); H5Sclose(space_id); H5Aclose(attr_id); H5Aclose(attr_out); diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 074f937..5e26242 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -35,8 +35,10 @@ static void usage(void); * * Comments: * - * Modifications: July 2004 - * Introduced the extra EC or NN option for SZIP + * Modifications: + * July 2004: Introduced the extra EC or NN option for SZIP + * October 2006: Added a new switch -n, that allows to write the dataset + * using a native type. The default to write is the file type. * *------------------------------------------------------------------------- */ @@ -104,6 +106,9 @@ int main(int argc, char **argv) else if (strcmp(argv[i], "-e") == 0) { read_info(argv[++i],&options); } + else if (strcmp(argv[i], "-n") == 0) { + options.use_native = 1; + } else if (argv[i][0] == '-') { error_msg(progname, " - is not a valid argument\n"); @@ -194,6 +199,7 @@ void usage(void) printf(" <dim_1 x dim_2 x ... dim_n>\n"); printf("\n"); printf("-e file File with the -f and -l options (only filter and layout flags)\n"); + printf("-n Use a native type when repacking. Default is the file type\n"); printf("-m size Do not apply the filter to objects which size in bytes\n"); printf(" is smaller than number. If no size is specified a minimum of\n"); printf(" 1024 bytes is assumed.\n"); diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c index 1f7c2e9..2a01f9e 100644 --- a/tools/h5repack/h5repack_refs.c +++ b/tools/h5repack/h5repack_refs.c @@ -61,9 +61,9 @@ int do_copy_refobjs(hid_t fidin, hid_t type_in; /* named type ID */ hid_t dcpl_id; /* dataset creation property list ID */ hid_t space_id; /* space ID */ - hid_t ftype_id; /* file data type ID */ - hid_t mtype_id; /* memory data type ID */ - size_t msize; /* memory size of memory type */ + hid_t ftype_id; /* file type ID */ + hid_t wtype_id; /* read/write type ID */ + size_t msize; /* size of type */ hsize_t nelmts; /* number of elements in dataset */ int rank; /* rank of dataset */ hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ @@ -124,11 +124,13 @@ int do_copy_refobjs(hid_t fidin, nelmts=1; for (j=0; j<rank; j++) nelmts*=dims[j]; + + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - if ((mtype_id=h5tools_get_native_type(ftype_id))<0) - goto error; - - if ((msize=H5Tget_size(mtype_id))==0) + if ((msize=H5Tget_size(wtype_id))==0) goto error; /*------------------------------------------------------------------------- @@ -157,7 +159,7 @@ int do_copy_refobjs(hid_t fidin, * we cannot just copy the buffers, but instead we recreate the reference *------------------------------------------------------------------------- */ - if (H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + if (H5Tequal(wtype_id, H5T_STD_REF_OBJ)) { H5G_obj_t obj_type; hid_t refobj_id; @@ -178,7 +180,7 @@ int do_copy_refobjs(hid_t fidin, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(dset_in,H5R_OBJECT,buf))<0) @@ -214,10 +216,10 @@ int do_copy_refobjs(hid_t fidin, * create/write dataset/close *------------------------------------------------------------------------- */ - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,space_id,dcpl_id))<0) goto error; if (nelmts) { - if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) goto error; } @@ -232,7 +234,7 @@ int do_copy_refobjs(hid_t fidin, * dataset region references *------------------------------------------------------------------------- */ - else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) + else if (H5Tequal(wtype_id, H5T_STD_REF_DSETREG)) { H5G_obj_t obj_type; hid_t refobj_id; @@ -252,7 +254,7 @@ int do_copy_refobjs(hid_t fidin, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(dset_in,H5R_DATASET_REGION,buf))<0) goto error; @@ -299,10 +301,10 @@ int do_copy_refobjs(hid_t fidin, * create/write dataset/close *------------------------------------------------------------------------- */ - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,space_id,dcpl_id))<0) goto error; if (nelmts) { - if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) goto error; } @@ -359,7 +361,7 @@ int do_copy_refobjs(hid_t fidin, if (H5Tclose(ftype_id)<0) goto error; - if (H5Tclose(mtype_id)<0) + if (H5Tclose(wtype_id)<0) goto error; if (H5Pclose(dcpl_id)<0) goto error; @@ -437,7 +439,7 @@ error: H5Dclose(dset_in); H5Dclose(dset_out); H5Tclose(ftype_id); - H5Tclose(mtype_id); + H5Tclose(wtype_id); H5Tclose(type_in); } H5E_END_TRY; return -1; @@ -473,9 +475,9 @@ static int copy_refs_attr(hid_t loc_in, hid_t attr_id; /* attr ID */ hid_t attr_out; /* attr ID */ hid_t space_id; /* space ID */ - hid_t ftype_id; /* file data type ID */ - hid_t mtype_id; /* memory data type ID */ - size_t msize; /* memory size of type */ + hid_t ftype_id; /* file type ID */ + hid_t wtype_id; /* read/write type ID */ + size_t msize; /* size of type */ hsize_t nelmts; /* number of elements in dataset */ int rank; /* rank of dataset */ hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ @@ -522,10 +524,12 @@ static int copy_refs_attr(hid_t loc_in, for (j=0; j<rank; j++) nelmts*=dims[j]; - if ((mtype_id=h5tools_get_native_type(ftype_id))<0) - goto error; + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - if ((msize=H5Tget_size(mtype_id))==0) + if ((msize=H5Tget_size(wtype_id))==0) goto error; /*------------------------------------------------------------------------- @@ -533,7 +537,7 @@ static int copy_refs_attr(hid_t loc_in, * we cannot just copy the buffers, but instead we recreate the reference *------------------------------------------------------------------------- */ - if (H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + if (H5Tequal(wtype_id, H5T_STD_REF_OBJ)) { H5G_obj_t obj_type; hid_t refobj_id; @@ -554,7 +558,7 @@ static int copy_refs_attr(hid_t loc_in, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Aread(attr_id,mtype_id,buf)<0) + if (H5Aread(attr_id,wtype_id,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(attr_id,H5R_OBJECT,buf))<0) @@ -592,7 +596,7 @@ static int copy_refs_attr(hid_t loc_in, if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) goto error; if (nelmts) { - if(H5Awrite(attr_out,mtype_id,refbuf)<0) + if(H5Awrite(attr_out,wtype_id,refbuf)<0) goto error; } @@ -610,7 +614,7 @@ static int copy_refs_attr(hid_t loc_in, * dataset region references *------------------------------------------------------------------------- */ - else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) + else if (H5Tequal(wtype_id, H5T_STD_REF_DSETREG)) { H5G_obj_t obj_type; hid_t refobj_id; @@ -631,7 +635,7 @@ static int copy_refs_attr(hid_t loc_in, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Aread(attr_id,mtype_id,buf)<0) + if (H5Aread(attr_id,wtype_id,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(attr_id,H5R_DATASET_REGION,buf))<0) goto error; @@ -679,7 +683,7 @@ static int copy_refs_attr(hid_t loc_in, if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) goto error; if (nelmts) { - if(H5Awrite(attr_out,mtype_id,refbuf)<0) + if(H5Awrite(attr_out,wtype_id,refbuf)<0) goto error; } if (H5Aclose(attr_out)<0) @@ -696,7 +700,7 @@ static int copy_refs_attr(hid_t loc_in, */ if (H5Tclose(ftype_id)<0) goto error; - if (H5Tclose(mtype_id)<0) goto error; + if (H5Tclose(wtype_id)<0) goto error; if (H5Sclose(space_id)<0) goto error; if (H5Aclose(attr_id)<0) goto error; } /* u */ @@ -706,7 +710,7 @@ static int copy_refs_attr(hid_t loc_in, error: H5E_BEGIN_TRY { H5Tclose(ftype_id); - H5Tclose(mtype_id); + H5Tclose(wtype_id); H5Sclose(space_id); H5Aclose(attr_id); H5Aclose(attr_out); @@ -715,9 +719,9 @@ error: } /*------------------------------------------------------------------------- - * Function: close_obj + * Function: close_obj * - * Purpose: Auxiliary function to close an object + * Purpose: Auxiliary function to close an object * *------------------------------------------------------------------------- */ @@ -744,9 +748,9 @@ static void close_obj(H5G_obj_t obj_type, hid_t obj_id) } /*------------------------------------------------------------------------- - * Function: MapIdToName + * Function: MapIdToName * - * Purpose: map an object ID to a name + * Purpose: map an object ID to a name * *------------------------------------------------------------------------- */ diff --git a/tools/h5repack/testh5repack.h b/tools/h5repack/testh5repack.h index c367009..dab2a8b 100644 --- a/tools/h5repack/testh5repack.h +++ b/tools/h5repack/testh5repack.h @@ -49,6 +49,8 @@ #define FNAME12OUT "test_nbitout.h5" #define FNAME13 "test_scaleoffset.h5" #define FNAME13OUT "test_scaleoffsetout.h5" +#define FNAME14 "test_big.h5" +#define FNAME14OUT "test_bigout.h5" int make_testfiles(void); diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c index 9ecef36..3692b3e 100644 --- a/tools/h5repack/testh5repack_main.c +++ b/tools/h5repack/testh5repack_main.c @@ -1171,6 +1171,25 @@ if (szip_can_encode) { #endif /*------------------------------------------------------------------------- + * test a big file + *------------------------------------------------------------------------- + */ + + TESTING(" big file"); + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack(FNAME14,FNAME14OUT,&pack_options) < 0) + TEST_ERROR; + if (h5diff(FNAME14,FNAME14OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME14OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + +/*------------------------------------------------------------------------- * end *------------------------------------------------------------------------- */ diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c index cc4b47d..319af3d 100644 --- a/tools/h5repack/testh5repack_make.c +++ b/tools/h5repack/testh5repack_make.c @@ -17,11 +17,14 @@ #include "h5repack.h" #include "testh5repack.h" -#define DIM1 40 -#define DIM2 20 -#define CDIM1 DIM1/2 -#define CDIM2 DIM2/2 -#define RANK 2 +#define DIM1 40 +#define DIM2 20 +#define CDIM1 DIM1/2 +#define CDIM2 DIM2/2 +#define RANK 2 +#define GBLL ((unsigned long_long) 1024*1024*1024) + + int make_all_objects(hid_t loc_id); int make_attributes(hid_t loc_id); @@ -38,6 +41,8 @@ int make_nbit(hid_t loc_id); int make_scaleoffset(hid_t loc_id); int make_all(hid_t loc_id); int make_fill(hid_t loc_id); +int make_big(hid_t loc_id); + /*------------------------------------------------------------------------- @@ -161,6 +166,17 @@ int make_testfiles(void) return -1; /*------------------------------------------------------------------------- + * create a file with all the filters + *------------------------------------------------------------------------- + */ + if((loc_id = H5Fcreate(FNAME11,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + if (make_all(loc_id)<0) + goto out; + if(H5Fclose(loc_id)<0) + return -1; + +/*------------------------------------------------------------------------- * create a file with the nbit filter *------------------------------------------------------------------------- */ @@ -183,18 +199,17 @@ int make_testfiles(void) return -1; /*------------------------------------------------------------------------- - * create a file with all the filters + * create a big file *------------------------------------------------------------------------- */ - if((loc_id = H5Fcreate(FNAME11,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + if((loc_id = H5Fcreate(FNAME14,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) return -1; - if (make_all(loc_id)<0) + if (make_big(loc_id)<0) goto out; if(H5Fclose(loc_id)<0) return -1; - - PASSED(); + return 0; out: @@ -737,13 +752,15 @@ int make_nbit(hid_t loc_id) #endif /*------------------------------------------------------------------------- - * close space and dcpl + * close *------------------------------------------------------------------------- */ if(H5Sclose(sid)<0) goto out; if(H5Pclose(dcpl)<0) goto out; + if (H5Tclose(dtid)<0) + goto out; return 0; @@ -1226,10 +1243,6 @@ out: return -1; } - - - - /*------------------------------------------------------------------------- * Function: make a file with an integer dataset with a fill value * @@ -1250,17 +1263,128 @@ int make_fill(hid_t loc_id) * H5T_INTEGER, write a fill value *------------------------------------------------------------------------- */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + goto out; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillvalue)<0) + goto out; + if ((sid = H5Screate_simple(2,dims,NULL))<0) + goto out; + if ((did = H5Dcreate(loc_id,"dset_fill",H5T_NATIVE_INT,sid,dcpl))<0) + goto out; + if (H5Dwrite(did,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto out; - dcpl = H5Pcreate(H5P_DATASET_CREATE); - H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillvalue); - sid = H5Screate_simple(2,dims,NULL); - did = H5Dcreate(loc_id,"dset_fill",H5T_NATIVE_INT,sid,dcpl); - H5Dwrite(did,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf); - H5Pclose(dcpl); - H5Dclose(did); - H5Sclose(sid); + /* close */ + if(H5Sclose(sid)<0) + goto out; + if(H5Pclose(dcpl)<0) + goto out; + if(H5Dclose(did)<0) + goto out; return 0; +out: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(sid); + H5Dclose(did); + } H5E_END_TRY; + return -1; + +} + +/*------------------------------------------------------------------------- + * Function: make a big file + * + * Purpose: used in test read by hyperslabs. writes a 1GB file by iterating + * trough 1MB hyperslabs + * + *------------------------------------------------------------------------- + */ +int make_big(hid_t loc_id) +{ + hid_t did; + hid_t f_sid; + hid_t m_sid; + hid_t tid; + hid_t dcpl; + hsize_t dims[1]={GBLL}; /* dataset dimensions */ + hsize_t hs_size[1]={GBLL/1024}; /* hyperslab dimensions */ + hsize_t chunk_dims[1]={hs_size[0]}; /* chunk dimensions */ + hsize_t hs_start[1]; + size_t size; + size_t nelmts=(size_t)hs_size[0]; + char fillvalue=-1; + char *buf=NULL; + int i, j, s; + char c; + + /* create */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + goto out; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_CHAR, &fillvalue)<0) + goto out; + if(H5Pset_chunk(dcpl, 1, chunk_dims)<0) + goto out; + if ((f_sid = H5Screate_simple(1,dims,NULL))<0) + goto out; + if ((did = H5Dcreate(loc_id,"big",H5T_NATIVE_CHAR,f_sid,dcpl))<0) + goto out; + if ((m_sid = H5Screate_simple(1, hs_size, hs_size))<0) + goto out; + if ((tid = H5Dget_type(did))<0) + goto out; + if ((size = H5Tget_size(tid))<=0) + goto out; + + /* create a evenly divided buffer from 0 to 127 */ + buf=(char *) HDmalloc((unsigned)(nelmts*size)); + s = 1024 / 127; + for (i=0, j=0, c=0; i<1024; j++, i++) + { + if ( j==s) + { + c++; + j=0; + }; + + HDmemset(buf, c, nelmts); + + hs_start[0] = i * GBLL/1024; + if (H5Sselect_hyperslab (f_sid,H5S_SELECT_SET,hs_start,NULL,hs_size, NULL)<0) + goto out; + if (H5Dwrite (did,H5T_NATIVE_CHAR,m_sid,f_sid,H5P_DEFAULT,buf)<0) + goto out; + + /* write only one hyperslab, for space considerations */ + if (i==0) + break; + + } + free(buf); + buf=NULL; + + /* close */ + if(H5Sclose(f_sid)<0) + goto out; + if(H5Sclose(m_sid)<0) + goto out; + if(H5Pclose(dcpl)<0) + goto out; + if(H5Dclose(did)<0) + goto out; + + return 0; + +out: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(f_sid); + H5Sclose(m_sid); + H5Dclose(did); + } H5E_END_TRY; + return -1; + } |