diff options
| -rw-r--r-- | MANIFEST | 1 | ||||
| -rw-r--r-- | tools/h5repack/h5repack.h | 5 | ||||
| -rwxr-xr-x | tools/h5repack/h5repack.sh.in | 4 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_copy.c | 352 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_main.c | 10 | ||||
| -rw-r--r-- | tools/h5repack/h5repack_refs.c | 74 | ||||
| -rw-r--r-- | tools/h5repack/testh5repack.h | 2 | ||||
| -rw-r--r-- | tools/h5repack/testh5repack_main.c | 19 | ||||
| -rw-r--r-- | tools/h5repack/testh5repack_make.c | 170 | ||||
| -rw-r--r-- | tools/lib/h5diff_dset.c | 336 | ||||
| -rw-r--r-- | tools/lib/h5tools.c | 11 | ||||
| -rw-r--r-- | tools/lib/h5tools.h | 17 | ||||
| -rw-r--r-- | tools/testfiles/test_big.h5 | bin | 0 -> 1052720 bytes |
13 files changed, 710 insertions, 291 deletions
@@ -1140,6 +1140,7 @@ ./tools/testfiles/tbin2.ddl ./tools/testfiles/tbin3.ddl ./tools/testfiles/tbin4.ddl +./tools/testfiles/test_big.h5 # Expected output from h5ls tests diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h index 29909b7..da82a05 100644 --- a/tools/h5repack/h5repack.h +++ b/tools/h5repack/h5repack.h @@ -99,8 +99,9 @@ typedef struct { filter_info_t filter_g; /*global filter INFO for the ALL case */ chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ H5D_layout_t layout_g; /*global layout information for the ALL case */ - int verbose; /*verbose mode */ - hsize_t threshold; /*minimum size to compress, in bytes */ + int verbose; /*verbose mode */ + hsize_t threshold; /*minimum size to compress, in bytes */ + int use_native; /*use a native type in write */ } pack_opt_t; diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index e2b770b..5769783 100755 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -409,6 +409,10 @@ TOOLTEST $arg7 TOOLTEST $arg8 TOOLTEST $arg9 +#native option +arg="test1.h5 -n" +TOOLTEST $arg + if test $nerrors -eq 0 ; then echo "All $H5REPACK tests passed." diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index b0677ef..7c530ab 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -16,13 +16,15 @@ #include <string.h> #include <stdio.h> #include "H5private.h" +#include "h5tools.h" #include "h5tools_utils.h" #include "h5repack.h" + extern char *progname; -#define PER(A,B) { per = 0; \ +#define PER(A,B) { per = 0; \ if (A!=0) \ per = (float)fabs(1-( (float)B / (float)A )); \ } @@ -254,8 +256,16 @@ out: * Programmer: Pedro Vicente, pvn@ncsa.uiuc.edu * * Date: October, 23, 2003 - * Modified: December, 03, 2004 - added a check for H5Dcreate; if the dataset - * cannot be created with the requested filter, use the input one + * + * Modifications: + * + * July 2004: Introduced the extra EC or NN option for SZIP + * + * December 2004: Added a check for H5Dcreate; if the dataset cannot be created + * with the requested filter, use the input one + * + * October 2006: Read/write using the file type by default. + * Read/write by hyperslabs for big datasets. * *------------------------------------------------------------------------- */ @@ -265,31 +275,30 @@ int do_copy_objects(hid_t fidin, trav_table_t *travt, pack_opt_t *options) /* repack options */ { - hid_t grp_in=-1; /* group ID */ - hid_t grp_out=-1; /* group ID */ - hid_t dset_in=-1; /* read dataset ID */ - hid_t dset_out=-1; /* write dataset ID */ - hid_t type_in=-1; /* named type ID */ - hid_t type_out=-1; /* named type ID */ - hid_t dcpl_id=-1; /* dataset creation property list ID */ - hid_t dcpl_out=-1; /* dataset creation property list ID */ - hid_t space_id=-1; /* space ID */ - hid_t ftype_id=-1; /* file data type ID */ - hid_t mtype_id=-1; /* memory data type ID */ - size_t msize; /* memory size of memory type */ - void *buf=NULL; /* data buffer */ - hsize_t nelmts; /* number of elements in dataset */ - int rank; /* rank of dataset */ - hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ - hsize_t dsize_in; /* input dataset size before filter */ - hsize_t dsize_out; /* output dataset size after filter */ - int next; /* external files */ - int apply_s; /* flag for apply filter to small dataset sizes */ - int apply_f; /* flag for apply filter to return error on H5Dcreate with filter */ - float per; /* percent utilization of storage */ - int i, j; - - + hid_t grp_in; /* group ID */ + hid_t grp_out; /* group ID */ + hid_t dset_in; /* read dataset ID */ + hid_t dset_out; /* write dataset ID */ + hid_t type_in; /* named type ID */ + hid_t type_out; /* named type ID */ + hid_t dcpl_id; /* dataset creation property list ID */ + hid_t dcpl_out; /* dataset creation property list ID */ + hid_t f_space_id; /* file space ID */ + hid_t ftype_id; /* file type ID */ + hid_t wtype_id; /* read/write type ID */ + size_t msize; /* size of type */ + hsize_t nelmts; /* number of elements in dataset */ + int rank; /* rank of dataset */ + hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ + hsize_t dsize_in; /* input dataset size before filter */ + hsize_t dsize_out; /* output dataset size after filter */ + int next; /* external files */ + int apply_s; /* flag for apply filter to small dataset sizes */ + int apply_f; /* flag for apply filter to return error on H5Dcreate */ + float per; /* percent utilization of storage */ + void *buf=NULL; /* buffer for raw data */ + void *sm_buf=NULL; /* buffer for raw data */ + int i, j; /*------------------------------------------------------------------------- * copy the suppplied object list @@ -305,7 +314,7 @@ int do_copy_objects(hid_t fidin, for ( i = 0; i < travt->nobjs; i++) { - buf=NULL; + buf = NULL; switch ( travt->objs[i].type ) { /*------------------------------------------------------------------------- @@ -345,27 +354,29 @@ int do_copy_objects(hid_t fidin, if ((dset_in=H5Dopen(fidin,travt->objs[i].name))<0) goto error; - if ((space_id=H5Dget_space(dset_in))<0) + if ((f_space_id=H5Dget_space(dset_in))<0) goto error; if ((ftype_id=H5Dget_type (dset_in))<0) goto error; if ((dcpl_id=H5Dget_create_plist(dset_in))<0) goto error; - if ((dcpl_out = H5Pcopy (dcpl_id))<0) - goto error; - if ( (rank=H5Sget_simple_extent_ndims(space_id))<0) + if ((dcpl_out = H5Pcopy (dcpl_id))<0) + goto error; + if ( (rank=H5Sget_simple_extent_ndims(f_space_id))<0) goto error; HDmemset(dims, 0, sizeof dims); - if ( H5Sget_simple_extent_dims(space_id,dims,NULL)<0) + if ( H5Sget_simple_extent_dims(f_space_id,dims,NULL)<0) goto error; nelmts=1; for (j=0; j<rank; j++) nelmts*=dims[j]; - if ((mtype_id=h5tools_get_native_type(ftype_id))<0) - goto error; + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - if ((msize=H5Tget_size(mtype_id))==0) + if ((msize=H5Tget_size(wtype_id))==0) goto error; /*------------------------------------------------------------------------- @@ -397,79 +408,160 @@ int do_copy_objects(hid_t fidin, * in a second traversal of the output file *------------------------------------------------------------------------- */ - if ( (H5T_REFERENCE!=H5Tget_class(mtype_id))) + if ( (H5T_REFERENCE!=H5Tget_class(wtype_id))) { /* get the storage size of the input dataset */ dsize_in=H5Dget_storage_size(dset_in); + /* check for datasets too small */ + if (nelmts*msize < options->threshold ) + apply_s=0; + + /* apply the filter */ + if (apply_s){ + if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0) + goto error; + } + + /*------------------------------------------------------------------------- + * create the output dataset; + * disable error checking in case the dataset cannot be created with the + * modified dcpl; in that case use the original instead + *------------------------------------------------------------------------- + */ + H5E_BEGIN_TRY { + dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_out); + } H5E_END_TRY; + if (dset_out==FAIL) + { + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,f_space_id,dcpl_id))<0) + goto error; + apply_f=0; + } + /*------------------------------------------------------------------------- - * read + * read/write *------------------------------------------------------------------------- */ if (nelmts) { - buf=(void *) HDmalloc((unsigned)(nelmts*msize)); - if ( buf==NULL){ - error_msg(progname, "cannot read into memory\n" ); - goto error; - } - if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - - /*------------------------------------------------------------------------- - * check for datasets too small - *------------------------------------------------------------------------- - */ + size_t need = (size_t)(nelmts*msize); /* bytes needed */ + if ( need < H5TOOLS_MALLOCSIZE ) + buf = HDmalloc(need); - if (nelmts*msize < options->threshold ) - apply_s=0; - - /*------------------------------------------------------------------------- - * apply the filter - *------------------------------------------------------------------------- - */ - if (apply_s){ - if (apply_filters(travt->objs[i].name,rank,dims,dcpl_out,options)<0) + if (buf != NULL ) + { + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; - } - + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto error; + } + + else /* possibly not enough memory, read/write by hyperslabs */ + + { + size_t p_type_nbytes = msize; /*size of memory type */ + hsize_t p_nelmts = nelmts; /*total selected elmts */ + hsize_t elmtno; /*counter */ + int carry; /*counter carry value */ + unsigned int vl_data = 0; /*contains VL datatypes */ + + /* stripmine info */ + hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ + hsize_t sm_nbytes; /*bytes per stripmine */ + hsize_t sm_nelmts; /*elements per stripmine*/ + hid_t sm_space; /*stripmine data space */ + + /* hyperslab info */ + hsize_t hs_offset[H5S_MAX_RANK];/*starting offset */ + hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ + hsize_t hs_nelmts; /*elements in request */ + hsize_t zero[8]; /*vector of zeros */ + + /* check if we have VL data in the dataset's datatype */ + if (H5Tdetect_class(wtype_id, H5T_VLEN) == TRUE) + vl_data = TRUE; + + /* + * determine the strip mine size and allocate a buffer. The strip mine is + * a hyperslab whose size is manageable. + */ + sm_nbytes = p_type_nbytes; + + for (i = rank; i > 0; --i) { + sm_size[i - 1] = MIN(dims[i - 1], H5TOOLS_BUFSIZE / sm_nbytes); + sm_nbytes *= sm_size[i - 1]; + assert(sm_nbytes > 0); + } + sm_buf = HDmalloc((size_t)sm_nbytes); + + sm_nelmts = sm_nbytes / p_type_nbytes; + sm_space = H5Screate_simple(1, &sm_nelmts, NULL); + + /* the stripmine loop */ + memset(hs_offset, 0, sizeof hs_offset); + memset(zero, 0, sizeof zero); + + for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) + { + /* calculate the hyperslab size */ + if (rank > 0) + { + for (i = 0, hs_nelmts = 1; i < rank; i++) + { + hs_size[i] = MIN(dims[i] - hs_offset[i], sm_size[i]); + hs_nelmts *= hs_size[i]; + } + + if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0) + goto error; + if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0) + goto error; + } + else + { + H5Sselect_all(f_space_id); + H5Sselect_all(sm_space); + hs_nelmts = 1; + } /* rank */ + + /* read/write */ + if (H5Dread(dset_in, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) + goto error; + if (H5Dwrite(dset_out, wtype_id, sm_space, f_space_id, H5P_DEFAULT, sm_buf) < 0) + goto error; + + /* reclaim any VL memory, if necessary */ + if(vl_data) + H5Dvlen_reclaim(wtype_id, sm_space, H5P_DEFAULT, sm_buf); + + /* calculate the next hyperslab offset */ + for (i = rank, carry = 1; i > 0 && carry; --i) + { + hs_offset[i - 1] += hs_size[i - 1]; + if (hs_offset[i - 1] == dims[i - 1]) + hs_offset[i - 1] = 0; + else + carry = 0; + } /* i */ + } /* elmtno */ + + H5Sclose(sm_space); + /* free */ + if (sm_buf!=NULL) + { + HDfree(sm_buf); + sm_buf=NULL; + } + } /* hyperslab read */ }/*nelmts*/ - /*------------------------------------------------------------------------- - * create; - * disable error checking in case the dataset cannot be created with the - * modified dcpl; in that case use the original instead - *------------------------------------------------------------------------- - */ - - H5E_BEGIN_TRY { - dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_out); - } H5E_END_TRY; - - - if (dset_out==FAIL) - { - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) - goto error; - - apply_f=0; - } - - /*------------------------------------------------------------------------- - * write dataset - *------------------------------------------------------------------------- - */ - - if (dsize_in && nelmts) { - if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) - goto error; - } - /*------------------------------------------------------------------------- * amount of compression used *------------------------------------------------------------------------- */ - if (options->verbose) { + if (options->verbose) + { if (apply_s && apply_f) { /* get the storage size of the input dataset */ @@ -487,7 +579,7 @@ int do_copy_objects(hid_t fidin, (int)options->threshold); if (apply_f==0 && options->verbose) - printf(" <warning: could not apply the filter to %s>\n", + printf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name); /*------------------------------------------------------------------------- @@ -500,11 +592,8 @@ int do_copy_objects(hid_t fidin, /*close */ if (H5Dclose(dset_out)<0) goto error; - - if (buf) - free(buf); - - }/*H5T_STD_REF_OBJ*/ + + }/*H5T_STD_REF_OBJ*/ }/*can_read*/ @@ -514,13 +603,13 @@ int do_copy_objects(hid_t fidin, */ if (H5Tclose(ftype_id)<0) goto error; - if (H5Tclose(mtype_id)<0) + if (H5Tclose(wtype_id)<0) goto error; if (H5Pclose(dcpl_id)<0) goto error; - if (H5Pclose(dcpl_out)<0) + if (H5Pclose(dcpl_out)<0) goto error; - if (H5Sclose(space_id)<0) + if (H5Sclose(f_space_id)<0) goto error; if (H5Dclose(dset_in)<0) goto error; @@ -581,18 +670,18 @@ int do_copy_objects(hid_t fidin, } break; -/*------------------------------------------------------------------------- - *------------------------------------------------------------------------- - */ - { - - } - break; - default: goto error; + } /* switch */ + + /* free */ + if (buf!=NULL) + { + HDfree(buf); + buf=NULL; } - } + + } /* i */ /*------------------------------------------------------------------------- * the root is a special case, we get an ID for the root group @@ -623,15 +712,24 @@ error: H5Gclose(grp_in); H5Gclose(grp_out); H5Pclose(dcpl_id); - H5Sclose(space_id); + H5Sclose(f_space_id); H5Dclose(dset_in); H5Dclose(dset_out); H5Tclose(ftype_id); - H5Tclose(mtype_id); + H5Tclose(wtype_id); H5Tclose(type_in); H5Tclose(type_out); - if (buf) - free(buf); + /* free */ + if (buf!=NULL) + { + HDfree(buf); + buf=NULL; + } + if (sm_buf!=NULL) + { + HDfree(sm_buf); + sm_buf=NULL; + } } H5E_END_TRY; return -1; @@ -663,9 +761,9 @@ int copy_attr(hid_t loc_in, hid_t attr_id; /* attr ID */ hid_t attr_out; /* attr ID */ hid_t space_id; /* space ID */ - hid_t ftype_id; /* file data type ID */ - hid_t mtype_id; /* memory data type ID */ - size_t msize; /* memory size of type */ + hid_t ftype_id; /* file type ID */ + hid_t wtype_id; /* read/write type ID */ + size_t msize; /* size of type */ void *buf=NULL; /* data buffer */ hsize_t nelmts; /* number of elements in dataset */ int rank; /* rank of dataset */ @@ -712,10 +810,12 @@ int copy_attr(hid_t loc_in, for (j=0; j<rank; j++) nelmts*=dims[j]; - if ((mtype_id=h5tools_get_native_type(ftype_id))<0) - goto error; + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - if ((msize=H5Tget_size(mtype_id))==0) + if ((msize=H5Tget_size(wtype_id))==0) goto error; /*------------------------------------------------------------------------- @@ -725,7 +825,7 @@ int copy_attr(hid_t loc_in, * the referenced objects *------------------------------------------------------------------------- */ - if ( ! H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + if ( ! H5Tequal(wtype_id, H5T_STD_REF_OBJ)) { @@ -739,7 +839,7 @@ int copy_attr(hid_t loc_in, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Aread(attr_id,mtype_id,buf)<0) + if (H5Aread(attr_id,wtype_id,buf)<0) goto error; /*------------------------------------------------------------------------- @@ -749,7 +849,7 @@ int copy_attr(hid_t loc_in, if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) goto error; - if(H5Awrite(attr_out,mtype_id,buf)<0) + if(H5Awrite(attr_out,wtype_id,buf)<0) goto error; /*close*/ @@ -773,7 +873,7 @@ int copy_attr(hid_t loc_in, */ if (H5Tclose(ftype_id)<0) goto error; - if (H5Tclose(mtype_id)<0) goto error; + if (H5Tclose(wtype_id)<0) goto error; if (H5Sclose(space_id)<0) goto error; if (H5Aclose(attr_id)<0) goto error; @@ -784,7 +884,7 @@ int copy_attr(hid_t loc_in, error: H5E_BEGIN_TRY { H5Tclose(ftype_id); - H5Tclose(mtype_id); + H5Tclose(wtype_id); H5Sclose(space_id); H5Aclose(attr_id); H5Aclose(attr_out); diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c index 074f937..5e26242 100644 --- a/tools/h5repack/h5repack_main.c +++ b/tools/h5repack/h5repack_main.c @@ -35,8 +35,10 @@ static void usage(void); * * Comments: * - * Modifications: July 2004 - * Introduced the extra EC or NN option for SZIP + * Modifications: + * July 2004: Introduced the extra EC or NN option for SZIP + * October 2006: Added a new switch -n, that allows to write the dataset + * using a native type. The default to write is the file type. * *------------------------------------------------------------------------- */ @@ -104,6 +106,9 @@ int main(int argc, char **argv) else if (strcmp(argv[i], "-e") == 0) { read_info(argv[++i],&options); } + else if (strcmp(argv[i], "-n") == 0) { + options.use_native = 1; + } else if (argv[i][0] == '-') { error_msg(progname, " - is not a valid argument\n"); @@ -194,6 +199,7 @@ void usage(void) printf(" <dim_1 x dim_2 x ... dim_n>\n"); printf("\n"); printf("-e file File with the -f and -l options (only filter and layout flags)\n"); + printf("-n Use a native type when repacking. Default is the file type\n"); printf("-m size Do not apply the filter to objects which size in bytes\n"); printf(" is smaller than number. If no size is specified a minimum of\n"); printf(" 1024 bytes is assumed.\n"); diff --git a/tools/h5repack/h5repack_refs.c b/tools/h5repack/h5repack_refs.c index 1f7c2e9..2a01f9e 100644 --- a/tools/h5repack/h5repack_refs.c +++ b/tools/h5repack/h5repack_refs.c @@ -61,9 +61,9 @@ int do_copy_refobjs(hid_t fidin, hid_t type_in; /* named type ID */ hid_t dcpl_id; /* dataset creation property list ID */ hid_t space_id; /* space ID */ - hid_t ftype_id; /* file data type ID */ - hid_t mtype_id; /* memory data type ID */ - size_t msize; /* memory size of memory type */ + hid_t ftype_id; /* file type ID */ + hid_t wtype_id; /* read/write type ID */ + size_t msize; /* size of type */ hsize_t nelmts; /* number of elements in dataset */ int rank; /* rank of dataset */ hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ @@ -124,11 +124,13 @@ int do_copy_refobjs(hid_t fidin, nelmts=1; for (j=0; j<rank; j++) nelmts*=dims[j]; + + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - if ((mtype_id=h5tools_get_native_type(ftype_id))<0) - goto error; - - if ((msize=H5Tget_size(mtype_id))==0) + if ((msize=H5Tget_size(wtype_id))==0) goto error; /*------------------------------------------------------------------------- @@ -157,7 +159,7 @@ int do_copy_refobjs(hid_t fidin, * we cannot just copy the buffers, but instead we recreate the reference *------------------------------------------------------------------------- */ - if (H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + if (H5Tequal(wtype_id, H5T_STD_REF_OBJ)) { H5G_obj_t obj_type; hid_t refobj_id; @@ -178,7 +180,7 @@ int do_copy_refobjs(hid_t fidin, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(dset_in,H5R_OBJECT,buf))<0) @@ -214,10 +216,10 @@ int do_copy_refobjs(hid_t fidin, * create/write dataset/close *------------------------------------------------------------------------- */ - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,space_id,dcpl_id))<0) goto error; if (nelmts) { - if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) goto error; } @@ -232,7 +234,7 @@ int do_copy_refobjs(hid_t fidin, * dataset region references *------------------------------------------------------------------------- */ - else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) + else if (H5Tequal(wtype_id, H5T_STD_REF_DSETREG)) { H5G_obj_t obj_type; hid_t refobj_id; @@ -252,7 +254,7 @@ int do_copy_refobjs(hid_t fidin, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Dread(dset_in,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + if (H5Dread(dset_in,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(dset_in,H5R_DATASET_REGION,buf))<0) goto error; @@ -299,10 +301,10 @@ int do_copy_refobjs(hid_t fidin, * create/write dataset/close *------------------------------------------------------------------------- */ - if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,mtype_id,space_id,dcpl_id))<0) + if ((dset_out=H5Dcreate(fidout,travt->objs[i].name,wtype_id,space_id,dcpl_id))<0) goto error; if (nelmts) { - if (H5Dwrite(dset_out,mtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) + if (H5Dwrite(dset_out,wtype_id,H5S_ALL,H5S_ALL,H5P_DEFAULT,refbuf)<0) goto error; } @@ -359,7 +361,7 @@ int do_copy_refobjs(hid_t fidin, if (H5Tclose(ftype_id)<0) goto error; - if (H5Tclose(mtype_id)<0) + if (H5Tclose(wtype_id)<0) goto error; if (H5Pclose(dcpl_id)<0) goto error; @@ -437,7 +439,7 @@ error: H5Dclose(dset_in); H5Dclose(dset_out); H5Tclose(ftype_id); - H5Tclose(mtype_id); + H5Tclose(wtype_id); H5Tclose(type_in); } H5E_END_TRY; return -1; @@ -473,9 +475,9 @@ static int copy_refs_attr(hid_t loc_in, hid_t attr_id; /* attr ID */ hid_t attr_out; /* attr ID */ hid_t space_id; /* space ID */ - hid_t ftype_id; /* file data type ID */ - hid_t mtype_id; /* memory data type ID */ - size_t msize; /* memory size of type */ + hid_t ftype_id; /* file type ID */ + hid_t wtype_id; /* read/write type ID */ + size_t msize; /* size of type */ hsize_t nelmts; /* number of elements in dataset */ int rank; /* rank of dataset */ hsize_t dims[H5S_MAX_RANK];/* dimensions of dataset */ @@ -522,10 +524,12 @@ static int copy_refs_attr(hid_t loc_in, for (j=0; j<rank; j++) nelmts*=dims[j]; - if ((mtype_id=h5tools_get_native_type(ftype_id))<0) - goto error; + if (options->use_native==1) + wtype_id = h5tools_get_native_type(ftype_id); + else + wtype_id = H5Tcopy(ftype_id); - if ((msize=H5Tget_size(mtype_id))==0) + if ((msize=H5Tget_size(wtype_id))==0) goto error; /*------------------------------------------------------------------------- @@ -533,7 +537,7 @@ static int copy_refs_attr(hid_t loc_in, * we cannot just copy the buffers, but instead we recreate the reference *------------------------------------------------------------------------- */ - if (H5Tequal(mtype_id, H5T_STD_REF_OBJ)) + if (H5Tequal(wtype_id, H5T_STD_REF_OBJ)) { H5G_obj_t obj_type; hid_t refobj_id; @@ -554,7 +558,7 @@ static int copy_refs_attr(hid_t loc_in, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Aread(attr_id,mtype_id,buf)<0) + if (H5Aread(attr_id,wtype_id,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(attr_id,H5R_OBJECT,buf))<0) @@ -592,7 +596,7 @@ static int copy_refs_attr(hid_t loc_in, if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) goto error; if (nelmts) { - if(H5Awrite(attr_out,mtype_id,refbuf)<0) + if(H5Awrite(attr_out,wtype_id,refbuf)<0) goto error; } @@ -610,7 +614,7 @@ static int copy_refs_attr(hid_t loc_in, * dataset region references *------------------------------------------------------------------------- */ - else if (H5Tequal(mtype_id, H5T_STD_REF_DSETREG)) + else if (H5Tequal(wtype_id, H5T_STD_REF_DSETREG)) { H5G_obj_t obj_type; hid_t refobj_id; @@ -631,7 +635,7 @@ static int copy_refs_attr(hid_t loc_in, error_msg(progname, "cannot read into memory\n" ); goto error; } - if (H5Aread(attr_id,mtype_id,buf)<0) + if (H5Aread(attr_id,wtype_id,buf)<0) goto error; if ((obj_type = H5Rget_obj_type(attr_id,H5R_DATASET_REGION,buf))<0) goto error; @@ -679,7 +683,7 @@ static int copy_refs_attr(hid_t loc_in, if ((attr_out=H5Acreate(loc_out,name,ftype_id,space_id,H5P_DEFAULT))<0) goto error; if (nelmts) { - if(H5Awrite(attr_out,mtype_id,refbuf)<0) + if(H5Awrite(attr_out,wtype_id,refbuf)<0) goto error; } if (H5Aclose(attr_out)<0) @@ -696,7 +700,7 @@ static int copy_refs_attr(hid_t loc_in, */ if (H5Tclose(ftype_id)<0) goto error; - if (H5Tclose(mtype_id)<0) goto error; + if (H5Tclose(wtype_id)<0) goto error; if (H5Sclose(space_id)<0) goto error; if (H5Aclose(attr_id)<0) goto error; } /* u */ @@ -706,7 +710,7 @@ static int copy_refs_attr(hid_t loc_in, error: H5E_BEGIN_TRY { H5Tclose(ftype_id); - H5Tclose(mtype_id); + H5Tclose(wtype_id); H5Sclose(space_id); H5Aclose(attr_id); H5Aclose(attr_out); @@ -715,9 +719,9 @@ error: } /*------------------------------------------------------------------------- - * Function: close_obj + * Function: close_obj * - * Purpose: Auxiliary function to close an object + * Purpose: Auxiliary function to close an object * *------------------------------------------------------------------------- */ @@ -744,9 +748,9 @@ static void close_obj(H5G_obj_t obj_type, hid_t obj_id) } /*------------------------------------------------------------------------- - * Function: MapIdToName + * Function: MapIdToName * - * Purpose: map an object ID to a name + * Purpose: map an object ID to a name * *------------------------------------------------------------------------- */ diff --git a/tools/h5repack/testh5repack.h b/tools/h5repack/testh5repack.h index c367009..dab2a8b 100644 --- a/tools/h5repack/testh5repack.h +++ b/tools/h5repack/testh5repack.h @@ -49,6 +49,8 @@ #define FNAME12OUT "test_nbitout.h5" #define FNAME13 "test_scaleoffset.h5" #define FNAME13OUT "test_scaleoffsetout.h5" +#define FNAME14 "test_big.h5" +#define FNAME14OUT "test_bigout.h5" int make_testfiles(void); diff --git a/tools/h5repack/testh5repack_main.c b/tools/h5repack/testh5repack_main.c index 9ecef36..3692b3e 100644 --- a/tools/h5repack/testh5repack_main.c +++ b/tools/h5repack/testh5repack_main.c @@ -1171,6 +1171,25 @@ if (szip_can_encode) { #endif /*------------------------------------------------------------------------- + * test a big file + *------------------------------------------------------------------------- + */ + + TESTING(" big file"); + if (h5repack_init (&pack_options, 0)<0) + TEST_ERROR; + if (h5repack(FNAME14,FNAME14OUT,&pack_options) < 0) + TEST_ERROR; + if (h5diff(FNAME14,FNAME14OUT,NULL,NULL,&diff_options) == 1) + TEST_ERROR; + if (h5repack_verify(FNAME14OUT,&pack_options)<=0) + TEST_ERROR; + if (h5repack_end (&pack_options)<0) + TEST_ERROR; + PASSED(); + + +/*------------------------------------------------------------------------- * end *------------------------------------------------------------------------- */ diff --git a/tools/h5repack/testh5repack_make.c b/tools/h5repack/testh5repack_make.c index cc4b47d..319af3d 100644 --- a/tools/h5repack/testh5repack_make.c +++ b/tools/h5repack/testh5repack_make.c @@ -17,11 +17,14 @@ #include "h5repack.h" #include "testh5repack.h" -#define DIM1 40 -#define DIM2 20 -#define CDIM1 DIM1/2 -#define CDIM2 DIM2/2 -#define RANK 2 +#define DIM1 40 +#define DIM2 20 +#define CDIM1 DIM1/2 +#define CDIM2 DIM2/2 +#define RANK 2 +#define GBLL ((unsigned long_long) 1024*1024*1024) + + int make_all_objects(hid_t loc_id); int make_attributes(hid_t loc_id); @@ -38,6 +41,8 @@ int make_nbit(hid_t loc_id); int make_scaleoffset(hid_t loc_id); int make_all(hid_t loc_id); int make_fill(hid_t loc_id); +int make_big(hid_t loc_id); + /*------------------------------------------------------------------------- @@ -161,6 +166,17 @@ int make_testfiles(void) return -1; /*------------------------------------------------------------------------- + * create a file with all the filters + *------------------------------------------------------------------------- + */ + if((loc_id = H5Fcreate(FNAME11,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + return -1; + if (make_all(loc_id)<0) + goto out; + if(H5Fclose(loc_id)<0) + return -1; + +/*------------------------------------------------------------------------- * create a file with the nbit filter *------------------------------------------------------------------------- */ @@ -183,18 +199,17 @@ int make_testfiles(void) return -1; /*------------------------------------------------------------------------- - * create a file with all the filters + * create a big file *------------------------------------------------------------------------- */ - if((loc_id = H5Fcreate(FNAME11,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) + if((loc_id = H5Fcreate(FNAME14,H5F_ACC_TRUNC,H5P_DEFAULT,H5P_DEFAULT))<0) return -1; - if (make_all(loc_id)<0) + if (make_big(loc_id)<0) goto out; if(H5Fclose(loc_id)<0) return -1; - - PASSED(); + return 0; out: @@ -737,13 +752,15 @@ int make_nbit(hid_t loc_id) #endif /*------------------------------------------------------------------------- - * close space and dcpl + * close *------------------------------------------------------------------------- */ if(H5Sclose(sid)<0) goto out; if(H5Pclose(dcpl)<0) goto out; + if (H5Tclose(dtid)<0) + goto out; return 0; @@ -1226,10 +1243,6 @@ out: return -1; } - - - - /*------------------------------------------------------------------------- * Function: make a file with an integer dataset with a fill value * @@ -1250,17 +1263,128 @@ int make_fill(hid_t loc_id) * H5T_INTEGER, write a fill value *------------------------------------------------------------------------- */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + goto out; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillvalue)<0) + goto out; + if ((sid = H5Screate_simple(2,dims,NULL))<0) + goto out; + if ((did = H5Dcreate(loc_id,"dset_fill",H5T_NATIVE_INT,sid,dcpl))<0) + goto out; + if (H5Dwrite(did,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf)<0) + goto out; - dcpl = H5Pcreate(H5P_DATASET_CREATE); - H5Pset_fill_value(dcpl, H5T_NATIVE_INT, &fillvalue); - sid = H5Screate_simple(2,dims,NULL); - did = H5Dcreate(loc_id,"dset_fill",H5T_NATIVE_INT,sid,dcpl); - H5Dwrite(did,H5T_NATIVE_INT,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf); - H5Pclose(dcpl); - H5Dclose(did); - H5Sclose(sid); + /* close */ + if(H5Sclose(sid)<0) + goto out; + if(H5Pclose(dcpl)<0) + goto out; + if(H5Dclose(did)<0) + goto out; return 0; +out: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(sid); + H5Dclose(did); + } H5E_END_TRY; + return -1; + +} + +/*------------------------------------------------------------------------- + * Function: make a big file + * + * Purpose: used in test read by hyperslabs. writes a 1GB file by iterating + * trough 1MB hyperslabs + * + *------------------------------------------------------------------------- + */ +int make_big(hid_t loc_id) +{ + hid_t did; + hid_t f_sid; + hid_t m_sid; + hid_t tid; + hid_t dcpl; + hsize_t dims[1]={GBLL}; /* dataset dimensions */ + hsize_t hs_size[1]={GBLL/1024}; /* hyperslab dimensions */ + hsize_t chunk_dims[1]={hs_size[0]}; /* chunk dimensions */ + hsize_t hs_start[1]; + size_t size; + size_t nelmts=(size_t)hs_size[0]; + char fillvalue=-1; + char *buf=NULL; + int i, j, s; + char c; + + /* create */ + if ((dcpl = H5Pcreate(H5P_DATASET_CREATE))<0) + goto out; + if (H5Pset_fill_value(dcpl, H5T_NATIVE_CHAR, &fillvalue)<0) + goto out; + if(H5Pset_chunk(dcpl, 1, chunk_dims)<0) + goto out; + if ((f_sid = H5Screate_simple(1,dims,NULL))<0) + goto out; + if ((did = H5Dcreate(loc_id,"big",H5T_NATIVE_CHAR,f_sid,dcpl))<0) + goto out; + if ((m_sid = H5Screate_simple(1, hs_size, hs_size))<0) + goto out; + if ((tid = H5Dget_type(did))<0) + goto out; + if ((size = H5Tget_size(tid))<=0) + goto out; + + /* create a evenly divided buffer from 0 to 127 */ + buf=(char *) HDmalloc((unsigned)(nelmts*size)); + s = 1024 / 127; + for (i=0, j=0, c=0; i<1024; j++, i++) + { + if ( j==s) + { + c++; + j=0; + }; + + HDmemset(buf, c, nelmts); + + hs_start[0] = i * GBLL/1024; + if (H5Sselect_hyperslab (f_sid,H5S_SELECT_SET,hs_start,NULL,hs_size, NULL)<0) + goto out; + if (H5Dwrite (did,H5T_NATIVE_CHAR,m_sid,f_sid,H5P_DEFAULT,buf)<0) + goto out; + + /* write only one hyperslab, for space considerations */ + if (i==0) + break; + + } + free(buf); + buf=NULL; + + /* close */ + if(H5Sclose(f_sid)<0) + goto out; + if(H5Sclose(m_sid)<0) + goto out; + if(H5Pclose(dcpl)<0) + goto out; + if(H5Dclose(did)<0) + goto out; + + return 0; + +out: + H5E_BEGIN_TRY { + H5Pclose(dcpl); + H5Sclose(f_sid); + H5Sclose(m_sid); + H5Dclose(did); + } H5E_END_TRY; + return -1; + } diff --git a/tools/lib/h5diff_dset.c b/tools/lib/h5diff_dset.c index 012552f..630b408 100644 --- a/tools/lib/h5diff_dset.c +++ b/tools/lib/h5diff_dset.c @@ -191,33 +191,36 @@ hsize_t diff_datasetid( hid_t did1, const char *obj2_name, diff_opt_t *options ) { - hid_t sid1; - hid_t sid2; - hid_t f_tid1; - hid_t f_tid2; /* file data type */ - hid_t m_tid1; - hid_t m_tid2; /* memory data type */ - size_t m_size1; - size_t m_size2; /* size of type in memory */ - H5T_sign_t sign1; - H5T_sign_t sign2; /* sign of type */ - int rank1; - int rank2; - void *buf1=NULL; - void *buf2=NULL; - hsize_t nelmts1; - hsize_t nelmts2; - hsize_t dims1[H5S_MAX_RANK]; - hsize_t dims2[H5S_MAX_RANK]; - hsize_t maxdim1[H5S_MAX_RANK]; - hsize_t maxdim2[H5S_MAX_RANK]; - const char *name1=NULL; /* relative names */ - const char *name2=NULL; - hsize_t storage_size1; - hsize_t storage_size2; - hsize_t nfound=0; /* number of differences found */ - int cmp=1; /* do diff or not */ - int i; + hid_t sid1; + hid_t sid2; + hid_t f_tid1; + hid_t f_tid2; + hid_t m_tid1; + hid_t m_tid2; + size_t m_size1; + size_t m_size2; + H5T_sign_t sign1; + H5T_sign_t sign2; + int rank1; + int rank2; + hsize_t nelmts1; + hsize_t nelmts2; + hsize_t dims1[H5S_MAX_RANK]; + hsize_t dims2[H5S_MAX_RANK]; + hsize_t maxdim1[H5S_MAX_RANK]; + hsize_t maxdim2[H5S_MAX_RANK]; + const char *name1=NULL; /* relative names */ + const char *name2=NULL; + hsize_t storage_size1; + hsize_t storage_size2; + hsize_t nfound=0; /* number of differences found */ + int cmp=1; /* do diff or not */ + void *buf1=NULL; + void *buf2=NULL; + void *sm_buf1=NULL; + void *sm_buf2=NULL; + size_t need; /* bytes needed for malloc */ + int i; /* Get the dataspace handle */ if ( (sid1 = H5Dget_space(did1)) < 0 ) @@ -272,23 +275,22 @@ hsize_t diff_datasetid( hid_t did1, options->not_cmp=1; } - /*------------------------------------------------------------------------- * check for comparable TYPE and SPACE *------------------------------------------------------------------------- */ if (diff_can_type(f_tid1, - f_tid2, - rank1, - rank2, - dims1, - dims2, - maxdim1, - maxdim2, - obj1_name, - obj2_name, - options)!=1) + f_tid2, + rank1, + rank2, + dims1, + dims2, + maxdim1, + maxdim2, + obj1_name, + obj2_name, + options)!=1) { cmp=0; options->not_cmp=1; @@ -307,11 +309,6 @@ hsize_t diff_datasetid( hid_t did1, m_size1 = H5Tget_size( m_tid1 ); m_size2 = H5Tget_size( m_tid2 ); -#if defined (H5DIFF_DEBUG) - if (obj1_name) - print_sizes(obj1_name,obj2_name,f_tid1,f_tid2,m_tid1,m_tid2); -#endif - /*------------------------------------------------------------------------- * check for different signed/unsigned types *------------------------------------------------------------------------- @@ -353,7 +350,6 @@ hsize_t diff_datasetid( hid_t did1, nelmts2 *= dims2[i]; } - /* only assert if the space is the same */ assert(nelmts1==nelmts2); /*------------------------------------------------------------------------- @@ -381,56 +377,176 @@ hsize_t diff_datasetid( hid_t did1, m_size2 = H5Tget_size( m_tid2 ); } -#if defined (H5DIFF_DEBUG) - printf("WARNING: Size was upgraded\n"); - if (obj1_name) - print_sizes(obj1_name,obj2_name,f_tid1,f_tid2,m_tid1,m_tid2); -#endif } assert(m_size1==m_size2); - - buf1 = (void *) HDmalloc((unsigned) (nelmts1*m_size1)); - buf2 = (void *) HDmalloc((unsigned) (nelmts2*m_size2)); - - if ( buf1 == NULL || buf2 == NULL ) - { - printf( "cannot read into memory\n" ); - goto error; + /* print names */ + if (obj1_name) { + name1=diff_basename(obj1_name); + } + if (obj2_name) { + name2=diff_basename(obj2_name); } + /*------------------------------------------------------------------------- - * read + * read/compare *------------------------------------------------------------------------- */ - if ( H5Dread(did1,m_tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf1) < 0 ) - goto error; - - if ( H5Dread(did2,m_tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf2) < 0 ) - goto error; - -/*------------------------------------------------------------------------- - * array compare - *------------------------------------------------------------------------- - */ - if (obj1_name) { - name1=diff_basename(obj1_name); + need = (size_t)(nelmts1*m_size1); /* bytes needed */ + if ( need < H5TOOLS_MALLOCSIZE ) + { + buf1 = HDmalloc(need); + buf2 = HDmalloc(need); } - if (obj2_name) { - name2=diff_basename(obj2_name); + + if ( buf1!=NULL && buf2!=NULL) + { + if ( H5Dread(did1,m_tid1,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf1) < 0 ) + goto error; + if ( H5Dread(did2,m_tid2,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf2) < 0 ) + goto error; + + /* array diff */ + nfound = diff_array(buf1, + buf2, + nelmts1, + rank1, + dims1, + options, + name1, + name2, + m_tid1, + did1, + did2); } - nfound = diff_array(buf1, - buf2, - nelmts1, - rank1, - dims1, - options, - name1, - name2, - m_tid1, - did1, - did2); + + else /* possibly not enough memory, read/compare by hyperslabs */ + + { + size_t p_type_nbytes = m_size1; /*size of memory type */ + hsize_t p_nelmts = nelmts1; /*total selected elmts */ + hsize_t elmtno; /*counter */ + int carry; /*counter carry value */ + unsigned int vl_data = 0; /*contains VL datatypes */ + + /* stripmine info */ + hsize_t sm_size[H5S_MAX_RANK]; /*stripmine size */ + hsize_t sm_nbytes; /*bytes per stripmine */ + hsize_t sm_nelmts; /*elements per stripmine*/ + hid_t sm_space; /*stripmine data space */ + + /* hyperslab info */ + hsize_t hs_offset[H5S_MAX_RANK]; /*starting offset */ + hsize_t hs_size[H5S_MAX_RANK]; /*size this pass */ + unsigned char *sm_buf1 = NULL; /*buffer for raw data */ + unsigned char *sm_buf2 = NULL; /*buffer for raw data */ + hsize_t hs_nelmts; /*elements in request */ + hsize_t zero[8]; /*vector of zeros */ + + /* check if we have VL data in the dataset's datatype */ + if (H5Tdetect_class(m_tid1, H5T_VLEN) == TRUE) + vl_data = TRUE; + + /* + * determine the strip mine size and allocate a buffer. The strip mine is + * a hyperslab whose size is manageable. + */ + sm_nbytes = p_type_nbytes; + + for (i = rank1; i > 0; --i) { + sm_size[i - 1] = MIN(dims1[i - 1], H5TOOLS_BUFSIZE / sm_nbytes); + sm_nbytes *= sm_size[i - 1]; + assert(sm_nbytes > 0); + } + + sm_buf1 = malloc((size_t)sm_nbytes); + sm_buf2 = malloc((size_t)sm_nbytes); + + sm_nelmts = sm_nbytes / p_type_nbytes; + sm_space = H5Screate_simple(1, &sm_nelmts, NULL); + + /* the stripmine loop */ + memset(hs_offset, 0, sizeof hs_offset); + memset(zero, 0, sizeof zero); + + for (elmtno = 0; elmtno < p_nelmts; elmtno += hs_nelmts) + { + /* calculate the hyperslab size */ + if (rank1 > 0) + { + for (i = 0, hs_nelmts = 1; i < rank1; i++) + { + hs_size[i] = MIN(dims1[i] - hs_offset[i], sm_size[i]); + hs_nelmts *= hs_size[i]; + } + if (H5Sselect_hyperslab(sid1, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0) + goto error; + if (H5Sselect_hyperslab(sid2, H5S_SELECT_SET, hs_offset, NULL, hs_size, NULL)<0) + goto error; + if (H5Sselect_hyperslab(sm_space, H5S_SELECT_SET, zero, NULL, &hs_nelmts, NULL)<0) + goto error; + } + else + { + H5Sselect_all(sid1); + H5Sselect_all(sid2); + H5Sselect_all(sm_space); + hs_nelmts = 1; + } /* rank */ + + if ( H5Dread(did1,m_tid1,sm_space,sid1,H5P_DEFAULT,sm_buf1) < 0 ) + goto error; + if ( H5Dread(did2,m_tid2,sm_space,sid2,H5P_DEFAULT,sm_buf2) < 0 ) + goto error; + + /* array diff */ + nfound = diff_array(sm_buf1, + sm_buf2, + hs_nelmts, + rank1, + dims1, + options, + name1, + name2, + m_tid1, + did1, + did2); + + /* reclaim any VL memory, if necessary */ + if(vl_data) + { + H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf1); + H5Dvlen_reclaim(m_tid1, sm_space, H5P_DEFAULT, sm_buf2); + } + + /* calculate the next hyperslab offset */ + for (i = rank1, carry = 1; i > 0 && carry; --i) + { + hs_offset[i - 1] += hs_size[i - 1]; + if (hs_offset[i - 1] == dims1[i - 1]) + hs_offset[i - 1] = 0; + else + carry = 0; + } /* i */ + } /* elmtno */ + + H5Sclose(sm_space); + /* free */ + if (sm_buf1!=NULL) + { + free(sm_buf1); + sm_buf1=NULL; + } + if (sm_buf2!=NULL) + { + free(sm_buf2); + sm_buf2=NULL; + } + + } /* hyperslab read */ + /*------------------------------------------------------------------------- * compare attributes * the if condition refers to cases when the dataset is a referenced object @@ -446,12 +562,29 @@ hsize_t diff_datasetid( hid_t did1, * close *------------------------------------------------------------------------- */ + + /* free */ + if (buf1!=NULL) + { + free(buf1); + buf1=NULL; + } + if (buf2!=NULL) + { + free(buf2); + buf2=NULL; + } + if (sm_buf1!=NULL) + { + free(sm_buf1); + sm_buf1=NULL; + } + if (sm_buf2!=NULL) + { + free(sm_buf2); + sm_buf2=NULL; + } - - if ( buf1) HDfree(buf1); - if ( buf2) HDfree(buf2); - /* close */ - /* disable error reporting */ H5E_BEGIN_TRY { H5Sclose(sid1); H5Sclose(sid2); @@ -459,16 +592,35 @@ hsize_t diff_datasetid( hid_t did1, H5Tclose(f_tid2); H5Tclose(m_tid1); H5Tclose(m_tid2); - /* enable error reporting */ } H5E_END_TRY; return nfound; error: options->err_stat=1; - if ( buf1) HDfree(buf1); - if ( buf2) HDfree(buf2); - /* close */ + + /* free */ + if (buf1!=NULL) + { + free(buf1); + buf1=NULL; + } + if (buf2!=NULL) + { + free(buf2); + buf2=NULL; + } + if (sm_buf1!=NULL) + { + free(sm_buf1); + sm_buf1=NULL; + } + if (sm_buf2!=NULL) + { + free(sm_buf2); + sm_buf2=NULL; + } + /* disable error reporting */ H5E_BEGIN_TRY { H5Sclose(sid1); diff --git a/tools/lib/h5tools.c b/tools/lib/h5tools.c index 362b070..8279ec9 100644 --- a/tools/lib/h5tools.c +++ b/tools/lib/h5tools.c @@ -29,17 +29,6 @@ #include "h5tools_utils.h" #include "H5private.h" -/* - * The output functions need a temporary buffer to hold a piece of the - * dataset while it's being printed. This constant sets the limit on the - * size of that temporary buffer in bytes. For efficiency's sake, choose the - * largest value suitable for your machine (for testing use a small value). - */ -#if 1 -#define H5TOOLS_BUFSIZE (1024 * 1024) -#else -#define H5TOOLS_BUFSIZE (1024) -#endif /* 1 */ #define ALIGN(A,Z) ((((A) + (Z) - 1) / (Z)) * (Z)) diff --git a/tools/lib/h5tools.h b/tools/lib/h5tools.h index 357154b..70d8734 100644 --- a/tools/lib/h5tools.h +++ b/tools/lib/h5tools.h @@ -30,6 +30,23 @@ #define END_OF_DATA 0x0002 /* + * The output functions need a temporary buffer to hold a piece of the + * dataset while it's being printed. This constant sets the limit on the + * size of that temporary buffer in bytes. For efficiency's sake, choose the + * largest value suitable for your machine (for testing use a small value). + */ +#if 1 +#define H5TOOLS_BUFSIZE (1024 * 1024) +#else +#define H5TOOLS_BUFSIZE (1024) +#endif + +/* + * Maximum size used in a call to malloc + */ +#define H5TOOLS_MALLOCSIZE ((size_t)1024 * 1024 * 1024) + +/* * Information about how to format output. */ typedef struct h5tool_format_t { diff --git a/tools/testfiles/test_big.h5 b/tools/testfiles/test_big.h5 Binary files differnew file mode 100644 index 0000000..b7815ff --- /dev/null +++ b/tools/testfiles/test_big.h5 |