diff options
author | Jonathan Kim <jkm@hdfgroup.org> | 2012-09-26 18:10:01 (GMT) |
---|---|---|
committer | Jonathan Kim <jkm@hdfgroup.org> | 2012-09-26 18:10:01 (GMT) |
commit | 06e9ab4420adb7a6ae04514adcb134d27114e410 (patch) | |
tree | 493a66696596e465c5d780b2cb9854c7efa31781 /tools/h5repack | |
parent | 87271a753ddd14a32959f11a19fb1e44d745daf5 (diff) | |
download | hdf5-06e9ab4420adb7a6ae04514adcb134d27114e410.zip hdf5-06e9ab4420adb7a6ae04514adcb134d27114e410.tar.gz hdf5-06e9ab4420adb7a6ae04514adcb134d27114e410.tar.bz2 |
[svn-r22814] Purpose:
HDFFV-8012 - h5repack changes max dims and cause failure if only "-f none" is used without changing layout for chunked dataset when a chunk dim is bigger than a dataset dim
Description:
"h5repack -f <obj>:NONE <file.h5> out.h5" command failed if source file contains chunked dataset and a chunk dim is bigger than a dataset dim.
Another issue is that the command changed max dims if chunk dim is smaller than the dataset dim.
These issue occurred when dataset size is smaller than 64k (compact size limit)
Fixed them.
Merged from HDF5 trunk r22805
Tested:
jam (linux32-LE), koala (linux64-LE), ostrich (linuxppc64-BE), tejeda (mac32-LE), linew (solaris-BE), Windows (32-LE cmake), cmake (jam)
Diffstat (limited to 'tools/h5repack')
-rw-r--r-- | tools/h5repack/CMakeLists.txt | 22 | ||||
-rw-r--r-- | tools/h5repack/h5repack.c | 1 | ||||
-rw-r--r-- | tools/h5repack/h5repack.sh.in | 21 | ||||
-rw-r--r-- | tools/h5repack/h5repack_copy.c | 20 | ||||
-rw-r--r-- | tools/h5repack/h5repacktst.c | 101 | ||||
-rw-r--r-- | tools/h5repack/testfiles/h5repack_layout3.h5 | bin | 966904 -> 491840 bytes |
6 files changed, 143 insertions, 22 deletions
diff --git a/tools/h5repack/CMakeLists.txt b/tools/h5repack/CMakeLists.txt index 3ee1cba..7351fd5 100644 --- a/tools/h5repack/CMakeLists.txt +++ b/tools/h5repack/CMakeLists.txt @@ -800,6 +800,28 @@ ADD_H5_VERIFY_TEST (chunk2conti "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CONT ADD_H5_TEST (chunk2compa "TEST" h5repack_layout3.h5 -l chunk_unlimit1:COMPA) ADD_H5_VERIFY_TEST (chunk2compa "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CHUNK) +#-------------------------------------------------------------------------- +# Test -f for some specific cases. Chunked dataset with unlimited max dims. +# (HDFFV-8012) +#-------------------------------------------------------------------------- +# - should not fail +# - should not change max dims from unlimit + +# chunk dim is bigger than dataset dim. ( dset size < 64k ) +ADD_H5_TEST (error1 "TEST" h5repack_layout3.h5 -f chunk_unlimit1:NONE) +ADD_H5_VERIFY_TEST (error1 "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED) + +# chunk dim is bigger than dataset dim. ( dset size > 64k ) +ADD_H5_TEST (error2 "TEST" h5repack_layout3.h5 -f chunk_unlimit2:NONE) +ADD_H5_VERIFY_TEST (error2 "TEST" 0 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED) + +# chunk dims are smaller than dataset dims. ( dset size < 64k ) +ADD_H5_TEST (error3 "TEST" h5repack_layout3.h5 -f chunk_unlimit3:NONE) +ADD_H5_VERIFY_TEST (error3 "TEST" 0 h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED) + +# file input - should not fail +ADD_H5_TEST (error4 "TEST" h5repack_layout3.h5 -f NONE) + # Native option # Do not use FILE1, as the named dtype will be converted to native, and h5diff will # report a difference. diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 6517ae1..621cc52 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -93,6 +93,7 @@ int h5repack_init(pack_opt_t *options, HDmemset(options, 0, sizeof(pack_opt_t)); options->min_comp = 1024; options->verbose = verbose; + options->layout_g = H5D_LAYOUT_ERROR; for ( n = 0; n < H5_REPACK_MAX_NFILTERS; n++) { diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index 2614dd6..a732ffb 100644 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -818,6 +818,27 @@ VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CONTI TOOLTEST_MAIN h5repack_layout3.h5 -l chunk_unlimit1:COMPA VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CHUNK +#-------------------------------------------------------------------------- +# Test -f for some specific cases. Chunked dataset with unlimited max dims. +# (HDFFV-8012) +#-------------------------------------------------------------------------- +# - should not fail +# - should not change max dims from unlimit + +# chunk dim is bigger than dataset dim. ( dset size < 64k ) +TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit1:NONE +VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED +# chunk dim is bigger than dataset dim. ( dset size > 64k ) +TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit2:NONE +VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED + +# chunk dims are smaller than dataset dims. ( dset size < 64k ) +TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit3:NONE +VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED + +# file input - should not fail +TOOLTEST h5repack_layout3.h5 -f NONE + # Native option # Do not use FILE1, as the named dtype will be converted to native, and h5diff will # report a difference. diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 94bd565..cfd542a 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -732,6 +732,7 @@ int do_copy_objects(hid_t fidin, void *hslab_buf=NULL; /* hyperslab buffer for raw data */ int has_filter; /* current object has a filter */ int req_filter; /* there was a request for a filter */ + int req_obj_layout=0; /* request layout to current object */ unsigned crt_order_flags; /* group creation order flag */ unsigned i; unsigned u; @@ -863,6 +864,22 @@ int do_copy_objects(hid_t fidin, } } + /* check if layout change requested individual object */ + if (options->layout_g != H5D_LAYOUT_ERROR) + { + pack_info_t *pckinfo; + /* any dataset is specified */ + if (options->op_tbl->nelems > 0) + { + /* check if object exist */ + pckinfo = options_get_object (travt->objs[i].name, options->op_tbl); + if (pckinfo) + { + req_obj_layout = 1; + } + } + } + /* early detection of references */ if((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) goto error; @@ -982,6 +999,9 @@ int do_copy_objects(hid_t fidin, goto error; } + /* only if layout change requested for entire file or + * individual obj */ + if (options->all_layout > 0 || req_obj_layout == 1) /*------------------------------------------------- * Unset the unlimited max dims if convert to other * than chunk layouts, because unlimited max dims diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index 5cc1785..a22b0fe 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -3078,65 +3078,119 @@ out: */ #define DIM1_L3 300 #define DIM2_L3 200 +/* small size */ +#define SDIM1_L3 4 +#define SDIM2_L3 50 static int make_layout3(hid_t loc_id) { - hid_t dcpl=-1; /* dataset creation property list */ - hid_t sid=-1; /* dataspace ID */ - hsize_t dims[RANK]={DIM1_L3,DIM2_L3}; + hid_t dcpl1=-1; /* dataset creation property list */ + hid_t dcpl2=-1; /* dataset creation property list */ + hid_t dcpl3=-1; /* dataset creation property list */ + hid_t sid1=-1; /* dataspace ID */ + hid_t sid2=-1; /* dataspace ID */ + hsize_t dims1[RANK]={DIM1_L3,DIM2_L3}; + hsize_t dims2[RANK]={SDIM1_L3,SDIM2_L3}; hsize_t maxdims[RANK]={H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[RANK]={DIM1_L3*2,5}; - int buf[DIM1_L3][DIM2_L3]; + hsize_t chunk_dims1[RANK]={DIM1_L3*2,5}; + hsize_t chunk_dims2[RANK]={SDIM1_L3 + 2, SDIM2_L3/2}; + hsize_t chunk_dims3[RANK]={SDIM1_L3 - 2, SDIM2_L3/2}; + int buf1[DIM1_L3][DIM2_L3]; + int buf2[SDIM1_L3][SDIM2_L3]; int i, j, n; + /* init buf1 */ for (i=n=0; i<DIM1_L3; i++) { for (j=0; j<DIM2_L3; j++) { - buf[i][j]=n++; + buf1[i][j]=n++; + } + } + + /* init buf2 */ + for (i=n=0; i<SDIM1_L3; i++) + { + for (j=0; j<SDIM2_L3; j++) + { + buf2[i][j]=n++; } } /*------------------------------------------------------------------------- - * make several dataset with several layout options + * make chunked dataset with + * - dset maxdims are UNLIMIT + * - a chunk dim is bigger than dset dim + * - dset size bigger than compact max (64K) *------------------------------------------------------------------------- */ /* create a space */ - if((sid = H5Screate_simple(RANK, dims, maxdims)) < 0) + if((sid1 = H5Screate_simple(RANK, dims1, maxdims)) < 0) return -1; /* create a dataset creation property list; the same DCPL is used for all dsets */ - if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + if ((dcpl1 = H5Pcreate(H5P_DATASET_CREATE)) < 0) { goto out; } + if(H5Pset_chunk(dcpl1, RANK, chunk_dims1) < 0) + goto out; + if (make_dset(loc_id,"chunk_unlimit1",sid1,dcpl1,buf1) < 0) + { + goto out; + } /*------------------------------------------------------------------------- - * H5D_CHUNKED + * make chunked dataset with + * - dset maxdims are UNLIMIT + * - a chunk dim is bigger than dset dim + * - dset size smaller than compact (64K) *------------------------------------------------------------------------- */ - if(H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + + /* create a space */ + if((sid2 = H5Screate_simple(RANK, dims2, maxdims)) < 0) + return -1; + /* create a dataset creation property list; the same DCPL is used for all dsets */ + if ((dcpl2 = H5Pcreate(H5P_DATASET_CREATE)) < 0) goto out; - if (make_dset(loc_id,"chunk_unlimit1",sid,dcpl,buf) < 0) - { + + if(H5Pset_chunk(dcpl2, RANK, chunk_dims2) < 0) goto out; - } - if(H5Pset_chunk(dcpl, RANK, chunk_dims) < 0) + if (make_dset(loc_id,"chunk_unlimit2",sid2,dcpl2,buf2) < 0) goto out; - if (make_dset(loc_id,"chunk_unlimit2",sid,dcpl,buf) < 0) - { + /*------------------------------------------------------------------------- + * make chunked dataset with + * - dset maxdims are UNLIMIT + * - a chunk dims are smaller than dset dims + * - dset size smaller than compact (64K) + *------------------------------------------------------------------------- + */ + /* create a dataset creation property list; the same DCPL is used for all dsets */ + if ((dcpl3 = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto out; + + if(H5Pset_chunk(dcpl3, RANK, chunk_dims3) < 0) + goto out; + + if (make_dset(loc_id,"chunk_unlimit3",sid2,dcpl3,buf2) < 0) goto out; - } /*------------------------------------------------------------------------- * close space and dcpl *------------------------------------------------------------------------- */ - if(H5Sclose(sid) < 0) + if(H5Sclose(sid1) < 0) goto out; - if(H5Pclose(dcpl) < 0) + if(H5Sclose(sid2) < 0) + goto out; + if(H5Pclose(dcpl1) < 0) + goto out; + if(H5Pclose(dcpl2) < 0) + goto out; + if(H5Pclose(dcpl3) < 0) goto out; return 0; @@ -3144,8 +3198,11 @@ int make_layout3(hid_t loc_id) out: H5E_BEGIN_TRY { - H5Pclose(dcpl); - H5Sclose(sid); + H5Sclose(sid1); + H5Sclose(sid2); + H5Pclose(dcpl1); + H5Pclose(dcpl2); + H5Pclose(dcpl3); } H5E_END_TRY; return -1; } diff --git a/tools/h5repack/testfiles/h5repack_layout3.h5 b/tools/h5repack/testfiles/h5repack_layout3.h5 Binary files differindex 5319031..7351502 100644 --- a/tools/h5repack/testfiles/h5repack_layout3.h5 +++ b/tools/h5repack/testfiles/h5repack_layout3.h5 |