From 06e9ab4420adb7a6ae04514adcb134d27114e410 Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 26 Sep 2012 13:10:01 -0500 Subject: [svn-r22814] Purpose: HDFFV-8012 - h5repack changes max dims and cause failure if only "-f none" is used without changing layout for chunked dataset when a chunk dim is bigger than a dataset dim Description: "h5repack -f :NONE out.h5" command failed if source file contains chunked dataset and a chunk dim is bigger than a dataset dim. Another issue is that the command changed max dims if chunk dim is smaller than the dataset dim. These issue occurred when dataset size is smaller than 64k (compact size limit) Fixed them. Merged from HDF5 trunk r22805 Tested: jam (linux32-LE), koala (linux64-LE), ostrich (linuxppc64-BE), tejeda (mac32-LE), linew (solaris-BE), Windows (32-LE cmake), cmake (jam) --- release_docs/RELEASE.txt | 7 ++ tools/h5repack/CMakeLists.txt | 22 ++++++ tools/h5repack/h5repack.c | 1 + tools/h5repack/h5repack.sh.in | 21 ++++++ tools/h5repack/h5repack_copy.c | 20 ++++++ tools/h5repack/h5repacktst.c | 101 +++++++++++++++++++++------ tools/h5repack/testfiles/h5repack_layout3.h5 | Bin 966904 -> 491840 bytes 7 files changed, 150 insertions(+), 22 deletions(-) diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 61c1828..2dc0527 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -125,6 +125,13 @@ Bug Fixes since HDF5-1.8.9 Tools ----- + - h5repack: "h5repack -f NONE file1.h5 out.h5" command failed if + source file contains chunked dataset and a chunk dim is bigger than + the dataset dim. Another issue is that the command changed max dims + if chunk dim is smaller than the dataset dim. + These issue occurred when dataset size is smaller than 64k (compact + size limit) Fixed both. + HDFFV-8012 (JKM 2012/09/24) - h5diff: Fixed not to accumulate attribute difference to dataset difference in verbose mode (-v, -r), which caused incorrect difference between dataset and group/datatype object if attribute diff --git a/tools/h5repack/CMakeLists.txt b/tools/h5repack/CMakeLists.txt index 3ee1cba..7351fd5 100644 --- a/tools/h5repack/CMakeLists.txt +++ b/tools/h5repack/CMakeLists.txt @@ -800,6 +800,28 @@ ADD_H5_VERIFY_TEST (chunk2conti "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CONT ADD_H5_TEST (chunk2compa "TEST" h5repack_layout3.h5 -l chunk_unlimit1:COMPA) ADD_H5_VERIFY_TEST (chunk2compa "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CHUNK) +#-------------------------------------------------------------------------- +# Test -f for some specific cases. Chunked dataset with unlimited max dims. +# (HDFFV-8012) +#-------------------------------------------------------------------------- +# - should not fail +# - should not change max dims from unlimit + +# chunk dim is bigger than dataset dim. ( dset size < 64k ) +ADD_H5_TEST (error1 "TEST" h5repack_layout3.h5 -f chunk_unlimit1:NONE) +ADD_H5_VERIFY_TEST (error1 "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED) + +# chunk dim is bigger than dataset dim. ( dset size > 64k ) +ADD_H5_TEST (error2 "TEST" h5repack_layout3.h5 -f chunk_unlimit2:NONE) +ADD_H5_VERIFY_TEST (error2 "TEST" 0 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED) + +# chunk dims are smaller than dataset dims. ( dset size < 64k ) +ADD_H5_TEST (error3 "TEST" h5repack_layout3.h5 -f chunk_unlimit3:NONE) +ADD_H5_VERIFY_TEST (error3 "TEST" 0 h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED) + +# file input - should not fail +ADD_H5_TEST (error4 "TEST" h5repack_layout3.h5 -f NONE) + # Native option # Do not use FILE1, as the named dtype will be converted to native, and h5diff will # report a difference. diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c index 6517ae1..621cc52 100644 --- a/tools/h5repack/h5repack.c +++ b/tools/h5repack/h5repack.c @@ -93,6 +93,7 @@ int h5repack_init(pack_opt_t *options, HDmemset(options, 0, sizeof(pack_opt_t)); options->min_comp = 1024; options->verbose = verbose; + options->layout_g = H5D_LAYOUT_ERROR; for ( n = 0; n < H5_REPACK_MAX_NFILTERS; n++) { diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index 2614dd6..a732ffb 100644 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -818,6 +818,27 @@ VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CONTI TOOLTEST_MAIN h5repack_layout3.h5 -l chunk_unlimit1:COMPA VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CHUNK +#-------------------------------------------------------------------------- +# Test -f for some specific cases. Chunked dataset with unlimited max dims. +# (HDFFV-8012) +#-------------------------------------------------------------------------- +# - should not fail +# - should not change max dims from unlimit + +# chunk dim is bigger than dataset dim. ( dset size < 64k ) +TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit1:NONE +VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED +# chunk dim is bigger than dataset dim. ( dset size > 64k ) +TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit2:NONE +VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED + +# chunk dims are smaller than dataset dims. ( dset size < 64k ) +TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit3:NONE +VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED + +# file input - should not fail +TOOLTEST h5repack_layout3.h5 -f NONE + # Native option # Do not use FILE1, as the named dtype will be converted to native, and h5diff will # report a difference. diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index 94bd565..cfd542a 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -732,6 +732,7 @@ int do_copy_objects(hid_t fidin, void *hslab_buf=NULL; /* hyperslab buffer for raw data */ int has_filter; /* current object has a filter */ int req_filter; /* there was a request for a filter */ + int req_obj_layout=0; /* request layout to current object */ unsigned crt_order_flags; /* group creation order flag */ unsigned i; unsigned u; @@ -863,6 +864,22 @@ int do_copy_objects(hid_t fidin, } } + /* check if layout change requested individual object */ + if (options->layout_g != H5D_LAYOUT_ERROR) + { + pack_info_t *pckinfo; + /* any dataset is specified */ + if (options->op_tbl->nelems > 0) + { + /* check if object exist */ + pckinfo = options_get_object (travt->objs[i].name, options->op_tbl); + if (pckinfo) + { + req_obj_layout = 1; + } + } + } + /* early detection of references */ if((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0) goto error; @@ -982,6 +999,9 @@ int do_copy_objects(hid_t fidin, goto error; } + /* only if layout change requested for entire file or + * individual obj */ + if (options->all_layout > 0 || req_obj_layout == 1) /*------------------------------------------------- * Unset the unlimited max dims if convert to other * than chunk layouts, because unlimited max dims diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index 5cc1785..a22b0fe 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -3078,65 +3078,119 @@ out: */ #define DIM1_L3 300 #define DIM2_L3 200 +/* small size */ +#define SDIM1_L3 4 +#define SDIM2_L3 50 static int make_layout3(hid_t loc_id) { - hid_t dcpl=-1; /* dataset creation property list */ - hid_t sid=-1; /* dataspace ID */ - hsize_t dims[RANK]={DIM1_L3,DIM2_L3}; + hid_t dcpl1=-1; /* dataset creation property list */ + hid_t dcpl2=-1; /* dataset creation property list */ + hid_t dcpl3=-1; /* dataset creation property list */ + hid_t sid1=-1; /* dataspace ID */ + hid_t sid2=-1; /* dataspace ID */ + hsize_t dims1[RANK]={DIM1_L3,DIM2_L3}; + hsize_t dims2[RANK]={SDIM1_L3,SDIM2_L3}; hsize_t maxdims[RANK]={H5S_UNLIMITED, H5S_UNLIMITED}; - hsize_t chunk_dims[RANK]={DIM1_L3*2,5}; - int buf[DIM1_L3][DIM2_L3]; + hsize_t chunk_dims1[RANK]={DIM1_L3*2,5}; + hsize_t chunk_dims2[RANK]={SDIM1_L3 + 2, SDIM2_L3/2}; + hsize_t chunk_dims3[RANK]={SDIM1_L3 - 2, SDIM2_L3/2}; + int buf1[DIM1_L3][DIM2_L3]; + int buf2[SDIM1_L3][SDIM2_L3]; int i, j, n; + /* init buf1 */ for (i=n=0; i