From a6b6b75a79248a57867a243b52ef9a6bca196aff Mon Sep 17 00:00:00 2001 From: Jonathan Kim Date: Wed, 15 Sep 2010 13:32:07 -0500 Subject: [svn-r19389] Purpose: Fix for Bug1896 h5repack - changing layout to COMPACT does not work Description: Make h5repack be able to convert a layout to COMPACT for small size dataset as default. Also add verifying layout changes in our test script. Tested: jam, amani, heiwa --- MANIFEST | 1 + release_docs/RELEASE.txt | 2 + tools/h5repack/h5repack.sh.in | 182 ++++++++++++++++++++++++--- tools/h5repack/h5repack_copy.c | 12 +- tools/h5repack/h5repacktst.c | 89 +++++++++++++ tools/h5repack/testfiles/h5repack_layout2.h5 | Bin 0 -> 6808 bytes 6 files changed, 264 insertions(+), 22 deletions(-) create mode 100644 tools/h5repack/testfiles/h5repack_layout2.h5 diff --git a/MANIFEST b/MANIFEST index abe0c4b..db7e347 100644 --- a/MANIFEST +++ b/MANIFEST @@ -1639,6 +1639,7 @@ ./tools/h5repack/testfiles/h5repack_hlink.h5 ./tools/h5repack/testfiles/h5repack_layout.h5 ./tools/h5repack/testfiles/h5repack_layouto.h5 +./tools/h5repack/testfiles/h5repack_layout2.h5 ./tools/h5repack/testfiles/h5repack_early.h5 ./tools/h5repack/testfiles/h5repack_szip.h5 ./tools/h5repack/testfiles/h5repack_deflate.h5 diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 87f6e40..172f87a 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -466,6 +466,8 @@ Bug Fixes since HDF5-1.8.0 release Tools ----- + - Make h5repack be able to convert a layout to COMPACT for small size + dataset as default. bug#1896 (JKM 2010/09/15) - Change h5ls not to manipulate special characters in object name or attribute name for smart display. bug#1784 (JKM 2010/06/28) - Fixed h5ls to return exit code 1 (error) when non-existent file is diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in index d889b55..5a60227 100755 --- a/tools/h5repack/h5repack.sh.in +++ b/tools/h5repack/h5repack.sh.in @@ -35,7 +35,12 @@ H5REPACK=h5repack # The tool name H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary H5DIFF=../h5diff/h5diff # The h5diff tool name -H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary +H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary + +H5DUMP=../h5dump/h5dump # The h5dump tool name +H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary + +GREP='grep' H5DETECTSZIP=testh5repack_detect_szip H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP @@ -59,6 +64,7 @@ FILE14=h5repack_layouto.h5 # A file with an older version of the layout mes # (copy of test/tlayouto.h5) FILE15=h5repack_named_dtypes.h5 FILE16=tfamily%05d.h5 # located in common testfiles folder +FILE18=h5repack_layout2.h5 FILE_REF=h5repack_refs.h5 FILE_ATTR_REF=h5repack_attr_refs.h5 @@ -115,8 +121,21 @@ DIFFTEST() # Call h5repack # + + +# call TOOLTEST_MAIN and delete $output file TOOLTEST() { + echo $@ + TOOLTEST_MAIN $@ + path=`pwd` + outfile=$path/out.$1 + rm -f $outfile +} + +# TOOLTEST main function, doesn't delete $output file +TOOLTEST_MAIN() +{ # Run test. TESTING $H5REPACK $@ @@ -133,7 +152,98 @@ TOOLTEST() echo " PASSED" DIFFTEST $infile $outfile fi +} + +#------------------------------------------ +# Verifying layouts of a dataset +VERIFY_LAYOUT_DSET() +{ + path=`pwd` + outfile=$path/out.$1 + layoutfile=$path/layout.$1 + dset=$2 + expectlayout=$3 + + #--------------------------------- + # check the layout from a dataset + VERIFY "Layout" + $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile + $GREP $expectlayout $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " PASSED" + else + echo " FAILED" + fi + + # clean up tmp files rm -f $outfile + rm -f $layoutfile +} + +#---------------------------------------- +# Verifying layouts from entire file +VERIFY_LAYOUT_ALL() +{ + path=`pwd` + outfile=$path/out.$1 + layoutfile=$path/layout.$1 + expectlayout=$2 + + #--------------------------------- + # check the layout from a dataset + # check if the other layouts still exsit + VERIFY "Layout " + # if CONTIGUOUS + if [ $expectlayout = "CONTIGUOUS" ]; then + $H5DUMP_BIN -pH $outfile > $layoutfile + $GREP "COMPACT" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + else + $GREP "CHUNKED" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + else + echo " PASSED" + fi + fi + else + # if COMPACT + if [ $expectlayout = "COMPACT" ]; then + $H5DUMP_BIN -pH $outfile > $layoutfile + $GREP "CHUNKED" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + else + $GREP "CONTIGUOUS" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + else + echo " PASSED" + fi + fi + else + # if CHUNKED + if [ $expectlayout = "CHUNKED" ]; then + $H5DUMP_BIN -pH $outfile > $layoutfile + $GREP "CONTIGUOUS" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + else + $GREP "COMPACT" $layoutfile > /dev/null + if [ $? -eq 0 ]; then + echo " FAILED" + else + echo " PASSED" + fi + fi + fi + fi + fi + + # clean up tmp files + rm -f $outfile + rm -f $layoutfile } # same as TOOLTEST, but it uses the old syntax -i input_file -o output_file @@ -494,32 +604,64 @@ else TOOLTEST $arg fi - ######################################################### # layout options (these files have no filters) ######################################################### +TOOLTEST_MAIN $FILE4 --layout dset2:CHUNK=20x10 +VERIFY_LAYOUT_DSET $FILE4 dset2 CHUNKED -TOOLTEST $FILE4 --layout=dset2:CHUNK=20x10 -TOOLTEST $FILE4 -l CHUNK=20x10 -TOOLTEST $FILE4 -l dset2:CONTI -TOOLTEST $FILE4 -l CONTI -TOOLTEST $FILE4 -l dset2:COMPA -TOOLTEST $FILE4 -l COMPA +TOOLTEST_MAIN $FILE4 -l CHUNK=20x10 +VERIFY_LAYOUT_ALL $FILE4 CHUNKED +TOOLTEST_MAIN $FILE4 -l dset2:CONTI +VERIFY_LAYOUT_DSET $FILE4 dset2 CONTIGUOUS + +TOOLTEST_MAIN $FILE4 -l CONTI +VERIFY_LAYOUT_ALL $FILE4 CONTIGUOUS + +TOOLTEST_MAIN $FILE4 -l dset2:COMPA +VERIFY_LAYOUT_DSET $FILE4 dset2 COMPACT + +TOOLTEST_MAIN $FILE4 -l COMPA +VERIFY_LAYOUT_ALL $FILE4 COMPACT ################################################################ # layout conversions (file has no filters) ############################################################### -TOOLTEST $FILE4 -l dset_compact:CONTI -TOOLTEST $FILE4 -l dset_compact:CHUNK=2x5 -TOOLTEST $FILE4 -l dset_compact:COMPA -TOOLTEST $FILE4 -l dset_contiguous:COMPA -TOOLTEST $FILE4 -l dset_contiguous:CHUNK=3x6 -TOOLTEST $FILE4 -l dset_contiguous:CONTI -TOOLTEST $FILE4 -l dset_chunk:COMPA -TOOLTEST $FILE4 -l dset_chunk:CONTI -TOOLTEST $FILE4 -l dset_chunk:CHUNK=18x13 +TOOLTEST_MAIN $FILE4 -l dset_compact:CONTI +VERIFY_LAYOUT_DSET $FILE4 dset_compact CONTIGUOUS + +TOOLTEST_MAIN $FILE4 -l dset_compact:CHUNK=2x5 +VERIFY_LAYOUT_DSET $FILE4 dset_compact CHUNKED + +TOOLTEST_MAIN $FILE4 -l dset_compact:COMPA +VERIFY_LAYOUT_DSET $FILE4 dset_compact COMPACT + +TOOLTEST_MAIN $FILE4 -l dset_contiguous:COMPA +VERIFY_LAYOUT_DSET $FILE4 dset_contiguous COMPACT + +TOOLTEST_MAIN $FILE4 -l dset_contiguous:CHUNK=3x6 +VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CHUNKED + +TOOLTEST_MAIN $FILE4 -l dset_contiguous:CONTI +VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CONTIGUOUS + +TOOLTEST_MAIN $FILE4 -l dset_chunk:COMPA +VERIFY_LAYOUT_DSET $FILE4 dset_chunk COMPACT + +TOOLTEST_MAIN $FILE4 -l dset_chunk:CONTI +VERIFY_LAYOUT_DSET $FILE4 dset_chunk CONTIGUOUS + +TOOLTEST_MAIN $FILE4 -l dset_chunk:CHUNK=18x13 +VERIFY_LAYOUT_DSET $FILE4 dset_chunk CHUNKED + +# test convert small size dataset ( < 1k) to compact layout without -m +TOOLTEST_MAIN $FILE18 -l contig_small:COMPA +VERIFY_LAYOUT_DSET $FILE18 contig_small COMPACT + +TOOLTEST_MAIN $FILE18 -l chunked_small_fixed:COMPA +VERIFY_LAYOUT_DSET $FILE18 chunked_small_fixed COMPACT # Native option # Do not use FILE1, as the named dtype will be converted to native, and h5diff will @@ -533,7 +675,8 @@ arg="$FILE4 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else - TOOLTEST $arg + TOOLTEST_MAIN $arg + VERIFY_LAYOUT_ALL $FILE4 CHUNKED fi # latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters) @@ -541,7 +684,8 @@ arg="$FILE4 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg else - TOOLTEST $arg + TOOLTEST_MAIN $arg + VERIFY_LAYOUT_ALL $FILE4 CHUNKED fi # several global filters diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c index b736c22..f58a303 100644 --- a/tools/h5repack/h5repack_copy.c +++ b/tools/h5repack/h5repack_copy.c @@ -789,9 +789,15 @@ int do_copy_objects(hid_t fidin, /* get the storage size of the input dataset */ dsize_in=H5Dget_storage_size(dset_in); - /* check for datasets too small */ - if (nelmts*msize < options->min_comp ) - apply_s=0; + /* check for small size datasets (less than 1k) except + * changing to COMPACT. For the reference, COMPACT is limited + * by size 64K by library. + */ + if (options->layout_g != H5D_COMPACT) + { + if ( nelmts*msize < options->min_comp ) + apply_s=0; + } /* apply the filter */ if (apply_s) diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c index a9efe25..0f8be17 100644 --- a/tools/h5repack/h5repacktst.c +++ b/tools/h5repack/h5repacktst.c @@ -76,6 +76,9 @@ #define FNAME17 "h5repack_named_dtypes.h5" #define FNAME17OUT "h5repack_named_dtypes_out.h5" +#define FNAME18 "h5repack_layout2.h5" +#define FNAME18OUT "h5repack_layout2_out.h5" + #define FNAME_UB "ublock.bin" /* obj and region references */ @@ -120,6 +123,7 @@ static int make_attributes(hid_t loc_id); static int make_hlinks(hid_t loc_id); static int make_early(void); static int make_layout(hid_t loc_id); +static int make_layout2(hid_t loc_id); #ifdef H5_HAVE_FILTER_SZIP static int make_szip(hid_t loc_id); #endif /* H5_HAVE_FILTER_SZIP */ @@ -1643,6 +1647,19 @@ int make_testfiles(void) return -1; /*------------------------------------------------------------------------- + * create a file for layout conversion test + *------------------------------------------------------------------------- + */ + if((fid = H5Fcreate(FNAME18, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0) + return -1; + + if(make_layout2(fid) < 0) + goto out; + + if(H5Fclose(fid) < 0) + return -1; + + /*------------------------------------------------------------------------- * create a file for the H5D_ALLOC_TIME_EARLY test *------------------------------------------------------------------------- */ @@ -2934,6 +2951,78 @@ out: } /*------------------------------------------------------------------------- + * Function: make_layout2 + * + * Purpose: create datasets with contiguous and chunked layouts: + * + * contig_small: < 1k, fixed dims datspace + * chunked_small_fixed: < 1k, fixed dims dataspace + * + *------------------------------------------------------------------------- + */ +#define S_DIM1 4 +#define S_DIM2 10 +#define CONTIG_S "contig_small" +#define CHUNKED_S_FIX "chunked_small_fixed" + +static +int make_layout2(hid_t loc_id) +{ + + hid_t contig_dcpl = -1; /* dataset creation property list */ + hid_t chunked_dcpl = -1; /* dataset creation property list */ + + int i, j, n; /* Local index variables */ + int ret_value = -1; /* Return value */ + hid_t s_sid = -1; /* dataspace ID */ + + hsize_t s_dims[RANK] = {S_DIM1,S_DIM2}; /* Dataspace (< 1 k) */ + hsize_t chunk_dims[RANK] = {S_DIM1/2, S_DIM2/2}; /* Dimension sizes for chunks */ + + int s_buf[S_DIM1][S_DIM2]; /* Temporary buffer */ + + for(i = n = 0; i < S_DIM1; i++) { + for (j = 0; j < S_DIM2; j++) { + s_buf[i][j] = n++; + } + } + + /* Create dataspaces */ + if((s_sid = H5Screate_simple(RANK, s_dims, NULL)) < 0) + goto out; + + /* Create contiguous datasets */ + if((contig_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto out; + if(H5Pset_layout(contig_dcpl, H5D_CONTIGUOUS) < 0) + goto out; + if(make_dset(loc_id, CONTIG_S, s_sid, contig_dcpl, s_buf) < 0) + goto out; + + /* Create chunked datasets */ + if((chunked_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0) + goto out; + if(H5Pset_chunk(chunked_dcpl, RANK, chunk_dims) < 0) + goto out; + if(make_dset(loc_id, CHUNKED_S_FIX, s_sid, chunked_dcpl, s_buf) < 0) + goto out; + + ret_value = 0; + +out: + H5E_BEGIN_TRY { + H5Pclose(contig_dcpl); + H5Pclose(chunked_dcpl); + + H5Sclose(s_sid); + + } H5E_END_TRY; + + return(ret_value); + +} /* make_layout2() */ + +/*------------------------------------------------------------------------- * Function: make a file with an integer dataset with a fill value * * Purpose: test copy of fill values diff --git a/tools/h5repack/testfiles/h5repack_layout2.h5 b/tools/h5repack/testfiles/h5repack_layout2.h5 new file mode 100644 index 0000000..6e77e41 Binary files /dev/null and b/tools/h5repack/testfiles/h5repack_layout2.h5 differ -- cgit v0.12