summaryrefslogtreecommitdiffstats
path: root/tools/h5repack
diff options
context:
space:
mode:
Diffstat (limited to 'tools/h5repack')
-rwxr-xr-xtools/h5repack/h5repack.sh.in182
-rw-r--r--tools/h5repack/h5repack_copy.c12
-rw-r--r--tools/h5repack/h5repacktst.c89
-rw-r--r--tools/h5repack/testfiles/h5repack_layout2.h5bin0 -> 6808 bytes
4 files changed, 261 insertions, 22 deletions
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index d889b55..5a60227 100755
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -35,7 +35,12 @@ H5REPACK=h5repack # The tool name
H5REPACK_BIN=`pwd`/$H5REPACK # The path of the tool binary
H5DIFF=../h5diff/h5diff # The h5diff tool name
-H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary
+H5DIFF_BIN=`pwd`/$H5DIFF # The path of the h5diff tool binary
+
+H5DUMP=../h5dump/h5dump # The h5dump tool name
+H5DUMP_BIN=`pwd`/$H5DUMP # The path of the h5dump tool binary
+
+GREP='grep'
H5DETECTSZIP=testh5repack_detect_szip
H5DETECTSZIP_BIN=`pwd`/$H5DETECTSZIP
@@ -59,6 +64,7 @@ FILE14=h5repack_layouto.h5 # A file with an older version of the layout mes
# (copy of test/tlayouto.h5)
FILE15=h5repack_named_dtypes.h5
FILE16=tfamily%05d.h5 # located in common testfiles folder
+FILE18=h5repack_layout2.h5
FILE_REF=h5repack_refs.h5
FILE_ATTR_REF=h5repack_attr_refs.h5
@@ -115,8 +121,21 @@ DIFFTEST()
# Call h5repack
#
+
+
+# call TOOLTEST_MAIN and delete $output file
TOOLTEST()
{
+ echo $@
+ TOOLTEST_MAIN $@
+ path=`pwd`
+ outfile=$path/out.$1
+ rm -f $outfile
+}
+
+# TOOLTEST main function, doesn't delete $output file
+TOOLTEST_MAIN()
+{
# Run test.
TESTING $H5REPACK $@
@@ -133,7 +152,98 @@ TOOLTEST()
echo " PASSED"
DIFFTEST $infile $outfile
fi
+}
+
+#------------------------------------------
+# Verifying layouts of a dataset
+VERIFY_LAYOUT_DSET()
+{
+ path=`pwd`
+ outfile=$path/out.$1
+ layoutfile=$path/layout.$1
+ dset=$2
+ expectlayout=$3
+
+ #---------------------------------
+ # check the layout from a dataset
+ VERIFY "Layout"
+ $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile
+ $GREP $expectlayout $layoutfile > /dev/null
+ if [ $? -eq 0 ]; then
+ echo " PASSED"
+ else
+ echo " FAILED"
+ fi
+
+ # clean up tmp files
rm -f $outfile
+ rm -f $layoutfile
+}
+
+#----------------------------------------
+# Verifying layouts from entire file
+VERIFY_LAYOUT_ALL()
+{
+ path=`pwd`
+ outfile=$path/out.$1
+ layoutfile=$path/layout.$1
+ expectlayout=$2
+
+ #---------------------------------
+ # check the layout from a dataset
+ # check if the other layouts still exsit
+ VERIFY "Layout "
+ # if CONTIGUOUS
+ if [ $expectlayout = "CONTIGUOUS" ]; then
+ $H5DUMP_BIN -pH $outfile > $layoutfile
+ $GREP "COMPACT" $layoutfile > /dev/null
+ if [ $? -eq 0 ]; then
+ echo " FAILED"
+ else
+ $GREP "CHUNKED" $layoutfile > /dev/null
+ if [ $? -eq 0 ]; then
+ echo " FAILED"
+ else
+ echo " PASSED"
+ fi
+ fi
+ else
+ # if COMPACT
+ if [ $expectlayout = "COMPACT" ]; then
+ $H5DUMP_BIN -pH $outfile > $layoutfile
+ $GREP "CHUNKED" $layoutfile > /dev/null
+ if [ $? -eq 0 ]; then
+ echo " FAILED"
+ else
+ $GREP "CONTIGUOUS" $layoutfile > /dev/null
+ if [ $? -eq 0 ]; then
+ echo " FAILED"
+ else
+ echo " PASSED"
+ fi
+ fi
+ else
+ # if CHUNKED
+ if [ $expectlayout = "CHUNKED" ]; then
+ $H5DUMP_BIN -pH $outfile > $layoutfile
+ $GREP "CONTIGUOUS" $layoutfile > /dev/null
+ if [ $? -eq 0 ]; then
+ echo " FAILED"
+ else
+ $GREP "COMPACT" $layoutfile > /dev/null
+ if [ $? -eq 0 ]; then
+ echo " FAILED"
+ else
+ echo " PASSED"
+ fi
+ fi
+ fi
+ fi
+ fi
+
+ # clean up tmp files
+ rm -f $outfile
+ rm -f $layoutfile
}
# same as TOOLTEST, but it uses the old syntax -i input_file -o output_file
@@ -494,32 +604,64 @@ else
TOOLTEST $arg
fi
-
#########################################################
# layout options (these files have no filters)
#########################################################
+TOOLTEST_MAIN $FILE4 --layout dset2:CHUNK=20x10
+VERIFY_LAYOUT_DSET $FILE4 dset2 CHUNKED
-TOOLTEST $FILE4 --layout=dset2:CHUNK=20x10
-TOOLTEST $FILE4 -l CHUNK=20x10
-TOOLTEST $FILE4 -l dset2:CONTI
-TOOLTEST $FILE4 -l CONTI
-TOOLTEST $FILE4 -l dset2:COMPA
-TOOLTEST $FILE4 -l COMPA
+TOOLTEST_MAIN $FILE4 -l CHUNK=20x10
+VERIFY_LAYOUT_ALL $FILE4 CHUNKED
+TOOLTEST_MAIN $FILE4 -l dset2:CONTI
+VERIFY_LAYOUT_DSET $FILE4 dset2 CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l CONTI
+VERIFY_LAYOUT_ALL $FILE4 CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l dset2:COMPA
+VERIFY_LAYOUT_DSET $FILE4 dset2 COMPACT
+
+TOOLTEST_MAIN $FILE4 -l COMPA
+VERIFY_LAYOUT_ALL $FILE4 COMPACT
################################################################
# layout conversions (file has no filters)
###############################################################
-TOOLTEST $FILE4 -l dset_compact:CONTI
-TOOLTEST $FILE4 -l dset_compact:CHUNK=2x5
-TOOLTEST $FILE4 -l dset_compact:COMPA
-TOOLTEST $FILE4 -l dset_contiguous:COMPA
-TOOLTEST $FILE4 -l dset_contiguous:CHUNK=3x6
-TOOLTEST $FILE4 -l dset_contiguous:CONTI
-TOOLTEST $FILE4 -l dset_chunk:COMPA
-TOOLTEST $FILE4 -l dset_chunk:CONTI
-TOOLTEST $FILE4 -l dset_chunk:CHUNK=18x13
+TOOLTEST_MAIN $FILE4 -l dset_compact:CONTI
+VERIFY_LAYOUT_DSET $FILE4 dset_compact CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l dset_compact:CHUNK=2x5
+VERIFY_LAYOUT_DSET $FILE4 dset_compact CHUNKED
+
+TOOLTEST_MAIN $FILE4 -l dset_compact:COMPA
+VERIFY_LAYOUT_DSET $FILE4 dset_compact COMPACT
+
+TOOLTEST_MAIN $FILE4 -l dset_contiguous:COMPA
+VERIFY_LAYOUT_DSET $FILE4 dset_contiguous COMPACT
+
+TOOLTEST_MAIN $FILE4 -l dset_contiguous:CHUNK=3x6
+VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CHUNKED
+
+TOOLTEST_MAIN $FILE4 -l dset_contiguous:CONTI
+VERIFY_LAYOUT_DSET $FILE4 dset_contiguous CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l dset_chunk:COMPA
+VERIFY_LAYOUT_DSET $FILE4 dset_chunk COMPACT
+
+TOOLTEST_MAIN $FILE4 -l dset_chunk:CONTI
+VERIFY_LAYOUT_DSET $FILE4 dset_chunk CONTIGUOUS
+
+TOOLTEST_MAIN $FILE4 -l dset_chunk:CHUNK=18x13
+VERIFY_LAYOUT_DSET $FILE4 dset_chunk CHUNKED
+
+# test convert small size dataset ( < 1k) to compact layout without -m
+TOOLTEST_MAIN $FILE18 -l contig_small:COMPA
+VERIFY_LAYOUT_DSET $FILE18 contig_small COMPACT
+
+TOOLTEST_MAIN $FILE18 -l chunked_small_fixed:COMPA
+VERIFY_LAYOUT_DSET $FILE18 chunked_small_fixed COMPACT
# Native option
# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
@@ -533,7 +675,8 @@ arg="$FILE4 --layout CHUNK=20x10 --filter GZIP=1 --minimum=10 --native --latest
if test $USE_FILTER_DEFLATE != "yes" ; then
SKIP $arg
else
- TOOLTEST $arg
+ TOOLTEST_MAIN $arg
+ VERIFY_LAYOUT_ALL $FILE4 CHUNKED
fi
# latest file format with short switches. use FILE4=h5repack_layout.h5 (no filters)
@@ -541,7 +684,8 @@ arg="$FILE4 -l CHUNK=20x10 -f GZIP=1 -m 10 -n -L -c 8 -d 6 -s 8[:dtype]"
if test $USE_FILTER_DEFLATE != "yes" ; then
SKIP $arg
else
- TOOLTEST $arg
+ TOOLTEST_MAIN $arg
+ VERIFY_LAYOUT_ALL $FILE4 CHUNKED
fi
# several global filters
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index b736c22..f58a303 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -789,9 +789,15 @@ int do_copy_objects(hid_t fidin,
/* get the storage size of the input dataset */
dsize_in=H5Dget_storage_size(dset_in);
- /* check for datasets too small */
- if (nelmts*msize < options->min_comp )
- apply_s=0;
+ /* check for small size datasets (less than 1k) except
+ * changing to COMPACT. For the reference, COMPACT is limited
+ * by size 64K by library.
+ */
+ if (options->layout_g != H5D_COMPACT)
+ {
+ if ( nelmts*msize < options->min_comp )
+ apply_s=0;
+ }
/* apply the filter */
if (apply_s)
diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c
index a9efe25..0f8be17 100644
--- a/tools/h5repack/h5repacktst.c
+++ b/tools/h5repack/h5repacktst.c
@@ -76,6 +76,9 @@
#define FNAME17 "h5repack_named_dtypes.h5"
#define FNAME17OUT "h5repack_named_dtypes_out.h5"
+#define FNAME18 "h5repack_layout2.h5"
+#define FNAME18OUT "h5repack_layout2_out.h5"
+
#define FNAME_UB "ublock.bin"
/* obj and region references */
@@ -120,6 +123,7 @@ static int make_attributes(hid_t loc_id);
static int make_hlinks(hid_t loc_id);
static int make_early(void);
static int make_layout(hid_t loc_id);
+static int make_layout2(hid_t loc_id);
#ifdef H5_HAVE_FILTER_SZIP
static int make_szip(hid_t loc_id);
#endif /* H5_HAVE_FILTER_SZIP */
@@ -1643,6 +1647,19 @@ int make_testfiles(void)
return -1;
/*-------------------------------------------------------------------------
+ * create a file for layout conversion test
+ *-------------------------------------------------------------------------
+ */
+ if((fid = H5Fcreate(FNAME18, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ return -1;
+
+ if(make_layout2(fid) < 0)
+ goto out;
+
+ if(H5Fclose(fid) < 0)
+ return -1;
+
+ /*-------------------------------------------------------------------------
* create a file for the H5D_ALLOC_TIME_EARLY test
*-------------------------------------------------------------------------
*/
@@ -2934,6 +2951,78 @@ out:
}
/*-------------------------------------------------------------------------
+ * Function: make_layout2
+ *
+ * Purpose: create datasets with contiguous and chunked layouts:
+ *
+ * contig_small: < 1k, fixed dims datspace
+ * chunked_small_fixed: < 1k, fixed dims dataspace
+ *
+ *-------------------------------------------------------------------------
+ */
+#define S_DIM1 4
+#define S_DIM2 10
+#define CONTIG_S "contig_small"
+#define CHUNKED_S_FIX "chunked_small_fixed"
+
+static
+int make_layout2(hid_t loc_id)
+{
+
+ hid_t contig_dcpl = -1; /* dataset creation property list */
+ hid_t chunked_dcpl = -1; /* dataset creation property list */
+
+ int i, j, n; /* Local index variables */
+ int ret_value = -1; /* Return value */
+ hid_t s_sid = -1; /* dataspace ID */
+
+ hsize_t s_dims[RANK] = {S_DIM1,S_DIM2}; /* Dataspace (< 1 k) */
+ hsize_t chunk_dims[RANK] = {S_DIM1/2, S_DIM2/2}; /* Dimension sizes for chunks */
+
+ int s_buf[S_DIM1][S_DIM2]; /* Temporary buffer */
+
+ for(i = n = 0; i < S_DIM1; i++) {
+ for (j = 0; j < S_DIM2; j++) {
+ s_buf[i][j] = n++;
+ }
+ }
+
+ /* Create dataspaces */
+ if((s_sid = H5Screate_simple(RANK, s_dims, NULL)) < 0)
+ goto out;
+
+ /* Create contiguous datasets */
+ if((contig_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ goto out;
+ if(H5Pset_layout(contig_dcpl, H5D_CONTIGUOUS) < 0)
+ goto out;
+ if(make_dset(loc_id, CONTIG_S, s_sid, contig_dcpl, s_buf) < 0)
+ goto out;
+
+ /* Create chunked datasets */
+ if((chunked_dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ goto out;
+ if(H5Pset_chunk(chunked_dcpl, RANK, chunk_dims) < 0)
+ goto out;
+ if(make_dset(loc_id, CHUNKED_S_FIX, s_sid, chunked_dcpl, s_buf) < 0)
+ goto out;
+
+ ret_value = 0;
+
+out:
+ H5E_BEGIN_TRY {
+ H5Pclose(contig_dcpl);
+ H5Pclose(chunked_dcpl);
+
+ H5Sclose(s_sid);
+
+ } H5E_END_TRY;
+
+ return(ret_value);
+
+} /* make_layout2() */
+
+/*-------------------------------------------------------------------------
* Function: make a file with an integer dataset with a fill value
*
* Purpose: test copy of fill values
diff --git a/tools/h5repack/testfiles/h5repack_layout2.h5 b/tools/h5repack/testfiles/h5repack_layout2.h5
new file mode 100644
index 0000000..6e77e41
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_layout2.h5
Binary files differ
generated by cgit v0.12 at 2024-11-19 18:55:39 (GMT)