summaryrefslogtreecommitdiffstats
path: root/tools/h5repack
diff options
context:
space:
mode:
Diffstat (limited to 'tools/h5repack')
-rw-r--r--tools/h5repack/CMakeLists.txt31
-rw-r--r--tools/h5repack/h5repack.c9
-rw-r--r--tools/h5repack/h5repack.h1
-rw-r--r--tools/h5repack/h5repack.sh.in51
-rw-r--r--tools/h5repack/h5repack_copy.c46
-rw-r--r--tools/h5repack/h5repack_main.c2
-rw-r--r--tools/h5repack/h5repack_opttable.c13
-rw-r--r--tools/h5repack/h5repacktst.c102
-rw-r--r--tools/h5repack/testfiles/h5repack_layout3.h5bin0 -> 966904 bytes
9 files changed, 232 insertions, 23 deletions
diff --git a/tools/h5repack/CMakeLists.txt b/tools/h5repack/CMakeLists.txt
index ae7a658..98d5c5a 100644
--- a/tools/h5repack/CMakeLists.txt
+++ b/tools/h5repack/CMakeLists.txt
@@ -96,6 +96,7 @@ IF (BUILD_TESTING)
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layouto.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout2.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout3.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_named_dtypes.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_nbit.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_objs.h5
@@ -322,10 +323,12 @@ IF (BUILD_TESTING)
-E remove
./testfiles/h5repack_filters.h5.out
./testfiles/h5repack_filters.h5.out.err
- ./testfiles/h5repack_layout2.h5-v.out
- ./testfiles/h5repack_layout2.h5-v.out.err
./testfiles/h5repack_layout.h5-v.out
./testfiles/h5repack_layout.h5-v.out.err
+ ./testfiles/h5repack_layout2.h5-v.out
+ ./testfiles/h5repack_layout2.h5-v.out.err
+ ./testfiles/h5repack_layout3.h5-v.out
+ ./testfiles/h5repack_layout3.h5-v.out.err
./testfiles/out.tfamily%05d.h5
./testfiles/out.h5diff_attr1.h5
./testfiles/out.h5repack_attr.h5
@@ -336,9 +339,10 @@ IF (BUILD_TESTING)
./testfiles/out.h5repack_filters.h5
./testfiles/out.h5repack_fletcher.h5
./testfiles/out.h5repack_hlink.h5
- ./testfiles/out.h5repack_layout2.h5
./testfiles/out.h5repack_layout.h5
./testfiles/out.h5repack_layouto.h5
+ ./testfiles/out.h5repack_layout2.h5
+ ./testfiles/out.h5repack_layout3.h5
./testfiles/out.h5repack_named_dtypes.h5
./testfiles/out.h5repack_nbit.h5
./testfiles/out.h5repack_objs.h5
@@ -369,9 +373,10 @@ IF (BUILD_TESTING)
h5repack_fletcher_out.h5
h5repack_hlink.h5
h5repack_hlink_out.h5
- h5repack_layout2.h5
h5repack_layout.h5
h5repack_layout_out.h5
+ h5repack_layout2.h5
+ h5repack_layout3.h5
h5repack_named_dtypes.h5
h5repack_named_dtypes_out.h5
h5repack_nbit.h5
@@ -777,6 +782,24 @@ IF (BUILD_TESTING)
ADD_H5_TEST (contig_small_fixed_compa "TEST" ${FILE18} -l chunked_small_fixed:COMPA)
ADD_H5_VERIFY_TEST (contig_small_fixed_compa "TEST" 0 ${FILE18} chunked_small_fixed COMPACT)
+#---------------------------------------------------------------------------
+# Test file contains chunked datasets (need multiple dsets) with
+# unlimited max dims. (HDFFV-7933)
+# Use first dset to test.
+#---------------------------------------------------------------------------
+# chunk to chunk - specify chunk dim bigger than any current dim
+ADD_H5_TEST (chunk2chunk "TEST" h5repack_layout3.h5 -l chunk_unlimit1:CHUNK=100x300)
+ADD_H5_VERIFY_TEST (chunk2chunk "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CHUNK)
+
+# chunk to contiguous
+ADD_H5_TEST (chunk2conti "TEST" h5repack_layout3.h5 -l chunk_unlimit1:CONTI)
+ADD_H5_VERIFY_TEST (chunk2conti "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CONTI)
+
+# chunk to compact - convert big dataset (should be > 64k) for this purpose,
+# should remain as original layout (chunk)
+ADD_H5_TEST (chunk2compa "TEST" h5repack_layout3.h5 -l chunk_unlimit1:COMPA)
+ADD_H5_VERIFY_TEST (chunk2compa "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CHUNK)
+
# Native option
# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
# report a difference.
diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c
index 9778fdd..292f4ec 100644
--- a/tools/h5repack/h5repack.c
+++ b/tools/h5repack/h5repack.c
@@ -206,10 +206,12 @@ int h5repack_addlayout(const char* str,
if (obj_list==NULL)
return -1;
- /* set global layout option */
+ /* set layout option */
+ options->layout_g = pack.layout;
+
+ /* no individual dataset specified */
if (options->all_layout==1 )
{
- options->layout_g=pack.layout;
if (pack.layout==H5D_CHUNKED)
{
/* -2 means the NONE option, remove chunking
@@ -228,6 +230,7 @@ int h5repack_addlayout(const char* str,
}
}
+ /* individual dataset specified */
if (options->all_layout==0)
options_add_layout(obj_list,
n_objs,
@@ -504,7 +507,7 @@ int copy_attr(hid_t loc_in,
if (type_class == H5T_COMPOUND) {
int nmembers = H5Tget_nmembers(wtype_id) ;
for (j=0; j<nmembers; j++) {
- hid_t mtid = H5Tget_member_type( wtype_id, j );
+ hid_t mtid = H5Tget_member_type( wtype_id, (unsigned) j );
H5T_class_t mtclass = H5Tget_class(mtid);
H5Tclose(mtid);
diff --git a/tools/h5repack/h5repack.h b/tools/h5repack/h5repack.h
index 3907178..69e36fc 100644
--- a/tools/h5repack/h5repack.h
+++ b/tools/h5repack/h5repack.h
@@ -28,6 +28,7 @@
#define MAX_VAR_DIMS 32 /* max per variable dimensions */
#define FORMAT_OBJ " %-27s %s\n" /* obj type, name */
#define FORMAT_OBJ_ATTR " %-27s %s\n" /* obj type, name */
+#define MAX_COMPACT_DSIZE 64512 /* max data size for compact layout. -1k for header size */
/*-------------------------------------------------------------------------
* data structures for command line options
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index 236c555..e05b03f 100644
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -94,6 +94,7 @@ $SRC_H5REPACK_TESTFILES/h5repack_hlink.h5
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5
$SRC_H5REPACK_TESTFILES/h5repack_layouto.h5
$SRC_H5REPACK_TESTFILES/h5repack_layout2.h5
+$SRC_H5REPACK_TESTFILES/h5repack_layout3.h5
$SRC_H5REPACK_TESTFILES/h5repack_named_dtypes.h5
$SRC_H5REPACK_TESTFILES/h5repack_nbit.h5
$SRC_H5REPACK_TESTFILES/h5repack_objs.h5
@@ -165,7 +166,7 @@ TESTING() {
# beginning with the word "Verifying".
#
VERIFY() {
- MESSAGE "Verifying h5diff output $*"
+ MESSAGE "Verifying $*"
}
# Print a message that a test has been skipped (because a required filter
@@ -179,7 +180,7 @@ SKIP() {
#
DIFFTEST()
{
- VERIFY $@
+ VERIFY h5diff output $@
$RUNSERIAL $H5DIFF_BIN -q "$@"
RET=$?
if [ $RET != 0 ] ; then
@@ -235,7 +236,7 @@ VERIFY_LAYOUT_DSET()
#---------------------------------
# check the layout from a dataset
- VERIFY "Layout"
+ VERIFY "a dataset layout"
$RUNSERIAL $H5DUMP_BIN -d $dset -pH $outfile > $layoutfile
$GREP $expectlayout $layoutfile > /dev/null
if [ $? -eq 0 ]; then
@@ -260,7 +261,7 @@ VERIFY_LAYOUT_ALL()
#---------------------------------
# check the layout from a dataset
# check if the other layouts still exsit
- VERIFY "Layout "
+ VERIFY "layouts"
# if CONTIGUOUS
if [ $expectlayout = "CONTIGUOUS" ]; then
$RUNSERIAL $H5DUMP_BIN -pH $outfile > $layoutfile
@@ -360,7 +361,8 @@ TOOLTEST1()
rm -f $outfile
}
-# Call h5repack and compare output to a text file for -v option
+# This is same as TOOLTEST() with comparing display output
+# from -v option
#
TOOLTESTV()
{
@@ -374,10 +376,20 @@ TOOLTESTV()
outfile=$TESTDIR/out.$1
shift
$RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile >$actual 2>$actual_err
- cp $actual $actual_sav
+ RET=$?
+ if [ $RET != 0 ] ; then
+ echo "*FAILED*"
+ nerrors="`expr $nerrors + 1`"
+ else
+ echo " PASSED"
+ DIFFTEST $infile $outfile
+ fi
+
+ # display output compare
STDOUT_FILTER $actual
cat $actual_err >> $actual
+ VERIFY output from $H5REPACK $@
if cmp -s $expect $actual; then
echo " PASSED"
else
@@ -387,7 +399,8 @@ TOOLTESTV()
test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /'
fi
- rm -f $actual $actual_err $actual_sav
+ rm -f $actual $actual_err
+ rm -f $outfile
}
# TOOLTEST_META:
@@ -559,7 +572,8 @@ arg="h5repack_filters.h5 -v -f /dset_deflate:GZIP=9"
if test $USE_FILTER_DEFLATE != "yes" ; then
SKIP $arg
else
- TOOLTEST $arg
+ # compare output
+ TOOLTESTV $arg
fi
###########################################################
@@ -778,6 +792,24 @@ VERIFY_LAYOUT_DSET h5repack_layout2.h5 contig_small COMPACT
TOOLTEST_MAIN h5repack_layout2.h5 -l chunked_small_fixed:COMPA
VERIFY_LAYOUT_DSET h5repack_layout2.h5 chunked_small_fixed COMPACT
+#---------------------------------------------------------------------------
+# Test file contains chunked datasets (need multiple dsets) with
+# unlimited max dims. (HDFFV-7933)
+# Use first dset to test.
+#---------------------------------------------------------------------------
+# chunk to chunk - specify chunk dim bigger than any current dim
+TOOLTEST_MAIN h5repack_layout3.h5 -l chunk_unlimit1:CHUNK=100x300
+VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CHUNK
+
+# chunk to contiguous
+TOOLTEST_MAIN h5repack_layout3.h5 -l chunk_unlimit1:CONTI
+VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CONTI
+
+# chunk to compact - convert big dataset (should be > 64k) for this purpose,
+# should remain as original layout (chunk)
+TOOLTEST_MAIN h5repack_layout3.h5 -l chunk_unlimit1:COMPA
+VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CHUNK
+
# Native option
# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
# report a difference.
@@ -856,8 +888,7 @@ TOOLTEST h5repack_attr_refs.h5
# Note: this test is experimental for sharing test file among tools
TOOLTEST h5diff_attr1.h5
-# tests for metadata block size option ('-M')
-TOOLTEST_META h5repack_layout.h5 -M 8192
+# tests for metadata block size option
TOOLTEST_META h5repack_layout.h5 --metadata_block_size=8192
if test $nerrors -eq 0 ; then
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index d38a15c..957d3c8 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -778,6 +778,8 @@ int do_copy_objects(hid_t fidin,
unsigned u;
int is_ref=0;
htri_t is_named;
+ hbool_t limit_maxdims;
+ hsize_t size_dset;
/*-------------------------------------------------------------------------
@@ -794,8 +796,10 @@ int do_copy_objects(hid_t fidin,
for ( i = 0; i < travt->nobjs; i++)
{
-
+ /* init variables per obj */
buf = NULL;
+ limit_maxdims = FALSE;
+
switch ( travt->objs[i].type )
{
@@ -970,6 +974,9 @@ int do_copy_objects(hid_t fidin,
if((msize = H5Tget_size(wtype_id)) == 0)
goto error;
+ /* size of current dset */
+ size_dset = nelmts * msize;
+
/*-------------------------------------------------------------------------
* check if the dataset creation property list has filters that
* are not registered in the current configuration
@@ -999,7 +1006,7 @@ int do_copy_objects(hid_t fidin,
*/
if (options->layout_g != H5D_COMPACT)
{
- if ( nelmts*msize < options->min_comp )
+ if ( size_dset < options->min_comp )
apply_s=0;
}
@@ -1016,9 +1023,38 @@ int do_copy_objects(hid_t fidin,
goto error;
}
- /* unset the unlimimted dimensions, which cannot be applied to layout other than chunked. */
- if (options->layout_g != H5D_CHUNKED) {
- H5Sset_extent_simple( f_space_id, rank, dims, NULL );
+ /*-------------------------------------------------
+ * Unset the unlimited max dims if convert to other
+ * than chunk layouts, because unlimited max dims
+ * only can be applied to chunk layout.
+ * Also perform only for targeted dataset
+ * Also check for size limit to convert to compact
+ *-------------------------------------------------*/
+ if (options->layout_g != H5D_CHUNKED)
+ {
+ /* any dataset is specified */
+ if (options->op_tbl->nelems > 0)
+ {
+ /* if current obj match specified obj */
+ if (options_get_object (travt->objs[i].name, options->op_tbl))
+ limit_maxdims = TRUE;
+ }
+ else /* no dataset is specified */
+ {
+ limit_maxdims = TRUE;
+ }
+
+ /* if convert to COMPACT */
+ if (options->layout_g == H5D_COMPACT)
+ {
+ /* should be smaller than 64K */
+ if ( size_dset > MAX_COMPACT_DSIZE )
+ limit_maxdims = FALSE;
+ }
+
+ /* unset unlimited max dims */
+ if (limit_maxdims)
+ H5Sset_extent_simple( f_space_id, rank, dims, NULL );
}
/*-------------------------------------------------------------------------
diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c
index 6004084..f61c961 100644
--- a/tools/h5repack/h5repack_main.c
+++ b/tools/h5repack/h5repack_main.c
@@ -466,6 +466,8 @@ void parse_command_line(int argc, const char **argv, pack_opt_t* options)
options->fs_threshold = (hsize_t)HDatol( opt_arg );
break;
+ default:
+ break;
} /* switch */
diff --git a/tools/h5repack/h5repack_opttable.c b/tools/h5repack/h5repack_opttable.c
index d0fcff3..4ec7fcf 100644
--- a/tools/h5repack/h5repack_opttable.c
+++ b/tools/h5repack/h5repack_opttable.c
@@ -388,11 +388,22 @@ pack_info_t* options_get_object( const char *path,
pack_opttbl_t *table )
{
unsigned int i;
+ const char tbl_path[MAX_NC_NAME];
+
for ( i = 0; i < table->nelems; i++)
{
+ /* make full path (start with "/") to compare correctly */
+ if (HDstrncmp(table->objs[i].path, "/", 1))
+ {
+ HDstrcpy(tbl_path, "/");
+ HDstrcat(tbl_path, table->objs[i].path);
+ }
+ else
+ HDstrcpy(tbl_path, table->objs[i].path);
+
/* found it */
- if (HDstrcmp(table->objs[i].path,path)==0)
+ if (HDstrcmp(tbl_path, path)==0)
{
return (&table->objs[i]);
}
diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c
index 6b15f0a..bc94eed 100644
--- a/tools/h5repack/h5repacktst.c
+++ b/tools/h5repack/h5repacktst.c
@@ -124,6 +124,7 @@ static int make_hlinks(hid_t loc_id);
static int make_early(void);
static int make_layout(hid_t loc_id);
static int make_layout2(hid_t loc_id);
+static int make_layout3(hid_t loc_id);
#ifdef H5_HAVE_FILTER_SZIP
static int make_szip(hid_t loc_id);
#endif /* H5_HAVE_FILTER_SZIP */
@@ -1712,6 +1713,19 @@ int make_testfiles(void)
return -1;
/*-------------------------------------------------------------------------
+ * for test layout conversions form chunk with unlimited max dims
+ *-------------------------------------------------------------------------
+ */
+ if((fid = H5Fcreate("h5repack_layout3.h5", H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT)) < 0)
+ return -1;
+
+ if(make_layout3(fid) < 0)
+ goto out;
+
+ if(H5Fclose(fid) < 0)
+ return -1;
+
+ /*-------------------------------------------------------------------------
* create a file for the H5D_ALLOC_TIME_EARLY test
*-------------------------------------------------------------------------
*/
@@ -3075,6 +3089,94 @@ out:
} /* make_layout2() */
/*-------------------------------------------------------------------------
+* Function: make_layout3
+*
+* Purpose: make chunked datasets with unlimited max dim and chunk dim is
+* bigger than current dim. (HDFFV-7933)
+* Test for converting chunk to chunk , chunk to conti and chunk
+* to compact.
+* - The chunk to chunk changes layout bigger than any current dim
+* again.
+* - The chunk to compact test dataset bigger than 64K, should
+* remain original layout.*
+*
+*-------------------------------------------------------------------------
+*/
+#define DIM1_L3 300
+#define DIM2_L3 200
+static
+int make_layout3(hid_t loc_id)
+{
+ hid_t dcpl=-1; /* dataset creation property list */
+ hid_t sid=-1; /* dataspace ID */
+ hsize_t dims[RANK]={DIM1_L3,DIM2_L3};
+ hsize_t maxdims[RANK]={H5S_UNLIMITED, H5S_UNLIMITED};
+ hsize_t chunk_dims[RANK]={DIM1_L3*2,5};
+ int buf[DIM1_L3][DIM2_L3];
+ int i, j, n;
+
+ for (i=n=0; i<DIM1_L3; i++)
+ {
+ for (j=0; j<DIM2_L3; j++)
+ {
+ buf[i][j]=n++;
+ }
+ }
+
+ /*-------------------------------------------------------------------------
+ * make several dataset with several layout options
+ *-------------------------------------------------------------------------
+ */
+ /* create a space */
+ if((sid = H5Screate_simple(RANK, dims, maxdims)) < 0)
+ return -1;
+ /* create a dataset creation property list; the same DCPL is used for all dsets */
+ if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ {
+ goto out;
+ }
+
+
+ /*-------------------------------------------------------------------------
+ * H5D_CHUNKED
+ *-------------------------------------------------------------------------
+ */
+ if(H5Pset_chunk(dcpl, RANK, chunk_dims) < 0)
+ goto out;
+ if (make_dset(loc_id,"chunk_unlimit1",sid,dcpl,buf) < 0)
+ {
+ goto out;
+ }
+
+ if(H5Pset_chunk(dcpl, RANK, chunk_dims) < 0)
+ goto out;
+
+ if (make_dset(loc_id,"chunk_unlimit2",sid,dcpl,buf) < 0)
+ {
+ goto out;
+ }
+
+ /*-------------------------------------------------------------------------
+ * close space and dcpl
+ *-------------------------------------------------------------------------
+ */
+ if(H5Sclose(sid) < 0)
+ goto out;
+ if(H5Pclose(dcpl) < 0)
+ goto out;
+
+ return 0;
+
+out:
+ H5E_BEGIN_TRY
+ {
+ H5Pclose(dcpl);
+ H5Sclose(sid);
+ } H5E_END_TRY;
+ return -1;
+}
+
+/*-------------------------------------------------------------------------
* Function: make a file with an integer dataset with a fill value
*
* Purpose: test copy of fill values
diff --git a/tools/h5repack/testfiles/h5repack_layout3.h5 b/tools/h5repack/testfiles/h5repack_layout3.h5
new file mode 100644
index 0000000..5319031
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_layout3.h5
Binary files differ
generated by cgit v0.12 at 2024-11-07 05:46:45 (GMT)