summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorJonathan Kim <jkm@hdfgroup.org>2012-09-24 23:13:57 (GMT)
committerJonathan Kim <jkm@hdfgroup.org>2012-09-24 23:13:57 (GMT)
commit50e3990f2a9e3bba421460d8d5cb21bb6f6e98fe (patch)
tree764c642d62a1082e02cb0004871dc33b81dbff08
parent528bd520b83136e9c50ac6cff4ac116fc8054e7a (diff)
downloadhdf5-50e3990f2a9e3bba421460d8d5cb21bb6f6e98fe.zip
hdf5-50e3990f2a9e3bba421460d8d5cb21bb6f6e98fe.tar.gz
hdf5-50e3990f2a9e3bba421460d8d5cb21bb6f6e98fe.tar.bz2
[svn-r22805] Purpose:
HDFFV-8012 - h5repack changes max dims and cause failure if only "-f none" is used without changing layout for chunked dataset when a chunk dim is bigger than a dataset dim Description: "h5repack -f <obj>:NONE <file.h5> out.h5" command failed if source file contains chunked dataset and a chunk dim is bigger than a dataset dim. Another issue is that the command changed max dims if chunk dim is smaller than the dataset dim. These issue occurred when dataset size is smaller than 64k (compact size limit) Fixed them. Tested: jam (linux32-LE), koala (linux64-LE), ostrich (linuxppc64-BE), tejeda (mac32-LE), linew (solaris-BE), Windows (32-LE cmake), cmake (jam)
-rw-r--r--release_docs/RELEASE.txt7
-rw-r--r--tools/h5repack/CMakeLists.txt22
-rw-r--r--tools/h5repack/h5repack.c1
-rw-r--r--tools/h5repack/h5repack.sh.in21
-rw-r--r--tools/h5repack/h5repack_copy.c20
-rw-r--r--tools/h5repack/h5repacktst.c101
-rw-r--r--tools/h5repack/testfiles/h5repack_layout3.h5bin966904 -> 491840 bytes
7 files changed, 150 insertions, 22 deletions
diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt
index ef1b8fa..d3459f5 100644
--- a/release_docs/RELEASE.txt
+++ b/release_docs/RELEASE.txt
@@ -714,6 +714,13 @@ Bug Fixes since HDF5-1.8.0 release
Tools
-----
+ - h5repack: "h5repack -f NONE file1.h5 out.h5" command failed if
+ source file contains chunked dataset and a chunk dim is bigger than
+ the dataset dim. Another issue is that the command changed max dims
+ if chunk dim is smaller than the dataset dim.
+ These issue occurred when dataset size is smaller than 64k (compact
+ size limit) Fixed both.
+ HDFFV-8012 (JKM 2012/09/24)
- h5diff: Fixed not to accumulate attribute difference to dataset
difference in verbose mode (-v, -r), which caused incorrect
difference between dataset and group/datatype object if attribute
diff --git a/tools/h5repack/CMakeLists.txt b/tools/h5repack/CMakeLists.txt
index 3ee1cba..7351fd5 100644
--- a/tools/h5repack/CMakeLists.txt
+++ b/tools/h5repack/CMakeLists.txt
@@ -800,6 +800,28 @@ ADD_H5_VERIFY_TEST (chunk2conti "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CONT
ADD_H5_TEST (chunk2compa "TEST" h5repack_layout3.h5 -l chunk_unlimit1:COMPA)
ADD_H5_VERIFY_TEST (chunk2compa "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 CHUNK)
+#--------------------------------------------------------------------------
+# Test -f for some specific cases. Chunked dataset with unlimited max dims.
+# (HDFFV-8012)
+#--------------------------------------------------------------------------
+# - should not fail
+# - should not change max dims from unlimit
+
+# chunk dim is bigger than dataset dim. ( dset size < 64k )
+ADD_H5_TEST (error1 "TEST" h5repack_layout3.h5 -f chunk_unlimit1:NONE)
+ADD_H5_VERIFY_TEST (error1 "TEST" 0 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED)
+
+# chunk dim is bigger than dataset dim. ( dset size > 64k )
+ADD_H5_TEST (error2 "TEST" h5repack_layout3.h5 -f chunk_unlimit2:NONE)
+ADD_H5_VERIFY_TEST (error2 "TEST" 0 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED)
+
+# chunk dims are smaller than dataset dims. ( dset size < 64k )
+ADD_H5_TEST (error3 "TEST" h5repack_layout3.h5 -f chunk_unlimit3:NONE)
+ADD_H5_VERIFY_TEST (error3 "TEST" 0 h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED)
+
+# file input - should not fail
+ADD_H5_TEST (error4 "TEST" h5repack_layout3.h5 -f NONE)
+
# Native option
# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
# report a difference.
diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c
index 292f4ec..38a000a 100644
--- a/tools/h5repack/h5repack.c
+++ b/tools/h5repack/h5repack.c
@@ -95,6 +95,7 @@ h5repack_init(pack_opt_t *options, int verbose, H5F_file_space_type_t strategy,
HDmemset(options, 0, sizeof(pack_opt_t));
options->min_comp = 1024;
options->verbose = verbose;
+ options->layout_g = H5D_LAYOUT_ERROR;
for ( n = 0; n < H5_REPACK_MAX_NFILTERS; n++)
{
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index 2614dd6..a732ffb 100644
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -818,6 +818,27 @@ VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CONTI
TOOLTEST_MAIN h5repack_layout3.h5 -l chunk_unlimit1:COMPA
VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 CHUNK
+#--------------------------------------------------------------------------
+# Test -f for some specific cases. Chunked dataset with unlimited max dims.
+# (HDFFV-8012)
+#--------------------------------------------------------------------------
+# - should not fail
+# - should not change max dims from unlimit
+
+# chunk dim is bigger than dataset dim. ( dset size < 64k )
+TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit1:NONE
+VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED
+# chunk dim is bigger than dataset dim. ( dset size > 64k )
+TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit2:NONE
+VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED
+
+# chunk dims are smaller than dataset dims. ( dset size < 64k )
+TOOLTEST_MAIN h5repack_layout3.h5 -f chunk_unlimit3:NONE
+VERIFY_LAYOUT_DSET h5repack_layout3.h5 chunk_unlimit3 H5S_UNLIMITED
+
+# file input - should not fail
+TOOLTEST h5repack_layout3.h5 -f NONE
+
# Native option
# Do not use FILE1, as the named dtype will be converted to native, and h5diff will
# report a difference.
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index cf55d7f..02337fd 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -773,6 +773,7 @@ int do_copy_objects(hid_t fidin,
void *hslab_buf=NULL; /* hyperslab buffer for raw data */
int has_filter; /* current object has a filter */
int req_filter; /* there was a request for a filter */
+ int req_obj_layout=0; /* request layout to current object */
unsigned crt_order_flags; /* group creation order flag */
unsigned i;
unsigned u;
@@ -904,6 +905,22 @@ int do_copy_objects(hid_t fidin,
}
}
+ /* check if layout change requested individual object */
+ if (options->layout_g != H5D_LAYOUT_ERROR)
+ {
+ pack_info_t *pckinfo;
+ /* any dataset is specified */
+ if (options->op_tbl->nelems > 0)
+ {
+ /* check if object exist */
+ pckinfo = options_get_object (travt->objs[i].name, options->op_tbl);
+ if (pckinfo)
+ {
+ req_obj_layout = 1;
+ }
+ }
+ }
+
/* early detection of references */
if((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0)
goto error;
@@ -1023,6 +1040,9 @@ int do_copy_objects(hid_t fidin,
goto error;
}
+ /* only if layout change requested for entire file or
+ * individual obj */
+ if (options->all_layout > 0 || req_obj_layout == 1)
/*-------------------------------------------------
* Unset the unlimited max dims if convert to other
* than chunk layouts, because unlimited max dims
diff --git a/tools/h5repack/h5repacktst.c b/tools/h5repack/h5repacktst.c
index 4f1707b..791e268 100644
--- a/tools/h5repack/h5repacktst.c
+++ b/tools/h5repack/h5repacktst.c
@@ -3104,65 +3104,119 @@ out:
*/
#define DIM1_L3 300
#define DIM2_L3 200
+/* small size */
+#define SDIM1_L3 4
+#define SDIM2_L3 50
static
int make_layout3(hid_t loc_id)
{
- hid_t dcpl=-1; /* dataset creation property list */
- hid_t sid=-1; /* dataspace ID */
- hsize_t dims[RANK]={DIM1_L3,DIM2_L3};
+ hid_t dcpl1=-1; /* dataset creation property list */
+ hid_t dcpl2=-1; /* dataset creation property list */
+ hid_t dcpl3=-1; /* dataset creation property list */
+ hid_t sid1=-1; /* dataspace ID */
+ hid_t sid2=-1; /* dataspace ID */
+ hsize_t dims1[RANK]={DIM1_L3,DIM2_L3};
+ hsize_t dims2[RANK]={SDIM1_L3,SDIM2_L3};
hsize_t maxdims[RANK]={H5S_UNLIMITED, H5S_UNLIMITED};
- hsize_t chunk_dims[RANK]={DIM1_L3*2,5};
- int buf[DIM1_L3][DIM2_L3];
+ hsize_t chunk_dims1[RANK]={DIM1_L3*2,5};
+ hsize_t chunk_dims2[RANK]={SDIM1_L3 + 2, SDIM2_L3/2};
+ hsize_t chunk_dims3[RANK]={SDIM1_L3 - 2, SDIM2_L3/2};
+ int buf1[DIM1_L3][DIM2_L3];
+ int buf2[SDIM1_L3][SDIM2_L3];
int i, j, n;
+ /* init buf1 */
for (i=n=0; i<DIM1_L3; i++)
{
for (j=0; j<DIM2_L3; j++)
{
- buf[i][j]=n++;
+ buf1[i][j]=n++;
+ }
+ }
+
+ /* init buf2 */
+ for (i=n=0; i<SDIM1_L3; i++)
+ {
+ for (j=0; j<SDIM2_L3; j++)
+ {
+ buf2[i][j]=n++;
}
}
/*-------------------------------------------------------------------------
- * make several dataset with several layout options
+ * make chunked dataset with
+ * - dset maxdims are UNLIMIT
+ * - a chunk dim is bigger than dset dim
+ * - dset size bigger than compact max (64K)
*-------------------------------------------------------------------------
*/
/* create a space */
- if((sid = H5Screate_simple(RANK, dims, maxdims)) < 0)
+ if((sid1 = H5Screate_simple(RANK, dims1, maxdims)) < 0)
return -1;
/* create a dataset creation property list; the same DCPL is used for all dsets */
- if ((dcpl = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ if ((dcpl1 = H5Pcreate(H5P_DATASET_CREATE)) < 0)
{
goto out;
}
+ if(H5Pset_chunk(dcpl1, RANK, chunk_dims1) < 0)
+ goto out;
+ if (make_dset(loc_id,"chunk_unlimit1",sid1,dcpl1,buf1) < 0)
+ {
+ goto out;
+ }
/*-------------------------------------------------------------------------
- * H5D_CHUNKED
+ * make chunked dataset with
+ * - dset maxdims are UNLIMIT
+ * - a chunk dim is bigger than dset dim
+ * - dset size smaller than compact (64K)
*-------------------------------------------------------------------------
*/
- if(H5Pset_chunk(dcpl, RANK, chunk_dims) < 0)
+
+ /* create a space */
+ if((sid2 = H5Screate_simple(RANK, dims2, maxdims)) < 0)
+ return -1;
+ /* create a dataset creation property list; the same DCPL is used for all dsets */
+ if ((dcpl2 = H5Pcreate(H5P_DATASET_CREATE)) < 0)
goto out;
- if (make_dset(loc_id,"chunk_unlimit1",sid,dcpl,buf) < 0)
- {
+
+ if(H5Pset_chunk(dcpl2, RANK, chunk_dims2) < 0)
goto out;
- }
- if(H5Pset_chunk(dcpl, RANK, chunk_dims) < 0)
+ if (make_dset(loc_id,"chunk_unlimit2",sid2,dcpl2,buf2) < 0)
goto out;
- if (make_dset(loc_id,"chunk_unlimit2",sid,dcpl,buf) < 0)
- {
+ /*-------------------------------------------------------------------------
+ * make chunked dataset with
+ * - dset maxdims are UNLIMIT
+ * - a chunk dims are smaller than dset dims
+ * - dset size smaller than compact (64K)
+ *-------------------------------------------------------------------------
+ */
+ /* create a dataset creation property list; the same DCPL is used for all dsets */
+ if ((dcpl3 = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ goto out;
+
+ if(H5Pset_chunk(dcpl3, RANK, chunk_dims3) < 0)
+ goto out;
+
+ if (make_dset(loc_id,"chunk_unlimit3",sid2,dcpl3,buf2) < 0)
goto out;
- }
/*-------------------------------------------------------------------------
* close space and dcpl
*-------------------------------------------------------------------------
*/
- if(H5Sclose(sid) < 0)
+ if(H5Sclose(sid1) < 0)
goto out;
- if(H5Pclose(dcpl) < 0)
+ if(H5Sclose(sid2) < 0)
+ goto out;
+ if(H5Pclose(dcpl1) < 0)
+ goto out;
+ if(H5Pclose(dcpl2) < 0)
+ goto out;
+ if(H5Pclose(dcpl3) < 0)
goto out;
return 0;
@@ -3170,8 +3224,11 @@ int make_layout3(hid_t loc_id)
out:
H5E_BEGIN_TRY
{
- H5Pclose(dcpl);
- H5Sclose(sid);
+ H5Sclose(sid1);
+ H5Sclose(sid2);
+ H5Pclose(dcpl1);
+ H5Pclose(dcpl2);
+ H5Pclose(dcpl3);
} H5E_END_TRY;
return -1;
}
diff --git a/tools/h5repack/testfiles/h5repack_layout3.h5 b/tools/h5repack/testfiles/h5repack_layout3.h5
index 5319031..7351502 100644
--- a/tools/h5repack/testfiles/h5repack_layout3.h5
+++ b/tools/h5repack/testfiles/h5repack_layout3.h5
Binary files differ