summaryrefslogtreecommitdiffstats
path: root/tools
diff options
context:
space:
mode:
authorAllen Byrne <byrn@hdfgroup.org>2019-10-02 20:02:37 (GMT)
committerAllen Byrne <byrn@hdfgroup.org>2019-10-02 20:02:37 (GMT)
commitd7b76fdf9d39d98bf4fcf48bf342dcc7b18da3cd (patch)
tree04d4a355c483f5c68c1d41f6bd2bd3216984fe11 /tools
parent0728ef8352f9b6a6fb697282f31a071a812d0a9e (diff)
downloadhdf5-d7b76fdf9d39d98bf4fcf48bf342dcc7b18da3cd.zip
hdf5-d7b76fdf9d39d98bf4fcf48bf342dcc7b18da3cd.tar.gz
hdf5-d7b76fdf9d39d98bf4fcf48bf342dcc7b18da3cd.tar.bz2
HDFFV-10919 merge external storage repack
Diffstat (limited to 'tools')
-rw-r--r--tools/h5repack/CMakeLists.txt13
-rw-r--r--tools/h5repack/CMakeTests.cmake126
-rw-r--r--tools/h5repack/Makefile.am7
-rw-r--r--tools/h5repack/h5repack.c238
-rw-r--r--tools/h5repack/h5repack.sh.in113
-rw-r--r--tools/h5repack/h5repack_copy.c108
-rw-r--r--tools/h5repack/h5repack_main.c283
-rw-r--r--tools/h5repack/h5repackgentest.c413
-rw-r--r--tools/h5repack/testfiles/h5repack-help.txt17
-rw-r--r--tools/h5repack/testfiles/h5repack_f32le.h5bin0 -> 2336 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_f32le_ex-0.datbin0 -> 288 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_f32le_ex.h5bin0 -> 1464 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_1d.h5bin0 -> 2096 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_1d_ex-0.dat1
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_1d_ex-1.datbin0 -> 24 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_1d_ex.h5bin0 -> 1504 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_2d.h5bin0 -> 2304 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_2d_ex-0.datbin0 -> 256 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_2d_ex.h5bin0 -> 1472 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_3d.h5bin0 -> 4096 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_3d_ex-0.datbin0 -> 2048 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_int32le_3d_ex.h5bin0 -> 1472 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_uint8be.h5bin0 -> 2304 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_uint8be_ex-0.datbin0 -> 64 bytes
-rw-r--r--tools/h5repack/testfiles/h5repack_uint8be_ex-1.dat1
-rw-r--r--tools/h5repack/testfiles/h5repack_uint8be_ex-2.dat1
-rw-r--r--tools/h5repack/testfiles/h5repack_uint8be_ex-3.dat1
-rw-r--r--tools/h5repack/testfiles/h5repack_uint8be_ex.h5bin0 -> 1608 bytes
-rw-r--r--tools/h5repack/testh5repack_detect_szip.c4
-rw-r--r--tools/lib/io_timer.c2
30 files changed, 993 insertions, 335 deletions
diff --git a/tools/h5repack/CMakeLists.txt b/tools/h5repack/CMakeLists.txt
index fb293f1..b108f31 100644
--- a/tools/h5repack/CMakeLists.txt
+++ b/tools/h5repack/CMakeLists.txt
@@ -112,6 +112,19 @@ if (BUILD_TESTING)
)
endif ()
+# --------------------------------------------------------------------
+# Add the h5repack test executable
+# --------------------------------------------------------------------
+ if (HDF5_BUILD_GENERATORS AND NOT ONLY_SHARED_LIBS)
+ add_executable (h5repackgentest ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/h5repackgentest.c)
+ target_include_directories (h5repackgentest PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")
+ TARGET_C_PROPERTIES (h5repackgentest STATIC)
+ target_link_libraries (h5repackgentest PRIVATE ${HDF5_TOOLS_LIB_TARGET} ${HDF5_LIB_TARGET})
+ set_target_properties (h5repackgentest PROPERTIES FOLDER generator/tools)
+
+ #add_test (NAME h5repackgentest COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5repackgentest>)
+ endif ()
+
include (CMakeTests.cmake)
endif ()
diff --git a/tools/h5repack/CMakeTests.cmake b/tools/h5repack/CMakeTests.cmake
index b1d9a05..a05c4d7 100644
--- a/tools/h5repack/CMakeTests.cmake
+++ b/tools/h5repack/CMakeTests.cmake
@@ -25,10 +25,23 @@
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_deflate.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_early.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_ext.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_f32le.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_f32le_ex-0.dat
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_f32le_ex.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fill.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_filters.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_fletcher.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_hlink.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_1d.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_1d_ex-0.dat
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_1d_ex-1.dat
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_1d_ex.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_2d.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_2d_ex-0.dat
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_2d_ex.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_3d.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_3d_ex-0.dat
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_int32le_3d_ex.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layouto.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_layout2.h5
@@ -41,6 +54,12 @@
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_shuffle.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_soffset.h5
${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_szip.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be.h5
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex-0.dat
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex-1.dat
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex-2.dat
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex-3.dat
+ ${HDF5_TOOLS_H5REPACK_SOURCE_DIR}/testfiles/h5repack_uint8be_ex.h5
# h5diff/testfile
${HDF5_TOOLS_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr1.h5
# tools/testfiles
@@ -551,6 +570,104 @@
endif ()
endmacro ()
+ macro (ADD_H5_EXTERNAL_TEST testname testtype testfile)
+ # canonical file = h5repack_${testfile}.h5 - preexist
+ # external file = h5repack_${testfile}_ex.h5 - preexist
+ # repacked file = h5repack_${testfile}_rp.h5 - created
+ # external data file = h5repack_${testfile}_ex-0.dat
+ if ("${testtype}" STREQUAL "SKIP")
+ if (NOT HDF5_ENABLE_USING_MEMCHECKER)
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}
+ COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${testfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${testfile}"
+ )
+ set_property(TEST H5REPACK_EXTERNAL-${testname} PROPERTY DISABLED)
+ endif ()
+ else ()
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}-clear-objects
+ COMMAND ${CMAKE_COMMAND} -E remove h5repack_${testfile}_rp.h5
+ )
+ set_tests_properties (H5REPACK_EXTERNAL-${testname}-clear-objects PROPERTIES
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+ FIXTURES_REQUIRED clear_h5repack
+ )
+ # make sure external data file 0 is available
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}_CPY
+ COMMAND ${CMAKE_COMMAND} -E copy_if_different
+ "${PROJECT_SOURCE_DIR}/testfiles/h5repack_${testfile}_ex-0.dat" "${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_ex-0.dat"
+ )
+ set_tests_properties (H5REPACK_EXTERNAL-${testname}_CPY PROPERTIES
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+ DEPENDS H5REPACK_EXTERNAL-${testname}-clear-objects
+ )
+ # comparison of known files
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}_DFF1
+ COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5diff${tgt_ext}> --enable-error-stack ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}.h5 ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_ex.h5
+ )
+ set_tests_properties (H5REPACK_EXTERNAL-${testname}_DFF1 PROPERTIES
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+ DEPENDS H5REPACK_EXTERNAL-${testname}_CPY
+ )
+ # repack the external file to the repacked file
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}
+ COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5repack${tgt_ext}> --enable-error-stack ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_ex.h5 ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_rp.h5
+ )
+ set_tests_properties (H5REPACK_EXTERNAL-${testname} PROPERTIES
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+ DEPENDS H5REPACK_EXTERNAL-${testname}_DFF1
+ )
+ # comparison of repacked file to known files
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}_DFF2
+ COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5diff${tgt_ext}> --enable-error-stack ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_rp.h5 ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}.h5
+ )
+ set_tests_properties (H5REPACK_EXTERNAL-${testname}_DFF2 PROPERTIES
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+ DEPENDS H5REPACK_EXTERNAL-${testname}
+ )
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}_DFF3
+ COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5diff${tgt_ext}> --enable-error-stack ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_rp.h5 ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_ex.h5
+ )
+ set_tests_properties (H5REPACK_EXTERNAL-${testname}_DFF3 PROPERTIES
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+ DEPENDS H5REPACK_EXTERNAL-${testname}_DFF2
+ )
+ # invalidate external file by removing its first data file
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}_DATA_RMV
+ COMMAND ${CMAKE_COMMAND} -E remove h5repack_${testfile}_ex-0.dat
+ )
+ set_tests_properties (H5REPACK_EXTERNAL-${testname}_DATA_RMV PROPERTIES
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+ DEPENDS H5REPACK_EXTERNAL-${testname}_DFF3
+ )
+ # verify comparison of repacked file to known file
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}_DFF4
+ COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5diff${tgt_ext}> --enable-error-stack ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_rp.h5 ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}.h5
+ )
+ set_tests_properties (H5REPACK_EXTERNAL-${testname}_DFF4 PROPERTIES
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+ DEPENDS H5REPACK_EXTERNAL-${testname}_DATA_RMV
+ )
+ # verify comparison of repacked file to known external file fails
+ add_test (
+ NAME H5REPACK_EXTERNAL-${testname}_DFF_FAIL
+ COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5diff${tgt_ext}> --enable-error-stack ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_rp.h5 ${PROJECT_BINARY_DIR}/testfiles/h5repack_${testfile}_ex.h5
+ )
+ set_tests_properties (H5REPACK_EXTERNAL-${testname}_DFF_FAIL PROPERTIES
+ WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles"
+ DEPENDS H5REPACK_EXTERNAL-${testname}_DFF4
+ WILL_FAIL "true"
+ )
+ endif ()
+ endmacro ()
+
##############################################################################
##############################################################################
### T H E T E S T S ###
@@ -1109,6 +1226,15 @@
ADD_H5_TEST_META (meta_long h5repack_layout.h5 --metadata_block_size=8192)
##############################################################################
+### E X T E R N A L S T O R A G E T E S T S
+##############################################################################
+ADD_H5_EXTERNAL_TEST (ext_f32le "TEST" f32le -l CONTI)
+ADD_H5_EXTERNAL_TEST (ext_int32le_1d "TEST" int32le_1d -l CONTI)
+ADD_H5_EXTERNAL_TEST (ext_int32le_2d "TEST" int32le_2d -l CONTI)
+ADD_H5_EXTERNAL_TEST (ext_int32le_3d "TEST" int32le_3d -l CONTI)
+ADD_H5_EXTERNAL_TEST (ext_uint8be "TEST" uint8be -l CONTI)
+
+##############################################################################
### P L U G I N T E S T S
##############################################################################
if (BUILD_SHARED_LIBS)
diff --git a/tools/h5repack/Makefile.am b/tools/h5repack/Makefile.am
index dac7aa9..284bca3 100644
--- a/tools/h5repack/Makefile.am
+++ b/tools/h5repack/Makefile.am
@@ -19,12 +19,12 @@
include $(top_srcdir)/config/commence.am
# Include src, test, and tools/lib directories
-AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib
+AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/test -I$(top_srcdir)/tools/lib -I$(top_srcdir)/tools/h5repack
# Test programs and scripts
if BUILD_TESTS_CONDITIONAL
TEST_SCRIPT=h5repack.sh
-TEST_PROG=h5repacktst
+TEST_PROG=h5repacktst h5repackgentest
noinst_PROGRAMS=testh5repack_detect_szip
if HAVE_SHARED_CONDITIONAL
if USE_PLUGINS_CONDITIONAL
@@ -87,7 +87,8 @@ endif
# Temporary files. *.h5 are generated by h5repack. They should
# copied to the testfiles/ directory if update is required.
-CHECK_CLEANFILES+=*.h5 *.bin testfiles/h5diff_attr1.h5 testfiles/tfamily*.h5
+# *.dat files may be created by h5repackgentest.
+CHECK_CLEANFILES+=*.h5 *.dat *.bin testfiles/h5diff_attr1.h5 testfiles/tfamily*.h5
DISTCLEANFILES=h5repack.sh h5repack_plugin.sh
include $(top_srcdir)/config/conclude.am
diff --git a/tools/h5repack/h5repack.c b/tools/h5repack/h5repack.c
index b6f332a..0fd30ac 100644
--- a/tools/h5repack/h5repack.c
+++ b/tools/h5repack/h5repack.c
@@ -41,7 +41,9 @@ static int have_request(pack_opt_t *options);
* Return: 0, ok, -1, fail
*-------------------------------------------------------------------------
*/
-int h5repack(const char* infile, const char* outfile, pack_opt_t *options) {
+int
+h5repack(const char* infile, const char* outfile, pack_opt_t *options)
+{
/* check input */
if (check_options(options) < 0)
return -1;
@@ -55,7 +57,7 @@ int h5repack(const char* infile, const char* outfile, pack_opt_t *options) {
return -1;
return 0;
-}
+} /* end h5repack() */
/*-------------------------------------------------------------------------
* Function: h5repack_init
@@ -84,7 +86,7 @@ h5repack_init(pack_opt_t *options, int verbose, hbool_t latest)
}
return (options_table_init(&(options->op_tbl)));
-}
+} /* end h5repack_init() */
/*-------------------------------------------------------------------------
* Function: h5repack_end
@@ -93,9 +95,11 @@ h5repack_init(pack_opt_t *options, int verbose, hbool_t latest)
*-------------------------------------------------------------------------
*/
-int h5repack_end(pack_opt_t *options) {
+int
+h5repack_end(pack_opt_t *options)
+{
return options_table_free(options->op_tbl);
-}
+} /* end h5repack_end() */
/*-------------------------------------------------------------------------
* Function: h5repack_addfilter
@@ -136,7 +140,7 @@ h5repack_addfilter(const char *str, pack_opt_t *options)
HDfree(obj_list);
return 0;
-}
+} /* end h5repack_addfilter() */
/*-------------------------------------------------------------------------
* Function: h5repack_addlayout
@@ -173,7 +177,7 @@ h5repack_addlayout(const char *str, pack_opt_t *options)
if (pack.layout == H5D_CHUNKED) {
/* -2 means the NONE option, remove chunking
and set the global layout to contiguous */
- if (pack.chunk.rank == -2)
+ if (pack.chunk.rank == -2) /* TODO: fix 'magic number' */
options->layout_g = H5D_CONTIGUOUS;
/* otherwise set the global chunking type */
else {
@@ -190,10 +194,10 @@ h5repack_addlayout(const char *str, pack_opt_t *options)
HDfree(obj_list);
ret_value = 0;
- }
+ } /* end if obj_list exists */
return ret_value;
-}
+} /* end h5repack_addlayout() */
/* Note: The below copy_named_datatype(), named_datatype_free(), copy_attr()
* were located in h5repack_copy.c as static prior to bugfix1726.
@@ -213,8 +217,9 @@ h5repack_addlayout(const char *str, pack_opt_t *options)
* to free the stack.
*-------------------------------------------------------------------------
*/
-hid_t copy_named_datatype(hid_t type_in, hid_t fidout,
- named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options) {
+hid_t
+copy_named_datatype(hid_t type_in, hid_t fidout, named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options)
+{
named_dt_t *dt = *named_dt_head_p; /* Stack pointer */
named_dt_t *dt_ret = NULL; /* Datatype to return */
H5O_info_t oinfo; /* Object info of input dtype */
@@ -248,13 +253,14 @@ hid_t copy_named_datatype(hid_t type_in, hid_t fidout,
/* Check if this type is the one requested */
if (oinfo.addr == dt->addr_in) {
dt_ret = dt;
- } /* end if */
- } /* end if */
- } /* end for */
- } /* end else */
+ }
+ } /* end if named datatype */
+ } /* end for each object in traversal table */
+ } /* end else (create the stack) */
/* Handle the case that the requested datatype was not found. This is
- * possible if the datatype was committed anonymously in the input file. */
+ * possible if the datatype was committed anonymously in the input file.
+ */
if (!dt_ret) {
/* Push the new datatype onto the stack */
if (NULL == (dt_ret = (named_dt_t *)HDmalloc(sizeof(named_dt_t))))
@@ -265,10 +271,11 @@ hid_t copy_named_datatype(hid_t type_in, hid_t fidout,
/* Update the address and id */
dt_ret->addr_in = oinfo.addr;
dt_ret->id_out = -1;
- } /* end if */
+ } /* end if requested datatype not found */
/* If the requested datatype does not yet exist in the output file, copy it
- * anonymously */
+ * anonymously
+ */
if (dt_ret->id_out < 0) {
if (options->use_native == 1)
dt_ret->id_out = H5Tget_native_type(type_in, H5T_DIR_DEFAULT);
@@ -278,18 +285,20 @@ hid_t copy_named_datatype(hid_t type_in, hid_t fidout,
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_native_type-H5Tcopy failed");
if (H5Tcommit_anon(fidout, dt_ret->id_out, H5P_DEFAULT, H5P_DEFAULT) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tcommit_anon failed");
- } /* end if */
+ } /* end if named datatype not yet in output file */
/* Set return value */
ret_value = dt_ret->id_out;
- /* Increment the ref count on id_out, because the calling function will try to close it */
- if(H5Iinc_ref(ret_value) < 0)
+ /* Increment the ref count on id_out, because the calling function will try
+ * to close it. (TODO: fix scope envy)
+ */
+ if (H5Iinc_ref(ret_value) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Iinc_ref failed");
done:
return ret_value;
-} /* end copy_named_datatype */
+} /* end copy_named_datatype() */
/*-------------------------------------------------------------------------
* Function: named_datatype_free
@@ -297,7 +306,9 @@ done:
* Purpose: Frees the stack of named datatypes.
*-------------------------------------------------------------------------
*/
-int named_datatype_free(named_dt_t **named_dt_head_p, int ignore_err) {
+int
+named_datatype_free(named_dt_t **named_dt_head_p, int ignore_err)
+{
named_dt_t *dt = *named_dt_head_p;
int ret_value = -1;
@@ -308,13 +319,13 @@ int named_datatype_free(named_dt_t **named_dt_head_p, int ignore_err) {
dt = dt->next;
HDfree(*named_dt_head_p);
*named_dt_head_p = dt;
- } /* end while */
+ }
ret_value = 0;
done:
return (ret_value);
-} /* end named_datatype_free */
+} /* end named_datatype_free() */
/*-------------------------------------------------------------------------
* Function: copy_attr
@@ -328,8 +339,7 @@ done:
*-------------------------------------------------------------------------
*/
int
-copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
- trav_table_t *travt, pack_opt_t *options)
+copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p, trav_table_t *travt, pack_opt_t *options)
{
int ret_value = 0;
hid_t attr_id = -1; /* attr ID */
@@ -362,7 +372,6 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
if ((attr_id = H5Aopen_by_idx(loc_in, ".", H5_INDEX_CRT_ORDER, H5_ITER_INC, (hsize_t) u, H5P_DEFAULT, H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aopen_by_idx failed");
- /* get name */
if (H5Aget_name(attr_id, (size_t) 255, name) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed");
@@ -384,17 +393,17 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
if ((wtype_id = copy_named_datatype(ftype_id, fidout, named_dt_head_p, travt, options)) < 0) {
H5Fclose(fidout);
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_named_datatype failed");
- } /* end if */
+ }
if (H5Fclose(fidout) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Fclose failed");
- } /* end if */
+ } /* end if datatype is committed and we have a traversal table */
else {
if (options->use_native == 1)
wtype_id = H5Tget_native_type(ftype_id, H5T_DIR_DEFAULT);
else
wtype_id = H5Tcopy(ftype_id);
- } /* end else */
+ } /* end else: uncommitted datatype and/or no traversal table */
/* get the dataspace handle */
if ((space_id = H5Aget_space(attr_id)) < 0)
@@ -411,12 +420,12 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
if ((msize = H5Tget_size(wtype_id)) == 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_size failed");
- /*-------------------------------------------------------------------------
- * object references are a special case. We cannot just copy the buffers,
- * but instead we recreate the reference.
- * This is done on a second sweep of the file that just copies the referenced
- * objects at copy_refs_attr()
- *-------------------------------------------------------------------------
+ /*---------------------------------------------------------------------
+ * object references are a special case. We cannot just copy the
+ * buffers, but instead we recreate the reference.
+ * This is done on a second sweep of the file that just copies the
+ * referenced objects at copy_refs_attr().
+ *---------------------------------------------------------------------
*/
type_class = H5Tget_class(wtype_id);
is_ref = (type_class == H5T_REFERENCE);
@@ -427,7 +436,7 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
is_ref = (is_ref || (H5Tget_class(base_type) == H5T_REFERENCE));
if (H5Tclose(base_type) < 0)
H5TOOLS_INFO(H5E_tools_min_id_g, "H5Tclose base_type failed");
- }
+ } /* end if type_class is variable length or array */
if (type_class == H5T_COMPOUND) {
int nmembers = H5Tget_nmembers(wtype_id);
@@ -442,13 +451,13 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
is_ref = 1;
break;
}
- } /* for (j=0; i<nmembers; j++) */
- } /* if (type_class == H5T_COMPOUND) */
+ } /* end for each member */
+ } /* end if type_class is H5T_COMPOUND */
if (!is_ref) {
- /*-------------------------------------------------------------------------
+ /*-----------------------------------------------------------------
* read to memory
- *-------------------------------------------------------------------------
+ *-----------------------------------------------------------------
*/
buf = (void *)HDmalloc((size_t)(nelmts * msize));
@@ -458,9 +467,9 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
if (H5Aread(attr_id, wtype_id, buf) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aread failed");
- /*-------------------------------------------------------------------------
+ /*-----------------------------------------------------------------
* copy
- *-------------------------------------------------------------------------
+ *-----------------------------------------------------------------
*/
if ((attr_out = H5Acreate2(loc_out, name, wtype_id, space_id, H5P_DEFAULT, H5P_DEFAULT)) < 0)
@@ -483,9 +492,9 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
if (options->verbose)
HDprintf(FORMAT_OBJ_ATTR, "attr", name);
- /*-------------------------------------------------------------------------
+ /*---------------------------------------------------------------------
* close
- *-------------------------------------------------------------------------
+ *---------------------------------------------------------------------
*/
if (H5Sclose(space_id) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sclose failed");
@@ -499,19 +508,18 @@ copy_attr(hid_t loc_in, hid_t loc_out, named_dt_t **named_dt_head_p,
if (H5Aclose(attr_id) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Aclose failed");
attr_id = -1;
- } /* for u */
+ } /* for u (each attribute) */
done:
H5E_BEGIN_TRY {
if (buf) {
- /* Check if we have VL data and string in the attribute's datatype that must
- * be reclaimed */
+ /* Check if we have VL data and string in the attribute's
+ * datatype that must be reclaimed
+ */
if (TRUE == h5tools_detect_vlen(wtype_id))
H5Dvlen_reclaim(wtype_id, space_id, H5P_DEFAULT, buf);
-
- /* Free buf */
HDfree(buf);
- } /* end if */
+ }
H5Aclose(attr_out);
H5Sclose(space_id);
@@ -523,25 +531,27 @@ done:
return ret_value;
} /* end copy_attr() */
-/*-------------------------------------------------------------------------
+/*-----------------------------------------------------------------------------
* Function: check_options
*
* Purpose: print options, checks for invalid options
*
* Return: void, return -1 on error
- *-------------------------------------------------------------------------
+ *-----------------------------------------------------------------------------
*/
-static int check_options(pack_opt_t *options) {
+static int
+check_options(pack_opt_t *options)
+{
int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */
unsigned int i;
int k, j, has_cp = 0, has_ck = 0;
char slayout[30];
/*-------------------------------------------------------------------------
- * objects to layout
- *-------------------------------------------------------------------------
- */
- if (options->verbose && have_request(options) /* only print if requested */) {
+ * Objects to layout
+ *-------------------------------------------------------------------------
+ */
+ if (options->verbose && have_request(options)) {
if (options->all_layout == 1) {
HDprintf("All objects to modify layout are...\n");
switch (options->layout_g) {
@@ -572,7 +582,7 @@ static int check_options(pack_opt_t *options) {
}
else
HDprintf("No all objects to modify layout\n");
- }/* verbose */
+ } /* end if verbose */
for (i = 0; i < options->op_tbl->nelems; i++) {
char* name = options->op_tbl->objs[i].path;
@@ -586,22 +596,22 @@ static int check_options(pack_opt_t *options) {
}
has_ck = 1;
}
- else if (options->op_tbl->objs[i].chunk.rank == -2) {
+ else if (options->op_tbl->objs[i].chunk.rank == -2) { /* TODO: replace 'magic number' */
if (options->verbose)
HDprintf(" <%s> %s\n", name, "NONE (contiguous)");
has_ck = 1;
}
- }
+ } /* end for each object in options */
if (options->all_layout == 1 && has_ck)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid chunking input: 'all' option is present with other objects");
/*-------------------------------------------------------------------------
- * objects to filter
- *-------------------------------------------------------------------------
- */
+ * Objects to filter
+ *-------------------------------------------------------------------------
+ */
- if (options->verbose && have_request(options) /* only print if requested */) {
+ if (options->verbose && have_request(options)) {
if (options->all_filter == 1) {
HDprintf("All objects to apply filter are...\n");
for (k = 0; k < options->n_filter_g; k++) {
@@ -625,12 +635,12 @@ static int check_options(pack_opt_t *options) {
default:
HDprintf(" User Defined %d\n", filtn);
break;
- } /* k */
- };
- }
+ } /* end switch */
+ } /* end for each filter */
+ } /* end if options->all_filter == 1 (TODO: meaning) */
else
HDprintf("No all objects to apply filter\n");
- } /* verbose */
+ } /* end if verbose */
for (i = 0; i < options->op_tbl->nelems; i++) {
pack_info_t pack = options->op_tbl->objs[i];
@@ -638,24 +648,26 @@ static int check_options(pack_opt_t *options) {
for (j = 0; j < pack.nfilters; j++) {
if (options->verbose) {
- if(pack.filter[j].filtn >= 0) {
- if(pack.filter[j].filtn > H5Z_FILTER_SCALEOFFSET)
+ if (pack.filter[j].filtn >= 0) {
+ if (pack.filter[j].filtn > H5Z_FILTER_SCALEOFFSET) {
HDprintf(" <%s> with %s filter %d\n", name, get_sfilter(pack.filter[j].filtn), pack.filter[j].filtn);
- else
+ }
+ else {
HDprintf(" <%s> with %s filter\n", name, get_sfilter(pack.filter[j].filtn));
+ }
}
}
has_cp = 1;
- } /* j */
- } /* i */
+ } /* end for each filter */
+ } /* end for each object in options table */
if (options->all_filter == 1 && has_cp)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid compression input: 'all' option is present with other objects");
/*-------------------------------------------------------------------------
- * check options for the latest format
- *-------------------------------------------------------------------------
- */
+ * Check options for the latest format
+ *-------------------------------------------------------------------------
+ */
if (options->grp_compact < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid maximum number of links to store as header messages");
@@ -667,10 +679,10 @@ static int check_options(pack_opt_t *options) {
if (options->msg_size[i] < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid shared message size");
- /*--------------------------------------------------------------------------------
- * verify new user userblock options; file name must be present
- *---------------------------------------------------------------------------------
- */
+ /*------------------------------------------------------------------------
+ * Verify new user userblock options; file name must be present
+ *------------------------------------------------------------------------
+ */
if (options->ublock_filename != NULL && options->ublock_size == 0) {
if (options->verbose) {
HDprintf("Warning: user block size missing for file %s. Assigning a default size of 1024...\n", options->ublock_filename);
@@ -681,35 +693,37 @@ static int check_options(pack_opt_t *options) {
if (options->ublock_filename == NULL && options->ublock_size != 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "file name missing for user block", options->ublock_filename);
- /*--------------------------------------------------------------------------------
- * verify alignment options; threshold is zero default but alignment not
- *---------------------------------------------------------------------------------
- */
+ /*------------------------------------------------------------------------
+ * Verify alignment options; threshold is zero default but alignment not
+ *------------------------------------------------------------------------
+ */
if (options->alignment == 0 && options->threshold != 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "alignment for H5Pset_alignment missing");
done:
return ret_value;
-}
+} /* end check_options() */
/*-------------------------------------------------------------------------
* Function: check_objects
*
- * Purpose: locate all HDF5 objects in the file and compare with user
- * supplied list
+ * Purpose: Locate all HDF5 objects in the file and compare with user-supplied
+ * list.
*
* Return: 0, ok, -1 no
*-------------------------------------------------------------------------
*/
-static int check_objects(const char* fname, pack_opt_t *options) {
- int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */
+static int
+check_objects(const char* fname, pack_opt_t *options)
+{
hid_t fid = -1;
hid_t did = -1;
hid_t sid = -1;
unsigned int i;
int ifil;
trav_table_t *travt = NULL;
+ int ret_value = 0; /*no need to LEAVE() on ERROR: HERR_INIT(int, SUCCEED) */
/* nothing to do */
if (options->op_tbl->nelems == 0)
@@ -745,7 +759,9 @@ static int check_objects(const char* fname, pack_opt_t *options) {
HDprintf("Opening file. Searching %zu objects to modify ...\n", travt->nobjs);
for (i = 0; i < options->op_tbl->nelems; i++) {
- char* name = options->op_tbl->objs[i].path;
+ pack_info_t obj = options->op_tbl->objs[i];
+ char* name = obj.path;
+
if (options->verbose)
HDprintf(" <%s>", name);
@@ -756,24 +772,24 @@ static int check_objects(const char* fname, pack_opt_t *options) {
if (options->verbose)
HDprintf("...Found\n");
- for (ifil = 0; ifil < options->op_tbl->objs[i].nfilters; ifil++) {
- if (options->op_tbl->objs[i].filter[ifil].filtn < 0)
+ for (ifil = 0; ifil < obj.nfilters; ifil++) {
+ if (obj.filter[ifil].filtn < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "invalid filter");
/* check for extra filter conditions */
- switch (options->op_tbl->objs[i].filter[ifil].filtn) {
+ switch (obj.filter[ifil].filtn) {
/* chunk size must be smaller than pixels per block */
case H5Z_FILTER_SZIP:
{
int j;
hsize_t csize = 1;
- unsigned ppb = options->op_tbl->objs[i].filter[ifil].cd_values[0];
+ unsigned ppb = obj.filter[ifil].cd_values[0];
hsize_t dims[H5S_MAX_RANK];
int rank;
- if (options->op_tbl->objs[i].chunk.rank > 0) {
- rank = options->op_tbl->objs[i].chunk.rank;
+ if (obj.chunk.rank > 0) {
+ rank = obj.chunk.rank;
for (j = 0; j < rank; j++)
- csize *= options->op_tbl->objs[i].chunk.chunk_lengths[j];
+ csize *= obj.chunk.chunk_lengths[j];
}
else {
if ((did = H5Dopen2(fid, name, H5P_DEFAULT)) < 0)
@@ -797,13 +813,13 @@ static int check_objects(const char* fname, pack_opt_t *options) {
HDprintf(" <warning: SZIP settings, chunk size is smaller than pixels per block>\n");
HGOTO_DONE(0);
}
- }
+ } /* end case SZIP */
break;
default:
break;
- }
- } /* for ifil */
- } /* for i */
+ } /* end switch */
+ } /* for ifil (each user-defined filter) */
+ } /* for i (each object in options traversal table) */
done:
H5E_BEGIN_TRY {
@@ -814,7 +830,7 @@ done:
if (travt)
trav_table_free(travt);
return ret_value;
-}
+} /* end check_objects() */
/*-------------------------------------------------------------------------
* Function: have_request
@@ -824,14 +840,15 @@ done:
* Return: 1 yes, 0 no
*-------------------------------------------------------------------------
*/
-static int have_request(pack_opt_t *options) {
+static int
+have_request(pack_opt_t *options)
+{
if (options->all_filter || options->all_layout || options->op_tbl->nelems)
return 1;
return 0;
-
-}
+} /* end have_request() */
/*-------------------------------------------------------------------------
* Function: get_sfilter
@@ -841,8 +858,9 @@ static int have_request(pack_opt_t *options) {
* Return: name of filter, exit on error
*-------------------------------------------------------------------------
*/
-
-static const char* get_sfilter(H5Z_filter_t filtn) {
+static const char *
+get_sfilter(H5Z_filter_t filtn)
+{
if (filtn < 0)
return NULL;
else if (filtn == H5Z_FILTER_NONE)
@@ -861,5 +879,5 @@ static const char* get_sfilter(H5Z_filter_t filtn) {
return "SOFF";
else
return "UD";
-}
+} /* end get_sfilter() */
diff --git a/tools/h5repack/h5repack.sh.in b/tools/h5repack/h5repack.sh.in
index febc063..4710a2d 100644
--- a/tools/h5repack/h5repack.sh.in
+++ b/tools/h5repack/h5repack.sh.in
@@ -101,6 +101,16 @@ $SRC_H5REPACK_TESTFILES/h5repack_refs.h5
$SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5
$SRC_H5REPACK_TESTFILES/h5repack_soffset.h5
$SRC_H5REPACK_TESTFILES/h5repack_szip.h5
+$SRC_H5REPACK_TESTFILES/h5repack_f32le.h5
+$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d.h5
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex.h5
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be.h5
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex.h5
$SRC_H5DIFF_TESTFILES/h5diff_attr1.h5
$SRC_TOOLS_TESTFILES/tfamily00000.h5
$SRC_TOOLS_TESTFILES/tfamily00001.h5
@@ -123,9 +133,18 @@ $SRC_H5REPACK_TESTFILES/ublock.bin
$SRC_H5REPACK_TESTFILES/h5repack.info
$SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl
$SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl
+$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-1.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex-0.dat
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl
$SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst
$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-0.dat
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-1.dat
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-2.dat
+$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-3.dat
$SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst
"
@@ -403,8 +422,94 @@ VERIFY_LAYOUT_ALL()
rm -f $layoutfile
}
+# -----------------------------------------------------------------------------
+# Expect h5diff to fail
+# -----------------------------------------------------------------------------
+DIFFFAIL()
+{
+ VERIFY h5diff unequal $@
+ (
+ cd $TESTDIR
+ $RUNSERIAL $H5DIFF_BIN -q "$@"
+ )
+ RET=$?
+ if [ $RET == 0 ] ; then
+ echo "*FAILED*"
+ nerrors="`expr $nerrors + 1`"
+ else
+ echo " PASSED"
+ fi
+}
+
+# -----------------------------------------------------------------------------
+# Catchall test for repacking with external files
+# Loops over all (internally-listed) cases and applies the given arguments
+# to h5repack.
+# Each test file family must be listed in the for loop at the start of the
+# 'function'.
+# Expects case files to be of format:
+# + h5repack_<NAME>.h5
+# + h5repack_<NAME>_ex.h5
+# + h5repack_<NAME>_ex-<N>.dat
+# ...where NAME is part of the internal list, and N is a positive decimal
+# number; multiple external files (*.dat) are allowed per file, but they must
+# follow the pattern and be in contiguous numerical sequence starting at 0.
+# -----------------------------------------------------------------------------
+VERIFY_EXTERNAL_CONSOLIDATION()
+{
+ for name in f32le int32le_1d int32le_2d int32le_3d uint8be
+ do
+ basename="h5repack_${name}"
+ base_ext="${basename}_ex"
+ base_rpkg="${basename}_rp"
+ canonical="${basename}.h5"
+ external="${base_ext}.h5"
+ ex_dat="${base_ext}-0.dat"
+ ex_dat_mv="__${ext_dat}"
+ repacked="${base_rpkg}.h5"
+ prev_nerrors=$nerrors
+
+ TESTING external consolidation on $name with h5repack $@
+ echo " . . ."
+
+ # baseline comparison of known files
+ DIFFTEST $canonical $external
+
+ # repack the external file, verify integrity
+ ( cd $TESTDIR
+ $H5REPACK_BIN $@ $external $repacked
+ )
+ DIFFTEST $repacked $canonical
+ DIFFTEST $repacked $external
+
+ # invalidate external file by moving its first data file; verify
+ ( cd $TESTDIR
+ mv $ex_dat $ex_dat_mv
+ )
+ DIFFFAIL $canonical $external
+ DIFFFAIL $repacked $external
+ DIFFTEST $canonical $repacked
+
+ # reset test state for re-runs with different h5repack arguments
+ ( cd $TESTDIR
+ mv $ex_dat_mv $ex_dat
+ rm $repacked
+ )
+
+ # flag if any of the above checks failed
+ if [ $prev_nerrors != $nerrors ]
+ then
+ echo ". . . *FAILED*"
+ nerrors="`expr $prev_nerrors + 1`"
+ else
+ echo ". . . PASSED"
+ fi
+ done
+} # end VERIFY_EXTERNAL_CONSOLIDATION
+
+# -----------------------------------------------------------------------------
# same as TOOLTEST, but it uses the old syntax -i input_file -o output_file
-#
+# -----------------------------------------------------------------------------
TOOLTEST0()
{
infile=$2
@@ -695,6 +800,7 @@ fi
##############################################################################
### T H E T E S T S
##############################################################################
+
# prepare for test
COPY_TESTFILES_TO_TESTDIR
@@ -1077,6 +1183,11 @@ TOOLTEST HDFFV-7840 h5diff_attr1.h5
TOOLTEST_META meta_short h5repack_layout.h5 -M 8192
TOOLTEST_META meta_long h5repack_layout.h5 --metadata_block_size=8192
+########################################
+# Testing external storage
+########################################
+VERIFY_EXTERNAL_CONSOLIDATION -l CONTI
+
# Clean up temporary files/directories
CLEAN_TESTFILES_AND_TESTDIR
diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 02a3c00..cdee6eb 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -80,8 +80,8 @@ static herr_t walk_error_callback(H5_ATTR_UNUSED unsigned n, const H5E_error2_t
* -1 no
*-------------------------------------------------------------------------
*/
-
-int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
+int
+copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
{
int ret_value = 0;
hid_t fidin = -1;
@@ -126,6 +126,14 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed to close property list");
}
+ /* Create file access property list */
+ if((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+ HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcreate failed to create file access property list");
+
+ if(options->latest)
+ if(H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
+ HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_libver_bounds failed to set format version bounds");
+
/* Check if we need to create a non-default file creation property list */
if (options->latest || ub_size > 0) {
/* Create file creation property list */
@@ -141,8 +149,7 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
/* Adjust group creation parameters for root group */
/* (So that it is created in "dense storage" form) */
- if (H5Pset_link_phase_change(fcpl, (unsigned) options->grp_compact,
- (unsigned) options->grp_indexed) < 0)
+ if (H5Pset_link_phase_change(fcpl, (unsigned) options->grp_compact, (unsigned) options->grp_indexed) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_link_phase_change failed to adjust group creation parameters for root group");
for (i = 0; i < 5; i++) {
@@ -187,12 +194,6 @@ int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_shared_mesg_index failed to configure the specified shared object header message index");
} /* if (nindex>0) */
- /* Create file access property list */
- if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
- HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcreate failed to create file access property list");
-
- if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
- HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_libver_bounds failed to set property for using latest version of the format");
} /* end if */
} /* end if */
#if defined (H5REPACK_DEBUG_USER_BLOCK)
@@ -333,7 +334,7 @@ done:
trav_table_free(travt);
return ret_value;
-}
+} /* end copy_objects() */
/*-------------------------------------------------------------------------
* Function: Get_hyperslab
@@ -489,7 +490,7 @@ Get_hyperslab(hid_t dcpl_id, int rank_dset, hsize_t dims_dset[],
done:
return ret_value;
-}
+} /* end Get_hyperslab() */
/*-------------------------------------------------------------------------
* Function: do_copy_objects
@@ -545,7 +546,8 @@ done:
*-------------------------------------------------------------------------
*/
-int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
+int
+do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
pack_opt_t *options) /* repack options */
{
int ret_value = 0;
@@ -580,7 +582,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
unsigned crt_order_flags; /* group creation order flag */
unsigned i;
unsigned u;
- unsigned uf;
+ int ifil;
int is_ref = 0;
htri_t is_named;
hbool_t limit_maxdims;
@@ -686,8 +688,8 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
if (options->op_tbl->objs) {
for (u = 0; u < options->op_tbl->nelems; u++) {
if (HDstrcmp(travt->objs[i].name, options->op_tbl->objs[u].path) == 0)
- for (uf = 0; uf < options->op_tbl->objs[uf].nfilters; uf++) {
- if (options->op_tbl->objs[u].filter[uf].filtn > 0)
+ for (ifil = 0; ifil < options->op_tbl->objs[ifil].nfilters; ifil++) {
+ if (options->op_tbl->objs[u].filter[ifil].filtn > 0)
req_filter = 1;
}
}
@@ -745,8 +747,6 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed");
if ((dcpl_in = H5Dget_create_plist(dset_in)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed");
- if ((dcpl_out = H5Pcopy(dcpl_in)) < 0)
- HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcopy failed");
if ((rank = H5Sget_simple_extent_ndims(f_space_id)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_ndims failed");
HDmemset(dims, 0, sizeof dims);
@@ -755,6 +755,19 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
if (H5Dget_space_status(dset_in, &space_status) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space_status failed");
+ /* If the input dataset has external storage, it must be contiguous.
+ * Accordingly, there would be no filter or chunk properties to preserve,
+ * so create a new DCPL.
+ * Otherwise, copy dcpl_in.
+ */
+ if (H5Pget_external_count(dcpl_in)) {
+ if ((dcpl_out = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+ HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcreate failed");
+ }
+ else if ((dcpl_out = H5Pcopy(dcpl_in)) < 0) {
+ HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcopy failed");
+ }
+
nelmts = 1;
for (j = 0; j < rank; j++)
nelmts *= dims[j];
@@ -765,7 +778,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
wtype_id = H5Tget_native_type(ftype_id, H5T_DIR_DEFAULT);
else
wtype_id = H5Tcopy(ftype_id);
- } /* end if */
+ }
if ((msize = H5Tget_size(wtype_id)) == 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_size failed");
@@ -809,7 +822,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
/* only if layout change requested for entire file or
* individual obj */
- if (options->all_layout > 0 || req_obj_layout == 1)
+ if (options->all_layout > 0 || req_obj_layout == 1) {
/*-------------------------------------------------
* Unset the unlimited max dims if convert to other
* than chunk layouts, because unlimited max dims
@@ -836,7 +849,8 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
/* unset unlimited max dims */
if (limit_maxdims)
H5Sset_extent_simple(f_space_id, rank, dims, NULL);
- }
+ } /* end if not chunked */
+ } /* end if layout change requested for entire file or individual object */
/*-------------------------------------------------------------------------
* create the output dataset;
@@ -853,7 +867,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
if ((dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_in, H5P_DEFAULT)) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dcreate2 failed");
apply_f = 0;
- }
+ } /* end if retry dataset create */
/*-------------------------------------------------------------------------
* read/write
@@ -914,7 +928,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
dset_layout = H5Pget_layout(dcpl_out);
if (dset_layout == H5D_CHUNKED)
dcpl_tmp = dcpl_out; /* writing dataset */
- else { /* if reading dataset is chunked */
+ else {
dset_layout = H5Pget_layout(dcpl_in);
if (dset_layout == H5D_CHUNKED)
dcpl_tmp = dcpl_in; /* reading dataset */
@@ -925,6 +939,8 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "Get_hyperslab failed");
hslab_buf = HDmalloc((size_t)hslab_nbytes);
+ if (hslab_buf == NULL)
+ HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "can't allocate space for hyperslab");
hslab_nelmts = hslab_nbytes / p_type_nbytes;
hslab_space = H5Screate_simple(1, &hslab_nelmts, NULL);
@@ -951,12 +967,12 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sselect_hyperslab failed");
if (H5Sselect_hyperslab(hslab_space, H5S_SELECT_SET, zero, NULL, &hs_select_nelmts, NULL) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sselect_hyperslab failed");
- }
+ } /* end if rank > 0 */
else {
H5Sselect_all(f_space_id);
H5Sselect_all(hslab_space);
hs_select_nelmts = 1;
- } /* rank */
+ } /* end (else) rank == 0 */
if(H5Dread(dset_in, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dread failed");
@@ -975,8 +991,8 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
hs_sel_offset[k - 1] = 0;
else
carry = 0;
- } /* k */
- } /* elmtno */
+ }
+ } /* end for (hyperslab selection loop) */
H5Sclose(hslab_space);
/* free */
@@ -984,11 +1000,11 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
HDfree(hslab_buf);
hslab_buf = NULL;
}
- } /* hyperslab read */
- } /* if (nelmts>0 && space_status==H5D_SPACE_STATUS_NOT_ALLOCATED) */
+ } /* end if reading/writing by hyperslab */
+ } /* end if (nelmts > 0 && space_status != H5D_SPACE_STATUS_NOT_ALLOCATED) */
/*-------------------------------------------------------------------------
- * amount of compression used
+ * print amount of compression used
*-------------------------------------------------------------------------
*/
if (options->verbose) {
@@ -1008,14 +1024,14 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
print_dataset_info(dcpl_in, travt->objs[i].name, ratio, 0);
/* print a message that the filter was not applied
- (in case there was a filter)
+ * (in case there was a filter)
*/
if (has_filter && apply_s == 0)
HDprintf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", travt->objs[i].name, (int) options->min_comp);
if (has_filter && apply_f == 0)
HDprintf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name);
- } /* verbose */
+ } /* end if verbose (print compression) */
/*-------------------------------------------------------------------------
* copy attrs
@@ -1027,11 +1043,11 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
/*close */
if (H5Dclose(dset_out) < 0)
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
- }/*!H5T_REFERENCE*/
- }/*h5tools_canreadf*/
+ } /* end if not a reference */
+ } /* end if h5tools_canreadf (filter availability check) */
/*-------------------------------------------------------------------------
- * close
+ * Close
*-------------------------------------------------------------------------
*/
if (H5Tclose(ftype_id) < 0)
@@ -1048,7 +1064,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
}
/*-------------------------------------------------------------------------
- * we do not have request for filter/chunking use H5Ocopy instead
+ * We do not have request for filter/chunking; use H5Ocopy instead
*-------------------------------------------------------------------------
*/
else {
@@ -1080,7 +1096,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed");
/*-------------------------------------------------------------------------
- * copy attrs manually
+ * Copy attrs manually
*-------------------------------------------------------------------------
*/
if ((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0)
@@ -1097,7 +1113,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
if (options->verbose)
HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name);
- } /* end do we have request for filter/chunking */
+ } /* end whether we have request for filter/chunking */
break;
/*-------------------------------------------------------------------------
@@ -1157,8 +1173,8 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
default:
HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "Object type not found");
} /* switch */
- } /* end for */
- } /* end if */
+ } /* end for each object to traverse */
+ } /* end if there are objects */
done:
@@ -1196,7 +1212,7 @@ done:
HDfree(hslab_buf);
return ret_value;
-}
+} /* end do_copy_objects() */
/*-------------------------------------------------------------------------
* Function: print_dataset_info
@@ -1290,8 +1306,8 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr)
default:
HDstrcat(strfilter, "UD ");
break;
- } /* switch */
- }/*i*/
+ } /* end switch */
+ } /* end for each filter */
if (!pr)
HDprintf(FORMAT_OBJ, "dset", objname);
@@ -1304,7 +1320,7 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr)
HDstrcat(str, temp);
HDprintf(FORMAT_OBJ, str, objname);
}
-}
+} /* end print_dataset_info() */
/*-------------------------------------------------------------------------
* Function: copy_user_block
@@ -1374,7 +1390,7 @@ done:
HDclose(outfid);
return ret_value;
-}
+} /* end copy_user_block() */
/*-------------------------------------------------------------------------
* Function: print_user_block
@@ -1447,6 +1463,6 @@ done:
HDclose(fh);
return;
-}
+} /* end print_user_block() */
#endif
diff --git a/tools/h5repack/h5repack_main.c b/tools/h5repack/h5repack_main.c
index 9c96e56..bc6a097 100644
--- a/tools/h5repack/h5repack_main.c
+++ b/tools/h5repack/h5repack_main.c
@@ -23,7 +23,8 @@ static void leave(int ret) H5_ATTR_NORETURN;
/* module-scoped variables */
-static int has_i_o = 0;
+static int has_i = 0;
+static int has_o = 0;
const char *infile = NULL;
const char *outfile = NULL;
@@ -33,28 +34,28 @@ const char *outfile = NULL;
*/
static const char *s_opts = "hVvf:l:m:e:nLc:d:s:u:b:M:t:a:i:o:q:z:E";
static struct long_options l_opts[] = {
- { "help", no_arg, 'h' },
- { "version", no_arg, 'V' },
- { "verbose", no_arg, 'v' },
- { "filter", require_arg, 'f' },
- { "layout", require_arg, 'l' },
- { "minimum", require_arg, 'm' },
- { "file", require_arg, 'e' },
- { "native", no_arg, 'n' },
- { "latest", no_arg, 'L' },
- { "compact", require_arg, 'c' },
- { "indexed", require_arg, 'd' },
- { "ssize", require_arg, 's' },
- { "ublock", require_arg, 'u' },
- { "block", require_arg, 'b' },
+ { "help", no_arg, 'h' },
+ { "version", no_arg, 'V' },
+ { "verbose", no_arg, 'v' },
+ { "filter", require_arg, 'f' },
+ { "layout", require_arg, 'l' },
+ { "minimum", require_arg, 'm' },
+ { "file", require_arg, 'e' },
+ { "native", no_arg, 'n' },
+ { "latest", no_arg, 'L' },
+ { "compact", require_arg, 'c' },
+ { "indexed", require_arg, 'd' },
+ { "ssize", require_arg, 's' },
+ { "ublock", require_arg, 'u' },
+ { "block", require_arg, 'b' },
{ "metadata_block_size", require_arg, 'M' },
- { "threshold", require_arg, 't' },
- { "alignment", require_arg, 'a' },
- { "infile", require_arg, 'i' }, /* -i for backward compability */
- { "outfile", require_arg, 'o' }, /* -o for backward compability */
- { "sort_by", require_arg, 'q' },
- { "sort_order", require_arg, 'z' },
- { "enable-error-stack", no_arg, 'E' },
+ { "threshold", require_arg, 't' },
+ { "alignment", require_arg, 'a' },
+ { "infile", require_arg, 'i' }, /* for backward compability */
+ { "outfile", require_arg, 'o' }, /* for backward compability */
+ { "sort_by", require_arg, 'q' },
+ { "sort_order", require_arg, 'z' },
+ { "enable-error-stack", no_arg, 'E' },
{ NULL, 0, '\0' }
};
@@ -77,7 +78,8 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " -v, --verbose Verbose mode, print object information\n");
PRINTVALSTREAM(rawoutstream, " -V, --version Print version number and exit\n");
PRINTVALSTREAM(rawoutstream, " -n, --native Use a native HDF5 type when repacking\n");
- PRINTVALSTREAM(rawoutstream, " --enable-error-stack Prints messages from the HDF5 error stack as they occur\n");
+ PRINTVALSTREAM(rawoutstream, " --enable-error-stack Prints messages from the HDF5 error stack as they\n");
+ PRINTVALSTREAM(rawoutstream, " occur\n");
PRINTVALSTREAM(rawoutstream, " -L, --latest Use latest version of file format\n");
PRINTVALSTREAM(rawoutstream, " -c L1, --compact=L1 Maximum number of links in header messages\n");
PRINTVALSTREAM(rawoutstream, " -d L2, --indexed=L2 Minimum number of links in the indexed format\n");
@@ -100,8 +102,10 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " U - is a filename.\n");
PRINTVALSTREAM(rawoutstream, " T - is an integer\n");
PRINTVALSTREAM(rawoutstream, " A - is an integer greater than zero\n");
- PRINTVALSTREAM(rawoutstream, " Q - is the sort index type for the input file. It can be \"name\" or \"creation_order\" (default)\n");
- PRINTVALSTREAM(rawoutstream, " Z - is the sort order type for the input file. It can be \"descending\" or \"ascending\" (default)\n");
+ PRINTVALSTREAM(rawoutstream, " Q - is the sort index type for the input file. It can be \"name\" or\n");
+ PRINTVALSTREAM(rawoutstream, " \"creation_order\" (default)\n");
+ PRINTVALSTREAM(rawoutstream, " Z - is the sort order type for the input file. It can be \"descending\" or\n");
+ PRINTVALSTREAM(rawoutstream, " \"ascending\" (default)\n");
PRINTVALSTREAM(rawoutstream, " B - is the user block size, any value that is 512 or greater and is\n");
PRINTVALSTREAM(rawoutstream, " a power of 2 (1024 default)\n");
PRINTVALSTREAM(rawoutstream, " F - is the shared object header message type, any of <dspace|dtype|fill|\n");
@@ -132,9 +136,9 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " NBIT (no parameter)\n");
PRINTVALSTREAM(rawoutstream, " SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type\n");
PRINTVALSTREAM(rawoutstream, " is either IN or DS\n");
- PRINTVALSTREAM(rawoutstream, " UD=<filter_number,filter_flag,cd_value_count,value_1[,value_2,...,value_N]>\n");
- PRINTVALSTREAM(rawoutstream, " required values for filter_number,filter_flag,cd_value_count,value_1\n");
- PRINTVALSTREAM(rawoutstream, " optional values for value_2 to value_N\n");
+ PRINTVALSTREAM(rawoutstream, " UD=<filter_number,filter_flag,cd_value_count,value1[,value2,...,valueN]>\n");
+ PRINTVALSTREAM(rawoutstream, " Required values: filter_number, filter_flag, cd_value_count, value1\n");
+ PRINTVALSTREAM(rawoutstream, " Optional values: value2 to valueN\n");
PRINTVALSTREAM(rawoutstream, " NONE (no parameter)\n");
PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " LAYT - is a string with the format:\n");
@@ -171,7 +175,7 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, "4) h5repack -L -c 10 -s 20:dtype file1 file2\n");
PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, " Using latest file format with maximum compact group size of 10 and\n");
- PRINTVALSTREAM(rawoutstream, " and minimum shared datatype size of 20\n");
+ PRINTVALSTREAM(rawoutstream, " minimum shared datatype size of 20\n");
PRINTVALSTREAM(rawoutstream, "\n");
PRINTVALSTREAM(rawoutstream, "5) h5repack -f SHUF -f GZIP=1 file1 file2\n");
PRINTVALSTREAM(rawoutstream, "\n");
@@ -215,7 +219,7 @@ int read_info(const char *filename, pack_opt_t *options)
int i, rc = 1;
int ret_value = EXIT_SUCCESS;
- if ((fp = HDfopen(filename, "r")) == (FILE *) NULL) {
+ if (NULL == (fp = HDfopen(filename, "r"))) {
error_msg("cannot open options file %s\n", filename);
h5tools_setstatus(EXIT_FAILURE);
ret_value = EXIT_FAILURE;
@@ -224,90 +228,46 @@ int read_info(const char *filename, pack_opt_t *options)
/* cycle until end of file reached */
while (1) {
- rc = fscanf(fp, "%s", stype);
- if (rc == -1)
- break;
-
- /*-------------------------------------------------------------------------
- * filter
- *-------------------------------------------------------------------------
- */
- if (HDstrcmp(stype,"-f") == 0) {
- /* find begining of info */
- i = 0;
- c = '0';
- while (c != ' ') {
- if(fscanf(fp, "%c", &c) < 0 && HDferror(fp)) {
- error_msg("fscanf error\n");
- h5tools_setstatus(EXIT_FAILURE);
- ret_value = EXIT_FAILURE;
- goto done;
- } /* end if */
- if (HDfeof(fp))
- break;
- }
- c = '0';
- /* go until end */
- while (c != ' ') {
- if(fscanf(fp, "%c", &c) < 0 && HDferror(fp)) {
- error_msg("fscanf error\n");
- h5tools_setstatus(EXIT_FAILURE);
- ret_value = EXIT_FAILURE;
- goto done;
- } /* end if */
- comp_info[i] = c;
- i++;
- if (HDfeof(fp))
- break;
- if (c == 10 /*eol*/)
- break;
- }
- comp_info[i - 1] = '\0'; /*cut the last " */
- if (h5repack_addfilter(comp_info, options) == -1) {
- error_msg("could not add compression option\n");
+ /* Info indicator must be for layout or filter */
+ if (HDstrcmp(stype,"-l") && HDstrcmp(stype, "-f")) {
+ error_msg("bad file format for %s", filename);
+ h5tools_setstatus(EXIT_FAILURE);
+ ret_value = EXIT_FAILURE;
+ goto done;
+ }
+
+ /* find begining of info */
+ i = 0;
+ c = '0';
+ while (c != ' ') {
+ if (fscanf(fp, "%c", &c) < 0 && HDferror(fp)) {
+ error_msg("fscanf error\n");
h5tools_setstatus(EXIT_FAILURE);
ret_value = EXIT_FAILURE;
goto done;
}
+ if (HDfeof(fp))
+ break;
}
- /*-------------------------------------------------------------------------
- * layout
- *-------------------------------------------------------------------------
- */
- else if (HDstrcmp(stype,"-l") == 0) {
-
- /* find begining of info */
- i = 0;
- c = '0';
- while (c != ' ') {
- if(fscanf(fp, "%c", &c) < 0 && HDferror(fp)) {
- error_msg("fscanf error\n");
- h5tools_setstatus(EXIT_FAILURE);
- ret_value = EXIT_FAILURE;
- goto done;
- } /* end if */
- if (HDfeof(fp))
- break;
- }
- c = '0';
- /* go until end */
- while (c != ' ') {
- if(fscanf(fp, "%c", &c) < 0 && HDferror(fp)) {
- error_msg("fscanf error\n");
- h5tools_setstatus(EXIT_FAILURE);
- ret_value = EXIT_FAILURE;
- goto done;
- } /* end if */
- comp_info[i] = c;
- i++;
- if (HDfeof(fp))
- break;
- if (c == 10 /*eol*/)
- break;
+ c = '0';
+ /* go until end */
+ while (c != ' ') {
+ if (fscanf(fp, "%c", &c) < 0 && HDferror(fp)) {
+ error_msg("fscanf error\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ ret_value = EXIT_FAILURE;
+ goto done;
}
- comp_info[i - 1] = '\0'; /*cut the last " */
+ comp_info[i++] = c;
+ if (HDfeof(fp))
+ break;
+ if (c == 10 /*eol*/)
+ break;
+ }
+ comp_info[i - 1] = '\0'; /*cut the last " */
+ if (!HDstrcmp(stype, "-l")) {
if (h5repack_addlayout(comp_info, options) == -1) {
error_msg("could not add chunck option\n");
h5tools_setstatus(EXIT_FAILURE);
@@ -315,17 +275,15 @@ int read_info(const char *filename, pack_opt_t *options)
goto done;
}
}
- /*-------------------------------------------------------------------------
- * not valid
- *-------------------------------------------------------------------------
- */
else {
- error_msg("bad file format for %s", filename);
- h5tools_setstatus(EXIT_FAILURE);
- ret_value = EXIT_FAILURE;
- goto done;
+ if (h5repack_addfilter(comp_info, options) == -1) {
+ error_msg("could not add compression option\n");
+ h5tools_setstatus(EXIT_FAILURE);
+ ret_value = EXIT_FAILURE;
+ goto done;
+ }
}
- }
+ } /* end while info-read cycling */
done:
if (fp)
@@ -349,9 +307,9 @@ set_sort_by(const char *form)
{
H5_index_t idx_type = H5_INDEX_UNKNOWN;
- if (HDstrcmp(form,"name")==0) /* H5_INDEX_NAME */
+ if (!HDstrcmp(form, "name"))
idx_type = H5_INDEX_NAME;
- else if (HDstrcmp(form,"creation_order")==0) /* H5_INDEX_CRT_ORDER */
+ else if (!HDstrcmp(form, "creation_order"))
idx_type = H5_INDEX_CRT_ORDER;
return idx_type;
@@ -372,9 +330,9 @@ set_sort_order(const char *form)
{
H5_iter_order_t iter_order = H5_ITER_UNKNOWN;
- if (HDstrcmp(form,"ascending")==0) /* H5_ITER_INC */
+ if (!HDstrcmp(form, "ascending"))
iter_order = H5_ITER_INC;
- else if (HDstrcmp(form,"descending")==0) /* H5_ITER_DEC */
+ else if (!HDstrcmp(form, "descending"))
iter_order = H5_ITER_DEC;
return iter_order;
@@ -393,19 +351,19 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
int ret_value = 0;
/* parse command line options */
- while ((opt = get_option(argc, argv, s_opts, l_opts)) != EOF) {
+ while (EOF != (opt = get_option(argc, argv, s_opts, l_opts))) {
switch ((char) opt) {
/* -i for backward compatibility */
case 'i':
infile = opt_arg;
- has_i_o = 1;
+ has_i++;
break;
/* -o for backward compatibility */
case 'o':
outfile = opt_arg;
- has_i_o = 1;
+ has_o++;
break;
case 'h':
@@ -484,10 +442,10 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
{
int idx = 0;
int ssize = 0;
- char *msgPtr = HDstrchr( opt_arg, ':');
+ char *msgPtr = HDstrchr(opt_arg, ':');
options->latest = TRUE; /* must use latest format */
if (msgPtr == NULL) {
- ssize = HDatoi( opt_arg );
+ ssize = HDatoi(opt_arg);
for (idx = 0; idx < 5; idx++)
options->msg_size[idx] = ssize;
}
@@ -497,15 +455,15 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
HDstrcpy(msgType, msgPtr + 1);
msgPtr[0] = '\0';
ssize = HDatoi( opt_arg );
- if (HDstrncmp(msgType, "dspace",6) == 0)
+ if (!HDstrncmp(msgType, "dspace", 6))
options->msg_size[0] = ssize;
- else if (HDstrncmp(msgType, "dtype", 5) == 0)
+ else if (!HDstrncmp(msgType, "dtype", 5))
options->msg_size[1] = ssize;
- else if (HDstrncmp(msgType, "fill", 4) == 0)
+ else if (!HDstrncmp(msgType, "fill", 4))
options->msg_size[2] = ssize;
- else if (HDstrncmp(msgType, "pline", 5) == 0)
+ else if (!HDstrncmp(msgType, "pline", 5))
options->msg_size[3] = ssize;
- else if (HDstrncmp(msgType, "attr", 4) == 0)
+ else if (!HDstrncmp(msgType, "attr", 4))
options->msg_size[4] = ssize;
}
}
@@ -538,7 +496,7 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
break;
case 'q':
- if((sort_by = set_sort_by(opt_arg)) < 0) {
+ if (H5_INDEX_UNKNOWN == set_sort_by(opt_arg)) {
error_msg(" failed to set sort by form <%s>\n", opt_arg);
h5tools_setstatus(EXIT_FAILURE);
ret_value = -1;
@@ -547,7 +505,7 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
break;
case 'z':
- if((sort_order = set_sort_order(opt_arg)) < 0) {
+ if (set_sort_order(opt_arg) == H5_ITER_UNKNOWN) {
error_msg(" failed to set sort order form <%s>\n", opt_arg);
h5tools_setstatus(EXIT_FAILURE);
ret_value = -1;
@@ -561,18 +519,35 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
default:
break;
- } /* switch */
- } /* while */
-
- if (has_i_o == 0) {
- /* check for file names to be processed */
- if (argc <= opt_ind || argv[opt_ind + 1] == NULL) {
- error_msg("missing file names\n");
- usage(h5tools_getprogname());
- h5tools_setstatus(EXIT_FAILURE);
- ret_value = -1;
- }
- }
+ } /* end switch */
+ } /* end while there are more options to parse */
+
+ /* If neither -i nor -o given, get in and out files positionally */
+ if (0 == (has_i + has_o)) {
+ if (argv[opt_ind] != NULL && argv[opt_ind + 1] != NULL) {
+ infile = argv[opt_ind];
+ outfile = argv[opt_ind + 1];
+
+ if (!HDstrcmp(infile, outfile)) {
+ error_msg("file names cannot be the same\n");
+ usage(h5tools_getprogname());
+ h5tools_setstatus(EXIT_FAILURE);
+ ret_value = -1;
+ }
+ }
+ else {
+ error_msg("file names missing\n");
+ usage(h5tools_getprogname());
+ h5tools_setstatus(EXIT_FAILURE);
+ ret_value = -1;
+ }
+ }
+ else if (has_i != 1 || has_o != 1) {
+ error_msg("filenames must be either both -i -o or both positional\n");
+ usage(h5tools_getprogname());
+ h5tools_setstatus(EXIT_FAILURE);
+ ret_value = -1;
+ }
done:
return ret_value;
@@ -629,28 +604,6 @@ int main(int argc, const char **argv)
if (parse_command_line(argc, argv, &options) < 0)
goto done;
- /* get file names if they were not yet got */
- if (has_i_o == 0) {
-
- if (argv[opt_ind] != NULL && argv[opt_ind + 1] != NULL) {
- infile = argv[opt_ind];
- outfile = argv[opt_ind + 1];
-
- if ( HDstrcmp( infile, outfile ) == 0) {
- error_msg("file names cannot be the same\n");
- usage(h5tools_getprogname());
- h5tools_setstatus(EXIT_FAILURE);
- goto done;
- }
- }
- else {
- error_msg("file names missing\n");
- usage(h5tools_getprogname());
- h5tools_setstatus(EXIT_FAILURE);
- goto done;
- }
- }
-
if (enable_error_stack > 0) {
H5Eset_auto2(H5E_DEFAULT, func, edata);
H5Eset_auto2(H5tools_ERR_STACK_g, tools_func, tools_edata);
diff --git a/tools/h5repack/h5repackgentest.c b/tools/h5repack/h5repackgentest.c
new file mode 100644
index 0000000..12883b7
--- /dev/null
+++ b/tools/h5repack/h5repackgentest.c
@@ -0,0 +1,413 @@
+/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
+ * Copyright by The HDF Group. *
+ * Copyright by the Board of Trustees of the University of Illinois. *
+ * All rights reserved. *
+ * *
+ * This file is part of HDF5. The full HDF5 copyright notice, including *
+ * terms governing use, modification, and redistribution, is contained in *
+ * the COPYING file, which can be found at the root of the source code *
+ * distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. *
+ * If you do not have access to either file, you may request a copy from *
+ * help@hdfgroup.org. *
+ * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
+
+/* For generating files for verifying h5repack with external storage. . .
+ *
+ * Each case file should follow the format of:
+ * + h5repack_<NAME>.h5
+ * + h5repack_<NAME>_ex.h5
+ * + h5repack_<NAME>_ex-<N>.dat
+ * ...where NAME idenfities the type, and N is a positive decimal number;
+ * multiple external files (*.dat) are allowed per file, but they must
+ * follow the pattern and be in contiguous numerical sequence starting at 0.
+ *
+ * Each file typename must be added to the listing for
+ * `VERIFY_EXTERNAL_CONSOLIDATION` in h5repack.sh
+ *
+ * There is no restriction on the name, number, or structure of datasets and
+ * groups in HDF5 file.
+ *
+ * The inluded datatypes should be more than adequate to verify the correctness
+ * of the behavior -- if one type can be consolidated from external storage,
+ * then thay all can.
+ */
+
+#include "hdf5.h"
+#include "H5private.h"
+
+#define MAX_NAME_SIZE 256
+#define FILE_INT32LE_1 "h5repack_int32le_1d"
+#define FILE_INT32LE_2 "h5repack_int32le_2d"
+#define FILE_INT32LE_3 "h5repack_int32le_3d"
+#define FILE_UINT8BE "h5repack_uint8be"
+#define FILE_F32LE "h5repack_f32le"
+
+#define H5REPACKGENTEST_OOPS ret_value = -1; goto done;
+
+#define H5REPACKGENTEST_COMMON_CLEANUP(dcpl, file, space) { \
+ if ((dcpl) != H5P_DEFAULT && (dcpl) != H5I_INVALID_HID) { \
+ (void)H5Pclose((dcpl)); \
+ } \
+ if ((file) != H5I_INVALID_HID) { (void)H5Fclose((file)); } \
+ if ((space) != H5I_INVALID_HID) { (void)H5Sclose((space)); } \
+}
+
+struct external_def {
+ hsize_t type_size;
+ unsigned n_elts_per_file;
+ unsigned n_elts_total;
+};
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Helper function to create and write a dataset to file.
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+__make_dataset(
+ hid_t file_id,
+ const char *dset_name,
+ hid_t mem_type_id,
+ hid_t space_id,
+ hid_t dcpl_id,
+ void *wdata)
+{
+ hid_t dset_id = H5I_INVALID_HID;
+ int ret_value = 0;
+
+ dset_id = H5Dcreate2(
+ file_id,
+ dset_name,
+ mem_type_id,
+ space_id,
+ H5P_DEFAULT,
+ dcpl_id,
+ H5P_DEFAULT);
+ if (dset_id == H5I_INVALID_HID) {
+ H5REPACKGENTEST_OOPS;
+ }
+
+ if (H5Dwrite(
+ dset_id,
+ mem_type_id,
+ H5S_ALL,
+ H5S_ALL,
+ H5P_DEFAULT,
+ wdata)
+ < 0)
+ {
+ H5REPACKGENTEST_OOPS;
+ }
+
+done:
+ if (dset_id != H5I_INVALID_HID) { (void)H5Dclose(dset_id); }
+ return ret_value;
+} /* end __make_dataset() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Helper function to populate the DCPL external storage list.
+ * Creates external files for the DCPL, with each file name following the
+ * convention "<filename>_ex-<num>.dat". Will append `n_external_files` to
+ * the filename list, with each file having space for `n_elts` items of the
+ * type (of size `elt_size`). The numeric inputs are not sanity-checked.
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+__set_dcpl_external_list(
+ hid_t dcpl,
+ const char *filename,
+ unsigned n_elts_per_file,
+ unsigned n_elts_total,
+ hsize_t elt_size)
+{
+ char name[MAX_NAME_SIZE];
+ unsigned n_external_files = 0;
+ unsigned i = 0;
+
+ if (NULL == filename || '\0' == *filename) {
+ return -1;
+ }
+
+ n_external_files = n_elts_total / n_elts_per_file;
+ if (n_elts_total != (n_external_files * n_elts_per_file)) {
+ return -1;
+ }
+
+ for (i = 0; i < n_external_files; i++) {
+ if (snprintf(name, MAX_NAME_SIZE, "%s_ex-%u.dat", filename, i) >= MAX_NAME_SIZE) {
+ return -1;
+ }
+ if (H5Pset_external(dcpl, name, 0, n_elts_per_file * elt_size) < 0) {
+ return -1;
+ }
+ }
+ return 0;
+} /* end __set_dcpl_external_list() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Generalized utility function to write a file with the specified data and
+ * dataset configuration. If `ext` is provided, will attempt to use external
+ * storage.
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+__make_file(
+ const char *basename,
+ struct external_def *ext,
+ hid_t type_id,
+ hsize_t rank,
+ hsize_t *dims,
+ void *wdata)
+{
+ char filename[MAX_NAME_SIZE];
+ hid_t file_id = H5I_INVALID_HID;
+ hid_t dcpl_id = H5P_DEFAULT;
+ hid_t space_id = H5I_INVALID_HID;
+ int ret_value = 0;
+
+ if (snprintf(filename,
+ MAX_NAME_SIZE,
+ "%s%s.h5",
+ basename,
+ (NULL != ext) ? "_ex" : "")
+ >= MAX_NAME_SIZE)
+ {
+ H5REPACKGENTEST_OOPS;
+ }
+
+ if (NULL != ext) {
+ dcpl_id = H5Pcreate(H5P_DATASET_CREATE);
+ if (dcpl_id == H5I_INVALID_HID) {
+ H5REPACKGENTEST_OOPS;
+ }
+ if (__set_dcpl_external_list(
+ dcpl_id,
+ basename,
+ ext->n_elts_per_file,
+ ext->n_elts_total,
+ ext->type_size)
+ < 0)
+ {
+ H5REPACKGENTEST_OOPS;
+ }
+ }
+
+ space_id = H5Screate_simple(rank, dims, NULL);
+ if (space_id == H5I_INVALID_HID) {
+ H5REPACKGENTEST_OOPS;
+ }
+
+ file_id = H5Fcreate(filename, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
+ if (file_id == H5I_INVALID_HID) {
+ H5REPACKGENTEST_OOPS;
+ }
+
+ if (__make_dataset(
+ file_id,
+ "dset",
+ type_id,
+ space_id,
+ dcpl_id,
+ wdata)
+ < 0)
+ {
+ H5REPACKGENTEST_OOPS;
+ }
+
+done:
+ H5REPACKGENTEST_COMMON_CLEANUP(dcpl_id, file_id, space_id);
+ return ret_value;
+} /* end __make_file() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_int32le_1d(hbool_t external)
+{
+ int32_t wdata[12];
+ hsize_t dims[] = {12};
+ struct external_def *def_ptr = NULL;
+ struct external_def def = { (hsize_t)sizeof(int32_t), 6, 12 };
+ int32_t n = 0;
+ int ret_value = 0;
+
+ /* Generate values
+ */
+ for (n = 0; n < 12; n++) {
+ wdata[n] = n-6;
+ }
+
+ def_ptr = (TRUE == external) ? (&def) : NULL;
+ if (__make_file(FILE_INT32LE_1, def_ptr, H5T_STD_I32LE, 1, dims, wdata) < 0) {
+ ret_value = -1;
+ }
+
+ return ret_value;
+} /* end generate_int32le_1d() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_int32le_2d(hbool_t external)
+{
+ int32_t wdata[64];
+ hsize_t dims[] = {8, 8};
+ struct external_def *def_ptr = NULL;
+ struct external_def def = { (hsize_t)sizeof(int32_t), 64, 64 };
+ int32_t n = 0;
+ int ret_value = 0;
+
+ /* Generate values
+ */
+ for (n = 0; n < 64; n++) {
+ wdata[n] = n-32;
+ }
+
+ def_ptr = (TRUE == external) ? (&def) : NULL;
+ if (__make_file(FILE_INT32LE_2, def_ptr, H5T_STD_I32LE, 2, dims, wdata) < 0) {
+ ret_value = -1;
+ }
+
+ return ret_value;
+} /* end generate_int32le_2d() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_int32le_3d(hbool_t external)
+{
+ hsize_t dims[] = {8, 8, 8};
+ int32_t wdata[512]; /* 8^3, from dims */
+ struct external_def *def_ptr = NULL;
+ struct external_def def = { (hsize_t)sizeof(int32_t), 512, 512 };
+ int32_t n = 0;
+ int i = 0;
+ int j = 0;
+ int k = 0;
+ int ret_value = 0;
+
+ /* generate values, alternating positive and negative
+ */
+ for (i=0, n=0; i < dims[0]; i++) {
+ for (j=0; j < dims[1]; j++) {
+ for (k=0; k < dims[2]; k++, n++) {
+ wdata[n] = (k + j*512 + i*4096) * ((n&1) ? (-1) : (1));
+ }
+ }
+ }
+
+ def_ptr = (TRUE == external) ? (&def) : NULL;
+ if (__make_file(FILE_INT32LE_3, def_ptr, H5T_STD_I32LE, 3, dims, wdata) < 0) {
+ ret_value = -1;
+ }
+
+ return ret_value;
+} /* end generate_int32le_3d() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_uint8be(hbool_t external)
+{
+ hsize_t dims[] = {4, 8, 8};
+ uint8_t wdata[256]; /* 4*8*8, from dims */
+ struct external_def *def_ptr = NULL;
+ struct external_def def = { (hsize_t)sizeof(uint8_t), 64, 256 };
+ uint8_t n = 0;
+ int i = 0;
+ int j = 0;
+ int k = 0;
+ int ret_value = 0;
+
+ /* Generate values, ping-pong from ends of range
+ */
+ for (i=0, n=0; i < dims[0]; i++) {
+ for (j=0; j < dims[1]; j++) {
+ for (k=0; k < dims[2]; k++, n++) {
+ wdata[n] = n * ((n&1) ? (-1) : (1));
+ }
+ }
+ }
+
+ def_ptr = (TRUE == external) ? (&def) : NULL;
+ if (__make_file(FILE_UINT8BE, def_ptr, H5T_STD_U8BE, 3, dims, wdata) < 0) {
+ ret_value = -1;
+ }
+
+ return ret_value;
+} /* end generate_uint8be() */
+
+
+/* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
+ * Returns 0 on success, -1 on failure.
+ */
+static int
+generate_f32le(hbool_t external)
+{
+ hsize_t dims[] = {12, 6};
+ float wdata[72]; /* 12*6, from dims */
+ struct external_def *def_ptr = NULL;
+ struct external_def def = {
+ (hsize_t)sizeof(float),
+ 72,
+ 72
+ };
+ float n = 0;
+ int i = 0;
+ int j = 0;
+ int k = 0;
+ int ret_value = 0;
+
+ /* Generate values */
+ for (i=0, k=0, n=0; i < dims[0]; i++) {
+ for (j=0; j < dims[1]; j++, k++, n++) {
+ wdata[k] = n * 801.1 * ((k % 5 == 1) ? (-1) : (1));
+ }
+ }
+
+ def_ptr = (TRUE == external) ? (&def) : NULL;
+ if (__make_file(FILE_F32LE, def_ptr, H5T_IEEE_F32LE, 2, dims, wdata) < 0) {
+ ret_value = -1;
+ }
+
+ return ret_value;
+} /* end generate_f32le() */
+
+
+/* ----------------------------------------------------------------------------
+ * Create files.
+ * Return 0 on success, nonzero on failure.
+ */
+int
+main(void)
+{
+ int i = 0;
+ int ret_value = 0;
+
+ for (i=0; i < 2; i++) {
+ hbool_t external = (i&1) ? TRUE : FALSE;
+ if (ret_value == 0) { ret_value -= generate_int32le_1d(external); }
+ if (ret_value == 0) { ret_value -= generate_int32le_2d(external); }
+ if (ret_value == 0) { ret_value -= generate_int32le_3d(external); }
+ if (ret_value == 0) { ret_value -= generate_uint8be(external); }
+ if (ret_value == 0) { ret_value -= generate_f32le(external); }
+ } /* end for external data storage or not */
+
+ if (ret_value != 0) {
+ HDprintf("A problem occurred!\n");
+ }
+
+ return ret_value;
+} /* end main() */
+
+
diff --git a/tools/h5repack/testfiles/h5repack-help.txt b/tools/h5repack/testfiles/h5repack-help.txt
index 8161157..05b8b21 100644
--- a/tools/h5repack/testfiles/h5repack-help.txt
+++ b/tools/h5repack/testfiles/h5repack-help.txt
@@ -6,7 +6,8 @@ usage: h5repack [OPTIONS] file1 file2
-v, --verbose Verbose mode, print object information
-V, --version Print version number and exit
-n, --native Use a native HDF5 type when repacking
- --enable-error-stack Prints messages from the HDF5 error stack as they occur
+ --enable-error-stack Prints messages from the HDF5 error stack as they
+ occur
-L, --latest Use latest version of file format
-c L1, --compact=L1 Maximum number of links in header messages
-d L2, --indexed=L2 Minimum number of links in the indexed format
@@ -29,8 +30,10 @@ usage: h5repack [OPTIONS] file1 file2
U - is a filename.
T - is an integer
A - is an integer greater than zero
- Q - is the sort index type for the input file. It can be "name" or "creation_order" (default)
- Z - is the sort order type for the input file. It can be "descending" or "ascending" (default)
+ Q - is the sort index type for the input file. It can be "name" or
+ "creation_order" (default)
+ Z - is the sort order type for the input file. It can be "descending" or
+ "ascending" (default)
B - is the user block size, any value that is 512 or greater and is
a power of 2 (1024 default)
F - is the shared object header message type, any of <dspace|dtype|fill|
@@ -61,9 +64,9 @@ usage: h5repack [OPTIONS] file1 file2
NBIT (no parameter)
SOFF=<scale_factor,scale_type> scale_factor is an integer and scale_type
is either IN or DS
- UD=<filter_number,filter_flag,cd_value_count,value_1[,value_2,...,value_N]>
- required values for filter_number,filter_flag,cd_value_count,value_1
- optional values for value_2 to value_N
+ UD=<filter_number,filter_flag,cd_value_count,value1[,value2,...,valueN]>
+ Required values: filter_number, filter_flag, cd_value_count, value1
+ Optional values: value2 to valueN
NONE (no parameter)
LAYT - is a string with the format:
@@ -100,7 +103,7 @@ Examples of use:
4) h5repack -L -c 10 -s 20:dtype file1 file2
Using latest file format with maximum compact group size of 10 and
- and minimum shared datatype size of 20
+ minimum shared datatype size of 20
5) h5repack -f SHUF -f GZIP=1 file1 file2
diff --git a/tools/h5repack/testfiles/h5repack_f32le.h5 b/tools/h5repack/testfiles/h5repack_f32le.h5
new file mode 100644
index 0000000..03a9b57
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_f32le.h5
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_f32le_ex-0.dat b/tools/h5repack/testfiles/h5repack_f32le_ex-0.dat
new file mode 100644
index 0000000..8378fc2
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_f32le_ex-0.dat
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_f32le_ex.h5 b/tools/h5repack/testfiles/h5repack_f32le_ex.h5
new file mode 100644
index 0000000..ca6c49d
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_f32le_ex.h5
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_int32le_1d.h5 b/tools/h5repack/testfiles/h5repack_int32le_1d.h5
new file mode 100644
index 0000000..7d60566
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_1d.h5
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_int32le_1d_ex-0.dat b/tools/h5repack/testfiles/h5repack_int32le_1d_ex-0.dat
new file mode 100644
index 0000000..f75ada5
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_1d_ex-0.dat
@@ -0,0 +1 @@
+ \ No newline at end of file
diff --git a/tools/h5repack/testfiles/h5repack_int32le_1d_ex-1.dat b/tools/h5repack/testfiles/h5repack_int32le_1d_ex-1.dat
new file mode 100644
index 0000000..1f8f3b6
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_1d_ex-1.dat
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_int32le_1d_ex.h5 b/tools/h5repack/testfiles/h5repack_int32le_1d_ex.h5
new file mode 100644
index 0000000..ba4b534
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_1d_ex.h5
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_int32le_2d.h5 b/tools/h5repack/testfiles/h5repack_int32le_2d.h5
new file mode 100644
index 0000000..2f32bd1
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_2d.h5
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_int32le_2d_ex-0.dat b/tools/h5repack/testfiles/h5repack_int32le_2d_ex-0.dat
new file mode 100644
index 0000000..c6bfc64
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_2d_ex-0.dat
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_int32le_2d_ex.h5 b/tools/h5repack/testfiles/h5repack_int32le_2d_ex.h5
new file mode 100644
index 0000000..f6cce59
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_2d_ex.h5
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_int32le_3d.h5 b/tools/h5repack/testfiles/h5repack_int32le_3d.h5
new file mode 100644
index 0000000..7a61ef5
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_3d.h5
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_int32le_3d_ex-0.dat b/tools/h5repack/testfiles/h5repack_int32le_3d_ex-0.dat
new file mode 100644
index 0000000..b505eb4
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_3d_ex-0.dat
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_int32le_3d_ex.h5 b/tools/h5repack/testfiles/h5repack_int32le_3d_ex.h5
new file mode 100644
index 0000000..7c4b750
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_int32le_3d_ex.h5
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_uint8be.h5 b/tools/h5repack/testfiles/h5repack_uint8be.h5
new file mode 100644
index 0000000..0e25211
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_uint8be.h5
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_uint8be_ex-0.dat b/tools/h5repack/testfiles/h5repack_uint8be_ex-0.dat
new file mode 100644
index 0000000..f95b90a
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_uint8be_ex-0.dat
Binary files differ
diff --git a/tools/h5repack/testfiles/h5repack_uint8be_ex-1.dat b/tools/h5repack/testfiles/h5repack_uint8be_ex-1.dat
new file mode 100644
index 0000000..011ab48
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_uint8be_ex-1.dat
@@ -0,0 +1 @@
+@BDFHJLNPRTVXZ\^`bdfhjlnprtvxz|~ \ No newline at end of file
diff --git a/tools/h5repack/testfiles/h5repack_uint8be_ex-2.dat b/tools/h5repack/testfiles/h5repack_uint8be_ex-2.dat
new file mode 100644
index 0000000..069b96d
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_uint8be_ex-2.dat
@@ -0,0 +1 @@
+}{ywusqomkigeca_][YWUSQOMKIGECA \ No newline at end of file
diff --git a/tools/h5repack/testfiles/h5repack_uint8be_ex-3.dat b/tools/h5repack/testfiles/h5repack_uint8be_ex-3.dat
new file mode 100644
index 0000000..240d8ca
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_uint8be_ex-3.dat
@@ -0,0 +1 @@
+?=;97531/-+)'%#!  \ No newline at end of file
diff --git a/tools/h5repack/testfiles/h5repack_uint8be_ex.h5 b/tools/h5repack/testfiles/h5repack_uint8be_ex.h5
new file mode 100644
index 0000000..8787188
--- /dev/null
+++ b/tools/h5repack/testfiles/h5repack_uint8be_ex.h5
Binary files differ
diff --git a/tools/h5repack/testh5repack_detect_szip.c b/tools/h5repack/testh5repack_detect_szip.c
index 6e7a24e..6b86dd5 100644
--- a/tools/h5repack/testh5repack_detect_szip.c
+++ b/tools/h5repack/testh5repack_detect_szip.c
@@ -50,10 +50,10 @@ int main(void)
#ifdef H5_HAVE_FILTER_SZIP
if (h5tools_can_encode(H5Z_FILTER_SZIP) == 1) {
- printf("yes\n");
+ HDprintf("yes\n");
return(1);
}
#endif /* H5_HAVE_FILTER_SZIP */
- printf("no\n");
+ HDprintf("no\n");
return(0);
}
diff --git a/tools/lib/io_timer.c b/tools/lib/io_timer.c
index ab184d3..4ad2423 100644
--- a/tools/lib/io_timer.c
+++ b/tools/lib/io_timer.c
@@ -133,7 +133,7 @@ io_time_t *
set_time(io_time_t *pt, timer_type t, int start_stop)
{
/* sanity check */
- assert(pt);
+ HDassert(pt);
switch(pt->type){
#ifdef H5_HAVE_PARALLEL