diff options
| author | Allen Byrne <byrn@hdfgroup.org> | 2020-08-14 17:16:58 (GMT) |
|---|---|---|
| committer | Allen Byrne <byrn@hdfgroup.org> | 2020-08-14 17:16:58 (GMT) |
| commit | 500d87fd1f5d6068feeb71119bc5d22843203fef (patch) | |
| tree | d98efc114fc684fd835aa31168457e72c833af99 | |
| parent | 044ee6f88c78a70eb3694d8c73fa74c77fde9689 (diff) | |
| download | hdf5-500d87fd1f5d6068feeb71119bc5d22843203fef.zip hdf5-500d87fd1f5d6068feeb71119bc5d22843203fef.tar.gz hdf5-500d87fd1f5d6068feeb71119bc5d22843203fef.tar.bz2 | |
HDFFV-9984 Add options to merge/prune external links during repack
31 files changed, 1511 insertions, 120 deletions
@@ -2663,6 +2663,7 @@ #test files for h5repack ./tools/test/h5repack/testfiles/README ./tools/test/h5repack/testfiles/bounds_latest_latest.h5 +./tools/test/h5repack/testfiles/h5copy_extlinks_src.h5 ./tools/test/h5repack/testfiles/h5repack_aggr.h5 ./tools/test/h5repack/testfiles/h5repack_attr.h5 ./tools/test/h5repack/testfiles/h5repack_attr_refs.h5 @@ -2741,6 +2742,24 @@ ./tools/test/h5repack/testfiles/4_vds.h5-vds_compa-v.ddl ./tools/test/h5repack/testfiles/attrregion.tattrreg.h5.ddl ./tools/test/h5repack/testfiles/dataregion.tdatareg.h5.ddl +./tools/test/h5repack/testfiles/textlink-merge.textlink.h5.tst +./tools/test/h5repack/testfiles/textlink-mergeprune.textlink.h5.ddl +./tools/test/h5repack/testfiles/textlink-prune.textlink.h5.ddl +./tools/test/h5repack/testfiles/textlinkfar-merge.textlinkfar.h5.tst +./tools/test/h5repack/testfiles/textlinkfar-mergeprune.textlinkfar.h5.ddl +./tools/test/h5repack/testfiles/textlinkfar-prune.textlinkfar.h5.ddl +./tools/test/h5repack/testfiles/textlinksrc-merge.textlinksrc.h5.tst +./tools/test/h5repack/testfiles/textlinksrc-mergeprune.textlinksrc.h5.ddl +./tools/test/h5repack/testfiles/textlinksrc-prune.textlinksrc.h5.ddl +./tools/test/h5repack/testfiles/textlinktar-merge.textlinktar.h5.tst +./tools/test/h5repack/testfiles/textlinktar-mergeprune.textlinktar.h5.ddl +./tools/test/h5repack/testfiles/textlinktar-prune.textlinktar.h5.ddl +./tools/test/h5repack/testfiles/tsoftlinks-merge.tsoftlinks.h5.tst +./tools/test/h5repack/testfiles/tsoftlinks-mergeprune.tsoftlinks.h5.ddl +./tools/test/h5repack/testfiles/tsoftlinks-prune.tsoftlinks.h5.ddl +./tools/test/h5repack/testfiles/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst +./tools/test/h5repack/testfiles/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5.ddl +./tools/test/h5repack/testfiles/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5.ddl # jam utility and tests ./tools/src/h5jam/Makefile.am diff --git a/release_docs/RELEASE.txt b/release_docs/RELEASE.txt index 9cb9d53..45dee05 100644 --- a/release_docs/RELEASE.txt +++ b/release_docs/RELEASE.txt @@ -699,6 +699,17 @@ New Features Tools: ------ + - h5repack added options to control how external links are handled. + + Currently h5repack preserves external links and cannot copy and merge + data from the external files. Two options, merge and prune, were added to + control how to merge data from an external link into the resulting file. + --merge Follow external soft link recursively and merge data. + --prune Do not follow external soft links and remove link. + --merge --prune Follow external link, merge data and remove dangling link. + + (ADB - 2020/08/05, HDFFV-9984) + - h5repack was fixed to repack the reference attributes properly. The code line that checks if the update of reference inside a compound datatype is misplaced outside the code block loop that carries out the diff --git a/tools/lib/h5tools_utils.c b/tools/lib/h5tools_utils.c index 1a1c2db..6167dd9 100644 --- a/tools/lib/h5tools_utils.c +++ b/tools/lib/h5tools_utils.c @@ -901,7 +901,7 @@ tmpfile(void) * link_info->trg_path must be freed out of this function *-------------------------------------------------------------------------*/ int -H5tools_get_symlink_info(hid_t file_id, const char * linkpath, h5tool_link_info_t *link_info, hbool_t get_obj_type) +H5tools_get_symlink_info(hid_t file_id, const char *linkpath, h5tool_link_info_t *link_info, hbool_t get_obj_type) { htri_t l_ret; H5O_info2_t trg_oinfo; diff --git a/tools/src/h5copy/h5copy.c b/tools/src/h5copy/h5copy.c index 5b19ae7..a6e4d4e 100644 --- a/tools/src/h5copy/h5copy.c +++ b/tools/src/h5copy/h5copy.c @@ -212,8 +212,8 @@ main (int argc, const char *argv[]) unsigned flag = 0; unsigned verbose = 0; unsigned parents = 0; - hid_t ocpl_id = (-1); /* Object copy property list */ - hid_t lcpl_id = (-1); /* Link creation property list */ + hid_t ocpl_id = H5I_INVALID_HID; /* Object copy property list */ + hid_t lcpl_id = H5I_INVALID_HID; /* Link creation property list */ int opt; int li_ret; h5tool_link_info_t linkinfo; diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h index caa1166..b124fdb 100644 --- a/tools/src/h5repack/h5repack.h +++ b/tools/src/h5repack/h5repack.h @@ -101,25 +101,27 @@ typedef struct { /* all the above, ready to go to the hrepack call */ typedef struct { - pack_opttbl_t *op_tbl; /*table with all -c and -f options */ - int all_layout; /*apply the layout to all objects */ - int all_filter; /*apply the filter to all objects */ + pack_opttbl_t *op_tbl; /* table with all -c and -f options */ + int all_layout; /* apply the layout to all objects */ + int all_filter; /* apply the filter to all objects */ filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */ - int n_filter_g; /*number of global filters */ - chunk_info_t chunk_g; /*global chunk INFO for the ALL case */ - H5D_layout_t layout_g; /*global layout information for the ALL case */ - int verbose; /*verbose mode */ - hsize_t min_comp; /*minimum size to compress, in bytes */ - int use_native; /*use a native type in write */ - hbool_t latest; /*pack file with the latest file format */ - H5F_libver_t low_bound; /* The file's low bound as in H5Fset_libver_bounds() */ - H5F_libver_t high_bound; /* The file's high bound as in H5Fset_libver_bounds() */ - hid_t fin_fapl; /* FAPL to use for opening the input file */ - hid_t fout_fapl; /* FAPL to use for opening/creating the output file */ - int grp_compact; /* Set the maximum number of links to store as header messages in the group */ - int grp_indexed; /* Set the minimum number of links to store in the indexed format */ - int msg_size[8]; /* Minimum size of shared messages: dataspace, - datatype, fill value, filter pipleline, attribute */ + int n_filter_g; /* number of global filters */ + chunk_info_t chunk_g; /* global chunk INFO for the ALL case */ + H5D_layout_t layout_g; /* global layout information for the ALL case */ + int verbose; /* verbose mode */ + hbool_t merge; /* Merge external file. */ + hbool_t prune; /* Don't follow external file. */ + hsize_t min_comp; /* minimum size to compress, in bytes */ + int use_native; /* use a native type in write */ + hbool_t latest; /* pack file with the latest file format */ + H5F_libver_t low_bound; /* The file's low bound as in H5Fset_libver_bounds() */ + H5F_libver_t high_bound; /* The file's high bound as in H5Fset_libver_bounds() */ + hid_t fin_fapl; /* FAPL to use for opening the input file */ + hid_t fout_fapl; /* FAPL to use for opening/creating the output file */ + int grp_compact; /* Set the maximum number of links to store as header messages in the group */ + int grp_indexed; /* Set the minimum number of links to store in the indexed format */ + int msg_size[8]; /* Minimum size of shared messages: dataspace, + datatype, fill value, filter pipleline, attribute */ const char *ublock_filename; /* user block file name */ hsize_t ublock_size; /* user block size */ hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */ diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c index b1d6ab4..8f3496f 100644 --- a/tools/src/h5repack/h5repack_copy.c +++ b/tools/src/h5repack/h5repack_copy.c @@ -594,6 +594,8 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, hid_t f_space_id = H5I_INVALID_HID; /* file space ID */ hid_t ftype_id = H5I_INVALID_HID; /* file type ID */ hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */ + hid_t ocpl_id = H5I_INVALID_HID; /* property to pass copy options */ + hid_t lcpl_id = H5I_INVALID_HID; /* link creation property list */ named_dt_t *named_dt_head = NULL; /* Pointer to the stack of named datatypes copied */ size_t msize; /* size of type */ hsize_t nelmts; /* number of elements in dataset */ @@ -610,6 +612,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, int req_filter; /* there was a request for a filter */ int req_obj_layout = 0; /* request layout to current object */ unsigned crt_order_flags; /* group creation order flag */ + h5tool_link_info_t linkinfo; unsigned i; unsigned u; int ifil; @@ -619,6 +622,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, hsize_t size_dset; int ret_value = 0; + /* init linkinfo struct */ + HDmemset(&linkinfo, 0, sizeof(h5tool_link_info_t)); + /*------------------------------------------------------------------------- * copy the supplied object list *------------------------------------------------------------------------- @@ -1137,26 +1143,25 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, *------------------------------------------------------------------------- */ else { - hid_t pid = H5I_INVALID_HID; - /* create property to pass copy options */ - if ((pid = H5Pcreate(H5P_OBJECT_COPY)) < 0) + if ((ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Pcreate failed"); /* set options for object copy */ - if (H5Pset_copy_object(pid, H5O_COPY_WITHOUT_ATTR_FLAG) < 0) + if (H5Pset_copy_object(ocpl_id, H5O_COPY_WITHOUT_ATTR_FLAG) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed"); if (H5Ocopy(fidin, /* Source file or group identifier */ travt->objs[i].name, /* Name of the source object to be copied */ fidout, /* Destination file or group identifier */ travt->objs[i].name, /* Name of the destination object */ - pid, /* Properties which apply to the copy */ + ocpl_id, /* Properties which apply to the copy */ H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */ H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed"); - if (H5Pclose(pid) < 0) + if (H5Pclose(ocpl_id) < 0) H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed"); + ocpl_id = H5I_INVALID_HID; /*------------------------------------------------------------------------- * Copy attrs manually @@ -1228,11 +1233,61 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, if (options->verbose) HDprintf(FORMAT_OBJ, "link", travt->objs[i].name); - if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0) - H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed"); + /* Check -X option. */ + if (options->merge) { + if (H5tools_get_symlink_info(fidin, travt->objs[i].name, &linkinfo, 1) == 0) { + /* dangling link */ + if (options->prune) { + HDprintf("Pruned %s.\n", travt->objs[i].name); + } + else { + if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed"); + } + } + else { + /* valid link */ + /* create property to pass copy options */ + if ((ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pcreate create property failed"); - if (options->verbose) - HDprintf(FORMAT_OBJ, "link", travt->objs[i].name); + /* set options for object copy */ + if (H5Pset_copy_object(ocpl_id, H5O_COPY_EXPAND_EXT_LINK_FLAG) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed"); + + /* Create link creation property list */ + if((lcpl_id = H5Pcreate(H5P_LINK_CREATE)) < 0) { + H5TOOLS_GOTO_ERROR((-1), "H5Pcreate link creation property failed"); + } + + /* Set flag for intermediate group creation */ + if (H5Pset_create_intermediate_group(lcpl_id, TRUE) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pset_create_intermediate_group failed"); + + if (H5Ocopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, ocpl_id, lcpl_id) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed"); + + if (H5Pclose(lcpl_id) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed"); + + if (H5Pclose(ocpl_id) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed"); + } + + /* free link info path */ + if (linkinfo.trg_path) + HDfree(linkinfo.trg_path); + linkinfo.trg_path = NULL; + } /* options->merge */ + else { + if (options->prune) { + HDprintf("Pruned %s.\n", travt->objs[i].name); + } + else { + if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0) + H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed"); + } + } break; default: @@ -1249,15 +1304,22 @@ done: if (named_datatype_free(&named_dt_head, 0) < 0) H5TOOLS_ERROR((-1), "named_datatype_free failed"); } - else + else { H5E_BEGIN_TRY { named_datatype_free(&named_dt_head, 1); } H5E_END_TRY; + } + + /* free link info path */ + if (linkinfo.trg_path) + HDfree(linkinfo.trg_path); H5E_BEGIN_TRY { H5Gclose(grp_in); H5Gclose(grp_out); + H5Pclose(lcpl_id); + H5Pclose(ocpl_id); H5Pclose(dcpl_in); H5Pclose(gcpl_in); H5Pclose(gcpl_out); diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c index 0ad61c0..cf0c611 100644 --- a/tools/src/h5repack/h5repack_main.c +++ b/tools/src/h5repack/h5repack_main.c @@ -32,36 +32,38 @@ const char *outfile = NULL; * Command-line options: The user can specify short or long-named * parameters. */ -static const char *s_opts = "hVvf:l:m:e:nLj:k:c:d:s:u:b:M:t:a:i:o:S:P:T:G:q:z:E"; +static const char *s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:vz:EG:LM:P:S:T:VXW1:2:3:4:5:6:"; static struct long_options l_opts[] = { - { "help", no_arg, 'h' }, - { "version", no_arg, 'V' }, - { "verbose", no_arg, 'v' }, + { "alignment", require_arg, 'a' }, + { "block", require_arg, 'b' }, + { "compact", require_arg, 'c' }, + { "indexed", require_arg, 'd' }, + { "file", require_arg, 'e' }, { "filter", require_arg, 'f' }, + { "help", no_arg, 'h' }, + { "infile", require_arg, 'i' }, /* for backward compability */ + { "low", require_arg, 'j' }, + { "high", require_arg, 'k' }, { "layout", require_arg, 'l' }, { "minimum", require_arg, 'm' }, - { "file", require_arg, 'e' }, { "native", no_arg, 'n' }, - { "latest", no_arg, 'L' }, - { "low", require_arg, 'j' }, - { "high", require_arg, 'k' }, - { "compact", require_arg, 'c' }, - { "indexed", require_arg, 'd' }, + { "outfile", require_arg, 'o' }, /* for backward compability */ + { "sort_by", require_arg, 'q' }, { "ssize", require_arg, 's' }, + { "threshold", require_arg, 't' }, { "ublock", require_arg, 'u' }, - { "block", require_arg, 'b' }, + { "verbose", no_arg, 'v' }, + { "sort_order", require_arg, 'z' }, + { "enable-error-stack", no_arg, 'E' }, + { "fs_pagesize", require_arg, 'G' }, + { "latest", no_arg, 'L' }, { "metadata_block_size", require_arg, 'M' }, - { "threshold", require_arg, 't' }, - { "alignment", require_arg, 'a' }, - { "infile", require_arg, 'i' }, /* for backward compability */ - { "outfile", require_arg, 'o' }, /* for backward compability */ - { "fs_strategy", require_arg, 'S' }, { "fs_persist", require_arg, 'P' }, + { "fs_strategy", require_arg, 'S' }, { "fs_threshold", require_arg, 'T' }, - { "fs_pagesize", require_arg, 'G' }, - { "sort_by", require_arg, 'q' }, - { "sort_order", require_arg, 'z' }, - { "enable-error-stack", no_arg, 'E' }, + { "version", no_arg, 'V' }, + { "merge", no_arg, 'X' }, + { "prune", no_arg, 'W' }, { "src-vol-value", require_arg, '1' }, { "src-vol-name", require_arg, '2' }, { "src-vol-info", require_arg, '3' }, @@ -113,6 +115,9 @@ static void usage(const char *prog) { PRINTVALSTREAM(rawoutstream, " --high=BOUND The high bound for library release versions to use\n"); PRINTVALSTREAM(rawoutstream, " when creating objects in the file\n"); PRINTVALSTREAM(rawoutstream, " (default is H5F_LIBVER_LATEST)\n"); + PRINTVALSTREAM(rawoutstream, " --merge Follow external soft link recursively and merge data\n"); + PRINTVALSTREAM(rawoutstream, " --prune Do not follow external soft links and remove link\n"); + PRINTVALSTREAM(rawoutstream, " --merge --prune Follow external link, merge data and remove dangling link\n"); PRINTVALSTREAM(rawoutstream, " -c L1, --compact=L1 Maximum number of links in header messages\n"); PRINTVALSTREAM(rawoutstream, " -d L2, --indexed=L2 Minimum number of links in the indexed format\n"); PRINTVALSTREAM(rawoutstream, " -s S[:F], --ssize=S[:F] Shared object header message minimum size\n"); @@ -545,6 +550,14 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options) options->high_bound = bound; break; + case 'X': + options->merge = TRUE; + break; + + case 'W': + options->prune = TRUE; + break; + case 'c': options->grp_compact = HDatoi( opt_arg ); if (options->grp_compact > 0) diff --git a/tools/test/h5diff/h5diffgentest.c b/tools/test/h5diff/h5diffgentest.c index afdf46c..173a5ce 100644 --- a/tools/test/h5diff/h5diffgentest.c +++ b/tools/test/h5diff/h5diffgentest.c @@ -8035,7 +8035,6 @@ void test_double_epsilon(const char *fname1, const char *fname2) { hid_t fid1 = H5I_INVALID_HID, fid2 = H5I_INVALID_HID; hsize_t dims1[2] = { 4, 7 }; - hsize_t dims2[2] = { 4, 7 }; double wdata[4][7]; int i, j; diff --git a/tools/test/h5repack/CMakeTests.cmake b/tools/test/h5repack/CMakeTests.cmake index 4823d07..2a34438 100644 --- a/tools/test/h5repack/CMakeTests.cmake +++ b/tools/test/h5repack/CMakeTests.cmake @@ -72,6 +72,14 @@ ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5repack_paged_persist.h5 # h5diff/testfile ${HDF5_TOOLS_TEST_H5DIFF_SOURCE_DIR}/testfiles/h5diff_attr1.h5 + # tools/testfiles for external links + ${HDF5_TOOLS_DIR}/testfiles/tsoftlinks.h5 + ${HDF5_TOOLS_DIR}/testfiles/textlinkfar.h5 + ${HDF5_TOOLS_DIR}/testfiles/textlinksrc.h5 + ${HDF5_TOOLS_DIR}/testfiles/textlinktar.h5 + ${HDF5_TOOLS_DIR}/testfiles/textlink.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5copy_extlinks_src.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5copy_extlinks_trg.h5 # tools/testfiles ${HDF5_TOOLS_DIR}/testfiles/tfamily00000.h5 ${HDF5_TOOLS_DIR}/testfiles/tfamily00001.h5 @@ -128,6 +136,13 @@ ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/plugin_version_test.h5repack_layout.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/plugin_zero.h5repack_layout.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/plugin_none.h5repack_layout.UD.h5 + # tools/testfiles for external links + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/tsoftlinks-merge.tsoftlinks.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlinkfar-merge.textlinkfar.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlinksrc-merge.textlinksrc.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlinktar-merge.textlinktar.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlink-merge.textlink.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5 ) set (LIST_DDL_TEST_FILES @@ -154,6 +169,19 @@ # refs ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/attrregion.tattrreg.h5 ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/dataregion.tdatareg.h5 + # tools/testfiles for external links + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlinkfar-prune.textlinkfar.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlinksrc-prune.textlinksrc.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlinktar-prune.textlinktar.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlink-prune.textlink.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/tsoftlinks-prune.tsoftlinks.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlinkfar-mergeprune.textlinkfar.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlinksrc-mergeprune.textlinksrc.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlinktar-mergeprune.textlinktar.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/textlink-mergeprune.textlink.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/tsoftlinks-mergeprune.tsoftlinks.h5 + ${HDF5_TOOLS_TEST_H5REPACK_SOURCE_DIR}/testfiles/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5 ) foreach (h5_file ${LIST_HDF5_TEST_FILES}) @@ -276,7 +304,7 @@ endif () endmacro () - macro (ADD_H5_CMP_TEST testname testfilter testtype resultcode resultfile) + macro (ADD_H5_FILTER_TEST testname testfilter testtype resultcode resultfile) if ("${testtype}" STREQUAL "SKIP") if (NOT HDF5_ENABLE_USING_MEMCHECKER) add_test ( @@ -410,6 +438,48 @@ endif () endmacro () + macro (ADD_H5_DIFF_TEST testname testtype resultcode testfile) + if ("${testtype}" STREQUAL "SKIP") + if (NOT HDF5_ENABLE_USING_MEMCHECKER) + add_test ( + NAME H5REPACK_DIFF-${testname} + COMMAND ${CMAKE_COMMAND} -E echo "SKIP ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${testfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${testfile}" + ) + set_property(TEST H5REPACK_DIFF-${testname} PROPERTY DISABLED) + endif () + else () + add_test ( + NAME H5REPACK_DIFF-${testname}-clear-objects + COMMAND ${CMAKE_COMMAND} -E remove testfiles/out-${testname}.${testfile} + ) + set_tests_properties (H5REPACK_DIFF-${testname}-clear-objects PROPERTIES + FIXTURES_REQUIRED clear_h5repack + ) + add_test ( + NAME H5REPACK_DIFF-${testname} + COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5repack${tgt_file_ext}> --enable-error-stack ${ARGN} ${PROJECT_BINARY_DIR}/testfiles/${testfile} ${PROJECT_BINARY_DIR}/testfiles/out-${testname}.${testfile} + ) + set_tests_properties (H5REPACK_DIFF-${testname} PROPERTIES + DEPENDS H5REPACK_DIFF-${testname}-clear-objects + ) + add_test ( + NAME H5REPACK_DIFF-${testname}_DFF + COMMAND "${CMAKE_COMMAND}" + -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}" + -D "TEST_PROGRAM=$<TARGET_FILE:h5diff${tgt_file_ext}>" + -D "TEST_ARGS:STRING=-v;--enable-error-stack;${testfile};out-${testname}.${testfile}" + -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles" + -D "TEST_OUTPUT=out-${testname}.${testfile}.out" + -D "TEST_EXPECT=${resultcode}" + -D "TEST_REFERENCE=${testname}.${testfile}.tst" + -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake" + ) + set_tests_properties (H5REPACK_DIFF-${testname}_DFF PROPERTIES + DEPENDS H5REPACK_DIFF-${testname} + ) + endif () + endmacro () + macro (ADD_H5_STAT_TEST testname testtype resultcode statarg resultfile) if ("${testtype}" STREQUAL "SKIP") if (NOT HDF5_ENABLE_USING_MEMCHECKER) @@ -1148,7 +1218,7 @@ if (NOT USE_FILTER_DEFLATE) set (TESTTYPE "SKIP") endif () - ADD_H5_CMP_TEST (gzip_verbose_filters "O?...ing file[^\n]+\n" ${TESTTYPE} 0 ${arg}) + ADD_H5_FILTER_TEST (gzip_verbose_filters "O?...ing file[^\n]+\n" ${TESTTYPE} 0 ${arg}) ########################################################### # the following tests assume the input files have filters @@ -1540,6 +1610,45 @@ ADD_H5_EXTERNAL_TEST (ext_int32le_3d "TEST" int32le_3d -l CONTI) ADD_H5_EXTERNAL_TEST (ext_uint8be "TEST" uint8be -l CONTI) ############################################################################## +### E X T E R N A L L I N K T E S T S +############################################################################## +### HDFFV-11128 needs fixed to enable the following test +#ADD_H5_DMP_TEST (h5copy_extlinks_src-base "TEST" 0 h5copy_extlinks_src.h5 --enable-error-stack) +ADD_H5_DMP_TEST (tsoftlinks-base "TEST" 0 tsoftlinks.h5 --enable-error-stack) +ADD_H5_DMP_TEST (textlink-base "TEST" 0 textlink.h5 --enable-error-stack) +ADD_H5_DMP_TEST (textlinkfar-base "TEST" 0 textlinkfar.h5 --enable-error-stack) +ADD_H5_DMP_TEST (textlinksrc-base "TEST" 0 textlinksrc.h5 --enable-error-stack) +ADD_H5_DMP_TEST (textlinktar-base "TEST" 0 textlinktar.h5 --enable-error-stack) + +### HDFFV-11128 needs fixed to enable the following test +#ADD_H5_DIFF_TEST (textlinksrc-merge "TEST" 1 textlinksrc.h5 --merge) +ADD_H5_DIFF_TEST (tsoftlinks-merge "TEST" 1 tsoftlinks.h5 --merge) +ADD_H5_DIFF_TEST (textlink-merge "TEST" 0 textlink.h5 --merge) +### HDFFV-11128 needs fixed to enable the following test +#ADD_H5_DIFF_TEST (textlinkfar-merge "TEST" 1 textlinkfar.h5 --merge) +### HDFFV-11128 needs fixed to enable the following test +#ADD_H5_DIFF_TEST (textlinktar-merge "TEST" 1 textlinktar.h5 --merge) +### HDFFV-11128 needs fixed to enable the following test +#ADD_H5_DIFF_TEST (h5copy_extlinks_src-merge "TEST" 0 h5copy_extlinks_src.h5 --merge) + +ADD_H5_DMP_TEST (h5copy_extlinks_src-prune "TEST" 0 h5copy_extlinks_src.h5 --prune --enable-error-stack) +ADD_H5_DMP_TEST (tsoftlinks-prune "TEST" 0 tsoftlinks.h5 --prune --enable-error-stack) +ADD_H5_DMP_TEST (textlink-prune "TEST" 0 textlink.h5 --prune --enable-error-stack) +ADD_H5_DMP_TEST (textlinkfar-prune "TEST" 0 textlinkfar.h5 --prune --enable-error-stack) +ADD_H5_DMP_TEST (textlinksrc-prune "TEST" 0 textlinksrc.h5 --prune --enable-error-stack) +ADD_H5_DMP_TEST (textlinktar-prune "TEST" 0 textlinktar.h5 --prune --enable-error-stack) + +ADD_H5_DMP_TEST (h5copy_extlinks_src-mergeprune "TEST" 0 h5copy_extlinks_src.h5 --merge --prune --enable-error-stack) +ADD_H5_DMP_TEST (tsoftlinks-mergeprune "TEST" 0 tsoftlinks.h5 --merge --prune --enable-error-stack) +ADD_H5_DMP_TEST (textlink-mergeprune "TEST" 0 textlink.h5 --merge --prune --enable-error-stack) +### HDFFV-11128 needs fixed to enable the following test +#ADD_H5_DMP_TEST (textlinkfar-mergeprune "TEST" 0 textlinkfar.h5 --merge --prune --enable-error-stack) +### HDFFV-11128 needs fixed to enable the following test +#ADD_H5_DMP_TEST (textlinksrc-mergeprune "TEST" 0 textlinksrc.h5 --merge --prune --enable-error-stack) +### HDFFV-11128 needs fixed to enable the following test +#ADD_H5_DMP_TEST (textlinktar-mergeprune "TEST" 0 textlinktar.h5 --merge --prune --enable-error-stack) + +############################################################################## ### P L U G I N T E S T S ############################################################################## if (BUILD_SHARED_LIBS) diff --git a/tools/test/h5repack/h5repack.sh.in b/tools/test/h5repack/h5repack.sh.in index ad6fef8..3d4e0ea 100644 --- a/tools/test/h5repack/h5repack.sh.in +++ b/tools/test/h5repack/h5repack.sh.in @@ -92,10 +92,18 @@ $SRC_H5REPACK_TESTFILES/h5repack_attr_refs.h5 $SRC_H5REPACK_TESTFILES/h5repack_deflate.h5 $SRC_H5REPACK_TESTFILES/h5repack_early.h5 $SRC_H5REPACK_TESTFILES/h5repack_ext.h5 +$SRC_H5REPACK_TESTFILES/h5repack_f32le.h5 +$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex.h5 $SRC_H5REPACK_TESTFILES/h5repack_fill.h5 $SRC_H5REPACK_TESTFILES/h5repack_filters.h5 $SRC_H5REPACK_TESTFILES/h5repack_fletcher.h5 $SRC_H5REPACK_TESTFILES/h5repack_hlink.h5 +$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d.h5 +$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex.h5 +$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d.h5 +$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex.h5 +$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d.h5 +$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex.h5 $SRC_H5REPACK_TESTFILES/h5repack_layout.h5 $SRC_H5REPACK_TESTFILES/h5repack_layouto.h5 $SRC_H5REPACK_TESTFILES/h5repack_layout2.h5 @@ -109,23 +117,26 @@ $SRC_H5REPACK_TESTFILES/h5repack_refs.h5 $SRC_H5REPACK_TESTFILES/h5repack_shuffle.h5 $SRC_H5REPACK_TESTFILES/h5repack_soffset.h5 $SRC_H5REPACK_TESTFILES/h5repack_szip.h5 +$SRC_H5REPACK_TESTFILES/h5repack_uint8be.h5 +$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex.h5 +########fsm#files######## $SRC_H5REPACK_TESTFILES/h5repack_aggr.h5 $SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_nopersist.h5 $SRC_H5REPACK_TESTFILES/h5repack_fsm_aggr_persist.h5 $SRC_H5REPACK_TESTFILES/h5repack_none.h5 $SRC_H5REPACK_TESTFILES/h5repack_paged_nopersist.h5 $SRC_H5REPACK_TESTFILES/h5repack_paged_persist.h5 -$SRC_H5REPACK_TESTFILES/h5repack_f32le.h5 -$SRC_H5REPACK_TESTFILES/h5repack_f32le_ex.h5 -$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d.h5 -$SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex.h5 -$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d.h5 -$SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex.h5 -$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d.h5 -$SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex.h5 -$SRC_H5REPACK_TESTFILES/h5repack_uint8be.h5 -$SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex.h5 +########h5diff/testfile######## $SRC_H5DIFF_TESTFILES/h5diff_attr1.h5 +########tools/testfiles#for#external#links######## +$SRC_TOOLS_TESTFILES/tsoftlinks.h5 +$SRC_TOOLS_TESTFILES/textlinkfar.h5 +$SRC_TOOLS_TESTFILES/textlinksrc.h5 +$SRC_TOOLS_TESTFILES/textlinktar.h5 +$SRC_TOOLS_TESTFILES/textlink.h5 +$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src.h5 +$SRC_H5REPACK_TESTFILES/h5copy_extlinks_trg.h5 +########tools/testfiles######## $SRC_TOOLS_TESTFILES/tfamily00000.h5 $SRC_TOOLS_TESTFILES/tfamily00001.h5 $SRC_TOOLS_TESTFILES/tfamily00002.h5 @@ -138,6 +149,8 @@ $SRC_TOOLS_TESTFILES/tfamily00008.h5 $SRC_TOOLS_TESTFILES/tfamily00009.h5 $SRC_TOOLS_TESTFILES/tfamily00010.h5 $SRC_TOOLS_TESTFILES/tordergr.h5 +########reference#conversion#files######## +########tools/testfiles/vds######## $SRC_TOOLS_TESTFILES/vds/1_a.h5 $SRC_TOOLS_TESTFILES/vds/1_b.h5 $SRC_TOOLS_TESTFILES/vds/1_c.h5 @@ -168,33 +181,61 @@ $SRC_H5REPACK_TESTFILES/h5repack-help.txt $SRC_H5REPACK_TESTFILES/h5repack_ext.bin $SRC_H5REPACK_TESTFILES/ublock.bin $SRC_H5REPACK_TESTFILES/h5repack.info -$SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl -$SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl +########dat#files######## $SRC_H5REPACK_TESTFILES/h5repack_f32le_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_int32le_1d_ex-1.dat $SRC_H5REPACK_TESTFILES/h5repack_int32le_2d_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_int32le_3d_ex-0.dat -$SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl -$SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst -$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst -$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-0.dat $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-1.dat $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-2.dat $SRC_H5REPACK_TESTFILES/h5repack_uint8be_ex-3.dat -$SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst -$SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_chunk20x10x5-v.ddl -$SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl -$SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl -$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl -$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl +############### +$SRC_H5REPACK_TESTFILES/crtorder.tordergr.h5.ddl +$SRC_H5REPACK_TESTFILES/deflate_limit.h5repack_layout.h5.ddl +$SRC_H5REPACK_TESTFILES/h5repack_layout.h5.ddl +$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-plugin_test.ddl +########fsm#files######## $SRC_H5REPACK_TESTFILES/SP.h5repack_fsm_aggr_nopersist.h5.ddl $SRC_H5REPACK_TESTFILES/S.h5repack_fsm_aggr_persist.h5.ddl $SRC_H5REPACK_TESTFILES/STG.h5repack_none.h5.ddl $SRC_H5REPACK_TESTFILES/GS.h5repack_paged_nopersist.h5.ddl $SRC_H5REPACK_TESTFILES/SP.h5repack_paged_persist.h5.ddl $SRC_H5REPACK_TESTFILES/SPT.h5repack_aggr.h5.ddl +########vds#files######## +$SRC_H5REPACK_TESTFILES/1_vds.h5-vds_dset_chunk20x10x5-v.ddl +$SRC_H5REPACK_TESTFILES/2_vds.h5-vds_chunk3x6x9-v.ddl +$SRC_H5REPACK_TESTFILES/3_1_vds.h5-vds_chunk2x5x8-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_compa-v.ddl +$SRC_H5REPACK_TESTFILES/4_vds.h5-vds_conti-v.ddl +########refs#files######## +$SRC_H5REPACK_TESTFILES/attrregion.tattrreg.h5.ddl +$SRC_H5REPACK_TESTFILES/dataregion.tdatareg.h5.ddl +########external#links#files######## +$SRC_H5REPACK_TESTFILES/textlinkfar-prune.textlinkfar.h5.ddl +$SRC_H5REPACK_TESTFILES/textlinksrc-prune.textlinksrc.h5.ddl +$SRC_H5REPACK_TESTFILES/textlinktar-prune.textlinktar.h5.ddl +$SRC_H5REPACK_TESTFILES/textlink-prune.textlink.h5.ddl +$SRC_H5REPACK_TESTFILES/tsoftlinks-prune.tsoftlinks.h5.ddl +$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5.ddl +$SRC_H5REPACK_TESTFILES/textlinkfar-mergeprune.textlinkfar.h5.ddl +$SRC_H5REPACK_TESTFILES/textlinksrc-mergeprune.textlinksrc.h5.ddl +$SRC_H5REPACK_TESTFILES/textlinktar-mergeprune.textlinktar.h5.ddl +$SRC_H5REPACK_TESTFILES/textlink-mergeprune.textlink.h5.ddl +$SRC_H5REPACK_TESTFILES/tsoftlinks-mergeprune.tsoftlinks.h5.ddl +$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5.ddl +########tst#files######## +$SRC_H5REPACK_TESTFILES/h5repack_filters.h5-gzip_verbose_filters.tst +$SRC_H5REPACK_TESTFILES/h5repack_layout.h5-dset2_chunk_20x10-errstk.tst +$SRC_H5REPACK_TESTFILES/plugin_test.h5repack_layout.h5.tst +########external#links#tst#files######## +$SRC_H5REPACK_TESTFILES/tsoftlinks-merge.tsoftlinks.h5.tst +$SRC_H5REPACK_TESTFILES/textlinkfar-merge.textlinkfar.h5.tst +$SRC_H5REPACK_TESTFILES/textlinksrc-merge.textlinksrc.h5.tst +$SRC_H5REPACK_TESTFILES/textlinktar-merge.textlinktar.h5.tst +$SRC_H5REPACK_TESTFILES/textlink-merge.textlink.h5.tst +$SRC_H5REPACK_TESTFILES/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst " # @@ -271,6 +312,12 @@ SKIP() { echo " -SKIP-" } +############################################################################## +############################################################################## +### T H E T E S T S M A C R O S ### +############################################################################## +############################################################################## + # Call the h5diff tool # DIFFTEST() @@ -619,25 +666,6 @@ VERIFY_INVALIDBOUNDS() } # end of VERIFY_INVALIDBOUNDS # ----------------------------------------------------------------------------- -# Expect h5diff to fail -# ----------------------------------------------------------------------------- -DIFFFAIL() -{ - VERIFY h5diff unequal $@ - ( - cd $TESTDIR - $RUNSERIAL $H5DIFF_BIN -q "$@" - ) - RET=$? - if [ $RET == 0 ] ; then - echo "*FAILED*" - nerrors="`expr $nerrors + 1`" - else - echo " PASSED" - fi -} - -# ----------------------------------------------------------------------------- # Catchall test for repacking with external files # Loops over all (internally-listed) cases and applies the given arguments # to h5repack. @@ -805,6 +833,121 @@ TOOLTESTV() rm -f $outfile } +# This is same as TOOLTESTV() with comparing display output +# with actual filename swapped +# +TOOLTESTSV() +{ + expect="$TESTDIR/$1.$2.tst" + actual="$TESTDIR/`basename $2 .ddl`.out" + actual_err="$TESTDIR/`basename $2 .ddl`.err" + + infile=$2 + outfile=out-$1.$2 + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + VERIFY h5diff unequal $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DIFF_BIN -v $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET == 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + fi + + # display output compare + STDOUT_FILTER $actual + cat $actual_err >> $actual + + VERIFY output from $H5REPACK $@ + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.tst) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + +# This is same as TOOLTESTSV() but expects a diff fail +# +TOOLFAILSV() +{ + expect="$TESTDIR/$1.$2.tst" + actual="$TESTDIR/`basename $2 .ddl`.out" + actual_err="$TESTDIR/`basename $2 .ddl`.err" + + infile=$2 + outfile=out-$1.$2 + shift + shift + + # Run test. + TESTING $H5REPACK $@ + ( + cd $TESTDIR + $RUNSERIAL $H5REPACK_BIN "$@" $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET != 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + VERIFY h5diff unequal $@ + ( + cd $TESTDIR + $RUNSERIAL $H5DIFF_BIN -v $infile $outfile + ) >$actual 2>$actual_err + RET=$? + if [ $RET == 0 ] ; then + echo "*FAILED*" + nerrors="`expr $nerrors + 1`" + else + echo " PASSED" + fi + fi + + # display output compare + STDOUT_FILTER $actual + cat $actual_err >> $actual + + VERIFY output from $H5REPACK $@ + if cmp -s $expect $actual; then + echo " PASSED" + else + echo "*FAILED*" + echo " Expected result (*.tst) differs from actual result (*.out)" + nerrors="`expr $nerrors + 1`" + test yes = "$verbose" && diff -c $expect $actual |sed 's/^/ /' + fi + + rm -f $actual $actual_err + rm -f $outfile +} + # same as TOOLTESTV but filters error stack output and compares to an error file # Extract file name, line number, version and thread IDs because they may be different # ADD_H5ERR_MASK_TEST @@ -1356,44 +1499,28 @@ TOOLTEST_STAT SPT $arg # layout options (these files have no filters) ######################################################### VERIFY_LAYOUT_DSET dset2_chunk_20x10 h5repack_layout.h5 dset2 CHUNKED --layout dset2:CHUNK=20x10 - VERIFY_LAYOUT_ALL chunk_20x10 h5repack_layout.h5 CHUNKED -l CHUNK=20x10 - VERIFY_LAYOUT_DSET dset2_conti h5repack_layout.h5 dset2 CONTIGUOUS -l dset2:CONTI - VERIFY_LAYOUT_ALL conti h5repack_layout.h5 CONTIGUOUS -l CONTI - VERIFY_LAYOUT_DSET dset2_compa h5repack_layout.h5 dset2 COMPACT -l dset2:COMPA - VERIFY_LAYOUT_ALL compa h5repack_layout.h5 COMPACT -l COMPA - TOOLTESTM dset2_chunk_20x10-errstk h5repack_layout.h5 --layout=dset2:CHUNK=20x10x5 --enable-error-stack ################################################################ # layout conversions (file has no filters) ############################################################### - VERIFY_LAYOUT_DSET dset_compa_conti h5repack_layout.h5 dset_compact CONTIGUOUS -l dset_compact:CONTI - VERIFY_LAYOUT_DSET dset_compa_chunk h5repack_layout.h5 dset_compact CHUNKED -l dset_compact:CHUNK=2x5 - VERIFY_LAYOUT_DSET dset_compa_compa h5repack_layout.h5 dset_compact COMPACT -l dset_compact:COMPA - VERIFY_LAYOUT_DSET dset_conti_compa h5repack_layout.h5 dset_contiguous COMPACT -l dset_contiguous:COMPA - VERIFY_LAYOUT_DSET dset_conti_chunk h5repack_layout.h5 dset_contiguous CHUNKED -l dset_contiguous:CHUNK=3x6 - VERIFY_LAYOUT_DSET dset_conti_conti h5repack_layout.h5 dset_contiguous CONTIGUOUS -l dset_contiguous:CONTI - VERIFY_LAYOUT_DSET chunk_compa h5repack_layout.h5 dset_chunk COMPACT -l dset_chunk:COMPA - VERIFY_LAYOUT_DSET chunk_conti h5repack_layout.h5 dset_chunk CONTIGUOUS -l dset_chunk:CONTI - VERIFY_LAYOUT_DSET chunk_18x13 h5repack_layout.h5 dset_chunk CHUNKED -l dset_chunk:CHUNK=18x13 # test convert small size dataset ( < 1k) to compact layout without -m VERIFY_LAYOUT_DSET contig_small_compa h5repack_layout2.h5 contig_small COMPACT -l contig_small:COMPA - VERIFY_LAYOUT_DSET contig_small_fixed_compa h5repack_layout2.h5 chunked_small_fixed COMPACT -l chunked_small_fixed:COMPA #--------------------------------------------------------------------------- @@ -1421,6 +1548,7 @@ VERIFY_LAYOUT_DSET chunk2compa h5repack_layout3.h5 chunk_unlimit1 CHUNK -l chunk # chunk dim is bigger than dataset dim. ( dset size < 64k ) VERIFY_LAYOUT_DSET error1 h5repack_layout3.h5 chunk_unlimit1 H5S_UNLIMITED -f chunk_unlimit1:NONE # chunk dim is bigger than dataset dim. ( dset size > 64k ) + VERIFY_LAYOUT_DSET error2 h5repack_layout3.h5 chunk_unlimit2 H5S_UNLIMITED -f chunk_unlimit2:NONE # chunk dims are smaller than dataset dims. ( dset size < 64k ) @@ -1435,7 +1563,6 @@ TOOLTEST error4 h5repack_layout3.h5 -f NONE # (dset size < 64K) and with unlimited max dims on a condition as follow. # (HDFFV-8214) #-------------------------------------------------------------------------- - # chunk dim is bigger than dataset dim. should succeed. VERIFY_LAYOUT_DSET ckdim_biger h5repack_layout3.h5 chunk_unlimit2 CONTI -l chunk_unlimit2:CONTI # chunk dim is smaller than dataset dim. should succeed. @@ -1466,7 +1593,6 @@ else fi # several global filters - arg="h5repack_layout.h5 --filter GZIP=1 --filter SHUF" if test $USE_FILTER_DEFLATE != "yes" ; then SKIP $arg @@ -1562,9 +1688,15 @@ else VERIFY_LAYOUT_VDS vds_conti 4_vds.h5 vds_dset CONTIGUOUS -l vds_dset:CONTI fi -######################################################### -# Testing version bounds -######################################################### +################################################################ +# reference new api conversions +############################################################### +#TOOLTEST_DUMP attrregion tattrreg.h5 +#TOOLTEST_DUMP dataregion tdatareg.h5 + +############################################################################## +### V E R S I O N B O U N D S T E S T S +############################################################################## # -j 0 -k 2, superblock will be 0 VERIFY_SUPERBLOCK 0 2 0 h5repack_layout.h5 -j 0 -k 2 h5repack_layout.h5 # -j 1 -k 2, superblock will be 2 @@ -1574,11 +1706,49 @@ VERIFY_SUPERBLOCK 2 2 3 h5repack_layout.h5 -j 2 -k 2 h5repack_layout.h5 # -j 0 -k 1, file cannot be opened VERIFY_INVALIDBOUNDS 0 1 bounds_latest_latest.h5 -######################################## -# Testing external storage -######################################## +############################################################################## +### E X T E R N A L S T O R A G E T E S T S +############################################################################## VERIFY_EXTERNAL_CONSOLIDATION -l CONTI +############################################################################## +### E X T E R N A L L I N K T E S T S +############################################################################## +### HDFFV-11128 needs fixed to enable the following test +#TOOLTEST_DUMP h5copy_extlinks_src-base h5copy_extlinks_src.h5 --enable-error-stack +TOOLTEST_DUMP tsoftlinks-base tsoftlinks.h5 --enable-error-stack +TOOLTEST_DUMP textlink-base textlink.h5 --enable-error-stack +TOOLTEST_DUMP textlinkfar-base textlinkfar.h5 --enable-error-stack +TOOLTEST_DUMP textlinksrc-base textlinksrc.h5 --enable-error-stack +TOOLTEST_DUMP textlinktar-base textlinktar.h5 --enable-error-stack + +TOOLTESTSV h5copy_extlinks_src-merge h5copy_extlinks_src.h5 --merge +TOOLFAILSV tsoftlinks-merge tsoftlinks.h5 --merge +TOOLTESTSV textlink-merge textlink.h5 --merge +### HDFFV-11128 needs fixed to enable the following test +#TOOLFAILSV textlinkfar-merge textlinkfar.h5 --merge +### HDFFV-11128 needs fixed to enable the following test +#TOOLFAILSV textlinksrc-merge textlinksrc.h5 --merge +### HDFFV-11128 needs fixed to enable the following test +#TOOLFAILSV textlinktar-merge textlinktar.h5 --merge + +TOOLTEST_DUMP h5copy_extlinks_src-prune h5copy_extlinks_src.h5 --prune --enable-error-stack +TOOLTEST_DUMP tsoftlinks-prune tsoftlinks.h5 --prune --enable-error-stack +TOOLTEST_DUMP textlink-prune textlink.h5 --prune --enable-error-stack +TOOLTEST_DUMP textlinkfar-prune textlinkfar.h5 --prune --enable-error-stack +TOOLTEST_DUMP textlinksrc-prune textlinksrc.h5 --prune --enable-error-stack +TOOLTEST_DUMP textlinktar-prune textlinktar.h5 --prune --enable-error-stack + +TOOLTEST_DUMP h5copy_extlinks_src-mergeprune h5copy_extlinks_src.h5 --merge --prune --enable-error-stack +TOOLTEST_DUMP tsoftlinks-mergeprune tsoftlinks.h5 --merge --prune --enable-error-stack +TOOLTEST_DUMP textlink-mergeprune textlink.h5 --merge --prune --enable-error-stack +### HDFFV-11128 needs fixed to enable the following test +#TOOLTEST_DUMP textlinkfar-mergeprune textlinkfar.h5 --merge --prune --enable-error-stack +### HDFFV-11128 needs fixed to enable the following test +#TOOLTEST_DUMP textlinksrc-mergeprune textlinksrc.h5 --merge --prune --enable-error-stack +### HDFFV-11128 needs fixed to enable the following test +#TOOLTEST_DUMP textlinktar-mergeprune textlinktar.h5 --merge --prune --enable-error-stack + # Clean up temporary files/directories CLEAN_TESTFILES_AND_TESTDIR diff --git a/tools/test/h5repack/testfiles/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst b/tools/test/h5repack/testfiles/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst new file mode 100644 index 0000000..52f215d --- /dev/null +++ b/tools/test/h5repack/testfiles/h5copy_extlinks_src-merge.h5copy_extlinks_src.h5.tst @@ -0,0 +1,26 @@ + +file1 file2 +--------------------------------------- + x x / + x x /group_ext + x x /group_ext/extlink_datatype + x x /group_ext/extlink_dset + x x /group_ext/extlink_grp + x x /group_ext/extlink_notyet1 + x x /group_ext/extlink_notyet2 + +group : </> and </> +0 differences found +group : </group_ext> and </group_ext> +0 differences found +Not comparable: </group_ext/extlink_datatype> is of type H5G_UDLINK and </group_ext/extlink_datatype> is of type H5G_TYPE +Not comparable: </group_ext/extlink_dset> is of type H5G_UDLINK and </group_ext/extlink_dset> is of type H5G_DATASET +Not comparable: </group_ext/extlink_grp> is of type H5G_UDLINK and </group_ext/extlink_grp> is of type H5G_GROUP +external link: </group_ext/extlink_notyet1> and </group_ext/extlink_notyet1> +0 differences found +external link: </group_ext/extlink_notyet2> and </group_ext/extlink_notyet2> +0 differences found +-------------------------------- +Some objects are not comparable +-------------------------------- +Use -c for a list of objects without details of differences. diff --git a/tools/test/h5repack/testfiles/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5.ddl b/tools/test/h5repack/testfiles/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5.ddl new file mode 100644 index 0000000..ad8a320 --- /dev/null +++ b/tools/test/h5repack/testfiles/h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5.ddl @@ -0,0 +1,28 @@ +HDF5 "out-h5copy_extlinks_src-mergeprune.h5copy_extlinks_src.h5" { +GROUP "/" { + GROUP "group_ext" { + DATATYPE "extlink_datatype" H5T_STD_I32LE; + DATASET "extlink_dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 2200 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + GROUP "extlink_grp" { + } + } +} +} diff --git a/tools/test/h5repack/testfiles/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5.ddl b/tools/test/h5repack/testfiles/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5.ddl new file mode 100644 index 0000000..b7d7bca --- /dev/null +++ b/tools/test/h5repack/testfiles/h5copy_extlinks_src-prune.h5copy_extlinks_src.h5.ddl @@ -0,0 +1,6 @@ +HDF5 "out-h5copy_extlinks_src-prune.h5copy_extlinks_src.h5" { +GROUP "/" { + GROUP "group_ext" { + } +} +} diff --git a/tools/test/h5repack/testfiles/h5copy_extlinks_src.h5 b/tools/test/h5repack/testfiles/h5copy_extlinks_src.h5 Binary files differnew file mode 100644 index 0000000..7b8621e --- /dev/null +++ b/tools/test/h5repack/testfiles/h5copy_extlinks_src.h5 diff --git a/tools/test/h5repack/testfiles/h5copy_extlinks_trg.h5 b/tools/test/h5repack/testfiles/h5copy_extlinks_trg.h5 Binary files differnew file mode 100644 index 0000000..3a0242d --- /dev/null +++ b/tools/test/h5repack/testfiles/h5copy_extlinks_trg.h5 diff --git a/tools/test/h5repack/testfiles/h5repack-help.txt b/tools/test/h5repack/testfiles/h5repack-help.txt index 00fae24..e12838d 100644 --- a/tools/test/h5repack/testfiles/h5repack-help.txt +++ b/tools/test/h5repack/testfiles/h5repack-help.txt @@ -29,6 +29,9 @@ usage: h5repack [OPTIONS] file1 file2 --high=BOUND The high bound for library release versions to use when creating objects in the file (default is H5F_LIBVER_LATEST) + --merge Follow external soft link recursively and merge data + --prune Do not follow external soft links and remove link + --merge --prune Follow external link, merge data and remove dangling link -c L1, --compact=L1 Maximum number of links in header messages -d L2, --indexed=L2 Minimum number of links in the indexed format -s S[:F], --ssize=S[:F] Shared object header message minimum size diff --git a/tools/test/h5repack/testfiles/textlink-merge.textlink.h5.tst b/tools/test/h5repack/testfiles/textlink-merge.textlink.h5.tst new file mode 100644 index 0000000..a7ff71d --- /dev/null +++ b/tools/test/h5repack/testfiles/textlink-merge.textlink.h5.tst @@ -0,0 +1,13 @@ + +file1 file2 +--------------------------------------- + x x / + x x /extlink1 + x x /extlink2 + +group : </> and </> +0 differences found +external link: </extlink1> and </extlink1> +0 differences found +external link: </extlink2> and </extlink2> +0 differences found diff --git a/tools/test/h5repack/testfiles/textlink-mergeprune.textlink.h5.ddl b/tools/test/h5repack/testfiles/textlink-mergeprune.textlink.h5.ddl new file mode 100644 index 0000000..7206ae9 --- /dev/null +++ b/tools/test/h5repack/testfiles/textlink-mergeprune.textlink.h5.ddl @@ -0,0 +1,4 @@ +HDF5 "out-textlink-mergeprune.textlink.h5" { +GROUP "/" { +} +} diff --git a/tools/test/h5repack/testfiles/textlink-prune.textlink.h5.ddl b/tools/test/h5repack/testfiles/textlink-prune.textlink.h5.ddl new file mode 100644 index 0000000..83db344 --- /dev/null +++ b/tools/test/h5repack/testfiles/textlink-prune.textlink.h5.ddl @@ -0,0 +1,4 @@ +HDF5 "out-textlink-prune.textlink.h5" { +GROUP "/" { +} +} diff --git a/tools/test/h5repack/testfiles/textlinkfar-merge.textlinkfar.h5.tst b/tools/test/h5repack/testfiles/textlinkfar-merge.textlinkfar.h5.tst new file mode 100644 index 0000000..6673d39 --- /dev/null +++ b/tools/test/h5repack/testfiles/textlinkfar-merge.textlinkfar.h5.tst @@ -0,0 +1,20 @@ + +file1 file2 +--------------------------------------- + x x / + x x /src_file + x /src_file/ext2soft_link1 + x /src_file/ext2softdangle_link1 + x /src_file/ext_link1 + x /src_file/ext_link2 + x /src_file/ext_link3 + x /src_file/ext_link4 + x /src_file/ext_link5 + +group : </> and </> +0 differences found +Not comparable: </src_file> is of type H5G_UDLINK and </src_file> is of type H5G_GROUP +-------------------------------- +Some objects are not comparable +-------------------------------- +Use -c for a list of objects without details of differences. diff --git a/tools/test/h5repack/testfiles/textlinkfar-mergeprune.textlinkfar.h5.ddl b/tools/test/h5repack/testfiles/textlinkfar-mergeprune.textlinkfar.h5.ddl new file mode 100644 index 0000000..7ba3308 --- /dev/null +++ b/tools/test/h5repack/testfiles/textlinkfar-mergeprune.textlinkfar.h5.ddl @@ -0,0 +1,152 @@ +HDF5 "out-textlinkfar-mergeprune.textlinkfar.h5" { +GROUP "/" { + GROUP "src_file" { + EXTERNAL_LINK "ext2soft_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dset1" + DATASET "/soft_dset1" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2848 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + } + EXTERNAL_LINK "ext2softdangle_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dangle" + } + EXTERNAL_LINK "ext_link1" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group" + GROUP "group" { + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 3136 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + EXTERNAL_LINK "elink_t1" { + TARGETFILE "textlinksrc.h5" + TARGETPATH "/" + GROUP "/" { + EXTERNAL_LINK "ext2soft_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dset1" + DATASET "/soft_dset1" { + HARDLINK "/dset1" + } + } + EXTERNAL_LINK "ext2softdangle_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dangle" + } + EXTERNAL_LINK "ext_link1" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group" + GROUP "group" { + HARDLINK "/group" + } + } + EXTERNAL_LINK "ext_link2" { + TARGETFILE "textlinktar.h5" + TARGETPATH "dset" + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 3160 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + } + EXTERNAL_LINK "ext_link3" { + TARGETFILE "textlinktar.h5" + TARGETPATH "type" + DATATYPE "type" H5T_STD_I32LE; + } + EXTERNAL_LINK "ext_link4" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group/elink_t2" + } + EXTERNAL_LINK "ext_link5" { + TARGETFILE "textlinktar.h5" + TARGETPATH "empty_group" + GROUP "empty_group" { + } + } + } + } + EXTERNAL_LINK "elink_t2" { + TARGETFILE "textlinksrc.h5" + TARGETPATH "/ext_link4" + } + GROUP "subgroup" { + GROUP "link_to_group" { + HARDLINK "/group" + } + } + } + } + EXTERNAL_LINK "ext_link2" { + TARGETFILE "textlinktar.h5" + TARGETPATH "dset" + DATASET "dset" { + HARDLINK "/dset" + } + } + EXTERNAL_LINK "ext_link3" { + TARGETFILE "textlinktar.h5" + TARGETPATH "type" + DATATYPE "type" HARDLINK "/type" + } + EXTERNAL_LINK "ext_link4" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group/elink_t2" + } + EXTERNAL_LINK "ext_link5" { + TARGETFILE "textlinktar.h5" + TARGETPATH "empty_group" + GROUP "empty_group" { + HARDLINK "/empty_group" + } + } + } +} +} diff --git a/tools/test/h5repack/testfiles/textlinkfar-prune.textlinkfar.h5.ddl b/tools/test/h5repack/testfiles/textlinkfar-prune.textlinkfar.h5.ddl new file mode 100644 index 0000000..30f8f1b --- /dev/null +++ b/tools/test/h5repack/testfiles/textlinkfar-prune.textlinkfar.h5.ddl @@ -0,0 +1,4 @@ +HDF5 "out-textlinkfar-prune.textlinkfar.h5" { +GROUP "/" { +} +} diff --git a/tools/test/h5repack/testfiles/textlinksrc-merge.textlinksrc.h5.tst b/tools/test/h5repack/testfiles/textlinksrc-merge.textlinksrc.h5.tst new file mode 100644 index 0000000..d5a5902 --- /dev/null +++ b/tools/test/h5repack/testfiles/textlinksrc-merge.textlinksrc.h5.tst @@ -0,0 +1,32 @@ + +file1 file2 +--------------------------------------- + x x / + x x /ext2soft_link1 + x x /ext2softdangle_link1 + x x /ext_link1 + x /ext_link1/dset + x /ext_link1/elink_t1 + x /ext_link1/elink_t2 + x /ext_link1/subgroup + x /ext_link1/subgroup/link_to_group + x x /ext_link2 + x x /ext_link3 + x x /ext_link4 + x x /ext_link5 + +group : </> and </> +0 differences found +Not comparable: </ext2soft_link1> is of type H5G_UDLINK and </ext2soft_link1> is of type H5G_DATASET +external link: </ext2softdangle_link1> and </ext2softdangle_link1> +0 differences found +Not comparable: </ext_link1> is of type H5G_UDLINK and </ext_link1> is of type H5G_GROUP +Not comparable: </ext_link2> is of type H5G_UDLINK and </ext_link2> is of type H5G_DATASET +Not comparable: </ext_link3> is of type H5G_UDLINK and </ext_link3> is of type H5G_TYPE +external link: </ext_link4> and </ext_link4> +0 differences found +Not comparable: </ext_link5> is of type H5G_UDLINK and </ext_link5> is of type H5G_GROUP +-------------------------------- +Some objects are not comparable +-------------------------------- +Use -c for a list of objects without details of differences. diff --git a/tools/test/h5repack/testfiles/textlinksrc-mergeprune.textlinksrc.h5.ddl b/tools/test/h5repack/testfiles/textlinksrc-mergeprune.textlinksrc.h5.ddl new file mode 100644 index 0000000..5b3c740 --- /dev/null +++ b/tools/test/h5repack/testfiles/textlinksrc-mergeprune.textlinksrc.h5.ddl @@ -0,0 +1,187 @@ +HDF5 "out-textlinksrc-mergeprune.textlinksrc.h5" { +GROUP "/" { + DATASET "ext2soft_link1" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2048 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + GROUP "ext_link1" { + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 2080 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + EXTERNAL_LINK "elink_t1" { + TARGETFILE "textlinksrc.h5" + TARGETPATH "/" + GROUP "/" { + EXTERNAL_LINK "ext2soft_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dset1" + DATASET "/soft_dset1" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2848 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + } + EXTERNAL_LINK "ext2softdangle_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dangle" + } + EXTERNAL_LINK "ext_link1" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group" + GROUP "group" { + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 3136 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + EXTERNAL_LINK "elink_t1" { + TARGETFILE "textlinksrc.h5" + TARGETPATH "/" + GROUP "/" { + HARDLINK "/" + } + } + EXTERNAL_LINK "elink_t2" { + TARGETFILE "textlinksrc.h5" + TARGETPATH "/ext_link4" + } + GROUP "subgroup" { + GROUP "link_to_group" { + HARDLINK "/group" + } + } + } + } + EXTERNAL_LINK "ext_link2" { + TARGETFILE "textlinktar.h5" + TARGETPATH "dset" + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 3160 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + } + EXTERNAL_LINK "ext_link3" { + TARGETFILE "textlinktar.h5" + TARGETPATH "type" + DATATYPE "type" H5T_STD_I32LE; + } + EXTERNAL_LINK "ext_link4" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group/elink_t2" + } + EXTERNAL_LINK "ext_link5" { + TARGETFILE "textlinktar.h5" + TARGETPATH "empty_group" + GROUP "empty_group" { + } + } + } + } + EXTERNAL_LINK "elink_t2" { + TARGETFILE "textlinksrc.h5" + TARGETPATH "/ext_link4" + } + GROUP "subgroup" { + GROUP "link_to_group" { + HARDLINK "/ext_link1" + } + } + } + DATASET "ext_link2" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 2104 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + DATATYPE "ext_link3" H5T_STD_I32LE; + GROUP "ext_link5" { + } +} +} diff --git a/tools/test/h5repack/testfiles/textlinksrc-prune.textlinksrc.h5.ddl b/tools/test/h5repack/testfiles/textlinksrc-prune.textlinksrc.h5.ddl new file mode 100644 index 0000000..65d35ef --- /dev/null +++ b/tools/test/h5repack/testfiles/textlinksrc-prune.textlinksrc.h5.ddl @@ -0,0 +1,4 @@ +HDF5 "out-textlinksrc-prune.textlinksrc.h5" { +GROUP "/" { +} +} diff --git a/tools/test/h5repack/testfiles/textlinktar-merge.textlinktar.h5.tst b/tools/test/h5repack/testfiles/textlinktar-merge.textlinktar.h5.tst new file mode 100644 index 0000000..ccf4f7b --- /dev/null +++ b/tools/test/h5repack/testfiles/textlinktar-merge.textlinktar.h5.tst @@ -0,0 +1,44 @@ + +file1 file2 +--------------------------------------- + x x / + x x /dset + x x /empty_group + x x /group + x x /group/dset + x x /group/elink_t1 + x /group/elink_t1/ext2soft_link1 + x /group/elink_t1/ext2softdangle_link1 + x /group/elink_t1/ext_link1 + x /group/elink_t1/ext_link2 + x /group/elink_t1/ext_link3 + x /group/elink_t1/ext_link4 + x /group/elink_t1/ext_link5 + x x /group/elink_t2 + x x /group/subgroup + x x /group/subgroup/link_to_group + x x /type + +group : </> and </> +0 differences found +dataset: </dset> and </dset> +0 differences found +group : </empty_group> and </empty_group> +0 differences found +group : </group> and </group> +0 differences found +dataset: </group/dset> and </group/dset> +0 differences found +Not comparable: </group/elink_t1> is of type H5G_UDLINK and </group/elink_t1> is of type H5G_GROUP +external link: </group/elink_t2> and </group/elink_t2> +0 differences found +group : </group/subgroup> and </group/subgroup> +0 differences found +group : </group/subgroup/link_to_group> and </group/subgroup/link_to_group> +0 differences found +datatype: </type> and </type> +0 differences found +-------------------------------- +Some objects are not comparable +-------------------------------- +Use -c for a list of objects without details of differences. diff --git a/tools/test/h5repack/testfiles/textlinktar-mergeprune.textlinktar.h5.ddl b/tools/test/h5repack/testfiles/textlinktar-mergeprune.textlinktar.h5.ddl new file mode 100644 index 0000000..c52e34d --- /dev/null +++ b/tools/test/h5repack/testfiles/textlinktar-mergeprune.textlinktar.h5.ddl @@ -0,0 +1,200 @@ +HDF5 "out-textlinktar-mergeprune.textlinktar.h5" { +GROUP "/" { + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 2048 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + GROUP "empty_group" { + } + GROUP "group" { + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 2072 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + GROUP "elink_t1" { + EXTERNAL_LINK "ext2soft_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dset1" + DATASET "/soft_dset1" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2848 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + } + EXTERNAL_LINK "ext2softdangle_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dangle" + } + EXTERNAL_LINK "ext_link1" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group" + GROUP "group" { + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 3136 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + EXTERNAL_LINK "elink_t1" { + TARGETFILE "textlinksrc.h5" + TARGETPATH "/" + GROUP "/" { + EXTERNAL_LINK "ext2soft_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dset1" + DATASET "/soft_dset1" { + HARDLINK "/dset1" + } + } + EXTERNAL_LINK "ext2softdangle_link1" { + TARGETFILE "tsoftlinks.h5" + TARGETPATH "/soft_dangle" + } + EXTERNAL_LINK "ext_link1" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group" + GROUP "group" { + HARDLINK "/group" + } + } + EXTERNAL_LINK "ext_link2" { + TARGETFILE "textlinktar.h5" + TARGETPATH "dset" + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 3160 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + } + EXTERNAL_LINK "ext_link3" { + TARGETFILE "textlinktar.h5" + TARGETPATH "type" + DATATYPE "type" H5T_STD_I32LE; + } + EXTERNAL_LINK "ext_link4" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group/elink_t2" + } + EXTERNAL_LINK "ext_link5" { + TARGETFILE "textlinktar.h5" + TARGETPATH "empty_group" + GROUP "empty_group" { + } + } + } + } + EXTERNAL_LINK "elink_t2" { + TARGETFILE "textlinksrc.h5" + TARGETPATH "/ext_link4" + } + GROUP "subgroup" { + GROUP "link_to_group" { + HARDLINK "/group" + } + } + } + } + EXTERNAL_LINK "ext_link2" { + TARGETFILE "textlinktar.h5" + TARGETPATH "dset" + DATASET "dset" { + HARDLINK "/dset" + } + } + EXTERNAL_LINK "ext_link3" { + TARGETFILE "textlinktar.h5" + TARGETPATH "type" + DATATYPE "type" HARDLINK "/type" + } + EXTERNAL_LINK "ext_link4" { + TARGETFILE "textlinktar.h5" + TARGETPATH "group/elink_t2" + } + EXTERNAL_LINK "ext_link5" { + TARGETFILE "textlinktar.h5" + TARGETPATH "empty_group" + GROUP "empty_group" { + HARDLINK "/empty_group" + } + } + } + GROUP "subgroup" { + GROUP "link_to_group" { + HARDLINK "/group" + } + } + } + DATATYPE "type" H5T_STD_I32LE; +} +} diff --git a/tools/test/h5repack/testfiles/textlinktar-prune.textlinktar.h5.ddl b/tools/test/h5repack/testfiles/textlinktar-prune.textlinktar.h5.ddl new file mode 100644 index 0000000..6787784 --- /dev/null +++ b/tools/test/h5repack/testfiles/textlinktar-prune.textlinktar.h5.ddl @@ -0,0 +1,52 @@ +HDF5 "out-textlinktar-prune.textlinktar.h5" { +GROUP "/" { + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 2048 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + GROUP "empty_group" { + } + GROUP "group" { + DATASET "dset" { + DATATYPE H5T_STD_I32LE + DATASPACE SIMPLE { ( 6 ) / ( 6 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 24 + OFFSET 2072 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + GROUP "subgroup" { + GROUP "link_to_group" { + HARDLINK "/group" + } + } + } + DATATYPE "type" H5T_STD_I32LE; +} +} diff --git a/tools/test/h5repack/testfiles/tsoftlinks-merge.tsoftlinks.h5.tst b/tools/test/h5repack/testfiles/tsoftlinks-merge.tsoftlinks.h5.tst new file mode 100644 index 0000000..27c33d7 --- /dev/null +++ b/tools/test/h5repack/testfiles/tsoftlinks-merge.tsoftlinks.h5.tst @@ -0,0 +1,53 @@ + +file1 file2 +--------------------------------------- + x x / + x x /dset1 + x x /dset2 + x x /dtype + x x /group1 + x x /group1/soft_dangle + x x /group1/soft_dset1 + x x /group1/soft_dset2 + x x /group1/soft_dtype + x x /group1/soft_empty_grp + x x /group_empty + x x /soft_dangle + x x /soft_dset1 + x x /soft_dtype + x x /soft_empty_grp + x x /soft_group1 + x /soft_group1/soft_dangle + x /soft_group1/soft_dset1 + x /soft_group1/soft_dset2 + x /soft_group1/soft_dtype + x /soft_group1/soft_empty_grp + +group : </> and </> +0 differences found +dataset: </dset1> and </dset1> +0 differences found +dataset: </dset2> and </dset2> +0 differences found +datatype: </dtype> and </dtype> +0 differences found +group : </group1> and </group1> +0 differences found +link : </group1/soft_dangle> and </group1/soft_dangle> +0 differences found +Not comparable: </group1/soft_dset1> is of type H5G_LINK and </group1/soft_dset1> is of type H5G_DATASET +Not comparable: </group1/soft_dset2> is of type H5G_LINK and </group1/soft_dset2> is of type H5G_DATASET +Not comparable: </group1/soft_dtype> is of type H5G_LINK and </group1/soft_dtype> is of type H5G_TYPE +Not comparable: </group1/soft_empty_grp> is of type H5G_LINK and </group1/soft_empty_grp> is of type H5G_GROUP +group : </group_empty> and </group_empty> +0 differences found +link : </soft_dangle> and </soft_dangle> +0 differences found +Not comparable: </soft_dset1> is of type H5G_LINK and </soft_dset1> is of type H5G_DATASET +Not comparable: </soft_dtype> is of type H5G_LINK and </soft_dtype> is of type H5G_TYPE +Not comparable: </soft_empty_grp> is of type H5G_LINK and </soft_empty_grp> is of type H5G_GROUP +Not comparable: </soft_group1> is of type H5G_LINK and </soft_group1> is of type H5G_GROUP +-------------------------------- +Some objects are not comparable +-------------------------------- +Use -c for a list of objects without details of differences. diff --git a/tools/test/h5repack/testfiles/tsoftlinks-mergeprune.tsoftlinks.h5.ddl b/tools/test/h5repack/testfiles/tsoftlinks-mergeprune.tsoftlinks.h5.ddl new file mode 100644 index 0000000..fa4e12d --- /dev/null +++ b/tools/test/h5repack/testfiles/tsoftlinks-mergeprune.tsoftlinks.h5.ddl @@ -0,0 +1,127 @@ +HDF5 "out-tsoftlinks-mergeprune.tsoftlinks.h5" { +GROUP "/" { + DATASET "dset1" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2048 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + DATASET "dset2" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2080 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + DATATYPE "dtype" H5T_STD_I32BE; + GROUP "group1" { + DATASET "soft_dset1" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2112 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + DATASET "soft_dset2" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2144 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + DATATYPE "soft_dtype" H5T_STD_I32BE; + GROUP "soft_empty_grp" { + } + } + GROUP "group_empty" { + } + DATASET "soft_dset1" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2176 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + DATATYPE "soft_dtype" H5T_STD_I32BE; + GROUP "soft_empty_grp" { + } + GROUP "soft_group1" { + SOFTLINK "soft_dangle" { + LINKTARGET "not_yet" + } + SOFTLINK "soft_dset1" { + LINKTARGET "/dset1" + } + SOFTLINK "soft_dset2" { + LINKTARGET "/dset2" + } + SOFTLINK "soft_dtype" { + LINKTARGET "/dtype" + } + SOFTLINK "soft_empty_grp" { + LINKTARGET "/group_empty" + } + } +} +} diff --git a/tools/test/h5repack/testfiles/tsoftlinks-prune.tsoftlinks.h5.ddl b/tools/test/h5repack/testfiles/tsoftlinks-prune.tsoftlinks.h5.ddl new file mode 100644 index 0000000..de58ef2 --- /dev/null +++ b/tools/test/h5repack/testfiles/tsoftlinks-prune.tsoftlinks.h5.ddl @@ -0,0 +1,47 @@ +HDF5 "out-tsoftlinks-prune.tsoftlinks.h5" { +GROUP "/" { + DATASET "dset1" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2048 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + DATASET "dset2" { + DATATYPE H5T_STD_I32BE + DATASPACE SIMPLE { ( 4, 2 ) / ( 4, 2 ) } + STORAGE_LAYOUT { + CONTIGUOUS + SIZE 32 + OFFSET 2080 + } + FILTERS { + NONE + } + FILLVALUE { + FILL_TIME H5D_FILL_TIME_IFSET + VALUE H5D_FILL_VALUE_DEFAULT + } + ALLOCATION_TIME { + H5D_ALLOC_TIME_LATE + } + } + DATATYPE "dtype" H5T_STD_I32BE; + GROUP "group1" { + } + GROUP "group_empty" { + } +} +} |