summaryrefslogtreecommitdiffstats
path: root/tools/src
diff options
context:
space:
mode:
authorAllen Byrne <byrn@hdfgroup.org>2020-08-14 17:16:58 (GMT)
committerAllen Byrne <byrn@hdfgroup.org>2020-08-14 17:16:58 (GMT)
commit500d87fd1f5d6068feeb71119bc5d22843203fef (patch)
treed98efc114fc684fd835aa31168457e72c833af99 /tools/src
parent044ee6f88c78a70eb3694d8c73fa74c77fde9689 (diff)
downloadhdf5-500d87fd1f5d6068feeb71119bc5d22843203fef.zip
hdf5-500d87fd1f5d6068feeb71119bc5d22843203fef.tar.gz
hdf5-500d87fd1f5d6068feeb71119bc5d22843203fef.tar.bz2
HDFFV-9984 Add options to merge/prune external links during repack
Diffstat (limited to 'tools/src')
-rw-r--r--tools/src/h5copy/h5copy.c4
-rw-r--r--tools/src/h5repack/h5repack.h38
-rw-r--r--tools/src/h5repack/h5repack_copy.c84
-rw-r--r--tools/src/h5repack/h5repack_main.c53
4 files changed, 128 insertions, 51 deletions
diff --git a/tools/src/h5copy/h5copy.c b/tools/src/h5copy/h5copy.c
index 5b19ae7..a6e4d4e 100644
--- a/tools/src/h5copy/h5copy.c
+++ b/tools/src/h5copy/h5copy.c
@@ -212,8 +212,8 @@ main (int argc, const char *argv[])
unsigned flag = 0;
unsigned verbose = 0;
unsigned parents = 0;
- hid_t ocpl_id = (-1); /* Object copy property list */
- hid_t lcpl_id = (-1); /* Link creation property list */
+ hid_t ocpl_id = H5I_INVALID_HID; /* Object copy property list */
+ hid_t lcpl_id = H5I_INVALID_HID; /* Link creation property list */
int opt;
int li_ret;
h5tool_link_info_t linkinfo;
diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h
index caa1166..b124fdb 100644
--- a/tools/src/h5repack/h5repack.h
+++ b/tools/src/h5repack/h5repack.h
@@ -101,25 +101,27 @@ typedef struct {
/* all the above, ready to go to the hrepack call */
typedef struct {
- pack_opttbl_t *op_tbl; /*table with all -c and -f options */
- int all_layout; /*apply the layout to all objects */
- int all_filter; /*apply the filter to all objects */
+ pack_opttbl_t *op_tbl; /* table with all -c and -f options */
+ int all_layout; /* apply the layout to all objects */
+ int all_filter; /* apply the filter to all objects */
filter_info_t filter_g[H5_REPACK_MAX_NFILTERS]; /*global filter array for the ALL case */
- int n_filter_g; /*number of global filters */
- chunk_info_t chunk_g; /*global chunk INFO for the ALL case */
- H5D_layout_t layout_g; /*global layout information for the ALL case */
- int verbose; /*verbose mode */
- hsize_t min_comp; /*minimum size to compress, in bytes */
- int use_native; /*use a native type in write */
- hbool_t latest; /*pack file with the latest file format */
- H5F_libver_t low_bound; /* The file's low bound as in H5Fset_libver_bounds() */
- H5F_libver_t high_bound; /* The file's high bound as in H5Fset_libver_bounds() */
- hid_t fin_fapl; /* FAPL to use for opening the input file */
- hid_t fout_fapl; /* FAPL to use for opening/creating the output file */
- int grp_compact; /* Set the maximum number of links to store as header messages in the group */
- int grp_indexed; /* Set the minimum number of links to store in the indexed format */
- int msg_size[8]; /* Minimum size of shared messages: dataspace,
- datatype, fill value, filter pipleline, attribute */
+ int n_filter_g; /* number of global filters */
+ chunk_info_t chunk_g; /* global chunk INFO for the ALL case */
+ H5D_layout_t layout_g; /* global layout information for the ALL case */
+ int verbose; /* verbose mode */
+ hbool_t merge; /* Merge external file. */
+ hbool_t prune; /* Don't follow external file. */
+ hsize_t min_comp; /* minimum size to compress, in bytes */
+ int use_native; /* use a native type in write */
+ hbool_t latest; /* pack file with the latest file format */
+ H5F_libver_t low_bound; /* The file's low bound as in H5Fset_libver_bounds() */
+ H5F_libver_t high_bound; /* The file's high bound as in H5Fset_libver_bounds() */
+ hid_t fin_fapl; /* FAPL to use for opening the input file */
+ hid_t fout_fapl; /* FAPL to use for opening/creating the output file */
+ int grp_compact; /* Set the maximum number of links to store as header messages in the group */
+ int grp_indexed; /* Set the minimum number of links to store in the indexed format */
+ int msg_size[8]; /* Minimum size of shared messages: dataspace,
+ datatype, fill value, filter pipleline, attribute */
const char *ublock_filename; /* user block file name */
hsize_t ublock_size; /* user block size */
hsize_t meta_block_size; /* metadata aggregation block size (for H5Pset_meta_block_size) */
diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c
index b1d6ab4..8f3496f 100644
--- a/tools/src/h5repack/h5repack_copy.c
+++ b/tools/src/h5repack/h5repack_copy.c
@@ -594,6 +594,8 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
hid_t f_space_id = H5I_INVALID_HID; /* file space ID */
hid_t ftype_id = H5I_INVALID_HID; /* file type ID */
hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */
+ hid_t ocpl_id = H5I_INVALID_HID; /* property to pass copy options */
+ hid_t lcpl_id = H5I_INVALID_HID; /* link creation property list */
named_dt_t *named_dt_head = NULL; /* Pointer to the stack of named datatypes copied */
size_t msize; /* size of type */
hsize_t nelmts; /* number of elements in dataset */
@@ -610,6 +612,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
int req_filter; /* there was a request for a filter */
int req_obj_layout = 0; /* request layout to current object */
unsigned crt_order_flags; /* group creation order flag */
+ h5tool_link_info_t linkinfo;
unsigned i;
unsigned u;
int ifil;
@@ -619,6 +622,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
hsize_t size_dset;
int ret_value = 0;
+ /* init linkinfo struct */
+ HDmemset(&linkinfo, 0, sizeof(h5tool_link_info_t));
+
/*-------------------------------------------------------------------------
* copy the supplied object list
*-------------------------------------------------------------------------
@@ -1137,26 +1143,25 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
*-------------------------------------------------------------------------
*/
else {
- hid_t pid = H5I_INVALID_HID;
-
/* create property to pass copy options */
- if ((pid = H5Pcreate(H5P_OBJECT_COPY)) < 0)
+ if ((ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0)
H5TOOLS_GOTO_ERROR((-1), "H5Pcreate failed");
/* set options for object copy */
- if (H5Pset_copy_object(pid, H5O_COPY_WITHOUT_ATTR_FLAG) < 0)
+ if (H5Pset_copy_object(ocpl_id, H5O_COPY_WITHOUT_ATTR_FLAG) < 0)
H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed");
if (H5Ocopy(fidin, /* Source file or group identifier */
travt->objs[i].name, /* Name of the source object to be copied */
fidout, /* Destination file or group identifier */
travt->objs[i].name, /* Name of the destination object */
- pid, /* Properties which apply to the copy */
+ ocpl_id, /* Properties which apply to the copy */
H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */
H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed");
- if (H5Pclose(pid) < 0)
+ if (H5Pclose(ocpl_id) < 0)
H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
+ ocpl_id = H5I_INVALID_HID;
/*-------------------------------------------------------------------------
* Copy attrs manually
@@ -1228,11 +1233,61 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
if (options->verbose)
HDprintf(FORMAT_OBJ, "link", travt->objs[i].name);
- if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0)
- H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed");
+ /* Check -X option. */
+ if (options->merge) {
+ if (H5tools_get_symlink_info(fidin, travt->objs[i].name, &linkinfo, 1) == 0) {
+ /* dangling link */
+ if (options->prune) {
+ HDprintf("Pruned %s.\n", travt->objs[i].name);
+ }
+ else {
+ if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0)
+ H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed");
+ }
+ }
+ else {
+ /* valid link */
+ /* create property to pass copy options */
+ if ((ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0)
+ H5TOOLS_GOTO_ERROR((-1), "H5Pcreate create property failed");
- if (options->verbose)
- HDprintf(FORMAT_OBJ, "link", travt->objs[i].name);
+ /* set options for object copy */
+ if (H5Pset_copy_object(ocpl_id, H5O_COPY_EXPAND_EXT_LINK_FLAG) < 0)
+ H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed");
+
+ /* Create link creation property list */
+ if((lcpl_id = H5Pcreate(H5P_LINK_CREATE)) < 0) {
+ H5TOOLS_GOTO_ERROR((-1), "H5Pcreate link creation property failed");
+ }
+
+ /* Set flag for intermediate group creation */
+ if (H5Pset_create_intermediate_group(lcpl_id, TRUE) < 0)
+ H5TOOLS_GOTO_ERROR((-1), "H5Pset_create_intermediate_group failed");
+
+ if (H5Ocopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, ocpl_id, lcpl_id) < 0)
+ H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed");
+
+ if (H5Pclose(lcpl_id) < 0)
+ H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
+
+ if (H5Pclose(ocpl_id) < 0)
+ H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
+ }
+
+ /* free link info path */
+ if (linkinfo.trg_path)
+ HDfree(linkinfo.trg_path);
+ linkinfo.trg_path = NULL;
+ } /* options->merge */
+ else {
+ if (options->prune) {
+ HDprintf("Pruned %s.\n", travt->objs[i].name);
+ }
+ else {
+ if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0)
+ H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed");
+ }
+ }
break;
default:
@@ -1249,15 +1304,22 @@ done:
if (named_datatype_free(&named_dt_head, 0) < 0)
H5TOOLS_ERROR((-1), "named_datatype_free failed");
}
- else
+ else {
H5E_BEGIN_TRY {
named_datatype_free(&named_dt_head, 1);
} H5E_END_TRY;
+ }
+
+ /* free link info path */
+ if (linkinfo.trg_path)
+ HDfree(linkinfo.trg_path);
H5E_BEGIN_TRY
{
H5Gclose(grp_in);
H5Gclose(grp_out);
+ H5Pclose(lcpl_id);
+ H5Pclose(ocpl_id);
H5Pclose(dcpl_in);
H5Pclose(gcpl_in);
H5Pclose(gcpl_out);
diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c
index 0ad61c0..cf0c611 100644
--- a/tools/src/h5repack/h5repack_main.c
+++ b/tools/src/h5repack/h5repack_main.c
@@ -32,36 +32,38 @@ const char *outfile = NULL;
* Command-line options: The user can specify short or long-named
* parameters.
*/
-static const char *s_opts = "hVvf:l:m:e:nLj:k:c:d:s:u:b:M:t:a:i:o:S:P:T:G:q:z:E";
+static const char *s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:vz:EG:LM:P:S:T:VXW1:2:3:4:5:6:";
static struct long_options l_opts[] = {
- { "help", no_arg, 'h' },
- { "version", no_arg, 'V' },
- { "verbose", no_arg, 'v' },
+ { "alignment", require_arg, 'a' },
+ { "block", require_arg, 'b' },
+ { "compact", require_arg, 'c' },
+ { "indexed", require_arg, 'd' },
+ { "file", require_arg, 'e' },
{ "filter", require_arg, 'f' },
+ { "help", no_arg, 'h' },
+ { "infile", require_arg, 'i' }, /* for backward compability */
+ { "low", require_arg, 'j' },
+ { "high", require_arg, 'k' },
{ "layout", require_arg, 'l' },
{ "minimum", require_arg, 'm' },
- { "file", require_arg, 'e' },
{ "native", no_arg, 'n' },
- { "latest", no_arg, 'L' },
- { "low", require_arg, 'j' },
- { "high", require_arg, 'k' },
- { "compact", require_arg, 'c' },
- { "indexed", require_arg, 'd' },
+ { "outfile", require_arg, 'o' }, /* for backward compability */
+ { "sort_by", require_arg, 'q' },
{ "ssize", require_arg, 's' },
+ { "threshold", require_arg, 't' },
{ "ublock", require_arg, 'u' },
- { "block", require_arg, 'b' },
+ { "verbose", no_arg, 'v' },
+ { "sort_order", require_arg, 'z' },
+ { "enable-error-stack", no_arg, 'E' },
+ { "fs_pagesize", require_arg, 'G' },
+ { "latest", no_arg, 'L' },
{ "metadata_block_size", require_arg, 'M' },
- { "threshold", require_arg, 't' },
- { "alignment", require_arg, 'a' },
- { "infile", require_arg, 'i' }, /* for backward compability */
- { "outfile", require_arg, 'o' }, /* for backward compability */
- { "fs_strategy", require_arg, 'S' },
{ "fs_persist", require_arg, 'P' },
+ { "fs_strategy", require_arg, 'S' },
{ "fs_threshold", require_arg, 'T' },
- { "fs_pagesize", require_arg, 'G' },
- { "sort_by", require_arg, 'q' },
- { "sort_order", require_arg, 'z' },
- { "enable-error-stack", no_arg, 'E' },
+ { "version", no_arg, 'V' },
+ { "merge", no_arg, 'X' },
+ { "prune", no_arg, 'W' },
{ "src-vol-value", require_arg, '1' },
{ "src-vol-name", require_arg, '2' },
{ "src-vol-info", require_arg, '3' },
@@ -113,6 +115,9 @@ static void usage(const char *prog) {
PRINTVALSTREAM(rawoutstream, " --high=BOUND The high bound for library release versions to use\n");
PRINTVALSTREAM(rawoutstream, " when creating objects in the file\n");
PRINTVALSTREAM(rawoutstream, " (default is H5F_LIBVER_LATEST)\n");
+ PRINTVALSTREAM(rawoutstream, " --merge Follow external soft link recursively and merge data\n");
+ PRINTVALSTREAM(rawoutstream, " --prune Do not follow external soft links and remove link\n");
+ PRINTVALSTREAM(rawoutstream, " --merge --prune Follow external link, merge data and remove dangling link\n");
PRINTVALSTREAM(rawoutstream, " -c L1, --compact=L1 Maximum number of links in header messages\n");
PRINTVALSTREAM(rawoutstream, " -d L2, --indexed=L2 Minimum number of links in the indexed format\n");
PRINTVALSTREAM(rawoutstream, " -s S[:F], --ssize=S[:F] Shared object header message minimum size\n");
@@ -545,6 +550,14 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
options->high_bound = bound;
break;
+ case 'X':
+ options->merge = TRUE;
+ break;
+
+ case 'W':
+ options->prune = TRUE;
+ break;
+
case 'c':
options->grp_compact = HDatoi( opt_arg );
if (options->grp_compact > 0)