HDFFV-9984 Add options to merge/prune external links during repack

4 files changed, 128 insertions, 51 deletions

diff --git a/tools/src/h5copy/h5copy.c b/tools/src/h5copy/h5copy.c
index 5b19ae7..a6e4d4e 100644
--- a/tools/src/h5copy/h5copy.c
+++ b/tools/src/h5copy/h5copy.c
@@ -212,8 +212,8 @@ main (int argc, const char *argv[])
     unsigned     flag = 0;
     unsigned     verbose = 0;
     unsigned     parents = 0;
-    hid_t        ocpl_id = (-1);          /* Object copy property list */
-    hid_t        lcpl_id = (-1);          /* Link creation property list */
+    hid_t        ocpl_id = H5I_INVALID_HID;          /* Object copy property list */
+    hid_t        lcpl_id = H5I_INVALID_HID;          /* Link creation property list */
     int          opt;
     int          li_ret;
     h5tool_link_info_t linkinfo;
diff --git a/tools/src/h5repack/h5repack.h b/tools/src/h5repack/h5repack.h
index caa1166..b124fdb 100644
--- a/tools/src/h5repack/h5repack.h
+++ b/tools/src/h5repack/h5repack.h
@@ -101,25 +101,27 @@ typedef struct {
 
 /* all the above, ready to go to the hrepack call */
 typedef struct {
-    pack_opttbl_t   *op_tbl;        /*table with all -c and -f options */
-    int             all_layout;     /*apply the layout to all objects */
-    int             all_filter;     /*apply the filter to all objects */
+    pack_opttbl_t   *op_tbl;           /* table with all -c and -f options */
+    int             all_layout;        /* apply the layout to all objects */
+    int             all_filter;        /* apply the filter to all objects */
     filter_info_t   filter_g[H5_REPACK_MAX_NFILTERS];    /*global filter array for the ALL case */
-    int             n_filter_g;     /*number of global filters */
-    chunk_info_t    chunk_g;        /*global chunk INFO for the ALL case */
-    H5D_layout_t    layout_g;       /*global layout information for the ALL case */
-    int             verbose;        /*verbose mode */
-    hsize_t         min_comp;       /*minimum size to compress, in bytes */
-    int             use_native;     /*use a native type in write */
-    hbool_t         latest;         /*pack file with the latest file format */
-    H5F_libver_t    low_bound;      /* The file's low bound as in H5Fset_libver_bounds() */
-    H5F_libver_t    high_bound;     /* The file's high bound as in H5Fset_libver_bounds() */
-    hid_t           fin_fapl;       /* FAPL to use for opening the input file */
-    hid_t           fout_fapl;      /* FAPL to use for opening/creating the output file */
-    int             grp_compact;    /* Set the maximum number of links to store as header messages in the group */
-    int             grp_indexed;    /* Set the minimum number of links to store in the indexed format */
-    int             msg_size[8];    /* Minimum size of shared messages: dataspace,
-                                       datatype, fill value, filter pipleline, attribute */
+    int             n_filter_g;        /* number of global filters */
+    chunk_info_t    chunk_g;           /* global chunk INFO for the ALL case */
+    H5D_layout_t    layout_g;          /* global layout information for the ALL case */
+    int             verbose;           /* verbose mode */
+    hbool_t         merge;             /* Merge external file. */
+    hbool_t         prune;             /* Don't follow external file. */
+    hsize_t         min_comp;          /* minimum size to compress, in bytes */
+    int             use_native;        /* use a native type in write */
+    hbool_t         latest;            /* pack file with the latest file format */
+    H5F_libver_t    low_bound;         /* The file's low bound as in H5Fset_libver_bounds() */
+    H5F_libver_t    high_bound;        /* The file's high bound as in H5Fset_libver_bounds() */
+    hid_t           fin_fapl;          /* FAPL to use for opening the input file */
+    hid_t           fout_fapl;         /* FAPL to use for opening/creating the output file */
+    int             grp_compact;       /* Set the maximum number of links to store as header messages in the group */
+    int             grp_indexed;       /* Set the minimum number of links to store in the indexed format */
+    int             msg_size[8];       /* Minimum size of shared messages: dataspace,
+                                          datatype, fill value, filter pipleline, attribute */
     const char      *ublock_filename;  /* user block file name */
     hsize_t         ublock_size;       /* user block size */
     hsize_t         meta_block_size;   /* metadata aggregation block size (for H5Pset_meta_block_size) */
diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c
index b1d6ab4..8f3496f 100644
--- a/tools/src/h5repack/h5repack_copy.c
+++ b/tools/src/h5repack/h5repack_copy.c
@@ -594,6 +594,8 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
     hid_t f_space_id = H5I_INVALID_HID; /* file space ID */
     hid_t ftype_id = H5I_INVALID_HID; /* file type ID */
     hid_t wtype_id = H5I_INVALID_HID; /* read/write type ID */
+    hid_t ocpl_id = H5I_INVALID_HID;  /* property to pass copy options */
+    hid_t lcpl_id = H5I_INVALID_HID;  /* link creation property list */
     named_dt_t *named_dt_head = NULL; /* Pointer to the stack of named datatypes copied */
     size_t msize;        /* size of type */
     hsize_t nelmts;      /* number of elements in dataset */
@@ -610,6 +612,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
     int req_filter;      /* there was a request for a filter */
     int req_obj_layout = 0; /* request layout to current object */
     unsigned crt_order_flags; /* group creation order flag */
+    h5tool_link_info_t linkinfo;
     unsigned i;
     unsigned u;
     int ifil;
@@ -619,6 +622,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
     hsize_t size_dset;
     int ret_value = 0;
 
+    /* init linkinfo struct */
+    HDmemset(&linkinfo, 0, sizeof(h5tool_link_info_t));
+
     /*-------------------------------------------------------------------------
      * copy the supplied object list
      *-------------------------------------------------------------------------
@@ -1137,26 +1143,25 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                  *-------------------------------------------------------------------------
                  */
                 else {
-                    hid_t pid = H5I_INVALID_HID;
-
                     /* create property to pass copy options */
-                    if ((pid = H5Pcreate(H5P_OBJECT_COPY)) < 0)
+                    if ((ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0)
                         H5TOOLS_GOTO_ERROR((-1), "H5Pcreate failed");
 
                     /* set options for object copy */
-                    if (H5Pset_copy_object(pid, H5O_COPY_WITHOUT_ATTR_FLAG) < 0)
+                    if (H5Pset_copy_object(ocpl_id, H5O_COPY_WITHOUT_ATTR_FLAG) < 0)
                         H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed");
 
                     if (H5Ocopy(fidin, /* Source file or group identifier */
                             travt->objs[i].name, /* Name of the source object to be copied */
                             fidout, /* Destination file or group identifier  */
                             travt->objs[i].name, /* Name of the destination object  */
-                            pid, /* Properties which apply to the copy   */
+                            ocpl_id, /* Properties which apply to the copy   */
                             H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */
                         H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed");
 
-                    if (H5Pclose(pid) < 0)
+                    if (H5Pclose(ocpl_id) < 0)
                         H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
+                    ocpl_id = H5I_INVALID_HID;
 
                     /*-------------------------------------------------------------------------
                      * Copy attrs manually
@@ -1228,11 +1233,61 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                 if (options->verbose)
                     HDprintf(FORMAT_OBJ, "link", travt->objs[i].name);
 
-                if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0)
-                    H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed");
+                /* Check -X option. */
+                if (options->merge) {
+                    if (H5tools_get_symlink_info(fidin, travt->objs[i].name, &linkinfo, 1) == 0) {
+                        /* dangling link */
+                        if (options->prune) {
+                            HDprintf("Pruned %s.\n", travt->objs[i].name);
+                        }
+                        else {
+                            if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0)
+                                H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed");
+                        }
+                    }
+                    else {
+                        /* valid link */
+                        /* create property to pass copy options */
+                        if ((ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0)
+                            H5TOOLS_GOTO_ERROR((-1), "H5Pcreate create property failed");
 
-                if (options->verbose)
-                    HDprintf(FORMAT_OBJ, "link", travt->objs[i].name);
+                        /* set options for object copy */
+                        if (H5Pset_copy_object(ocpl_id, H5O_COPY_EXPAND_EXT_LINK_FLAG) < 0)
+                            H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed");
+
+                        /* Create link creation property list */
+                        if((lcpl_id = H5Pcreate(H5P_LINK_CREATE)) < 0) {
+                            H5TOOLS_GOTO_ERROR((-1), "H5Pcreate link creation property failed");
+                        }
+
+                        /* Set flag for intermediate group creation */
+                        if (H5Pset_create_intermediate_group(lcpl_id, TRUE) < 0)
+                            H5TOOLS_GOTO_ERROR((-1), "H5Pset_create_intermediate_group failed");
+
+                        if (H5Ocopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, ocpl_id, lcpl_id) < 0)
+                            H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed");
+
+                        if (H5Pclose(lcpl_id) < 0)
+                            H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
+
+                        if (H5Pclose(ocpl_id) < 0)
+                            H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
+                    }
+
+                    /* free link info path */
+                    if (linkinfo.trg_path)
+                        HDfree(linkinfo.trg_path);
+                    linkinfo.trg_path = NULL;
+                } /* options->merge */
+                else {
+                    if (options->prune) {
+                        HDprintf("Pruned %s.\n", travt->objs[i].name);
+                    }
+                    else {
+                        if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT, H5P_DEFAULT) < 0)
+                            H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed");
+                    }
+                }
                 break;
 
             default:
@@ -1249,15 +1304,22 @@ done:
         if (named_datatype_free(&named_dt_head, 0) < 0)
             H5TOOLS_ERROR((-1), "named_datatype_free failed");
     }
-    else
+    else {
         H5E_BEGIN_TRY {
             named_datatype_free(&named_dt_head, 1);
         } H5E_END_TRY;
+    }
+
+    /* free link info path */
+    if (linkinfo.trg_path)
+        HDfree(linkinfo.trg_path);
 
     H5E_BEGIN_TRY
     {
         H5Gclose(grp_in);
         H5Gclose(grp_out);
+        H5Pclose(lcpl_id);
+        H5Pclose(ocpl_id);
         H5Pclose(dcpl_in);
         H5Pclose(gcpl_in);
         H5Pclose(gcpl_out);
diff --git a/tools/src/h5repack/h5repack_main.c b/tools/src/h5repack/h5repack_main.c
index 0ad61c0..cf0c611 100644
--- a/tools/src/h5repack/h5repack_main.c
+++ b/tools/src/h5repack/h5repack_main.c
@@ -32,36 +32,38 @@ const char *outfile = NULL;
  * Command-line options: The user can specify short or long-named
  * parameters.
  */
-static const char *s_opts = "hVvf:l:m:e:nLj:k:c:d:s:u:b:M:t:a:i:o:S:P:T:G:q:z:E";
+static const char *s_opts = "a:b:c:d:e:f:hi:j:k:l:m:no:q:s:t:u:vz:EG:LM:P:S:T:VXW1:2:3:4:5:6:";
 static struct long_options l_opts[] = {
-    { "help",                no_arg,      'h' },
-    { "version",             no_arg,      'V' },
-    { "verbose",             no_arg,      'v' },
+    { "alignment",           require_arg, 'a' },
+    { "block",               require_arg, 'b' },
+    { "compact",             require_arg, 'c' },
+    { "indexed",             require_arg, 'd' },
+    { "file",                require_arg, 'e' },
     { "filter",              require_arg, 'f' },
+    { "help",                no_arg,      'h' },
+    { "infile",              require_arg, 'i' }, /* for backward compability */
+    { "low",                 require_arg, 'j' },
+    { "high",                require_arg, 'k' },
     { "layout",              require_arg, 'l' },
     { "minimum",             require_arg, 'm' },
-    { "file",                require_arg, 'e' },
     { "native",              no_arg,      'n' },
-    { "latest",              no_arg,      'L' },
-    { "low",                 require_arg, 'j' },
-    { "high",                require_arg, 'k' },
-    { "compact",             require_arg, 'c' },
-    { "indexed",             require_arg, 'd' },
+    { "outfile",             require_arg, 'o' }, /* for backward compability */
+    { "sort_by",             require_arg, 'q' },
     { "ssize",               require_arg, 's' },
+    { "threshold",           require_arg, 't' },
     { "ublock",              require_arg, 'u' },
-    { "block",               require_arg, 'b' },
+    { "verbose",             no_arg,      'v' },
+    { "sort_order",          require_arg, 'z' },
+    { "enable-error-stack",  no_arg,      'E' },
+    { "fs_pagesize",         require_arg, 'G' },
+    { "latest",              no_arg,      'L' },
     { "metadata_block_size", require_arg, 'M' },
-    { "threshold",           require_arg, 't' },
-    { "alignment",           require_arg, 'a' },
-    { "infile",              require_arg, 'i' }, /* for backward compability */
-    { "outfile",             require_arg, 'o' }, /* for backward compability */
-    { "fs_strategy",         require_arg, 'S' },
     { "fs_persist",          require_arg, 'P' },
+    { "fs_strategy",         require_arg, 'S' },
     { "fs_threshold",        require_arg, 'T' },
-    { "fs_pagesize",         require_arg, 'G' },
-    { "sort_by",             require_arg, 'q' },
-    { "sort_order",          require_arg, 'z' },
-    { "enable-error-stack",  no_arg,      'E' },
+    { "version",             no_arg,      'V' },
+    { "merge",               no_arg,      'X' },
+    { "prune",               no_arg,      'W' },
     { "src-vol-value",       require_arg, '1' },
     { "src-vol-name",        require_arg, '2' },
     { "src-vol-info",        require_arg, '3' },
@@ -113,6 +115,9 @@ static void usage(const char *prog) {
     PRINTVALSTREAM(rawoutstream, "   --high=BOUND            The high bound for library release versions to use\n");
     PRINTVALSTREAM(rawoutstream, "                           when creating objects in the file\n");
     PRINTVALSTREAM(rawoutstream, "                           (default is H5F_LIBVER_LATEST)\n");
+    PRINTVALSTREAM(rawoutstream, "   --merge                 Follow external soft link recursively and merge data\n");
+    PRINTVALSTREAM(rawoutstream, "   --prune                 Do not follow external soft links and remove link\n");
+    PRINTVALSTREAM(rawoutstream, "   --merge --prune         Follow external link, merge data and remove dangling link\n");
     PRINTVALSTREAM(rawoutstream, "   -c L1, --compact=L1     Maximum number of links in header messages\n");
     PRINTVALSTREAM(rawoutstream, "   -d L2, --indexed=L2     Minimum number of links in the indexed format\n");
     PRINTVALSTREAM(rawoutstream, "   -s S[:F], --ssize=S[:F] Shared object header message minimum size\n");
@@ -545,6 +550,14 @@ int parse_command_line(int argc, const char **argv, pack_opt_t* options)
                 options->high_bound = bound;
                 break;
 
+            case 'X':
+                options->merge = TRUE;
+                break;
+
+            case 'W':
+                options->prune = TRUE;
+                break;
+
             case 'c':
                 options->grp_compact = HDatoi( opt_arg );
                 if (options->grp_compact > 0)