HDF5 1 8 merge of recent changes and tools (#353)

* Update supported platforms

* Merge PR#3 changes from develop

* # WARNING: head commit changed in the meantime

Merge gcc 10 diagnostics option from develop

Merge CMake changes from develop
Merge warnings from develop
Merge #318 OSX changes from develop
Merge tools changes from develop
Merge test macros from develop

* Format updates

* Fix missing semicolon and format fix

* Format update

* Correct actions, remove java option

* Update autotools build files

* Add testfiles

* Fix configure issue with make flags

* Init fapls to default

* Update generated files and fix h5repack id closure

* update format

1 files changed, 94 insertions, 58 deletions

diff --git a/tools/h5repack/h5repack_copy.c b/tools/h5repack/h5repack_copy.c
index 7ee6399..2df22fa 100644
--- a/tools/h5repack/h5repack_copy.c
+++ b/tools/h5repack/h5repack_copy.c
@@ -33,23 +33,6 @@
 /* size of buffer/# of bytes to xfer at a time when copying userblock */
 #define USERBLOCK_XFER_SIZE 512
 
-/* check H5Dread()/H5Dwrite() error, e.g. memory allocation error inside the library. */
-#define CHECK_H5DRW_ERROR(_fun, _fail, _did, _mtid, _msid, _fsid, _pid, _buf)                                \
-    {                                                                                                        \
-        H5E_BEGIN_TRY                                                                                        \
-        {                                                                                                    \
-            if (_fun(_did, _mtid, _msid, _fsid, _pid, _buf) < 0) {                                           \
-                hid_t _err_num = 0;                                                                          \
-                char  _msg[80];                                                                              \
-                H5Ewalk2(H5E_DEFAULT, H5E_WALK_DOWNWARD, walk_error_callback, &_err_num);                    \
-                H5Eget_msg(_err_num, NULL, _msg, (size_t)80);                                                \
-                error_msg("%s %s -- %s\n", #_fun, "failed", _msg);                                           \
-                HGOTO_DONE(_fail)                                                                            \
-            }                                                                                                \
-        }                                                                                                    \
-        H5E_END_TRY;                                                                                         \
-    }
-
 /*-------------------------------------------------------------------------
  * local functions
  *-------------------------------------------------------------------------
@@ -62,17 +45,6 @@ static int  copy_user_block(const char *infile, const char *outfile, hsize_t siz
 #if defined(H5REPACK_DEBUG_USER_BLOCK)
 static void print_user_block(const char *filename, hid_t fid);
 #endif
-static herr_t walk_error_callback(unsigned n, const H5E_error2_t *err_desc, void *udata);
-
-/* get the major number from the error stack. */
-static herr_t
-walk_error_callback(H5_ATTR_UNUSED unsigned n, const H5E_error2_t *err_desc, void *udata)
-{
-    if (err_desc)
-        *((hid_t *)udata) = err_desc->maj_num;
-
-    return 0;
-}
 
 /*-------------------------------------------------------------------------
  * Function: copy_objects
@@ -92,7 +64,6 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options)
     hid_t         grp_in  = H5I_INVALID_HID; /* group ID */
     hid_t         gcpl_in = H5I_INVALID_HID; /* group creation property list */
     hid_t         fcpl    = H5P_DEFAULT;     /* file creation property list ID */
-    hid_t         fapl    = H5P_DEFAULT;     /* file access property list ID */
     trav_table_t *travt   = NULL;
     hsize_t       ub_size = 0;     /* size of user block */
     unsigned      crt_order_flags; /* group creation order flag */
@@ -102,7 +73,8 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options)
      * open input file
      *-------------------------------------------------------------------------
      */
-    if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, H5P_DEFAULT, NULL, NULL, (size_t)0)) < 0)
+    if ((fidin = h5tools_fopen(fnamein, H5F_ACC_RDONLY, options->fin_fapl,
+                               (options->fin_fapl == H5P_DEFAULT) ? FALSE : TRUE, NULL, (size_t)0)) < 0)
         H5TOOLS_GOTO_ERROR((-1), "h5tools_fopen failed <%s>: %s", fnamein, H5FOPENERROR);
 
     /* get user block size and file space strategy/threshold */
@@ -130,11 +102,12 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options)
     }
 
     /* Create file access property list */
-    if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
-        H5TOOLS_GOTO_ERROR((-1), "H5Pcreate failed to create file access property list");
+    if (options->fout_fapl == H5P_DEFAULT)
+        if ((options->fout_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+            H5TOOLS_GOTO_ERROR((-1), "H5Pcreate failed to create file access property list");
 
     if (options->latest)
-        if (H5Pset_libver_bounds(fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
+        if (H5Pset_libver_bounds(options->fout_fapl, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
             H5TOOLS_GOTO_ERROR((-1), "H5Pset_libver_bounds failed to set format version bounds");
 
     /* Check if we need to create a non-default file creation property list */
@@ -230,12 +203,12 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options)
      */
     if (options->alignment > 0) {
         /* either use the FAPL already created or create a new one */
-        if (fapl == H5P_DEFAULT)
+        if (options->fout_fapl == H5P_DEFAULT)
             /* create a file access property list */
-            if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+            if ((options->fout_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
                 H5TOOLS_GOTO_ERROR((-1), "H5Pcreate failed to create file access property list");
 
-        if (H5Pset_alignment(fapl, options->threshold, options->alignment) < 0)
+        if (H5Pset_alignment(options->fout_fapl, options->threshold, options->alignment) < 0)
             H5TOOLS_GOTO_ERROR((-1), "H5Pset_alignment failed to set alignment");
     }
 
@@ -245,12 +218,12 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options)
      */
     if (options->meta_block_size > 0) {
         /* either use the FAPL already created or create a new one */
-        if (fapl == H5P_DEFAULT)
+        if (options->fout_fapl == H5P_DEFAULT)
             /* create a file access property list */
-            if ((fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
+            if ((options->fout_fapl = H5Pcreate(H5P_FILE_ACCESS)) < 0)
                 H5TOOLS_GOTO_ERROR((-1), "H5Pcreate failed to create file access property list");
 
-        if (H5Pset_meta_block_size(fapl, options->meta_block_size) < 0)
+        if (H5Pset_meta_block_size(options->fout_fapl, options->meta_block_size) < 0)
             H5TOOLS_GOTO_ERROR((-1), "H5Pset_meta_block_size failed to set metadata block size");
     }
 
@@ -275,7 +248,7 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options)
     if (options->verbose)
         HDprintf("Making new file ...\n");
 
-    if ((fidout = H5Fcreate(fnameout, H5F_ACC_TRUNC, fcpl, fapl)) < 0)
+    if ((fidout = H5Fcreate(fnameout, H5F_ACC_TRUNC, fcpl, options->fout_fapl)) < 0)
         H5TOOLS_GOTO_ERROR((-1), "H5Fcreate could not create file <%s>:", fnameout);
 
     /*-------------------------------------------------------------------------
@@ -331,15 +304,14 @@ copy_objects(const char *fnamein, const char *fnameout, pack_opt_t *options)
 done:
     H5E_BEGIN_TRY
     {
-        H5Pclose(fcpl_in);
-        H5Pclose(gcpl_in);
-        H5Pclose(fapl);
         H5Pclose(fcpl);
+        H5Pclose(options->fout_fapl);
+        options->fout_fapl = H5P_DEFAULT;
+        H5Pclose(gcpl_in);
         H5Gclose(grp_in);
-        H5Fclose(fidin);
+        H5Pclose(fcpl_in);
         H5Fclose(fidout);
         H5Fclose(fidin);
-        H5Fclose(fidout);
     }
     H5E_END_TRY;
     if (travt)
@@ -574,6 +546,8 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti
     hid_t              f_space_id    = H5I_INVALID_HID; /* file space ID */
     hid_t              ftype_id      = H5I_INVALID_HID; /* file type ID */
     hid_t              wtype_id      = H5I_INVALID_HID; /* read/write type ID */
+    hid_t              ocpl_id       = H5I_INVALID_HID; /* property to pass copy options */
+    hid_t              lcpl_id       = H5I_INVALID_HID; /* link creation property list */
     named_dt_t *       named_dt_head = NULL;            /* Pointer to the stack of named datatypes copied */
     size_t             msize;                           /* size of type */
     hsize_t            nelmts;                          /* number of elements in dataset */
@@ -590,6 +564,7 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti
     int                req_filter;         /* there was a request for a filter */
     int                req_obj_layout = 0; /* request layout to current object */
     unsigned           crt_order_flags;    /* group creation order flag */
+    h5tool_link_info_t linkinfo;
     unsigned           i;
     unsigned           u;
     int                ifil;
@@ -599,6 +574,9 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti
     hsize_t            size_dset;
     int                ret_value = 0;
 
+    /* init linkinfo struct */
+    HDmemset(&linkinfo, 0, sizeof(h5tool_link_info_t));
+
     /*-------------------------------------------------------------------------
      * copy the supplied object list
      *-------------------------------------------------------------------------
@@ -1105,26 +1083,25 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti
                      *-------------------------------------------------------------------------
                      */
                     else {
-                        hid_t pid = H5I_INVALID_HID;
-
                         /* create property to pass copy options */
-                        if ((pid = H5Pcreate(H5P_OBJECT_COPY)) < 0)
+                        if ((ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0)
                             H5TOOLS_GOTO_ERROR((-1), "H5Pcreate failed");
 
                         /* set options for object copy */
-                        if (H5Pset_copy_object(pid, H5O_COPY_WITHOUT_ATTR_FLAG) < 0)
+                        if (H5Pset_copy_object(ocpl_id, H5O_COPY_WITHOUT_ATTR_FLAG) < 0)
                             H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed");
 
                         if (H5Ocopy(fidin,               /* Source file or group identifier */
                                     travt->objs[i].name, /* Name of the source object to be copied */
                                     fidout,              /* Destination file or group identifier  */
                                     travt->objs[i].name, /* Name of the destination object  */
-                                    pid,                 /* Properties which apply to the copy   */
+                                    ocpl_id,             /* Properties which apply to the copy   */
                                     H5P_DEFAULT) < 0)    /* Properties which apply to the new hard link */
                             H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed");
 
-                        if (H5Pclose(pid) < 0)
+                        if (H5Pclose(ocpl_id) < 0)
                             H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
+                        ocpl_id = H5I_INVALID_HID;
 
                         /*-------------------------------------------------------------------------
                          * Copy attrs manually
@@ -1197,12 +1174,64 @@ do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt, pack_opt_t *opti
                     if (options->verbose)
                         HDprintf(FORMAT_OBJ, "link", travt->objs[i].name);
 
-                    if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT,
-                                H5P_DEFAULT) < 0)
-                        H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed");
+                    /* Check -X option. */
+                    if (options->merge) {
+                        if (H5tools_get_symlink_info(fidin, travt->objs[i].name, &linkinfo, 1) == 0) {
+                            /* dangling link */
+                            if (options->prune) {
+                                HDprintf("Pruned %s.\n", travt->objs[i].name);
+                            }
+                            else {
+                                if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name,
+                                            H5P_DEFAULT, H5P_DEFAULT) < 0)
+                                    H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed");
+                            }
+                        }
+                        else {
+                            /* valid link */
+                            /* create property to pass copy options */
+                            if ((ocpl_id = H5Pcreate(H5P_OBJECT_COPY)) < 0)
+                                H5TOOLS_GOTO_ERROR((-1), "H5Pcreate create property failed");
+
+                            /* set options for object copy */
+                            if (H5Pset_copy_object(ocpl_id, H5O_COPY_EXPAND_EXT_LINK_FLAG) < 0)
+                                H5TOOLS_GOTO_ERROR((-1), "H5Pset_copy_object failed");
+
+                            /* Create link creation property list */
+                            if ((lcpl_id = H5Pcreate(H5P_LINK_CREATE)) < 0) {
+                                H5TOOLS_GOTO_ERROR((-1), "H5Pcreate link creation property failed");
+                            }
+
+                            /* Set flag for intermediate group creation */
+                            if (H5Pset_create_intermediate_group(lcpl_id, TRUE) < 0)
+                                H5TOOLS_GOTO_ERROR((-1), "H5Pset_create_intermediate_group failed");
+
+                            if (H5Ocopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, ocpl_id,
+                                        lcpl_id) < 0)
+                                H5TOOLS_GOTO_ERROR((-1), "H5Ocopy failed");
+
+                            if (H5Pclose(lcpl_id) < 0)
+                                H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
+
+                            if (H5Pclose(ocpl_id) < 0)
+                                H5TOOLS_GOTO_ERROR((-1), "H5Pclose failed");
+                        }
 
-                    if (options->verbose)
-                        HDprintf(FORMAT_OBJ, "link", travt->objs[i].name);
+                        /* free link info path */
+                        if (linkinfo.trg_path)
+                            HDfree(linkinfo.trg_path);
+                        linkinfo.trg_path = NULL;
+                    } /* options->merge */
+                    else {
+                        if (options->prune) {
+                            HDprintf("Pruned %s.\n", travt->objs[i].name);
+                        }
+                        else {
+                            if (H5Lcopy(fidin, travt->objs[i].name, fidout, travt->objs[i].name, H5P_DEFAULT,
+                                        H5P_DEFAULT) < 0)
+                                H5TOOLS_GOTO_ERROR((-1), "H5Lcopy failed");
+                        }
+                    }
                     break;
 
                 default:
@@ -1219,14 +1248,21 @@ done:
         if (named_datatype_free(&named_dt_head, 0) < 0)
             H5TOOLS_ERROR((-1), "named_datatype_free failed");
     }
-    else
+    else {
         H5E_BEGIN_TRY { named_datatype_free(&named_dt_head, 1); }
-    H5E_END_TRY;
+        H5E_END_TRY;
+    }
+
+    /* free link info path */
+    if (linkinfo.trg_path)
+        HDfree(linkinfo.trg_path);
 
     H5E_BEGIN_TRY
     {
         H5Gclose(grp_in);
         H5Gclose(grp_out);
+        H5Pclose(lcpl_id);
+        H5Pclose(ocpl_id);
         H5Pclose(dcpl_in);
         H5Pclose(gcpl_in);
         H5Pclose(gcpl_out);