Merge pull request #1756 in HDFFV/hdf5 from ~JAKE.SMITH/hdf5:bugfix/repack_external_storage to develop

* commit '9a388325ebaec339f67ddd210ca31ddcc3ede973':
  Update to current external file population.
  Revert else-if to single line.
  Change print calls to HD-wrapped. Minor formatting tweaks (inc. regressions and in-file style-matching).
  Refactor h5repackgentest to have more modular internals, for easier addition of new tests. Update manifest and test script for new hdf5 external-storage .h5 files.
  Refactoring.
  Add copyright/disclaimer to repack gentest. Minor formatting tweaks.
  Minor formatting adjustments.
  Undo some formatting modifications. Update MANIFEST.
  Add forgotten files to include from h5repack testfiles for tests.
  Fix failing automated reporting. Add 8-bit big-endian testfile.
  First pass at repack consolidating external storage if layout given. Checks out manually, tests still complain.
  Formatting changes, mostly.
  Add initial failing test for h5repack copying external data into new file.
  Modify code comments for clarity. Some minor reformatting.
  Some minor reformatting. Replace tabs with spaces.
  Add test that uses the family member FAPL setting.
  refactor h5repack_main.c -- remove some redundancy, formatting
  Fit help message to 80 character width and clarify some details.

1 files changed, 152 insertions, 69 deletions

diff --git a/tools/src/h5repack/h5repack_copy.c b/tools/src/h5repack/h5repack_copy.c
index 13adb7c..398d632 100644
--- a/tools/src/h5repack/h5repack_copy.c
+++ b/tools/src/h5repack/h5repack_copy.c
@@ -58,8 +58,8 @@ static void print_user_block(const char *filename, hid_t fid);
  *          -1 no
  *-------------------------------------------------------------------------
  */
-
-int copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
+int
+copy_objects(const char* fnamein, const char* fnameout, pack_opt_t *options)
 {
     int           ret_value = 0;
     hid_t         fidin = -1;
@@ -365,7 +365,7 @@ done:
         trav_table_free(travt);
 
     return ret_value;
-}
+} /* end copy_objects() */
 
 /*-------------------------------------------------------------------------
  * Function: Get_hyperslab
@@ -521,7 +521,7 @@ Get_hyperslab(hid_t dcpl_id, int rank_dset, hsize_t dims_dset[],
 
 done:
     return ret_value;
-}
+} /* end Get_hyperslab() */
 
 /*-------------------------------------------------------------------------
  * Function: do_copy_objects
@@ -577,7 +577,8 @@ done:
  *-------------------------------------------------------------------------
  */
 
-int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
+int
+do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
         pack_opt_t *options) /* repack options */
 {
     int   ret_value = 0;
@@ -677,7 +678,11 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                 }
                 else {
                     if (options->grp_compact > 0 || options->grp_indexed > 0)
-                        if (H5Pset_link_phase_change(gcpl_out, (unsigned) options->grp_compact, (unsigned) options->grp_indexed) < 0)
+                        if (H5Pset_link_phase_change(
+                                gcpl_out,
+                                (unsigned) options->grp_compact,
+                                (unsigned) options->grp_indexed)
+                            < 0)
                             HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_link_phase_change failed");
 
                     if ((grp_out = H5Gcreate2(fidout, travt->objs[i].name, H5P_DEFAULT, gcpl_out, H5P_DEFAULT)) < 0)
@@ -777,8 +782,6 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_type failed");
                     if ((dcpl_in = H5Dget_create_plist(dset_in)) < 0)
                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_create_plist failed");
-                    if ((dcpl_out = H5Pcopy(dcpl_in)) < 0)
-                        HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcopy failed");
                     if ((rank = H5Sget_simple_extent_ndims(f_space_id)) < 0)
                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sget_simple_extent_ndims failed");
                     HDmemset(dims, 0, sizeof dims);
@@ -787,6 +790,20 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                     if (H5Dget_space_status(dset_in, &space_status) < 0)
                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dget_space_status failed");
 
+                    /* If the input dataset has external storage, it must be contiguous.
+                     * Accordingly, there would be no filter or chunk properties to preserve,
+                     * so create a new DCPL.
+                     * Otherwise, copy dcpl_in.
+                     */
+                    if (H5Pget_external_count(dcpl_in)) {
+                        if ((dcpl_out = H5Pcreate(H5P_DATASET_CREATE)) < 0)
+                            HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcreate failed");
+                    }
+                    else
+                    if ((dcpl_out = H5Pcopy(dcpl_in)) < 0) {
+                        HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pcopy failed");
+                    }
+
                     nelmts = 1;
                     for (j = 0; j < rank; j++)
                         nelmts *= dims[j];
@@ -797,7 +814,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                             wtype_id = H5Tget_native_type(ftype_id, H5T_DIR_DEFAULT);
                         else
                             wtype_id = H5Tcopy(ftype_id);
-                    } /* end if */
+                    }
 
                     if ((msize = H5Tget_size(wtype_id)) == 0)
                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Tget_size failed");
@@ -830,18 +847,29 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                              * changing to COMPACT. For the reference, COMPACT is limited
                              * by size 64K by library.
                              */
-                            if (options->layout_g != H5D_COMPACT)
-                                if (size_dset < options->min_comp)
+                            if (options->layout_g != H5D_COMPACT) {
+                                if (size_dset < options->min_comp) {
                                     apply_s = 0;
+                                }
+                            }
 
                             /* apply the filter */
-                            if (apply_s)
-                                if (apply_filters(travt->objs[i].name, rank, dims, msize, dcpl_out, options, &has_filter) < 0)
+                            if (apply_s) {
+                                if (apply_filters(
+                                        travt->objs[i].name, 
+                                        rank, 
+                                        dims, 
+                                        msize, 
+                                        dcpl_out, 
+                                        options, 
+                                        &has_filter)
+                                    < 0)
                                     HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "apply_filters failed");
+                            }
 
                             /* only if layout change requested for entire file or
                              * individual obj */
-                            if (options->all_layout > 0 || req_obj_layout == 1)
+                            if (options->all_layout > 0 || req_obj_layout == 1) {
                                 /*-------------------------------------------------
                                  * Unset the unlimited max dims if convert to other
                                  * than chunk layouts, because unlimited max dims
@@ -861,14 +889,14 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
 
                                     /* if convert to COMPACT */
                                     if (options->layout_g == H5D_COMPACT)
-                                        /* should be smaller than 64K */
                                         if (size_dset > MAX_COMPACT_DSIZE)
                                             limit_maxdims = FALSE;
 
                                     /* unset unlimited max dims */
                                     if (limit_maxdims)
                                         H5Sset_extent_simple(f_space_id, rank, dims, NULL);
-                                }
+                                } /* end if not chunked */
+                            } /* end if layout change requested for entire file or individual object */
 
                             /*-------------------------------------------------------------------------
                              * create the output dataset;
@@ -876,16 +904,39 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                              * modified dcpl; in that case use the original instead
                              *-------------------------------------------------------------------------
                              */
-                            dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_out, H5P_DEFAULT);
-                            if (dset_out == FAIL) {
-                                H5Epush2(H5tools_ERR_STACK_g, __FILE__, FUNC, __LINE__, H5tools_ERR_CLS_g, H5E_tools_g, H5E_tools_min_id_g, "H5Dcreate2 failed");
-                                if (options->verbose)
-                                    HDprintf(" warning: could not create dataset <%s>. Applying original settings\n", travt->objs[i].name);
-
-                                if ((dset_out = H5Dcreate2(fidout, travt->objs[i].name, wtype_id, f_space_id, H5P_DEFAULT, dcpl_in, H5P_DEFAULT)) < 0)
+                            dset_out = H5Dcreate2(fidout,
+                                    travt->objs[i].name,
+                                    wtype_id,
+                                    f_space_id,
+                                    H5P_DEFAULT,
+                                    dcpl_out,
+                                    H5P_DEFAULT);
+                            /* if unable to create, retry with original DCPL */
+                            if (dset_out < 0) {
+                                H5Epush2(H5tools_ERR_STACK_g,
+                                        __FILE__,
+                                        FUNC,
+                                        __LINE__,
+                                        H5tools_ERR_CLS_g,
+                                        H5E_tools_g,
+                                        H5E_tools_min_id_g,
+                                        "H5Dcreate2 failed");
+                                if (options->verbose) {
+                                    HDprintf(" warning: could not create dataset <%s>. Applying original settings\n",
+                                            travt->objs[i].name);
+                                }
+                                dset_out = H5Dcreate2(
+                                        fidout,
+                                        travt->objs[i].name,
+                                        wtype_id,
+                                        f_space_id,
+                                        H5P_DEFAULT,
+                                        dcpl_in,
+                                        H5P_DEFAULT);
+                                if (dset_out < 0)
                                     HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dcreate2 failed");
                                 apply_f = 0;
-                            }
+                            } /* end if retry dataset create */
 
                             /*-------------------------------------------------------------------------
                              * read/write
@@ -909,8 +960,8 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                                     if (TRUE == H5Tdetect_class(wtype_id, H5T_VLEN))
                                         if (H5Dvlen_reclaim(wtype_id, f_space_id, H5P_DEFAULT, buf) < 0)
                                             HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dvlen_reclaim failed");
-                                    /* free */
-                                    if (buf != NULL) {
+
+                                    if (buf != NULL) { /* TODO: is buf potentially released by H5Dvlen_reclaim()? */
                                         HDfree(buf);
                                         buf = NULL;
                                     }
@@ -946,17 +997,25 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                                     dset_layout = H5Pget_layout(dcpl_out);
                                     if (dset_layout == H5D_CHUNKED)
                                         dcpl_tmp = dcpl_out; /* writing dataset */
-                                    else { /* if reading dataset is chunked */
+                                    else {
                                         dset_layout = H5Pget_layout(dcpl_in);
                                         if (dset_layout == H5D_CHUNKED)
                                             dcpl_tmp = dcpl_in; /* reading dataset */
                                     }
 
-                                    /* get hyperslab dims and size in byte */
-                                    if (Get_hyperslab(dcpl_tmp, rank, dims, p_type_nbytes, hslab_dims, &hslab_nbytes) < 0)
+                                    if (Get_hyperslab(
+                                            dcpl_tmp,
+                                            rank,
+                                            dims,
+                                            p_type_nbytes,
+                                            hslab_dims,
+                                            &hslab_nbytes)
+                                        < 0)
                                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "Get_hyperslab failed");
 
                                     hslab_buf = HDmalloc((size_t)hslab_nbytes);
+                                    if (hslab_buf == NULL)
+                                        HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "can't allocate space for hyperslab");
 
                                     hslab_nelmts = hslab_nbytes / p_type_nbytes;
                                     hslab_space = H5Screate_simple(1, &hslab_nelmts, NULL);
@@ -974,28 +1033,57 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                                              */
                                             for (k = 0, hs_select_nelmts = 1; k < rank; k++) {
                                                 /* MIN() is used to get the remaining edge portion if exist.
-                                                 * "dims[k] - hs_sel_offset[k]" is remaining edge portion that is smaller then the hyperslab.*/
+                                                 * "dims[k] - hs_sel_offset[k]" is remaining edge portion that is
+                                                 * smaller then the hyperslab.
+                                                 */
                                                 hs_sel_count[k] = MIN(dims[k] - hs_sel_offset[k], hslab_dims[k]);
                                                 hs_select_nelmts *= hs_sel_count[k];
                                             }
 
-                                            if (H5Sselect_hyperslab(f_space_id, H5S_SELECT_SET, hs_sel_offset, NULL, hs_sel_count, NULL) < 0)
+                                            if (H5Sselect_hyperslab(
+                                                    f_space_id,
+                                                    H5S_SELECT_SET,
+                                                    hs_sel_offset,
+                                                    NULL,
+                                                    hs_sel_count,
+                                                    NULL)
+                                                 < 0)
                                                 HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sselect_hyperslab failed");
-                                            if (H5Sselect_hyperslab(hslab_space, H5S_SELECT_SET, zero, NULL, &hs_select_nelmts, NULL) < 0)
+                                            if (H5Sselect_hyperslab(
+                                                    hslab_space,
+                                                    H5S_SELECT_SET,
+                                                    zero,
+                                                    NULL,
+                                                    &hs_select_nelmts,
+                                                    NULL)
+                                                < 0)
                                                 HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Sselect_hyperslab failed");
-                                        }
+                                        } /* end if rank > 0 */
                                         else {
                                             H5Sselect_all(f_space_id);
                                             H5Sselect_all(hslab_space);
                                             hs_select_nelmts = 1;
-                                        } /* rank */
-
-                                        if(H5Dread(dset_in, wtype_id, hslab_space, f_space_id, H5P_DEFAULT, hslab_buf) < 0)
+                                        } /* end (else) rank  == 0 */
+
+                                        if (H5Dread(
+                                                dset_in,
+                                                wtype_id,
+                                                hslab_space,
+                                                f_space_id,
+                                                H5P_DEFAULT,
+                                                hslab_buf)
+                                            < 0)
                                             HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dread failed");
-                                        if(H5Dwrite(dset_out, wtype_id,  hslab_space, f_space_id, H5P_DEFAULT, hslab_buf) < 0)
+                                        if (H5Dwrite(
+                                                dset_out,
+                                                wtype_id,
+                                                hslab_space,
+                                                f_space_id,
+                                                H5P_DEFAULT,
+                                                hslab_buf)
+                                            < 0)
                                             HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dwrite failed");
 
-                                        /* reclaim any VL memory, if necessary */
                                         if (vl_data)
                                             H5Dvlen_reclaim(wtype_id, hslab_space, H5P_DEFAULT, hslab_buf);
 
@@ -1007,20 +1095,19 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                                                 hs_sel_offset[k - 1] = 0;
                                             else
                                                 carry = 0;
-                                        } /* k */
-                                    } /* elmtno */
+                                        }
+                                    } /* end for (hyperslab selection loop) */
 
                                     H5Sclose(hslab_space);
-                                    /* free */
                                     if (hslab_buf != NULL) {
                                         HDfree(hslab_buf);
                                         hslab_buf = NULL;
                                     }
-                                } /* hyperslab read */
-                            } /* if (nelmts>0 && space_status==H5D_SPACE_STATUS_NOT_ALLOCATED) */
+                                } /* end if reading/writing by hyperslab */
+                            } /* end if (nelmts > 0 && space_status != H5D_SPACE_STATUS_NOT_ALLOCATED) */
 
                             /*-------------------------------------------------------------------------
-                             * amount of compression used
+                             * print amount of compression used
                              *-------------------------------------------------------------------------
                              */
                             if (options->verbose) {
@@ -1040,14 +1127,17 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                                     print_dataset_info(dcpl_in, travt->objs[i].name, ratio, 0);
 
                                 /* print a message that the filter was not applied
-                                 (in case there was a filter)
+                                 * (in case there was a filter)
                                  */
-                                if (has_filter && apply_s == 0)
-                                    HDprintf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n", travt->objs[i].name, (int) options->min_comp);
+                                if (has_filter && apply_s == 0) {
+                                    HDprintf(" <warning: filter not applied to %s. dataset smaller than %d bytes>\n",
+                                            travt->objs[i].name,
+                                            (int) options->min_comp);
+                                }
 
                                 if (has_filter && apply_f == 0)
                                     HDprintf(" <warning: could not apply the filter to %s>\n", travt->objs[i].name);
-                            } /* verbose */
+                            } /* end if verbose (print compression) */
 
                             /*-------------------------------------------------------------------------
                              * copy attrs
@@ -1056,14 +1146,13 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                             if (copy_attr(dset_in, dset_out, &named_dt_head, travt, options) < 0)
                                 HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "copy_attr failed");
 
-                            /*close */
                             if (H5Dclose(dset_out) < 0)
                                 HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
-                        }/*!H5T_REFERENCE*/
-                    }/*h5tools_canreadf*/
+                        } /* end if not a reference */
+                    } /* end if h5tools_canreadf (filter availability check) */
 
                     /*-------------------------------------------------------------------------
-                     * close
+                     * Close
                      *-------------------------------------------------------------------------
                      */
                     if (H5Tclose(ftype_id) < 0)
@@ -1080,7 +1169,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Dclose failed");
                 }
                 /*-------------------------------------------------------------------------
-                 * we do not have request for filter/chunking use H5Ocopy instead
+                 * We do not have request for filter/chunking; use H5Ocopy instead
                  *-------------------------------------------------------------------------
                  */
                 else {
@@ -1094,11 +1183,6 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                     if (H5Pset_copy_object(pid, H5O_COPY_WITHOUT_ATTR_FLAG) < 0)
                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pset_copy_object failed");
 
-                    /*-------------------------------------------------------------------------
-                     * do the copy
-                     *-------------------------------------------------------------------------
-                     */
-
                     if (H5Ocopy(fidin, /* Source file or group identifier */
                             travt->objs[i].name, /* Name of the source object to be copied */
                             fidout, /* Destination file or group identifier  */
@@ -1107,12 +1191,11 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                             H5P_DEFAULT) < 0) /* Properties which apply to the new hard link */
                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Ocopy failed");
 
-                    /* close property */
                     if (H5Pclose(pid) < 0)
                         HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "H5Pclose failed");
 
                     /*-------------------------------------------------------------------------
-                     * copy attrs manually
+                     * Copy attrs manually
                      *-------------------------------------------------------------------------
                      */
                     if ((dset_in = H5Dopen2(fidin, travt->objs[i].name, H5P_DEFAULT)) < 0)
@@ -1129,7 +1212,7 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
                     if (options->verbose)
                         HDprintf(FORMAT_OBJ, "dset", travt->objs[i].name);
 
-                } /* end do we have request for filter/chunking */
+                } /* end whether we have request for filter/chunking */
                 break;
 
             /*-------------------------------------------------------------------------
@@ -1189,8 +1272,8 @@ int do_copy_objects(hid_t fidin, hid_t fidout, trav_table_t *travt,
             default:
                 HGOTO_ERROR(FAIL, H5E_tools_min_id_g, "Object type not found");
             } /* switch */
-        } /* end for */
-    } /* end if */
+        } /* end for each object to traverse */
+    } /* end if there are objects */
 
 done:
 
@@ -1228,7 +1311,7 @@ done:
         HDfree(hslab_buf);
 
     return ret_value;
-}
+} /* end do_copy_objects() */
 
 /*-------------------------------------------------------------------------
  * Function: print_dataset_info
@@ -1322,8 +1405,8 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr)
             default:
                 HDstrcat(strfilter, "UD ");
                 break;
-        } /* switch */
-    }/*i*/
+        } /* end switch */
+    } /* end for each filter */
 
     if (!pr)
         HDprintf(FORMAT_OBJ, "dset", objname);
@@ -1336,7 +1419,7 @@ print_dataset_info(hid_t dcpl_id, char *objname, double ratio, int pr)
         HDstrcat(str, temp);
         HDprintf(FORMAT_OBJ, str, objname);
     }
-}
+} /* end print_dataset_info() */
 
 /*-------------------------------------------------------------------------
  * Function: copy_user_block
@@ -1406,7 +1489,7 @@ done:
         HDclose(outfid);
 
     return ret_value;
-}
+} /* end copy_user_block() */
 
 /*-------------------------------------------------------------------------
  * Function: print_user_block
@@ -1479,6 +1562,6 @@ done:
         HDclose(fh);
 
     return;
-}
+} /* end print_user_block() */
 #endif