Add support for parallel tools based on the 3rd party library mpiFileUtils (libMFU) … (#1177)

Adds tool h5dwalk and configure options to enable building it.

Co-authored-by: Richard Warren <Richard.Warren@hdfgroup.org>
Co-authored-by: github-actions <41898282+github-actions[bot]@users.noreply.github.com>
Co-authored-by: Richard.Warren <richard.warren@jelly.ad.hdfgroup.org>
Co-authored-by: Larry Knox <lrknox@hdfgroup.org>

8 files changed, 502 insertions, 0 deletions

diff --git a/utils/tools/test/CMakeLists.txt b/utils/tools/test/CMakeLists.txt
new file mode 100644
index 0000000..0f5335d
--- /dev/null
+++ b/utils/tools/test/CMakeLists.txt
@@ -0,0 +1,8 @@
+cmake_minimum_required (VERSION 3.12)
+project (HDF5_TOOLS_TEST C)
+
+#-- Add the h5diff tests
+if (HDF5_BUILD_PARALLEL_TOOLS)
+   add_subdirectory (h5dwalk)
+endif()
+
diff --git a/utils/tools/test/Makefile.am b/utils/tools/test/Makefile.am
new file mode 100644
index 0000000..88104f2
--- /dev/null
+++ b/utils/tools/test/Makefile.am
@@ -0,0 +1,32 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5.  The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+##
+## Makefile.am
+## Run automake to generate a Makefile.in from this file.
+##
+#
+# Tools HDF5 Makefile(.in)
+#
+
+include $(top_srcdir)/config/commence.am
+
+if PARALLEL_TOOLS_CONDITIONAL
+  H5DWALK=h5dwalk
+else
+  H5DWALK=
+endif
+
+CONFIG=ordered
+
+# All subdirectories
+SUBDIRS=$(H5DWALK)
+
+include $(top_srcdir)/config/conclude.am
diff --git a/utils/tools/test/h5dwalk/CMakeLists.txt b/utils/tools/test/h5dwalk/CMakeLists.txt
new file mode 100644
index 0000000..5f6c992
--- /dev/null
+++ b/utils/tools/test/h5dwalk/CMakeLists.txt
@@ -0,0 +1,15 @@
+cmake_minimum_required (VERSION 3.12)
+project (HDF5_TOOLS_TEST_H5DWALK)
+
+if (HDF5_BUILD_PARALLEL_TOOLS)
+  add_custom_command(
+    OUTPUT ${HDF5_TOOLS_DIR}/test/demo_destfiles.test
+    COMMAND bash -c ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh
+    ARGS ${HDF5_TOOLS_DIR}/test ${CMAKE_BINARY_DIR}/bin
+    DEPENDS ${HDF5_TOOLS_SRC_H5DWALK_SOURCE_DIR}/copy_demo_files.sh
+  )
+endif ()
+
+if (HDF5_TEST_TOOLS AND HDF5_TEST_SERIAL)
+  include (CMakeTests.cmake)
+endif ()
diff --git a/utils/tools/test/h5dwalk/CMakeTests.cmake b/utils/tools/test/h5dwalk/CMakeTests.cmake
new file mode 100644
index 0000000..b9e52c5
--- /dev/null
+++ b/utils/tools/test/h5dwalk/CMakeTests.cmake
@@ -0,0 +1,56 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5.  The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+
+##############################################################################
+##############################################################################
+###           T E S T I N G                                                ###
+##############################################################################
+##############################################################################
+
+  file (MAKE_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles")
+
+
+##############################################################################
+##############################################################################
+###           T H E   T E S T S  M A C R O S                               ###
+##############################################################################
+##############################################################################
+
+  macro (ADD_H5_TEST resultfile resultcode)
+    # If using memchecker add tests without using scripts
+    if (HDF5_ENABLE_USING_MEMCHECKER)
+      message("Entered ADD_H5_TEST - 0")
+      add_test (NAME H5DWALK-${resultfile} COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR} $<TARGET_FILE:h5dwalk${tgt_file_ext}> ${ARGN})
+      set_tests_properties (H5DWALK-${resultfile} PROPERTIES
+		WORKING_DIRECTORY "${PROJECT_BINARY_DIR}/testfiles")
+      if ("${resultcode}" STREQUAL "1")
+        set_tests_properties (H5DWALK-${resultfile} PROPERTIES WILL_FAIL "true")
+      endif ()
+    else ()
+      # Remove any output file left over from previous test run
+      add_test (
+          NAME H5DWALK-${resultfile}
+          COMMAND "${CMAKE_COMMAND}"
+              -D "TEST_EMULATOR=${CMAKE_CROSSCOMPILING_EMULATOR}"
+              -D "TEST_PROGRAM=$<TARGET_FILE:h5dwalk${tgt_file_ext}>"
+              -D "TEST_ARGS=${ARGN}"
+              -D "TEST_FOLDER=${PROJECT_BINARY_DIR}/testfiles"
+              -D "TEST_OUTPUT=${resultfile}.out"
+              -D "TEST_EXPECT=${resultcode}"
+              -D "TEST_REFERENCE=${resultfile}.h5dwalk"
+              -D "TEST_LIBRARY_DIRECTORY=${LL_PATH}"
+              -P "${HDF_RESOURCES_EXT_DIR}/runTest.cmake"
+      )
+    endif ()
+  endmacro ()
+
+  ADD_H5_TEST(help-1 0 -h)
diff --git a/utils/tools/test/h5dwalk/Makefile.am b/utils/tools/test/h5dwalk/Makefile.am
new file mode 100644
index 0000000..c32dd0f
--- /dev/null
+++ b/utils/tools/test/h5dwalk/Makefile.am
@@ -0,0 +1,43 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5.  The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+##
+## Makefile.am
+## Run automake to generate a Makefile.in from this file.
+#
+# HDF5 Library Makefile(.in)
+#
+
+include $(top_srcdir)/config/commence.am
+
+# Include src directory
+AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib
+
+install-examples:
+	@echo "Creating demo files" && \
+	. copy_demo_files.sh $(top_srcdir)/src
+
+bin_SCRIPTS:install-examples
+
+#test script and program
+TEST_SCRIPT=testh5dwalk.sh copy_demo_files.sh
+check_SCRIPTS=$(TEST_SCRIPT)
+SCRIPT_DEPEND=../../h5dwalk/h5dwalk$(EXEEXT)
+
+# Tell automake to clean h5redeploy script
+CLEANFILES=
+
+# These were generated by configure.  Remove them only when distclean.
+DISTCLEANFILES=testh5dwalk.sh copy_demo_files.sh
+
+# All programs rely on hdf5 library and h5tools library
+LDADD=$(LIBH5TOOLS) $(LIBHDF5)
+
+include $(top_srcdir)/config/conclude.am
diff --git a/utils/tools/test/h5dwalk/copy_demo_files.sh.in b/utils/tools/test/h5dwalk/copy_demo_files.sh.in
new file mode 100644
index 0000000..f20bf43
--- /dev/null
+++ b/utils/tools/test/h5dwalk/copy_demo_files.sh.in
@@ -0,0 +1,86 @@
+#! /bin/sh
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+srcdir=@srcdir@
+TOP_BUILDDIR=..
+
+# Determine if backward compatibility options enabled
+DEPRECATED_SYMBOLS="yes"
+
+EXIT_SUCCESS=0
+EXIT_FAILURE=1
+
+CP='cp'
+
+THIS_DIR=`pwd`
+SRC_TOOLS_DIR=$srcdir/../../../../tools
+
+nerrors=0
+verbose=yes
+exit_code=$EXIT_SUCCESS
+
+
+# Add Testing files into the local testfiles directory::
+TESTDIR=./testfiles
+test -d $TESTDIR || mkdir $TESTDIR
+
+echo "HDF5 \"$THIS_DIR/testfiles/h5diff_basic1.h5\" {" > "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt
+echo "FILE_CONTENTS {
+ group      /
+ group      /g1
+ dataset    /g1/d1
+ dataset    /g1/d2
+ dataset    /g1/dset1
+ dataset    /g1/dset10
+ dataset    /g1/dset11
+ dataset    /g1/dset12
+ dataset    /g1/dset3
+ dataset    /g1/dset5
+ dataset    /g1/dset6
+ dataset    /g1/dset7
+ dataset    /g1/dset8
+ dataset    /g1/dset9
+ dataset    /g1/fp1
+ dataset    /g1/fp15
+ dataset    /g1/fp16
+ dataset    /g1/fp17
+ dataset    /g1/fp18
+ dataset    /g1/fp18_COPY
+ dataset    /g1/fp19
+ dataset    /g1/fp19_COPY
+ dataset    /g1/fp2
+ dataset    /g1/fp20
+ dataset    /g1/fp20_COPY
+ dataset    /g1/ld
+ }
+}" >> "$THIS_DIR"/testfiles/h5diff_basic1.h5_h5dump.txt
+
+# Create the help-1.txt output file for '-h' validation
+echo "
+Usage: h5dwalk [options] <path> ...
+
+Options:
+  -i, --input <file>      - read list from file
+  -o, --output <file>     - write output summary to the named file.
+  -E, --error  <file>     - write processed errors to file in text format
+  -l, --log_text <dir>    - write individual tool outputs to a file. Logs can be written to an optional named directory.
+  -T, --tool <executable> - name of the HDF5 tool to invoke
+  -h, --help              - print usage
+
+For more information see https://mpifileutils.readthedocs.io. 
+" > "$THIS_DIR"/testfiles/help-1.txt
+
+# Make a copy of the help-1.txt output file for --help validation
+$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-1.txt
+$CP "$srcdir"/help.h5dwalk "$THIS_DIR"/testfiles/help-2.txt
+# Make a copy of a simple HDF5 datafile which will be used as input for h5dump -n (see the expected output above)
+$CP "$SRC_TOOLS_DIR"/test/h5diff/testfiles/h5diff_basic1.h5 "$THIS_DIR"/testfiles
diff --git a/utils/tools/test/h5dwalk/help.h5dwalk b/utils/tools/test/h5dwalk/help.h5dwalk
new file mode 100644
index 0000000..986cbba
--- /dev/null
+++ b/utils/tools/test/h5dwalk/help.h5dwalk
@@ -0,0 +1,13 @@
+
+Usage: h5dwalk [options] <path> ...
+
+Options:
+  -i, --input <file>      - read list from file
+  -o, --output <file>     - write output summary to the named file.
+  -E, --error  <file>     - write processed errors to file in text format
+  -l, --log_text <dir>    - write individual tool outputs to a file. Logs can be written to an optional named directory.
+  -T, --tool <executable> - name of the HDF5 tool to invoke
+  -h, --help              - print usage
+
+For more information see https://mpifileutils.readthedocs.io. 
+
diff --git a/utils/tools/test/h5dwalk/testh5dwalk.sh.in b/utils/tools/test/h5dwalk/testh5dwalk.sh.in
new file mode 100644
index 0000000..4f6dbde
--- /dev/null
+++ b/utils/tools/test/h5dwalk/testh5dwalk.sh.in
@@ -0,0 +1,249 @@
+#! /bin/sh
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5.  The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://www.hdfgroup.org/licenses.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+
+# Tests for the h5dwalk tool
+
+#
+#
+
+srcdir=@srcdir@
+
+# Determine which filters are available
+USE_FILTER_SZIP="@USE_FILTER_SZIP@"
+USE_FILTER_DEFLATE="@USE_FILTER_DEFLATE@"
+
+
+TESTNAME=h5dwalk
+
+EXIT_SUCCESS=0
+EXIT_FAILURE=1
+
+THIS_DIR="`pwd`"
+ROOTDIR="`cd ../../../..; pwd`"
+cd "$THIS_DIR"
+TOP_DIR="$ROOTDIR"
+
+
+H5DWALK=../../h5dwalk/h5dwalk
+H5DWALK_BIN="$TOP_DIR/utils/tools/h5dwalk/h5dwalk"
+
+
+H5DUMP="$TOP_DIR/src/h5dump/h5dump"
+H5DUMP_BIN="$TOP_DIR/tools/src/h5dump/h5dump"
+
+RM='rm -rf'
+CMP='cmp -s'
+DIFF='diff -c'
+CP='cp'
+DIRNAME='dirname'
+LS='ls'
+AWK='awk'
+WC='wc'
+
+nerrors=0
+verbose=yes
+
+export LD_LIBRARY_PATH=@LL_PATH@
+
+# source dirs
+SRC_TOOLS="$TOP_DIR/tools/test"
+SRC_TOOLS_TESTFILES="$SRC_TOOLS/testfiles"
+
+# testfiles source dirs for tools
+SRC_H5LS_TESTFILES="$SRC_TOOLS_TESTFILES"
+SRC_H5DUMP_TESTFILES="$SRC_TOOLS_TESTFILES"
+SRC_H5DIFF_TESTFILES="$SRC_TOOLS/h5diff/testfiles"
+SRC_H5COPY_TESTFILES="$SRC_TOOLS/h5copy/testfiles"
+SRC_H5REPACK_TESTFILES="$SRC_TOOLS/h5repack/testfiles"
+SRC_H5JAM_TESTFILES="$SRC_TOOLS/h5jam/testfiles"
+SRC_H5DWALK_TESTFILES="$SRC_TOOLS/h5dwalk/testfiles"
+SRC_H5IMPORT_TESTFILES="$SRC_TOOLS/h5import/testfiles"
+
+TESTDIR=./testfiles
+test -d $TESTDIR || mkdir $TESTDIR
+
+echo "SRC_H5DIFF_TESTFILES = $SRC_H5DIFF_TESTFILES"
+echo "Creating demo files"
+. ./copy_demo_files.sh
+
+
+
+CLEAN_TESTFILES_AND_TESTDIR()
+{
+    echo "cleaning logfiles"
+    $RM $TESTDIR/*log*
+}
+
+# Print a line-line message left justified in a field of 70 characters
+# beginning with the word "Testing".
+#
+TESTING() {
+   SPACES="                                                               "
+   echo "Testing $* $SPACES" | cut -c1-70 | tr -d '\012'
+}
+
+# Run a test and print PASS or *FAIL*.  If a test fails then increment
+# the `nerrors' global variable and (if $verbose is set) display the
+# difference between the actual output and the expected output. The
+# expected output is given as the first argument to this function and
+# the actual output file is calculated by replacing the `.ddl' with
+# `.out'.  The actual output is not removed if $HDF5_NOCLEANUP has a
+# non-zero value.
+#
+TOOLTEST() {
+   expect="$TESTDIR/$1"
+   expect_err="$TESTDIR/`basename $1`.err"
+   actual="$TESTDIR/`basename $1`.out"
+   actual_err="$TESTDIR/`basename $1`.out.err"
+   actual_sav=${actual}-sav
+   actual_err_sav=${actual_err}-sav
+   shift
+
+   # Run test.
+
+   TESTING $H5DWALK $@
+   (
+      cd $TESTDIR
+      $RUNSERIAL $H5DWALK_BIN $@
+   ) 1> $actual 2> $actual_err
+
+   # save actual and actual_err in case they are needed later.
+   cp $actual $actual_sav
+   cp $actual_err $actual_err_sav
+
+   if [ ! -f $expect ]; then
+      # Compare error files if the expect file doesn't exist.
+      if $CMP $expect_err $actual_err; then
+         echo " PASSED"
+      else
+         echo "*FAILED*"
+         echo "    Expected result (*.err) differs from actual result (*.out.err)"
+         nerrors="`expr $nerrors + 1`"
+         test yes = "$verbose" && $DIFF $expect_err $actual_err |sed 's/^/    /'
+      fi
+   elif $CMP $expect $actual; then
+      echo " PASSED"
+   else
+      echo "*FAILED*"
+      echo "    Expected result (*.ddl) differs from actual result (*.out)"
+      nerrors="`expr $nerrors + 1`"
+      test yes = "$verbose" && $DIFF $expect $actual |sed 's/^/    /'
+   fi
+
+   # Clean up output file
+   if test -z "$HDF5_NOCLEANUP"; then
+      rm -f $actual $actual_err $actual_sav $actual_err_sav
+   fi
+}
+
+TOOL_LOGTEST() {
+   expect="$TESTDIR/`basename $1`.txt"
+   expect_err="$TESTDIR/`basename $1`.err"
+   actual="$TESTDIR/`basename $1`.log"
+   actual_err="$TESTDIR/`basename $1`.out.err"
+   actual_sav=${actual}-sav
+   actual_err_sav=${actual_err}-sav
+   shift
+
+   echo "running logtest"
+
+   # Run test.
+   TESTING $H5DWALK $@
+   (
+      cd $TESTDIR
+      $RUNSERIAL $H5DWALK_BIN $@
+
+   ) 1> $actual 2> $actual_err
+   expect_len="`wc -l < $expect`"
+
+   if [ ! -f $actual ]; then
+      echo "*FAILED*"
+      echo "    The expected .log file is missing"
+      echo "    Perhaps the test failed to run?"
+   else
+      actual_len="`wc -l < $actual`"
+      if [ $actual_len -eq $expect_len ]; then
+         echo " PASSED"
+      else
+         echo "*FAILED*"
+         echo "    The generated .log file length does not match the expected length.  $actual_len != $expected_len"
+      fi
+   fi
+
+   # Clean up output file
+   if test -z "$HDF5_NOCLEANUP"; then
+      rm -f $actual $actual_err $actual_sav $actual_err_sav
+   fi
+}
+
+TOOL_CHK_LOGLEN() {
+   expect=$1
+   shift
+
+   echo "running tool_chk_loglen"
+
+   # Run test.
+   TESTING $H5DWALK $@
+   (
+      cd $TESTDIR
+      $RUNSERIAL $H5DWALK_BIN $@
+   )
+	   
+   expect_len="`wc -l < $expect`"
+   if [ "$expect_len" -gt 0 ]; then
+	   echo " PASSED"
+   else
+       echo "*FAILED*"
+       echo "    The generated .log file is empty!."
+   fi
+
+   # Clean up output file
+   if test -z "$HDF5_NOCLEANUP"; then
+      rm -f $expect
+   fi
+}
+
+
+# Print a "SKIP" message
+SKIP() {
+    TESTING $H5DWALK $@6
+    echo  " -SKIP-"
+}
+
+
+
+##############################################################################
+##############################################################################
+###              T H E   T E S T S                                ###
+##############################################################################
+##############################################################################
+
+TOOLTEST help-1.txt -h
+TOOLTEST help-2.txt --help
+TOOL_LOGTEST h5diff_basic1.h5_h5dump -l -T $H5DUMP_BIN -n ./h5diff_basic1.h5
+TOOL_CHK_LOGLEN showme-h5dump.log -o `pwd`/showme-h5dump.log -T $H5DUMP_BIN -n `pwd`
+
+
+#
+#
+# Clean up temporary files/directories
+CLEAN_TESTFILES_AND_TESTDIR
+
+if test $nerrors -eq 0 ; then
+    echo "All $TESTNAME tests passed."
+    exit $EXIT_SUCCESS
+else
+    echo "$TESTNAME tests failed with $nerrors errors."
+    exit $EXIT_FAILURE
+fi
+