Merge pull request #1334 in HDFFV/hdf5 from ~LRKNOX/hdf5_lrk:develop to develop

* commit '7bf6e98bd921c4ef1899144ab8a1f0d66a141141':
  Move wait_H5init.cmake up to config/cmake.
  Changes to CMake code to pause cmake/ctest, waiting for batch jobs to run and finish.
  Really don't run parallel tests without option specifying script.
  Add script and changes to wait for H5Tinit.c Add more sample batch scripts, specifically for sbatch, not for knl cross compile. Don't run parallel tests when no parallel test script is configured in HDF5options.cmake.

9 files changed, 170 insertions, 59 deletions

diff --git a/MANIFEST b/MANIFEST
index 82b17a5..95d8b39 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -92,8 +92,12 @@
 ./bin/timekeeper      _DO_NOT_DISTRIBUTE_
 ./bin/trace
 ./bin/yodconfigure
+./bin/batch/knl_ctestP.sl
+./bin/batch/knl_ctestS.sl
 ./bin/batch/knl_ctestSP.sl
 ./bin/batch/knl_H5detect.sl.in.cmake
+./bin/batch/sbatch_ctestS.sl
+./bin/batch/sbatch_ctestP.sl
 ./bin/batch/sbatch_ctestSP.sl
 ./bin/pkgscrpts/h5rmflags                  _DO_NOT_DISTRIBUTE_
 ./bin/pkgscrpts/makeHDF5BinaryTarfiles.pl  _DO_NOT_DISTRIBUTE_
@@ -3192,6 +3196,7 @@
 ./config/cmake/UseJavaSymlinks.cmake
 ./config/cmake/userblockTest.cmake
 ./config/cmake/vfdTest.cmake
+./config/cmake/wait_H5Tinit.cmake
 
 ./config/cmake_ext_mod/ConfigureChecks.cmake
 ./config/cmake_ext_mod/CTestCustom.cmake
diff --git a/bin/batch/knl_ctestP.sl b/bin/batch/knl_ctestP.sl
new file mode 100644
index 0000000..3a03fe2
--- /dev/null
+++ b/bin/batch/knl_ctestP.sl
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+#SBATCH -p knl -C quad,flat
+#SBATCH --nodes=1
+#SBATCH -t 00:30:00
+#SBATCH --mail-type=BEGIN,END,FAIL
+##SBATCH --mail-user=<username>@sandia.gov
+#SBATCH --export=ALL
+#SBATCH --job-name=h5_ctestS
+
+cd build
+#run parallel tests except t_cache_image test
+CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM -E t_cache_image -C Release -T test"
+
+echo "Run $CMD. Test output will be in build/ctestP.out"
+$CMD >& ctestP.out
+echo "Done running $CMD"
+
diff --git a/bin/batch/knl_ctestS.sl b/bin/batch/knl_ctestS.sl
new file mode 100644
index 0000000..5b77c5b
--- /dev/null
+++ b/bin/batch/knl_ctestS.sl
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#SBATCH -p knl -C quad,flat
+#SBATCH --nodes=1
+#SBATCH -t 00:30:00
+#SBATCH --mail-type=BEGIN,END,FAIL
+##SBATCH --mail-user=<username>@sandia.gov
+#SBATCH --export=ALL
+#SBATCH --job-name=h5_ctestS
+
+cd build
+CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM -C Release -j 32 -T test"
+
+echo "Run $CMD. Test output will be in build/ctestS.out"
+$CMD  >& ctestS.out
+echo "Done running $CMD"
+
diff --git a/bin/batch/sbatch_ctestP.sl b/bin/batch/sbatch_ctestP.sl
new file mode 100644
index 0000000..e967dce
--- /dev/null
+++ b/bin/batch/sbatch_ctestP.sl
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#SBATCH --nodes=1
+#SBATCH -t 00:30:00
+#SBATCH --mail-type=BEGIN,END,FAIL
+##SBATCH --mail-user=<username>@sandia.gov
+#SBATCH --export=ALL
+#SBATCH --job-name=h5_ctestS
+
+
+cd build
+CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM -E t_cache_image -C Release -T test"
+
+echo "Run $CMD. Test output will be in build/ctestP.out"
+$CMD >& ctestP.out
+echo "Done running $CMD"
+
diff --git a/bin/batch/sbatch_ctestS.sl b/bin/batch/sbatch_ctestS.sl
new file mode 100644
index 0000000..a103356
--- /dev/null
+++ b/bin/batch/sbatch_ctestS.sl
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#SBATCH --nodes=1
+#SBATCH -t 00:30:00
+#SBATCH --mail-type=BEGIN,END,FAIL
+##SBATCH --mail-user=<username>@sandia.gov
+#SBATCH --export=ALL
+#SBATCH --job-name=h5_ctestS
+
+
+cd build
+CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM -C Release -j 32 -T test"
+
+echo "Run $CMD. Test output will be in build/ctestS.out"
+$CMD  >& ctestS.out
+echo "Done running $CMD"
diff --git a/bin/batch/sbatch_ctestSP.sl b/bin/batch/sbatch_ctestSP.sl
index 56da347..5d79d9b 100644
--- a/bin/batch/sbatch_ctestSP.sl
+++ b/bin/batch/sbatch_ctestSP.sl
@@ -1,6 +1,5 @@
 #!/bin/bash
 
-#SBATCH -p knl -C quad,flat
 #SBATCH --nodes=1
 #SBATCH -t 01:00:00
 #SBATCH --mail-type=BEGIN,END,FAIL
@@ -8,8 +7,6 @@
 #SBATCH --export=ALL
 #SBATCH --job-name=h5_ctestS
 
-module load cmake
-module load craype-hugepages4M
 
 cd build
 CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM -C Release -j 32 -T test"
diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake
index a3f2229..9534b1e 100755
--- a/config/cmake/scripts/CTestScript.cmake
+++ b/config/cmake/scripts/CTestScript.cmake
@@ -267,41 +267,64 @@ message (STATUS "Dashboard script configuration:\n${vars}\n")
       if (NOT LOCAL_BATCH_TEST)
         ctest_test (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args} RETURN_VALUE res)
       else ()
-        execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_NAME})
+        file(STRINGS ${CTEST_BINARY_DIRECTORY}/Testing/TAG TAG_CONTENTS REGEX "^2([0-9]+)[-]([0-9]+)$")
+        execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${CTEST_SOURCE_DIRECTORY}/bin/batch/${LOCAL_BATCH_SCRIPT_NAME})
+        message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
+        execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+        while(result)
+          ctest_sleep(60)
+          execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+        endwhile(result)
+        if (LOCAL_BATCH_SCRIPT_PARALLEL_NAME)
+          if (LOCAL_SUBMIT)
+            ctest_submit (PARTS Test)
+          endif ()
+          message(STATUS "Found ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml for serial tests. Renaming to SerialTest.xml")
+          file (RENAME ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/SerialTest.xml)
+          unset(result CACHE)
+          execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${CTEST_SOURCE_DIRECTORY}/bin/batch/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
+          message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
+          execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+          while(result)
+            ctest_sleep(60)
+            execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
+          endwhile(result)
+          message(STATUS "Found ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml for parallel tests.")
+        endif()
       endif ()
-      if (LOCAL_SUBMIT)

-        ctest_submit (PARTS Test)

-      endif ()

-      if (${res} LESS 0 OR ${res} GREATER 0)

-        file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n")

-      endif ()

-    else ()

-      ctest_memcheck (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args})

-      if (LOCAL_SUBMIT)

-        ctest_submit (PARTS MemCheck)

-      endif ()

-    endif ()

-    if (LOCAL_COVERAGE_TEST)

-      ctest_coverage (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND)

-      if (LOCAL_SUBMIT)

-        ctest_submit (PARTS Coverage)

-      endif ()

-    endif ()

-  endif ()

-

-  if (NOT LOCAL_MEMCHECK_TEST AND NOT LOCAL_NO_PACKAGE AND NOT LOCAL_SKIP_BUILD)

-    ##-----------------------------------------------

-    ## Package the product

-    ##-----------------------------------------------

-    execute_process (COMMAND cpack -C ${CTEST_CONFIGURATION_TYPE} -V

-      WORKING_DIRECTORY ${CTEST_BINARY_DIRECTORY}

-      RESULT_VARIABLE cpackResult

-      OUTPUT_VARIABLE cpackLog

-      ERROR_VARIABLE cpackLog.err

-    )

-    file (WRITE ${CTEST_BINARY_DIRECTORY}/cpack.log "${cpackLog.err}" "${cpackLog}")

-    if (cpackResult GREATER 0)

-      file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed packaging: ${cpackResult}:${cpackLog.err} \n")

-    endif ()

-  endif ()

-#-----------------------------------------------------------------------------

+      if (LOCAL_SUBMIT)
+        ctest_submit (PARTS Test)
+      endif ()
+      if (${res} LESS 0 OR ${res} GREATER 0)
+        file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed Tests: ${res}\n")
+      endif ()
+    else ()
+      ctest_memcheck (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND ${ctest_test_args})
+      if (LOCAL_SUBMIT)
+        ctest_submit (PARTS MemCheck)
+      endif ()
+    endif ()
+    if (LOCAL_COVERAGE_TEST)
+      ctest_coverage (BUILD "${CTEST_BINARY_DIRECTORY}" APPEND)
+      if (LOCAL_SUBMIT)
+        ctest_submit (PARTS Coverage)
+      endif ()
+    endif ()
+  endif ()
+
+  if (NOT LOCAL_MEMCHECK_TEST AND NOT LOCAL_NO_PACKAGE AND NOT LOCAL_SKIP_BUILD)
+    ##-----------------------------------------------
+    ## Package the product
+    ##-----------------------------------------------
+    execute_process (COMMAND cpack -C ${CTEST_CONFIGURATION_TYPE} -V
+      WORKING_DIRECTORY ${CTEST_BINARY_DIRECTORY}
+      RESULT_VARIABLE cpackResult
+      OUTPUT_VARIABLE cpackLog
+      ERROR_VARIABLE cpackLog.err
+    )
+    file (WRITE ${CTEST_BINARY_DIRECTORY}/cpack.log "${cpackLog.err}" "${cpackLog}")
+    if (cpackResult GREATER 0)
+      file (APPEND ${CTEST_SCRIPT_DIRECTORY}/FailedCTest.txt "Failed packaging: ${cpackResult}:${cpackLog.err} \n")
+    endif ()
+  endif ()
+#-----------------------------------------------------------------------------
diff --git a/config/cmake/wait_H5Tinit.cmake b/config/cmake/wait_H5Tinit.cmake
new file mode 100644
index 0000000..c0b4d06
--- /dev/null
+++ b/config/cmake/wait_H5Tinit.cmake
@@ -0,0 +1,11 @@
+cmake_minimum_required (VERSION 3.10)
+
+message(STATUS "Check for existence of ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c")
+execute_process(COMMAND ls ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c RESULT_VARIABLE H5TI_result OUTPUT_QUIET ERROR_QUIET)
+while(H5TI_result)
+  ctest_sleep(30)
+  message(STATUS "Checking again for existence of ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c")
+  execute_process(COMMAND ls ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c RESULT_VARIABLE H5TI_result OUTPUT_QUIET ERROR_QUIET)
+endwhile(H5TI_result)
+file (TOUCH "${HDF5_BINARY_DIR}/H5Tinit_created")
+message(STATUS "Found ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c")
diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt
index 31a38be..bca5a92 100644
--- a/src/CMakeLists.txt
+++ b/src/CMakeLists.txt
@@ -947,39 +947,46 @@ if (NOT EXISTS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c)
         ${HDF5_BINARY_DIR}/${HDF5_BATCH_H5DETECT_SCRIPT} ESCAPE_QUOTES @ONLY
     )
     add_custom_command (
-        OUTPUT ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.tmp
+        OUTPUT ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
         COMMAND ${HDF5_BATCH_CMD}
         ARGS ${HDF5_BINARY_DIR}/${HDF5_BATCH_H5DETECT_SCRIPT}
+        COMMAND ${CMAKE_COMMAND}
+        ARGS -E echo "Executed batch command to create H5Tinit.c"
         DEPENDS H5detect
         WORKING_DIRECTORY ${HDF5_GENERATED_SOURCE_DIR}
     )
+    add_custom_target (gen_H5Tinit
+        COMMAND ${CMAKE_COMMAND} -P ${HDF5_SOURCE_DIR}/config/cmake/wait_H5Tinit.cmake
+    )
+    if (BUILD_SHARED_LIBS)
+      add_custom_command (
+          OUTPUT     ${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c
+          COMMAND    ${CMAKE_COMMAND}
+          ARGS       -E copy_if_different "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c" "${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c"
+          DEPENDS    gen_H5Tinit ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit_created
+      )
+      set_source_files_properties (${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c PROPERTIES GENERATED TRUE)
+    endif ()
   else ()
     add_custom_command (
-        OUTPUT ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.tmp
+        OUTPUT ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
         COMMAND ${CMAKE_CROSSCOMPILING_EMULATOR}$<TARGET_FILE:H5detect>
-        ARGS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.tmp
+        ARGS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
         DEPENDS H5detect
         WORKING_DIRECTORY ${HDF5_GENERATED_SOURCE_DIR}
     )
+    if (BUILD_SHARED_LIBS)
+      add_custom_command (
+          OUTPUT     ${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c
+          COMMAND    ${CMAKE_COMMAND}
+          ARGS       -E copy_if_different "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c" "${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c"
+          DEPENDS    ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
+      )
+      set_source_files_properties (${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c PROPERTIES GENERATED TRUE)
+    endif ()
   endif ()
-  add_custom_target (gen_H5Tinit ALL DEPENDS ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.tmp)
-  add_custom_command (
-      OUTPUT     ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
-      COMMAND    ${CMAKE_COMMAND}
-      ARGS       -E copy_if_different "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.tmp" "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c"
-      DEPENDS    gen_H5Tinit ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.tmp
-  )
   set_source_files_properties (${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c PROPERTIES GENERATED TRUE)
 endif ()
-if (BUILD_SHARED_LIBS)
-  add_custom_command (
-      OUTPUT     ${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c
-      COMMAND    ${CMAKE_COMMAND}
-      ARGS       -E copy_if_different "${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c" "${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c"
-      DEPENDS    ${HDF5_GENERATED_SOURCE_DIR}/H5Tinit.c
-  )
-  set_source_files_properties (${HDF5_GENERATED_SOURCE_DIR}/shared/H5Tinit.c PROPERTIES GENERATED TRUE)
-endif ()
 
 add_executable (H5make_libsettings ${HDF5_SRC_DIR}/H5make_libsettings.c)
 target_include_directories(H5make_libsettings PRIVATE "${HDF5_SRC_DIR};${HDF5_BINARY_DIR};$<$<BOOL:${HDF5_ENABLE_PARALLEL}>:${MPI_C_INCLUDE_DIRS}>")