From cf6f2f039d8e1c98bdd9690357e2b4670f23ae4e Mon Sep 17 00:00:00 2001
From: "M. Scot Breitenfeld" <brtnfld@hdfgroup.org>
Date: Thu, 9 May 2019 16:45:22 -0500
Subject: Adding cross-compilation for theta (ANL) and qsub (COBALT) batch
controls.
---
MANIFEST | 2 +
bin/batch/ctest.qsub.in.cmake | 21 +++++++++++
config/cmake/scripts/CTestScript.cmake | 12 +++++-
config/cmake/scripts/HDF5config.cmake | 2 +-
config/cmake/scripts/HPC/qsub-HDF5options.cmake | 49 +++++++++++++++++++++++++
5 files changed, 83 insertions(+), 3 deletions(-)
create mode 100755 bin/batch/ctest.qsub.in.cmake
create mode 100644 config/cmake/scripts/HPC/qsub-HDF5options.cmake
diff --git a/MANIFEST b/MANIFEST
index 41edffb..5021503 100644
--- a/MANIFEST
+++ b/MANIFEST
@@ -103,6 +103,7 @@
./bin/batch/knl_ctestP.sl.in.cmake
./bin/batch/knl_ctestS.sl.in.cmake
./bin/batch/knl_H5detect.sl.in.cmake
+./bin/batch/ctest.qsub.in.cmake
./bin/batch/ray_ctestP.lsf.in.cmake
./bin/batch/ray_ctestS.lsf.in.cmake
./bin/batch/raybsub
@@ -3407,6 +3408,7 @@
# CMake-specific HPC Scripts
./config/cmake/scripts/HPC/sbatch-HDF5options.cmake
./config/cmake/scripts/HPC/bsub-HDF5options.cmake
+./config/cmake/scripts/HPC/qsub-HDF5options.cmake
./config/cmake/scripts/HPC/raybsub-HDF5options.cmake
# Files generated by autogen
diff --git a/bin/batch/ctest.qsub.in.cmake b/bin/batch/ctest.qsub.in.cmake
new file mode 100755
index 0000000..f1d9b00
--- /dev/null
+++ b/bin/batch/ctest.qsub.in.cmake
@@ -0,0 +1,21 @@
+#!/bin/bash -l
+if [ $# -gt 0 ]; then
+ SUMMARY_FILE=$1
+fi
+ACCOUNT_ID=@ACCOUNT_ID@
+
+echo "Run parallel test command. Test output will be in build/${SUMMARY_FILE}"
+CTEST_CMD=`which ctest`
+
+#SKIPTESTS <<KEYWORD:script inserts list of skips tests here -- don't remove>>
+
+cd @HDF5_BINARY_DIR@
+if [[ $SUMMARY_FILE == *"ctestS"* ]]; then
+ CMD="${CTEST_CMD} . -E MPI_TEST_ -C Release -j 32 -T test"
+else
+ CMD="${CTEST_CMD} . -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test "
+fi
+
+qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE}
+
+echo "Done running ctest parallel command."
diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake
index e234057..e1fe85a 100644
--- a/config/cmake/scripts/CTestScript.cmake
+++ b/config/cmake/scripts/CTestScript.cmake
@@ -280,7 +280,11 @@ message (STATUS "Dashboard script configuration:\n${vars}\n")
if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
else ()
- execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
+ if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub")
+ execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestS.out)
+ else ()
+ execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME})
+ endif()
endif ()
message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
@@ -299,7 +303,11 @@ message (STATUS "Dashboard script configuration:\n${vars}\n")
if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub")
execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
else ()
- execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
+ if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub")
+ execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestP.out)
+ else ()
+ execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME})
+ endif ()
endif ()
message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml")
execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET)
diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake
index 0d85b42..3ae48ce 100644
--- a/config/cmake/scripts/HDF5config.cmake
+++ b/config/cmake/scripts/HDF5config.cmake
@@ -44,7 +44,7 @@ set (CTEST_SOURCE_VERSEXT "")
#CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo
#CTEST_SOURCE_NAME - name of source folder; HDF5-1.10.0
#MODEL - CDash group name
-#HPC - run alternate configurations for HPC machines; sbatch, bsub, raybsub
+#HPC - run alternate configurations for HPC machines; sbatch, bsub, raybsub, qsub
#MPI - enable MPI;
if (DEFINED CTEST_SCRIPT_ARG)
# transform ctest script arguments of the form
diff --git a/config/cmake/scripts/HPC/qsub-HDF5options.cmake b/config/cmake/scripts/HPC/qsub-HDF5options.cmake
new file mode 100644
index 0000000..d131e4f
--- /dev/null
+++ b/config/cmake/scripts/HPC/qsub-HDF5options.cmake
@@ -0,0 +1,49 @@
+#
+# Copyright by The HDF Group.
+# All rights reserved.
+#
+# This file is part of HDF5. The full HDF5 copyright notice, including
+# terms governing use, modification, and redistribution, is contained in
+# the COPYING file, which can be found at the root of the source code
+# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases.
+# If you do not have access to either file, you may request a copy from
+# help@hdfgroup.org.
+#
+#############################################################################################
+#### Change default configuration of options in config/cmake/cacheinit.cmake file ###
+#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ###
+#############################################################################################
+
+### uncomment/comment and change the following lines for other configuration options
+
+#############################################################################################
+### enable parallel builds
+if (DEFINED MPI)
+ # maximum parallel processor count for build and test ####
+ set (MAX_PROC_COUNT 8)
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PARALLEL:BOOL=ON")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_CPP_LIB:BOOL=OFF")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_JAVA:BOOL=OFF")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_THREADSAFE:BOOL=OFF")
+endif ()
+#############################################################################################
+### options to run test scripts in batch commands
+set (LOCAL_BATCH_SCRIPT_NAME "ctest.qsub")
+if (DEFINED KNL)
+ ### some additions and alternatives to cross compile on haswell for knl
+ set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl")
+ set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake")
+endif ()
+set (LOCAL_BATCH_SCRIPT_COMMAND "qsub")
+set (LOCAL_BATCH_TEST "TRUE")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=aprun")
+# Option to suppress writing job statistics; to avoid issues with h5diff comparisons.
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_PREFLAGS:STRING=-q")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-n")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6")
+set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DACCOUNT_ID:STRING=${LOCAL_BATCH_SCRIPT_ARGS}")
+
+#############################################################################################
+#############################################################################################
--
cgit v0.12