From cf6f2f039d8e1c98bdd9690357e2b4670f23ae4e Mon Sep 17 00:00:00 2001 From: "M. Scot Breitenfeld" Date: Thu, 9 May 2019 16:45:22 -0500 Subject: Adding cross-compilation for theta (ANL) and qsub (COBALT) batch controls. --- MANIFEST | 2 + bin/batch/ctest.qsub.in.cmake | 21 +++++++++++ config/cmake/scripts/CTestScript.cmake | 12 +++++- config/cmake/scripts/HDF5config.cmake | 2 +- config/cmake/scripts/HPC/qsub-HDF5options.cmake | 49 +++++++++++++++++++++++++ 5 files changed, 83 insertions(+), 3 deletions(-) create mode 100755 bin/batch/ctest.qsub.in.cmake create mode 100644 config/cmake/scripts/HPC/qsub-HDF5options.cmake diff --git a/MANIFEST b/MANIFEST index 41edffb..5021503 100644 --- a/MANIFEST +++ b/MANIFEST @@ -103,6 +103,7 @@ ./bin/batch/knl_ctestP.sl.in.cmake ./bin/batch/knl_ctestS.sl.in.cmake ./bin/batch/knl_H5detect.sl.in.cmake +./bin/batch/ctest.qsub.in.cmake ./bin/batch/ray_ctestP.lsf.in.cmake ./bin/batch/ray_ctestS.lsf.in.cmake ./bin/batch/raybsub @@ -3407,6 +3408,7 @@ # CMake-specific HPC Scripts ./config/cmake/scripts/HPC/sbatch-HDF5options.cmake ./config/cmake/scripts/HPC/bsub-HDF5options.cmake +./config/cmake/scripts/HPC/qsub-HDF5options.cmake ./config/cmake/scripts/HPC/raybsub-HDF5options.cmake # Files generated by autogen diff --git a/bin/batch/ctest.qsub.in.cmake b/bin/batch/ctest.qsub.in.cmake new file mode 100755 index 0000000..f1d9b00 --- /dev/null +++ b/bin/batch/ctest.qsub.in.cmake @@ -0,0 +1,21 @@ +#!/bin/bash -l +if [ $# -gt 0 ]; then + SUMMARY_FILE=$1 +fi +ACCOUNT_ID=@ACCOUNT_ID@ + +echo "Run parallel test command. Test output will be in build/${SUMMARY_FILE}" +CTEST_CMD=`which ctest` + +#SKIPTESTS <> + +cd @HDF5_BINARY_DIR@ +if [[ $SUMMARY_FILE == *"ctestS"* ]]; then + CMD="${CTEST_CMD} . -E MPI_TEST_ -C Release -j 32 -T test" +else + CMD="${CTEST_CMD} . -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test " +fi + +qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE} + +echo "Done running ctest parallel command." diff --git a/config/cmake/scripts/CTestScript.cmake b/config/cmake/scripts/CTestScript.cmake index e234057..e1fe85a 100644 --- a/config/cmake/scripts/CTestScript.cmake +++ b/config/cmake/scripts/CTestScript.cmake @@ -280,7 +280,11 @@ message (STATUS "Dashboard script configuration:\n${vars}\n") if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub") execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME}) else () - execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME}) + if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub") + execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestS.out) + else () + execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME}) + endif() endif () message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml") execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET) @@ -299,7 +303,11 @@ message (STATUS "Dashboard script configuration:\n${vars}\n") if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "raybsub") execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}) else () - execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}) + if (LOCAL_BATCH_SCRIPT_COMMAND STREQUAL "qsub") + execute_process (COMMAND ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_NAME} ctestP.out) + else () + execute_process (COMMAND ${LOCAL_BATCH_SCRIPT_COMMAND} ${LOCAL_BATCH_SCRIPT_ARGS} ${CTEST_BINARY_DIRECTORY}/${LOCAL_BATCH_SCRIPT_PARALLEL_NAME}) + endif () endif () message(STATUS "Check for existence of ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml") execute_process(COMMAND ls ${CTEST_BINARY_DIRECTORY}/Testing/${TAG_CONTENTS}/Test.xml RESULT_VARIABLE result OUTPUT_QUIET ERROR_QUIET) diff --git a/config/cmake/scripts/HDF5config.cmake b/config/cmake/scripts/HDF5config.cmake index 0d85b42..3ae48ce 100644 --- a/config/cmake/scripts/HDF5config.cmake +++ b/config/cmake/scripts/HDF5config.cmake @@ -44,7 +44,7 @@ set (CTEST_SOURCE_VERSEXT "") #CTEST_CONFIGURATION_TYPE - Release, Debug, RelWithDebInfo #CTEST_SOURCE_NAME - name of source folder; HDF5-1.10.0 #MODEL - CDash group name -#HPC - run alternate configurations for HPC machines; sbatch, bsub, raybsub +#HPC - run alternate configurations for HPC machines; sbatch, bsub, raybsub, qsub #MPI - enable MPI; if (DEFINED CTEST_SCRIPT_ARG) # transform ctest script arguments of the form diff --git a/config/cmake/scripts/HPC/qsub-HDF5options.cmake b/config/cmake/scripts/HPC/qsub-HDF5options.cmake new file mode 100644 index 0000000..d131e4f --- /dev/null +++ b/config/cmake/scripts/HPC/qsub-HDF5options.cmake @@ -0,0 +1,49 @@ +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# +############################################################################################# +#### Change default configuration of options in config/cmake/cacheinit.cmake file ### +#### format: set(ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DXXX:YY=ZZZZ") ### +############################################################################################# + +### uncomment/comment and change the following lines for other configuration options + +############################################################################################# +### enable parallel builds +if (DEFINED MPI) + # maximum parallel processor count for build and test #### + set (MAX_PROC_COUNT 8) + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_PARALLEL:BOOL=ON") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_CPP_LIB:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_BUILD_JAVA:BOOL=OFF") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DHDF5_ENABLE_THREADSAFE:BOOL=OFF") +endif () +############################################################################################# +### options to run test scripts in batch commands +set (LOCAL_BATCH_SCRIPT_NAME "ctest.qsub") +if (DEFINED KNL) + ### some additions and alternatives to cross compile on haswell for knl + set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl") + set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake") +endif () +set (LOCAL_BATCH_SCRIPT_COMMAND "qsub") +set (LOCAL_BATCH_TEST "TRUE") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_TEST:BOOL=ON") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DLOCAL_BATCH_SCRIPT_NAME:STRING=${LOCAL_BATCH_SCRIPT_NAME}") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_EXECUTABLE:STRING=aprun") +# Option to suppress writing job statistics; to avoid issues with h5diff comparisons. +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_PREFLAGS:STRING=-q") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_NUMPROC_FLAG:STRING=-n") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DMPIEXEC_MAX_NUMPROCS:STRING=6") +set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DACCOUNT_ID:STRING=${LOCAL_BATCH_SCRIPT_ARGS}") + +############################################################################################# +############################################################################################# -- cgit v0.12