diff options
author | Larry Knox <lrknox@hdfgroup.org> | 2019-01-31 20:31:38 (GMT) |
---|---|---|
committer | Larry Knox <lrknox@hdfgroup.org> | 2019-01-31 20:31:38 (GMT) |
commit | b0e202a206a8cbbcf4adfc870e3f4fec3a0371f7 (patch) | |
tree | f0e0c7c10b88ba799a4b2bab4a5d173f2b0f1240 /bin | |
parent | b561c2fce8c96dbc97a6a1f01b49a2240094d55a (diff) | |
parent | bae744199b69eb7c3f5eda2441cca30cb8c6175f (diff) | |
download | hdf5-b0e202a206a8cbbcf4adfc870e3f4fec3a0371f7.zip hdf5-b0e202a206a8cbbcf4adfc870e3f4fec3a0371f7.tar.gz hdf5-b0e202a206a8cbbcf4adfc870e3f4fec3a0371f7.tar.bz2 |
Merge pull request #1508 in HDFFV/hdf5 from ~LRKNOX/hdf5_lrk:develop to develop
* commit 'bae744199b69eb7c3f5eda2441cca30cb8c6175f':
Add more specific batch scripts. Add script raybsub to handle submitting .lsf files on ray with bsub < script.lsf syntax that we couldn't handle with CMake. Add hpc-cmake-tgz option for bin/release.
Account info can be submitted as an argument to run the batch scripts. Special versions for sandia are not needed.
Add HDF5options.cmake files for serial and parallel tests on HPC machines (where tests are to be run as batch jobs. Add README_HPC file with instructions for setting up configuration to use batch scripts for running tests on HPC machines and some pointers for cross compiling.
Remove knl configuration from ctestS.sl.in.cmake. Add sandia batch files with account info.
Add batch files with batch parameters specific to cori and ray. Release.txt entry for skipping long double dt_arith tests on ppc64.
Diffstat (limited to 'bin')
-rw-r--r-- | bin/batch/cori_ctestP.sl.in.cmake | 17 | ||||
-rw-r--r-- | bin/batch/cori_ctestS.sl.in.cmake | 16 | ||||
-rw-r--r-- | bin/batch/cori_knl_ctestP.sl.in.cmake | 17 | ||||
-rw-r--r-- | bin/batch/cori_knl_ctestS.sl.in.cmake | 16 | ||||
-rw-r--r-- | bin/batch/ctestP.sl.in.cmake | 4 | ||||
-rw-r--r-- | bin/batch/ctestS.sl.in.cmake | 2 | ||||
-rw-r--r-- | bin/batch/knl_ctestP.sl.in.cmake | 2 | ||||
-rw-r--r-- | bin/batch/knl_ctestS.sl.in.cmake | 2 | ||||
-rw-r--r-- | bin/batch/ray_ctestP.lsf.in.cmake | 20 | ||||
-rw-r--r-- | bin/batch/ray_ctestS.lsf.in.cmake | 18 | ||||
-rw-r--r-- | bin/batch/raybsub | 7 | ||||
-rwxr-xr-x | bin/release | 102 |
12 files changed, 215 insertions, 8 deletions
diff --git a/bin/batch/cori_ctestP.sl.in.cmake b/bin/batch/cori_ctestP.sl.in.cmake new file mode 100644 index 0000000..6aa4916 --- /dev/null +++ b/bin/batch/cori_ctestP.sl.in.cmake @@ -0,0 +1,17 @@ +#!/bin/bash + +#SBATCH -C haswell +#SBATCH --nodes=1 +#SBATCH -t 00:30:00 +#SBATCH --mail-type=BEGIN,END,FAIL +##SBATCH --mail-user=<username>@sandia.gov +#SBATCH --export=ALL +#SBATCH --job-name=h5_ctestP + +cd @HDF5_BINARY_DIR@ +CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM|ph5example -E t_cache_image -C Release -T test" + +echo "Run $CMD. Test output will be in build/ctestP.out" +$CMD >& ctestP.out +echo "Done running $CMD" + diff --git a/bin/batch/cori_ctestS.sl.in.cmake b/bin/batch/cori_ctestS.sl.in.cmake new file mode 100644 index 0000000..502db3f --- /dev/null +++ b/bin/batch/cori_ctestS.sl.in.cmake @@ -0,0 +1,16 @@ +#!/bin/bash + +#SBATCH -C haswell +#SBATCH --nodes=1 +#SBATCH -t 00:30:00 +#SBATCH --mail-type=BEGIN,END,FAIL +##SBATCH --mail-user=<username>@sandia.gov +#SBATCH --export=ALL +#SBATCH --job-name=h5_ctestS + +cd @HDF5_BINARY_DIR@ +CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM|ph5example -C Release -j 32 -T test" + +echo "Run $CMD. Test output will be in build/ctestS.out" +$CMD >& ctestS.out +echo "Done running $CMD" diff --git a/bin/batch/cori_knl_ctestP.sl.in.cmake b/bin/batch/cori_knl_ctestP.sl.in.cmake new file mode 100644 index 0000000..90dd52a --- /dev/null +++ b/bin/batch/cori_knl_ctestP.sl.in.cmake @@ -0,0 +1,17 @@ +#!/bin/bash + +#SBATCH -C knl,quad,cache +#SBATCH --nodes=1 +#SBATCH -t 00:30:00 +#SBATCH --mail-type=BEGIN,END,FAIL +##SBATCH --mail-user=<username>@sandia.gov +#SBATCH --export=ALL +#SBATCH --job-name=h5_ctestP + +cd @HDF5_BINARY_DIR@ +CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM|ph5example -E t_cache_image -C Release -T test" + +echo "Run $CMD. Test output will be in build/ctestP.out" +$CMD >& ctestP.out +echo "Done running $CMD" + diff --git a/bin/batch/cori_knl_ctestS.sl.in.cmake b/bin/batch/cori_knl_ctestS.sl.in.cmake new file mode 100644 index 0000000..0ec7bf7 --- /dev/null +++ b/bin/batch/cori_knl_ctestS.sl.in.cmake @@ -0,0 +1,16 @@ +#!/bin/bash + +#SBATCH -C knl,quad, cache +#SBATCH --nodes=1 +#SBATCH -t 00:30:00 +#SBATCH --mail-type=BEGIN,END,FAIL +##SBATCH --mail-user=<username>@sandia.gov +#SBATCH --export=ALL +#SBATCH --job-name=h5_ctestS + +cd @HDF5_BINARY_DIR@ +CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM|ph5example -C Release -j 32 -T test" + +echo "Run $CMD. Test output will be in build/ctestS.out" +$CMD >& ctestS.out +echo "Done running $CMD" diff --git a/bin/batch/ctestP.sl.in.cmake b/bin/batch/ctestP.sl.in.cmake index 230a642..9fed14d 100644 --- a/bin/batch/ctestP.sl.in.cmake +++ b/bin/batch/ctestP.sl.in.cmake @@ -5,10 +5,10 @@ #SBATCH --mail-type=BEGIN,END,FAIL ##SBATCH --mail-user=<username>@sandia.gov #SBATCH --export=ALL -#SBATCH --job-name=h5_ctestS +#SBATCH --job-name=h5_ctestP cd @HDF5_BINARY_DIR@ -CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM -E t_cache_image -C Release -T test" +CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM|ph5example -E t_cache_image -C Release -T test" echo "Run $CMD. Test output will be in build/ctestP.out" $CMD >& ctestP.out diff --git a/bin/batch/ctestS.sl.in.cmake b/bin/batch/ctestS.sl.in.cmake index 9802e13..f772be9 100644 --- a/bin/batch/ctestS.sl.in.cmake +++ b/bin/batch/ctestS.sl.in.cmake @@ -8,7 +8,7 @@ #SBATCH --job-name=h5_ctestS cd @HDF5_BINARY_DIR@ -CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM -C Release -j 32 -T test" +CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM|ph5example -C Release -j 32 -T test" echo "Run $CMD. Test output will be in build/ctestS.out" $CMD >& ctestS.out diff --git a/bin/batch/knl_ctestP.sl.in.cmake b/bin/batch/knl_ctestP.sl.in.cmake index 687804b..8d786da 100644 --- a/bin/batch/knl_ctestP.sl.in.cmake +++ b/bin/batch/knl_ctestP.sl.in.cmake @@ -10,7 +10,7 @@ cd @HDF5_BINARY_DIR@ #run parallel tests except t_cache_image test -CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM -E t_cache_image -C Release -T test" +CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM|ph5example -E t_cache_image -C Release -T test" echo "Run $CMD. Test output will be in build/ctestP.out" $CMD >& ctestP.out diff --git a/bin/batch/knl_ctestS.sl.in.cmake b/bin/batch/knl_ctestS.sl.in.cmake index de1335c..b983228 100644 --- a/bin/batch/knl_ctestS.sl.in.cmake +++ b/bin/batch/knl_ctestS.sl.in.cmake @@ -9,7 +9,7 @@ #SBATCH --job-name=h5_ctestS cd @HDF5_BINARY_DIR@ -CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM -C Release -j 32 -T test" +CMD="ctest . -E TEST_PAR|PH5DIFF|PERFORM|ph5example -C Release -j 32 -T test" echo "Run $CMD. Test output will be in build/ctestS.out" $CMD >& ctestS.out diff --git a/bin/batch/ray_ctestP.lsf.in.cmake b/bin/batch/ray_ctestP.lsf.in.cmake new file mode 100644 index 0000000..6b5316a --- /dev/null +++ b/bin/batch/ray_ctestP.lsf.in.cmake @@ -0,0 +1,20 @@ +#!/bin/tcsh +### LSF syntax +#BSUB -n 6 #number of nodes +#BSUB -R "span[ptile=6]" +#BSUB -W 30 #walltime in minutes +#BSUB -G guests #account +#BSUB -e ctestPerrors.txt #stderr +#BSUB -o ctestPoutput.txt #stdout +#BSUB -J hdf5_ctestP #job +##BSUB -q pbatch #queue to use +#BSUB -q pdebug + +##date; hostname +##echo -n 'JobID is '; echo $LSB_JOBID + +cd @HDF5_BINARY_DIR@ +echo "Run parallel test command. Test output will be in build/ctestP.out" +ctest . -R 'TEST_PAR|PH5DIFF|PERFORM|ph5example' -E t_cache_image -C Release -T test >& ctestP.out + +echo "Done running ctest parallel command." diff --git a/bin/batch/ray_ctestS.lsf.in.cmake b/bin/batch/ray_ctestS.lsf.in.cmake new file mode 100644 index 0000000..cbc7130 --- /dev/null +++ b/bin/batch/ray_ctestS.lsf.in.cmake @@ -0,0 +1,18 @@ +#!/bin/tcsh +### LSF syntax +#BSUB -n 1 #number of nodes +#BSUB -W 29 #walltime in minutes +#BSUB -G guests #account +#BSUB -e ctestSerrors.txt #stderr +#BSUB -o ctestSoutput.txt #stdout +#BSUB -J hdf5_ctestS #job +##BSUB -q pbatch #queue to use +#BSUB -q pdebug + +cd @HDF5_BINARY_DIR@ +echo "Run command. Test output will be in build/ctestS.out" +ctest . -E 'TEST_PAR|H5DIFF|PERFORM|ph5example' -C Release -j 32 -T test >& ctestS.out + +##$CMD >& ctestS.out +echo "Done running command." + diff --git a/bin/batch/raybsub b/bin/batch/raybsub new file mode 100644 index 0000000..19dceef --- /dev/null +++ b/bin/batch/raybsub @@ -0,0 +1,7 @@ +#!/bin/tcsh + +# ray.llnl.gov requires a '<' with bsub for submitting .lsf batch jobs. +# CMake is reluctant to pass the '<', so we put it in this script and use +# the script to submit the bsub command on ray. + +bsub < $1 diff --git a/bin/release b/bin/release index e43be17..1cf1b13 100755 --- a/bin/release +++ b/bin/release @@ -55,13 +55,21 @@ for compressing the resulting tar archive (if none are given then tar -- use tar and don't do any compressing. gzip -- use gzip with "-9" and append ".gz" to the output name. + bzip2 -- use bzip2 with "-9" and append ".bz2" to the output name. + zip -- convert all text files to DOS style and form a zip file for Windows use. cmake-tgz -- create a tar file using the gzip default level with a build-unix.sh command file and all other CMake files needed to build HDF5 source using CMake on unix machines. - bzip2 -- use bzip2 with "-9" and append ".bz2" to the output name. - zip -- convert all text files to DOS style and form a zip file for Windows use. cmake-zip -- convert all text files to DOS style and create a zip file inluding cmake scripts and .bat files to build HDF5 source using CMake on Windows. + hpc-cmake-tgz + -- create a tar file using the gzip default level with a build-unix.sh + command file and all other CMake files needed to build HDF5 source + using CMake on unix machines, with HDF5options.cmake files for serial + and parallel builds on machines requiring batch jobs to run tests. + The default is for parallel build, with serial only build by changing + the HDF5options.cmake symlink to ser-HDF5options.cmake. More + information is available in the README_HPC file. doc -- produce the latest doc tree in addition to the archive. An md5 checksum is produced for each archive created and stored in the md5 file. @@ -256,7 +264,7 @@ tar2cmakezip() } # Function name: tar2cmaketgz -# Convert the release tarball to a Windows zipball with files to run CMake build. +# Convert the release tarball to a gzipped tar file with files to run CMake build. # # Programmer: Larry Knox # Creation date: 2017-02-20 @@ -330,6 +338,94 @@ tar2cmaketgz() rm -rf $cmgztmpdir } +# Function name: tar2hpccmaketgz +# Convert the release tarball to a gzipped tarfile with files to run CMake build +# and HDF5options.cmake files for parallel or serial only builds where build +# tests are run on compute nodes using batch scripts. +# +# Programmer: Larry Knox +# Creation date: 2019-01-28 +# +# Modifications +# +# Steps: +# 1. untar the tarball in a temporary directory; +# Note: do this in a temporary directory to avoid changing +# the original source directory which may be around. +# 2. add build-unix.sh script. +# 3. add SZIP.tar.gz, ZLib.tar.gz and cmake files to top level directory. +# 4. create gzipped tar file with these contents: +# build-unix.sh script +# hdf5-<version> source code directory extracted from tar file +# CTestScript.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts +# HDF5config.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts +# HDF5options.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts +# SZip.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake +# ZLib.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake +# +# 5. For HPC-CMake tgz file the following are also needed in the top-level directory: +# README_HPC copied from release_docs +# ser-HDF5options.cmake copied from <hdf5 source code>/config/cmake/scripts/HPC +# par-HDF5options.cmake copied from <hdf5 source code>/config/cmake/scripts/HPC +# HDF5options.cmake symlink to par-HDF5options.cmake +# + +# Parameters: +# $1 version +# $2 release tarball +# $3 output zipball file name +# +# Returns 0 if successful; 1 otherwise +# + # need function to create another temporary directory, extract the + # $tmpdir/$HDF5_VERS.tar into it, create build-unix.sh, + # add CTestScript.cmake, HDF5config.cmake, SZIP.tar.gz + # ZLib.tar.gz, HDF5 examples, and then tar.gz it. +tar2hpccmaketgz() +{ + if [ $# -ne 3 ]; then + echo "usage: tar2cmaketgz <tarfilename> <tgzfilename>" + return 1 + fi + cmgztmpdir=/tmp/cmgztmpdir$$ + cmgztmpsubdir=$cmgztmpdir/CMake-$HDF5_VERS + mkdir -p $cmgztmpsubdir + version=$1 + tarfile=$2 + tgzfile=$3 + + # step 1: untar tarball in cmgztmpdir + (cd $cmgztmpsubdir; tar xf -) < $tarfile + # sanity check + if [ ! -d $cmgztmpsubdir/$version ]; then + echo "untar did not create $cmgztmpsubdir/$version source dir" + # cleanup + rm -rf $cmgztmpdir + return 1 + fi + + + # step 2: add build-unix.sh script + (cd $cmgztmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log" > build-unix.sh; chmod 755 build-unix.sh) + + # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files + cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.10.10-Source.tar.gz $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpsubdir + + cp $cmgztmpsubdir/$version/release_docs/README_HPC $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/ser-HDF5options.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/par-HDF5options.cmake $cmgztmpsubdir + (cd $cmgztmpsubdir; ln -s par-HDF5options.cmake HDF5options.cmake) + tar czf $DEST/CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 + + # cleanup + rm -rf $cmgztmpdir +} + # This command must be run at the top level of the hdf5 source directory. # Verify this requirement. if [ ! \( -f configure.ac -a -f bin/release \) ]; then |