Add script for building HDF5 with CMake on HPC machines that use sbatch.

Add README file for inclusion in CMake-hdf5-<version>.tar.gz file.
Add updated README_HPC file with cross compile instructions.

3 files changed, 235 insertions, 102 deletions

diff --git a/bin/release b/bin/release
index 96c2e78..ad1365c 100755
--- a/bin/release
+++ b/bin/release
@@ -59,17 +59,9 @@ for compressing the resulting tar archive (if none are given then
     zip		-- convert all text files to DOS style and form a zip file for Windows use.
     cmake-tgz   -- create a tar file using the gzip default level with a build-unix.sh 
                    command file and all other CMake files needed to build HDF5 source
-                   using CMake on unix machines.
+                   using CMake on unix machines, including HPC machines.
     cmake-zip   -- convert all text files to DOS style and create a zip file inluding cmake
                    scripts and .bat files to build HDF5 source using CMake on Windows. 
-    hpc-cmake-tgz  
-                -- create a tar file using the gzip default level with a build-unix.sh 
-                   command file and all other CMake files needed to build HDF5 source
-                   using CMake on unix machines, with HDF5options.cmake files for serial
-                   and parallel builds on machines requiring batch jobs to run tests.
-                   The default is for parallel build, with serial only build by changing
-                   the HDF5options.cmake symlink to ser-HDF5options.cmake.  More 
-                   information is available in the README_HPC file.
     doc         -- produce the latest doc tree in addition to the archive.
 
 An md5 checksum is produced for each archive created and stored in the md5 file.
@@ -279,12 +271,14 @@ tar2cmakezip()
 # 3. add SZIP.tar.gz, ZLib.tar.gz and cmake files to top level directory.
 # 4. create gzipped tar file with these contents:
 #        build-unix.sh          script
+#        build-unix-hpc.sh      script
 #        hdf5-<version>         source code directory extracted from tar file
 #        CTestScript.cmake      cmake file copied from <hdf5 source code>/config/cmake/scripts
 #        HDF5config.cmake       cmake file copied from <hdf5 source code>/config/cmake/scripts
 #        HDF5options.cmake      cmake file copied from <hdf5 source code>/config/cmake/scripts
 #        SZip.tar.gz            copied from /mnt/scr1/pre-release/hdf5/CMake
 #        ZLib.tar.gz            copied from /mnt/scr1/pre-release/hdf5/CMake
+#        README_HDF5_CMake      copied from <hdf5 source code>/release_docs
 
 
 # Parameters:
@@ -322,8 +316,9 @@ tar2cmaketgz()
     fi
 
 
-    # step 2: add build-unix.sh script
+    # step 2: add build-unix.sh and build-unix-hpc.sh scripts
     (cd $cmgztmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log" > build-unix.sh; chmod 755 build-unix.sh)
+    (cd $cmgztmpsubdir; echo "ctest -S HDF5config.cmake,HPC=sbatch,MPI=true -C Release -V -O hdf5.log" > build-unix-hpc.sh; chmod 755 build-unix-hpc.sh)
 
     # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files
     cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmgztmpsubdir
@@ -332,99 +327,13 @@ tar2cmaketgz()
     cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpsubdir
+    cp $cmgztmpsubdir/$version/release_docs/README_HDF5_CMake $cmgztmpsubdir
     tar czf $DEST/CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 
                  
     # cleanup
     rm -rf $cmgztmpdir
 }
 
-# Function name: tar2hpccmaketgz
-# Convert the release tarball to a gzipped tarfile with files to run CMake build
-# and HDF5options.cmake files for parallel or serial only builds where build
-# tests are run on compute nodes using batch scripts.
-#
-# Programmer: Larry Knox
-# Creation date: 2019-01-28
-#
-# Modifications
-#
-# Steps:
-# 1. untar the tarball in a temporary directory;
-#    Note: do this in a temporary directory to avoid changing
-#    the original source directory which may be around.
-# 2. add build-unix.sh script.
-# 3. add SZIP.tar.gz, ZLib.tar.gz and cmake files to top level directory.
-# 4. create gzipped tar file with these contents:
-#        build-unix.sh          script
-#        hdf5-<version>         source code directory extracted from tar file
-#        CTestScript.cmake      cmake file copied from <hdf5 source code>/config/cmake/scripts
-#        HDF5config.cmake       cmake file copied from <hdf5 source code>/config/cmake/scripts
-#        HDF5options.cmake      cmake file copied from <hdf5 source code>/config/cmake/scripts
-#        SZip.tar.gz            copied from /mnt/scr1/pre-release/hdf5/CMake
-#        ZLib.tar.gz            copied from /mnt/scr1/pre-release/hdf5/CMake
-#
-# 5. For HPC-CMake tgz file the following are also needed in the top-level directory:
-#        README_HPC                 copied from release_docs
-#        ser-HDF5options.cmake      copied from <hdf5 source code>/config/cmake/scripts/HPC
-#        par-HDF5options.cmake      copied from <hdf5 source code>/config/cmake/scripts/HPC
-#        HDF5options.cmake          symlink to par-HDF5options.cmake
-#
-
-# Parameters:
-# $1 version
-# $2 release tarball
-# $3 output zipball file name
-#
-# Returns 0 if successful; 1 otherwise
-#
- # need function to create another temporary directory, extract the
-            # $tmpdir/$HDF5_VERS.tar into it, create build-unix.sh,
-            # add CTestScript.cmake, HDF5config.cmake, SZIP.tar.gz
-            # ZLib.tar.gz, HDF5 examples, and then tar.gz it.
-tar2hpccmaketgz()
-{
-    if [ $# -ne 3 ]; then
-	echo "usage: tar2hpccmaketgz <tarfilename> <tgzfilename>"
-	return 1
-    fi
-    cmgztmpdir=/tmp/cmgztmpdir$$
-    cmgztmpsubdir=$cmgztmpdir/HPC-CMake-$HDF5_VERS
-    mkdir -p $cmgztmpsubdir
-    version=$1
-    tarfile=$2
-    tgzfile=$3
-
-    # step 1: untar tarball in cmgztmpdir
-    (cd $cmgztmpsubdir; tar xf -) < $tarfile
-    # sanity check
-    if [ ! -d $cmgztmpsubdir/$version ]; then
-	echo "untar did not create $cmgztmpsubdir/$version source dir"
-	# cleanup
-	rm -rf $cmgztmpdir
-	return 1
-    fi
-
-
-    # step 2: add build-unix.sh script
-    (cd $cmgztmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log" > build-unix.sh; chmod 755 build-unix.sh)
-
-    # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files
-    cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmgztmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.12.4-Source.tar.gz $cmgztmpsubdir
-    cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir
-    cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir
-
-    cp $cmgztmpsubdir/$version/release_docs/README_HPC $cmgztmpsubdir
-    cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/ser-HDF5options.cmake $cmgztmpsubdir
-    cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/par-HDF5options.cmake $cmgztmpsubdir
-    (cd $cmgztmpsubdir; ln -s par-HDF5options.cmake HDF5options.cmake)
-    tar czf $DEST/HPC-CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 
-                 
-    # cleanup
-    rm -rf $cmgztmpdir
-}
-
 # This command must be run at the top level of the hdf5 source directory.
 # Verify this requirement.
 if [ ! \( -f configure.ac -a -f bin/release \) ]; then
@@ -609,11 +518,6 @@ for comp in $methods; do
 	    tar2cmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/CMake-$HDF5_VERS.tar.gz 1>&2
 	    (cd $DEST; md5sum CMake-$HDF5_VERS.tar.gz >> $MD5file) 
             ;;
-        hpc-cmake-tgz)
-	    test "$verbose" && echo "   Creating HPC-CMake tar.gz file..." 1>&2
-	    tar2hpccmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/HPC-CMake-$HDF5_VERS.tar.gz 1>&2
-	    (cd $DEST; md5sum HPC-CMake-$HDF5_VERS.tar.gz >> $MD5file) 
-            ;;
 	bzip2)
 	    test "$verbose" && echo "   Running bzip2..." 1>&2
 	    bzip2 -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.bz2
diff --git a/release_docs/README_HDF5_CMake b/release_docs/README_HDF5_CMake
new file mode 100644
index 0000000..2b81f8e
--- /dev/null
+++ b/release_docs/README_HDF5_CMake
@@ -0,0 +1,23 @@
+This tar file contains 
+
+    build-unix.sh       script to build HDF5 with CMake on unix machines
+    buidl-unix-hpc.sh   script to build HDF5 with CMake on unix machines and run
+                        tests with batch scripts (sbatch).
+    CTestScript.cmake   
+    HDF5config.cmake    CMake scripts for building HDF5
+    HDF5options.cmake
+    hdf5-1.10.5-pre1    HDF5 1.10.5-pre1 source
+    SZip.tar.gz         source for building SZIP
+    ZLib.tar.gz         source for building ZLIB
+
+For more information about building HDF5 with CMake, see USING_HDF5_CMake.txt in
+hdf5-1.10.5-pre1/release_docs, or 
+https://portal.hdfgroup.org/display/support/Building+HDF5+with+CMake.
+
+For more information about building HDF5 with CMake on HPC machines, including 
+cross compiling on Cray XC40, see README_HPC in hdf5-1.10.5-pre1/release_docs.
+
+
+
+
+
diff --git a/release_docs/README_HPC b/release_docs/README_HPC
new file mode 100644
index 0000000..67a5d6c
--- /dev/null
+++ b/release_docs/README_HPC
@@ -0,0 +1,206 @@
+************************************************************************
+*    Using CMake to build and test HDF5 source on HPC machines         *
+************************************************************************ 
+
+                    Contents
+
+Section I:     Prerequisites
+Section II:    Obtain HDF5 source
+Section III:   Using ctest command to build and test
+Section IV:    Cross compiling
+Section V:     Manual alternatives
+Section VI:    Other cross compiling options
+
+************************************************************************
+
+========================================================================
+I. Prerequisites
+========================================================================
+    1. Create a working directory that is accessible from the compute nodes for 
+       running tests; the working directory should be in a scratch space or a
+       parallel file system space since testing will use this space.  Building
+       from HDF5 source in a 'home' directory typically results in test 
+       failures and should be avoided.
+	  
+    2. Load modules for desired compilers, module for cmake version 3.10 or greater, 
+       and set any needed environment variables for compilers (i.e., CC, FC, CXX).
+       Unload any problematic modules (i.e., craype-hugepages2M).
+
+========================================================================
+II. Obtain HDF5 source
+========================================================================
+Obtain HDF5 source code from the HDF5 repository using a git command or 
+from a release tar file in a working directory:
+
+    git clone https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5.git 
+        [-b branch] [source directory]
+
+If no branch is specified, then the 'develop' version will be checked out.
+If no source directory is specified, then the source will be located in the 
+'hdf5' directory. The Cmake scripts expect the source to be in a directory 
+named hdf5-<version string>, where 'version string' uses the format '1.xx.xx'.
+For example, for the current 'develop' version, the "hdf5" directory should 
+be renamed "hdf5-1.11.4", or for the first hdf5_1_10_5 pre-release version, 
+it should be renamed "hdf5-1.10.5-pre1".
+
+If the version number is not known a priori, the version string 
+can be obtained by running bin/h5vers in the top level directory of the source clone, and
+the source directory renamed 'hdf5-<version string>'.
+
+Release or snapshot tar files may also be extracted and used.
+
+========================================================================
+III. Using ctest command to build and test
+========================================================================
+
+The ctest command [1]:
+
+  ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log
+
+will configure, build, test and package HDF5 from the downloaded source 
+after the setup steps outlined below are followed.
+
+CMake option variables are available to allow running test programs in batch 
+scripts on compute nodes and to cross-compile for compute node hardware using 
+a cross-compiling emulator.  The setup steps will make default settings for
+parallel or serial only builds available to the CMake command.  
+
+    1. For the current 'develop' version the "hdf5" directory should be renamed 
+       "hdf5-1.11.4".
+
+    2. Three cmake script files need to be copied to the working directory, or
+       have symbolic links to them, created in the working directory:
+   
+       hdf5-1.11.4/config/cmake/scripts/HDF5config.cmake
+       hdf5-1.11.4/config/cmake/scripts/CTestScript.cmake
+       hdf5-1.11.4/config/cmake/scripts/HDF5options.cmake 
+
+       should be copied to the working directory.
+
+    3. The resulting contents of the working directory are then:
+
+       CTestScript.cmake
+       HDF5config.cmake
+       HDF5options.cmake
+       hdf5-1.11.4
+
+       Additionally, when the ctest command runs [1], it will add a build directory 
+       in the working directory.
+
+    4. The following options (among others) can be added to the ctest
+       command [1], following '-S HDF5config.cmake,' and separated by ',':
+
+       HPC=sbatch   (or 'bsub' or 'raybsub') indicates which type of batch
+                     files to use for running tests.  If omitted, test 
+                     will run on the local machine or login node.
+
+       KNL=true      to cross-compile for KNL compute nodes on CrayXC40
+                     (see section IV)
+
+       MPI=true      enables parallel, disables c++, java, and threadsafe
+
+       LOCAL_BATCH_SCRIPT_ARGS="--account=<account#>"  to supply user account
+                                                       information for batch jobs
+
+       The HPC options will add BUILD_GENERATOR=Unix for the three HPC options.
+       An example ctest command for a parallel build on a system using sbatch is
+   
+         ctest -S HDF5config.cmake,HPC=sbatch,MPI=true -C Release -V -O hdf5.log
+
+       Adding the option 'KNL=true' to the above list will compile for KNL nodes, 
+       for example, on 'mutrino' and other CrayXC40 machines.
+
+       Changing -V to -VV will produce more logging information in HDF5.log. 
+
+    More detailed CMake information can be found in the HDF5 source in 
+    release_docs/INSTALL_CMake.txt.
+
+========================================================================
+IV. Cross-compiling
+========================================================================
+For cross-compiling on Cray, set environment variables CC=cc, FC=ftn 
+and CXX=CC (for c++) after all compiler modules are loaded since switching
+compiler modules may unset or reset these variables.
+
+CMake provides options for cross-compiling.  To cross-compile for KNL hardware 
+on mutrino and other CrayXC40 machines, add HPC=sbatch,KNL=true to the 
+ctest command line.  This will set the following options from the 
+config/cmake/scripts/HPC/sbatch-HDF5options.cmake file:
+
+    set (COMPILENODE_HWCOMPILE_MODULE "craype-haswell")
+    set (COMPUTENODE_HWCOMPILE_MODULE "craype-mic-knl")
+    set (LOCAL_BATCH_SCRIPT_NAME "knl_ctestS.sl")
+    set (LOCAL_BATCH_SCRIPT_PARALLEL_NAME "knl_ctestP.sl")
+    set (ADD_BUILD_OPTIONS "${ADD_BUILD_OPTIONS} -DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake")
+
+On the Cray XC40 the craype-haswell module is needed for configuring, and the 
+craype-mic-knl module is needed for building to run on the KNL nodes. CMake
+with the above options will swap modules after configuring is complete, 
+but before compiling programs for KNL.
+
+The sbatch script arguments for running jobs on KNL nodes may differ on CrayXC40
+machines other than mutrino.  The batch scripts knl_ctestS.sl and knl_ctestP.sl 
+have the correct arguments for mutrino:  "#SBATCH -p knl -C quad,cache".  For 
+cori, another CrayXC40, that line is replaced by "#SBATCH -C knl,quad,cache".  
+For cori (and other machines), the values in LOCAL_BATCH_SCRIPT_NAME and 
+LOCAL_BATCH_SCRIPT_PARALLEL_NAME in the config/cmake/scripts/HPC/sbatch-HDF5options.cmake 
+file can be replaced by cori_knl_ctestS.sl and cori_knl_ctestS.sl, or the lines 
+can be edited in the batch files in hdf5-1.11.4/bin/batch.
+
+========================================================================
+V. Manual alternatives
+========================================================================
+If using ctest is undesirable, one can create a build directory and run the cmake 
+configure command, for example
+
+"/projects/Mutrino/hpcsoft/cle6.0/common/cmake/3.10.2/bin/cmake" 
+-C "<working directory>/hdf5-1.11.4/config/cmake/cacheinit.cmake" 
+-DCMAKE_BUILD_TYPE:STRING=Release -DHDF5_BUILD_FORTRAN:BOOL=ON 
+-DHDF5_BUILD_JAVA:BOOL=OFF 
+-DCMAKE_INSTALL_PREFIX:PATH=<working directory>/HDF_Group/HDF5/1.11.4 
+-DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=OFF -DHDF5_ENABLE_SZIP_SUPPORT:BOOL=OFF 
+-DHDF5_ENABLE_PARALLEL:BOOL=ON -DHDF5_BUILD_CPP_LIB:BOOL=OFF 
+-DHDF5_BUILD_JAVA:BOOL=OFF -DHDF5_ENABLE_THREADSAFE:BOOL=OFF 
+-DHDF5_PACKAGE_EXTLIBS:BOOL=ON -DLOCAL_BATCH_TEST:BOOL=ON 
+-DMPIEXEC_EXECUTABLE:STRING=srun -DMPIEXEC_NUMPROC_FLAG:STRING=-n 
+-DMPIEXEC_MAX_NUMPROCS:STRING=6 
+-DCMAKE_TOOLCHAIN_FILE:STRING=config/toolchain/crayle.cmake 
+-DLOCAL_BATCH_SCRIPT_NAME:STRING=knl_ctestS.sl 
+-DLOCAL_BATCH_SCRIPT_PARALLEL_NAME:STRING=knl_ctestP.sl -DSITE:STRING=mutrino 
+-DBUILDNAME:STRING=par-knl_GCC493-SHARED-Linux-4.4.156-94.61.1.16335.0.PTF.1107299-default-x86_64 
+"-GUnix Makefiles" "" "<working directory>/hdf5-1.11.4"
+
+followed by make and batch jobs to run tests.
+
+To cross-compile on CrayXC40, run the configure command with the craype-haswell 
+module loaded, then switch to the craype-mic-knl module for the build process.
+
+Tests on machines using slurm can be run with 
+
+"sbatch -p knl -C quad,cache ctestS.sl" 
+
+or 
+
+"sbatch -p knl -C quad,cache ctestP.sl" 
+
+for parallel builds.
+ 
+Tests on machines using LSF will typically use "bsub ctestS.lsf", etc.
+
+========================================================================
+VI.    Other cross compiling options
+========================================================================
+Settings for two other cross-compiling options are also in the config/toolchain 
+files which do not seem to be necessary with the Cray PrgEnv-* modules
+
+1.  HDF5_USE_PREGEN.  This option, along with the HDF5_USE_PREGEN_DIR CMake 
+    variable would allow the use of an appropriate H5Tinit.c file with type 
+	information generated on a compute node to be used when cross compiling 
+	for those compute nodes.  The use of the variables in lines 110 and 111 
+	of HDF5options.cmake file seem to preclude needing this option with the 
+	available Cray modules and CMake option.
+
+2.  HDF5_BATCH_H5DETECT and associated CMake variables.  This option when 
+    properly configured will run H5detect in a batch job on a compute node 
+	at the beginning of the CMake build process.  It was also found to be 
+	unnecessary with the available Cray modules and CMake options.