Merge branch 'develop' into c_stdlib_posix_call_cleanup

13 files changed, 224 insertions, 13 deletions

diff --git a/bin/batch/cori_ctestP.sl.in.cmake b/bin/batch/cori_ctestP.sl.in.cmake
new file mode 100644
index 0000000..287fe3c
--- /dev/null
+++ b/bin/batch/cori_ctestP.sl.in.cmake
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#SBATCH -C haswell
+#SBATCH --nodes=1
+#SBATCH -t 00:30:00
+#SBATCH --mail-type=BEGIN,END,FAIL
+##SBATCH --mail-user=<username>@sandia.gov
+#SBATCH --export=ALL
+#SBATCH --job-name=h5_ctestP
+
+cd @HDF5_BINARY_DIR@
+CMD="ctest . -R MPI_TEST_ -E t_cache_image -C Release -T test"
+
+echo "Run $CMD. Test output will be in build/ctestP.out"
+$CMD >& ctestP.out
+echo "Done running $CMD"
+
diff --git a/bin/batch/cori_ctestS.sl.in.cmake b/bin/batch/cori_ctestS.sl.in.cmake
new file mode 100644
index 0000000..50bef6b
--- /dev/null
+++ b/bin/batch/cori_ctestS.sl.in.cmake
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#SBATCH -C haswell
+#SBATCH --nodes=1
+#SBATCH -t 00:30:00
+#SBATCH --mail-type=BEGIN,END,FAIL
+##SBATCH --mail-user=<username>@sandia.gov
+#SBATCH --export=ALL
+#SBATCH --job-name=h5_ctestS
+
+cd @HDF5_BINARY_DIR@
+CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
+
+echo "Run $CMD. Test output will be in build/ctestS.out"
+$CMD  >& ctestS.out
+echo "Done running $CMD"
diff --git a/bin/batch/cori_knl_ctestP.sl.in.cmake b/bin/batch/cori_knl_ctestP.sl.in.cmake
new file mode 100644
index 0000000..82378ec
--- /dev/null
+++ b/bin/batch/cori_knl_ctestP.sl.in.cmake
@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#SBATCH -C knl,quad,cache
+#SBATCH --nodes=1
+#SBATCH -t 00:30:00
+#SBATCH --mail-type=BEGIN,END,FAIL
+##SBATCH --mail-user=<username>@sandia.gov
+#SBATCH --export=ALL
+#SBATCH --job-name=h5_ctestP
+
+cd @HDF5_BINARY_DIR@
+CMD="ctest . -R MPI_TEST_ -E t_cache_image -C Release -T test"
+
+echo "Run $CMD. Test output will be in build/ctestP.out"
+$CMD >& ctestP.out
+echo "Done running $CMD"
+
diff --git a/bin/batch/cori_knl_ctestS.sl.in.cmake b/bin/batch/cori_knl_ctestS.sl.in.cmake
new file mode 100644
index 0000000..c6d9e83
--- /dev/null
+++ b/bin/batch/cori_knl_ctestS.sl.in.cmake
@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#SBATCH -C knl,quad,cache
+#SBATCH --nodes=1
+#SBATCH -t 00:30:00
+#SBATCH --mail-type=BEGIN,END,FAIL
+##SBATCH --mail-user=<username>@sandia.gov
+#SBATCH --export=ALL
+#SBATCH --job-name=h5_ctestS
+
+cd @HDF5_BINARY_DIR@
+CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
+
+echo "Run $CMD. Test output will be in build/ctestS.out"
+$CMD  >& ctestS.out
+echo "Done running $CMD"
diff --git a/bin/batch/ctestP.sl.in.cmake b/bin/batch/ctestP.sl.in.cmake
index 230a642..8d216d6 100644
--- a/bin/batch/ctestP.sl.in.cmake
+++ b/bin/batch/ctestP.sl.in.cmake
@@ -5,10 +5,10 @@
 #SBATCH --mail-type=BEGIN,END,FAIL
 ##SBATCH --mail-user=<username>@sandia.gov
 #SBATCH --export=ALL
-#SBATCH --job-name=h5_ctestS
+#SBATCH --job-name=h5_ctestP
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM -E t_cache_image -C Release -T test"
+CMD="ctest . -R MPI_TEST_ -E t_cache_image -C Release -T test"
 
 echo "Run $CMD. Test output will be in build/ctestP.out"
 $CMD >& ctestP.out
diff --git a/bin/batch/ctestS.lsf.in.cmake b/bin/batch/ctestS.lsf.in.cmake
index 5692574..7ef5af5 100644
--- a/bin/batch/ctestS.lsf.in.cmake
+++ b/bin/batch/ctestS.lsf.in.cmake
@@ -11,7 +11,7 @@
 
 cd @HDF5_BINARY_DIR@
 echo "Run command. Test output will be in build/ctestS.out"
-ctest . -E 'TEST_PAR|H5DIFF|PERFORM' -C Release -j 32 -T test >& ctestS.out
+ctest . -E 'TEST_PAR|PH5DIFF|PERFORM' -C Release -j 32 -T test >& ctestS.out
 
 ##$CMD  >& ctestS.out
 echo "Done running command."
diff --git a/bin/batch/ctestS.sl.in.cmake b/bin/batch/ctestS.sl.in.cmake
index 9802e13..4f96bb9 100644
--- a/bin/batch/ctestS.sl.in.cmake
+++ b/bin/batch/ctestS.sl.in.cmake
@@ -8,7 +8,7 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM -C Release -j 32 -T test"
+CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
 
 echo "Run $CMD. Test output will be in build/ctestS.out"
 $CMD  >& ctestS.out
diff --git a/bin/batch/knl_ctestP.sl.in.cmake b/bin/batch/knl_ctestP.sl.in.cmake
index 687804b..79dfb89 100644
--- a/bin/batch/knl_ctestP.sl.in.cmake
+++ b/bin/batch/knl_ctestP.sl.in.cmake
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-#SBATCH -C knl,quad,cache
+#SBATCH -p knl -C quad,cache
 #SBATCH --nodes=1
 #SBATCH -t 00:30:00
 #SBATCH --mail-type=BEGIN,END,FAIL
@@ -10,7 +10,7 @@
 
 cd @HDF5_BINARY_DIR@
 #run parallel tests except t_cache_image test
-CMD="ctest . -R TEST_PAR|PH5DIFF|PERFORM -E t_cache_image -C Release -T test"
+CMD="ctest . -R MPI_TEST_ -E t_cache_image -C Release -T test"
 
 echo "Run $CMD. Test output will be in build/ctestP.out"
 $CMD >& ctestP.out
diff --git a/bin/batch/knl_ctestS.sl.in.cmake b/bin/batch/knl_ctestS.sl.in.cmake
index de1335c..af6353b 100644
--- a/bin/batch/knl_ctestS.sl.in.cmake
+++ b/bin/batch/knl_ctestS.sl.in.cmake
@@ -1,6 +1,6 @@
 #!/bin/bash
 
-#SBATCH -C knl,quad,cache
+#SBATCH -p knl -C quad,cache
 #SBATCH --nodes=1
 #SBATCH -t 00:30:00
 #SBATCH --mail-type=BEGIN,END,FAIL
@@ -9,7 +9,7 @@
 #SBATCH --job-name=h5_ctestS
 
 cd @HDF5_BINARY_DIR@
-CMD="ctest . -E TEST_PAR|H5DIFF|PERFORM -C Release -j 32 -T test"
+CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test"
 
 echo "Run $CMD. Test output will be in build/ctestS.out"
 $CMD  >& ctestS.out
diff --git a/bin/batch/ray_ctestP.lsf.in.cmake b/bin/batch/ray_ctestP.lsf.in.cmake
new file mode 100644
index 0000000..082276a
--- /dev/null
+++ b/bin/batch/ray_ctestP.lsf.in.cmake
@@ -0,0 +1,20 @@
+#!/bin/tcsh
+### LSF syntax
+#BSUB -n 6                       #number of nodes
+#BSUB -R "span[ptile=6]"
+#BSUB -W 30                      #walltime in minutes
+#BSUB -G guests                  #account
+#BSUB -e ctestPerrors.txt        #stderr
+#BSUB -o ctestPoutput.txt        #stdout
+#BSUB -J hdf5_ctestP             #job
+##BSUB -q pbatch                 #queue to use
+#BSUB -q pdebug
+
+##date; hostname
+##echo -n 'JobID is '; echo $LSB_JOBID
+
+cd @HDF5_BINARY_DIR@
+echo "Run parallel test command. Test output will be in build/ctestP.out"
+ctest . -R 'MPI_TEST_' -E t_cache_image -C Release -T test >& ctestP.out
+
+echo "Done running ctest parallel command."
diff --git a/bin/batch/ray_ctestS.lsf.in.cmake b/bin/batch/ray_ctestS.lsf.in.cmake
new file mode 100644
index 0000000..da20438
--- /dev/null
+++ b/bin/batch/ray_ctestS.lsf.in.cmake
@@ -0,0 +1,18 @@
+#!/bin/tcsh
+### LSF syntax
+#BSUB -n 1                       #number of nodes
+#BSUB -W 29                      #walltime in minutes
+#BSUB -G guests                  #account
+#BSUB -e ctestSerrors.txt        #stderr
+#BSUB -o ctestSoutput.txt        #stdout
+#BSUB -J hdf5_ctestS             #job
+##BSUB -q pbatch                 #queue to use
+#BSUB -q pdebug
+
+cd @HDF5_BINARY_DIR@
+echo "Run command. Test output will be in build/ctestS.out"
+ctest . -E 'MPI_TEST_' -C Release -j 32 -T test >& ctestS.out
+
+##$CMD  >& ctestS.out
+echo "Done running command."
+
diff --git a/bin/batch/raybsub b/bin/batch/raybsub
new file mode 100755
index 0000000..19dceef
--- /dev/null
+++ b/bin/batch/raybsub
@@ -0,0 +1,7 @@
+#!/bin/tcsh
+
+# ray.llnl.gov requires a '<' with bsub for submitting .lsf batch jobs.
+# CMake is reluctant to pass the '<', so we put it in this script and use
+# the script to submit the bsub command on ray.
+
+bsub < $1
diff --git a/bin/release b/bin/release
index e43be17..96c2e78 100755
--- a/bin/release
+++ b/bin/release
@@ -55,13 +55,21 @@ for compressing the resulting tar archive (if none are given then
 
     tar		-- use tar and don't do any compressing.
     gzip	-- use gzip with "-9" and append ".gz" to the output name.
+    bzip2       -- use bzip2 with "-9" and append ".bz2" to the output name.
+    zip		-- convert all text files to DOS style and form a zip file for Windows use.
     cmake-tgz   -- create a tar file using the gzip default level with a build-unix.sh 
                    command file and all other CMake files needed to build HDF5 source
                    using CMake on unix machines.
-    bzip2       -- use bzip2 with "-9" and append ".bz2" to the output name.
-    zip		-- convert all text files to DOS style and form a zip file for Windows use.
     cmake-zip   -- convert all text files to DOS style and create a zip file inluding cmake
                    scripts and .bat files to build HDF5 source using CMake on Windows. 
+    hpc-cmake-tgz  
+                -- create a tar file using the gzip default level with a build-unix.sh 
+                   command file and all other CMake files needed to build HDF5 source
+                   using CMake on unix machines, with HDF5options.cmake files for serial
+                   and parallel builds on machines requiring batch jobs to run tests.
+                   The default is for parallel build, with serial only build by changing
+                   the HDF5options.cmake symlink to ser-HDF5options.cmake.  More 
+                   information is available in the README_HPC file.
     doc         -- produce the latest doc tree in addition to the archive.
 
 An md5 checksum is produced for each archive created and stored in the md5 file.
@@ -225,7 +233,7 @@ tar2cmakezip()
     # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files
     cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmziptmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmziptmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.10.10-Source.tar.gz $cmziptmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.12.4-Source.zip $cmziptmpsubdir
     cp $cmziptmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmziptmpsubdir
     cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmziptmpsubdir
     cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmziptmpsubdir
@@ -256,7 +264,7 @@ tar2cmakezip()
 }
 
 # Function name: tar2cmaketgz
-# Convert the release tarball to a Windows zipball with files to run CMake build.
+# Convert the release tarball to a gzipped tar file with files to run CMake build.
 #
 # Programmer: Larry Knox
 # Creation date: 2017-02-20
@@ -320,7 +328,7 @@ tar2cmaketgz()
     # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files
     cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmgztmpsubdir
     cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir
-    cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.10.10-Source.tar.gz $cmgztmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.12.4-Source.tar.gz $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir
     cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpsubdir
@@ -330,6 +338,93 @@ tar2cmaketgz()
     rm -rf $cmgztmpdir
 }
 
+# Function name: tar2hpccmaketgz
+# Convert the release tarball to a gzipped tarfile with files to run CMake build
+# and HDF5options.cmake files for parallel or serial only builds where build
+# tests are run on compute nodes using batch scripts.
+#
+# Programmer: Larry Knox
+# Creation date: 2019-01-28
+#
+# Modifications
+#
+# Steps:
+# 1. untar the tarball in a temporary directory;
+#    Note: do this in a temporary directory to avoid changing
+#    the original source directory which may be around.
+# 2. add build-unix.sh script.
+# 3. add SZIP.tar.gz, ZLib.tar.gz and cmake files to top level directory.
+# 4. create gzipped tar file with these contents:
+#        build-unix.sh          script
+#        hdf5-<version>         source code directory extracted from tar file
+#        CTestScript.cmake      cmake file copied from <hdf5 source code>/config/cmake/scripts
+#        HDF5config.cmake       cmake file copied from <hdf5 source code>/config/cmake/scripts
+#        HDF5options.cmake      cmake file copied from <hdf5 source code>/config/cmake/scripts
+#        SZip.tar.gz            copied from /mnt/scr1/pre-release/hdf5/CMake
+#        ZLib.tar.gz            copied from /mnt/scr1/pre-release/hdf5/CMake
+#
+# 5. For HPC-CMake tgz file the following are also needed in the top-level directory:
+#        README_HPC                 copied from release_docs
+#        ser-HDF5options.cmake      copied from <hdf5 source code>/config/cmake/scripts/HPC
+#        par-HDF5options.cmake      copied from <hdf5 source code>/config/cmake/scripts/HPC
+#        HDF5options.cmake          symlink to par-HDF5options.cmake
+#
+
+# Parameters:
+# $1 version
+# $2 release tarball
+# $3 output zipball file name
+#
+# Returns 0 if successful; 1 otherwise
+#
+ # need function to create another temporary directory, extract the
+            # $tmpdir/$HDF5_VERS.tar into it, create build-unix.sh,
+            # add CTestScript.cmake, HDF5config.cmake, SZIP.tar.gz
+            # ZLib.tar.gz, HDF5 examples, and then tar.gz it.
+tar2hpccmaketgz()
+{
+    if [ $# -ne 3 ]; then
+	echo "usage: tar2hpccmaketgz <tarfilename> <tgzfilename>"
+	return 1
+    fi
+    cmgztmpdir=/tmp/cmgztmpdir$$
+    cmgztmpsubdir=$cmgztmpdir/HPC-CMake-$HDF5_VERS
+    mkdir -p $cmgztmpsubdir
+    version=$1
+    tarfile=$2
+    tgzfile=$3
+
+    # step 1: untar tarball in cmgztmpdir
+    (cd $cmgztmpsubdir; tar xf -) < $tarfile
+    # sanity check
+    if [ ! -d $cmgztmpsubdir/$version ]; then
+	echo "untar did not create $cmgztmpsubdir/$version source dir"
+	# cleanup
+	rm -rf $cmgztmpdir
+	return 1
+    fi
+
+
+    # step 2: add build-unix.sh script
+    (cd $cmgztmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log" > build-unix.sh; chmod 755 build-unix.sh)
+
+    # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files
+    cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmgztmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir
+    cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.12.4-Source.tar.gz $cmgztmpsubdir
+    cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir
+    cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir
+
+    cp $cmgztmpsubdir/$version/release_docs/README_HPC $cmgztmpsubdir
+    cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/ser-HDF5options.cmake $cmgztmpsubdir
+    cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/par-HDF5options.cmake $cmgztmpsubdir
+    (cd $cmgztmpsubdir; ln -s par-HDF5options.cmake HDF5options.cmake)
+    tar czf $DEST/HPC-CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 
+                 
+    # cleanup
+    rm -rf $cmgztmpdir
+}
+
 # This command must be run at the top level of the hdf5 source directory.
 # Verify this requirement.
 if [ ! \( -f configure.ac -a -f bin/release \) ]; then
@@ -514,6 +609,11 @@ for comp in $methods; do
 	    tar2cmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/CMake-$HDF5_VERS.tar.gz 1>&2
 	    (cd $DEST; md5sum CMake-$HDF5_VERS.tar.gz >> $MD5file) 
             ;;
+        hpc-cmake-tgz)
+	    test "$verbose" && echo "   Creating HPC-CMake tar.gz file..." 1>&2
+	    tar2hpccmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/HPC-CMake-$HDF5_VERS.tar.gz 1>&2
+	    (cd $DEST; md5sum HPC-CMake-$HDF5_VERS.tar.gz >> $MD5file) 
+            ;;
 	bzip2)
 	    test "$verbose" && echo "   Running bzip2..." 1>&2
 	    bzip2 -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.bz2