From be7382c2e2bbebb8677dbe399743412e1856533e Mon Sep 17 00:00:00 2001 From: Larry Knox Date: Fri, 15 Feb 2019 14:13:50 -0600 Subject: Add release function to create HPC-CMake tar.gz file. --- bin/release | 324 +++++++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 212 insertions(+), 112 deletions(-) diff --git a/bin/release b/bin/release index 3d1cb26..96c2e78 100755 --- a/bin/release +++ b/bin/release @@ -39,29 +39,37 @@ USAGE() { cat << EOF Usage: $0 -d [--docver BRANCHNAME] [-h] [--nocheck] [--private] ... - -d DIR The name of the directory where the releas(es) should be + -d DIR The name of the directory where the releas(es) should be placed. --docver BRANCHNAME This is added for 1.8 and beyond to get the correct version of documentation files from the hdf5docs repository. BRANCHNAME for v1.8 should be hdf5_1_8. -h print the help page. - --nocheck Ignore errors in MANIFEST file. - --private Make a private release with today's date in version information. - + --nocheck Ignore errors in MANIFEST file. + --private Make a private release with today's date in version information. + This must be run at the top level of the source directory. The other command-line options are the names of the programs to use for compressing the resulting tar archive (if none are given then "tar" is assumed): - tar -- use tar and don't do any compressing. - gzip -- use gzip with "-9" and append ".gz" to the output name. - cmake-tgz -- create a tar file using the gzip default level with a build-unix.sh + tar -- use tar and don't do any compressing. + gzip -- use gzip with "-9" and append ".gz" to the output name. + bzip2 -- use bzip2 with "-9" and append ".bz2" to the output name. + zip -- convert all text files to DOS style and form a zip file for Windows use. + cmake-tgz -- create a tar file using the gzip default level with a build-unix.sh command file and all other CMake files needed to build HDF5 source using CMake on unix machines. - bzip2 -- use bzip2 with "-9" and append ".bz2" to the output name. - zip -- convert all text files to DOS style and form a zip file for Windows use. cmake-zip -- convert all text files to DOS style and create a zip file inluding cmake - scripts and .bat files to build HDF5 source using CMake on Windows. + scripts and .bat files to build HDF5 source using CMake on Windows. + hpc-cmake-tgz + -- create a tar file using the gzip default level with a build-unix.sh + command file and all other CMake files needed to build HDF5 source + using CMake on unix machines, with HDF5options.cmake files for serial + and parallel builds on machines requiring batch jobs to run tests. + The default is for parallel build, with serial only build by changing + the HDF5options.cmake symlink to ser-HDF5options.cmake. More + information is available in the README_HPC file. doc -- produce the latest doc tree in addition to the archive. An md5 checksum is produced for each archive created and stored in the md5 file. @@ -114,8 +122,8 @@ EOF tar2zip() { if [ $# -ne 3 ]; then - echo "usage: tar2zip " - return 1 + echo "usage: tar2zip " + return 1 fi ztmpdir=/tmp/ztmpdir$$ mkdir -p $ztmpdir @@ -127,23 +135,23 @@ tar2zip() (cd $ztmpdir; tar xf -) < $tarfile # sanity check if [ ! -d $ztmpdir/$version ]; then - echo "untar did not create $ztmpdir/$version source dir" - # cleanup - rm -rf $ztmpdir - return 1 + echo "untar did not create $ztmpdir/$version source dir" + # cleanup + rm -rf $ztmpdir + return 1 fi # step 2: convert text files # There maybe a simpler way to do this. # options used in unix2dos: - # -k Keep the date stamp + # -k Keep the date stamp # -q quiet mode # grep redirect output to /dev/null because -q or -s are not portable. find $ztmpdir/$version | \ - while read inf; do \ - if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ - unix2dos -q -k $inf; \ - fi\ - done + while read inf; do \ + if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ + unix2dos -q -k $inf; \ + fi\ + done # step 3: make zipball # -9 maximum compression # -y Store symbolic links as such in the zip archive @@ -194,8 +202,8 @@ tar2zip() tar2cmakezip() { if [ $# -ne 3 ]; then - echo "usage: tar2cmakezip " - return 1 + echo "usage: tar2cmakezip " + return 1 fi cmziptmpdir=/tmp/cmziptmpdir$$ cmziptmpsubdir=$cmziptmpdir/CMake-$HDF5_VERS @@ -208,10 +216,10 @@ tar2cmakezip() (cd $cmziptmpsubdir; tar xf -) < $tarfile # sanity check if [ ! -d $cmziptmpsubdir/$version ]; then - echo "untar did not create $cmziptmpsubdir/$version source dir" - # cleanup - rm -rf $cmziptmpdir - return 1 + echo "untar did not create $cmziptmpsubdir/$version source dir" + # cleanup + rm -rf $cmziptmpdir + return 1 fi # step 2: add batch file for building CMake on window @@ -225,7 +233,7 @@ tar2cmakezip() # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmziptmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmziptmpsubdir - cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.10.13-Source.tar.gz $cmziptmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.12.4-Source.zip $cmziptmpsubdir cp $cmziptmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmziptmpsubdir cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmziptmpsubdir cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmziptmpsubdir @@ -233,15 +241,15 @@ tar2cmakezip() # step 4: convert text files # There maybe a simpler way to do this. # options used in unix2dos: - # -k Keep the date stamp + # -k Keep the date stamp # -q quiet mode # grep redirect output to /dev/null because -q or -s are not portable. find $cmziptmpsubdir/$version | \ - while read inf; do \ - if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ - unix2dos -q -k $inf; \ - fi\ - done + while read inf; do \ + if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ + unix2dos -q -k $inf; \ + fi\ + done # step 3: make zipball # -9 maximum compression @@ -256,7 +264,7 @@ tar2cmakezip() } # Function name: tar2cmaketgz -# Convert the release tarball to a Windows zipball with files to run CMake build. +# Convert the release tarball to a gzipped tar file with files to run CMake build. # # Programmer: Larry Knox # Creation date: 2017-02-20 @@ -293,8 +301,8 @@ tar2cmakezip() tar2cmaketgz() { if [ $# -ne 3 ]; then - echo "usage: tar2cmaketgz " - return 1 + echo "usage: tar2cmaketgz " + return 1 fi cmgztmpdir=/tmp/cmgztmpdir$$ cmgztmpsubdir=$cmgztmpdir/CMake-$HDF5_VERS @@ -307,10 +315,93 @@ tar2cmaketgz() (cd $cmgztmpsubdir; tar xf -) < $tarfile # sanity check if [ ! -d $cmgztmpsubdir/$version ]; then - echo "untar did not create $cmgztmpsubdir/$version source dir" + echo "untar did not create $cmgztmpsubdir/$version source dir" + # cleanup + rm -rf $cmgztmpdir + return 1 + fi + + + # step 2: add build-unix.sh script + (cd $cmgztmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log" > build-unix.sh; chmod 755 build-unix.sh) + + # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files + cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.12.4-Source.tar.gz $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpsubdir + tar czf $DEST/CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 + # cleanup rm -rf $cmgztmpdir - return 1 +} + +# Function name: tar2hpccmaketgz +# Convert the release tarball to a gzipped tarfile with files to run CMake build +# and HDF5options.cmake files for parallel or serial only builds where build +# tests are run on compute nodes using batch scripts. +# +# Programmer: Larry Knox +# Creation date: 2019-01-28 +# +# Modifications +# +# Steps: +# 1. untar the tarball in a temporary directory; +# Note: do this in a temporary directory to avoid changing +# the original source directory which may be around. +# 2. add build-unix.sh script. +# 3. add SZIP.tar.gz, ZLib.tar.gz and cmake files to top level directory. +# 4. create gzipped tar file with these contents: +# build-unix.sh script +# hdf5- source code directory extracted from tar file +# CTestScript.cmake cmake file copied from /config/cmake/scripts +# HDF5config.cmake cmake file copied from /config/cmake/scripts +# HDF5options.cmake cmake file copied from /config/cmake/scripts +# SZip.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake +# ZLib.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake +# +# 5. For HPC-CMake tgz file the following are also needed in the top-level directory: +# README_HPC copied from release_docs +# ser-HDF5options.cmake copied from /config/cmake/scripts/HPC +# par-HDF5options.cmake copied from /config/cmake/scripts/HPC +# HDF5options.cmake symlink to par-HDF5options.cmake +# + +# Parameters: +# $1 version +# $2 release tarball +# $3 output zipball file name +# +# Returns 0 if successful; 1 otherwise +# + # need function to create another temporary directory, extract the + # $tmpdir/$HDF5_VERS.tar into it, create build-unix.sh, + # add CTestScript.cmake, HDF5config.cmake, SZIP.tar.gz + # ZLib.tar.gz, HDF5 examples, and then tar.gz it. +tar2hpccmaketgz() +{ + if [ $# -ne 3 ]; then + echo "usage: tar2hpccmaketgz " + return 1 + fi + cmgztmpdir=/tmp/cmgztmpdir$$ + cmgztmpsubdir=$cmgztmpdir/HPC-CMake-$HDF5_VERS + mkdir -p $cmgztmpsubdir + version=$1 + tarfile=$2 + tgzfile=$3 + + # step 1: untar tarball in cmgztmpdir + (cd $cmgztmpsubdir; tar xf -) < $tarfile + # sanity check + if [ ! -d $cmgztmpsubdir/$version ]; then + echo "untar did not create $cmgztmpsubdir/$version source dir" + # cleanup + rm -rf $cmgztmpdir + return 1 fi @@ -320,12 +411,16 @@ tar2cmaketgz() # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmgztmpsubdir cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir - cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.10.13-Source.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.12.4-Source.tar.gz $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir - cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpsubdir - tar czf $DEST/CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 + cp $cmgztmpsubdir/$version/release_docs/README_HPC $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/ser-HDF5options.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/par-HDF5options.cmake $cmgztmpsubdir + (cd $cmgztmpsubdir; ln -s par-HDF5options.cmake HDF5options.cmake) + tar czf $DEST/HPC-CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 + # cleanup rm -rf $cmgztmpdir } @@ -347,7 +442,7 @@ check=yes release_date=`date +%F` today=`date +%Y%m%d` pmode='no' -tmpdir="../#release_tmp.$$" # tmp work directory +tmpdir="../#release_tmp.$$" # tmp work directory DOC_URL=https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5doc.git CPPLUS_RM_NAME=cpplus_RM MAINT_MODE_ENABLED="" @@ -364,11 +459,11 @@ fi RESTORE_VERSION() { if [ X-${VERS_OLD} != X- ]; then - echo restoring version information back to $VERS_OLD - rm -f config/lt_vers.am - cp $tmpdir/lt_vers.am config/lt_vers.am - bin/h5vers -s $VERS_OLD - VERS_OLD= + echo restoring version information back to $VERS_OLD + rm -f config/lt_vers.am + cp $tmpdir/lt_vers.am config/lt_vers.am + bin/h5vers -s $VERS_OLD + VERS_OLD= fi } @@ -378,32 +473,32 @@ while [ -n "$1" ]; do arg=$1 shift case "$arg" in - -d) - DEST=$1 - shift - ;; - --nocheck) - check=no - ;; - -h) - USAGE - exit 0 - ;; - --private) - pmode=yes - ;; + -d) + DEST=$1 + shift + ;; + --nocheck) + check=no + ;; + -h) + USAGE + exit 0 + ;; + --private) + pmode=yes + ;; --docver) DOCVERSION=$1 shift ;; - -*) - echo "Unknown switch: $arg" 1>&2 - USAGE - exit 1 - ;; - *) - methods="$methods $arg" - ;; + -*) + echo "Unknown switch: $arg" 1>&2 + USAGE + exit 1 + ;; + *) + methods="$methods $arg" + ;; esac done @@ -453,12 +548,12 @@ if [ "X$fail" = "Xyes" ]; then echo "" echo "Note! If you are running bin/release in a development branch" echo "later than v 1.8 the MANIFEST check is expected to fail when" - echo "autogen.sh has not been run successfully. Either run autogen.sh " + echo "autogen.sh has not been run successfully. Either run autogen.sh " echo "with /usr/hdf/bin/AUTOTOOLS at the beginning of PATH or add the" echo "--nocheck argument to the bin/release command." exit 1 else - echo "Continuing anyway..." + echo "Continuing anyway..." fi fi @@ -492,7 +587,7 @@ test "$verbose" && echo " Running tar..." 1>&2 ( \ cd $tmpdir; \ tar cf $HDF5_VERS.tar $HDF5_VERS/Makefile \ - `sed 's/^\.\//hdf5-'$VERS'\//' $MANIFEST` || exit 1 \ + `sed 's/^\.\//hdf5-'$VERS'\//' $MANIFEST` || exit 1 \ ) # Compress @@ -500,56 +595,61 @@ MD5file=$HDF5_VERS.md5 cp /dev/null $DEST/$MD5file for comp in $methods; do case $comp in - tar) - cp -p $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.tar - (cd $DEST; md5sum $HDF5_VERS.tar >> $MD5file) - ;; - gzip) - test "$verbose" && echo " Running gzip..." 1>&2 - gzip -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.gz - (cd $DEST; md5sum $HDF5_VERS.tar.gz >> $MD5file) - ;; + tar) + cp -p $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.tar + (cd $DEST; md5sum $HDF5_VERS.tar >> $MD5file) + ;; + gzip) + test "$verbose" && echo " Running gzip..." 1>&2 + gzip -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.gz + (cd $DEST; md5sum $HDF5_VERS.tar.gz >> $MD5file) + ;; cmake-tgz) - test "$verbose" && echo " Creating CMake tar.gz file..." 1>&2 - tar2cmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/CMake-$HDF5_VERS.tar.gz 1>&2 - (cd $DEST; md5sum CMake-$HDF5_VERS.tar.gz >> $MD5file) + test "$verbose" && echo " Creating CMake tar.gz file..." 1>&2 + tar2cmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/CMake-$HDF5_VERS.tar.gz 1>&2 + (cd $DEST; md5sum CMake-$HDF5_VERS.tar.gz >> $MD5file) ;; - bzip2) - test "$verbose" && echo " Running bzip2..." 1>&2 - bzip2 -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.bz2 - (cd $DEST; md5sum $HDF5_VERS.tar.bz2 >> $MD5file) - ;; - zip) - test "$verbose" && echo " Creating zip ball..." 1>&2 - tar2zip $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.zip 1>&2 - (cd $DEST; md5sum $HDF5_VERS.zip >> $MD5file) - ;; + hpc-cmake-tgz) + test "$verbose" && echo " Creating HPC-CMake tar.gz file..." 1>&2 + tar2hpccmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/HPC-CMake-$HDF5_VERS.tar.gz 1>&2 + (cd $DEST; md5sum HPC-CMake-$HDF5_VERS.tar.gz >> $MD5file) + ;; + bzip2) + test "$verbose" && echo " Running bzip2..." 1>&2 + bzip2 -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.bz2 + (cd $DEST; md5sum $HDF5_VERS.tar.bz2 >> $MD5file) + ;; + zip) + test "$verbose" && echo " Creating zip ball..." 1>&2 + tar2zip $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.zip 1>&2 + (cd $DEST; md5sum $HDF5_VERS.zip >> $MD5file) + ;; cmake-zip) test "$verbose" && echo " Creating CMake-zip ball..." 1>&2 tar2cmakezip $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/CMake-$HDF5_VERS.zip 1>&2 (cd $DEST; md5sum CMake-$HDF5_VERS.zip >> $MD5file) ;; - doc) + doc) if [ "${DOCVERSION}" = "" ]; then DOCVERSION=master fi - test "$verbose" && echo " Creating docs..." 1>&2 - # Check out docs from git repo - (cd $tmpdir; git clone -q $DOC_URL ${DOCVERSION} > /dev/null) || exit 1 + test "$verbose" && echo " Creating docs..." 1>&2 + # Check out docs from git repo + (cd $tmpdir; git clone -q $DOC_URL ${DOCVERSION} > /dev/null) || exit 1 # Create doxygen C++ RM - (cd c++/src && doxygen cpp_doc_config > /dev/null ) || exit 1 - # Replace version of C++ RM with just-created version - rm -rf $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME || exit 1 - mv c++/src/$CPPLUS_RM_NAME $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME || exit 1 + (cd c++/src && doxygen cpp_doc_config > /dev/null ) || exit 1 + # Replace version of C++ RM with just-created version + rm -rf $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME || exit 1 + mv c++/src/$CPPLUS_RM_NAME $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME || exit 1 # Compress the docs and move them to the release area - mv $tmpdir/${DOCVERSION} $tmpdir/${HDF5_VERS}_docs || exit 1 - (cd $tmpdir && tar cf ${HDF5_VERS}_docs.tar ${HDF5_VERS}_docs) || exit 1 - mv $tmpdir/${HDF5_VERS}_docs.tar $DEST || exit 1 - ;; - *) - echo "***Error*** Unknown method $comp" - exit 1 - ;; + mv $tmpdir/${DOCVERSION} $tmpdir/${HDF5_VERS}_docs || exit 1 + (cd $tmpdir && tar cf ${HDF5_VERS}_docs.tar ${HDF5_VERS}_docs) || exit 1 + mv $tmpdir/${HDF5_VERS}_docs.tar $DEST || exit 1 + ;; + *) + echo "***Error*** Unknown method $comp" + exit 1 + ;; esac done -- cgit v0.12