#!/bin/sh # # Copyright by The HDF Group. # Copyright by the Board of Trustees of the University of Illinois. # All rights reserved. # # This file is part of HDF5. The full HDF5 copyright notice, including # terms governing use, modification, and redistribution, is contained in # the COPYING file, which can be found at the root of the source code # distribution tree, or in https://www.hdfgroup.org/licenses. # If you do not have access to either file, you may request a copy from # help@hdfgroup.org. # This script should be run nightly from cron. It checks out the source # from the source repository and compares it against the previous # snapshot. If anything significant changed then a new snapshot is # created, the minor version number is incremented, and the change is # checked back into the source repository. # # function definitions TIMESTAMP() { echo "=====" "$1": "`date`" "=====" } EXIT_BANNER() { TIMESTAMP "Exit $PROGNAME with status=$?" } # Show current total disk usage. DISKUSAGE() { du -ks | \ ( read x y; echo "Disk Usage=$x KB" ) } # function provided for testing software downloaded as tar files. A version of # this function that properly extracts the downloaded files can be provided in # the snapshots-${sw}-overrides file. EXTRACT() { echo "Error: ${SWVERSION} is in source repository - does not need extraction." } # Standard procedure for checking out or updating source code from an hdfgroup # git repository. Override the function for other repositories or procedures. SOURCE_CHECKOUT() { if test -n $GIT_URL; then if [ -n "$AUTOGEN" ]; then echo "Creating fresh clone of $GIT_URL in $BASEDIR/current_src" # Check out the current version from source repository. (cd $BASEDIR; rm -rf current_src if test -z $GIT_BRANCH; then echo "Testing empty branch $GIT_BRANCH." git clone $GIT_URL current_src else echo "Testing branch $GIT_BRANCH." git clone $GIT_URL -b $GIT_BRANCH current_src fi ) || exit 1 else echo "Creating fresh clone of $GIT_URL in $BASEDIR/current" # Check out the current version from source repository. (cd $BASEDIR; rm -rf current if test -n $GIT_BRANCH; then git clone $GIT_URL -b $GIT_BRANCH current else git clone $GIT_URL current fi ) || exit 1 fi else echo "Warning! Source directory ("current") is not checked out from git." fi } # Standard procedure for running the configure command in a build (test) # directory RUNCONFIGURE() { if [ "${CURRENT}" != "${TESTDIR}" -a "$CPSRC" = "yes" ]; then echo "Copying source files to ${TESTDIR}." cp -pr ${CURRENT}/* ${TESTDIR} cd ${TESTDIR} ./${CONFIGURE} elif [ -n "${AUTOGEN}" ]; then ${CURRENTSRC}/${CONFIGURE} else ${CURRENT}/${CONFIGURE} fi } # Sometimes "make distclean" doesn't adequately remove files from the previous # build. If a build (test) directory was used, its contents can be entirely # deleted to provide a clean start. If the test is building in the source # directory, the contents can't be deleted, so run "make distclean". DISTCLEAN() { if [ "${srcdir}" = "yes" -a -n "${SRCDIRNAME}" -a -d ${BASEDIR}/TestDir/${SRCDIRNAME} ]; then echo "Remove contents of $SRCDIRNAME.\n" rm -rf ${BASEDIR}/TestDir/${SRCDIRNAME}/* else echo "$MAKE distclean" (cd ${TESTDIR} && ${MAKE} distclean) fi } # Several of the software packages tested do not support make check-install. # Those that support it should have a version of this function in their # override with the following lines: # TIMESTAMP "check-install $1" # ${MAKE} check-install $1 CHECKINSTALL() { echo "check-install is not supported for ${SWVERSION}" } # Function for hdf4 and hdf5 to override to check in changes after snapshot. # Safety measure to avoid unintended checkins to other repositories. COMMITSNAPSHOT() { echo "original hdf5 script committed code changes back into git." } DISPLAYUSAGE() { set - cat <<EOF Usage: $PROGNAME [all] [checkout] [ftp <URL> [diff] [test] [srcdir] [release] [help] [clean] [distclean] [echo] [deploy <dir>] [deploydir <dir>] [zlib <zlib_path>] [releasedir <dir>] [srcdirname <dir>] [check-vfd] [check-passthrough-vol] [exec <command>] [module-load <module-list>] [op-configure <option>] [--<option>] all: Run all commands (checkout, test & release) [Default is all] checkout: Run source checkout diff: Run diff on current and previous versions. Exit 0 if no significant differences are found. Otherwise, non-zero. deploy: deploy binary to directory <dir> deploydir: use <dir> as the default directory for deployment test: Run test release: Run release clean: Run make clean distclean:Run make distclean echo: Turn on echo mode (set -x) setenv <name> <value>: Set environment variable <name> to <value>. setenvN <N> <name> <value> ...: Set environment variable with <N> values. E.g., setenvN 3 x a b c is same as setenv x="a b c". srcdir: Use srcdir option (does not imply other commands) "snapshot srcdir" is equivalent to "snapshot srcdir all" "snapshot srcdir checkout" is equivalent to "snapshot checkout" srcdirname <dir>: Use <dir> as the srcdir testing directory if srcdir is choosen. If <dir> starts with '-', it is append to the default name E.g., "snapshot srcdir srcdirname -xx" uses hostname-xx [Default is hostname] help: Print this message echo: Turn on shell echo zlib <zlib_path>: Use <zlib_path> as the ZLIB locations [Default is $ZLIB_default] releasedir <dir>: Use <dir> as the release directory [Default is $ReleaseDir_default] check-vfd: Run make check-vfd instead of just make check. check-passthrough-vol: Run make check-passthrough-vol instead of just make check. NOTE: Will only succeed with passthrough VOL connectors that use the native VOL connector as the terminal connector. exttest <testscript>; Run testscript; exec <command>: Run <command>; module-load <module-list>: Load modules in comma-separated <module-list>; op-configure <option>: Pass <option> to the configure command E.g., "snapshot op-configure --enable-parallel" configures for parallel mode --<option>: Pass --<option> to the configure command E.g., "snapshot --enable-parallel" configures for parallel mode EOF exit $errcode } # MAIN # SGI /bin/sh replaces $0 as function name if used in a function. # Set the name here to avoid that ambiguity and better style too. PROGNAME=$0 SNAPSHOTNAME= HDFREPOS= DOCVERSION="" MODULELIST="" if [ -f bin/snapshot_params ]; then . bin/snapshot_params echo "Added snapshot_params." fi if [ -z "$SWVER" -a -f bin/snapshot_version ] then . bin/snapshot_version echo "Added snapshot_version." fi if [ -n ${HDFREPOS} -a -f bin/snapshot-${HDFREPOS}-overrides ]; then . bin/snapshot-${HDFREPOS}-overrides echo "Added snapshot-${HDFREPOS}-overrides." fi echo "=====================================" echo "$PROGNAME $*" echo "=====================================" TIMESTAMP MAIN uname -a # setup exit banner message trap EXIT_BANNER 0 1 2 9 15 # Dump environment variables before option parsing echo ===Dumping environment variables before option parsing === printenv | sort echo ===Done Dumping environment variables before option parsing === # snapshots release directory. Default relative to $BASEDIR. ReleaseDir_default=release_dir # Where is the zlib library? # At NCSA, half of the machines have it in /usr/lib, the other half at # /usr/ncsa/lib. Leave it unset. ZLIB_default= ZLIB=$ZLIB_default # What compression methods to use? (md5 does checksum). Doc was apparently # added as a compression method to create a separate tarfile containing the # documentation files for v 1.8 and above. if [ "${SWVERSION}" = "hdf5_1_6" ]; then METHODS="gzip bzip2 md5" else METHODS="gzip bzip2 doc" fi # Use User's MAKE if set. Else use generic make. MAKE=${MAKE:-make} # Default check action. CHECKVAL=check # # Command options cmd="all" test_opt="" errcode=0 AUTOGEN="" EXTTEST="" EXEC_CMD_ARG="" while [ $# -gt 0 ] ; do case "$1" in all) cmd="all" ;; checkout-autogen) cmdcheckout="checkout" AUTOGEN="autogen" cmd="" ;; checkout) cmdcheckout="checkout" cmd="" ;; ftp) echo "Setting ftp flags in snapshot script" cmdcheckout="checkout" cmdftp="ftp" cmd="" shift if [ $# -lt 1 ]; then echo "URL missing" errcode=1 cmd="help" break fi ftp_url="$1" echo "ftp_url is $ftp_url" ;; diff) cmddiff="diff" cmd="" ;; deploy) # deploy the built binary. shift if [ $# -lt 1 ]; then echo "deploy <dir> missing" errcode=1 cmd="help" break fi cmddeploy="deploy" DEPLOYDIRNAME="$1" ;; deploydir) # default directory for deployment. shift if [ $# -lt 1 ]; then echo "deploydir <dir> missing" errcode=1 cmd="help" break fi deploydir="$1" ;; test) cmdtest="test" cmd="" ;; setenv) # set environment variable shift eval $1="$2" export $1 shift ;; setenvN) # set environment variable with $1 values # e.g., setenvN 3 x a b c is same as setenv x="a b c". # a kludge now--the extra single quotes are needed # else eval complains. shift envN=$1 shift envname=$1 envalue= while test $envN -gt 0; do shift envalue="$envalue $1" envN=`expr $envN - 1` done eval $envname="'$envalue'" export $envname ;; srcdir) #use srcdir option for test srcdir="yes" ;; srcdirname) shift if [ $# -lt 1 ]; then echo "srcdirname <dir> missing" errcode=1 cmd="help" break fi SRCDIRNAME="$1" ;; release) cmdrel="release" cmd="" ;; autogen-release) cmdrel="autogen-release" cmd="" ;; clean | distclean) cmdclean="$1" cmd="" ;; help) cmd="help" break ;; echo) set -x break ;; zlib) shift if [ $# -lt 1 ]; then echo "ZLIB information missing" errcode=1 cmd="help" break fi ZLIB="$1" ;; releasedir) shift if [ $# -lt 1 ]; then echo "Release directory name missing" errcode=1 cmd="help" break fi ReleaseDir="$1" ;; exttest) shift if [ $# -lt 1 ]; then echo "exttest script name missing" errcode=1 cmd="help" break fi cmd="" EXTTEST="$1" ;; exec) shift if [ $# -lt 1 ]; then echo "exec command name missing" errcode=1 cmd="help" break fi cmd="" EXEC_CMD_ARG="$@" # exit the parsing while loop since all arguments have been consummed. break ;; check-vfd) CHECKVAL=check-vfd ;; check-passthrough-vol) CHECKVAL=check-passthrough-vol ;; module-load) shift if [ $# -lt 1 ]; then echo "missing module list to load" errcode=1 cmd="help" break fi MODULELIST="$1" ;; --*) OP_CONFIGURE="$OP_CONFIGURE $1" ;; op-configure) shift if [ $# -lt 1 ]; then echo "op-configure option missing" errcode=1 cmd="help" break fi OP_CONFIGURE="$OP_CONFIGURE $1" ;; *) echo "Unkown option $1" errcode=1 cmd="help" break ;; esac shift done if [ -n "$MODULELIST" ]; then . ~/.bashrc module use /opt/pkgs/modules/all # load module command will take a space separated list of modules. # If we have a comma separated list, convert ',' to ' '. MODULELIST="$( echo -e "$MODULELIST" | tr ',' ' ' )" module load $MODULELIST fi # Dump environment variables after option parsing echo ===Dumping environment variables after option parsing === printenv | sort echo ===Done Dumping environment variables after option parsing === if [ "$cmd" = help ]; then DISPLAYUSAGE fi # Setup the proper configure option (--with-zlib) to use zlib library # provide ZLIB is non-empty. ZLIB=${ZLIB:+"--with-zlib="$ZLIB} # Adding --prefix as a configure option will put the path to the deploy # directory in the initial libhdf5*.la files if [ -n "$DEPLOYDIRNAME" ]; then OP_CONFIGURE="$OP_CONFIGURE --prefix=${deploydir}/${DEPLOYDIRNAME}" fi CONFIGURE="configure $OP_CONFIGURE" # echo "Configure command is $CONFIGURE" # Execute the requests snapshot=yes BASEDIR=${HOME}/snapshots-${SNAPSHOTNAME} if [ ! -d ${BASEDIR} ]; then echo "BASEDIR ($BASEDIR) does not exist" exit 1 fi CURRENT=${BASEDIR}/current PREVIOUS=${BASEDIR}/previous ReleaseDir=${ReleaseDir:=${BASEDIR}/${ReleaseDir_default}} HOSTNAME=`hostname | cut -f1 -d.` # no domain part # Try finding a version of diff that supports the -I option too. DIFF=diff for d in `echo $PATH | sed -e 's/:/ /g'` ; do test -x $d/diff && $d/diff -I XYZ /dev/null /dev/null > /dev/null 2>&1 && DIFF=$d/diff && break done #============================= # Run source checkout #============================= if [ "$cmd" = "all" -o -n "$cmdcheckout" ]; then TIMESTAMP "checkout" # ${BASEDIR}/bin is now updated from git by EveningMaint or DailyMaint # to avoid updating the scripts in ${BASEDIR}/bin while they are running. if [ -z "$AUTOGEN" ]; then # If there is a Makefile in ${CURRENT}, the last test done in it # has not been distclean'ed. They would interfere with other # --srcdir build since make considers the files in ${CURRENT} # take precedence over files in its own build-directory. Run # a "make distclean" to clean them all out. This is not really # part of the "checkout" functions but this is the most convenient # spot to do the distclean. We will also continue the checkout process # regardless of the return code of distclean. ( cd ${CURRENT}; test -f Makefile && ${MAKE} distclean) fi # echo "cmdftp is $cmdftp; ftp_url is $ftp_url" if [ -n "$cmdftp" ]; then echo "Get the NetCDF4 source from their ftp server." echo "Command executed is: 2>&1 wget -N $ftp_url" cd ${BASEDIR}; WGET_OUTPUT="`2>&1 wget -N $ftp_url`" errcode=$? if [[ $errcode -ne 0 ]]; then exit $errcode fi if [ $? -ne 0 ];then echo $0: "$WGET_OUTPUT" Exiting. exit 1 fi # echo "Wget output was $WGET_OUTPUT" if echo "$WGET_OUTPUT" | fgrep 'not retrieving' &> /dev/null then echo "Snapshot unchanged" else echo "New snapshot downloaded" EXTRACT fi else SOURCE_CHECKOUT fi fi # Do source checkout #============================= # Run Test the HDF5 library #============================= if [ "$cmd" = "all" -o -n "$cmdtest" -o -n "$cmddiff" ]; then TIMESTAMP "Run Tests" # setup if srcdir is used. if [ -z "$srcdir" ]; then TESTDIR=${CURRENT} else #create TESTDIR if not exist yet case "$SRCDIRNAME" in "") SRCDIRNAME=$HOSTNAME ;; -*) SRCDIRNAME="$HOSTNAME$SRCDIRNAME" ;; esac TESTDIR=${BASEDIR}/TestDir/${SRCDIRNAME} test -d ${TESTDIR} || mkdir ${TESTDIR} # set TESTDIR to use the direct path to the local test directory # rather than the path through ${BASEDIR}. cd ${TESTDIR} TESTDIR=`pwd -P` cd ${CURRENT} fi # Make sure current version exists and is clean if [ -d ${TESTDIR} ]; then DISTCLEAN else errcode=$? snapshot=no exit $errcode fi # Compare it with the previous version. Compare only files listed in # the MANIFEST plus the MANIFEST itself. if [ -d ${PREVIOUS} ]; then if [ -z "${AUTOGEN}" ]; then CURRENTSRC=${CURRENT} else CURRENTSRC=${BASEDIR}/current_src fi if (${DIFF} -c ${PREVIOUS}/MANIFEST ${CURRENTSRC}/MANIFEST); then snapshot=no for src in `grep '^\.' ${CURRENTSRC}/MANIFEST|expand|cut -f1 -d' '`; do if ${DIFF} -I H5_VERS_RELEASE -I " released on " \ -I " currently under development" \ ${PREVIOUS}/$src ${CURRENTSRC}/$src then : #continue else snapshot=yes break fi done fi fi # if diff is choosen, exit 0 if no significant differences are found. # otherwise, exit 1. This includes cases of other failures. if [ -n "$cmddiff" ]; then if [ $snapshot = no ]; then exit 0 else exit 1 fi fi #============================= # Execute command if defined #============================= #echo BEFORE EXEC command #echo EXEC_CMD_ARG=${EXEC_CMD_ARG} if [ -n "$EXEC_CMD_ARG" ]; then TIMESTAMP ${EXEC_CMD_ARG} TESTDIR=${BASEDIR}/TestDir/${SRCDIRNAME} test -d ${TESTDIR} || mkdir ${TESTDIR} if cd ${TESTDIR}; then # clean up the directory before executing the command # Do we need to clean first? # rm -rf * # # If EXEC_CMD_ARG starts with a '/', it has an absolute path, else it is # relative to the BASEDIR. case "$EXEC_CMD_ARG" in /*) ${EXEC_CMD_ARG} ;; *) ${BASEDIR}/${EXEC_CMD_ARG} ;; esac errcode=$? else echo "${TESTDIR} not accessible" errcode=1 fi # exit snapshot since nothing else to do, for now. exit $errcode fi # Build, run tests and install procedures if [ "$snapshot" = "yes" ] && [ "$NOMAKE" != "yes" ]; then FAIL_SECTION="" if [ -f ${TESTDIR}/failsection ]; then rm ${TESTDIR}/failsection fi if (cd ${TESTDIR} && \ TIMESTAMP "configure" && echo "configure" > ${TESTDIR}/failsection && \ RUNCONFIGURE && \ sleep 2 && \ TIMESTAMP "make" && echo "make" > ${TESTDIR}/failsection && \ ${MAKE} && DISKUSAGE \ TIMESTAMP ${CHECKVAL} && echo "make check" > ${TESTDIR}/failsection && \ ${MAKE} ${CHECKVAL} && DISKUSAGE \ TIMESTAMP "install" && echo "make install" > ${TESTDIR}/failsection && \ ${MAKE} install && DISKUSAGE \ TIMESTAMP "check-install" && echo "make check-install" > ${TESTDIR}/failsection && \ CHECKINSTALL && DISKUSAGE \ TIMESTAMP "uninstall" && echo "make uninstall" > ${TESTDIR}/failsection && \ ${MAKE} uninstall && DISKUSAGE); then : else errcode=$? FAIL_SECTION=`cat ${TESTDIR}/failsection` echo "Failed running ${FAIL_SECTION}" snapshot=no exit $errcode fi elif [ $CPSRC ]; then cp -pr ${CURRENT}/* ${TESTDIR} else cmdclean="" fi fi # Test the HDF5 library # Run external test if configured #============================= #============================= #if [ -d "$CURRENT" ]; then if [ "$EXTTEST" != "" ]; then TIMESTAMP ${EXTTEST} TESTDIR=${BASEDIR}/TestDir/${SRCDIRNAME} test -d ${TESTDIR} || mkdir ${TESTDIR} cd ${TESTDIR} sleep 1 TIMESTAMP $pwd ls ${BASEDIR}/${EXTTEST} errcode=$? exit $errcode fi #============================= # Run deployment if requested. #============================= if [ -n "$DEPLOYDIRNAME" ]; then # The daily tests deploy to .../hdf5/... or .../hdf4/... except on cobalt where the # deploy directory is in .../HDF5/... lc will take care of this. If hdf4 or hdf5 # either upper or lower case isn't in the path, RELEASE.txt won't be found unless # it is in $CURRENT. POS4=`perl -e "print index(lc(\"${deploydir}/${DEPLOYDIRNAME}\"), 'hdf4')"` POS5=`perl -e "print index(lc(\"${deploydir}/${DEPLOYDIRNAME}\"), 'hdf5')"` if [ "${POS4}" -ge "0" ]; then RELEASE_TXT_LOC="release_notes" elif [ "${POS5}" -ge "0" ]; then RELEASE_TXT_LOC="release_docs" else RELEASE_TXT_LOC="" fi if [ "$snapshot" = "yes" ]; then TIMESTAMP "deploy" if (cd ${TESTDIR} && ${BASEDIR}/bin/deploy ${deploydir}/${DEPLOYDIRNAME} && \ TIMESTAMP "clean" && \ ${MAKE} clean && \ TIMESTAMP "check-install prefix=${deploydir}/${DEPLOYDIRNAME}" && \ CHECKINSTALL prefix=${deploydir}/${DEPLOYDIRNAME}); then cd ${CURRENT} cp ${RELEASE_TXT_LOC}/RELEASE.txt ${deploydir}/${DEPLOYDIRNAME} cp COPYING ${deploydir}/${DEPLOYDIRNAME} #: #continue else errcode=$? exit $errcode fi fi fi # Deploy #============================= # Run Release snapshot, update version, and commit to source repository #============================= if [ "$cmd" = "all" -o -n "$cmdrel" ]; then if [ "$snapshot" = "yes" ]; then TIMESTAMP "release" DISTCLEAN ( # Turn on exit on error in the sub-shell so that it does not # commit source if errors encounter here. set -e if [ "$cmdrel" = "autogen-release" ]; then cd ${BASEDIR}/current_src else cd ${CURRENT} fi if [ "$HDFREPOS" = "hdf4" ]; then RELEASE_VERSION="`perl bin/h4vers -v`" echo "Making snapshot release ($RELEASE_VERSION) to ${ReleaseDir}..." bin/release -d $ReleaseDir $METHODS perl bin/h4vers -i elif [ "$HDFREPOS" = "hdf5" ]; then RELEASE_VERSION="`perl bin/h5vers -v`" echo "Making snapshot release ($RELEASE_VERSION) to ${ReleaseDir}..." if [ "${DOCVERSION}" ]; then bin/release -d $ReleaseDir --docver ${DOCVERSION} $METHODS else bin/release -d $ReleaseDir $METHODS fi perl bin/h5vers -i else echo "need real release steps. For now, only move current version to previous" fi COMMITSNAPSHOT ) errcode=$? fi # Replace the previous version with the current version. # Should check if the errcode of the release process but there # are other failures after release was done (e.g. h5vers or git failures) # that should allow the replacement to occure. rm -rf ${PREVIOUS} mv ${CURRENT} ${PREVIOUS} fi #Release snapshot #============================= # Clean the test area. Default is no clean. #============================= if [ -n "$cmdclean" ] && [ "$NOMAKE" != "yes" ]; then TIMESTAMP "clean" # setup if srcdir is used. if [ -z "$srcdir" ]; then TESTDIR=${CURRENT} else case "$SRCDIRNAME" in "") SRCDIRNAME=$HOSTNAME ;; -*) SRCDIRNAME="$HOSTNAME$SRCDIRNAME" ;; esac TESTDIR=${BASEDIR}/TestDir/${SRCDIRNAME} fi # clean it if (cd ${TESTDIR} && ${MAKE} $cmdclean ) then : else errcode=$? snapshot=no exit $errcode fi fi # Clean the Test directory exit $errcode