diff options
Diffstat (limited to 'bin')
33 files changed, 2041 insertions, 288 deletions
diff --git a/bin/Makefile.am b/bin/Makefile.am new file mode 100644 index 0000000..a39b695 --- /dev/null +++ b/bin/Makefile.am @@ -0,0 +1,57 @@ +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## +## Makefile.am +## Run automake to generate a Makefile.in from this file. +# +# HDF5 Library Makefile(.in) +# + +include $(top_srcdir)/config/commence.am + +# Include src directory +AM_CPPFLAGS+=-I$(top_srcdir)/src -I$(top_srcdir)/tools/lib + +# These are our main targets +bin_SCRIPTS=h5redeploy + +# Tell automake to clean h5redeploy script +CLEANFILES=h5redeploy + +# These were generated by configure. Remove them only when distclean. +DISTCLEANFILES=h5cc + +# All programs rely on hdf5 library and h5tools library +LDADD=$(LIBH5TOOLS) $(LIBHDF5) + +# How to build h5redeploy script +h5redeploy: h5redeploy.in + @cp $(srcdir)/$@.in $@ + +# h5cc needs custom install and uninstall rules, since it may be +# named h5pcc if hdf5 is being built in parallel mode. +if BUILD_PARALLEL_CONDITIONAL + H5CC_NAME=h5pcc +else + H5CC_NAME=h5cc +endif + +$(DESTDIR)$(bindir): + echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \ + $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; + +install-exec-local: $(DESTDIR)$(bindir) + @$(INSTALL) h5cc $(DESTDIR)$(bindir)/$(H5CC_NAME) +uninstall-local: + @$(RM) $(DESTDIR)$(bindir)/$(H5CC_NAME) + +include $(top_srcdir)/config/conclude.am diff --git a/bin/batch/ctest.qsub.in.cmake b/bin/batch/ctest.qsub.in.cmake new file mode 100755 index 0000000..702fca7 --- /dev/null +++ b/bin/batch/ctest.qsub.in.cmake @@ -0,0 +1,21 @@ +#!/bin/bash -l +if [ $# -gt 0 ]; then + SUMMARY_FILE=$1 +fi +ACCOUNT_ID=@ACCOUNT_ID@ + +echo "Run parallel test command. Test output will be in build/${SUMMARY_FILE}" +CTEST_CMD=`which ctest` + +#SKIPTESTS <<KEYWORD:script inserts list of skips tests here -- don't remove>> + +cd @HDF5_BINARY_DIR@ +if [[ $SUMMARY_FILE == *"ctestS"* ]]; then + CMD="${CTEST_CMD} . -E MPI_TEST_ -C Release -j 32 -T test" +else + CMD="${CTEST_CMD} . -R MPI_TEST_ ${SKIP_TESTS} -C Release -T test" +fi + +qsub -t 60 -n 1 -q debug-flat-quad -A ${ACCOUNT_ID} ${CMD} >& ${SUMMARY_FILE} + +echo "Done running ctest parallel command." diff --git a/bin/batch/ctestP.lsf.in.cmake b/bin/batch/ctestP.lsf.in.cmake new file mode 100644 index 0000000..3fdd5ce --- /dev/null +++ b/bin/batch/ctestP.lsf.in.cmake @@ -0,0 +1,19 @@ +#!/bin/tcsh +### LSF syntax +#BSUB -nnodes 1 #number of nodes +#BSUB -W 30 #walltime in minutes +#BSUB -G guests #account +#BSUB -e ctestPerrors.txt #stderr +#BSUB -o ctestPoutput.txt #stdout +#BSUB -J hdf5_ctestP #job +##BSUB -q pbatch #queue to use +#BSUB -q pdebug + +##date; hostname +##echo -n 'JobID is '; echo $LSB_JOBID + +cd @HDF5_BINARY_DIR@ +echo "Run parallel test command. Test output will be in build/ctestP.out" +ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out + +echo "Done running ctest parallel command." diff --git a/bin/batch/ctestP.sl.in.cmake b/bin/batch/ctestP.sl.in.cmake new file mode 100644 index 0000000..1069fa9 --- /dev/null +++ b/bin/batch/ctestP.sl.in.cmake @@ -0,0 +1,14 @@ +#!/bin/bash + +#SBATCH --nodes=1 +#SBATCH -t 00:30:00 +#SBATCH --mail-type=BEGIN,END,FAIL +##SBATCH --mail-user=<username>@sandia.gov +#SBATCH --export=ALL +#SBATCH --job-name=h5_ctestP + +cd @HDF5_BINARY_DIR@ +ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out + +echo "Done running ctestP.sl" + diff --git a/bin/batch/ctestS.lsf.in.cmake b/bin/batch/ctestS.lsf.in.cmake new file mode 100644 index 0000000..a01d39b --- /dev/null +++ b/bin/batch/ctestS.lsf.in.cmake @@ -0,0 +1,18 @@ +#!/bin/tcsh +### LSF syntax +#BSUB -nnodes 1 #number of nodes +#BSUB -W 29 #walltime in minutes +#BSUB -G guests #account +#BSUB -e ctestSerrors.txt #stderr +#BSUB -o ctestSoutput.txt #stdout +#BSUB -J hdf5_ctestS #job +##BSUB -q pbatch #queue to use +#BSUB -q pdebug + +cd @HDF5_BINARY_DIR@ +echo "Run command. Test output will be in build/ctestS.out" +ctest . -E MPI_TEST_ -C Release -j 32 -T test >& ctestS.out + +##$CMD >& ctestS.out +echo "Done running command." + diff --git a/bin/batch/ctestS.sl.in.cmake b/bin/batch/ctestS.sl.in.cmake new file mode 100644 index 0000000..4f96bb9 --- /dev/null +++ b/bin/batch/ctestS.sl.in.cmake @@ -0,0 +1,15 @@ +#!/bin/bash + +#SBATCH --nodes=1 +#SBATCH -t 00:30:00 +#SBATCH --mail-type=BEGIN,END,FAIL +##SBATCH --mail-user=<username>@sandia.gov +#SBATCH --export=ALL +#SBATCH --job-name=h5_ctestS + +cd @HDF5_BINARY_DIR@ +CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test" + +echo "Run $CMD. Test output will be in build/ctestS.out" +$CMD >& ctestS.out +echo "Done running $CMD" diff --git a/bin/batch/knl_H5detect.sl.in.cmake b/bin/batch/knl_H5detect.sl.in.cmake new file mode 100644 index 0000000..39a3ef3 --- /dev/null +++ b/bin/batch/knl_H5detect.sl.in.cmake @@ -0,0 +1,20 @@ +#!/bin/bash + +#SBATCH -p knl -C quad +#SBATCH --nodes=1 +#SBATCH -t 00:10:00 +#SBATCH --mail-type=BEGIN,END,FAIL +#SBATCH --mail-user=<username>@sandia.gov +#SBATCH --export=ALL +#SBATCH --job-name=knl_h5detect + + +# Inputs: Build directory, output file name, executable file name (username/email if available). +PROGNAME=H5detect +OUTPUT=H5Tinit.c + +CMD="@HDF5_BINARY_DIR@/bin/${PROGNAME} @HDF5_GENERATED_SOURCE_DIR@/${OUTPUT}" +echo "Run $CMD" +srun -n 1 $CMD +echo "Done running $CMD" + diff --git a/bin/batch/knl_ctestP.sl.in.cmake b/bin/batch/knl_ctestP.sl.in.cmake new file mode 100644 index 0000000..f985fbb --- /dev/null +++ b/bin/batch/knl_ctestP.sl.in.cmake @@ -0,0 +1,16 @@ +#!/bin/bash + +#SBATCH -p knl -C quad,cache +#SBATCH --nodes=1 +#SBATCH -t 00:30:00 +#SBATCH --mail-type=BEGIN,END,FAIL +##SBATCH --mail-user=<username>@sandia.gov +#SBATCH --export=ALL +#SBATCH --job-name=h5_ctestP + +cd @HDF5_BINARY_DIR@ +#run parallel tests except t_cache_image test +ctest . -R MPI_TEST_ -C Release -T test >& ctestP.out + +echo "Done running $CMD" + diff --git a/bin/batch/knl_ctestS.sl.in.cmake b/bin/batch/knl_ctestS.sl.in.cmake new file mode 100644 index 0000000..af6353b --- /dev/null +++ b/bin/batch/knl_ctestS.sl.in.cmake @@ -0,0 +1,17 @@ +#!/bin/bash + +#SBATCH -p knl -C quad,cache +#SBATCH --nodes=1 +#SBATCH -t 00:30:00 +#SBATCH --mail-type=BEGIN,END,FAIL +##SBATCH --mail-user=<username>@sandia.gov +#SBATCH --export=ALL +#SBATCH --job-name=h5_ctestS + +cd @HDF5_BINARY_DIR@ +CMD="ctest . -E MPI_TEST_ -C Release -j 32 -T test" + +echo "Run $CMD. Test output will be in build/ctestS.out" +$CMD >& ctestS.out +echo "Done running $CMD" + diff --git a/bin/batch/ray_ctestP.lsf.in.cmake b/bin/batch/ray_ctestP.lsf.in.cmake new file mode 100644 index 0000000..7067a65 --- /dev/null +++ b/bin/batch/ray_ctestP.lsf.in.cmake @@ -0,0 +1,20 @@ +#!/bin/tcsh +### LSF syntax +#BSUB -n 6 #number of nodes +#BSUB -R "span[ptile=6]" +#BSUB -W 30 #walltime in minutes +#BSUB -G guests #account +#BSUB -e ctestPerrors.txt #stderr +#BSUB -o ctestPoutput.txt #stdout +#BSUB -J hdf5_ctestP #job +##BSUB -q pbatch #queue to use +#BSUB -q pdebug + +##date; hostname +##echo -n 'JobID is '; echo $LSB_JOBID + +cd @HDF5_BINARY_DIR@ +echo "Run parallel test command. Test output will be in build/ctestP.out" +ctest . -R 'MPI_TEST_' -C Release -T test >& ctestP.out + +echo "Done running ctest parallel command." diff --git a/bin/batch/ray_ctestS.lsf.in.cmake b/bin/batch/ray_ctestS.lsf.in.cmake new file mode 100644 index 0000000..da20438 --- /dev/null +++ b/bin/batch/ray_ctestS.lsf.in.cmake @@ -0,0 +1,18 @@ +#!/bin/tcsh +### LSF syntax +#BSUB -n 1 #number of nodes +#BSUB -W 29 #walltime in minutes +#BSUB -G guests #account +#BSUB -e ctestSerrors.txt #stderr +#BSUB -o ctestSoutput.txt #stdout +#BSUB -J hdf5_ctestS #job +##BSUB -q pbatch #queue to use +#BSUB -q pdebug + +cd @HDF5_BINARY_DIR@ +echo "Run command. Test output will be in build/ctestS.out" +ctest . -E 'MPI_TEST_' -C Release -j 32 -T test >& ctestS.out + +##$CMD >& ctestS.out +echo "Done running command." + diff --git a/bin/batch/raybsub b/bin/batch/raybsub new file mode 100755 index 0000000..19dceef --- /dev/null +++ b/bin/batch/raybsub @@ -0,0 +1,7 @@ +#!/bin/tcsh + +# ray.llnl.gov requires a '<' with bsub for submitting .lsf batch jobs. +# CMake is reluctant to pass the '<', so we put it in this script and use +# the script to submit the bsub command on ray. + +bsub < $1 diff --git a/bin/checkapi b/bin/checkapi index 6882dea..f5dcacc 100755 --- a/bin/checkapi +++ b/bin/checkapi @@ -1,4 +1,4 @@ -#!/usr/bin/perl -w +#!/usr/bin/env perl # # Copyright by The HDF Group. # Copyright by the Board of Trustees of the University of Illinois. @@ -13,6 +13,8 @@ # require 5.003; +use warnings; + # Purpose: insures that API functions aren't called internally. # Usage: checkapi H5*.c my $filename = ""; diff --git a/bin/checkposix b/bin/checkposix index 7ab741c..233d15c 100755 --- a/bin/checkposix +++ b/bin/checkposix @@ -1,5 +1,6 @@ -#!/usr/bin/perl -w +#!/usr/bin/env perl require 5.003; +use warnings; # # Copyright by The HDF Group. @@ -13,101 +14,165 @@ require 5.003; # If you do not have access to either file, you may request a copy from # help@hdfgroup.org. # -# Robb Matzke, matzke@llnl.gov -# 30 Aug 1997 +# Dana Robinson +# Spring 2019 +# (Original by Robb Matzke) # -# Purpose: Given the names of C source files this script will print the -# file name, line number, and function name of any function that -# doesn't begin with the letter `h' or `H' as stipulated by the -# HDF5 programming style guide. +# Purpose: Given the names of C source files this script will print the +# file name, line number, and function name of any function that +# doesn't begin with the letter 'h' or 'H' as stipulated by the +# HDF5 programming style guide. # -# Emacs users can run this script as the compile command and -# use `next-error' (usually bound to M-`) to find each name -# violation. - -if(<>) { - if($ARGV =~ /\//) { - ($filename) = ($ARGV =~ /^.*\/([A-Za-z0-9_]*)\.c$/); - } else { - ($filename) = ($ARGV =~ /([A-Za-z0-9_]*)\.c$/); +# Emacs users can run this script as the compile command and +# use 'next-error' (usually bound to M-`) to find each name +# violation. + +use File::Basename; + +# Loop over all files passed to the function +foreach $arg (@ARGV) { + + # Get the filename from the path + $filename = fileparse($arg); + + # Skip files that don't include H5private.h + # H5system. has to be inspected by hand since it wraps POSIX files + # + # H5detect and H5make_libsettings are created before the library exists + # so calls that link to function replacements won't work. We'll ignore + # it here. + # + # If a user specifies one file, process it no matter what so people + # can inspect files we normally skip (like H5system.c). + if($#ARGV gt 0 and $filename =~ /H5FDmulti|H5FDstdio|H5VLpassthru|H5system|H5detect|H5make_libsettings/) { + print "$filename is exempt from using Standard library macro wrappers\n"; + next; } - if($filename =~ /H5FDmulti|H5FDstdio/) { - print "$ARGV is exempt from using Standard library macro wrappers\n"; - } else { - while (<>) { - - # Get rid of comments by removing the inside part. - s|/\*.*?\*/||g; - if ($in_comment) { - if (/\*\//) { - s|.*?\*/||; - $in_comment = 0; - } else { - $_="\n"; - } - } elsif (m|/\*|) { - s|/\*.*||; - $in_comment = 1; - } - - # Get rid of string constants if they begin and end on this line. - s/([\'\"])([^\1]|\\\1)*?\1/$1$1/g; - - # Get rid of preprocessor directives - s/^\#.*//; - - # Skip callbacks invoked as methods in a struct - next if $_ =~ /\b(\)?->|\.)\(?([a-z_A-Z]\w*)\s*\(/; - - # Now find all function calls on this line which don't start with 'H' - while (($name)=/\b([a-z_A-GI-Z]\w*)\s*\(/) { - $_ = $'; + # Open the file + open(my $fh, "<", $arg) or do { + warn "NOTE: Unable to open $arg: !$\n"; + next; + }; + + # Loop over all lines in the file to find undecorated functions + while (<$fh>) { + + # Get rid of comments by removing the inside part. + s|/\*.*?\*/||g; + if ($in_comment) { + if (/\*\//) { + s|.*?\*/||; + $in_comment = 0; + } else { + $_="\n"; + } + } elsif (m|/\*|) { + s|/\*.*||; + $in_comment = 1; + } + + # Get rid of string constants if they begin and end on this line. + s/([\'\"])([^\1]|\\\1)*?\1/$1$1/g; + + # Get rid of preprocessor directives + s/^\#.*//; + + # Skip callbacks invoked as methods in a struct + next if $_ =~ /\b(\)?]?->|\.)\(?([a-z_A-Z]\w*)\s*\(/; + + # Now find all function calls on this line which don't start with 'H' + while (($name)=/\b([a-z_A-GI-Z]\w*)\s*\(/) { + $_ = $'; - # Ignore C statements that look sort of like function - # calls. - next if $name =~ /^(if|for|offsetof|return|sizeof|switch|while|void)$/; - - # Ignore things that get misdetected because of the simplified - # parsing that takes place here. - next if $name =~ /^(int|herr_t|_term_interface)$/; - - # These are really HDF5 functions/macros even though they don't - # start with `h' or `H'. - next if $name =~ /^FUNC_(ENTER|LEAVE)(_(NO)?API|_PACKAGE|_STATIC)?(_NOFS|_NOCLEAR|_NOINIT)?(_NOFUNC|_TAG)?$/; - next if $name =~ /^(BEGIN|END)_FUNC$/; - next if $name =~ /^U?INT(8|16|32|64)(ENCODE|DECODE)(_VAR)?$/; - next if $name =~ /^CI_(PRINT_STATS|INC_SRC|INC_DST)$/; - next if $name =~ /^(ABS|ADDR_OVERFLOW|ALL_MEMBERS|BOUND|CONSTR|DETECT_[I|F|M]|DOWN)$/; - next if $name =~ /^(MIN3?|MAX3?|NELMTS|POWER_OF_TWO|REGION_OVERFLOW)$/; - next if $name =~ /^(UNIQUE_MEMBERS)$/; - next if $name =~ /^addr_defined$/; - - # These functions/macros are exempt. - next if $name =~ /^(main|[fs]?printf|va_(start|arg|end))$/; - - # These are Windows system calls. Ignore them. - next if $name =~ /^(_get_osfhandle|GetFileInformationByHandle|SetFilePointer|GetLastError|SetEndOfFile)$/; - next if $name =~ /^(FindNextFile|FindClose|_tzset|Wgettimeofday|GetSystemTimeAsFileTime|Wgetlogin|GetUserName)$/; - - # These are MPI function calls. Ignore them. - next if $name =~ /^(MPI_|MPE_)/; - - # These are POSIX threads function calls. Ignore them. - next if $name =~ /^pthread_/; - - # These are Windows threads function calls. Ignore them. - next if $name =~ /^(_beginthread|(Initialize|Enter|Leave)CriticalSection|TlsAlloc)$/; - - # These are zlib & szlib function calls. Ignore them. - next if $name =~ /^(inflate|SZ_)/; - next if $name =~ /^compress2$/; - - print "$ARGV:$.: $name\n"; - } - - } continue { - close ARGV if eof; # reset line number + # Ignore C statements that look sort of like function + # calls. + next if $name =~ /^(if|for|offsetof|return|sizeof|switch|while|void)$/; + + # Ignore things that get misdetected because of the simplified + # parsing that takes place here. + next if $name =~ /^(int|herr_t|_term_interface|_term_package)$/; + + # These are really HDF5 functions/macros even though they don't + # start with `h' or `H'. + next if $name =~ /^FUNC_(ENTER|LEAVE)(_(NO)?API|_PACKAGE|_STATIC)?(_NOFS|_NOCLEAR|_NOINIT)?(_NOFUNC|_TAG)?$/; + next if $name =~ /^(BEGIN|END)_FUNC$/; + next if $name =~ /^U?INT(8|16|32|64)(ENCODE|DECODE)(_VAR)?$/; + next if $name =~ /^CI_(PRINT_STATS|INC_SRC|INC_DST)$/; + next if $name =~ /^(ABS|ADDR_OVERFLOW|ALL_MEMBERS|BOUND|CONSTR|DETECT_[I|F|M]|DOWN)$/; + next if $name =~ /^(MIN3?|MAX3?|NELMTS|POWER_OF_TWO|REGION_OVERFLOW)$/; + next if $name =~ /^(UNIQUE_MEMBERS|S_ISDIR)$/; + next if $name =~ /^addr_defined$/; + + # These functions/macros are exempt. + # op, cb, and OP are often spuriously flagged so ignore them. + next if $name =~ /^(main|op|cb|OP)$/; + + # This often appears in preprocessor lines that span multiple lines + next if $name =~ /^(defined)$/; + + # These are Windows system calls. Ignore them. + next if $name =~ /^(_get_osfhandle|GetFileInformationByHandle|SetFilePointer|GetLastError|SetEndOfFile)$/; + next if $name =~ /^(FindNextFile|FindClose|_tzset|Wgettimeofday|GetSystemTimeAsFileTime|Wgetlogin|GetUserName)$/; + next if $name =~ /^(DeleteCriticalSection|TlsFree|TlsGetValue|CreateThread)$/; + next if $name =~ /^(ExpandEnvironmentStringsA|LockFileEx|UnlockFileEx)$/; + next if $name =~ /^(DllMain|LocalAlloc|LocalFree)$/; + next if $name =~ /^(FindFirstFileA|FindNextFileA)$/; + next if $name =~ /^(_beginthread|(Initialize|Enter|Leave)CriticalSection|TlsAlloc)$/; + + # These are MPI function calls. Ignore them. + next if $name =~ /^(MPI_|MPE_)/; + + # These are POSIX threads function calls. Ignore them. + next if $name =~ /^pthread_/; + + # These are zlib & szlib function calls. Ignore them. + next if $name =~ /^(inflate|SZ_)/; + next if $name =~ /^compress2$/; + + # These is an H5Dfill function. Ignore it in this file. + if($filename =~ /H5Dfill/) { + next if $name =~ /^(alloc_func)$/; + } + + # These are H5Zscaleoffset functions. Ignore them in this file. + if($filename =~ /H5Zscaleoffset/) { + next if $name =~ /^(pow_fun|round_fun|abs_fun|lround_fun|llround_fun)$/; + } + + # TESTING (not comprehensive - just noise reduction) + + # Test macros and functions (testhdf5.h) + next if $name =~ /^(AddTest|TestErrPrintf|TestSummary|TestCleanup|TestShutdown)$/; + next if $name =~ /^(CHECK|CHECK_PTR|CHECK_PTR_NULL|CHECK_PTR_EQ|CHECK_I)$/; + next if $name =~ /^(VERIFY|VERIFY_STR|VERIFY|TYPE|MESSAGE|ERROR)$/; + + # Test macros and functions (h5test.h) + next if $name =~ /^(TESTING|PASSED|SKIPPED|FAIL_PUTS_ERROR|FAIL_STACK_ERROR|TEST_ERROR)$/; + next if $name =~ /^(GetTestExpress)$/; + + # Ignore functions that start with test_ or check_ + next if $name =~ /^test_/; + next if $name =~ /^check_/; + + # Ignore functions that start with h5_ + next if $name =~ /^h5_/; + + # Ignore usage functions + next if $name =~ /^usage$/; + + print "$filename:$.: $name\n"; } + } + + # Close the file + close($fh); +} + +if($#ARGV gt 0) { + print "\n"; + print "NOTE:\n"; + print "If any files were skipped due to being exempt, you can inspect them manually\n"; + print "by using this script on them one at a time, which will always process the file.\n"; } diff --git a/bin/cmakehdf5 b/bin/cmakehdf5 index 7f6453a..da1aef8 100755 --- a/bin/cmakehdf5 +++ b/bin/cmakehdf5 @@ -46,8 +46,9 @@ cacheinit=$srcdir/config/cmake/cacheinit.cmake build_cpp_lib=-DHDF5_BUILD_CPP_LIB:BOOL=OFF # C++ interface default off build_fortran=-DHDF5_BUILD_FORTRAN:BOOL=OFF # Fortran interface default off build_hl_lib=-DHDF5_BUILD_HL_LIB:BOOL=ON # High Level interface default on -build_threadsafe=-DHDF5_ENABLE_THREADSAFE:BOOL=OFF # Threadsafe feature default off +build_threadsafe=-DHDF5_ENABLE_THREADSAFE:BOOL=OFF # Threadsafe feature default off build_testing=-DBUILD_TESTING:BOOL=ON # Build tests default on +build_test_shell=-DTEST_SHELL_SCRIPTS:BOOL=ON # Run shell script tests default on build_tools=-DHDF5_BUILD_TOOLS:BOOL=ON # Build tools default on with_zlib=-DHDF5_ENABLE_Z_LIB_SUPPORT=ON # enable zlib filter default on with_szlib=-DHDF5_ENABLE_SZIP_SUPPORT=OFF # enables szip filter default off @@ -256,6 +257,12 @@ while [ $# -gt 0 ]; do --disable-testing) build_testing=-DBUILD_TESTING:BOOL=OFF ;; + --enable-shell-testing) + build_test_shell=-DTEST_SHELL_SCRIPTS:BOOL=ON + ;; + --disable-shell-testing) + build_test_shell=-DTEST_SHELL_SCRIPTS:BOOL=OFF + ;; --with-zlib) with_zlib=-DHDF5_ENABLE_Z_LIB_SUPPORT:BOOL=ON ;; @@ -342,6 +349,7 @@ STEP "Configure..." \ $build_threadsafe \ $shared_lib \ $build_testing \ + $build_test_shell \ $build_tools \ $with_zlib \ $with_szlib \ diff --git a/bin/debug-ohdr b/bin/debug-ohdr index 5b0a4b3..1363456 100755 --- a/bin/debug-ohdr +++ b/bin/debug-ohdr @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # # Copyright by The HDF Group. # Copyright by the Board of Trustees of the University of Illinois. diff --git a/bin/dependencies b/bin/dependencies index 82247da..367351a 100755 --- a/bin/dependencies +++ b/bin/dependencies @@ -1,4 +1,4 @@ -#!/usr/bin/perl -w +#!/usr/bin/env perl # # Copyright by The HDF Group. # Copyright by the Board of Trustees of the University of Illinois. @@ -11,6 +11,8 @@ # If you do not have access to either file, you may request a copy from # help@hdfgroup.org. # +use warnings; + my $depend_file; my $new_depend_file; my $srcdir; diff --git a/bin/distdep b/bin/distdep index 4643700..cd310e0 100755 --- a/bin/distdep +++ b/bin/distdep @@ -1,4 +1,7 @@ -#!/usr/bin/perl -p +#!/bin/sh +#! -*-perl-*- +eval 'exec perl -p -x -S $0 ${1+"$@"}' + if 0; # # Copyright by The HDF Group. # Copyright by the Board of Trustees of the University of Illinois. @@ -1,5 +1,6 @@ -#!/usr/local/bin/perl -w +#!/usr/bin/env perl require 5.003; +use warnings; use Text::Tabs; # NOTE: THE FORMAT OF HRETURN_ERROR AND HGOTO_ERROR MACROS HAS diff --git a/bin/genparser b/bin/genparser index 8cf6ec2..e6aee5b 100755 --- a/bin/genparser +++ b/bin/genparser @@ -218,7 +218,7 @@ perl -0777 -pi -e 's/int H5LTyyparse/hid_t H5LTyyparse/igs' ${path_to_hl_src}/H5 # will simply ignore them, but we want to avoid those warnings. for f in ${path_to_hl_src}/H5LTparse.c ${path_to_hl_src}/H5LTanalyze.c do - echo '#if __GNUC__ >= 4 && __GNUC_MINOR__ >=2 ' >> tmp.out + echo '#if defined __GNUC__ && 402 <= __GNUC__ * 100 + __GNUC_MINOR__ ' >> tmp.out echo '#pragma GCC diagnostic ignored "-Wconversion" ' >> tmp.out echo '#pragma GCC diagnostic ignored "-Wimplicit-function-declaration" ' >> tmp.out echo '#pragma GCC diagnostic ignored "-Wlarger-than=" ' >> tmp.out diff --git a/bin/h5cc.in b/bin/h5cc.in new file mode 100644 index 0000000..62a0884 --- /dev/null +++ b/bin/h5cc.in @@ -0,0 +1,392 @@ +#! /bin/sh +## +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +## + +# This tool is adapted from the mpicc command of the MPICH Software. + +############################################################################ +## ## +## Things You May Have to Modify: ## +## ## +## If the following paths don't point to the place were HDF5 is installed ## +## on your system (i.e., you received a binary distribution or moved the ## +## files from the originally installed directory to another directory) ## +## then modify them accordingly to represent the new paths. ## +## ## +############################################################################ +prefix="@prefix@" +exec_prefix="@exec_prefix@" +libdir="@libdir@" +includedir="@includedir@" +HL="@HL@" + +############################################################################ +## ## +## Things You Can Modify to Override HDF5 Library Build Components: ## +## ## +## (Advanced usage - know what you're doing - you're on your own here.) ## +## The four variables below can be used to insert paths and flags in ## +## CPPFLAGS, CFLAGS, LDFLAGS, or LIBS in the h5cc compile line: ## +## $CLINKER $H5BLD_CPPFLAGS $CPPFLAGS $H5BLD_CFLAGS $CFLAGS $LDFLAGS ## +## $LIBS $clibpath $link_objs $link_args $shared_link ## +## ## +## These settings can be overriden by setting HDF5_CFLAGS, ## +## HDF5_CPPFLAGS, HDF5_LDFLAGS, or HDF5_LIBS in the environment. ## +## ## +############################################################################ +CFLAGSBASE="" +CPPFLAGSBASE="" +LDFLAGSBASE="" +LIBSBASE="" + +############################################################################ +## ## +## You shouldn't have to modify anything below this line. ## +## ## +############################################################################ + +# Constants definitions +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +host_os="@host_os@" + +prog_name="`basename $0`" + +allargs="" +compile_args="" +libraries="" +link_args="" +link_objs="" +clibpath="" + +do_link="yes" +do_compile="no" +dash_o="no" +dash_c="no" +get_output_file="no" + +SHOW="eval" +CCBASE="@CC@" +CLINKERBASE="@CC@" + +# CFLAGS, CPPFLAGS and LDFLAGS are reserved for use by the script user. +# FLAGS brought from the hdf5 build are put in H5BLD_*FLAGS. + +# User's CPPFLAGS and CFLAGS come after their H5BLD counterparts. User's +# LDFLAGS come just before clibpath, user's LIBS come after $link_objs and +# before the hdf5 libraries in $link_args, followed by any external library +# paths and libraries from AM_LDFLAGS, LDFLAGS, AM_LIBS or LIBS carried in +# from the hdf5 build. The order of the flags is intended to give precedence +# to the user's flags. +H5BLD_CFLAGS="@AM_CFLAGS@ @CFLAGS@" +H5BLD_CPPFLAGS="@AM_CPPFLAGS@ @CPPFLAGS@" +H5BLD_LDFLAGS="@AM_LDFLAGS@ @LDFLAGS@" +H5BLD_LIBS="@LIBS@" + +CC="${HDF5_CC:-$CCBASE}" +CLINKER="${HDF5_CLINKER:-$CLINKERBASE}" +CFLAGS="${HDF5_CFLAGS:-$CFLAGSBASE}" +CPPFLAGS="${HDF5_CPPFLAGS:-$CPPFLAGSBASE}" +LDFLAGS="${HDF5_LDFLAGS:-$LDFLAGSBASE}" +LIBS="${HDF5_LIBS:-$LIBSBASE}" + +# If a static library is available, the default will be to use it. If the only +# available library is shared, it will be used by default. The user can +# override either default, although choosing an unavailable library will result +# in link errors. +STATIC_AVAILABLE="@enable_static@" +if test "${STATIC_AVAILABLE}" = "yes"; then + USE_SHARED_LIB="${HDF5_USE_SHLIB:-no}" +else + USE_SHARED_LIB="${HDF5_USE_SHLIB:-yes}" +fi + + +usage() { + # A wonderfully informative "usage" message. + echo "usage: $prog_name [OPTIONS] <compile line>" + echo " OPTIONS:" + echo " -help This help message." + echo " -echo Show all the shell commands executed" + echo " -prefix=DIR Prefix directory to find HDF5 lib/ and include/" + echo " subdirectories [default: $prefix]" + echo " -show Show the commands without executing them" + echo " -showconfig Show the HDF5 library configuration summary" + echo " -shlib Compile with shared HDF5 libraries [default for hdf5 built" + echo " without static libraries]" + echo " -noshlib Compile with static HDF5 libraries [default for hdf5 built" + echo " with static libraries]" + echo " " + echo " <compile line> - the normal compile line options for your compiler." + echo " $prog_name uses the same compiler you used to compile" + echo " HDF5. Check with your compiler's man pages for more" + echo " information on which options are needed." + echo " " + echo " You can override the compiler, linker, and whether or not to use static" + echo " or shared libraries to compile your program by setting the following" + echo " environment variables accordingly:" + echo " " + echo " HDF5_CC - use a different C compiler" + echo " HDF5_CLINKER - use a different linker" + echo " HDF5_USE_SHLIB=[yes|no] - use shared or static version of the HDF5 library" + echo " [default: no except when built with only" + echo " shared libraries]" + echo " " + echo " You can also add or change paths and flags to the compile line using" + echo " the following environment varibles or by assigning them to their counterparts" + echo " in the 'Things You Can Modify to Override...'" section of $prog_name + echo " " + echo " Variable Current value to be replaced" + echo " HDF5_CPPFLAGS \"$CPPFLAGSBASE\"" + echo " HDF5_CFLAGS \"$CFLAGSBASE\"" + echo " HDF5_LDFLAGS \"$LDFLAGSBASE\"" + echo " HDF5_LIBS \"$LIBSBASE\"" + echo " " + echo " Note that adding library paths to HDF5_LDFLAGS where another hdf5 version" + echo " is located may link your program with that other hdf5 library version." + echo " " + exit $EXIT_FAILURE +} + +# Show the configuration summary of the library recorded in the +# libhdf5.settings file reside in the lib directory. +showconfigure() +{ + cat ${libdir}/libhdf5.settings + status=$? +} + +# Main +status=$EXIT_SUCCESS + +if test "$#" = "0"; then + # No parameters specified, issue usage statement and exit. + usage +fi + +case "$CC" in + gcc) + kind="gcc" + ;; + mpicc|mpcc|mpicc_r) + # Is this gcc masquarading as an MPI compiler? + if test "`${CC} -v 2>&1 | sed -n 2p | cut -c1-3`" = "gcc"; then + kind="gcc" + else + # Nope + kind="$host_os" + fi + ;; + *) + kind="$host_os" + ;; +esac + +for arg in $@ ; do + if test "x$get_output_file" = "xyes"; then + link_args="$link_args $arg" + output_file="$arg" + get_output_file="no" + continue + fi + + case "$arg" in + -c) + allargs="$allargs $arg" + compile_args="$compile_args $arg" + + if test "x$do_link" = "xyes" -a -n "$output_file"; then + compile_args="$compile_args -o $output_file" + fi + + do_link="no" + dash_c="yes" + ;; + -o) + allargs="$allargs $arg" + dash_o="yes" + + if test "x$dash_c" = "xyes"; then + compile_args="$compile_args $arg" + else + link_args="$link_args $arg" + do_link="yes" + get_output_file="yes" + fi + ;; + -E|-M|-MT) + allargs="$allargs $arg" + compile_args="$compile_args $arg" + dash_c="yes" + do_link="no" + ;; + -l*) + libraries=" $libraries $arg " + allargs="$allargs $arg" + ;; + -prefix=*) + prefix="`expr "$arg" : '-prefix=\(.*\)'`" + ;; + -echo) + set -x + ;; + -show) + SHOW="echo" + ;; + -showconfig) + showconfigure + exit $status + ;; + -shlib) + USE_SHARED_LIB="yes" + ;; + -noshlib) + USE_SHARED_LIB="no" + ;; + -help) + usage + ;; + *\"*) + qarg="'"$arg"'" + allargs="$allargs $qarg" + ;; + *\'*) + qarg='\"'"$arg"'\"' + allargs="$allargs $qarg" + ;; + *) + allargs="$allargs $qarg" + + if test -s "$arg"; then + ext=`expr "$arg" : '.*\(\..*\)'` + + if test "x$ext" = "x.c"; then + do_compile="yes" + compile_args="$compile_args $arg" + fname=`basename $arg .c` + link_objs="$link_objs $fname.o" + elif test "x$ext" = "x.o"; then + if test "x$dash_c" = "xyes"; then + compile_args="$compile_args $arg" + else + do_link="yes" + link_objs="$link_objs $arg" + fi + elif test "x$ext" = "x.a"; then + # This is an archive that we're linking in + libraries=" $libraries $arg " + else + compile_args="$compile_args $arg" + link_args="$link_args $arg" + fi + else + compile_args="$compile_args $arg" + link_args="$link_args $arg" + fi + ;; + esac +done + +if test "$dash_c" = "yes" -a "$do_compile" = no -a "$do_link" = no ; then + # -c was specified. Force do_compile on. + do_compile=yes +fi + +if test "x$do_compile" = "xyes"; then + if test "x$dash_c" != "xyes"; then + compile_args="-c $compile_args" + fi + + $SHOW $CC -I$includedir $H5BLD_CPPFLAGS $CPPFLAGS $H5BLD_CFLAGS $CFLAGS $compile_args + status=$? + + if test "$status" != "0"; then + exit $status + fi +fi + +if test "x$do_link" = "xyes"; then + shared_link="" +# conditionnaly link with the hl library + if test "X$HL" = "Xhl"; then + libraries=" $libraries -lhdf5_hl -lhdf5 " + else + libraries=" $libraries -lhdf5 " + fi + link_args="$link_args -L${libdir}" + + case "$kind" in + gcc|linux*) + # MacOS X doesn't support the "-Wl,-rpath -Wl," style of linker flags. + # It appears to want none of them specified. + case "$host_os" in + darwin*) flag="" ;; + *) flag="-Wl,-rpath -Wl," ;; + esac + ;; + hpux*) flag="-Wl,+b -Wl," ;; + freebsd*|solaris*) flag="-R" ;; + rs6000*|aix*) flag="-L" ;; + sgi) flag="-rpath " ;; + *) flag="" ;; + esac + + if test -n "$flag"; then + shared_link="${flag}${libdir}" + fi + + if test "x$USE_SHARED_LIB" != "xyes"; then + # The "-lhdf5" & "-lhdf5_hl" flags are in here already...This is a static + # compile though, so change it to the static version (.a) of the library. + new_libraries="" + for lib in $libraries; do + case "$lib" in + -lhdf5) + new_libraries="$new_libraries ${libdir}/libhdf5.a" + ;; + -lhdf5_hl) + new_libraries="$new_libraries ${libdir}/libhdf5_hl.a" + ;; + *) + new_libraries="$new_libraries $lib" + ;; + esac + done + libraries="$new_libraries" + fi + + for lib in $libraries; do + if echo $link_args | grep " $lib " > /dev/null || + echo $link_args | grep " $lib$" > /dev/null; then + : + else + link_args="$link_args $lib " + fi + done + + # The LIBS are just a bunch of -l* libraries necessary for the HDF5 + # module. It's okay if they're included twice in the compile line. + link_args="$link_args $H5BLD_LDFLAGS $H5BLD_LIBS" + + # User's CPPFLAGS and CFLAGS come after their H5BLD counterparts. User's + # LDFLAGS come just before clibpath, user's LIBS come after $link_objs and + # before the hdf5 libraries in $link_args, followed by any external library + # paths and libraries from AM_LDFLAGS, LDFLAGS, AM_LIBS or LIBS carried in + # from the hdf5 build. The order of the flags is intended to give precedence + # to the user's flags. + $SHOW $CLINKER $H5BLD_CPPFLAGS $CPPFLAGS $H5BLD_CFLAGS $CFLAGS $LDFLAGS $clibpath $link_objs $LIBS $link_args $shared_link + status=$? +fi + +exit $status diff --git a/bin/h5redeploy.in b/bin/h5redeploy.in new file mode 100644 index 0000000..242459a --- /dev/null +++ b/bin/h5redeploy.in @@ -0,0 +1,216 @@ +#! /bin/sh +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +## Update HDF5 compiler tools after the HDF5 software has been installed ## +## in a new location. ## +## For help page, use "h5redeploy -help" ## +## ## + +# Constants definitions +EXIT_SUCCESS=0 +EXIT_FAILURE=1 + +# Function definitions + +# show help page +usage() { + # A wonderfully informative "usage" message. + echo "usage: $prog_name [OPTIONS]" + echo " OPTIONS:" + echo " -help|help This help message" + echo " -echo Show all the shell commands executed" + echo " -force No prompt, just do it" + echo " -prefix=DIR New directory to find HDF5 lib/ and include/" + echo " subdirectories [default: current directory]" + echo " -exec-prefix=DIR New directory to find HDF5 lib/" + echo " subdirectory [default: <prefix>]" + echo " -libdir=DIR New directory for the HDF5 lib directory" + echo " [default: <exec-prefix>/lib]" + echo " -includedir=DIR New directory for the HDF5 header files" + echo " [default: <prefix>/include]" + echo " -tool=TOOL Tool to update. TOOL must be in the current" + echo " directory and writable. [default: $h5tools]" + echo " -show Show the commands without executing them" + echo " " + exit $EXIT_FAILURE +} + +# display variable values +dump_vars(){ + echo "====Showing all variable values=====" + echo prefix=$prefix + echo h5tools=$h5tools + echo "====End Showing=====" +} + +# show actions to be taken +show_action() +{ + echo "Update the following tools because they are now installed at a new directory" + for t in $foundtools; do + echo "${t}:" + echo " current setting=`sed -e '/^prefix=/s/prefix=//p' -e d $t`" + echo " new setting="\""$prefix"\" + done +} + + +# Report Error message +ERROR() +{ + echo "***ERROR***" + echo "$1" +} + +# Main +# +############################################################################ +## Installation directories: ## +## prefix architecture-independent files. ## +## exec_prefix architecture-dependent files, default is <prefix>. ## +## libdir libraries, default is <exec_prefix>/lib. ## +## includedir header files, default is <prefix/include>. ## +## Not used here: ## +## bindir executables, <exec_prefix/bin>. ## +############################################################################ +# Initialization +h5tools="h5cc h5pcc h5fc h5pfc h5c++" # possible hdf5 tools +foundtools= # tools found and will be modified +fmode= # force mode, default is off +prefix= +exec_prefix= +libdir= +includedir= + +# Parse options +for arg in $@ ; do + case "$arg" in + -prefix=*) + prefix="`echo $arg | cut -f2 -d=`" + ;; + -exec-prefix=*) + exec_prefix="`echo $arg | cut -f2 -d=`" + ;; + -libdir=*) + libdir="`echo $arg | cut -f2 -d=`" + ;; + -includedir=*) + includedir="`echo $arg | cut -f2 -d=`" + ;; + -echo) + set -x + ;; + -show) + SHOW="echo" + ;; + -tool=*) + h5tools="`echo $arg | cut -f2 -d=`" + ;; + -help|help) + usage + ;; + -force) + fmode=yes + ;; + *) + ERROR "Unknown Option($arg)" + usage + exit $EXIT_FAILURE + ;; + esac +done + +# Set to default value, one above where i am, if not given by user +if [ -z "$prefix" ]; then + prefix=`(cd ..;pwd)` +fi +if [ -z "$exec_prefix" ]; then + exec_prefix='${prefix}' # use single quotes to prevent expansion of $ +fi +if [ -z "$libdir" ]; then + libdir='${exec_prefix}'/lib # use single quotes to prevent expansion of $ +fi +if [ -z "$includedir" ]; then + includedir='${prefix}'/include # use single quotes to prevent expansion of $ +fi + +for x in $h5tools; do + if [ -f $x ]; then + foundtools="$foundtools $x" + if [ ! -w $x ]; then + ERROR "h5tool($x) is not writable" + exit $EXIT_FAILURE + fi + fi +done + +if [ -z "$foundtools" ]; then + ERROR "found no tools to modify" + exit $EXIT_FAILURE +fi + +# Show actions to be taken and get consent +show_action +# Ask confirmation unless fmode is on +if [ x-$fmode = x- ]; then + echo "Continue? (yes/no)" + read ansx + ans=`echo $ansx | tr "[A-Z]" "[a-z]"` + if [ x-$ans != x-yes ]; then + echo ABORT. No tools changed. + exit $EXIT_FAILURE + fi +fi + + +# Create the update commands +CMDFILE=/tmp/h5redeploy.$$ +touch $CMDFILE +chmod 0600 $CMDFILE +echo "/^prefix=/c" >> $CMDFILE +echo prefix=\""$prefix"\" >> $CMDFILE +echo . >> $CMDFILE +echo "/^exec_prefix=/c" >> $CMDFILE +echo exec_prefix=\""$exec_prefix"\" >> $CMDFILE +echo . >> $CMDFILE +echo "/^libdir=/c" >> $CMDFILE +echo libdir=\""$libdir"\" >> $CMDFILE +echo . >> $CMDFILE +echo "/^includedir=/c" >> $CMDFILE +echo includedir=\""$includedir"\" >> $CMDFILE +echo . >> $CMDFILE +(echo w; echo q) >> $CMDFILE + + +# Update them +if [ "$SHOW" = "echo" ]; then + echo "===Update commands are:====" + cat $CMDFILE + echo "===End Update commands=====" +fi + +for t in $foundtools; do + echo Update $t ... + COMMAND="ed - $t" + if [ "$SHOW" = "echo" ]; then + echo $COMMAND + else + $COMMAND < $CMDFILE + fi +done + + +# Cleanup +rm -f $CMDFILE +exit $EXIT_SUCCESS @@ -198,7 +198,7 @@ my $H5_JAVA = $file; $H5_JAVA =~ s/[^\/]*$/..\/java\/src\/hdf\/hdf5lib\/H5.java/; die "unable to read file: $H5_JAVA\n" unless -r $file; my $TESTH5_JAVA = $file; -$TESTH5_JAVA =~ s/[^\/]*$/..\/java\/test\/TestH5.java/; +$TESTH5_JAVA =~ s/[^\/]*$/..\/java\/test\/TestH5.java/; die "unable to read file: $TESTH5_JAVA\n" unless -r $file; my $REPACK_LAYOUT_PLUGIN_VERSION = $file; $REPACK_LAYOUT_PLUGIN_VERSION =~ s/[^\/]*$/..\/tools\/test\/h5repack\/testfiles\/h5repack_layout.h5-plugin_version_test.ddl/; @@ -213,7 +213,7 @@ my (@curver) = getvers $contents; # Determine the new version number. my @newver; #new version if ($set) { - if ($set =~ /(\d+)\.(\d+)\.(\d+)(-([a-zA-Z]\w*))?/) { + if ($set =~ /(\d+)\.(\d+)\.(\d+)(-([\da-zA-Z]\w*))?/) { @newver = ($1, $2, $3, $5); } elsif ($set =~ /(\d+)\D+(\d+)\D+(\d+)(\s*\(([a-zA-Z]\w*)\))?\D*$/) { @newver = ($1, $2, $3, $5); @@ -354,7 +354,7 @@ if ($HDF5CONFIGCMAKE) { my $sub_rel_ver_str = ( $newver[3] eq "" ? sprintf("\"%s\"", "") - : sprintf("\"%s\"", "-".$newver[3].", currently under development") + : sprintf("\"%s\"", "-".$newver[3]) ); my $version_string = sprintf("\"%d.%d.%d\"", @newver[0,1,2]); @@ -377,7 +377,7 @@ if ($H5_JAVA) { my $version_string2 = sprintf("%d, %d, %d", @newver[0,1,2]); $data =~ s/\@version HDF5 .* <BR>/\@version HDF5 $version_string1 <BR>/; - $data =~ s/ public final static int LIB_VERSION\[\] = { \d*, \d*, \d* };/ public final static int LIB_VERSION[] = { $version_string2 };/; + $data =~ s/ public final static int LIB_VERSION\[\] = \{ \d*, \d*, \d* \};/ public final static int LIB_VERSION[] = \{ $version_string2 \};/; write_file($H5_JAVA, $data); } @@ -394,8 +394,8 @@ if ($TESTH5_JAVA) { my $version_string1 = sprintf("%d, %d, %d", @newver[0,1,2]); my $version_string2 = sprintf("int majnum = %d, minnum = %d, relnum = %d", @newver[0,1,2]); - $data =~ s/ int libversion\[\] = { .* };/ int libversion\[\] = { $version_string1 };/; - $data =~ s/ int majnum = \d*, minnum = \d*, relnum = \d*;/ $version_string2;/; + $data =~ s/ int libversion\[\] = \{ .* \};/ int libversion\[\] = \{ $version_string1 \};/; + $data =~ s/ int majnum = \d*, minnum = \d*, relnum = \d*;/ $version_string2;/; write_file($TESTH5_JAVA, $data); } @@ -405,12 +405,12 @@ if ($REPACK_LAYOUT_PLUGIN_VERSION) { my $data = read_file($REPACK_LAYOUT_PLUGIN_VERSION); my $version_string = sprintf("%d %d %d", @newver[0,1,2]); - $data =~ s/ PARAMS { 9 \d* \d* \d* }/ PARAMS { 9 $version_string }/g; + $data =~ s/ PARAMS \{ 9 \d* \d* \d* \}/ PARAMS \{ 9 $version_string \}/g; write_file($REPACK_LAYOUT_PLUGIN_VERSION, $data); } -# helper function to read the file for updating c++/src/cpp_doc_config, +# helper function to read the file for updating c++/src/cpp_doc_config, # config/cmake/scripts/HDF5Config.cmake, and java files. # The version string in that file is not at the top, so the string replacement # is not for the first line, and reading/writing the entire file as one string diff --git a/bin/iostats b/bin/iostats index f054b9c..e389992 100755 --- a/bin/iostats +++ b/bin/iostats @@ -1,4 +1,4 @@ -#!/usr/bin/perl +#!/usr/bin/env perl # # Copyright by The HDF Group. # Copyright by the Board of Trustees of the University of Illinois. diff --git a/bin/make_err b/bin/make_err index bfe8861..7f38591 100755 --- a/bin/make_err +++ b/bin/make_err @@ -1,6 +1,7 @@ -#!/usr/bin/perl -w +#!/usr/bin/env perl require 5.003; $indent=4; +use warnings; # # Copyright by The HDF Group. @@ -244,8 +245,8 @@ sub create_init ($) { print HEADER "/* Major error codes */\n"; print HEADER "/*********************/\n\n"; foreach $name (keys %major) { - print HEADER " "x(0*$indent),"assert(${name}_g==(-1));\n"; - print HEADER " "x(0*$indent),"if((msg = H5E_create_msg(cls, H5E_MAJOR, \"${major{$name}}\"))==NULL)\n"; + print HEADER " "x(0*$indent),"HDassert(${name}_g==(-1));\n"; + print HEADER " "x(0*$indent),"if((msg = H5E__create_msg(cls, H5E_MAJOR, \"${major{$name}}\"))==NULL)\n"; print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\")\n"; print HEADER " "x(0*$indent),"if((${name}_g = H5I_register(H5I_ERROR_MSG, msg, FALSE))<0)\n"; print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\")\n"; @@ -260,8 +261,8 @@ sub create_init ($) { # Iterate over all the minor errors in each section for $name ( @{$section_list{$sect_name}}) { - print HEADER " "x(0*$indent),"assert(${name}_g==(-1));\n"; - print HEADER " "x(0*$indent),"if((msg = H5E_create_msg(cls, H5E_MINOR, \"${minor{$name}}\"))==NULL)\n"; + print HEADER " "x(0*$indent),"HDassert(${name}_g==(-1));\n"; + print HEADER " "x(0*$indent),"if((msg = H5E__create_msg(cls, H5E_MINOR, \"${minor{$name}}\"))==NULL)\n"; print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTINIT, FAIL, \"error message initialization failed\")\n"; print HEADER " "x(0*$indent),"if((${name}_g = H5I_register(H5I_ERROR_MSG, msg, FALSE))<0)\n"; print HEADER " "x(1*$indent),"HGOTO_ERROR(H5E_ERROR, H5E_CANTREGISTER, FAIL, \"can't register error message\")\n"; diff --git a/bin/make_overflow b/bin/make_overflow index ccd640e..cee0126 100755 --- a/bin/make_overflow +++ b/bin/make_overflow @@ -1,6 +1,7 @@ -#!/usr/bin/perl -w +#!/usr/bin/env perl require 5.003; use strict; +use warnings; # Global settings diff --git a/bin/make_vers b/bin/make_vers index 7e7fba1..74b0ae2 100755 --- a/bin/make_vers +++ b/bin/make_vers @@ -1,5 +1,6 @@ -#!/usr/bin/perl -w +#!/usr/bin/env perl require 5.003; +use warnings; # Global settings # (The max_idx parameter is the only thing that needs to be changed when adding @@ -7,8 +8,8 @@ require 5.003; # is added (like support for 1.4, etc), the min_sup_idx parameter will # need to be decremented. - QAK) -# Max. library "index" (0 = v1.0, 1 = 1.2, etc) -$max_idx = 5; +# Max. library "index" (0 = v1.0, 1 = 1.2, 2 = 1.4, 3 = 1.6, 4 = 1.8, 5 = 1.10, 6 = 1.12, 7 = 1.14, etc) +$max_idx = 7; # Min. supported previous library version "index" (0 = v1.0, 1 = 1.2, etc) $min_sup_idx = 3; @@ -89,7 +90,8 @@ sub print_checkoptions ($) { my $curr_idx; # Current API version index # Print the option checking - print $fh "\n/* Issue error if contradicting macros have been defined. */\n"; + print $fh "\n\n/* Issue error if contradicting macros have been defined. */\n"; + print $fh "/* (Can't use an older (deprecated) API version if deprecated symbols have been disabled) */\n"; # Print the #ifdef print $fh "#if ("; @@ -118,26 +120,40 @@ sub print_checkoptions ($) { ############################################################################## # Print "global" API version macro settings # -sub print_globalapivers ($) { +sub print_globalapidefvers ($) { my $fh = shift; # File handle for output file my $curr_idx; # Current API version index # Print the descriptive comment - print $fh "\n\n/* If a particular \"global\" version of the library's interfaces is chosen,\n"; - print $fh " * set the versions for the API symbols affected.\n"; + print $fh "\n\n/* If a particular default \"global\" version of the library's interfaces is\n"; + print $fh " * chosen, set the corresponding version macro for API symbols.\n"; print $fh " *\n"; - print $fh " * Note: If an application has already chosen a particular version for an\n"; - print $fh " * API symbol, the individual API version macro takes priority.\n"; print $fh " */\n"; for $curr_idx ($min_sup_idx .. ($max_idx - 1)) { # Print API version ifdef - print $fh "#if defined(H5_USE_1", ($curr_idx * 2), "_API_DEFAULT) && !defined(H5_USE_1", ($curr_idx * 2), "_API)\n"; + print $fh "\n#if defined(H5_USE_1", ($curr_idx * 2), "_API_DEFAULT) && !defined(H5_USE_1", ($curr_idx * 2), "_API)\n"; # Print API version definition print $fh " " x $indent, "#define H5_USE_1", ($curr_idx * 2), "_API 1\n"; # Print API version endif - print $fh "#endif /* H5_USE_1", ($curr_idx * 2), "_API_DEFAULT && !H5_USE_1", ($curr_idx * 2), "_API */\n\n"; + print $fh "#endif /* H5_USE_1", ($curr_idx * 2), "_API_DEFAULT && !H5_USE_1", ($curr_idx * 2), "_API */\n"; } +} + +############################################################################## +# Print "global" API symbol version macro settings +# +sub print_globalapisymbolvers ($) { + my $fh = shift; # File handle for output file + my $curr_idx; # Current API version index + + # Print the descriptive comment + print $fh "\n\n/* If a particular \"global\" version of the library's interfaces is chosen,\n"; + print $fh " * set the versions for the API symbols affected.\n"; + print $fh " *\n"; + print $fh " * Note: If an application has already chosen a particular version for an\n"; + print $fh " * API symbol, the individual API version macro takes priority.\n"; + print $fh " */\n"; # Loop over supported older library APIs and define the appropriate macros for $curr_idx ($min_sup_idx .. ($max_idx - 1)) { @@ -338,7 +354,18 @@ sub parse_line ($) { my $vers_idx; # Index of version in array # Do some validation on the input - if(!( $_ =~ /v1[02468]/ || $_ =~ /v11[02468]/ )) { + # Note: v111 is allowed because H5O functions were prematurely versioned + # in HDF5 1.10. Because users were affected by this, the versioning + # was rescinded but the H5O version 2 functions were kept to be + # called directly. Now that the version macros are added in 1.12, + # along with a 3rd version of the H5O functions, the H5O function + # version for default api=v110 should be version 1 to work correctly + # with 1.10 applications that were using unversioned H5O functions, + # and the H5O function version should be version 3 for default api=v112 + # (the default api version for 1.12). Allowing a v111 entry and + # incrementing its index 13 lines below allows a version 2 that is + # never accessed via the H5O function macros. + if(!( $_ =~ /v1[02468]/ || $_ =~ /v11[02468]/ || $_ =~ /v111/ )) { die "bad version information: $name"; } if(exists($sym_versions{$_})) { @@ -351,6 +378,9 @@ sub parse_line ($) { #print "parse_line: _=$_\n"; # Get the index of the version ($vers_idx) = ($_ =~ /v1(\d+)/); + if($vers_idx == 11) { + $vers_idx++; + } $vers_idx /= 2; #print "parse_line: vers_idx='$vers_idx'\n"; push(@vers_nums, $vers_idx); @@ -443,8 +473,9 @@ sub create_public ($) { print_copyright(*HEADER); print_warning(*HEADER); print_startprotect(*HEADER, $file); + print_globalapidefvers(*HEADER); print_checkoptions(*HEADER); - print_globalapivers(*HEADER); + print_globalapisymbolvers(*HEADER); print_defaultapivers(*HEADER); print_endprotect(*HEADER, $file); @@ -478,7 +509,7 @@ for $file (@ARGV) { } } close SOURCE; - + # Create header files print "Generating '", $prefix, "H5version.h'\n"; create_public($prefix); diff --git a/bin/makehelp b/bin/makehelp index d6fbda2..4707397 100755 --- a/bin/makehelp +++ b/bin/makehelp @@ -40,6 +40,7 @@ make distclean: remove all files generated by make, make check, or make check-p: Only run parallel tests make check-s: Only run serial tests make check-vfd: Run tests with each virtual file driver +make check-vol: Run tests with each virtual object layer connector HDF5 uses Automake, so any standard Automake targets not listed here should also work. diff --git a/bin/release b/bin/release index 6358b0d..a34191e 100755 --- a/bin/release +++ b/bin/release @@ -39,29 +39,37 @@ USAGE() { cat << EOF Usage: $0 -d <dir> [--docver BRANCHNAME] [-h] [--nocheck] [--private] <methods> ... - -d DIR The name of the directory where the releas(es) should be + -d DIR The name of the directory where the releas(es) should be placed. --docver BRANCHNAME This is added for 1.8 and beyond to get the correct version of documentation files from the hdf5docs repository. BRANCHNAME for v1.8 should be hdf5_1_8. -h print the help page. - --nocheck Ignore errors in MANIFEST file. - --private Make a private release with today's date in version information. - + --nocheck Ignore errors in MANIFEST file. + --private Make a private release with today's date in version information. + This must be run at the top level of the source directory. The other command-line options are the names of the programs to use for compressing the resulting tar archive (if none are given then "tar" is assumed): - tar -- use tar and don't do any compressing. - gzip -- use gzip with "-9" and append ".gz" to the output name. - cmake-tgz -- create a tar file using the gzip default level with a build-unix.sh + tar -- use tar and don't do any compressing. + gzip -- use gzip with "-9" and append ".gz" to the output name. + bzip2 -- use bzip2 with "-9" and append ".bz2" to the output name. + zip -- convert all text files to DOS style and form a zip file for Windows use. + cmake-tgz -- create a tar file using the gzip default level with a build-unix.sh command file and all other CMake files needed to build HDF5 source using CMake on unix machines. - bzip2 -- use bzip2 with "-9" and append ".bz2" to the output name. - zip -- convert all text files to DOS style and form a zip file for Windows use. cmake-zip -- convert all text files to DOS style and create a zip file inluding cmake - scripts and .bat files to build HDF5 source using CMake on Windows. + scripts and .bat files to build HDF5 source using CMake on Windows. + hpc-cmake-tgz + -- create a tar file using the gzip default level with a build-unix.sh + command file and all other CMake files needed to build HDF5 source + using CMake on unix machines, with HDF5options.cmake files for serial + and parallel builds on machines requiring batch jobs to run tests. + The default is for parallel build, with serial only build by changing + the HDF5options.cmake symlink to ser-HDF5options.cmake. More + information is available in the README_HPC file. doc -- produce the latest doc tree in addition to the archive. An md5 checksum is produced for each archive created and stored in the md5 file. @@ -114,8 +122,8 @@ EOF tar2zip() { if [ $# -ne 3 ]; then - echo "usage: tar2zip <tarfilename> <zipfilename>" - return 1 + echo "usage: tar2zip <tarfilename> <zipfilename>" + return 1 fi ztmpdir=/tmp/ztmpdir$$ mkdir -p $ztmpdir @@ -127,23 +135,23 @@ tar2zip() (cd $ztmpdir; tar xf -) < $tarfile # sanity check if [ ! -d $ztmpdir/$version ]; then - echo "untar did not create $ztmpdir/$version source dir" - # cleanup - rm -rf $ztmpdir - return 1 + echo "untar did not create $ztmpdir/$version source dir" + # cleanup + rm -rf $ztmpdir + return 1 fi # step 2: convert text files # There maybe a simpler way to do this. # options used in unix2dos: - # -k Keep the date stamp + # -k Keep the date stamp # -q quiet mode # grep redirect output to /dev/null because -q or -s are not portable. find $ztmpdir/$version | \ - while read inf; do \ - if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ - unix2dos -q -k $inf; \ - fi\ - done + while read inf; do \ + if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ + unix2dos -q -k $inf; \ + fi\ + done # step 3: make zipball # -9 maximum compression # -y Store symbolic links as such in the zip archive @@ -169,14 +177,14 @@ tar2zip() # Note: do this in a temporary directory to avoid changing # the original source directory which may be around. # 2. add build-unix.sh script. -# 3. add SZIP.tar.gz, ZLib.tar.gz and cmake files to top level directory. +# 3. add LIBAEC.tar.gz, ZLib.tar.gz and cmake files to top level directory. # 4. create gzipped tar file with these contents: # build-unix.sh script # hdf5-<version> source code directory extracted from tar file # CTestScript.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts # HDF5config.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts # HDF5options.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts -# SZip.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake +# LIBAEC.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake # ZLib.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake @@ -188,58 +196,61 @@ tar2zip() # Returns 0 if successful; 1 otherwise # # need function to create another temporary directory, extract the - # $tmpdir/$HDF5_VERS.tar into it, add (create) build-unix.sh, - # CTestScript.cmake, HDF5config.cmake, SZIP.tar.gz and ZLib.tar.gz, - # and then tar.gz it. + # $tmpdir/$HDF5_VERS.tar into it, create build-VS*.bat files, + # add CTestScript.cmake, HDF5config.cmake, LIBAEC.tar.gz + # ZLib.tar.gz, HDF5 examples, and then zip it. tar2cmakezip() { if [ $# -ne 3 ]; then - echo "usage: tar2cmakezip <tarfilename> <tgzfilename>" - return 1 + echo "usage: tar2cmakezip <tarfilename> <zipfilename>" + return 1 fi cmziptmpdir=/tmp/cmziptmpdir$$ - mkdir -p $cmziptmpdir + cmziptmpsubdir=$cmziptmpdir/CMake-$HDF5_VERS + mkdir -p $cmziptmpsubdir version=$1 tarfile=$2 zipfile=$3 # step 1: untar tarball in cmgztmpdir - (cd $cmziptmpdir; tar xf -) < $tarfile + (cd $cmziptmpsubdir; tar xf -) < $tarfile # sanity check - if [ ! -d $cmziptmpdir/$version ]; then - echo "untar did not create $cmziptmpdir/$version source dir" - # cleanup - rm -rf $cmziptmpdir - return 1 + if [ ! -d $cmziptmpsubdir/$version ]; then + echo "untar did not create $cmziptmpsubdir/$version source dir" + # cleanup + rm -rf $cmziptmpdir + return 1 fi # step 2: add batch file for building CMake on window - cp /mnt/scr1/pre-release/hdf5/CMake/build-VS2013-32.bat $cmziptmpdir - cp /mnt/scr1/pre-release/hdf5/CMake/build-VS2013-64.bat $cmziptmpdir - cp /mnt/scr1/pre-release/hdf5/CMake/build-VS2015-32.bat $cmziptmpdir - cp /mnt/scr1/pre-release/hdf5/CMake/build-VS2015-64.bat $cmziptmpdir - cp /mnt/scr1/pre-release/hdf5/CMake/build-VS2017-32.bat $cmziptmpdir - cp /mnt/scr1/pre-release/hdf5/CMake/build-VS2017-64.bat $cmziptmpdir - - # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files - cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmziptmpdir - cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmziptmpdir - cp $cmziptmpdir/$version/config/cmake/scripts/CTestScript.cmake $cmziptmpdir - cp $cmziptmpdir/$version/config/cmake/scripts/HDF5config.cmake $cmziptmpdir - cp $cmziptmpdir/$version/config/cmake/scripts/HDF5options.cmake $cmziptmpdir + (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS2013 -C Release -V -O hdf5.log" > build-VS2013-32.bat; chmod 755 build-VS2013-32.bat) + (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201364 -C Release -V -O hdf5.log" > build-VS2013-64.bat; chmod 755 build-VS2013-64.bat) + (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS2015 -C Release -V -O hdf5.log" > build-VS2015-32.bat; chmod 755 build-VS2015-32.bat) + (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201564 -C Release -V -O hdf5.log" > build-VS2015-64.bat; chmod 755 build-VS2015-64.bat) + (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS2017 -C Release -V -O hdf5.log" > build-VS2017-32.bat; chmod 755 build-VS2017-32.bat) + (cd $cmziptmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=VS201764 -C Release -V -O hdf5.log" > build-VS2017-64.bat; chmod 755 build-VS2017-64.bat) + + # step 3: add LIBAEC.tar.gz, ZLib.tar.gz and cmake files + cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmziptmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmziptmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.1-Source.zip $cmziptmpsubdir + cp $cmziptmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmziptmpsubdir + cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmziptmpsubdir + cp $cmziptmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmziptmpsubdir # step 4: convert text files # There maybe a simpler way to do this. # options used in unix2dos: - # -k Keep the date stamp + # -k Keep the date stamp # -q quiet mode # grep redirect output to /dev/null because -q or -s are not portable. - find $cmziptmpdir/$version | \ - while read inf; do \ - if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ - unix2dos -q -k $inf; \ - fi\ - done + find $cmziptmpsubdir/$version | \ + while read inf; do \ + if file $inf | grep "$inf\: .*text" > /dev/null 2>&1 ; then \ + unix2dos -q -k $inf; \ + fi\ + done + # step 3: make zipball # -9 maximum compression # -y Store symbolic links as such in the zip archive @@ -253,7 +264,7 @@ tar2cmakezip() } # Function name: tar2cmaketgz -# Convert the release tarball to a Windows zipball with files to run CMake build. +# Convert the release tarball to a gzipped tar file with files to run CMake build. # # Programmer: Larry Knox # Creation date: 2017-02-20 @@ -265,14 +276,14 @@ tar2cmakezip() # Note: do this in a temporary directory to avoid changing # the original source directory which may be around. # 2. add build-unix.sh script. -# 3. add SZIP.tar.gz, ZLib.tar.gz and cmake files to top level directory. +# 3. add LIBAEC.tar.gz, ZLib.tar.gz and cmake files to top level directory. # 4. create gzipped tar file with these contents: # build-unix.sh script # hdf5-<version> source code directory extracted from tar file # CTestScript.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts # HDF5config.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts # HDF5options.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts -# SZip.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake +# LIBAEC.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake # ZLib.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake @@ -284,43 +295,132 @@ tar2cmakezip() # Returns 0 if successful; 1 otherwise # # need function to create another temporary directory, extract the - # $tmpdir/$HDF5_VERS.tar into it, add (create) build-unix.sh, - # CTestScript.cmake, HDF5config.cmake, SZIP.tar.gz and ZLib.tar.gz, - # and then tar.gz it. + # $tmpdir/$HDF5_VERS.tar into it, create build-unix.sh, + # add CTestScript.cmake, HDF5config.cmake, LIBAEC.tar.gz + # ZLib.tar.gz, HDF5 examples, and then tar.gz it. tar2cmaketgz() { if [ $# -ne 3 ]; then - echo "usage: tar2cmaketgz <tarfilename> <tgzfilename>" - return 1 + echo "usage: tar2cmaketgz <tarfilename> <tgzfilename>" + return 1 + fi + cmgztmpdir=/tmp/cmgztmpdir$$ + cmgztmpsubdir=$cmgztmpdir/CMake-$HDF5_VERS + mkdir -p $cmgztmpsubdir + version=$1 + tarfile=$2 + tgzfile=$3 + + # step 1: untar tarball in cmgztmpdir + (cd $cmgztmpsubdir; tar xf -) < $tarfile + # sanity check + if [ ! -d $cmgztmpsubdir/$version ]; then + echo "untar did not create $cmgztmpsubdir/$version source dir" + # cleanup + rm -rf $cmgztmpdir + return 1 + fi + + + # step 2: add build-unix.sh script + (cd $cmgztmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log" > build-unix.sh; chmod 755 build-unix.sh) + + # step 3: add LIBAEC.tar.gz, ZLib.tar.gz and cmake files + cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.1-Source.tar.gz $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpsubdir + tar czf $DEST/CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 + + # cleanup + rm -rf $cmgztmpdir +} + +# Function name: tar2hpccmaketgz +# Convert the release tarball to a gzipped tarfile with files to run CMake build +# and HDF5options.cmake files for parallel or serial only builds where build +# tests are run on compute nodes using batch scripts. +# +# Programmer: Larry Knox +# Creation date: 2019-01-28 +# +# Modifications +# +# Steps: +# 1. untar the tarball in a temporary directory; +# Note: do this in a temporary directory to avoid changing +# the original source directory which may be around. +# 2. add build-unix.sh script. +# 3. add LIBAEC.tar.gz, ZLib.tar.gz and cmake files to top level directory. +# 4. create gzipped tar file with these contents: +# build-unix.sh script +# hdf5-<version> source code directory extracted from tar file +# CTestScript.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts +# HDF5config.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts +# HDF5options.cmake cmake file copied from <hdf5 source code>/config/cmake/scripts +# LIBAEC.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake +# ZLib.tar.gz copied from /mnt/scr1/pre-release/hdf5/CMake +# +# 5. For HPC-CMake tgz file the following are also needed in the top-level directory: +# README_HPC copied from release_docs +# ser-HDF5options.cmake copied from <hdf5 source code>/config/cmake/scripts/HPC +# par-HDF5options.cmake copied from <hdf5 source code>/config/cmake/scripts/HPC +# HDF5options.cmake symlink to par-HDF5options.cmake +# + +# Parameters: +# $1 version +# $2 release tarball +# $3 output zipball file name +# +# Returns 0 if successful; 1 otherwise +# + # need function to create another temporary directory, extract the + # $tmpdir/$HDF5_VERS.tar into it, create build-unix.sh, + # add CTestScript.cmake, HDF5config.cmake, LIBAEC.tar.gz + # ZLib.tar.gz, HDF5 examples, and then tar.gz it. +tar2hpccmaketgz() +{ + if [ $# -ne 3 ]; then + echo "usage: tar2hpccmaketgz <tarfilename> <tgzfilename>" + return 1 fi cmgztmpdir=/tmp/cmgztmpdir$$ - mkdir -p $cmgztmpdir + cmgztmpsubdir=$cmgztmpdir/HPC-CMake-$HDF5_VERS + mkdir -p $cmgztmpsubdir version=$1 tarfile=$2 tgzfile=$3 # step 1: untar tarball in cmgztmpdir - (cd $cmgztmpdir; tar xf -) < $tarfile + (cd $cmgztmpsubdir; tar xf -) < $tarfile # sanity check - if [ ! -d $cmgztmpdir/$version ]; then - echo "untar did not create $cmgztmpdir/$version source dir" - # cleanup - rm -rf $cmgztmpdir - return 1 + if [ ! -d $cmgztmpsubdir/$version ]; then + echo "untar did not create $cmgztmpsubdir/$version source dir" + # cleanup + rm -rf $cmgztmpdir + return 1 fi # step 2: add build-unix.sh script - (cd $cmgztmpdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log" > build-unix.sh; chmod 755 build-unix.sh) - - # step 3: add SZIP.tar.gz, ZLib.tar.gz and cmake files - cp /mnt/scr1/pre-release/hdf5/CMake/SZip.tar.gz $cmgztmpdir - cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpdir - cp $cmgztmpdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpdir - cp $cmgztmpdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpdir - cp $cmgztmpdir/$version/config/cmake/scripts/HDF5options.cmake $cmgztmpdir - tar czf $DEST/CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 - + (cd $cmgztmpsubdir; echo "ctest -S HDF5config.cmake,BUILD_GENERATOR=Unix -C Release -V -O hdf5.log" > build-unix.sh; chmod 755 build-unix.sh) + + # step 3: add LIBAEC.tar.gz, ZLib.tar.gz and cmake files + cp /mnt/scr1/pre-release/hdf5/CMake/LIBAEC.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/ZLib.tar.gz $cmgztmpsubdir + cp /mnt/scr1/pre-release/hdf5/CMake/HDF5Examples-1.14.1-Source.tar.gz $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/CTestScript.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HDF5config.cmake $cmgztmpsubdir + + cp $cmgztmpsubdir/$version/release_docs/README_HPC $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/ser-HDF5options.cmake $cmgztmpsubdir + cp $cmgztmpsubdir/$version/config/cmake/scripts/HPC/par-HDF5options.cmake $cmgztmpsubdir + (cd $cmgztmpsubdir; ln -s par-HDF5options.cmake HDF5options.cmake) + tar czf $DEST/HPC-CMake-$HDF5_VERS.tar.gz -C $cmgztmpdir . || exit 1 + # cleanup rm -rf $cmgztmpdir } @@ -342,7 +442,7 @@ check=yes release_date=`date +%F` today=`date +%Y%m%d` pmode='no' -tmpdir="../#release_tmp.$$" # tmp work directory +tmpdir="../#release_tmp.$$" # tmp work directory DOC_URL=https://git@bitbucket.hdfgroup.org/scm/hdffv/hdf5doc.git CPPLUS_RM_NAME=cpplus_RM MAINT_MODE_ENABLED="" @@ -359,11 +459,11 @@ fi RESTORE_VERSION() { if [ X-${VERS_OLD} != X- ]; then - echo restoring version information back to $VERS_OLD - rm -f config/lt_vers.am - cp $tmpdir/lt_vers.am config/lt_vers.am - bin/h5vers -s $VERS_OLD - VERS_OLD= + echo restoring version information back to $VERS_OLD + rm -f config/lt_vers.am + cp $tmpdir/lt_vers.am config/lt_vers.am + bin/h5vers -s $VERS_OLD + VERS_OLD= fi } @@ -373,32 +473,32 @@ while [ -n "$1" ]; do arg=$1 shift case "$arg" in - -d) - DEST=$1 - shift - ;; - --nocheck) - check=no - ;; - -h) - USAGE - exit 0 - ;; - --private) - pmode=yes - ;; + -d) + DEST=$1 + shift + ;; + --nocheck) + check=no + ;; + -h) + USAGE + exit 0 + ;; + --private) + pmode=yes + ;; --docver) DOCVERSION=$1 shift ;; - -*) - echo "Unknown switch: $arg" 1>&2 - USAGE - exit 1 - ;; - *) - methods="$methods $arg" - ;; + -*) + echo "Unknown switch: $arg" 1>&2 + USAGE + exit 1 + ;; + *) + methods="$methods $arg" + ;; esac done @@ -445,9 +545,15 @@ fi bin/chkmanifest || fail=yes if [ "X$fail" = "Xyes" ]; then if [ $check = yes ]; then + echo "" + echo "Note! If you are running bin/release in a development branch" + echo "later than v 1.8 the MANIFEST check is expected to fail when" + echo "autogen.sh has not been run successfully. Either run autogen.sh " + echo "with /usr/hdf/bin/AUTOTOOLS at the beginning of PATH or add the" + echo "--nocheck argument to the bin/release command." exit 1 else - echo "Continuing anyway..." + echo "Continuing anyway..." fi fi @@ -481,7 +587,7 @@ test "$verbose" && echo " Running tar..." 1>&2 ( \ cd $tmpdir; \ tar cf $HDF5_VERS.tar $HDF5_VERS/Makefile \ - `sed 's/^\.\//hdf5-'$VERS'\//' $MANIFEST` || exit 1 \ + `sed 's/^\.\//hdf5-'$VERS'\//' $MANIFEST` || exit 1 \ ) # Compress @@ -489,56 +595,61 @@ MD5file=$HDF5_VERS.md5 cp /dev/null $DEST/$MD5file for comp in $methods; do case $comp in - tar) - cp -p $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.tar - (cd $DEST; md5sum $HDF5_VERS.tar >> $MD5file) - ;; - gzip) - test "$verbose" && echo " Running gzip..." 1>&2 - gzip -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.gz - (cd $DEST; md5sum $HDF5_VERS.tar.gz >> $MD5file) - ;; + tar) + cp -p $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.tar + (cd $DEST; md5sum $HDF5_VERS.tar >> $MD5file) + ;; + gzip) + test "$verbose" && echo " Running gzip..." 1>&2 + gzip -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.gz + (cd $DEST; md5sum $HDF5_VERS.tar.gz >> $MD5file) + ;; cmake-tgz) - test "$verbose" && echo " Creating CMake tar.gz file..." 1>&2 - tar2cmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/CMake-$HDF5_VERS.tar.gz 1>&2 - (cd $DEST; md5sum CMake-$HDF5_VERS.tar.gz >> $MD5file) + test "$verbose" && echo " Creating CMake tar.gz file..." 1>&2 + tar2cmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/CMake-$HDF5_VERS.tar.gz 1>&2 + (cd $DEST; md5sum CMake-$HDF5_VERS.tar.gz >> $MD5file) ;; - bzip2) - test "$verbose" && echo " Running bzip2..." 1>&2 - bzip2 -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.bz2 - (cd $DEST; md5sum $HDF5_VERS.tar.bz2 >> $MD5file) - ;; - zip) - test "$verbose" && echo " Creating zip ball..." 1>&2 - tar2zip $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.zip 1>&2 - (cd $DEST; md5sum $HDF5_VERS.zip >> $MD5file) - ;; + hpc-cmake-tgz) + test "$verbose" && echo " Creating HPC-CMake tar.gz file..." 1>&2 + tar2hpccmaketgz $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/HPC-CMake-$HDF5_VERS.tar.gz 1>&2 + (cd $DEST; md5sum HPC-CMake-$HDF5_VERS.tar.gz >> $MD5file) + ;; + bzip2) + test "$verbose" && echo " Running bzip2..." 1>&2 + bzip2 -9 <$tmpdir/$HDF5_VERS.tar >$DEST/$HDF5_VERS.tar.bz2 + (cd $DEST; md5sum $HDF5_VERS.tar.bz2 >> $MD5file) + ;; + zip) + test "$verbose" && echo " Creating zip ball..." 1>&2 + tar2zip $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/$HDF5_VERS.zip 1>&2 + (cd $DEST; md5sum $HDF5_VERS.zip >> $MD5file) + ;; cmake-zip) test "$verbose" && echo " Creating CMake-zip ball..." 1>&2 tar2cmakezip $HDF5_VERS $tmpdir/$HDF5_VERS.tar $DEST/CMake-$HDF5_VERS.zip 1>&2 (cd $DEST; md5sum CMake-$HDF5_VERS.zip >> $MD5file) ;; - doc) + doc) if [ "${DOCVERSION}" = "" ]; then DOCVERSION=master fi - test "$verbose" && echo " Creating docs..." 1>&2 - # Check out docs from git repo - (cd $tmpdir; git clone -q $DOC_URL ${DOCVERSION} > /dev/null) || exit 1 + test "$verbose" && echo " Creating docs..." 1>&2 + # Check out docs from git repo + (cd $tmpdir; git clone -q $DOC_URL ${DOCVERSION} > /dev/null) || exit 1 # Create doxygen C++ RM - (cd c++/src && doxygen cpp_doc_config > /dev/null ) || exit 1 - # Replace version of C++ RM with just-created version - rm -rf $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME || exit 1 - mv c++/src/$CPPLUS_RM_NAME $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME || exit 1 + (cd c++/src && doxygen cpp_doc_config > /dev/null ) || exit 1 + # Replace version of C++ RM with just-created version + rm -rf $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME || exit 1 + mv c++/src/$CPPLUS_RM_NAME $tmpdir/${DOCVERSION}/html/$CPPLUS_RM_NAME || exit 1 # Compress the docs and move them to the release area - mv $tmpdir/${DOCVERSION} $tmpdir/${HDF5_VERS}_docs || exit 1 - (cd $tmpdir && tar cf ${HDF5_VERS}_docs.tar ${HDF5_VERS}_docs) || exit 1 - mv $tmpdir/${HDF5_VERS}_docs.tar $DEST || exit 1 - ;; - *) - echo "***Error*** Unknown method $comp" - exit 1 - ;; + mv $tmpdir/${DOCVERSION} $tmpdir/${HDF5_VERS}_docs || exit 1 + (cd $tmpdir && tar cf ${HDF5_VERS}_docs.tar ${HDF5_VERS}_docs) || exit 1 + mv $tmpdir/${HDF5_VERS}_docs.tar $DEST || exit 1 + ;; + *) + echo "***Error*** Unknown method $comp" + exit 1 + ;; esac done diff --git a/bin/runbkgprog b/bin/runbkgprog new file mode 100755 index 0000000..f04ea89 --- /dev/null +++ b/bin/runbkgprog @@ -0,0 +1,88 @@ +#!/usr/bin/env perl +require 5.003; +use warnings; +$indent=4; + +# +# Copyright by The HDF Group. +# Copyright by the Board of Trustees of the University of Illinois. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the COPYING file, which can be found at the root of the source code +# distribution tree, or in https://support.hdfgroup.org/ftp/HDF5/releases. +# If you do not have access to either file, you may request a copy from +# help@hdfgroup.org. +# + +# Run program in background +# +use warnings; +use strict; + +use Carp; +use Time::HiRes; +use POSIX 'setsid'; + +my $child_pid; +my $child_proc; +my $cmd = $ARGV[0]; +my $debug = 1; + +print "\nStart child process\n"; +start_child(); +print "\nStarted child process\n"; + +sub start_child { + die "cannot execute cmd: $cmd" unless -x $cmd; + if ($^O eq 'MSWin32') { # Windows + require Win32::Process; + Win32::Process::Create($child_proc, $cmd, $cmd, 0, 0, ".") || confess "Could not spawn child: $!"; + $child_pid = $child_proc->GetProcessID(); + } + else { # Unix + $SIG{CHLD} = 'IGNORE'; + $child_pid = fork(); + unless (defined $child_pid) { + confess "Could not spawn child (Unix): $!"; + } + if ($child_pid == 0) { # child + unless ($debug) { + open STDIN, "<", "/dev/null" or die "Can't read /dev/null: $!"; + open STDOUT, ">", "/dev/null" or die "Can't write /dev/null: $!"; + } + setsid or warn "setsid cannot start a new session: $!"; + unless ($debug) { + open STDERR, '>&STDOUT' or die "Can't dup stdout: $!"; + } + local $| = 1; + unless (exec($cmd)) { + confess "Could not start child: $cmd: $!"; + CORE::exit(0); + } + } + # parent + $SIG{CHLD} = 'DEFAULT'; + } + # catch early child exit, e.g. if program path is incorrect + sleep(1.0); + POSIX::waitpid(-1, POSIX::WNOHANG()); # clean up any defunct child process + if (kill(0,$child_pid)) { + print "\nStarted child process id $child_pid\n"; + } + else { + warn "Child process exited quickly: $cmd: process $child_pid"; + } +} + +sub stop_child +{ + if ($^O eq 'MSWin32') { # Windows + Win32::Process::KillProcess($child_pid,0); + } + else { # Unix + kill 9, $child_pid || warn "could not kill process $child_pid: $!"; + } + print "Stopped child process id $child_pid\n"; +} diff --git a/bin/snapshot b/bin/snapshot index 5c78fc8..1218caa 100755 --- a/bin/snapshot +++ b/bin/snapshot @@ -130,9 +130,10 @@ DISPLAYUSAGE() set - cat <<EOF Usage: $PROGNAME [all] [checkout] [ftp <URL> [diff] [test] [srcdir] [release] [help] - [clean] [distclean] [echo] [deploy <dir>] [deploydir <dir>] - [zlib <zlib_path>] [releasedir <dir>] [srcdirname <dir>] [check-vfd] - [exec <command>] [module-load <module-list>] [op-configure <option>] + [clean] [distclean] [echo] [deploy <dir>] [deploydir <dir>] + [zlib <zlib_path>] [releasedir <dir>] [srcdirname <dir>] [check-vfd] + [check-passthrough-vol] + [exec <command>] [module-load <module-list>] [op-configure <option>] [--<option>] all: Run all commands (checkout, test & release) [Default is all] @@ -149,8 +150,8 @@ Usage: $PROGNAME [all] [checkout] [ftp <URL> [diff] [test] [srcdir] [release] [h setenv <name> <value>: Set environment variable <name> to <value>. setenvN <N> <name> <value> ...: - Set environment variable with <N> values. - E.g., setenvN 3 x a b c is same as setenv x="a b c". + Set environment variable with <N> values. + E.g., setenvN 3 x a b c is same as setenv x="a b c". srcdir: Use srcdir option (does not imply other commands) "snapshot srcdir" is equivalent to "snapshot srcdir all" "snapshot srcdir checkout" is equivalent to "snapshot checkout" @@ -168,7 +169,12 @@ Usage: $PROGNAME [all] [checkout] [ftp <URL> [diff] [test] [srcdir] [release] [h Use <dir> as the release directory [Default is $ReleaseDir_default] check-vfd: - Run make check-vfd instead of just make check. + Run make check-vfd instead of just make check. + check-passthrough-vol: + Run make check-passthrough-vol instead of just make check. + NOTE: Will only succeed with passthrough VOL connectors + that use the native VOL connector as the terminal + connector. exttest <testscript>; Run testscript; exec <command>: @@ -422,6 +428,9 @@ while [ $# -gt 0 ] ; do check-vfd) CHECKVAL=check-vfd ;; + check-passthrough-vol) + CHECKVAL=check-passthrough-vol + ;; module-load) shift if [ $# -lt 1 ]; then @@ -1,4 +1,4 @@ -#!/usr/bin/perl -w +#!/usr/bin/env perl ## # Copyright by The HDF Group. # Copyright by the Board of Trustees of the University of Illinois. @@ -12,6 +12,7 @@ # help@hdfgroup.org. ## require 5.003; +use warnings; $Source = ""; ############################################################################## @@ -42,10 +43,12 @@ $Source = ""; "H5D_space_status_t" => "Ds", "H5D_vds_view_t" => "Dv", "H5FD_mpio_xfer_t" => "Dt", + "H5FD_splitter_vfd_config_t" => "Dr", "herr_t" => "e", "H5E_direction_t" => "Ed", "H5E_error_t" => "Ee", "H5E_type_t" => "Et", + "H5ES_status_t" => "Es", "H5F_close_degree_t" => "Fd", "H5F_fspace_strategy_t" => "Ff", "H5F_file_space_type_t" => "Ff", @@ -67,7 +70,9 @@ $Source = ""; "unsigned" => "Iu", "unsigned int" => "Iu", "uint32_t" => "Iu", + "uint64_t" => "UL", "H5I_type_t" => "It", + "H5O_token_t" => "k", "H5G_link_t" => "Ll", #Same as H5L_type_t now "H5L_type_t" => "Ll", "MPI_Comm" => "Mc", @@ -76,7 +81,9 @@ $Source = ""; "off_t" => "o", "H5O_type_t" => "Ot", "H5P_class_t" => "p", - "hobj_ref_t" => "r", + "hobj_ref_t" => "Ro", + "hdset_reg_ref_t" => "Rd", + "H5R_ref_t" => "Rr", "H5R_type_t" => "Rt", "char" => "s", "unsigned char" => "s", @@ -95,8 +102,38 @@ $Source = ""; "H5T_str_t" => "Tz", "unsigned long" => "Ul", "unsigned long long" => "UL", + "H5VL_subclass_t" => "VS", + "H5VL_get_conn_lvl_t" => "VL", + "H5VL_attr_get_t" => "Va", + "H5VL_attr_optional_t" => "Vs", + "H5VL_attr_specific_t" => "Vb", + "H5VL_blob_specific_t" => "VB", + "H5VL_class_value_t" => "VC", + "H5VL_dataset_get_t" => "Vc", + "H5VL_dataset_specific_t" => "Vd", + "H5VL_dataset_optional_t" => "Vt", + "H5VL_datatype_get_t" => "Ve", + "H5VL_datatype_specific_t" => "Vf", + "H5VL_datatype_optional_t" => "Vu", + "H5VL_file_get_t" => "Vg", + "H5VL_file_specific_t" => "Vh", + "H5VL_file_optional_t" => "Vv", + "H5VL_group_get_t" => "Vi", + "H5VL_group_specific_t" => "Vj", + "H5VL_group_optional_t" => "Vw", + "H5VL_link_create_type_t" => "Vk", + "H5VL_link_get_t" => "Vl", + "H5VL_link_specific_t" => "Vm", + "H5VL_link_optional_t" => "Vx", + "H5VL_object_get_t" => "Vn", + "H5VL_object_specific_t" => "Vo", + "H5VL_object_optional_t" => "Vy", + "H5VL_request_specific_t" => "Vr", + "H5VL_request_optional_t" => "Vz", + "H5VL_blob_optional_t" => "VA", "void" => "x", "FILE" => "x", + "H5_alloc_stats_t" => "x", "H5A_operator_t" => "x", "H5A_operator1_t" => "x", "H5A_operator2_t" => "x", @@ -120,18 +157,29 @@ $Source = ""; "H5FD_t" => "x", "H5FD_class_t" => "x", "H5FD_stream_fapl_t" => "x", + "H5FD_ros3_fapl_t" => "x", + "H5FD_hdfs_fapl_t" => "x", "H5FD_file_image_callbacks_t" => "x", + "H5FD_mirror_fapl_t" => "x", "H5G_iterate_t" => "x", "H5G_info_t" => "x", "H5I_free_t" => "x", + "H5I_iterate_func_t" => "x", "H5I_search_func_t" => "x", "H5L_class_t" => "x", "H5L_elink_traverse_t" => "x", - "H5L_iterate_t" => "x", + "H5L_info1_t" => "x", + "H5L_info2_t" => "x", + "H5L_iterate1_t" => "x", + "H5L_iterate2_t" => "x", + "H5M_iterate_t" => 'x', "H5MM_allocate_t" => "x", "H5MM_free_t" => "x", - "H5O_info_t" => "x", - "H5O_iterate_t" => "x", + "H5O_info1_t" => "x", + "H5O_info2_t" => "x", + "H5O_native_info_t" => "x", + "H5O_iterate1_t" => "x", + "H5O_iterate2_t" => "x", "H5O_mcdt_search_cb_t" => "x", "H5P_cls_create_func_t" => "x", "H5P_cls_copy_func_t" => "x", @@ -147,6 +195,10 @@ $Source = ""; "H5T_cdata_t" => "x", "H5T_conv_t" => "x", "H5T_conv_except_func_t" => "x", + "H5VL_t" => "x", + "H5VL_class_t" => "x", + "H5VL_loc_params_t" => "x", + "H5VL_request_notify_t" => "x", "H5Z_func_t" => "x", "H5Z_filter_func_t" => "x", "va_list" => "x", @@ -190,6 +242,11 @@ sub argstring ($$$) { # certain type qualifiers, and indirection. $atype =~ s/^\bconst\b//; $atype =~ s/\bH5_ATTR_UNUSED\b//g; + $atype =~ s/\bH5_ATTR_DEPRECATED_USED\b//g; + $atype =~ s/\bH5_ATTR_NDEBUG_UNUSED\b//g; + $atype =~ s/\bH5_ATTR_DEBUG_API_USED\b//g; + $atype =~ s/\bH5_ATTR_PARALLEL_UNUSED\b//g; + $atype =~ s/\bH5_ATTR_PARALLEL_USED\b//g; $atype =~ s/\s+/ /g; $ptr = length $1 if $atype =~ s/(\*+)//; $atype =~ s/^\s+//; @@ -295,7 +352,7 @@ sub rewrite_func ($$$$$) { # Ignored due to NO TRACE comment. } elsif ($body =~ s/((\n[ \t]*)H5TRACE\d+\s*\(.*?\);)\n/"$2$trace"/es) { # Replaced an H5TRACE macro. - } elsif ($body=~s/((\n[ \t]*)FUNC_ENTER\w*\s*(\(.*?\))?;??)\n/"$1$2$trace"/es) { + } elsif ($body=~s/((\n[ \t]*)FUNC_ENTER\w*[ \t]*(\(.*?\))?;??)\n/"$1$2$trace"/es) { # Added an H5TRACE macro after a FUNC_ENTER macro. } else { errmesg $file, $name, "unable to insert tracing information"; diff --git a/bin/warnhist b/bin/warnhist new file mode 100755 index 0000000..a88474b --- /dev/null +++ b/bin/warnhist @@ -0,0 +1,523 @@ +#!/usr/bin/env perl +require 5.003; +use warnings; + +# +# Copyright by The HDF Group. +# All rights reserved. +# +# This file is part of HDF5. The full HDF5 copyright notice, including +# terms governing use, modification, and redistribution, is contained in +# the files COPYING and Copyright.html. COPYING can be found at the root +# of the source code distribution tree; Copyright.html can be found at the +# root level of an installed copy of the electronic HDF5 document set and +# is linked from the top-level documents page. It can also be found at +# http://hdfgroup.org/HDF5/doc/Copyright.html. If you do not have +# access to either file, you may request a copy from help@hdfgroup.org. +# +# Quincey Koziol, koziol@hdfgroup.org +# 9 Aug 2013 +# +# Purpose: Given an input file containing the output from a build of the +# library, gather the file names and line numbers, alias +# identical types of warnings together into a single bin and +# report the number of warnings for each type of warning, each file +# and the total count of warnings + +# Perl modules/settings +use strict; +use Getopt::Std; + +# Global variables, for accumulating information +my $totalcount = 0; +my %warn_count = (); +my $warncount; +my %warn_file = (); +my %warn_file_line = (); +my %file_count = (); +my $filecount; +my $ignorecount = 0; +my @ignorenames; +my %ignored_files = (); +my %warn_file_indices = (); +my %file_warn_indices = (); +my @warn_match_strings; +my @file_match_strings; +my %file_warn = (); +my %file_warn_line = (); +my $current_warning = 0; +my $current_file = 0; +my $warn_index; +my $last_c_name; +my $last_fort_name; +my $last_fort_line; +my $genericize = 1; + +# Display usage +sub do_help { + print "Usage: 'warnhist [-h, --help] [-t <prefix>] [-w <n>] [-W] [-f <n>] [-F] [-s <warning string list>] [-S <file string list] [-l] [-u] [-i <name list>] [file]'\n"; + print "\t-h, --help\tDisplay this usage\n"; + print "\t-t <prefix>\tTrim pathname prefix from filenames, <prefix>\n"; + print "\t-w <n>\tDisplay files for a given warning index list, <n>\n"; + print "\t\t<n> can be a single value, a range, or a comma separated list\n"; + print "\t\tFor example: '0' or '0,4' or '8-10' or '0,2-4,8-10,13'\n"; + print "\t-W\tDisplay files for all warnings\n"; + print "\t-f <n>\tDisplay warnings for a given file index list, <n>\n"; + print "\t\t<n> can be a single value, a range, or a comma separated list\n"; + print "\t\tFor example: '0' or '0,4' or '8-10' or '0,2-4,8-10,13'\n"; + print "\t-F\tDisplay warnings for all files\n"; + print "\t-s <warning string list>\tDisplay files for warnings which contain a string, <warning string list>\n"; + print "\t\t<warning string list> is a comma separated list, with no spaces\n"; + print "\t\tFor example: 'Wunused-dummy-argument' or 'Wunused-dummy-argument,Wunused-variable'\n"; + print "\t-S <file string list>\tDisplay warnings for files which contain a string, <file string list>\n"; + print "\t\t<file string list> is a comma separated list, with no spaces\n"; + print "\t\tFor example: 'H5Fint' or 'H5Fint,H5Gnode'\n"; + print "\t-l\tDisplay line nunbers for file/warning\n"; + print "\t-u\tLeave 'unique' types in warnings, instead of genericizing them\n"; + print "\t-i <name list>\tIgnore named files, <name list>\n"; + print "\t\t<name list> is a comma separated list, with no spaces\n"; + print "\t\tFor example: 'H5LTparse' or 'H5LTparse,H5LTanalyze'\n"; + print "\tfile\tFilename containing build output\n"; + print "\t\tIf no file is given, standard input is used.\n"; + exit; +} + +sub main::HELP_MESSAGE { + do_help(); +} + +# declare the Perl command line flags/options we want to allow +my %options=(); +getopts("FWhut:w:f:s:S:i:l", \%options); + +# Display usage, if requested +if($options{h}) { + do_help(); +} + +# Parse list of file names to ignore +if(exists $options{i}) { + @ignorenames = split /,/, $options{i}; +#print @ignorenames; +} + +# Parse list of warning indices to expand file names +if(exists $options{w}) { + my @tmp_indices; + + @tmp_indices = split /,/, $options{w}; +#print @tmp_indices; + for my $x (@tmp_indices) { +#print "x = '$x'\n"; + if($x =~ /\-/) { + my $start_index; + my $end_index; + +#print "matched = '$x'\n"; + ($start_index, $end_index) = split /\-/, $x; +#print "start_index = '$start_index', end_index = '$end_index'\n"; + for my $y ($start_index..$end_index) { +#print "y = '$y'\n"; + if(!exists $warn_file_indices{$y}) { + $warn_file_indices{$y} = $y; + } + } + } + else { + if(!exists $warn_file_indices{$x}) { + $warn_file_indices{$x} = $x; + } + } + } +#foreach (sort keys %warn_file_indices) { +# print "$_ : $warn_file_indices{$_}\n"; +#} +} + +# Parse list of warning strings to expand file names +if(exists $options{s}) { + @warn_match_strings = split /,/, $options{s}; +# print @warn_match_strings; +} + +# Parse list of file indices to expand warnings +if(exists $options{f}) { + my @tmp_indices; + + @tmp_indices = split /,/, $options{f}; +#print @tmp_indices; + for my $x (@tmp_indices) { +#print "x = '$x'\n"; + if($x =~ /\-/) { + my $start_index; + my $end_index; + +#print "matched = '$x'\n"; + ($start_index, $end_index) = split /\-/, $x; +#print "start_index = '$start_index', end_index = '$end_index'\n"; + for my $y ($start_index..$end_index) { +#print "y = '$y'\n"; + if(!exists $file_warn_indices{$y}) { + $file_warn_indices{$y} = $y; + } + } + } + else { + if(!exists $file_warn_indices{$x}) { + $file_warn_indices{$x} = $x; + } + } + } +#foreach (sort keys %warn_file_indices) { +# print "$_ : $warn_file_indices{$_}\n"; +#} +} + +# Parse list of warning strings for files to expand warnings +if(exists $options{S}) { + @file_match_strings = split /,/, $options{S}; +# print @file_match_strings; +} + +# Check if warnings should stay unique and not be "genericized" +if($options{u}) { + $genericize = 0; +} + +PARSE_LINES: +while (<>) { + my $name; + my $line; + my $prev_line; + my $toss; + my $offset; + my $warning; + my $extra; + my $extra2; + + # Retain last FORTRAN compile line, which comes a few lines before warning + if($_ =~ /.*\.[fF]90:.*/) { + ($last_fort_name, $last_fort_line, $toss) = split /\:/, $_; + ($last_fort_line, $toss) = split /\./, $last_fort_line; + } + + # Retain last C/C++ compile line, which possibly comes a few lines before warning + if($_ =~ /.*[A-Za-z0-9_]\.[cC]:.*/) { + ($last_c_name, $toss) = split /\:/, $_; + } + + # Retain C/C++ compile line, which comes with the line of warning + if($_ =~ /.*[A-Za-z0-9_]\.[chC]\(.*[0-9]\):.*#.*/) { + $last_c_name = $_; + } + + # Skip lines that don't have the word "warning" + next if $_ !~ /[Ww]arning/; + + # Skip warnings from linker + next if $_ =~ /ld: warning:/; + + # Skip warnings from build_py and install_lib + next if $_ =~ /warning: (build_py|install_lib)/; + + # "Hide" the C++ '::' symbol until we've parsed out the parts of the line + while($_ =~ /\:\:/) { + $_ =~ s/\:\:/@@@@/g; + } + + # Check for weird formatting of warning message + if($_ =~ /^cc1: warning:.*/) { + $name = $last_c_name; + $line = "??"; + ($toss, $toss, $warning, $extra, $extra2) = split /\:/, $_; + # Check for CMAKE build with warning on first line and no filename + } elsif($_ =~ /^\s*[Ww]arning:.*/) { + $name = $last_c_name; + $line = "??"; + ($toss, $warning, $extra, $extra2) = split /\:/, $_; + # Check for FORTRAN warning output + } elsif($_ =~ /^Warning:.*/) { + $name = $last_fort_name; + $line = $last_fort_line; + ($toss, $warning, $extra, $extra2) = split /\:/, $_; +#print "1:",$.,":",$_; +# $_ = <>; +#print "2:",$.,":",$_; +# if($_ =~ /^\sFC.*/) { +# $_ = <>; +#print "3:",$.,":",$_; +# } +# ($name, $line, $toss) = split /\:/, $_; +#print "4:","'",$name,"'","-","'",$line,"'","\n"; + # Check for improperly parsed filename or line + if($name =~ /^$/) { + print "Filename is a null string! Input line #$. is: '$_'"; + next + } + if($line =~ /^$/) { + print "Line is a null string! Input line #$. is: '$_'"; + next + } + # Check for non-GCC warning (Solaris/Oracle?) + } elsif($_ =~ /^\".*, line [0-9]+: *[Ww]arning:.*/) { + ($name, $toss, $warning, $extra, $extra2) = split /\:/, $_; + ($name, $line) = split /\,/, $name; + $name =~ s/^\"//g; + $name =~ s/\"$//g; + $line =~ s/^\s*line\s*//g; +# print "name:'", $name, "'-'", $line, "'\n"; +# print "warning:'", $warning, "'\n"; + # Check for Intel icc warning + } elsif($_ =~ /.*[A-Za-z0-9_]\.[chC]\(.*[0-9]\):.*#.*/) { + ($last_c_name, $toss, $warning) = split /\:/, $last_c_name; + ($name, $line) = split /\(/, $last_c_name; + $line =~ s/\)//g; + } else { + # Check for 'character offset' field appended to file & line # + # (This is probably specific to GCC) + if($_ =~ /^.*[0-9]+\:[0-9]+\:/) { + ($name, $line, $offset, $toss, $warning, $extra, $extra2) = split /\:/, $_; + } else { + ($name, $line, $toss, $warning, $extra, $extra2) = split /\:/, $_; + } + } + + # Check for extra ':' followed by more text in original warning string, + # and append the ':' and text back onto the parsed warning + # (Use 'length $extra' idiom to avoid warning when $extra is undefined) + if(length $extra ) { + $warning = join ':', $warning, $extra; + } + if(length $extra2 ) { + $warning = join ':', $warning, $extra2; + } + + # Restore the C++ '::' symbol now that we've parsed out the parts of the line + while($warning =~ /@@@@/) { + $warning =~ s/@@@@/\:\:/g; + } + + # Trim leading '..' paths from filename + while($name =~ /^\.\.\//) { + $name =~ s/^\.\.\///g; + } + + # Check for trimming prefix + if((exists $options{t}) && ($name =~ /$options{t}/)) { + $name =~ s/^$options{t}\///g; + } + + # Check for ignored file + if(exists $options{i}) { + for my $x (@ignorenames) { +#print "x = '$x'\n"; + if($name =~ /$x/) { +# print "matched name = '$name'\n"; + $ignorecount++; + if(!(exists $ignored_files{$name})) { + $ignored_files{$name} = $name; + } + next PARSE_LINES; + } + } + } + + # Check for improperly parsed warning (usually an undefined warning string) + if(!defined $warning) { + print "Warning Undefined! Input line is: '$_'"; + next + } + + # Get rid of leading & trailing whitespace + $warning =~ s/^\s//g; + $warning =~ s/\s$//g; + + # Check for improperly parsed warning + if($warning =~ /^$/) { + print "Warning is a null string! Input line is: '$_'"; + next + } + + # Convert all quotes to ' + $warning =~ s/‘/'/g; + $warning =~ s/’/'/g; + $warning =~ s/"/'/g; + +# +# These skipped messages & "genericizations" may be specific to GCC + + # Skip supplemental warning message + next if $warning =~ /near initialization for/; + + # Skip C++ supplemental warning message + next if $warning =~ /in call to/; + + # Skip GCC warning that should be a note + next if $_ =~ /\(this will be reported only once per input file\)/; + + if($genericize) { + # Eliminate C/C++ "{aka <some type>}" and "{aka '<some type>'}" info + if($warning =~ /\s(\{|\()aka '?[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#]+[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\ ]*'?(\}|\))/) { + $warning =~ s/\s(\{|\()aka '?[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#]+[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\ ]*'?(\}|\))//g; + } + + # Genericize C/C++ '<some type>', printf format '%<some format>', and + # "unknown warning group" into '-' + if($warning =~ /'[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\-\=]+[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\-\=\ ]*'/) { + $warning =~ s/'[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\-\=]+[A-Za-z_0-9\(\)\*\,\[\]\.\<\>\&\:\+\#\-\=\ ]*'/'-'/g; + } + if($warning =~ /'%[\#0\-\ \+]*[,;\:_]?[0-9\*]*\.?[0-9\*]*[hjltzL]*[aAcdeEfFgGinosuxX]'/) { + $warning =~ s/'%[\#0\-\ \+]*[,;\:_]?[0-9\*]*\.?[0-9\*]*[hjltzL]*[aAcdeEfFgGinosuxX]'/'-'/g; + } + + # Genericize C/C++ "<macro>" warnings into "-" + if($warning =~ /"[A-Za-z_0-9]*"/) { + $warning =~ s/"[A-Za-z_0-9]*"/"-"/g; + } + + # Genericize [GCC?] C/C++ warning text about suggessted attribute + if($warning =~ /attribute=[A-Za-z_0-9]*\]/) { + $warning =~ s/=[A-Za-z_0-9]*\]/=-\]/g; + } + + # Genericize FORTRAN "at (<n>)" into "at (-)", "REAL(<n>)" into "REAL(-)", + # and "INTEGER(<n>)" into "INTEGER(-)" + if($warning =~ /.*at\s\([0-9]+\).*/) { + $warning =~ s/at\s\([0-9]+\)/at \(-\)/g; + } + if($warning =~ /.*REAL\([0-9]+\).*/) { + $warning =~ s/REAL\([0-9]+\)/REAL\(-\)/g; + } + if($warning =~ /.*INTEGER\([0-9]+\).*/) { + $warning =~ s/INTEGER\([0-9]+\)/INTEGER\(-\)/g; + } + + # Genericize standalone numbers in warnings + if($warning =~ /(\s|')-?[0-9]+(\s|')/) { + $warning =~ s/-?[0-9]+/-/g; + } + + # Genericize unusual GCC/G++/GFORTRAN warnings that aren't handled above + if($warning =~ /\[deprecation\] [A-Za-z_0-9]*\([A-Za-z_,0-9]*\) in [A-Za-z_0-9]* has been deprecated.*/) { + $warning =~ s/[A-Za-z_0-9]*\([A-Za-z_,0-9]*\) in [A-Za-z_0-9]*/-\(-\) in -/g; + } + } +# print "warning = $warning\n"; +# <end possible GCC-specific code> + + # Check if we've already seen this warning on this line in this file + # (Can happen for warnings from inside header files) + if( !exists $warn_file_line{$warning}{$name}{$line} ) { + # Increment count for [generic] warning + $warn_count{$warning}++; + $warn_file{$warning}{$name}++; + $warn_file_line{$warning}{$name}{$line}++; + + # Increment count for filename + $file_count{$name}++; + $file_warn{$name}{$warning}++; + $file_warn_line{$name}{$warning}{$line}++; + + # Increment total count of warnings + $totalcount++; + } + +# print "name = $name\n"; +# print "line = $line\n"; +# print "offset = $offset\n"; +# print "warning = \"$warning\"\n"; +} + +print "Total unique [non-ignored] warnings: $totalcount\n"; +print "Total ignored warnings: $ignorecount\n"; +$warncount = keys %warn_count; +print "Total unique kinds of warnings: $warncount\n"; +$filecount = keys %file_count; +print "Total files with warnings: $filecount\n\n"; + +# Print warnings in decreasing frequency +print "# of Warnings by frequency (file count)\n"; +print "=======================================\n"; +for my $x (sort {$warn_count{$b} <=> $warn_count{$a}} keys(%warn_count)) { + printf ("[%2d] %4d (%2d) - %s\n", $current_warning++, $warn_count{$x}, scalar(keys %{$warn_file_line{$x}}), $x); + if((exists $options{W}) || (exists $options{w}) || (exists $options{s})) { + my $curr_index = $current_warning - 1; + my $match = 0; + + # Check for string from list in current warning + if(exists $options{s}) { + for my $y (@warn_match_strings) { +# print "y = '$y'\n"; + if($x =~ /$y/) { +# print "matched warning = '$x'\n"; + $match = 1; + last; + } + } + } + + # Check if current warning index matches + if((exists $warn_file_indices{$curr_index}) && $curr_index == $warn_file_indices{$curr_index}) { + $match = 1; + } + + if($match) { + for my $y (sort {$warn_file{$x}{$b} <=> $warn_file{$x}{$a}} keys(%{$warn_file{$x}})) { + printf ("\t%4d - %s\n", $warn_file{$x}{$y}, $y); + if(exists $options{l}) { + my $lines = join ", ", sort {$a <=> $b} keys %{$warn_file_line{$x}{$y}}; + printf("\t\tLines: $lines \n"); + } + } + } + } +} + +# Print warnings in decreasing frequency, by filename +print "\n# of Warnings by filename (warning type)\n"; +print "========================================\n"; +for my $x (sort {$file_count{$b} <=> $file_count{$a}} keys(%file_count)) { + printf ("[%3d] %4d (%2d) - %s\n", $current_file++, $file_count{$x}, scalar(keys %{$file_warn_line{$x}}), $x); + if((exists $options{F}) || (exists $options{f}) || (exists $options{S})) { + my $curr_index = $current_file - 1; + my $match = 0; + + # Check for string from list in current file + if(exists $options{S}) { + for my $y (@file_match_strings) { +# print "y = '$y'\n"; + if($x =~ /$y/) { +# print "matched warning = '$x'\n"; + $match = 1; + last; + } + } + } + + # Check if current file index matches + if((exists $file_warn_indices{$curr_index}) && $curr_index == $file_warn_indices{$curr_index}) { + $match = 1; + } + + if($match) { + for my $y (sort {$file_warn{$x}{$b} <=> $file_warn{$x}{$a}} keys(%{$file_warn{$x}})) { + printf ("\t%4d - %s\n", $file_warn{$x}{$y}, $y); + if(exists $options{l}) { + my $lines = join ", ", sort {$a <=> $b} keys %{$file_warn_line{$x}{$y}}; + printf("\t\tLines: $lines \n"); + } + } + } + } +} + +# Print names of files that were ignored +# Check for ignored file +if(exists $options{i}) { + print "\nIgnored filenames\n"; + print "=================\n"; + for my $x (sort keys(%ignored_files)) { + print "$x\n"; + } +} + |