diff options
Diffstat (limited to 'scripts')
| -rwxr-xr-x | scripts/gen_travis.py | 153 | ||||
| -rw-r--r-- | scripts/linux/before_install.sh | 13 | ||||
| -rw-r--r-- | scripts/windows/before_install.sh | 83 | ||||
| -rw-r--r-- | scripts/windows/before_script.sh | 20 | ||||
| -rw-r--r-- | scripts/windows/script.sh | 10 |
5 files changed, 226 insertions, 53 deletions
diff --git a/scripts/gen_travis.py b/scripts/gen_travis.py index e98ebeb..63e0054 100755 --- a/scripts/gen_travis.py +++ b/scripts/gen_travis.py @@ -6,6 +6,7 @@ from enum import Enum, auto LINUX = 'linux' OSX = 'osx' +WINDOWS = 'windows' AMD64 = 'amd64' @@ -13,28 +14,48 @@ ARM64 = 'arm64' PPC64LE = 'ppc64le' -TRAVIS_TEMPLATE = """ +TRAVIS_TEMPLATE = """\ # This config file is generated by ./scripts/gen_travis.py. # Do not edit by hand. -language: generic +# We use 'minimal', because 'generic' makes Windows VMs hang at startup. Also +# the software provided by 'generic' is simply not needed for our tests. +# Differences are explained here: +# https://docs.travis-ci.com/user/languages/minimal-and-generic/ +language: minimal dist: focal jobs: include: {jobs} +before_install: + - |- + if test -f "./scripts/$TRAVIS_OS_NAME/before_install.sh"; then + source ./scripts/$TRAVIS_OS_NAME/before_install.sh + fi + before_script: - - autoconf - - scripts/gen_travis.py > travis_script && diff .travis.yml travis_script - # If COMPILER_FLAGS are not empty, add them to CC and CXX - - ./configure ${{COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" \ + - |- + if test -f "./scripts/$TRAVIS_OS_NAME/before_script.sh"; then + source ./scripts/$TRAVIS_OS_NAME/before_script.sh + else + scripts/gen_travis.py > travis_script && diff .travis.yml travis_script + autoconf + # If COMPILER_FLAGS are not empty, add them to CC and CXX + ./configure ${{COMPILER_FLAGS:+ CC="$CC $COMPILER_FLAGS" \ CXX="$CXX $COMPILER_FLAGS"}} $CONFIGURE_FLAGS - - make -j3 - - make -j3 tests + make -j3 + make -j3 tests + fi script: - - make check + - |- + if test -f "./scripts/$TRAVIS_OS_NAME/script.sh"; then + source ./scripts/$TRAVIS_OS_NAME/script.sh + else + make check + fi """ @@ -44,6 +65,7 @@ class Option(object): COMPILER_FLAG = auto() CONFIGURE_FLAG = auto() MALLOC_CONF = auto() + FEATURE = auto() def __init__(self, type, value): self.type = type @@ -65,6 +87,10 @@ class Option(object): def as_malloc_conf(value): return Option(Option.Type.MALLOC_CONF, value) + @staticmethod + def as_feature(value): + return Option(Option.Type.FEATURE, value) + def __eq__(self, obj): return (isinstance(obj, Option) and obj.type == self.type and obj.value == self.value) @@ -81,13 +107,14 @@ MAX_UNUSUAL_OPTIONS = 2 GCC = Option.as_compiler('CC=gcc CXX=g++') CLANG = Option.as_compiler('CC=clang CXX=clang++') +CL = Option.as_compiler('CC=cl.exe CXX=cl.exe') -compiler_default = GCC compilers_unusual = [CLANG,] -compiler_flag_unusuals = [Option.as_compiler_flag(opt) for opt in ('-m32',)] +CROSS_COMPILE_32BIT = Option.as_feature('CROSS_COMPILE_32BIT') +feature_unusuals = [CROSS_COMPILE_32BIT] configure_flag_unusuals = [Option.as_configure_flag(opt) for opt in ( @@ -108,73 +135,75 @@ malloc_conf_unusuals = [Option.as_malloc_conf(opt) for opt in ( )] -all_unusuals = (compilers_unusual + compiler_flag_unusuals +all_unusuals = (compilers_unusual + feature_unusuals + configure_flag_unusuals + malloc_conf_unusuals) -gcc_multilib_set = False - - def get_extra_cflags(os, compiler): + if os == WINDOWS: + # For non-CL compilers under Windows (for now it's only MinGW-GCC), + # -fcommon needs to be specified to correctly handle multiple + # 'malloc_conf' symbols and such, which are declared weak under Linux. + # Weak symbols don't work with MinGW-GCC. + if compiler != CL.value: + return ['-fcommon'] + else: + return [] + # We get some spurious errors when -Warray-bounds is enabled. extra_cflags = ['-Werror', '-Wno-array-bounds'] if compiler == CLANG.value or os == OSX: extra_cflags += [ - '-Wno-unknown-warning-option', - '-Wno-ignored-attributes' - ] + '-Wno-unknown-warning-option', + '-Wno-ignored-attributes' + ] if os == OSX: extra_cflags += [ - '-Wno-deprecated-declarations', - ] + '-Wno-deprecated-declarations', + ] return extra_cflags # Formats a job from a combination of flags def format_job(os, arch, combination): - global gcc_multilib_set - - compiler = [x.value for x in combination if x.type == Option.Type.COMPILER] - assert(len(compiler) <= 1) - if not compiler: - compiler = compiler_default.value - else: - compiler = compiler[0] + compilers = [x.value for x in combination if x.type == Option.Type.COMPILER] + assert(len(compilers) <= 1) compiler_flags = [x.value for x in combination if x.type == Option.Type.COMPILER_FLAG] configure_flags = [x.value for x in combination if x.type == Option.Type.CONFIGURE_FLAG] malloc_conf = [x.value for x in combination if x.type == Option.Type.MALLOC_CONF] + features = [x.value for x in combination if x.type == Option.Type.FEATURE] if len(malloc_conf) > 0: configure_flags.append('--with-malloc-conf=' + ','.join(malloc_conf)) - job = "" - job += ' - os: {}\n'.format(os) - job += ' arch: {}\n'.format(arch) + if not compilers: + compiler = GCC.value + else: + compiler = compilers[0] - if '-m32' in compiler_flags and os == 'linux': - job += ' addons:' - if gcc_multilib_set: - job += ' *gcc_multilib\n' - else: - job += ' &gcc_multilib\n' - job += ' apt:\n' - job += ' packages:\n' - job += ' - gcc-multilib\n' - job += ' - g++-multilib\n' - gcc_multilib_set = True - - env_string = ('{} COMPILER_FLAGS="{}" CONFIGURE_FLAGS="{}" ' - 'EXTRA_CFLAGS="{}"'.format( + extra_environment_vars = '' + cross_compile = CROSS_COMPILE_32BIT.value in features + if os == LINUX and cross_compile: + compiler_flags.append('-m32') + + features_str = ' '.join([' {}=yes'.format(feature) for feature in features]) + + stringify = lambda arr, name: ' {}="{}"'.format(name, ' '.join(arr)) if arr else '' + env_string = '{}{}{}{}{}{}'.format( compiler, - ' '.join(compiler_flags), - ' '.join(configure_flags), - ' '.join(get_extra_cflags(os, compiler)))) + features_str, + stringify(compiler_flags, 'COMPILER_FLAGS'), + stringify(configure_flags, 'CONFIGURE_FLAGS'), + stringify(get_extra_cflags(os, compiler), 'EXTRA_CFLAGS'), + extra_environment_vars) + job = ' - os: {}\n'.format(os) + job += ' arch: {}\n'.format(arch) job += ' env: {}'.format(env_string) return job -def generate_unusual_combinations(max_unusual_opts): +def generate_unusual_combinations(unusuals, max_unusual_opts): """ Generates different combinations of non-standard compilers, compiler flags, configure flags and malloc_conf settings. @@ -182,20 +211,22 @@ def generate_unusual_combinations(max_unusual_opts): @param max_unusual_opts: Limit of unusual options per combination. """ return chain.from_iterable( - [combinations(all_unusuals, i) for i in range(max_unusual_opts + 1)]) + [combinations(unusuals, i) for i in range(max_unusual_opts + 1)]) def included(combination, exclude): """ Checks if the combination of options should be included in the Travis testing matrix. + + @param exclude: A list of options to be avoided. """ return not any(excluded in combination for excluded in exclude) -def generate_jobs(os, arch, exclude, max_unusual_opts): +def generate_jobs(os, arch, exclude, max_unusual_opts, unusuals=all_unusuals): jobs = [] - for combination in generate_unusual_combinations(max_unusual_opts): + for combination in generate_unusual_combinations(unusuals, max_unusual_opts): if included(combination, exclude): jobs.append(format_job(os, arch, combination)) return '\n'.join(jobs) @@ -210,7 +241,7 @@ def generate_linux(arch): exclude = [] if arch == PPC64LE: # Avoid 32 bit builds and clang on PowerPC - exclude = [Option.as_compiler_flag('-m32')] + compilers_unusual + exclude = (CROSS_COMPILE_32BIT, CLANG,) return generate_jobs(os, arch, exclude, max_unusual_opts) @@ -230,6 +261,19 @@ def generate_macos(arch): return generate_jobs(os, arch, exclude, max_unusual_opts) +def generate_windows(arch): + os = WINDOWS + + max_unusual_opts = 3 + unusuals = ( + Option.as_configure_flag('--enable-debug'), + CL, + CROSS_COMPILE_32BIT, + ) + return generate_jobs(os, arch, (), max_unusual_opts, unusuals) + + + def get_manual_jobs(): return """\ # Development build @@ -251,6 +295,9 @@ def main(): generate_linux(PPC64LE), generate_macos(AMD64), + + generate_windows(AMD64), + get_manual_jobs() )) diff --git a/scripts/linux/before_install.sh b/scripts/linux/before_install.sh new file mode 100644 index 0000000..6741746 --- /dev/null +++ b/scripts/linux/before_install.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -ev + +if [[ "$TRAVIS_OS_NAME" != "linux" ]]; then + echo "Incorrect \$TRAVIS_OS_NAME: expected linux, got $TRAVIS_OS_NAME" + exit 1 +fi + +if [[ "$CROSS_COMPILE_32BIT" == "yes" ]]; then + sudo apt-get update + sudo apt-get -y install gcc-multilib g++-multilib +fi diff --git a/scripts/windows/before_install.sh b/scripts/windows/before_install.sh new file mode 100644 index 0000000..2740c45 --- /dev/null +++ b/scripts/windows/before_install.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +set -e + +# The purpose of this script is to install build dependencies and set +# $build_env to a function that sets appropriate environment variables, +# to enable (mingw32|mingw64) environment if we want to compile with gcc, or +# (mingw32|mingw64) + vcvarsall.bat if we want to compile with cl.exe + +if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then + echo "Incorrect \$TRAVIS_OS_NAME: expected windows, got $TRAVIS_OS_NAME" + exit 1 +fi + +[[ ! -f C:/tools/msys64/msys2_shell.cmd ]] && rm -rf C:/tools/msys64 +choco uninstall -y mingw +choco upgrade --no-progress -y msys2 + +msys_shell_cmd="cmd //C RefreshEnv.cmd && set MSYS=winsymlinks:nativestrict && C:\\tools\\msys64\\msys2_shell.cmd" + +msys2() { $msys_shell_cmd -defterm -no-start -msys2 -c "$*"; } +mingw32() { $msys_shell_cmd -defterm -no-start -mingw32 -c "$*"; } +mingw64() { $msys_shell_cmd -defterm -no-start -mingw64 -c "$*"; } + +if [[ "$CROSS_COMPILE_32BIT" == "yes" ]]; then + mingw=mingw32 + mingw_gcc_package_arch=i686 +else + mingw=mingw64 + mingw_gcc_package_arch=x86_64 +fi + +if [[ "$CC" == *"gcc"* ]]; then + $mingw pacman -S --noconfirm --needed \ + autotools \ + git \ + mingw-w64-${mingw_gcc_package_arch}-make \ + mingw-w64-${mingw_gcc_package_arch}-gcc \ + mingw-w64-${mingw_gcc_package_arch}-binutils + build_env=$mingw +elif [[ "$CC" == *"cl"* ]]; then + $mingw pacman -S --noconfirm --needed \ + autotools \ + git \ + mingw-w64-${mingw_gcc_package_arch}-make \ + mingw-w64-${mingw_gcc_package_arch}-binutils + + # In order to use MSVC compiler (cl.exe), we need to correctly set some environment + # variables, namely PATH, INCLUDE, LIB and LIBPATH. The correct values of these + # variables are set by a batch script "vcvarsall.bat". The code below generates + # a batch script that calls "vcvarsall.bat" and prints the environment variables. + # + # Then, those environment variables are transformed from cmd to bash format and put + # into a script $apply_vsenv. If cl.exe needs to be used from bash, one can + # 'source $apply_vsenv' and it will apply the environment variables needed for cl.exe + # to be located and function correctly. + # + # At last, a function "mingw_with_msvc_vars" is generated which forwards user input + # into a correct mingw (32 or 64) subshell that automatically performs 'source $apply_vsenv', + # making it possible for autotools to discover and use cl.exe. + vcvarsall="vcvarsall.tmp.bat" + echo "@echo off" > $vcvarsall + echo "call \"c:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\\\vcvarsall.bat\" $USE_MSVC" >> $vcvarsall + echo "set" >> $vcvarsall + + apply_vsenv="./apply_vsenv.sh" + cmd //C $vcvarsall | grep -E "^PATH=" | sed -n -e 's/\(.*\)=\(.*\)/export \1=$PATH:"\2"/g' \ + -e 's/\([a-zA-Z]\):[\\\/]/\/\1\//g' \ + -e 's/\\/\//g' \ + -e 's/;\//:\//gp' > $apply_vsenv + cmd //C $vcvarsall | grep -E "^(INCLUDE|LIB|LIBPATH)=" | sed -n -e 's/\(.*\)=\(.*\)/export \1="\2"/gp' >> $apply_vsenv + + cat $apply_vsenv + mingw_with_msvc_vars() { $msys_shell_cmd -defterm -no-start -$mingw -c "source $apply_vsenv && ""$*"; } + build_env=mingw_with_msvc_vars + + rm -f $vcvarsall +else + echo "Unknown C compiler: $CC" + exit 1 +fi + +echo "Build environment function: $build_env" diff --git a/scripts/windows/before_script.sh b/scripts/windows/before_script.sh new file mode 100644 index 0000000..9d30aba --- /dev/null +++ b/scripts/windows/before_script.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then + echo "Incorrect \$TRAVIS_OS_NAME: expected windows, got $TRAVIS_OS_NAME" + exit 1 +fi + +$build_env autoconf +$build_env ./configure $CONFIGURE_FLAGS +# mingw32-make simply means "make", unrelated to mingw32 vs mingw64. +# Simply disregard the prefix and treat is as "make". +$build_env mingw32-make -j3 +# At the moment, it's impossible to make tests in parallel, +# seemingly due to concurrent writes to '.pdb' file. I don't know why +# that happens, because we explicitly supply '/Fs' to the compiler. +# Until we figure out how to fix it, we should build tests sequentially +# on Windows. +$build_env mingw32-make tests diff --git a/scripts/windows/script.sh b/scripts/windows/script.sh new file mode 100644 index 0000000..3a27f70 --- /dev/null +++ b/scripts/windows/script.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [[ "$TRAVIS_OS_NAME" != "windows" ]]; then + echo "Incorrect \$TRAVIS_OS_NAME: expected windows, got $TRAVIS_OS_NAME" + exit 1 +fi + +$build_env mingw32-make -k check |