From 889a7146ff8a1b3ca73cfa14e07fae7ae6ee706d Mon Sep 17 00:00:00 2001 From: Ryan Thornton Date: Mon, 16 Mar 2020 11:59:12 -0500 Subject: GoogleTestAddTests: Refactor into callable method Move test discovery logic into new gtest_discover_tests_impl method and make GoogleTestAddTests aware of whether it is being launched in CMake's script mode. When launched in script mode, gtest_discover_tests_impl is called passing arguments obtained from the definitions passed into the call to cmake. (i.e. cmake -P GoogleTestAddTests -D -D ...) This preserves the existing behavior assumed by GoogleTest.cmake. Unit tests are unchanged and still pass. Looking ahead, it also allows GoogleTestAddTests to be included in generated files and call gtest_discover_tests_impl to perform test discovery at test runtime with the new PRE_TEST discovery mode introduced later in this branch. My original approach attempted to call execute_process(cmake -P ...) in the generated file, the same way POST_BUILD is doing, but I ran into difficulties serializing the command arguments correctly. By exposing a way to call gtest_discover_tests_impl directly from our generated file, we remove a layer of shell quoting / parsing that our arguments have to survive, which simplifies the act of producing a generated file that behaves the same as its POST_BUILD counterpart. --- Modules/GoogleTestAddTests.cmake | 220 ++++++++++++++++++++++----------------- 1 file changed, 125 insertions(+), 95 deletions(-) diff --git a/Modules/GoogleTestAddTests.cmake b/Modules/GoogleTestAddTests.cmake index 753319f..65af4c2 100644 --- a/Modules/GoogleTestAddTests.cmake +++ b/Modules/GoogleTestAddTests.cmake @@ -3,21 +3,12 @@ cmake_minimum_required(VERSION ${CMAKE_VERSION}) -set(prefix "${TEST_PREFIX}") -set(suffix "${TEST_SUFFIX}") -set(extra_args ${TEST_EXTRA_ARGS}) -set(properties ${TEST_PROPERTIES}) -set(script) -set(suite) -set(tests) -set(tests_buffer) - -# Overwrite possibly existing ${CTEST_FILE} with empty file +# Overwrite possibly existing ${_CTEST_FILE} with empty file set(flush_tests_MODE WRITE) -# Flushes script to ${CTEST_FILE} +# Flushes script to ${_CTEST_FILE} macro(flush_script) - file(${flush_tests_MODE} "${CTEST_FILE}" "${script}") + file(${flush_tests_MODE} "${_CTEST_FILE}" "${script}") set(flush_tests_MODE APPEND) set(script "") @@ -48,98 +39,137 @@ macro(add_command NAME) unset(_script_len) endmacro() -# Run test executable to get list of available tests -if(NOT EXISTS "${TEST_EXECUTABLE}") - message(FATAL_ERROR - "Specified test executable does not exist.\n" - " Path: '${TEST_EXECUTABLE}'" +function(gtest_discover_tests_impl) + + cmake_parse_arguments( + "" + "" + "NO_PRETTY_TYPES;NO_PRETTY_VALUES;TEST_EXECUTABLE;TEST_EXECUTOR;TEST_WORKING_DIR;TEST_PREFIX;TEST_SUFFIX;TEST_LIST;CTEST_FILE;TEST_DISCOVERY_TIMEOUT;TEST_XML_OUTPUT_DIR" + "TEST_EXTRA_ARGS;TEST_PROPERTIES" + ${ARGN} ) -endif() -execute_process( - COMMAND ${TEST_EXECUTOR} "${TEST_EXECUTABLE}" --gtest_list_tests - WORKING_DIRECTORY "${TEST_WORKING_DIR}" - TIMEOUT ${TEST_DISCOVERY_TIMEOUT} - OUTPUT_VARIABLE output - RESULT_VARIABLE result -) -if(NOT ${result} EQUAL 0) - string(REPLACE "\n" "\n " output "${output}") - message(FATAL_ERROR - "Error running test executable.\n" - " Path: '${TEST_EXECUTABLE}'\n" - " Result: ${result}\n" - " Output:\n" - " ${output}\n" + + set(prefix "${_TEST_PREFIX}") + set(suffix "${_TEST_SUFFIX}") + set(extra_args ${_TEST_EXTRA_ARGS}) + set(properties ${_TEST_PROPERTIES}) + set(script) + set(suite) + set(tests) + set(tests_buffer) + + # Run test executable to get list of available tests + if(NOT EXISTS "${_TEST_EXECUTABLE}") + message(FATAL_ERROR + "Specified test executable does not exist.\n" + " Path: '${_TEST_EXECUTABLE}'" + ) + endif() + execute_process( + COMMAND ${_TEST_EXECUTOR} "${_TEST_EXECUTABLE}" --gtest_list_tests + WORKING_DIRECTORY "${_TEST_WORKING_DIR}" + TIMEOUT ${_TEST_DISCOVERY_TIMEOUT} + OUTPUT_VARIABLE output + RESULT_VARIABLE result ) -endif() + if(NOT ${result} EQUAL 0) + string(REPLACE "\n" "\n " output "${output}") + message(FATAL_ERROR + "Error running test executable.\n" + " Path: '${_TEST_EXECUTABLE}'\n" + " Result: ${result}\n" + " Output:\n" + " ${output}\n" + ) + endif() -string(REPLACE "\n" ";" output "${output}") - -# Parse output -foreach(line ${output}) - # Skip header - if(NOT line MATCHES "gtest_main\\.cc") - # Do we have a module name or a test name? - if(NOT line MATCHES "^ ") - # Module; remove trailing '.' to get just the name... - string(REGEX REPLACE "\\.( *#.*)?" "" suite "${line}") - if(line MATCHES "#" AND NOT NO_PRETTY_TYPES) - string(REGEX REPLACE "/[0-9]\\.+ +#.*= +" "/" pretty_suite "${line}") - else() - set(pretty_suite "${suite}") - endif() - string(REGEX REPLACE "^DISABLED_" "" pretty_suite "${pretty_suite}") - else() - # Test name; strip spaces and comments to get just the name... - string(REGEX REPLACE " +" "" test "${line}") - if(test MATCHES "#" AND NOT NO_PRETTY_VALUES) - string(REGEX REPLACE "/[0-9]+#GetParam..=" "/" pretty_test "${test}") - else() - string(REGEX REPLACE "#.*" "" pretty_test "${test}") - endif() - string(REGEX REPLACE "^DISABLED_" "" pretty_test "${pretty_test}") - string(REGEX REPLACE "#.*" "" test "${test}") - if(NOT TEST_XML_OUTPUT_DIR STREQUAL "") - set(TEST_XML_OUTPUT_PARAM "--gtest_output=xml:${TEST_XML_OUTPUT_DIR}/${prefix}${pretty_suite}.${pretty_test}${suffix}.xml") + string(REPLACE "\n" ";" output "${output}") + + # Parse output + foreach(line ${output}) + # Skip header + if(NOT line MATCHES "gtest_main\\.cc") + # Do we have a module name or a test name? + if(NOT line MATCHES "^ ") + # Module; remove trailing '.' to get just the name... + string(REGEX REPLACE "\\.( *#.*)?" "" suite "${line}") + if(line MATCHES "#" AND NOT _NO_PRETTY_TYPES) + string(REGEX REPLACE "/[0-9]\\.+ +#.*= +" "/" pretty_suite "${line}") + else() + set(pretty_suite "${suite}") + endif() + string(REGEX REPLACE "^DISABLED_" "" pretty_suite "${pretty_suite}") else() - unset(TEST_XML_OUTPUT_PARAM) - endif() - # ...and add to script - add_command(add_test - "${prefix}${pretty_suite}.${pretty_test}${suffix}" - ${TEST_EXECUTOR} - "${TEST_EXECUTABLE}" - "--gtest_filter=${suite}.${test}" - "--gtest_also_run_disabled_tests" - ${TEST_XML_OUTPUT_PARAM} - ${extra_args} - ) - if(suite MATCHES "^DISABLED" OR test MATCHES "^DISABLED") + # Test name; strip spaces and comments to get just the name... + string(REGEX REPLACE " +" "" test "${line}") + if(test MATCHES "#" AND NOT _NO_PRETTY_VALUES) + string(REGEX REPLACE "/[0-9]+#GetParam..=" "/" pretty_test "${test}") + else() + string(REGEX REPLACE "#.*" "" pretty_test "${test}") + endif() + string(REGEX REPLACE "^DISABLED_" "" pretty_test "${pretty_test}") + string(REGEX REPLACE "#.*" "" test "${test}") + if(NOT "${_TEST_XML_OUTPUT_DIR}" STREQUAL "") + set(TEST_XML_OUTPUT_PARAM "--gtest_output=xml:${_TEST_XML_OUTPUT_DIR}/${prefix}${pretty_suite}.${pretty_test}${suffix}.xml") + else() + unset(TEST_XML_OUTPUT_PARAM) + endif() + # ...and add to script + add_command(add_test + "${prefix}${pretty_suite}.${pretty_test}${suffix}" + ${_TEST_EXECUTOR} + "${_TEST_EXECUTABLE}" + "--gtest_filter=${suite}.${test}" + "--gtest_also_run_disabled_tests" + ${TEST_XML_OUTPUT_PARAM} + ${extra_args} + ) + if(suite MATCHES "^DISABLED" OR test MATCHES "^DISABLED") + add_command(set_tests_properties + "${prefix}${pretty_suite}.${pretty_test}${suffix}" + PROPERTIES DISABLED TRUE + ) + endif() add_command(set_tests_properties "${prefix}${pretty_suite}.${pretty_test}${suffix}" - PROPERTIES DISABLED TRUE + PROPERTIES + WORKING_DIRECTORY "${_TEST_WORKING_DIR}" + ${properties} ) - endif() - add_command(set_tests_properties - "${prefix}${pretty_suite}.${pretty_test}${suffix}" - PROPERTIES - WORKING_DIRECTORY "${TEST_WORKING_DIR}" - ${properties} - ) - list(APPEND tests_buffer "${prefix}${pretty_suite}.${pretty_test}${suffix}") - list(LENGTH tests_buffer tests_buffer_length) - if(${tests_buffer_length} GREATER "250") - flush_tests_buffer() + list(APPEND tests_buffer "${prefix}${pretty_suite}.${pretty_test}${suffix}") + list(LENGTH tests_buffer tests_buffer_length) + if(${tests_buffer_length} GREATER "250") + flush_tests_buffer() + endif() endif() endif() - endif() -endforeach() + endforeach() + + + # Create a list of all discovered tests, which users may use to e.g. set + # properties on the tests + flush_tests_buffer() + add_command(set ${_TEST_LIST} ${tests}) + # Write CTest script + flush_script() -# Create a list of all discovered tests, which users may use to e.g. set -# properties on the tests -flush_tests_buffer() -add_command(set ${TEST_LIST} ${tests}) +endfunction() -# Write CTest script -flush_script() +if(CMAKE_SCRIPT_MODE_FILE) + gtest_discover_tests_impl( + NO_PRETTY_TYPES ${NO_PRETTY_TYPES} + NO_PRETTY_VALUES ${NO_PRETTY_VALUES} + TEST_EXECUTABLE ${TEST_EXECUTABLE} + TEST_EXECUTOR ${TEST_EXECUTOR} + TEST_WORKING_DIR ${TEST_WORKING_DIR} + TEST_PREFIX ${TEST_PREFIX} + TEST_SUFFIX ${TEST_SUFFIX} + TEST_LIST ${TEST_LIST} + CTEST_FILE ${CTEST_FILE} + TEST_DISCOVERY_TIMEOUT ${TEST_DISCOVERY_TIMEOUT} + TEST_XML_OUTPUT_DIR ${TEST_XML_OUTPUT_DIR} + TEST_EXTRA_ARGS ${TEST_EXTRA_ARGS} + TEST_PROPERTIES ${TEST_PROPERTIES} + ) +endif() -- cgit v0.12 From 75e82a13db7627ad250baa48f315786e34330995 Mon Sep 17 00:00:00 2001 From: Ryan Thornton Date: Mon, 16 Mar 2020 12:04:35 -0500 Subject: GoogleTest: Add new DISCOVERY_MODE option to gtest_discover_tests Introducing a new DISCOVERY_MODE mode option, which provides greater control over when gtest_discover_tests perforsm test discovery. It has two supported modes: * POST_BUILD * PRE_TEST POST_BUILD is the default behavior, which adds a POST_BUILD command to perform test discovery after the test has been built. PRE_TEST is a new mode, which delays test discovery until test execution. DISCOVERY_MODE can be controlled in two ways: 1. Setting the DISCOVERY_MODE when calling gtest_discover_tests 2. Setting the global CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE prior to calling gtest_discover_tests By delaying test discovery until ctest runtime, we can provide better cross compile support, because we're more likely to be in an environment that can run the test executable. This was achieved by moving the command for test discovery into the generated ctest include file. In PRE_TEST mode, the generated include file now has the form: if(EXISTS "path/to/test.exe") if("path/to/test.exe" IS_NEWER_THAN "path/to/ctest_file") // command to discover tests // and generate ctest_file endif() include("path/to/ctest_file") endif() elseif() // test not built endif() Which generates the appropriate CTest files at runtime and regenerates the CTest files when the executable is updated. In the old approach, test discovery was always added as POST_BUILD step and the executable was ran in order to scan for tests. If the test executable failed to run for any reason, this resulted in a link failure. This failure could more commonly occur when cross compiling, because your build environment might not have the appropriate runtime environment dlls required to run the test executable. I also ran into the issue when compiling a Qt application using Qt Creator. Qt Creator manages its build and runtime environments separately and only adds the Qt dlls to the environment during runtime -- not during build. So when gtest_discover_tests ran my test executable, it promptly crashed because the environment wasn't correct, which resulted in a build failure. Setting the DISCOVERY_MODE to PRE_TEST fixed my build failure by delaying test discovery until runtime, because this allowed the test exe to run in the proper environment. A few non-trivial implementation details worth noting: 1. bracket_arguments In the PRE_TEST side, parameters whose contents might contain special characters, need to be handled with great care. To aid in escaping these list arguments, backslashes, and other special characters, and ensuring that they are preserved in the generated file, bracket arguments (i.e. [== <...> ==]) are used. For example: gtest_discover_tests( ... EXTRA_ARGS how now "\"brown\" cow" ) Generates a file with the following call: gtest_discover_tests_impl( ... TEST_EXTRA_ARGS [==[how;now;"brown" cow]==] ) This way the arguments are forwarded correctly. 2. multi-config generators Multi-Config generators (e.g. MSBuild) now work more correctly in PRE_TEST than POST_BUILD. PRE_TEST is more correct because it will generate a unique CTest file for each configuration. It generates a per config file responsible for test discovery: foo[1]_include-Debug.cmake foo[1]_include-MinSizeRel.cmake foo[1]_include-Release.cmake foo[1]_include-RelWithDebInfo.cmake A per config file for containing the google tests: foo[1]_tests-Debug.cmake And an outer ctest file: foo[1]_include-Debug.cmake That is generically written to include the correct configuration file by looking at the value of ${CTEST_CONFIGURATION_TYPE} when CTest runs. POST_BUILD, in contrast, preserves the existing functionality. Tests are disocvered based on the last configuration that was chosen and only a single file is produced: foo[1]_include.cmake foo[1]_tests.cmake But it runs with whatever executable requested by ctest --config, which means it's possible to run the wrong tests if some tests were enabled in one configuration but not another. --- Modules/GoogleTest.cmake | 133 ++++++++++++++++++++++++++++++++++++----------- 1 file changed, 104 insertions(+), 29 deletions(-) diff --git a/Modules/GoogleTest.cmake b/Modules/GoogleTest.cmake index 1d4398e..975ce6c 100644 --- a/Modules/GoogleTest.cmake +++ b/Modules/GoogleTest.cmake @@ -152,6 +152,7 @@ same as the Google Test name (i.e. ``suite.testcase``); see also [TEST_LIST var] [DISCOVERY_TIMEOUT seconds] [XML_OUTPUT_DIR dir] + [DISCOVERY_MODE ] ) ``gtest_discover_tests`` sets up a post-build command on the test executable @@ -244,6 +245,22 @@ same as the Google Test name (i.e. ``suite.testcase``); see also ``EXTRA_ARGS --gtest_output=xml`` to avoid race conditions writing the XML result output when using parallel test execution. + ``DISCOVERY_MODE`` + Provides greater control over when ``gtest_discover_tests``performs test + discovery. By default, ``POST_BUILD`` sets up a post-build command + to perform test discovery at build time. In certain scenarios, like + cross-compiling, this ``POST_BUILD`` behavior is not desirable. + By contrast, ``PRE_TEST`` delays test discovery until just prior to test + execution. This way test discovery occurs in the target environment + where the test has a better chance at finding appropriate runtime + dependencies. + + ``DISCOVERY_MODE`` defaults to the value of the + ``CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE`` variable if it is not + passed when calling ``gtest_discover_tests``. This provides a mechanism + for globally selecting a preferred test discovery behavior without having + to modify each call site. + #]=======================================================================] # Save project's policies @@ -376,11 +393,12 @@ function(gtest_add_tests) endfunction() #------------------------------------------------------------------------------ + function(gtest_discover_tests TARGET) cmake_parse_arguments( "" "NO_PRETTY_TYPES;NO_PRETTY_VALUES" - "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;DISCOVERY_TIMEOUT;XML_OUTPUT_DIR" + "TEST_PREFIX;TEST_SUFFIX;WORKING_DIRECTORY;TEST_LIST;DISCOVERY_TIMEOUT;XML_OUTPUT_DIR;DISCOVERY_MODE" "EXTRA_ARGS;PROPERTIES" ${ARGN} ) @@ -394,6 +412,12 @@ function(gtest_discover_tests TARGET) if(NOT _DISCOVERY_TIMEOUT) set(_DISCOVERY_TIMEOUT 5) endif() + if(NOT _DISCOVERY_MODE) + if(NOT CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE) + set(CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE "POST_BUILD") + endif() + set(_DISCOVERY_MODE ${CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE}) + endif() get_property( has_counter @@ -425,35 +449,86 @@ function(gtest_discover_tests TARGET) TARGET ${TARGET} PROPERTY CROSSCOMPILING_EMULATOR ) - add_custom_command( - TARGET ${TARGET} POST_BUILD - BYPRODUCTS "${ctest_tests_file}" - COMMAND "${CMAKE_COMMAND}" - -D "TEST_TARGET=${TARGET}" - -D "TEST_EXECUTABLE=$" - -D "TEST_EXECUTOR=${crosscompiling_emulator}" - -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" - -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}" - -D "TEST_PROPERTIES=${_PROPERTIES}" - -D "TEST_PREFIX=${_TEST_PREFIX}" - -D "TEST_SUFFIX=${_TEST_SUFFIX}" - -D "NO_PRETTY_TYPES=${_NO_PRETTY_TYPES}" - -D "NO_PRETTY_VALUES=${_NO_PRETTY_VALUES}" - -D "TEST_LIST=${_TEST_LIST}" - -D "CTEST_FILE=${ctest_tests_file}" - -D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}" - -D "TEST_XML_OUTPUT_DIR=${_XML_OUTPUT_DIR}" - -P "${_GOOGLETEST_DISCOVER_TESTS_SCRIPT}" - VERBATIM - ) - file(WRITE "${ctest_include_file}" - "if(EXISTS \"${ctest_tests_file}\")\n" - " include(\"${ctest_tests_file}\")\n" - "else()\n" - " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n" - "endif()\n" - ) + if(_DISCOVERY_MODE STREQUAL "POST_BUILD") + add_custom_command( + TARGET ${TARGET} POST_BUILD + BYPRODUCTS "${ctest_tests_file}" + COMMAND "${CMAKE_COMMAND}" + -D "TEST_TARGET=${TARGET}" + -D "TEST_EXECUTABLE=$" + -D "TEST_EXECUTOR=${crosscompiling_emulator}" + -D "TEST_WORKING_DIR=${_WORKING_DIRECTORY}" + -D "TEST_EXTRA_ARGS=${_EXTRA_ARGS}" + -D "TEST_PROPERTIES=${_PROPERTIES}" + -D "TEST_PREFIX=${_TEST_PREFIX}" + -D "TEST_SUFFIX=${_TEST_SUFFIX}" + -D "NO_PRETTY_TYPES=${_NO_PRETTY_TYPES}" + -D "NO_PRETTY_VALUES=${_NO_PRETTY_VALUES}" + -D "TEST_LIST=${_TEST_LIST}" + -D "CTEST_FILE=${ctest_tests_file}" + -D "TEST_DISCOVERY_TIMEOUT=${_DISCOVERY_TIMEOUT}" + -D "TEST_XML_OUTPUT_DIR=${_XML_OUTPUT_DIR}" + -P "${_GOOGLETEST_DISCOVER_TESTS_SCRIPT}" + VERBATIM + ) + + file(WRITE "${ctest_include_file}" + "if(EXISTS \"${ctest_tests_file}\")\n" + " include(\"${ctest_tests_file}\")\n" + "else()\n" + " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)\n" + "endif()\n" + ) + elseif(_DISCOVERY_MODE STREQUAL "PRE_TEST") + + get_property(GENERATOR_IS_MULTI_CONFIG GLOBAL + PROPERTY GENERATOR_IS_MULTI_CONFIG + ) + + if(GENERATOR_IS_MULTI_CONFIG) + set(ctest_tests_file "${ctest_file_base}_tests-$.cmake") + endif() + + string(CONCAT ctest_include_content + "if(EXISTS \"$\")" "\n" + " if(\"$\" IS_NEWER_THAN \"${ctest_tests_file}\")" "\n" + " include(GoogleTestAddTests)" "\n" + " gtest_discover_tests_impl(" "\n" + " TEST_EXECUTABLE" " [==[" "$" "]==]" "\n" + " TEST_EXECUTOR" " [==[" "${crosscompiling_emulator}" "]==]" "\n" + " TEST_WORKING_DIR" " [==[" "${_WORKING_DIRECTORY}" "]==]" "\n" + " TEST_EXTRA_ARGS" " [==[" "${_EXTRA_ARGS}" "]==]" "\n" + " TEST_PROPERTIES" " [==[" "${_PROPERTIES}" "]==]" "\n" + " TEST_PREFIX" " [==[" "${_TEST_PREFIX}" "]==]" "\n" + " TEST_SUFFIX" " [==[" "${_TEST_SUFFIX}" "]==]" "\n" + " NO_PRETTY_TYPES" " [==[" "${_NO_PRETTY_TYPES}" "]==]" "\n" + " NO_PRETTY_VALUES" " [==[" "${_NO_PRETTY_VALUES}" "]==]" "\n" + " TEST_LIST" " [==[" "${_TEST_LIST}" "]==]" "\n" + " CTEST_FILE" " [==[" "${ctest_tests_file}" "]==]" "\n" + " TEST_DISCOVERY_TIMEOUT" " [==[" "${_DISCOVERY_TIMEOUT}" "]==]" "\n" + " TEST_XML_OUTPUT_DIR" " [==[" "${_XML_OUTPUT_DIR}" "]==]" "\n" + " )" "\n" + " endif()" "\n" + " include(\"${ctest_tests_file}\")" "\n" + "else()" "\n" + " add_test(${TARGET}_NOT_BUILT ${TARGET}_NOT_BUILT)" "\n" + "endif()" "\n" + ) + + if(GENERATOR_IS_MULTI_CONFIG) + foreach(_config ${CMAKE_CONFIGURATION_TYPES}) + file(GENERATE OUTPUT "${ctest_file_base}_include-${_config}.cmake" CONTENT "${ctest_include_content}" CONDITION $) + endforeach() + file(WRITE "${ctest_include_file}" "include(\"${ctest_file_base}_include-\${CTEST_CONFIGURATION_TYPE}.cmake\")") + else() + file(GENERATE OUTPUT "${ctest_file_base}_include.cmake" CONTENT "${ctest_include_content}") + file(WRITE "${ctest_include_file}" "include(\"${ctest_file_base}_include.cmake\")") + endif() + + else() + message(SEND_ERROR "Unknown DISCOVERY_MODE: ${_DISCOVERY_MODE}") + endif() # Add discovered tests to directory TEST_INCLUDE_FILES set_property(DIRECTORY -- cgit v0.12 From 1ba4cb565eb0bf16a717863ed4945e93468cb146 Mon Sep 17 00:00:00 2001 From: Ryan Thornton Date: Mon, 16 Mar 2020 14:40:31 -0500 Subject: GoogleTest: Parameterize tests to check PRE_TEST/POST_BUILD discovery mode Now, the unit tests are ran twice -- once with POST_BUILD (i.e. default mode) and again with PRE_TEST (i.e. new discovery mode). Both modes of setting gtest discovery mode are also tested: 1. Using the global override (i.e. CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE) 2. Explicitly passing DISCOVERY_MODE in calls to gtest_discover_tests (in GoogleTestDiscoveryTimeout.cmake) The goal is to show that the new PRE_TEST discovery mode does not break existing behavior (i.e. should not break POST_BUILD mode) and should also pass the same tests in the same way. A few non trivial implementation details worth noting: 1. Refactoring discovery_timeout_test into own project Originally, I tried doing: ``` run_GoogleTest(POST_BUILD) run_GoogleTest(PRE_TEST) ``` Without changing the internal structure of run_GoogleTest. But since discovery_timeout_test is part of the same project as the other tests, and CTest include files always get evaluated and that's where test discovery occurs, this means every other test now notices the timeout problem when running in PRE_TEST mode. As a result, keeping the existing test structure meant that each existing test (and any new test) would need to have its own PRE_TEST / POST_BUILD variant for stderr and stdout in order to handle the case where discovery_timeout_test timed out. This exponential increase in test output files introduced unnecessary complexity and made it more cumbersome to work on test cases. Why should an unrelated test case care about discovery_timeout_test? So, to fix that issue, the tests were broken apart into two main groups: 1. run_GoogleTest_discovery_timeout (the test dealing with discovery_timeout_test) 2. run_GoogleTest (everything else) This isolates the PRE_TEST / POST_BUILD timeout variants to a single test case. And the other test cases remain unchanged -- further driving home the point that DISCOVERY_MODE shouldn't change existing behavior. 2. Different number of PRE_TEST / POST_BUILD file variants On the PRE_TEST path, different build systems / compilers (i.e. MSBuild and ninja/gcc) produces different build output when building discovery_timeout_test, but we don't actually care what it is, just as long as it builds successfully. This the fundamental difference in behavior between POST_BUILD (which would have failed) and PRE_TEST (which doesn't) and is the reason why we don't need a GoogleTest-discovery-build-result.txt or GoogleTest-discovery-build-stdout.txt 3. Fix flaky discovery timeout test The test expects to see: > Output: > timeout > case. But sometimes, the test would only produce: > Output: > timout In certain environments, specifically when built with OpenWatcom 1.4, and while the build server was under heavy load (i.e. running many tests in parallel), std::endl behaves inconsistently and doesn't completely flush std::cout when the program is terminated due to timeout. This results in inconsistent test failures because the actual output doesn't fully match what's expected. At first we tried adding an additional: std::cout << std::flush That didn't work. But using C-style printf() and fflush() appears to do the trick: > This time I managed to get on the machine while it was still busy doing other nightly builds > and could reproduce the problem reliably. With that I was finally able to find a fix. > It turns out my earlier hypothesis that C++ stream flushing was not working on the old compiler was correct, > but even .flush() is not enough. > I changed it to use C-style printf() and fflush() and now the test passes on that build. > -- Brad King Co-authored-by: Ryan Thornton Co-authored-by: Kevin Puetz --- ...t-discovery-POST_BUILD-timeout-build-result.txt | 1 + ...t-discovery-POST_BUILD-timeout-build-stdout.txt | 7 +++ ...st-discovery-POST_BUILD-timeout-test-result.txt | 1 + ...st-discovery-POST_BUILD-timeout-test-stderr.txt | 2 + ...st-discovery-POST_BUILD-timeout-test-stdout.txt | 18 +++++++ ...Test-discovery-PRE_TEST-timeout-test-stderr.txt | 8 +++ ...Test-discovery-PRE_TEST-timeout-test-stdout.txt | 1 + .../GoogleTest-discovery-timeout-build-result.txt | 1 - .../GoogleTest-discovery-timeout-build-stdout.txt | 7 --- .../GoogleTest-discovery-timeout-test-result.txt | 1 - .../GoogleTest-discovery-timeout-test-stderr.txt | 2 - .../GoogleTest-discovery-timeout-test-stdout.txt | 18 ------- Tests/RunCMake/GoogleTest/GoogleTest.cmake | 8 --- .../GoogleTest/GoogleTestDiscoveryTimeout.cmake | 14 +++++ Tests/RunCMake/GoogleTest/RunCMakeTest.cmake | 62 ++++++++++++++-------- Tests/RunCMake/GoogleTest/timeout_test.cpp | 7 +-- 16 files changed, 96 insertions(+), 62 deletions(-) create mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-build-result.txt create mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-build-stdout.txt create mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-result.txt create mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-stderr.txt create mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-stdout.txt create mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-stderr.txt create mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-stdout.txt delete mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-build-result.txt delete mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-build-stdout.txt delete mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-result.txt delete mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-stderr.txt delete mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-stdout.txt create mode 100644 Tests/RunCMake/GoogleTest/GoogleTestDiscoveryTimeout.cmake diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-build-result.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-build-result.txt new file mode 100644 index 0000000..d197c91 --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-build-result.txt @@ -0,0 +1 @@ +[^0] diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-build-stdout.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-build-stdout.txt new file mode 100644 index 0000000..3a6572c --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-build-stdout.txt @@ -0,0 +1,7 @@ +( *|[0-9]+>)CMake Error at .*GoogleTestAddTests.cmake:[0-9]+ \(message\): +( *|[0-9]+>) Error running test executable. +?( *|[0-9]+>) +( *|[0-9]+>) Path: '.*discovery_timeout_test(\.exe)?' +( *|[0-9]+>) Result: Process terminated due to timeout +( *|[0-9]+>) Output: +( *|[0-9]+>) + diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-result.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-result.txt new file mode 100644 index 0000000..d197c91 --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-result.txt @@ -0,0 +1 @@ +[^0] diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-stderr.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-stderr.txt new file mode 100644 index 0000000..f6be939 --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-stderr.txt @@ -0,0 +1,2 @@ +Unable to find executable: discovery_timeout_test_NOT_BUILT +Errors while running CTest diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-stdout.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-stdout.txt new file mode 100644 index 0000000..d9de3f8 --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-POST_BUILD-timeout-test-stdout.txt @@ -0,0 +1,18 @@ +Test project .*GoogleTest-discovery-timeout +[ \t]*Start [0-9]+: discovery_timeout_test_NOT_BUILT +Could not find executable discovery_timeout_test_NOT_BUILT +Looked in the following places: +discovery_timeout_test_NOT_BUILT +discovery_timeout_test_NOT_BUILT(\.exe)? +Debug/discovery_timeout_test_NOT_BUILT +Debug/discovery_timeout_test_NOT_BUILT(\.exe)? +Debug/discovery_timeout_test_NOT_BUILT +Debug/discovery_timeout_test_NOT_BUILT(\.exe)? +[^\n]+discovery_timeout_test_NOT_BUILT +\.+\*\*\*Not Run +[0-9.]+ sec ++ +0% tests passed, 1 tests failed out of 1 ++ +Total Test time \(real\) = +[0-9.]+ sec ++ +The following tests FAILED: +[^\n]+discovery_timeout_test_NOT_BUILT \(Not Run\) diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-stderr.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-stderr.txt new file mode 100644 index 0000000..75afe4a --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-stderr.txt @@ -0,0 +1,8 @@ +CMake Error at .*GoogleTestAddTests.cmake:[0-9]+ \(message\): +[ \t]*Error running test executable. ++ +[ \t]*Path: '.*discovery_timeout_test(\.exe)?' +[ \t]*Result: Process terminated due to timeout +[ \t]*Output: +[ \t]*timeout. +[ \t]*case diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-stdout.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-stdout.txt new file mode 100644 index 0000000..d65061f --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-PRE_TEST-timeout-test-stdout.txt @@ -0,0 +1 @@ +Test project .*GoogleTest-discovery-timeout diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-build-result.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-build-result.txt deleted file mode 100644 index d197c91..0000000 --- a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-build-result.txt +++ /dev/null @@ -1 +0,0 @@ -[^0] diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-build-stdout.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-build-stdout.txt deleted file mode 100644 index 3a6572c..0000000 --- a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-build-stdout.txt +++ /dev/null @@ -1,7 +0,0 @@ -( *|[0-9]+>)CMake Error at .*GoogleTestAddTests.cmake:[0-9]+ \(message\): -( *|[0-9]+>) Error running test executable. -?( *|[0-9]+>) -( *|[0-9]+>) Path: '.*discovery_timeout_test(\.exe)?' -( *|[0-9]+>) Result: Process terminated due to timeout -( *|[0-9]+>) Output: -( *|[0-9]+>) + diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-result.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-result.txt deleted file mode 100644 index d197c91..0000000 --- a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-result.txt +++ /dev/null @@ -1 +0,0 @@ -[^0] diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-stderr.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-stderr.txt deleted file mode 100644 index f6be939..0000000 --- a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-stderr.txt +++ /dev/null @@ -1,2 +0,0 @@ -Unable to find executable: discovery_timeout_test_NOT_BUILT -Errors while running CTest diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-stdout.txt b/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-stdout.txt deleted file mode 100644 index d4c4e7b..0000000 --- a/Tests/RunCMake/GoogleTest/GoogleTest-discovery-timeout-test-stdout.txt +++ /dev/null @@ -1,18 +0,0 @@ -Test project .*GoogleTest-build -[ \t]*Start [0-9]+: discovery_timeout_test_NOT_BUILT -Could not find executable discovery_timeout_test_NOT_BUILT -Looked in the following places: -discovery_timeout_test_NOT_BUILT -discovery_timeout_test_NOT_BUILT(\.exe)? -Debug/discovery_timeout_test_NOT_BUILT -Debug/discovery_timeout_test_NOT_BUILT(\.exe)? -Debug/discovery_timeout_test_NOT_BUILT -Debug/discovery_timeout_test_NOT_BUILT(\.exe)? -[^\n]+discovery_timeout_test_NOT_BUILT +\.+\*\*\*Not Run +[0-9.]+ sec -+ -0% tests passed, 1 tests failed out of 1 -+ -Total Test time \(real\) = +[0-9.]+ sec -+ -The following tests FAILED: -[^\n]+discovery_timeout_test_NOT_BUILT \(Not Run\) diff --git a/Tests/RunCMake/GoogleTest/GoogleTest.cmake b/Tests/RunCMake/GoogleTest/GoogleTest.cmake index 31808c6..4bc6b9d 100644 --- a/Tests/RunCMake/GoogleTest/GoogleTest.cmake +++ b/Tests/RunCMake/GoogleTest/GoogleTest.cmake @@ -49,11 +49,3 @@ gtest_discover_tests( DISCOVERY_TIMEOUT 20 PROPERTIES TIMEOUT 2 ) - -add_executable(discovery_timeout_test timeout_test.cpp) -target_compile_definitions(discovery_timeout_test PRIVATE discoverySleepSec=10) -gtest_discover_tests( - discovery_timeout_test - TEST_PREFIX discovery_ - DISCOVERY_TIMEOUT 2 -) diff --git a/Tests/RunCMake/GoogleTest/GoogleTestDiscoveryTimeout.cmake b/Tests/RunCMake/GoogleTest/GoogleTestDiscoveryTimeout.cmake new file mode 100644 index 0000000..7398faf --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTestDiscoveryTimeout.cmake @@ -0,0 +1,14 @@ +project(test_include_dirs) +include(CTest) +include(GoogleTest) + +enable_testing() + +add_executable(discovery_timeout_test timeout_test.cpp) +target_compile_definitions(discovery_timeout_test PRIVATE discoverySleepSec=10) +gtest_discover_tests( + discovery_timeout_test + TEST_PREFIX discovery_ + DISCOVERY_TIMEOUT 2 + DISCOVERY_MODE ${DISCOVERY_MODE} +) diff --git a/Tests/RunCMake/GoogleTest/RunCMakeTest.cmake b/Tests/RunCMake/GoogleTest/RunCMakeTest.cmake index 8070512..97a777b 100644 --- a/Tests/RunCMake/GoogleTest/RunCMakeTest.cmake +++ b/Tests/RunCMake/GoogleTest/RunCMakeTest.cmake @@ -1,6 +1,6 @@ include(RunCMake) -function(run_GoogleTest) +function(run_GoogleTest DISCOVERY_MODE) # Use a single build tree for a few tests without cleaning. set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/GoogleTest-build) set(RunCMake_TEST_NO_CLEAN 1) @@ -10,7 +10,7 @@ function(run_GoogleTest) file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") - run_cmake(GoogleTest) + run_cmake_with_options(GoogleTest -DCMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE=${DISCOVERY_MODE}) run_cmake_command(GoogleTest-build ${CMAKE_COMMAND} @@ -26,15 +26,6 @@ function(run_GoogleTest) --target property_timeout_test ) - set(RunCMake_TEST_OUTPUT_MERGE 1) - run_cmake_command(GoogleTest-discovery-timeout-build - ${CMAKE_COMMAND} - --build . - --config Debug - --target discovery_timeout_test - ) - set(RunCMake_TEST_OUTPUT_MERGE 0) - run_cmake_command(GoogleTest-test1 ${CMAKE_CTEST_COMMAND} -C Debug @@ -69,16 +60,9 @@ function(run_GoogleTest) -R property_timeout\\.case_with_discovery --no-label-summary ) - - run_cmake_command(GoogleTest-discovery-timeout-test - ${CMAKE_CTEST_COMMAND} - -C Debug - -R discovery_timeout_test - --no-label-summary - ) endfunction() -function(run_GoogleTestXML) +function(run_GoogleTestXML DISCOVERY_MODE) # Use a single build tree for a few tests without cleaning. set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/GoogleTestXML-build) set(RunCMake_TEST_NO_CLEAN 1) @@ -88,7 +72,7 @@ function(run_GoogleTestXML) file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") - run_cmake(GoogleTestXML) + run_cmake_with_options(GoogleTestXML -DCMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE=${DISCOVERY_MODE}) run_cmake_command(GoogleTestXML-discovery ${CMAKE_COMMAND} @@ -105,5 +89,39 @@ function(run_GoogleTestXML) ) endfunction() -run_GoogleTest() -run_GoogleTestXML() +function(run_GoogleTest_discovery_timeout DISCOVERY_MODE) + # Use a single build tree for a few tests without cleaning. + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/GoogleTest-discovery-timeout) + set(RunCMake_TEST_NO_CLEAN 1) + if(NOT RunCMake_GENERATOR_IS_MULTI_CONFIG) + set(RunCMake_TEST_OPTIONS -DCMAKE_BUILD_TYPE=Debug) + endif() + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + + run_cmake_with_options(GoogleTestDiscoveryTimeout -DDISCOVERY_MODE=${DISCOVERY_MODE}) + + set(RunCMake_TEST_OUTPUT_MERGE 1) + run_cmake_command(GoogleTest-discovery-${DISCOVERY_MODE}-timeout-build + ${CMAKE_COMMAND} + --build . + --config Debug + --target discovery_timeout_test + ) + set(RunCMake_TEST_OUTPUT_MERGE 0) + + run_cmake_command(GoogleTest-discovery-${DISCOVERY_MODE}-timeout-test + ${CMAKE_CTEST_COMMAND} + -C Debug + -R discovery_timeout_test + --no-label-sumary + ) +endfunction() + +foreach(DISCOVERY_MODE POST_BUILD PRE_TEST) + message("Testing ${DISCOVERY_MODE} discovery mode via CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE global override...") + run_GoogleTest(${DISCOVERY_MODE}) + run_GoogleTestXML(${DISCOVERY_MODE}) + message("Testing ${DISCOVERY_MODE} discovery mode via DISCOVERY_MODE option...") + run_GoogleTest_discovery_timeout(${DISCOVERY_MODE}) +endforeach() diff --git a/Tests/RunCMake/GoogleTest/timeout_test.cpp b/Tests/RunCMake/GoogleTest/timeout_test.cpp index b8ad055..5506269 100644 --- a/Tests/RunCMake/GoogleTest/timeout_test.cpp +++ b/Tests/RunCMake/GoogleTest/timeout_test.cpp @@ -4,9 +4,10 @@ # include #endif -#include #include +#include + void sleepFor(unsigned seconds) { #if defined(_WIN32) @@ -23,8 +24,8 @@ int main(int argc, char** argv) // invoked with --gtest_list_tests. Thus, we fake that here. This allows us // to test the module without actually needing Google Test. if (argc > 1 && std::string(argv[1]) == "--gtest_list_tests") { - std::cout << "timeout." << std::endl; - std::cout << " case" << std::endl; + printf("timeout.\n case\n"); + fflush(stdout); #ifdef discoverySleepSec sleepFor(discoverySleepSec); #endif -- cgit v0.12 From 3b4838b57fc70bd64b29044ccbdf582f14a0574d Mon Sep 17 00:00:00 2001 From: Ryan Thornton Date: Mon, 16 Mar 2020 15:16:38 -0500 Subject: GoogleTest: Add tests for MultiConfig discovery in PRE_TEST mode PRE_TEST makes it possible to properly distinguish between test cases that exist only in certain configurations. In the new test scenario, debug tests are disabled in release builds, and release tests are disabled in debug builds when a multi config generator is used. Note, this is a bit of a hack and *only* works for PRE_TEST mode. POST_BUILD makes no attempt to get this right. It preserves the status quo and you obtain the tests that were last discovered. See further discussion in !4078 Ideally, the POST_BUILD behavior could be fixed by using generator expressions in OUTPUT and BYPRODUCT expressions. Then you could do something like: set(ctest_include_file "${ctest_file_base}_include-$.cmake") set(ctest_tests_file "${ctest_file_base}_tests-$.cmake") Once #12877 lands, maybe this can be revisited. Co-authored-by: Ryan Thornton Co-authored-by: Kevin Puetz --- .../GoogleTest-configuration-debug-stdout.txt | 5 +++ .../GoogleTest-configuration-release-stdout.txt | 5 +++ .../GoogleTestDiscoveryMultiConfig.cmake | 14 ++++++++ Tests/RunCMake/GoogleTest/RunCMakeTest.cmake | 41 ++++++++++++++++++++++ Tests/RunCMake/GoogleTest/configuration_gtest.cpp | 23 ++++++++++++ 5 files changed, 88 insertions(+) create mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-configuration-debug-stdout.txt create mode 100644 Tests/RunCMake/GoogleTest/GoogleTest-configuration-release-stdout.txt create mode 100644 Tests/RunCMake/GoogleTest/GoogleTestDiscoveryMultiConfig.cmake create mode 100644 Tests/RunCMake/GoogleTest/configuration_gtest.cpp diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-configuration-debug-stdout.txt b/Tests/RunCMake/GoogleTest/GoogleTest-configuration-debug-stdout.txt new file mode 100644 index 0000000..1f5d1a5 --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTest-configuration-debug-stdout.txt @@ -0,0 +1,5 @@ +Test project .*GoogleTest-discovery-multi-config +[ \t]*Test #[0-9]+: configuration\.case_release \(Disabled\) +[ \t]*Test #[0-9]+: configuration\.case_debug ++ +Total Tests: 2 diff --git a/Tests/RunCMake/GoogleTest/GoogleTest-configuration-release-stdout.txt b/Tests/RunCMake/GoogleTest/GoogleTest-configuration-release-stdout.txt new file mode 100644 index 0000000..4f91664 --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTest-configuration-release-stdout.txt @@ -0,0 +1,5 @@ +Test project .*GoogleTest-discovery-multi-config +[ \t]*Test #[0-9]+: configuration\.case_release +[ \t]*Test #[0-9]+: configuration\.case_debug \(Disabled\) ++ +Total Tests: 2 diff --git a/Tests/RunCMake/GoogleTest/GoogleTestDiscoveryMultiConfig.cmake b/Tests/RunCMake/GoogleTest/GoogleTestDiscoveryMultiConfig.cmake new file mode 100644 index 0000000..1919dc1 --- /dev/null +++ b/Tests/RunCMake/GoogleTest/GoogleTestDiscoveryMultiConfig.cmake @@ -0,0 +1,14 @@ +project(test_include_dirs) +include(CTest) +include(GoogleTest) + +enable_testing() + +add_executable(configuration_gtest configuration_gtest.cpp) +target_compile_definitions(configuration_gtest PRIVATE $<$:DEBUG=1>) + +gtest_discover_tests( + configuration_gtest + PROPERTIES LABELS CONFIG + DISCOVERY_MODE PRE_TEST +) diff --git a/Tests/RunCMake/GoogleTest/RunCMakeTest.cmake b/Tests/RunCMake/GoogleTest/RunCMakeTest.cmake index 97a777b..6b9d458 100644 --- a/Tests/RunCMake/GoogleTest/RunCMakeTest.cmake +++ b/Tests/RunCMake/GoogleTest/RunCMakeTest.cmake @@ -118,6 +118,42 @@ function(run_GoogleTest_discovery_timeout DISCOVERY_MODE) ) endfunction() +function(run_GoogleTest_discovery_multi_config) + # Use a single build tree for a few tests without cleaning. + set(RunCMake_TEST_BINARY_DIR ${RunCMake_BINARY_DIR}/GoogleTest-discovery-multi-config) + set(RunCMake_TEST_NO_CLEAN 1) + file(REMOVE_RECURSE "${RunCMake_TEST_BINARY_DIR}") + file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}") + + run_cmake(GoogleTestDiscoveryMultiConfig) + + run_cmake_command(GoogleTest-build-release + ${CMAKE_COMMAND} + --build . + --config Release + --target configuration_gtest + ) + run_cmake_command(GoogleTest-build-debug + ${CMAKE_COMMAND} + --build . + --config Debug + --target configuration_gtest + ) + run_cmake_command(GoogleTest-configuration-release + ${CMAKE_CTEST_COMMAND} + -C Release + -L CONFIG + -N + ) + run_cmake_command(GoogleTest-configuration-debug + ${CMAKE_CTEST_COMMAND} + -C Debug + -L CONFIG + -N + ) + +endfunction() + foreach(DISCOVERY_MODE POST_BUILD PRE_TEST) message("Testing ${DISCOVERY_MODE} discovery mode via CMAKE_GTEST_DISCOVER_TESTS_DISCOVERY_MODE global override...") run_GoogleTest(${DISCOVERY_MODE}) @@ -125,3 +161,8 @@ foreach(DISCOVERY_MODE POST_BUILD PRE_TEST) message("Testing ${DISCOVERY_MODE} discovery mode via DISCOVERY_MODE option...") run_GoogleTest_discovery_timeout(${DISCOVERY_MODE}) endforeach() + +if(RunCMake_GENERATOR_IS_MULTI_CONFIG) + message("Testing PRE_TEST discovery multi configuration...") + run_GoogleTest_discovery_multi_config() +endif() diff --git a/Tests/RunCMake/GoogleTest/configuration_gtest.cpp b/Tests/RunCMake/GoogleTest/configuration_gtest.cpp new file mode 100644 index 0000000..3cbb134 --- /dev/null +++ b/Tests/RunCMake/GoogleTest/configuration_gtest.cpp @@ -0,0 +1,23 @@ +#include +#include + +int main(int argc, char** argv) +{ + // Note: GoogleTest.cmake doesn't actually depend on Google Test as such; + // it only requires that we produces output in the expected format when + // invoked with --gtest_list_tests. Thus, we fake that here. This allows us + // to test the module without actually needing Google Test. + if (argc > 1 && std::string(argv[1]) == "--gtest_list_tests") { + std::cout << "configuration." << std::endl; +#ifdef DEBUG + std::cout << " DISABLED_case_release" << std::endl; + std::cout << " case_debug" << std::endl; +#else + std::cout << " case_release" << std::endl; + std::cout << " DISABLED_case_debug" << std::endl; +#endif + return 0; + } + + return 1; +} -- cgit v0.12