From a3aa5596a11b67c0de860b9cb366038c3da2a4e2 Mon Sep 17 00:00:00 2001 From: Brad King Date: Tue, 27 Apr 2021 14:13:57 -0400 Subject: Tests: Isolate TestDriver build directories --- Tests/CMakeLists.txt | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index d6a20bc..cd1b465 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -1742,33 +1742,35 @@ if(BUILD_TESTING) add_test(testdriver1 ${CMAKE_CTEST_COMMAND} --build-and-test "${CMake_SOURCE_DIR}/Tests/TestDriver" - "${CMake_BINARY_DIR}/Tests/TestDriver" + "${CMake_BINARY_DIR}/Tests/TestDriver1" ${build_generator_args} --build-exe-dir "${CMake_BINARY_DIR}/Tests/Wrapping/bin" --build-project TestDriverTest --test-command TestDriverTest test1 ) + list(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/TestDriver1") add_test(testdriver2 ${CMAKE_CTEST_COMMAND} --build-and-test "${CMake_SOURCE_DIR}/Tests/TestDriver" - "${CMake_BINARY_DIR}/Tests/TestDriver" + "${CMake_BINARY_DIR}/Tests/TestDriver2" ${build_generator_args} --build-exe-dir "${CMake_BINARY_DIR}/Tests/Wrapping/bin" --build-project TestDriverTest --test-command TestDriverTest test2 ) + list(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/TestDriver2") add_test(testdriver3 ${CMAKE_CTEST_COMMAND} --build-and-test "${CMake_SOURCE_DIR}/Tests/TestDriver" - "${CMake_BINARY_DIR}/Tests/TestDriver" + "${CMake_BINARY_DIR}/Tests/TestDriver3" ${build_generator_args} --build-exe-dir "${CMake_BINARY_DIR}/Tests/Wrapping/bin" --build-project TestDriverTest --test-command TestDriverTest subdir/test3 ) - list(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/TestDriver") + list(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/TestDriver3") add_test(Dependency ${CMAKE_CTEST_COMMAND} --build-and-test -- cgit v0.12 From 3f6ff4b5dba01f2f58dca75546ebefd0830110a5 Mon Sep 17 00:00:00 2001 From: Sergey Bronnikov Date: Tue, 27 Apr 2021 11:01:21 +0300 Subject: create_test_sourcelist: add test driver option to run all tests New option `-A` passed to test binary allows to run all testcases at once and prints a report in a standard format - TestAnythingProtocol v.13 [1]. Execution of test whose names will be passed after an option will be skipped. Sample of output: TAP version 13 1..6 ok 1 TestCryptoHash # 0.030000 ok 2 TestCryptoRand # 0.008000 not ok 3 TestCryptoCipher # 0.005000 ok 4 TestCryptoProtectData # 0.000000 cbPlainText: 21 cbCipherText: 32 PlainText: MySecretPassword123! (cbPlainText = 21, cbCipherText = 32) Decrypted CipherText: MySecretPassword123! ok 5 TestCryptoProtectMemory # 0.014000 ok 6 TestCryptoCertEnumCertificatesInStore # 0.000000 1. https://testanything.org/ Fixes: #19367 --- Templates/TestDriver.cxx.in | 43 ++++++++++++++++++++++++++++++++++++++++++- Tests/CMakeLists.txt | 26 ++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/Templates/TestDriver.cxx.in b/Templates/TestDriver.cxx.in index 3e47d6a..e3efa79 100644 --- a/Templates/TestDriver.cxx.in +++ b/Templates/TestDriver.cxx.in @@ -2,6 +2,7 @@ #include /* NOLINT */ #include /* NOLINT */ #include /* NOLINT */ +#include #if defined(_MSC_VER) #pragma warning(disable : 4996) /* deprecation */ @@ -62,11 +63,23 @@ static char* lowercase(const char* string) return new_string; } +int isTestSkipped(const char *name, int n_skipped_tests, char *skipped_tests[]) { + int i; + for (i = 0; i < n_skipped_tests; i++) { + if (strcmp(name, skipped_tests[i]) == 0) { + return 1; + } + } + + return 0; +} + int main(int ac, char* av[]) { int i; int testNum = 0; int partial_match; + int run_all; char *arg; int testToRun = -1; @@ -95,15 +108,43 @@ int main(int ac, char* av[]) av++; } partial_match = 0; + run_all = 0; arg = CM_NULL; /* NOLINT */ - /* If partial match is requested. */ + /* If partial match or running all tests are requested. */ if (testToRun == -1 && ac > 1) { partial_match = (strcmp(av[1], "-R") == 0) ? 1 : 0; + run_all = (strcmp(av[1], "-A") == 0) ? 1 : 0; } if (partial_match != 0 && ac < 3) { printf("-R needs an additional parameter.\n"); return -1; } + if (run_all == 1) { + clock_t t; + int status = 0; + const char* status_message = NULL; + printf("TAP version 13\n"); + printf("1..%d\n", NumTests); + for (i = 0; i < NumTests; ++i) { + const char *name = cmakeGeneratedFunctionMapEntries[i].name; + if (ac > 2) { + if (isTestSkipped(name, ac - 2, av + 2) == 1) { + printf("ok %d %s # SKIP\n", i + 1, name); + continue; + } + } + t = clock(); + status = (*cmakeGeneratedFunctionMapEntries[i].func)(ac, av); + t = clock() - t; + double time_taken = ((double)t) / CLOCKS_PER_SEC; + status_message = (status == -1) ? "not ok" : "ok"; + printf("%s %d %s # %f\n", status_message, i + 1, name, time_taken); + } + printf("All tests finished.\n"); + + return 0; + } + if (testToRun == -1) { arg = lowercase(av[1 + partial_match]); } diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt index cd1b465..f9df95a 100644 --- a/Tests/CMakeLists.txt +++ b/Tests/CMakeLists.txt @@ -1772,6 +1772,32 @@ if(BUILD_TESTING) ) list(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/TestDriver3") + add_test(testdriver4 ${CMAKE_CTEST_COMMAND} + --build-and-test + "${CMake_SOURCE_DIR}/Tests/TestDriver" + "${CMake_BINARY_DIR}/Tests/TestDriver4" + ${build_generator_args} + --build-exe-dir "${CMake_BINARY_DIR}/Tests/Wrapping/bin" + --build-project TestDriverTest + --test-command TestDriverTest -A test2 + ) + list(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/TestDriver4") + + add_test(testdriver5 ${CMAKE_CTEST_COMMAND} + --build-and-test + "${CMake_SOURCE_DIR}/Tests/TestDriver" + "${CMake_BINARY_DIR}/Tests/TestDriver5" + ${build_generator_args} + --build-exe-dir "${CMake_BINARY_DIR}/Tests/Wrapping/bin" + --build-project TestDriverTest + --test-command TestDriverTest -A test2 + ) + list(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/TestDriver5") + set_tests_properties(testdriver5 PROPERTIES + PASS_REGULAR_EXPRESSION + "TAP version 13\n1\\.\\.3.+ok 1 test1 # [0-9]+\\.[0-9]+.*All tests finished." + ) + add_test(Dependency ${CMAKE_CTEST_COMMAND} --build-and-test "${CMake_SOURCE_DIR}/Tests/Dependency" -- cgit v0.12