From 13465306cfb7f6ff678c962758c4f46ab6a53ac2 Mon Sep 17 00:00:00 2001 From: scivision Date: Wed, 8 Nov 2023 14:17:14 -0500 Subject: Help: Clarify pass, fail and skip test properties and exit code The behavior when system-level failures occur was not previously defined. Clarify the behavior and provide an example for how to work around them. Affected test properties: - PASS_REGULAR_EXPRESSION - FAIL_REGULAR_EXPRESSION - SKIP_REGULAR_EXPRESSION - SKIP_RETURN_CODE Also update the existing WILL_FAIL test property docs to using the same consistent wording. --- Help/prop_test/FAIL_REGULAR_EXPRESSION.rst | 17 ++++++++---- Help/prop_test/PASS_REGULAR_EXPRESSION.rst | 43 +++++++++++++++++++++++++----- Help/prop_test/SKIP_REGULAR_EXPRESSION.rst | 31 ++++++++++++++++++--- Help/prop_test/SKIP_RETURN_CODE.rst | 37 ++++++++++++++++++++++++- Help/prop_test/WILL_FAIL.rst | 7 ++--- 5 files changed, 116 insertions(+), 19 deletions(-) diff --git a/Help/prop_test/FAIL_REGULAR_EXPRESSION.rst b/Help/prop_test/FAIL_REGULAR_EXPRESSION.rst index 1ec4517..e94856d 100644 --- a/Help/prop_test/FAIL_REGULAR_EXPRESSION.rst +++ b/Help/prop_test/FAIL_REGULAR_EXPRESSION.rst @@ -1,15 +1,22 @@ FAIL_REGULAR_EXPRESSION ----------------------- -If the output matches this regular expression the test will fail, -regardless of the process exit code. +If the test output (stdout or stderr) matches this regular expression the test +will fail, regardless of the process exit code. Tests that exceed the timeout +specified by :prop_test:`TIMEOUT` fail regardless of +``FAIL_REGULAR_EXPRESSION``. Any non-zero return code or system-level test +failures including segmentation faults, signal abort, or heap errors fail the +test even if the regular expression does not match. -If set, if the output matches one of specified regular expressions, -the test will fail. Example: +If set, if the output matches one of specified regular expressions, the test +will fail. Example: .. code-block:: cmake - set_tests_properties(mytest PROPERTIES + # test would pass, except for FAIL_REGULAR_EXPRESSION + add_test(NAME mytest COMMAND ${CMAKE_COMMAND} -E echo "Failed") + + set_property(TEST mytest PROPERTY FAIL_REGULAR_EXPRESSION "[^a-z]Error;ERROR;Failed" ) diff --git a/Help/prop_test/PASS_REGULAR_EXPRESSION.rst b/Help/prop_test/PASS_REGULAR_EXPRESSION.rst index 96468c0..9f92491 100644 --- a/Help/prop_test/PASS_REGULAR_EXPRESSION.rst +++ b/Help/prop_test/PASS_REGULAR_EXPRESSION.rst @@ -1,20 +1,49 @@ PASS_REGULAR_EXPRESSION ----------------------- -The output must match this regular expression for the test to pass. -The process exit code is ignored. +The test output (stdout or stderr) must match this regular expression +for the test to pass. The process exit code is ignored. Tests that exceed +the timeout specified by :prop_test:`TIMEOUT` still fail regardless of +``PASS_REGULAR_EXPRESSION``. System-level test failures including +segmentation faults, signal abort, or heap errors may fail the test even +if ``PASS_REGULAR_EXPRESSION`` is matched. -If set, the test output will be checked against the specified regular -expressions and at least one of the regular expressions has to match, -otherwise the test will fail. Example: +Example: .. code-block:: cmake - set_tests_properties(mytest PROPERTIES - PASS_REGULAR_EXPRESSION "TestPassed;All ok" + add_test(NAME mytest COMMAND ${CMAKE_COMMAND} -E echo "Passed this test") + + set_property(TEST mytest PROPERTY + PASS_REGULAR_EXPRESSION "pass;Passed" ) ``PASS_REGULAR_EXPRESSION`` expects a list of regular expressions. +To run a test that may have a system-level failure, but still pass if +``PASS_REGULAR_EXPRESSION`` matches, use a CMake command to wrap the +executable run. Note that this will prevent automatic handling of the +:prop_tgt:`CROSSCOMPILING_EMULATOR` target property. + +.. code-block:: cmake + + add_executable(main main.c) + + add_test(NAME sigabrt COMMAND ${CMAKE_COMMAND} -E env $) + + set_property(TEST sigabrt PROPERTY PROPERTY_REGULAR_EXPRESSION "pass;Passed") + +.. code-block:: c + + #include + #include + + int main(void){ + fprintf(stdout, "Passed\n"); + fflush(stdout); /* ensure the output buffer is seen */ + raise(SIGABRT); + return 0; + } + See also the :prop_test:`FAIL_REGULAR_EXPRESSION` and :prop_test:`SKIP_REGULAR_EXPRESSION` test properties. diff --git a/Help/prop_test/SKIP_REGULAR_EXPRESSION.rst b/Help/prop_test/SKIP_REGULAR_EXPRESSION.rst index 60038e4..9836412 100644 --- a/Help/prop_test/SKIP_REGULAR_EXPRESSION.rst +++ b/Help/prop_test/SKIP_REGULAR_EXPRESSION.rst @@ -3,19 +3,44 @@ SKIP_REGULAR_EXPRESSION .. versionadded:: 3.16 -If the output matches this regular expression the test will be marked as skipped. +If the test output (stderr or stdout) matches this regular expression the test +will be marked as skipped, regardless of the process exit code. Tests that +exceed the timeout specified by :prop_test:`TIMEOUT` still fail regardless of +``SKIP_REGULAR_EXPRESSION``. System-level test failures including segmentation +faults, signal abort, or heap errors may fail the test even if the regular +expression matches. -If set, if the output matches one of specified regular expressions, -the test will be marked as skipped. Example: +Example: .. code-block:: cmake + add_test(NAME mytest COMMAND ${CMAKE_COMMAND} -E echo "Skipped this test") + set_property(TEST mytest PROPERTY SKIP_REGULAR_EXPRESSION "[^a-z]Skip" "SKIP" "Skipped" ) ``SKIP_REGULAR_EXPRESSION`` expects a list of regular expressions. +To run a test that may have a system-level failure, but still skip if +``SKIP_REGULAR_EXPRESSION`` matches, use a CMake command to wrap the +executable run. Note that this will prevent automatic handling of the +:prop_tgt:`CROSSCOMPILING_EMULATOR` target property. + +.. code-block:: cmake + + add_executable(main main.c) + + add_test(NAME sigabrt COMMAND ${CMAKE_COMMAND} -E env $) + + set_property(TEST sigabrt PROPERTY SKIP_REGULAR_EXPRESSION "SIGABRT;[aA]bort") + +.. code-block:: c + + #include + + int main(void){ raise(SIGABRT); return 0; } + See also the :prop_test:`SKIP_RETURN_CODE`, :prop_test:`PASS_REGULAR_EXPRESSION`, and :prop_test:`FAIL_REGULAR_EXPRESSION` test properties. diff --git a/Help/prop_test/SKIP_RETURN_CODE.rst b/Help/prop_test/SKIP_RETURN_CODE.rst index 23c4c62..096eb09 100644 --- a/Help/prop_test/SKIP_RETURN_CODE.rst +++ b/Help/prop_test/SKIP_RETURN_CODE.rst @@ -9,4 +9,39 @@ a return code of the process can be specified that will mark the test as ``Not Run`` if it is encountered. Valid values are in the range of 0 to 255, inclusive. -See also the :prop_test:`SKIP_REGULAR_EXPRESSION` property. +Tests that exceed the timeout specified by :prop_test:`TIMEOUT` still fail +regardless of ``SKIP_RETURN_CODE``. +System-level test failures including segmentation faults, +signal abort, or heap errors may fail the test even if the return code matches. + +.. code-block:: cmake + + # cmake (1) defines this to return code 1 + add_test(NAME r1 COMMAND ${CMAKE_COMMAND} -E false) + + set_tests_properties(r1 PROPERTIES SKIP_RETURN_CODE 1) + + +To run a test that may have a system-level failure, but still skip if +``SKIP_RETURN_CODE`` matches, use a CMake command to wrap the executable run. +Note that this will prevent automatic handling of the +:prop_tgt:`CROSSCOMPILING_EMULATOR` target property. + +.. code-block:: cmake + + add_executable(main main.c) + + # cmake -E env returns 1 if the command fails in any way + add_test(NAME sigabrt COMMAND ${CMAKE_COMMAND} -E env $) + + set_property(TEST sigabrt PROPERTY SKIP_RETURN_CODE 1) + +.. code-block:: c + + #include + + int main(void){ raise(SIGABRT); return 0; } + + +To handle multiple types of cases that may need to be skipped, consider the +:prop_test:`SKIP_REGULAR_EXPRESSION` property. diff --git a/Help/prop_test/WILL_FAIL.rst b/Help/prop_test/WILL_FAIL.rst index 006b1c3..53437d2 100644 --- a/Help/prop_test/WILL_FAIL.rst +++ b/Help/prop_test/WILL_FAIL.rst @@ -5,8 +5,7 @@ If ``true``, inverts the pass / fail test criteria. Tests for which ``WILL_FAIL`` is ``true`` fail with return code 0 and pass with non-zero return code. Tests that exceed the timeout specified by :prop_test:`TIMEOUT` still fail regardless of ``WILL_FAIL``. - -Caveat: system-level test failures including segmentation faults, +System-level test failures including segmentation faults, signal abort, or heap errors may fail the test even if ``WILL_FAIL`` is true. Example of a test that would ordinarily pass, but fails because ``WILL_FAIL`` @@ -18,7 +17,9 @@ is ``true``: set_property(TEST failed PROPERTY WILL_FAIL true) To run a test that may have a system-level failure, but still pass if -``WILL_FAIL`` is set, use a CMake command to wrap the executable run like: +``WILL_FAIL`` is set, use a CMake command to wrap the executable run. +Note that this will prevent automatic handling of the +:prop_tgt:`CROSSCOMPILING_EMULATOR` target property. .. code-block:: cmake -- cgit v0.12