From 52eac4573d8e10c124aa92e8f12866c16e556d5b Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Wed, 21 Apr 2021 14:04:51 -0400 Subject: Help: Fix link to cdash.org from CTest manual --- Help/manual/ctest.1.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index 3d80cec..e35e162 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -1671,4 +1671,4 @@ See Also .. include:: LINKS.txt -.. _`CDash`: http://cdash.org/ +_`CDash`: https://cdash.org -- cgit v0.12 From 63b5ddcce2acff2a6496e6bed737cf6e2788bfce Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Wed, 21 Apr 2021 14:05:00 -0400 Subject: Tests: Add cases for CTest extra measurements from tests --- Tests/RunCMake/CMakeLists.txt | 4 +++- Tests/RunCMake/ctest_test/RunCMakeTest.cmake | 16 ++++++++++++++++ Tests/RunCMake/ctest_test/TestMeasurements-check.cmake | 17 +++++++++++++++++ 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 Tests/RunCMake/ctest_test/TestMeasurements-check.cmake diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt index db90a81..9bc4131 100644 --- a/Tests/RunCMake/CMakeLists.txt +++ b/Tests/RunCMake/CMakeLists.txt @@ -361,7 +361,9 @@ if(COVERAGE_COMMAND) endif() add_RunCMake_test(ctest_start) add_RunCMake_test(ctest_submit) -add_RunCMake_test(ctest_test) +add_RunCMake_test(ctest_test + -DIMAGE_DIR=${CMAKE_SOURCE_DIR}/Utilities/Sphinx/static +) add_RunCMake_test(ctest_disabled_test) add_RunCMake_test(ctest_skipped_test) add_RunCMake_test(ctest_update) diff --git a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake index 901ac11..b559e89 100644 --- a/Tests/RunCMake/ctest_test/RunCMakeTest.cmake +++ b/Tests/RunCMake/ctest_test/RunCMakeTest.cmake @@ -149,3 +149,19 @@ run_environment() # test for OUTPUT_JUNIT run_ctest_test(OutputJUnit OUTPUT_JUNIT junit.xml REPEAT UNTIL_FAIL:2) + +# Verify that extra measurements get reported. +function(run_measurements) + set(CASE_CMAKELISTS_SUFFIX_CODE [[ +add_test( + NAME double_measurement + COMMAND ${CMAKE_COMMAND} -E + echo 1.4847) +add_test( + NAME img_measurement + COMMAND ${CMAKE_COMMAND} -E + echo ]] ${IMAGE_DIR}/cmake-logo-16.png [[) + ]]) + run_ctest(TestMeasurements) +endfunction() +run_measurements() diff --git a/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake b/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake new file mode 100644 index 0000000..9ff9447 --- /dev/null +++ b/Tests/RunCMake/ctest_test/TestMeasurements-check.cmake @@ -0,0 +1,17 @@ +file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/TAG" _tag) +string(REGEX REPLACE "^([^\n]*)\n.*$" "\\1" _date "${_tag}") +file(READ "${RunCMake_TEST_BINARY_DIR}/Testing/${_date}/Test.xml" _test_contents) + +# Check double measurement. +if(NOT _test_contents MATCHES [[NamedMeasurement type="numeric/double" name="my_custom_value"]]) + string(APPEND RunCMake_TEST_FAILED + "Could not find expected tag for type='numeric/double' in Test.xml") +endif() +if(NOT _test_contents MATCHES "1.4847") + string(APPEND RunCMake_TEST_FAILED "Could not find expected measurement value in Test.xml") +endif() +# Check img measurement. +if(NOT _test_contents MATCHES [[NamedMeasurement name="TestImage" type="image/png" encoding="base64"]]) + string(APPEND RunCMake_TEST_FAILED + "Could not find expected tag for type='image/png' in Test.xml") +endif() -- cgit v0.12 From b60789a758ddd755be635ae60b24364ecf7fdd34 Mon Sep 17 00:00:00 2001 From: Zack Galbreath Date: Wed, 21 Apr 2021 14:05:11 -0400 Subject: Help: Document CTest custom test measurements --- Help/command/ctest_test.rst | 80 +++++++++++++++++++++++++++++++++++++++++++++ Help/manual/ctest.1.rst | 2 ++ 2 files changed, 82 insertions(+) diff --git a/Help/command/ctest_test.rst b/Help/command/ctest_test.rst index 9081b3f..c61d01e 100644 --- a/Help/command/ctest_test.rst +++ b/Help/command/ctest_test.rst @@ -170,3 +170,83 @@ The options are: See also the :variable:`CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE` and :variable:`CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE` variables. + +.. _`Additional Test Measurements`: + +Additional Test Measurements +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +CTest can parse the output of your tests for extra measurements to report +to CDash. + +When run as a :ref:`Dashboard Client`, CTest will include these custom +measurements in the ``Test.xml`` file that gets uploaded to CDash. + +Check the `CDash test measurement documentation +`_ +for more information on the types of test measurements that CDash recognizes. + +The following example demonstrates how to output a variety of custom test +measurements. + +.. code-block:: c++ + + std::cout << + "28.3" + << std::endl; + + std::cout << + "red" + << std::endl; + + std::cout << + "https://cmake.org" + << std::endl; + + std::cout << + "" << + "line 1.\n" << + " \033[31;1m line 2. Bold red, and indented!\033[0;0ml\n" << + "line 3. Not bold or indented...\n" << + "" << std::endl; + +Image Measurements +"""""""""""""""""" + +The following example demonstrates how to upload test images to CDash. + +.. code-block:: c++ + + std::cout << + "" << + "/dir/to/test_img.jpg" << std::endl; + + std::cout << + "" << + "/dir/to/valid_img.gif" << std::endl; + + std::cout << + " << + "/dir/to/img.png" + << std::endl; + +Images will be displayed together in an interactive comparison mode on CDash +if they are provided with two or more of the following names. + +* ``TestImage`` +* ``ValidImage`` +* ``BaselineImage`` +* ``DifferenceImage2`` + +By convention, ``TestImage`` is the image generated by your test, and +``ValidImage`` (or ``BaselineImage``) is basis of comparison used to determine +if the test passed or failed. + +If another image name is used it will be displayed by CDash as a static image +separate from the interactive comparison UI. + +Attached Files +"""""""""""""" + +To associate other types of files with a test, use the +:prop_test:`ATTACHED_FILES` or :prop_test:`ATTACHED_FILES_ON_FAIL` test properties. diff --git a/Help/manual/ctest.1.rst b/Help/manual/ctest.1.rst index e35e162..811997c 100644 --- a/Help/manual/ctest.1.rst +++ b/Help/manual/ctest.1.rst @@ -1095,6 +1095,8 @@ Configuration settings include: * `CTest Script`_ variable: :variable:`CTEST_TEST_TIMEOUT` * :module:`CTest` module variable: ``DART_TESTING_TIMEOUT`` +To report extra test values to CDash, see :ref:`Additional Test Measurements`. + .. _`CTest Coverage Step`: CTest Coverage Step -- cgit v0.12