summaryrefslogtreecommitdiffstats
path: root/Tests/RunCMake/ctest_test/RunCMakeTest.cmake
blob: e2c41a01a75735ebe63204969e0a6d63bde68938 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
include(RunCTest)
set(RunCMake_TEST_TIMEOUT 60)

set(CASE_CTEST_TEST_ARGS "")
set(CASE_CTEST_TEST_LOAD "")

function(run_ctest_test CASE_NAME)
  set(CASE_CTEST_TEST_ARGS "${ARGN}")
  run_ctest(${CASE_NAME})
endfunction()

run_ctest_test(TestQuiet QUIET)

# Tests for the 'Test Load' feature of ctest
#
# Spoof a load average value to make these tests more reliable.
set(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING} 7)
set(RunCTest_VERBOSE_FLAG -VV)
set(CASE_CMAKELISTS_SUFFIX_CODE [[
set_property(TEST RunCMakeVersion PROPERTY PROCESSORS 2)
]])

# Verify that new tests are started when the load average falls below
# our threshold.
run_ctest_test(TestLoadPass TEST_LOAD 8)

# Verify that new tests are not started when the load average exceeds
# our threshold and that they then run once the load average drops.
run_ctest_test(TestLoadWait0 TEST_LOAD 4 PARALLEL_LEVEL 8)
run_ctest_test(TestLoadWait1 TEST_LOAD 8 PARALLEL_LEVEL 8)

# Verify that when an invalid "TEST_LOAD" value is given, a warning
# message is displayed and the value is ignored.
run_ctest_test(TestLoadInvalid TEST_LOAD "ERR1")

# Verify that new tests are started when the load average falls below
# our threshold.
set(CASE_CTEST_TEST_LOAD 9)
run_ctest_test(CTestTestLoadPass)

# Verify that new tests are not started when the load average exceeds
# our threshold and that they then run once the load average drops.
set(CASE_CTEST_TEST_LOAD 6)
run_ctest_test(CTestTestLoadWait0 PARALLEL_LEVEL 8)
set(CASE_CTEST_TEST_LOAD 8)
run_ctest_test(CTestTestLoadWait1 PARALLEL_LEVEL 8)

# Verify that when an invalid "CTEST_TEST_LOAD" value is given,
# a warning message is displayed and the value is ignored.
set(CASE_CTEST_TEST_LOAD "ERR2")
run_ctest_test(CTestTestLoadInvalid)

# Verify that the "TEST_LOAD" value has higher precedence than
# the "CTEST_TEST_LOAD" value
set(CASE_CTEST_TEST_LOAD "ERR3")
run_ctest_test(TestLoadOrder TEST_LOAD "ERR4")

unset(ENV{__CTEST_FAKE_LOAD_AVERAGE_FOR_TESTING})
unset(CASE_CTEST_TEST_LOAD)
unset(CASE_CMAKELISTS_SUFFIX_CODE)
unset(RunCTest_VERBOSE_FLAG)

block()
  set(CASE_CMAKELISTS_SUFFIX_CODE [[
    add_test(NAME testNotRun COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/does_not_exist)
  ]])
  run_ctest_test(NotRun)
endblock()

function(run_TestChangeId)
  set(CASE_TEST_PREFIX_CODE [[
    set(CTEST_CHANGE_ID "<>1")
  ]])

  run_ctest(TestChangeId)
endfunction()
run_TestChangeId()

function(run_TestOutputSize)
  set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion)
  set(CASE_TEST_PREFIX_CODE [[
set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 10)
set(CTEST_CUSTOM_MAXIMUM_FAILED_TEST_OUTPUT_SIZE 12)
  ]])
  set(CASE_CMAKELISTS_SUFFIX_CODE [[
add_test(NAME PassingTest COMMAND ${CMAKE_COMMAND} -E echo PassingTestOutput)
add_test(NAME FailingTest COMMAND ${CMAKE_COMMAND} -E no_such_command)
  ]])

  run_ctest(TestOutputSize)
endfunction()
run_TestOutputSize()

# Test --test-output-truncation
function(run_TestOutputTruncation mode expected)
  set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion)
  set(TRUNCATED_OUTPUT ${expected})  # used in TestOutputTruncation-check.cmake
  string(CONCAT CASE_TEST_PREFIX_CODE "
set(CTEST_CUSTOM_MAXIMUM_PASSED_TEST_OUTPUT_SIZE 5)
set(CTEST_CUSTOM_TEST_OUTPUT_TRUNCATION ${mode})" )
  set(CASE_CMAKELISTS_SUFFIX_CODE "
add_test(NAME Truncation_${mode} COMMAND \${CMAKE_COMMAND} -E echo 123456789)")

  run_ctest(TestOutputTruncation_${mode})
endfunction()
run_TestOutputTruncation("head" "\\.\\.\\.6789")
run_TestOutputTruncation("middle" "12\\.\\.\\..*\\.\\.\\.89")
run_TestOutputTruncation("tail" "12345\\.\\.\\.")
run_TestOutputTruncation("bad" "")

run_ctest_test(TestRepeatBad1 REPEAT UNKNOWN:3)
run_ctest_test(TestRepeatBad2 REPEAT UNTIL_FAIL:-1)

function(run_TestRepeat case return_value )
  set(CASE_CTEST_TEST_ARGS RETURN_VALUE result EXCLUDE RunCMakeVersion ${ARGN})
  string(CONCAT suffix_code [[
add_test(NAME testRepeat
  COMMAND ${CMAKE_COMMAND} -D COUNT_FILE=${CMAKE_CURRENT_BINARY_DIR}/count.cmake
                           -P "]] "${RunCMake_SOURCE_DIR}/TestRepeat${case}" [[.cmake")
set_property(TEST testRepeat PROPERTY TIMEOUT 5)
  ]])
  string(APPEND CASE_CMAKELISTS_SUFFIX_CODE "${suffix_code}")

  run_ctest(TestRepeat${case})

  #write to end of the test file logic to Verify we get the expected
  #return code
  string(REPLACE "RETURN_VALUE:" "" return_value "${return_value}" )
  file(APPEND "${RunCMake_BINARY_DIR}/TestRepeat${case}/test.cmake"
"

  set(expected_result ${return_value})
  message(STATUS \${result})
  if(NOT result EQUAL expected_result)
    message(FATAL_ERROR \"expected a return value of: \${expected_result},
                         instead got: \${result}\")
  endif()
"
  )
endfunction()

run_TestRepeat(UntilFail RETURN_VALUE:1 REPEAT UNTIL_FAIL:3)
run_TestRepeat(UntilPass RETURN_VALUE:0 REPEAT UNTIL_PASS:3)
run_TestRepeat(AfterTimeout RETURN_VALUE:0 REPEAT AFTER_TIMEOUT:3)

# test repeat and not run tests interact correctly
set(CASE_CMAKELISTS_SUFFIX_CODE [[
  add_test(NAME testNotRun COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/does_not_exist)
  set_property(TEST testNotRun PROPERTY TIMEOUT 5)
  ]])
run_TestRepeat(NotRun RETURN_VALUE:1 REPEAT UNTIL_PASS:3)
unset(CASE_CMAKELISTS_SUFFIX_CODE)

# test --stop-on-failure
function(run_stop_on_failure)
  set(CASE_CTEST_TEST_ARGS EXCLUDE RunCMakeVersion)
  set(CASE_CMAKELISTS_SUFFIX_CODE [[
add_test(NAME StoppingTest COMMAND ${CMAKE_COMMAND} -E false)
add_test(NAME NotRunTest COMMAND ${CMAKE_COMMAND} -E true)
  ]])

  run_ctest_test(stop-on-failure STOP_ON_FAILURE)
endfunction()
run_stop_on_failure()


# test include/exclude tests from file
function(run_tests_from_file case)
  set(CASE_CTEST_TEST_ARGS ${ARGN})
  set(CASE_CMAKELISTS_SUFFIX_CODE [[
add_test(NAME Test1 COMMAND ${CMAKE_COMMAND} -E true)
add_test(NAME Test2 COMMAND ${CMAKE_COMMAND} -E true)
add_test(NAME Test11 COMMAND ${CMAKE_COMMAND} -E true)
  ]])

  run_ctest(TestsFromFile-${case})
endfunction()
run_tests_from_file(include INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
run_tests_from_file(exclude EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList.txt)
run_tests_from_file(include-missing INCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)
run_tests_from_file(exclude-missing EXCLUDE_FROM_FILE ${RunCMake_SOURCE_DIR}/TestsFromFile-TestList-missing.txt)


# Make sure environment gets logged
function(run_environment)
  set(ENV{BAD_ENVIRONMENT_VARIABLE} "Bad environment variable")
  set(CASE_CMAKELISTS_SUFFIX_CODE [[
set_property(TEST RunCMakeVersion PROPERTY ENVIRONMENT "ENV1=env1;ENV2=env2")
  ]])

  run_ctest(TestEnvironment)
endfunction()
run_environment()

# test for OUTPUT_JUNIT
run_ctest_test(OutputJUnit OUTPUT_JUNIT junit.xml REPEAT UNTIL_FAIL:2)

# Verify that extra measurements get reported.
function(run_measurements)
  set(CASE_CMAKELISTS_SUFFIX_CODE [[
add_test(
  NAME double_measurement
  COMMAND ${CMAKE_COMMAND} -E
  echo <DartMeasurement type="numeric/double" name="my_custom_value">1.4847</DartMeasurement>)
add_test(
  NAME double_measurement2
  COMMAND ${CMAKE_COMMAND} -E
  echo <CTestMeasurement type="numeric/double" name="another_custom_value">1.8474</CTestMeasurement>)
add_test(
  NAME img_measurement
  COMMAND ${CMAKE_COMMAND} -E
  echo <DartMeasurementFile name="TestImage" type="image/png">]] ${IMAGE_DIR}/cmake-logo-16.png [[</DartMeasurementFile>)
add_test(
  NAME img_measurement2
  COMMAND ${CMAKE_COMMAND} -E
  echo <CTestMeasurementFile name="TestImage2" type="image/png">]] ${IMAGE_DIR}/cmake-logo-16.png [[</CTestMeasurementFile>)
add_test(
  NAME file_measurement
  COMMAND ${CMAKE_COMMAND} -E
  echo <DartMeasurementFile name="my_test_input_data" type="file">]] ${IMAGE_DIR}/cmake-logo-16.png [[</DartMeasurementFile>)
add_test(
  NAME file_measurement2
  COMMAND ${CMAKE_COMMAND} -E
  echo <CTestMeasurementFile name="another_test_input_data" type="file">]] ${IMAGE_DIR}/cmake-logo-16.png [[</CTestMeasurementFile>)
  ]])
  run_ctest(TestMeasurements)
endfunction()
run_measurements()

# Verify that test output can override the Completion Status.
function(run_completion_status)
  set(CASE_CMAKELISTS_SUFFIX_CODE [[
add_test(
  NAME custom_details
  COMMAND ${CMAKE_COMMAND} -E
  echo test output\n<CTestDetails>CustomDetails</CTestDetails>\nmore output)
  ]])
  run_ctest(TestCompletionStatus)
endfunction()
run_completion_status()

# Verify that running ctest_test() multiple times with different label arguments
# doesn't break.
function(run_changing_labels)
  set(CASE_CMAKELISTS_SUFFIX_CODE [[
add_test(NAME a COMMAND ${CMAKE_COMMAND} -E true)
set_property(TEST a PROPERTY LABELS a)
add_test(NAME b COMMAND ${CMAKE_COMMAND} -E true)
set_property(TEST b PROPERTY LABELS b)
  ]])
  run_ctest(TestChangingLabels)
endfunction()
run_changing_labels()

# Verify that test output can add additional labels
function(run_extra_labels)
  set(CASE_CMAKELISTS_SUFFIX_CODE [[
add_test(
  NAME custom_labels
  COMMAND ${CMAKE_COMMAND} -E
  echo before\n<CTestLabel>label2</CTestLabel>\n<CTestLabel>label1</CTestLabel>\n<CTestLabel>label3</CTestLabel>\n<CTestLabel>label2</CTestLabel>\nafter)
set_tests_properties(custom_labels PROPERTIES LABELS "label1")
  ]])
  run_ctest(TestExtraLabels)
endfunction()
run_extra_labels()