Add an option for explicit BYPRODUCTS of custom commands (#14963)

A common idiom in CMake-based build systems is to have custom commands
that generate files not listed explicitly as outputs so that these
files do not have to be newer than the inputs.  The file modification
times of such "byproducts" are updated only when their content changes.
Then other build rules can depend on the byproducts explicitly so that
their dependents rebuild when the content of the original byproducts
really does change.

This "undeclared byproduct" approach is necessary for Makefile, VS, and
Xcode build tools because if a byproduct were listed as an output of a
rule then the rule would always rerun when the input is newer than the
byproduct but the byproduct may never be updated.

Ninja solves this problem by offering a 'restat' feature to check
whether an output was really modified after running a rule and tracking
the fact that it is up to date separately from its timestamp.  However,
Ninja also stats all dependencies up front and will only restat files
that are listed as outputs of rules with the 'restat' option enabled.
Therefore an undeclared byproduct that does not exist at the start of
the build will be considered missing and the build will fail even if
other dependencies would cause the byproduct to be available before its
dependents build.

CMake works around this limitation by adding 'phony' build rules for
custom command dependencies in the build tree that do not have any
explicit specification of what produces them.  This is not optimal
because it prevents Ninja from reporting an error when an input to a
rule really is missing.  A better approach is to allow projects to
explicitly specify the byproducts of their custom commands so that no
phony rules are needed for them.  In order to work with the non-Ninja
generators, the byproducts must be known separately from the outputs.

Add a new "BYPRODUCTS" option to the add_custom_command and
add_custom_target commands to specify byproducts explicitly.  Teach the
Ninja generator to specify byproducts as outputs of the custom commands.
In the case of POST_BUILD, PRE_LINK, and PRE_BUILD events on targets
that link, the byproducts must be specified as outputs of the link rule
that runs the commands.  Activate 'restat' for such rules so that Ninja
knows it needs to check the byproducts, but not for link rules that have
no byproducts.

22 files changed, 170 insertions, 0 deletions

diff --git a/Tests/CMakeLists.txt b/Tests/CMakeLists.txt
index 3e29683..ab28eca 100644
--- a/Tests/CMakeLists.txt
+++ b/Tests/CMakeLists.txt
@@ -716,6 +716,8 @@ ${CMake_BINARY_DIR}/bin/cmake -DDIR=dev -P ${CMake_SOURCE_DIR}/Utilities/Release
     )
   list(APPEND TEST_BUILD_DIRS "${CMake_BINARY_DIR}/Tests/CustomCommand")
 
+  ADD_TEST_MACRO(CustomCommandByproducts CustomCommandByproducts)
+
   ADD_TEST_MACRO(EmptyDepends ${CMAKE_CTEST_COMMAND})
 
   add_test(CustomCommandWorkingDirectory  ${CMAKE_CTEST_COMMAND}
diff --git a/Tests/CustomCommand/CMakeLists.txt b/Tests/CustomCommand/CMakeLists.txt
index e260070..915da0a 100644
--- a/Tests/CustomCommand/CMakeLists.txt
+++ b/Tests/CustomCommand/CMakeLists.txt
@@ -117,6 +117,7 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E echo " Copying doc1pre.txt to doc2post.txt."
   COMMAND ${CMAKE_COMMAND} -E copy ${PROJECT_BINARY_DIR}/doc1pre.txt
                                    ${PROJECT_BINARY_DIR}/doc2post.txt
+  BYPRODUCTS ${PROJECT_BINARY_DIR}/doc2post.txt
   COMMENT "Running TDocument post-build commands"
   )
 
diff --git a/Tests/CustomCommandByproducts/CMakeLists.txt b/Tests/CustomCommandByproducts/CMakeLists.txt
new file mode 100644
index 0000000..c39a536
--- /dev/null
+++ b/Tests/CustomCommandByproducts/CMakeLists.txt
@@ -0,0 +1,103 @@
+cmake_minimum_required(VERSION 3.1)
+project(CustomCommandByproducts C)
+
+# Generate a byproduct in a rule that runs in the target consuming it.
+add_custom_command(
+  OUTPUT timestamp1.txt
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct1.c.in byproduct1.c
+  BYPRODUCTS byproduct1.c
+  COMMAND ${CMAKE_COMMAND} -E touch timestamp1.txt
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/byproduct1.c.in
+  )
+
+# Generate a byproduct in a rule that runs in a dependency of the consumer.
+add_custom_command(
+  OUTPUT timestamp2.txt
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct2.c.in byproduct2.c
+  BYPRODUCTS byproduct2.c
+  COMMAND ${CMAKE_COMMAND} -E touch timestamp2.txt
+  DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/byproduct2.c.in
+  )
+add_custom_target(Producer2 DEPENDS timestamp2.txt)
+
+# Generate a byproduct in a custom target.
+add_custom_target(Producer3_4
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct3.c.in byproduct3.c
+  BYPRODUCTS byproduct3.c
+  )
+
+# Generate a byproduct in a custom target POST_BUILD command.
+add_custom_command(
+  TARGET Producer3_4 POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct4.c.in byproduct4.c
+  BYPRODUCTS byproduct4.c
+  )
+
+add_executable(ProducerExe ProducerExe.c)
+
+# Generate a byproduct in an executable POST_BUILD command.
+add_custom_command(
+  TARGET ProducerExe POST_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct5.c.in byproduct5.c
+  BYPRODUCTS byproduct5.c
+  )
+
+# Generate a byproduct in an executable PRE_LINK command.
+add_custom_command(
+  TARGET ProducerExe PRE_LINK
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct6.c.in byproduct6.c
+  BYPRODUCTS byproduct6.c
+  )
+
+# Generate a byproduct in an executable PRE_BUILD command.
+add_custom_command(
+  TARGET ProducerExe PRE_BUILD
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct7.c.in byproduct7.c
+  BYPRODUCTS byproduct7.c
+  )
+
+# Generate a byproduct in a custom command that consumes other byproducts.
+add_custom_command(OUTPUT timestamp8.txt
+  COMMAND ${CMAKE_COMMAND} -E copy_if_different
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct8.c.in byproduct8.c
+  COMMAND ${CMAKE_COMMAND} -E touch timestamp8.txt
+  BYPRODUCTS byproduct8.c
+  DEPENDS
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct2.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct3.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct4.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct5.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct6.c
+    ${CMAKE_CURRENT_BINARY_DIR}/byproduct7.c
+    ${CMAKE_CURRENT_SOURCE_DIR}/byproduct8.c.in
+  )
+
+# Add an executable consuming all the byproducts.
+add_executable(CustomCommandByproducts
+  CustomCommandByproducts.c
+  byproduct1.c timestamp1.txt
+  byproduct2.c
+  byproduct3.c
+  byproduct4.c
+  byproduct5.c
+  byproduct6.c
+  byproduct7.c
+  byproduct8.c timestamp8.txt
+  )
+add_dependencies(CustomCommandByproducts Producer2)
+add_dependencies(CustomCommandByproducts Producer3_4)
+add_dependencies(CustomCommandByproducts ProducerExe)
+
+if(CMAKE_GENERATOR STREQUAL "Ninja")
+  add_custom_target(CheckNinja ALL
+    COMMENT "Checking build.ninja"
+    COMMAND ${CMAKE_COMMAND} -P ${CMAKE_CURRENT_SOURCE_DIR}/ninja-check.cmake
+    )
+endif()
diff --git a/Tests/CustomCommandByproducts/CustomCommandByproducts.c b/Tests/CustomCommandByproducts/CustomCommandByproducts.c
new file mode 100644
index 0000000..d9db9e6
--- /dev/null
+++ b/Tests/CustomCommandByproducts/CustomCommandByproducts.c
@@ -0,0 +1,21 @@
+extern int byproduct1(void);
+extern int byproduct2(void);
+extern int byproduct3(void);
+extern int byproduct4(void);
+extern int byproduct5(void);
+extern int byproduct6(void);
+extern int byproduct7(void);
+extern int byproduct8(void);
+int main(void)
+{
+  return (
+    byproduct1() +
+    byproduct2() +
+    byproduct3() +
+    byproduct4() +
+    byproduct5() +
+    byproduct6() +
+    byproduct7() +
+    byproduct8() +
+    0);
+}
diff --git a/Tests/CustomCommandByproducts/ProducerExe.c b/Tests/CustomCommandByproducts/ProducerExe.c
new file mode 100644
index 0000000..78f2de1
--- /dev/null
+++ b/Tests/CustomCommandByproducts/ProducerExe.c
@@ -0,0 +1 @@
+int main(void) { return 0; }
diff --git a/Tests/CustomCommandByproducts/byproduct1.c.in b/Tests/CustomCommandByproducts/byproduct1.c.in
new file mode 100644
index 0000000..5c3cc24
--- /dev/null
+++ b/Tests/CustomCommandByproducts/byproduct1.c.in
@@ -0,0 +1 @@
+int byproduct1(void) { return 0; }
diff --git a/Tests/CustomCommandByproducts/byproduct2.c.in b/Tests/CustomCommandByproducts/byproduct2.c.in
new file mode 100644
index 0000000..eeb69ef
--- /dev/null
+++ b/Tests/CustomCommandByproducts/byproduct2.c.in
@@ -0,0 +1 @@
+int byproduct2(void) { return 0; }
diff --git a/Tests/CustomCommandByproducts/byproduct3.c.in b/Tests/CustomCommandByproducts/byproduct3.c.in
new file mode 100644
index 0000000..7d15310
--- /dev/null
+++ b/Tests/CustomCommandByproducts/byproduct3.c.in
@@ -0,0 +1 @@
+int byproduct3(void) { return 0; }
diff --git a/Tests/CustomCommandByproducts/byproduct4.c.in b/Tests/CustomCommandByproducts/byproduct4.c.in
new file mode 100644
index 0000000..8b243dd
--- /dev/null
+++ b/Tests/CustomCommandByproducts/byproduct4.c.in
@@ -0,0 +1 @@
+int byproduct4(void) { return 0; }
diff --git a/Tests/CustomCommandByproducts/byproduct5.c.in b/Tests/CustomCommandByproducts/byproduct5.c.in
new file mode 100644
index 0000000..47f5990
--- /dev/null
+++ b/Tests/CustomCommandByproducts/byproduct5.c.in
@@ -0,0 +1 @@
+int byproduct5(void) { return 0; }
diff --git a/Tests/CustomCommandByproducts/byproduct6.c.in b/Tests/CustomCommandByproducts/byproduct6.c.in
new file mode 100644
index 0000000..d70c14f
--- /dev/null
+++ b/Tests/CustomCommandByproducts/byproduct6.c.in
@@ -0,0 +1 @@
+int byproduct6(void) { return 0; }
diff --git a/Tests/CustomCommandByproducts/byproduct7.c.in b/Tests/CustomCommandByproducts/byproduct7.c.in
new file mode 100644
index 0000000..0be5006
--- /dev/null
+++ b/Tests/CustomCommandByproducts/byproduct7.c.in
@@ -0,0 +1 @@
+int byproduct7(void) { return 0; }
diff --git a/Tests/CustomCommandByproducts/byproduct8.c.in b/Tests/CustomCommandByproducts/byproduct8.c.in
new file mode 100644
index 0000000..abefd62
--- /dev/null
+++ b/Tests/CustomCommandByproducts/byproduct8.c.in
@@ -0,0 +1 @@
+int byproduct8(void) { return 0; }
diff --git a/Tests/CustomCommandByproducts/ninja-check.cmake b/Tests/CustomCommandByproducts/ninja-check.cmake
new file mode 100644
index 0000000..2fc3cc2
--- /dev/null
+++ b/Tests/CustomCommandByproducts/ninja-check.cmake
@@ -0,0 +1,20 @@
+file(READ build.ninja build_ninja)
+if("${build_ninja}" MATCHES [====[
+# Unknown Build Time Dependencies.
+# Tell Ninja that they may appear as side effects of build rules
+# otherwise ordered by order-only dependencies.
+
+((build [^:]*: phony [^\n]*
+)*)# ========]====])
+  set(phony "${CMAKE_MATCH_1}")
+  if(NOT phony)
+    message(STATUS "build.ninja correctly does not have extra phony rules")
+  else()
+    string(REGEX REPLACE "\n+$" "" phony "${phony}")
+    string(REGEX REPLACE "\n" "\n  " phony "  ${phony}")
+    message(FATAL_ERROR "build.ninja incorrectly has extra phony rules:\n"
+      "${phony}")
+  endif()
+else()
+  message(FATAL_ERROR "build.ninja is incorrectly missing expected block")
+endif()
diff --git a/Tests/RunCMake/add_custom_command/RunCMakeTest.cmake b/Tests/RunCMake/add_custom_command/RunCMakeTest.cmake
index d0f429a..2f5c938 100644
--- a/Tests/RunCMake/add_custom_command/RunCMakeTest.cmake
+++ b/Tests/RunCMake/add_custom_command/RunCMakeTest.cmake
@@ -6,4 +6,5 @@ run_cmake(BadArgument)
 run_cmake(NoArguments)
 run_cmake(NoOutputOrTarget)
 run_cmake(OutputAndTarget)
+run_cmake(SourceByproducts)
 run_cmake(SourceUsesTerminal)
diff --git a/Tests/RunCMake/add_custom_command/SourceByproducts-result.txt b/Tests/RunCMake/add_custom_command/SourceByproducts-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/add_custom_command/SourceByproducts-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/add_custom_command/SourceByproducts-stderr.txt b/Tests/RunCMake/add_custom_command/SourceByproducts-stderr.txt
new file mode 100644
index 0000000..a9cd64c
--- /dev/null
+++ b/Tests/RunCMake/add_custom_command/SourceByproducts-stderr.txt
@@ -0,0 +1,4 @@
+CMake Error at SourceByproducts.cmake:1 \(add_custom_command\):
+  add_custom_command BYPRODUCTS may not be specified with SOURCE signatures
+Call Stack \(most recent call first\):
+  CMakeLists.txt:3 \(include\)
diff --git a/Tests/RunCMake/add_custom_command/SourceByproducts.cmake b/Tests/RunCMake/add_custom_command/SourceByproducts.cmake
new file mode 100644
index 0000000..824f41d
--- /dev/null
+++ b/Tests/RunCMake/add_custom_command/SourceByproducts.cmake
@@ -0,0 +1 @@
+add_custom_command(SOURCE t TARGET t BYPRODUCTS b)
diff --git a/Tests/RunCMake/add_custom_target/ByproductsNoCommand-result.txt b/Tests/RunCMake/add_custom_target/ByproductsNoCommand-result.txt
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/Tests/RunCMake/add_custom_target/ByproductsNoCommand-result.txt
@@ -0,0 +1 @@
+1
diff --git a/Tests/RunCMake/add_custom_target/ByproductsNoCommand-stderr.txt b/Tests/RunCMake/add_custom_target/ByproductsNoCommand-stderr.txt
new file mode 100644
index 0000000..6c80ca6
--- /dev/null
+++ b/Tests/RunCMake/add_custom_target/ByproductsNoCommand-stderr.txt
@@ -0,0 +1,4 @@
+CMake Error at ByproductsNoCommand.cmake:1 \(add_custom_target\):
+  BYPRODUCTS may not be specified without any COMMAND
+Call Stack \(most recent call first\):
+  CMakeLists.txt:3 \(include\)
diff --git a/Tests/RunCMake/add_custom_target/ByproductsNoCommand.cmake b/Tests/RunCMake/add_custom_target/ByproductsNoCommand.cmake
new file mode 100644
index 0000000..6c142a2
--- /dev/null
+++ b/Tests/RunCMake/add_custom_target/ByproductsNoCommand.cmake
@@ -0,0 +1 @@
+add_custom_target(MyTarget BYPRODUCTS a b c d)
diff --git a/Tests/RunCMake/add_custom_target/RunCMakeTest.cmake b/Tests/RunCMake/add_custom_target/RunCMakeTest.cmake
index a612da9..92c4a38 100644
--- a/Tests/RunCMake/add_custom_target/RunCMakeTest.cmake
+++ b/Tests/RunCMake/add_custom_target/RunCMakeTest.cmake
@@ -2,4 +2,5 @@ include(RunCMake)
 
 run_cmake(NoArguments)
 run_cmake(BadTargetName)
+run_cmake(ByproductsNoCommand)
 run_cmake(UsesTerminalNoCommand)