VS: Avoid leaking child process output back to IDE (#14266)

The VS IDE sets the environment variable VS_UNICODE_OUTPUT when
executing build rules in order to tell MS tools to report output through
a back door instead of through stdout/stderr.  Unset this variable so
that CMake can capture or properly redirect all output from processes it
runs even when running inside a VS IDE build environment.

This generalizes the special cases fixed by commit 80d045b0 (When
GetPrerequisites.cmake runs dumpbin while running inside the VS IDE...,
2008-05-01) and commit 44aff73d (ExternalProject: Avoid bleed-through
output when logging, 2011-01-06), so drop special handling of
VS_UNICODE_OUTPUT in those instances.

3 files changed, 10 insertions, 4 deletions

diff --git a/Modules/ExternalProject.cmake b/Modules/ExternalProject.cmake
index 6afdb97..f7e6253 100644
--- a/Modules/ExternalProject.cmake
+++ b/Modules/ExternalProject.cmake
@@ -961,7 +961,7 @@ endif()
         set(sep ";")
       endif()
     endforeach()
-    set(code "set(ENV{VS_UNICODE_OUTPUT} \"\")\n${code}set(command \"${cmd}\")${code_execute_process}")
+    set(code "${code}set(command \"${cmd}\")${code_execute_process}")
     file(WRITE ${stamp_dir}/${name}-${step}-impl.cmake "${code}")
     set(command ${CMAKE_COMMAND} "-Dmake=\${make}" "-Dconfig=\${config}" -P ${stamp_dir}/${name}-${step}-impl.cmake)
   endif()
@@ -971,7 +971,6 @@ endif()
   set(logbase ${stamp_dir}/${name}-${step})
   file(WRITE ${script} "
 ${code_cygpath_make}
-set(ENV{VS_UNICODE_OUTPUT} \"\")
 set(command \"${command}\")
 execute_process(
   COMMAND \${command}
diff --git a/Modules/GetPrerequisites.cmake b/Modules/GetPrerequisites.cmake
index 18f449d..9e89788 100644
--- a/Modules/GetPrerequisites.cmake
+++ b/Modules/GetPrerequisites.cmake
@@ -635,7 +635,6 @@ function(get_prerequisites target prerequisites_var exclude_system recurse exepa
     set(gp_regex_fallback "")
     set(gp_regex_cmp_count 1)
     set(gp_tool_known 1)
-    set(ENV{VS_UNICODE_OUTPUT} "") # Block extra output from inside VS IDE.
   endif()
 
   if("${gp_tool}" STREQUAL "objdump")
diff --git a/Source/cmSystemTools.cxx b/Source/cmSystemTools.cxx
index aa5fc94..c52c266 100644
--- a/Source/cmSystemTools.cxx
+++ b/Source/cmSystemTools.cxx
@@ -1696,6 +1696,7 @@ cmSystemTools::SaveRestoreEnvironment::~SaveRestoreEnvironment()
 
 void cmSystemTools::EnableVSConsoleOutput()
 {
+#ifdef _WIN32
   // Visual Studio 8 2005 (devenv.exe or VCExpress.exe) will not
   // display output to the console unless this environment variable is
   // set.  We need it to capture the output of these build tools.
@@ -1703,8 +1704,15 @@ void cmSystemTools::EnableVSConsoleOutput()
   // either of these executables where NAME is created with
   // CreateNamedPipe.  This would bypass the internal buffering of the
   // output and allow it to be captured on the fly.
-#ifdef _WIN32
   cmSystemTools::PutEnv("vsconsoleoutput=1");
+
+# ifdef CMAKE_BUILD_WITH_CMAKE
+  // VS sets an environment variable to tell MS tools like "cl" to report
+  // output through a backdoor pipe instead of stdout/stderr.  Unset the
+  // environment variable to close this backdoor for any path of process
+  // invocations that passes through CMake so we can capture the output.
+  cmSystemTools::UnsetEnv("VS_UNICODE_OUTPUT");
+# endif
 #endif
 }