Ensure stdin, stdout, and stderr pipes are always open

On non-Windows platforms libuv assumes that file descriptors 0-2 are
always used for standard pipes and never for anything else.  Otherwise,
libuv may re-use one of these descriptors and then fail an assertion
when closing it.  Similarly, On Windows platforms our ConsoleBuf
implementation assumes that the standard handles are always open.

If CMake is run with any standard pipes closed, open them with
`/dev/null` or `NUL` to satisfy these assumptions.

Fixes: #19219

7 files changed, 73 insertions, 2 deletions

diff --git a/Source/CPack/cpack.cxx b/Source/CPack/cpack.cxx
index 3ceb824..58b9e70 100644
--- a/Source/CPack/cpack.cxx
+++ b/Source/CPack/cpack.cxx
@@ -98,6 +98,7 @@ static void cpackProgressCallback(const std::string& message, float /*unused*/)
 // this is CPack.
 int main(int argc, char const* const* argv)
 {
+  cmSystemTools::EnsureStdPipes();
 #if defined(_WIN32) && defined(CMAKE_BUILD_WITH_CMAKE)
   // Replace streambuf so we can output Unicode to console
   cmsys::ConsoleBuf::Manager consoleOut(std::cout);
diff --git a/Source/CursesDialog/ccmake.cxx b/Source/CursesDialog/ccmake.cxx
index 745c6bb..7caed0c 100644
--- a/Source/CursesDialog/ccmake.cxx
+++ b/Source/CursesDialog/ccmake.cxx
@@ -67,6 +67,7 @@ void onsig(int /*unused*/)
 
 int main(int argc, char const* const* argv)
 {
+  cmSystemTools::EnsureStdPipes();
   cmsys::Encoding::CommandLineArguments encoding_args =
     cmsys::Encoding::CommandLineArguments::Main(argc, argv);
   argc = encoding_args.argc();
diff --git a/Source/QtDialog/CMakeSetup.cxx b/Source/QtDialog/CMakeSetup.cxx
index 8d9a50c..c9ebba8 100644
--- a/Source/QtDialog/CMakeSetup.cxx
+++ b/Source/QtDialog/CMakeSetup.cxx
@@ -55,6 +55,7 @@ Q_IMPORT_PLUGIN(QWindowsVistaStylePlugin);
 
 int main(int argc, char** argv)
 {
+  cmSystemTools::EnsureStdPipes();
   cmsys::Encoding::CommandLineArguments encoding_args =
     cmsys::Encoding::CommandLineArguments::Main(argc, argv);
   int argc2 = encoding_args.argc();
diff --git a/Source/cmSystemTools.cxx b/Source/cmSystemTools.cxx
index bc853b7..17ed3f6 100644
--- a/Source/cmSystemTools.cxx
+++ b/Source/cmSystemTools.cxx
@@ -43,6 +43,7 @@
 #include <assert.h>
 #include <ctype.h>
 #include <errno.h>
+#include <fcntl.h>
 #include <iostream>
 #include <sstream>
 #include <stdio.h>
@@ -56,8 +57,6 @@
 #  include <windows.h>
 // include wincrypt.h after windows.h
 #  include <wincrypt.h>
-
-#  include <fcntl.h> /* _O_TEXT */
 #else
 #  include <sys/time.h>
 #  include <unistd.h>
@@ -2007,6 +2006,71 @@ int cmSystemTools::WaitForLine(cmsysProcess* process, std::string& line,
   }
 }
 
+#ifdef _WIN32
+static void EnsureStdPipe(DWORD fd)
+{
+  if (GetStdHandle(fd) != INVALID_HANDLE_VALUE) {
+    return;
+  }
+  SECURITY_ATTRIBUTES sa;
+  sa.nLength = sizeof(sa);
+  sa.lpSecurityDescriptor = NULL;
+  sa.bInheritHandle = TRUE;
+
+  HANDLE h = CreateFileW(
+    L"NUL",
+    fd == STD_INPUT_HANDLE ? FILE_GENERIC_READ
+                           : FILE_GENERIC_WRITE | FILE_READ_ATTRIBUTES,
+    FILE_SHARE_READ | FILE_SHARE_WRITE, &sa, OPEN_EXISTING, 0, NULL);
+
+  if (h == INVALID_HANDLE_VALUE) {
+    LPSTR message = NULL;
+    DWORD size = FormatMessageA(
+      FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM |
+        FORMAT_MESSAGE_IGNORE_INSERTS,
+      NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+      (LPSTR)&message, 0, NULL);
+    std::string msg = std::string(message, size);
+    LocalFree(message);
+    std::cerr << "failed to open NUL for missing stdio pipe: " << msg;
+    abort();
+  }
+
+  SetStdHandle(fd, h);
+}
+
+void cmSystemTools::EnsureStdPipes()
+{
+  EnsureStdPipe(STD_INPUT_HANDLE);
+  EnsureStdPipe(STD_OUTPUT_HANDLE);
+  EnsureStdPipe(STD_ERROR_HANDLE);
+}
+#else
+static void EnsureStdPipe(int fd)
+{
+  if (fcntl(fd, F_GETFD) != -1 || errno != EBADF) {
+    return;
+  }
+
+  int f = open("/dev/null", fd == STDIN_FILENO ? O_RDONLY : O_WRONLY);
+  if (f == -1) {
+    perror("failed to open /dev/null for missing stdio pipe");
+    abort();
+  }
+  if (f != fd) {
+    dup2(f, fd);
+    close(f);
+  }
+}
+
+void cmSystemTools::EnsureStdPipes()
+{
+  EnsureStdPipe(STDIN_FILENO);
+  EnsureStdPipe(STDOUT_FILENO);
+  EnsureStdPipe(STDERR_FILENO);
+}
+#endif
+
 void cmSystemTools::DoNotInheritStdPipes()
 {
 #ifdef _WIN32
diff --git a/Source/cmSystemTools.h b/Source/cmSystemTools.h
index 05bd351..8a87a37 100644
--- a/Source/cmSystemTools.h
+++ b/Source/cmSystemTools.h
@@ -435,6 +435,8 @@ public:
   // not get stuck waiting for all the output on the pipes.
   static void DoNotInheritStdPipes();
 
+  static void EnsureStdPipes();
+
   /** Copy the file create/access/modify times from the file named by
       the first argument to that named by the second.  */
   static bool CopyFileTime(const std::string& fromFile,
diff --git a/Source/cmakemain.cxx b/Source/cmakemain.cxx
index 5631d10..b137327 100644
--- a/Source/cmakemain.cxx
+++ b/Source/cmakemain.cxx
@@ -183,6 +183,7 @@ static void cmakemainProgressCallback(const std::string& m, float prog,
 
 int main(int ac, char const* const* av)
 {
+  cmSystemTools::EnsureStdPipes();
 #if defined(_WIN32) && defined(CMAKE_BUILD_WITH_CMAKE)
   // Replace streambuf so we can output Unicode to console
   cmsys::ConsoleBuf::Manager consoleOut(std::cout);
diff --git a/Source/ctest.cxx b/Source/ctest.cxx
index 461021b..3b3630f 100644
--- a/Source/ctest.cxx
+++ b/Source/ctest.cxx
@@ -143,6 +143,7 @@ static const char* cmDocumentationOptions[][2] = {
 // this is a test driver program for cmCTest.
 int main(int argc, char const* const* argv)
 {
+  cmSystemTools::EnsureStdPipes();
 #if defined(_WIN32) && defined(CMAKE_BUILD_WITH_CMAKE)
   // Replace streambuf so we can output Unicode to console
   cmsys::ConsoleBuf::Manager consoleOut(std::cout);