Ninja: Use new wincodepage tool to determine encoding

Ninja 1.11 and later uses UTF-8 on Windows when possible, and
includes a tool that reports the code page in use. Use this tool
to determine what encoding to write the Ninja files in.

Fixes: #21866

3 files changed, 66 insertions, 9 deletions

diff --git a/Source/cmGlobalNinjaGenerator.cxx b/Source/cmGlobalNinjaGenerator.cxx
index 7eac169..36be45c 100644
--- a/Source/cmGlobalNinjaGenerator.cxx
+++ b/Source/cmGlobalNinjaGenerator.cxx
@@ -9,6 +9,7 @@
 
 #include <cm/iterator>
 #include <cm/memory>
+#include <cm/string_view>
 #include <cmext/algorithm>
 #include <cmext/memory>
 
@@ -503,14 +504,7 @@ std::unique_ptr<cmLocalGenerator> cmGlobalNinjaGenerator::CreateLocalGenerator(
 
 codecvt::Encoding cmGlobalNinjaGenerator::GetMakefileEncoding() const
 {
-#ifdef _WIN32
-  // Ninja on Windows does not support non-ANSI characters.
-  // https://github.com/ninja-build/ninja/issues/1195
-  return codecvt::ANSI;
-#else
-  // No encoding conversion needed on other platforms.
-  return codecvt::None;
-#endif
+  return this->NinjaExpectedEncoding;
 }
 
 void cmGlobalNinjaGenerator::GetDocumentation(cmDocumentationEntry& entry)
@@ -731,6 +725,61 @@ void cmGlobalNinjaGenerator::CheckNinjaFeatures()
   this->NinjaSupportsMetadataOnRegeneration = !cmSystemTools::VersionCompare(
     cmSystemTools::OP_LESS, this->NinjaVersion.c_str(),
     RequiredNinjaVersionForMetadataOnRegeneration().c_str());
+#ifdef _WIN32
+  this->NinjaSupportsCodePage = !cmSystemTools::VersionCompare(
+    cmSystemTools::OP_LESS, this->NinjaVersion.c_str(),
+    RequiredNinjaVersionForCodePage().c_str());
+  if (this->NinjaSupportsCodePage) {
+    this->CheckNinjaCodePage();
+  } else {
+    this->NinjaExpectedEncoding = codecvt::ANSI;
+  }
+#endif
+}
+
+void cmGlobalNinjaGenerator::CheckNinjaCodePage()
+{
+  std::vector<std::string> command{ this->NinjaCommand, "-t", "wincodepage" };
+  std::string output;
+  std::string error;
+  int result;
+  if (!cmSystemTools::RunSingleCommand(command, &output, &error, &result,
+                                       nullptr, cmSystemTools::OUTPUT_NONE)) {
+    this->GetCMakeInstance()->IssueMessage(MessageType::FATAL_ERROR,
+                                           cmStrCat("Running\n '",
+                                                    cmJoin(command, "' '"),
+                                                    "'\n"
+                                                    "failed with:\n ",
+                                                    error));
+    cmSystemTools::SetFatalErrorOccured();
+  } else if (result == 0) {
+    std::istringstream outputStream(output);
+    std::string line;
+    bool found = false;
+    while (cmSystemTools::GetLineFromStream(outputStream, line)) {
+      if (cmHasLiteralPrefix(line, "Build file encoding: ")) {
+        cm::string_view lineView(line);
+        cm::string_view encoding =
+          lineView.substr(cmStrLen("Build file encoding: "));
+        if (encoding == "UTF-8") {
+          // Ninja expects UTF-8. We use that internally. No conversion needed.
+          this->NinjaExpectedEncoding = codecvt::None;
+        } else {
+          this->NinjaExpectedEncoding = codecvt::ANSI;
+        }
+        found = true;
+        break;
+      }
+    }
+    if (!found) {
+      this->GetCMakeInstance()->IssueMessage(
+        MessageType::WARNING,
+        "Could not determine Ninja's code page, defaulting to UTF-8");
+      this->NinjaExpectedEncoding = codecvt::None;
+    }
+  } else {
+    this->NinjaExpectedEncoding = codecvt::ANSI;
+  }
 }
 
 bool cmGlobalNinjaGenerator::CheckLanguages(
diff --git a/Source/cmGlobalNinjaGenerator.h b/Source/cmGlobalNinjaGenerator.h
index 0c919ef..9f31708 100644
--- a/Source/cmGlobalNinjaGenerator.h
+++ b/Source/cmGlobalNinjaGenerator.h
@@ -388,6 +388,7 @@ public:
   {
     return "1.10.2";
   }
+  static std::string RequiredNinjaVersionForCodePage() { return "1.11"; }
   bool SupportsConsolePool() const;
   bool SupportsImplicitOuts() const;
   bool SupportsManifestRestat() const;
@@ -474,6 +475,7 @@ private:
   std::string GetEditCacheCommand() const override;
   bool FindMakeProgram(cmMakefile* mf) override;
   void CheckNinjaFeatures();
+  void CheckNinjaCodePage();
   bool CheckLanguages(std::vector<std::string> const& languages,
                       cmMakefile* mf) const override;
   bool CheckFortran(cmMakefile* mf) const;
@@ -568,6 +570,9 @@ private:
   bool NinjaSupportsUnconditionalRecompactTool = false;
   bool NinjaSupportsMultipleOutputs = false;
   bool NinjaSupportsMetadataOnRegeneration = false;
+  bool NinjaSupportsCodePage = false;
+
+  codecvt::Encoding NinjaExpectedEncoding = codecvt::None;
 
   bool DiagnosedCxxModuleSupport = false;
 
diff --git a/Source/cmLocalNinjaGenerator.cxx b/Source/cmLocalNinjaGenerator.cxx
index 8ed411a..050c907 100644
--- a/Source/cmLocalNinjaGenerator.cxx
+++ b/Source/cmLocalNinjaGenerator.cxx
@@ -97,9 +97,12 @@ void cmLocalNinjaGenerator::Generate()
       // contains any non-ASCII characters and dependency checking will fail.
       // As a workaround, leave the msvc_deps_prefix UTF-8 encoded even though
       // the rest of the file is ANSI encoded.
-      if (GetConsoleOutputCP() == CP_UTF8 && GetACP() != CP_UTF8) {
+      if (GetConsoleOutputCP() == CP_UTF8 && GetACP() != CP_UTF8 &&
+          this->GetGlobalGenerator()->GetMakefileEncoding() != codecvt::None) {
         this->GetRulesFileStream().WriteRaw(showIncludesPrefix);
       } else {
+        // Ninja 1.11 and above uses the UTF-8 code page if it's supported, so
+        // in that case we can write it normally without using raw bytes.
         this->GetRulesFileStream() << showIncludesPrefix;
       }
 #else