AIX: Enable XCOFF editing to replace RPATH on installation

Avoid relinking before installation.

2 files changed, 128 insertions, 12 deletions

diff --git a/Source/cmGeneratorTarget.cxx b/Source/cmGeneratorTarget.cxx
index 0171d8a..9235faa 100644
--- a/Source/cmGeneratorTarget.cxx
+++ b/Source/cmGeneratorTarget.cxx
@@ -2009,10 +2009,10 @@ bool cmGeneratorTarget::NeedRelinkBeforeInstall(
     std::ostringstream w;
     /* clang-format off */
     w <<
-      "The install of the " << this->GetName() << " target requires "
-      "changing an RPATH from the build tree, but this is not supported "
-      "with the Ninja generator unless on an ELF-based platform.  The "
-      "CMAKE_BUILD_WITH_INSTALL_RPATH variable may be set to avoid this "
+      "The install of the " << this->GetName() << " target requires changing "
+      "an RPATH from the build tree, but this is not supported with the Ninja "
+      "generator unless on an ELF-based or XCOFF-based platform.  "
+      "The CMAKE_BUILD_WITH_INSTALL_RPATH variable may be set to avoid this "
       "relinking step."
       ;
     /* clang-format on */
@@ -2058,20 +2058,29 @@ bool cmGeneratorTarget::IsChrpathUsed(const std::string& config) const
     return true;
   }
 
-#if defined(CMake_USE_ELF_PARSER)
-  // Enable if the rpath flag uses a separator and the target uses ELF
-  // binaries.
+#if defined(CMake_USE_ELF_PARSER) || defined(CMake_USE_XCOFF_PARSER)
+  // Enable if the rpath flag uses a separator and the target uses
+  // binaries we know how to edit.
   std::string ll = this->GetLinkerLanguage(config);
   if (!ll.empty()) {
     std::string sepVar =
       cmStrCat("CMAKE_SHARED_LIBRARY_RUNTIME_", ll, "_FLAG_SEP");
     cmProp sep = this->Makefile->GetDefinition(sepVar);
     if (cmNonempty(sep)) {
-      // TODO: Add ELF check to ABI detection and get rid of
+      // TODO: Add binary format check to ABI detection and get rid of
       // CMAKE_EXECUTABLE_FORMAT.
       if (cmProp fmt =
             this->Makefile->GetDefinition("CMAKE_EXECUTABLE_FORMAT")) {
-        return (*fmt == "ELF");
+#  if defined(CMake_USE_ELF_PARSER)
+        if (*fmt == "ELF") {
+          return true;
+        }
+#  endif
+#  if defined(CMake_USE_XCOFF_PARSER)
+        if (*fmt == "XCOFF") {
+          return true;
+        }
+#  endif
       }
     }
   }
diff --git a/Source/cmSystemTools.cxx b/Source/cmSystemTools.cxx
index 8c5a657..0807590 100644
--- a/Source/cmSystemTools.cxx
+++ b/Source/cmSystemTools.cxx
@@ -53,6 +53,10 @@
 #  include "cmMachO.h"
 #endif
 
+#if defined(CMake_USE_XCOFF_PARSER)
+#  include "cmXCOFF.h"
+#endif
+
 #include <algorithm>
 #include <cassert>
 #include <cctype>
@@ -2360,9 +2364,9 @@ bool cmSystemTools::GuessLibraryInstallName(std::string const& fullPath,
   return false;
 }
 
-#if defined(CMake_USE_ELF_PARSER)
-std::string::size_type cmSystemToolsFindRPath(std::string const& have,
-                                              std::string const& want)
+#if defined(CMake_USE_ELF_PARSER) || defined(CMake_USE_XCOFF_PARSER)
+std::string::size_type cmSystemToolsFindRPath(cm::string_view const& have,
+                                              cm::string_view const& want)
 {
   std::string::size_type pos = 0;
   while (pos < have.size()) {
@@ -2404,6 +2408,7 @@ struct cmSystemToolsRPathInfo
 };
 #endif
 
+// FIXME: Dispatch if multiple formats are supported.
 #if defined(CMake_USE_ELF_PARSER)
 bool cmSystemTools::ChangeRPath(std::string const& file,
                                 std::string const& oldRPath,
@@ -2576,6 +2581,75 @@ bool cmSystemTools::ChangeRPath(std::string const& file,
   }
   return true;
 }
+#elif defined(CMake_USE_XCOFF_PARSER)
+bool cmSystemTools::ChangeRPath(std::string const& file,
+                                std::string const& oldRPath,
+                                std::string const& newRPath,
+                                bool removeEnvironmentRPath, std::string* emsg,
+                                bool* changed)
+{
+  if (changed) {
+    *changed = false;
+  }
+
+  bool chg = false;
+  cmXCOFF xcoff(file.c_str(), cmXCOFF::Mode::ReadWrite);
+  if (cm::optional<cm::string_view> maybeLibPath = xcoff.GetLibPath()) {
+    cm::string_view libPath = *maybeLibPath;
+    // Make sure the current rpath contains the old rpath.
+    std::string::size_type pos = cmSystemToolsFindRPath(libPath, oldRPath);
+    if (pos == std::string::npos) {
+      // If it contains the new rpath instead then it is okay.
+      if (cmSystemToolsFindRPath(libPath, newRPath) != std::string::npos) {
+        return true;
+      }
+      if (emsg) {
+        std::ostringstream e;
+        /* clang-format off */
+        e << "The current RPATH is:\n"
+          << "  " << libPath << "\n"
+          << "which does not contain:\n"
+          << "  " << oldRPath << "\n"
+          << "as was expected.";
+        /* clang-format on */
+        *emsg = e.str();
+      }
+      return false;
+    }
+
+    // The prefix is either empty or ends in a ':'.
+    cm::string_view prefix = libPath.substr(0, pos);
+    if (newRPath.empty() && !prefix.empty()) {
+      prefix.remove_suffix(1);
+    }
+
+    // The suffix is either empty or starts in a ':'.
+    cm::string_view suffix = libPath.substr(pos + oldRPath.length());
+
+    // Construct the new value which preserves the part of the path
+    // not being changed.
+    std::string newLibPath;
+    if (!removeEnvironmentRPath) {
+      newLibPath = std::string(prefix);
+    }
+    newLibPath += newRPath;
+    newLibPath += suffix;
+
+    chg = xcoff.SetLibPath(newLibPath);
+  }
+  if (!xcoff) {
+    if (emsg) {
+      *emsg = xcoff.GetErrorMessage();
+    }
+    return false;
+  }
+
+  // Everything was updated successfully.
+  if (changed) {
+    *changed = chg;
+  }
+  return true;
+}
 #else
 bool cmSystemTools::ChangeRPath(std::string const& /*file*/,
                                 std::string const& /*oldRPath*/,
@@ -2720,6 +2794,7 @@ int cmSystemTools::strverscmp(std::string const& lhs, std::string const& rhs)
   return cm_strverscmp(lhs.c_str(), rhs.c_str());
 }
 
+// FIXME: Dispatch if multiple formats are supported.
 #if defined(CMake_USE_ELF_PARSER)
 bool cmSystemTools::RemoveRPath(std::string const& file, std::string* emsg,
                                 bool* removed)
@@ -2861,6 +2936,28 @@ bool cmSystemTools::RemoveRPath(std::string const& file, std::string* emsg,
   }
   return true;
 }
+#elif defined(CMake_USE_XCOFF_PARSER)
+bool cmSystemTools::RemoveRPath(std::string const& file, std::string* emsg,
+                                bool* removed)
+{
+  if (removed) {
+    *removed = false;
+  }
+
+  cmXCOFF xcoff(file.c_str(), cmXCOFF::Mode::ReadWrite);
+  bool rm = xcoff.RemoveLibPath();
+  if (!xcoff) {
+    if (emsg) {
+      *emsg = xcoff.GetErrorMessage();
+    }
+    return false;
+  }
+
+  if (removed) {
+    *removed = rm;
+  }
+  return true;
+}
 #else
 bool cmSystemTools::RemoveRPath(std::string const& /*file*/,
                                 std::string* /*emsg*/, bool* /*removed*/)
@@ -2869,6 +2966,7 @@ bool cmSystemTools::RemoveRPath(std::string const& /*file*/,
 }
 #endif
 
+// FIXME: Dispatch if multiple formats are supported.
 bool cmSystemTools::CheckRPath(std::string const& file,
                                std::string const& newRPath)
 {
@@ -2894,6 +2992,15 @@ bool cmSystemTools::CheckRPath(std::string const& file,
     }
   }
   return false;
+#elif defined(CMake_USE_XCOFF_PARSER)
+  // Parse the XCOFF binary.
+  cmXCOFF xcoff(file.c_str());
+  if (cm::optional<cm::string_view> libPath = xcoff.GetLibPath()) {
+    if (cmSystemToolsFindRPath(*libPath, newRPath) != std::string::npos) {
+      return true;
+    }
+  }
+  return false;
 #else
   (void)file;
   (void)newRPath;