find_package: Enforce maximum nesting depth below maximum recursion depth

The stack usage for nested `find_package` calls is much larger than for
other kinds of recursion, so enforce a lower limit to avoid stack
overflow.

1 files changed, 15 insertions, 0 deletions

diff --git a/Source/cmFindPackageCommand.cxx b/Source/cmFindPackageCommand.cxx
index 8acfe83..b1e65af 100644
--- a/Source/cmFindPackageCommand.cxx
+++ b/Source/cmFindPackageCommand.cxx
@@ -976,6 +976,21 @@ bool cmFindPackageCommand::InitialPass(std::vector<std::string> const& args)
     }
   }
 
+  // Limit package nesting depth well below the recursion depth limit because
+  // find_package nesting uses more stack space than normal recursion.
+  {
+    static std::size_t const findPackageDepthMinMax = 100;
+    std::size_t const findPackageDepthMax = std::max(
+      this->Makefile->GetRecursionDepthLimit() / 2, findPackageDepthMinMax);
+    std::size_t const findPackageDepth =
+      this->Makefile->FindPackageRootPathStack.size() + 1;
+    if (findPackageDepth > findPackageDepthMax) {
+      this->SetError(cmStrCat("maximum nesting depth of ", findPackageDepthMax,
+                              " exceeded."));
+      return false;
+    }
+  }
+
   this->PushFindPackageRootPathStack();
 
   this->SetModuleVariables(components, componentVarDefs);