find_package: Enforce maximum nesting depth below maximum recursion depth

The stack usage for nested `find_package` calls is much larger than for
other kinds of recursion, so enforce a lower limit to avoid stack
overflow.

7 files changed, 39 insertions, 18 deletions

diff --git a/Source/cmFindPackageCommand.cxx b/Source/cmFindPackageCommand.cxx
index 8acfe83..b1e65af 100644
--- a/Source/cmFindPackageCommand.cxx
+++ b/Source/cmFindPackageCommand.cxx
@@ -976,6 +976,21 @@ bool cmFindPackageCommand::InitialPass(std::vector<std::string> const& args)
     }
   }
 
+  // Limit package nesting depth well below the recursion depth limit because
+  // find_package nesting uses more stack space than normal recursion.
+  {
+    static std::size_t const findPackageDepthMinMax = 100;
+    std::size_t const findPackageDepthMax = std::max(
+      this->Makefile->GetRecursionDepthLimit() / 2, findPackageDepthMinMax);
+    std::size_t const findPackageDepth =
+      this->Makefile->FindPackageRootPathStack.size() + 1;
+    if (findPackageDepth > findPackageDepthMax) {
+      this->SetError(cmStrCat("maximum nesting depth of ", findPackageDepthMax,
+                              " exceeded."));
+      return false;
+    }
+  }
+
   this->PushFindPackageRootPathStack();
 
   this->SetModuleVariables(components, componentVarDefs);
diff --git a/Tests/RunCMake/MaxRecursionDepth/find_package-default-script-stderr.txt b/Tests/RunCMake/MaxRecursionDepth/find_package-default-script-stderr.txt
index b8557ab..4e965e8 100644
--- a/Tests/RunCMake/MaxRecursionDepth/find_package-default-script-stderr.txt
+++ b/Tests/RunCMake/MaxRecursionDepth/find_package-default-script-stderr.txt
@@ -1,5 +1,7 @@
 [0-9]+
-CMake Error at .*/FindRecursivePackage\.cmake:1 \(message\):
-  Maximum recursion depth of [0-9]+ exceeded
+CMake Error at [^
+]*/Tests/RunCMake/MaxRecursionDepth/FindRecursivePackage.cmake:[0-9]+ \(find_package\):
+  find_package maximum nesting depth of [0-9]+ exceeded.
 Call Stack \(most recent call first\):
-  .*/FindRecursivePackage\.cmake:3 \(find_package\)
+  [^
+]*/Tests/RunCMake/MaxRecursionDepth/FindRecursivePackage.cmake:[0-9]+ \(find_package\)
diff --git a/Tests/RunCMake/MaxRecursionDepth/find_package-default-stderr.txt b/Tests/RunCMake/MaxRecursionDepth/find_package-default-stderr.txt
index 5d31e29..0119953 100644
--- a/Tests/RunCMake/MaxRecursionDepth/find_package-default-stderr.txt
+++ b/Tests/RunCMake/MaxRecursionDepth/find_package-default-stderr.txt
@@ -1,5 +1,5 @@
 [0-9]+
-CMake Error at FindRecursivePackage\.cmake:1 \(message\):
-  Maximum recursion depth of [0-9]+ exceeded
+CMake Error at FindRecursivePackage.cmake:[0-9]+ \(find_package\):
+  find_package maximum nesting depth of [0-9]+ exceeded.
 Call Stack \(most recent call first\):
-  FindRecursivePackage\.cmake:3 \(find_package\)
+  FindRecursivePackage.cmake:[0-9]+ \(find_package\)
diff --git a/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-env-script-stderr.txt b/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-env-script-stderr.txt
index b8557ab..4e965e8 100644
--- a/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-env-script-stderr.txt
+++ b/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-env-script-stderr.txt
@@ -1,5 +1,7 @@
 [0-9]+
-CMake Error at .*/FindRecursivePackage\.cmake:1 \(message\):
-  Maximum recursion depth of [0-9]+ exceeded
+CMake Error at [^
+]*/Tests/RunCMake/MaxRecursionDepth/FindRecursivePackage.cmake:[0-9]+ \(find_package\):
+  find_package maximum nesting depth of [0-9]+ exceeded.
 Call Stack \(most recent call first\):
-  .*/FindRecursivePackage\.cmake:3 \(find_package\)
+  [^
+]*/Tests/RunCMake/MaxRecursionDepth/FindRecursivePackage.cmake:[0-9]+ \(find_package\)
diff --git a/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-env-stderr.txt b/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-env-stderr.txt
index 5d31e29..0119953 100644
--- a/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-env-stderr.txt
+++ b/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-env-stderr.txt
@@ -1,5 +1,5 @@
 [0-9]+
-CMake Error at FindRecursivePackage\.cmake:1 \(message\):
-  Maximum recursion depth of [0-9]+ exceeded
+CMake Error at FindRecursivePackage.cmake:[0-9]+ \(find_package\):
+  find_package maximum nesting depth of [0-9]+ exceeded.
 Call Stack \(most recent call first\):
-  FindRecursivePackage\.cmake:3 \(find_package\)
+  FindRecursivePackage.cmake:[0-9]+ \(find_package\)
diff --git a/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-var-script-stderr.txt b/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-var-script-stderr.txt
index b8557ab..4e965e8 100644
--- a/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-var-script-stderr.txt
+++ b/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-var-script-stderr.txt
@@ -1,5 +1,7 @@
 [0-9]+
-CMake Error at .*/FindRecursivePackage\.cmake:1 \(message\):
-  Maximum recursion depth of [0-9]+ exceeded
+CMake Error at [^
+]*/Tests/RunCMake/MaxRecursionDepth/FindRecursivePackage.cmake:[0-9]+ \(find_package\):
+  find_package maximum nesting depth of [0-9]+ exceeded.
 Call Stack \(most recent call first\):
-  .*/FindRecursivePackage\.cmake:3 \(find_package\)
+  [^
+]*/Tests/RunCMake/MaxRecursionDepth/FindRecursivePackage.cmake:[0-9]+ \(find_package\)
diff --git a/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-var-stderr.txt b/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-var-stderr.txt
index 5d31e29..0119953 100644
--- a/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-var-stderr.txt
+++ b/Tests/RunCMake/MaxRecursionDepth/find_package-invalid-var-stderr.txt
@@ -1,5 +1,5 @@
 [0-9]+
-CMake Error at FindRecursivePackage\.cmake:1 \(message\):
-  Maximum recursion depth of [0-9]+ exceeded
+CMake Error at FindRecursivePackage.cmake:[0-9]+ \(find_package\):
+  find_package maximum nesting depth of [0-9]+ exceeded.
 Call Stack \(most recent call first\):
-  FindRecursivePackage\.cmake:3 \(find_package\)
+  FindRecursivePackage.cmake:[0-9]+ \(find_package\)