Merge topic 'cmake-presets-invalid-macro' into release-3.19

638557cbfe CMakePresets.json: Properly report macro expansion errors

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !5397

17 files changed, 268 insertions, 183 deletions

diff --git a/Source/QtDialog/QCMake.cxx b/Source/QtDialog/QCMake.cxx
index 9017a63..3789e93 100644
--- a/Source/QtDialog/QCMake.cxx
+++ b/Source/QtDialog/QCMake.cxx
@@ -157,8 +157,8 @@ void QCMake::setPreset(const QString& name, bool setBinary)
 
     if (!name.isNull()) {
       std::string presetName(name.toLocal8Bit());
-      auto const& preset = this->CMakePresetsFile.Presets[presetName];
-      auto expandedPreset = this->CMakePresetsFile.ExpandMacros(preset);
+      auto const& expandedPreset =
+        this->CMakePresetsFile.Presets[presetName].Expanded;
       if (expandedPreset) {
         if (setBinary) {
           QString binaryDir =
@@ -420,8 +420,7 @@ QCMakePropertyList QCMake::properties() const
 
   if (!this->PresetName.isNull()) {
     std::string presetName(this->PresetName.toLocal8Bit());
-    auto p = this->CMakePresetsFile.ExpandMacros(
-      this->CMakePresetsFile.Presets.at(presetName));
+    auto const& p = this->CMakePresetsFile.Presets.at(presetName).Expanded;
     if (p) {
       for (auto const& v : p->CacheVariables) {
         if (!v.second) {
@@ -537,7 +536,8 @@ void QCMake::loadPresets()
 
   QVector<QCMakePreset> presets;
   for (auto const& name : this->CMakePresetsFile.PresetOrder) {
-    auto const& p = this->CMakePresetsFile.Presets[name];
+    auto const& it = this->CMakePresetsFile.Presets[name];
+    auto const& p = it.Unexpanded;
     if (p.Hidden) {
       continue;
     }
@@ -554,12 +554,12 @@ void QCMake::loadPresets()
     preset.toolset = std::move(QString::fromLocal8Bit(p.Toolset.data()));
     preset.setGenConfig = !p.GeneratorConfig ||
       p.GeneratorConfig == cmCMakePresetsFile::CMakeGeneratorConfig::Default;
-    preset.enabled = std::find_if(this->AvailableGenerators.begin(),
-                                  this->AvailableGenerators.end(),
-                                  [&p](const cmake::GeneratorInfo& g) {
-                                    return g.name == p.Generator;
-                                  }) != this->AvailableGenerators.end() &&
-      this->CMakePresetsFile.ExpandMacros(p);
+    preset.enabled = it.Expanded &&
+      std::find_if(this->AvailableGenerators.begin(),
+                   this->AvailableGenerators.end(),
+                   [&p](const cmake::GeneratorInfo& g) {
+                     return g.name == p.Generator;
+                   }) != this->AvailableGenerators.end();
     presets.push_back(preset);
   }
   emit this->presetsChanged(presets);
diff --git a/Source/cmCMakePresetsFile.cxx b/Source/cmCMakePresetsFile.cxx
index 456094f..b3bb6df 100644
--- a/Source/cmCMakePresetsFile.cxx
+++ b/Source/cmCMakePresetsFile.cxx
@@ -29,6 +29,7 @@ enum class CycleStatus
 using ReadFileResult = cmCMakePresetsFile::ReadFileResult;
 using CacheVariable = cmCMakePresetsFile::CacheVariable;
 using UnexpandedPreset = cmCMakePresetsFile::UnexpandedPreset;
+using ExpandedPreset = cmCMakePresetsFile::ExpandedPreset;
 using CMakeGeneratorConfig = cmCMakePresetsFile::CMakeGeneratorConfig;
 
 constexpr int MIN_VERSION = 1;
@@ -312,9 +313,9 @@ void InheritOptionalBool(cm::optional<bool>& child,
  * that each preset has the required fields, either directly or through
  * inheritance.
  */
-ReadFileResult VisitPreset(std::map<std::string, UnexpandedPreset>& presets,
-                           UnexpandedPreset& preset,
-                           std::map<std::string, CycleStatus> cycleStatus)
+ReadFileResult VisitPreset(
+  std::map<std::string, cmCMakePresetsFile::PresetPair>& presets,
+  UnexpandedPreset& preset, std::map<std::string, CycleStatus> cycleStatus)
 {
   switch (cycleStatus[preset.Name]) {
     case CycleStatus::InProgress:
@@ -340,35 +341,38 @@ ReadFileResult VisitPreset(std::map<std::string, UnexpandedPreset>& presets,
       return ReadFileResult::INVALID_PRESET;
     }
 
-    if (!preset.User && parent->second.User) {
+    if (!preset.User && parent->second.Unexpanded.User) {
       return ReadFileResult::USER_PRESET_INHERITANCE;
     }
 
-    auto result = VisitPreset(presets, parent->second, cycleStatus);
+    auto result = VisitPreset(presets, parent->second.Unexpanded, cycleStatus);
     if (result != ReadFileResult::READ_OK) {
       return result;
     }
 
-    InheritString(preset.Generator, parent->second.Generator);
-    InheritString(preset.Architecture, parent->second.Architecture);
-    InheritString(preset.Toolset, parent->second.Toolset);
+    InheritString(preset.Generator, parent->second.Unexpanded.Generator);
+    InheritString(preset.Architecture, parent->second.Unexpanded.Architecture);
+    InheritString(preset.Toolset, parent->second.Unexpanded.Toolset);
     if (!preset.GeneratorConfig) {
-      preset.GeneratorConfig = parent->second.GeneratorConfig;
+      preset.GeneratorConfig = parent->second.Unexpanded.GeneratorConfig;
     }
-    InheritString(preset.BinaryDir, parent->second.BinaryDir);
-    InheritOptionalBool(preset.WarnDev, parent->second.WarnDev);
-    InheritOptionalBool(preset.ErrorDev, parent->second.ErrorDev);
-    InheritOptionalBool(preset.WarnDeprecated, parent->second.WarnDeprecated);
+    InheritString(preset.BinaryDir, parent->second.Unexpanded.BinaryDir);
+    InheritOptionalBool(preset.WarnDev, parent->second.Unexpanded.WarnDev);
+    InheritOptionalBool(preset.ErrorDev, parent->second.Unexpanded.ErrorDev);
+    InheritOptionalBool(preset.WarnDeprecated,
+                        parent->second.Unexpanded.WarnDeprecated);
     InheritOptionalBool(preset.ErrorDeprecated,
-                        parent->second.ErrorDeprecated);
+                        parent->second.Unexpanded.ErrorDeprecated);
     InheritOptionalBool(preset.WarnUninitialized,
-                        parent->second.WarnUninitialized);
-    InheritOptionalBool(preset.WarnUnusedCli, parent->second.WarnUnusedCli);
-    InheritOptionalBool(preset.WarnSystemVars, parent->second.WarnSystemVars);
-    for (auto const& v : parent->second.CacheVariables) {
+                        parent->second.Unexpanded.WarnUninitialized);
+    InheritOptionalBool(preset.WarnUnusedCli,
+                        parent->second.Unexpanded.WarnUnusedCli);
+    InheritOptionalBool(preset.WarnSystemVars,
+                        parent->second.Unexpanded.WarnSystemVars);
+    for (auto const& v : parent->second.Unexpanded.CacheVariables) {
       preset.CacheVariables.insert(v);
     }
-    for (auto const& v : parent->second.Environment) {
+    for (auto const& v : parent->second.Unexpanded.Environment) {
       preset.Environment.insert(v);
     }
   }
@@ -393,7 +397,7 @@ ReadFileResult VisitPreset(std::map<std::string, UnexpandedPreset>& presets,
 }
 
 ReadFileResult ComputePresetInheritance(
-  std::map<std::string, UnexpandedPreset>& presets)
+  std::map<std::string, cmCMakePresetsFile::PresetPair>& presets)
 {
   std::map<std::string, CycleStatus> cycleStatus;
   for (auto const& it : presets) {
@@ -401,7 +405,7 @@ ReadFileResult ComputePresetInheritance(
   }
 
   for (auto& it : presets) {
-    auto result = VisitPreset(presets, it.second, cycleStatus);
+    auto result = VisitPreset(presets, it.second.Unexpanded, cycleStatus);
     if (result != ReadFileResult::READ_OK) {
       return result;
     }
@@ -437,44 +441,111 @@ bool IsValidMacroNamespace(const std::string& str)
   return false;
 }
 
-bool VisitEnv(const cmCMakePresetsFile& file,
-              cmCMakePresetsFile::ExpandedPreset& preset,
-              std::map<std::string, CycleStatus>& envCycles,
-              std::string& value, CycleStatus& status);
+enum class ExpandMacroResult
+{
+  Ok,
+  Ignore,
+  Error,
+};
+
+ExpandMacroResult VisitEnv(const cmCMakePresetsFile& file,
+                           cmCMakePresetsFile::ExpandedPreset& preset,
+                           std::map<std::string, CycleStatus>& envCycles,
+                           std::string& value, CycleStatus& status);
+ExpandMacroResult ExpandMacros(const cmCMakePresetsFile& file,
+                               cmCMakePresetsFile::ExpandedPreset& preset,
+                               std::map<std::string, CycleStatus>& envCycles,
+                               std::string& out);
+ExpandMacroResult ExpandMacro(const cmCMakePresetsFile& file,
+                              cmCMakePresetsFile::ExpandedPreset& preset,
+                              std::map<std::string, CycleStatus>& envCycles,
+                              std::string& out,
+                              const std::string& macroNamespace,
+                              const std::string& macroName);
+
 bool ExpandMacros(const cmCMakePresetsFile& file,
-                  cmCMakePresetsFile::ExpandedPreset& preset,
-                  std::map<std::string, CycleStatus>& envCycles,
-                  std::string& out);
-bool ExpandMacro(const cmCMakePresetsFile& file,
-                 cmCMakePresetsFile::ExpandedPreset& preset,
-                 std::map<std::string, CycleStatus>& envCycles,
-                 std::string& out, const std::string& macroNamespace,
-                 const std::string& macroName);
-
-bool VisitEnv(const cmCMakePresetsFile& file,
-              cmCMakePresetsFile::ExpandedPreset& preset,
-              std::map<std::string, CycleStatus>& envCycles,
-              std::string& value, CycleStatus& status)
+                  const UnexpandedPreset& preset,
+                  cm::optional<ExpandedPreset>& out)
+{
+  out = preset;
+
+  std::map<std::string, CycleStatus> envCycles;
+  for (auto const& v : out->Environment) {
+    envCycles[v.first] = CycleStatus::Unvisited;
+  }
+
+  for (auto& v : out->Environment) {
+    if (v.second) {
+      switch (VisitEnv(file, *out, envCycles, *v.second, envCycles[v.first])) {
+        case ExpandMacroResult::Error:
+          return false;
+        case ExpandMacroResult::Ignore:
+          out.reset();
+          return true;
+        case ExpandMacroResult::Ok:
+          break;
+      }
+    }
+  }
+
+  std::string binaryDir = preset.BinaryDir;
+  switch (ExpandMacros(file, *out, envCycles, binaryDir)) {
+    case ExpandMacroResult::Error:
+      return false;
+    case ExpandMacroResult::Ignore:
+      out.reset();
+      return true;
+    case ExpandMacroResult::Ok:
+      break;
+  }
+  if (!cmSystemTools::FileIsFullPath(binaryDir)) {
+    binaryDir = cmStrCat(file.SourceDir, '/', binaryDir);
+  }
+  out->BinaryDir = cmSystemTools::CollapseFullPath(binaryDir);
+  cmSystemTools::ConvertToUnixSlashes(out->BinaryDir);
+
+  for (auto& variable : out->CacheVariables) {
+    if (variable.second) {
+      switch (ExpandMacros(file, *out, envCycles, variable.second->Value)) {
+        case ExpandMacroResult::Error:
+          return false;
+        case ExpandMacroResult::Ignore:
+          out.reset();
+          return true;
+        case ExpandMacroResult::Ok:
+          break;
+      }
+    }
+  }
+
+  return true;
+}
+
+ExpandMacroResult VisitEnv(const cmCMakePresetsFile& file,
+                           cmCMakePresetsFile::ExpandedPreset& preset,
+                           std::map<std::string, CycleStatus>& envCycles,
+                           std::string& value, CycleStatus& status)
 {
   if (status == CycleStatus::Verified) {
-    return true;
+    return ExpandMacroResult::Ok;
   }
   if (status == CycleStatus::InProgress) {
-    return false;
+    return ExpandMacroResult::Error;
   }
 
   status = CycleStatus::InProgress;
-  if (!ExpandMacros(file, preset, envCycles, value)) {
-    return false;
+  auto e = ExpandMacros(file, preset, envCycles, value);
+  if (e != ExpandMacroResult::Ok) {
+    return e;
   }
   status = CycleStatus::Verified;
-  return true;
+  return ExpandMacroResult::Ok;
 }
 
-bool ExpandMacros(const cmCMakePresetsFile& file,
-                  cmCMakePresetsFile::ExpandedPreset& preset,
-                  std::map<std::string, CycleStatus>& envCycles,
-                  std::string& out)
+ExpandMacroResult ExpandMacros(const cmCMakePresetsFile& file,
+                               cmCMakePresetsFile::ExpandedPreset& preset,
+                               std::map<std::string, CycleStatus>& envCycles,
+                               std::string& out)
 {
   std::string result;
   std::string macroNamespace;
@@ -521,9 +592,10 @@ bool ExpandMacros(const cmCMakePresetsFile& file,
 
       case State::MacroName:
         if (c == '}') {
-          if (!ExpandMacro(file, preset, envCycles, result, macroNamespace,
-                           macroName)) {
-            return false;
+          auto e = ExpandMacro(file, preset, envCycles, result, macroNamespace,
+                               macroName);
+          if (e != ExpandMacroResult::Ok) {
+            return e;
           }
           macroNamespace.clear();
           macroName.clear();
@@ -543,67 +615,72 @@ bool ExpandMacros(const cmCMakePresetsFile& file,
       result += macroNamespace;
       break;
     case State::MacroName:
-      return false;
+      return ExpandMacroResult::Error;
   }
 
   out = std::move(result);
-  return true;
+  return ExpandMacroResult::Ok;
 }
 
-bool ExpandMacro(const cmCMakePresetsFile& file,
-                 cmCMakePresetsFile::ExpandedPreset& preset,
-                 std::map<std::string, CycleStatus>& envCycles,
-                 std::string& out, const std::string& macroNamespace,
-                 const std::string& macroName)
+ExpandMacroResult ExpandMacro(const cmCMakePresetsFile& file,
+                              cmCMakePresetsFile::ExpandedPreset& preset,
+                              std::map<std::string, CycleStatus>& envCycles,
+                              std::string& out,
+                              const std::string& macroNamespace,
+                              const std::string& macroName)
 {
   if (macroNamespace.empty()) {
     if (macroName == "sourceDir") {
       out += file.SourceDir;
-      return true;
+      return ExpandMacroResult::Ok;
     }
     if (macroName == "sourceParentDir") {
       out += cmSystemTools::GetParentDirectory(file.SourceDir);
-      return true;
+      return ExpandMacroResult::Ok;
     }
     if (macroName == "presetName") {
       out += preset.Name;
-      return true;
+      return ExpandMacroResult::Ok;
     }
     if (macroName == "generator") {
       out += preset.Generator;
-      return true;
+      return ExpandMacroResult::Ok;
     }
     if (macroName == "dollar") {
       out += '$';
-      return true;
+      return ExpandMacroResult::Ok;
     }
   }
 
   if (macroNamespace == "env" && !macroName.empty()) {
     auto v = preset.Environment.find(macroName);
     if (v != preset.Environment.end() && v->second) {
-      if (!VisitEnv(file, preset, envCycles, *v->second,
-                    envCycles[macroName])) {
-        return false;
+      auto e =
+        VisitEnv(file, preset, envCycles, *v->second, envCycles[macroName]);
+      if (e != ExpandMacroResult::Ok) {
+        return e;
       }
       out += *v->second;
-      return true;
+      return ExpandMacroResult::Ok;
     }
   }
 
   if (macroNamespace == "env" || macroNamespace == "penv") {
     if (macroName.empty()) {
-      return false;
+      return ExpandMacroResult::Error;
     }
     const char* value = std::getenv(macroName.c_str());
     if (value) {
       out += value;
     }
-    return true;
+    return ExpandMacroResult::Ok;
   }
 
-  // "vendor" falls through to here
-  return false;
+  if (macroNamespace == "vendor") {
+    return ExpandMacroResult::Ignore;
+  }
+
+  return ExpandMacroResult::Error;
 }
 }
 
@@ -626,7 +703,7 @@ cmCMakePresetsFile::ReadFileResult cmCMakePresetsFile::ReadProjectPresets(
   this->PresetOrder.clear();
 
   std::vector<std::string> presetOrder;
-  std::map<std::string, UnexpandedPreset> presetMap;
+  std::map<std::string, PresetPair> presetMap;
 
   std::string filename = GetUserFilename(this->SourceDir);
   if (cmSystemTools::FileExists(filename)) {
@@ -656,6 +733,12 @@ cmCMakePresetsFile::ReadFileResult cmCMakePresetsFile::ReadProjectPresets(
     return result;
   }
 
+  for (auto& it : presetMap) {
+    if (!ExpandMacros(*this, it.second.Unexpanded, it.second.Expanded)) {
+      return ReadFileResult::INVALID_MACRO_EXPANSION;
+    }
+  }
+
   this->PresetOrder = std::move(presetOrder);
   this->Presets = std::move(presetMap);
   return ReadFileResult::READ_OK;
@@ -694,51 +777,16 @@ const char* cmCMakePresetsFile::ResultToString(ReadFileResult result)
       return "Cyclic preset inheritance";
     case ReadFileResult::USER_PRESET_INHERITANCE:
       return "Project preset inherits from user preset";
-    default:
-      return "Unknown error";
-  }
-}
-
-cm::optional<cmCMakePresetsFile::ExpandedPreset>
-cmCMakePresetsFile::ExpandMacros(const UnexpandedPreset& preset) const
-{
-  ExpandedPreset retval{ preset };
-
-  std::map<std::string, CycleStatus> envCycles;
-  for (auto const& v : retval.Environment) {
-    envCycles[v.first] = CycleStatus::Unvisited;
-  }
-
-  for (auto& v : retval.Environment) {
-    if (v.second &&
-        !VisitEnv(*this, retval, envCycles, *v.second, envCycles[v.first])) {
-      return cm::nullopt;
-    }
-  }
-
-  std::string binaryDir = preset.BinaryDir;
-  if (!::ExpandMacros(*this, retval, envCycles, binaryDir)) {
-    return cm::nullopt;
-  }
-  if (!cmSystemTools::FileIsFullPath(binaryDir)) {
-    binaryDir = cmStrCat(this->SourceDir, '/', binaryDir);
-  }
-  retval.BinaryDir = cmSystemTools::CollapseFullPath(binaryDir);
-  cmSystemTools::ConvertToUnixSlashes(retval.BinaryDir);
-
-  for (auto& variable : retval.CacheVariables) {
-    if (variable.second &&
-        !::ExpandMacros(*this, retval, envCycles, variable.second->Value)) {
-      return cm::nullopt;
-    }
+    case ReadFileResult::INVALID_MACRO_EXPANSION:
+      return "Invalid macro expansion";
   }
 
-  return cm::make_optional(retval);
+  return "Unknown error";
 }
 
 cmCMakePresetsFile::ReadFileResult cmCMakePresetsFile::ReadJSONFile(
   const std::string& filename, std::vector<std::string>& presetOrder,
-  std::map<std::string, UnexpandedPreset>& presetMap, bool user)
+  std::map<std::string, PresetPair>& presetMap, bool user)
 {
   cmsys::ifstream fin(filename.c_str());
   if (!fin) {
@@ -785,7 +833,7 @@ cmCMakePresetsFile::ReadFileResult cmCMakePresetsFile::ReadJSONFile(
     if (preset.Name.empty()) {
       return ReadFileResult::INVALID_PRESET;
     }
-    if (!presetMap.insert({ preset.Name, preset }).second) {
+    if (!presetMap.insert({ preset.Name, { preset, cm::nullopt } }).second) {
       return ReadFileResult::DUPLICATE_PRESETS;
     }
     presetOrder.push_back(preset.Name);
diff --git a/Source/cmCMakePresetsFile.h b/Source/cmCMakePresetsFile.h
index 70ec4c5..87797d7 100644
--- a/Source/cmCMakePresetsFile.h
+++ b/Source/cmCMakePresetsFile.h
@@ -102,8 +102,15 @@ public:
     }
   };
 
+  class PresetPair
+  {
+  public:
+    UnexpandedPreset Unexpanded;
+    cm::optional<ExpandedPreset> Expanded;
+  };
+
   std::string SourceDir;
-  std::map<std::string, UnexpandedPreset> Presets;
+  std::map<std::string, PresetPair> Presets;
   std::vector<std::string> PresetOrder;
 
   enum class ReadFileResult
@@ -123,6 +130,7 @@ public:
     DUPLICATE_PRESETS,
     CYCLIC_PRESET_INHERITANCE,
     USER_PRESET_INHERITANCE,
+    INVALID_MACRO_EXPANSION,
   };
 
   static std::string GetFilename(const std::string& sourceDir);
@@ -131,11 +139,9 @@ public:
                                     bool allowNoFiles = false);
   static const char* ResultToString(ReadFileResult result);
 
-  cm::optional<ExpandedPreset> ExpandMacros(
-    const UnexpandedPreset& preset) const;
-
 private:
-  ReadFileResult ReadJSONFile(
-    const std::string& filename, std::vector<std::string>& presetOrder,
-    std::map<std::string, UnexpandedPreset>& presetMap, bool user);
+  ReadFileResult ReadJSONFile(const std::string& filename,
+                              std::vector<std::string>& presetOrder,
+                              std::map<std::string, PresetPair>& presetMap,
+                              bool user);
 };
diff --git a/Source/cmake.cxx b/Source/cmake.cxx
index 2a14b03..5b6bd1e 100644
--- a/Source/cmake.cxx
+++ b/Source/cmake.cxx
@@ -1042,17 +1042,17 @@ void cmake::SetArgs(const std::vector<std::string>& args)
       this->PrintPresetList(settingsFile);
       return;
     }
-    if (preset->second.Hidden) {
+    if (preset->second.Unexpanded.Hidden) {
       cmSystemTools::Error(cmStrCat("Cannot use hidden preset in ",
                                     this->GetHomeDirectory(), ": \"",
                                     presetName, '"'));
       this->PrintPresetList(settingsFile);
       return;
     }
-    auto expandedPreset = settingsFile.ExpandMacros(preset->second);
+    auto const& expandedPreset = preset->second.Expanded;
     if (!expandedPreset) {
       cmSystemTools::Error(cmStrCat("Could not evaluate preset \"",
-                                    preset->second.Name,
+                                    preset->second.Unexpanded.Name,
                                     "\": Invalid macro expansion"));
       return;
     }
@@ -1464,13 +1464,12 @@ void cmake::PrintPresetList(const cmCMakePresetsFile& file) const
   std::vector<cmCMakePresetsFile::UnexpandedPreset> presets;
   for (auto const& p : file.PresetOrder) {
     auto const& preset = file.Presets.at(p);
-    if (!preset.Hidden &&
+    if (!preset.Unexpanded.Hidden && preset.Expanded &&
         std::find_if(generators.begin(), generators.end(),
                      [&preset](const GeneratorInfo& info) {
-                       return info.name == preset.Generator;
-                     }) != generators.end() &&
-        file.ExpandMacros(preset)) {
-      presets.push_back(preset);
+                       return info.name == preset.Unexpanded.Generator;
+                     }) != generators.end()) {
+      presets.push_back(preset.Unexpanded);
     }
   }
 
diff --git a/Tests/RunCMake/CMakePresets/CMakePresets.json.in b/Tests/RunCMake/CMakePresets/CMakePresets.json.in
index 18b9c3a..ea7df45 100644
--- a/Tests/RunCMake/CMakePresets/CMakePresets.json.in
+++ b/Tests/RunCMake/CMakePresets/CMakePresets.json.in
@@ -322,16 +322,6 @@
       "binaryDir": "${sourceDir}/build"
     },
     {
-      "name": "UnclosedMacro",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${sourceDir"
-    },
-    {
-      "name": "NoSuchMacro",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${noexist}"
-    },
-    {
       "name": "VendorMacro",
       "generator": "@RunCMake_GENERATOR@",
       "binaryDir": "$vendor{unknown.unknownMacro}"
@@ -342,31 +332,6 @@
       "binaryDir": "${sourceDir}/build"
     },
     {
-      "name": "EnvCycle",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${sourceDir}/build",
-      "environment": {
-        "ENV_1": "$env{ENV_2}",
-        "ENV_2": "$env{ENV_1}"
-      }
-    },
-    {
-      "name": "EmptyEnv",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${sourceDir}/build",
-      "cacheVariables": {
-        "MY_VAR": "$env{}"
-      }
-    },
-    {
-      "name": "EmptyPenv",
-      "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${sourceDir}/build",
-      "cacheVariables": {
-        "MY_VAR": "$penv{}"
-      }
-    },
-    {
       "name": "UseHiddenPreset",
       "generator": "@RunCMake_GENERATOR@",
       "binaryDir": "${sourceDir}/build",
diff --git a/Tests/RunCMake/CMakePresets/EmptyEnv-stderr.txt b/Tests/RunCMake/CMakePresets/EmptyEnv-stderr.txt
index 9c9c025..723ac21 100644
--- a/Tests/RunCMake/CMakePresets/EmptyEnv-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/EmptyEnv-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "EmptyEnv": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/EmptyEnv: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/EmptyEnv.json.in b/Tests/RunCMake/CMakePresets/EmptyEnv.json.in
new file mode 100644
index 0000000..ef0d575
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/EmptyEnv.json.in
@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "EmptyEnv",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "MY_VAR": "$env{}"
+      }
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CMakePresets/EmptyPenv-stderr.txt b/Tests/RunCMake/CMakePresets/EmptyPenv-stderr.txt
index 395c7b4..880cee6 100644
--- a/Tests/RunCMake/CMakePresets/EmptyPenv-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/EmptyPenv-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "EmptyPenv": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/EmptyPenv: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/EmptyPenv.json.in b/Tests/RunCMake/CMakePresets/EmptyPenv.json.in
new file mode 100644
index 0000000..9081fe5
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/EmptyPenv.json.in
@@ -0,0 +1,13 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "EmptyPenv",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "MY_VAR": "$penv{}"
+      }
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CMakePresets/EnvCycle-stderr.txt b/Tests/RunCMake/CMakePresets/EnvCycle-stderr.txt
index c8568f1..1d22b87 100644
--- a/Tests/RunCMake/CMakePresets/EnvCycle-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/EnvCycle-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "EnvCycle": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/EnvCycle: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/EnvCycle.json.in b/Tests/RunCMake/CMakePresets/EnvCycle.json.in
new file mode 100644
index 0000000..25a1349
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/EnvCycle.json.in
@@ -0,0 +1,14 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "EnvCycle",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir}/build",
+      "environment": {
+        "ENV_1": "$env{ENV_2}",
+        "ENV_2": "$env{ENV_1}"
+      }
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CMakePresets/ListPresets.json.in b/Tests/RunCMake/CMakePresets/ListPresets.json.in
index c7646c6..2ef3797 100644
--- a/Tests/RunCMake/CMakePresets/ListPresets.json.in
+++ b/Tests/RunCMake/CMakePresets/ListPresets.json.in
@@ -26,7 +26,7 @@
     {
       "name": "invalid-macro",
       "generator": "@RunCMake_GENERATOR@",
-      "binaryDir": "${noexist}"
+      "binaryDir": "$vendor{noexist}"
     },
     {
       "name": "ListPresetsHidden",
diff --git a/Tests/RunCMake/CMakePresets/NoSuchMacro-stderr.txt b/Tests/RunCMake/CMakePresets/NoSuchMacro-stderr.txt
index 08fa87b..7dafe62 100644
--- a/Tests/RunCMake/CMakePresets/NoSuchMacro-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/NoSuchMacro-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "NoSuchMacro": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/NoSuchMacro: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/NoSuchMacro.json.in b/Tests/RunCMake/CMakePresets/NoSuchMacro.json.in
new file mode 100644
index 0000000..94d0b76
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/NoSuchMacro.json.in
@@ -0,0 +1,10 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "NoSuchMacro",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${noexist}"
+    }
+  ]
+}
diff --git a/Tests/RunCMake/CMakePresets/RunCMakeTest.cmake b/Tests/RunCMake/CMakePresets/RunCMakeTest.cmake
index e0e54f4..dddf05f 100644
--- a/Tests/RunCMake/CMakePresets/RunCMakeTest.cmake
+++ b/Tests/RunCMake/CMakePresets/RunCMakeTest.cmake
@@ -121,6 +121,13 @@ run_cmake_presets(InvalidCMakeGeneratorConfig)
 run_cmake_presets(UnknownCMakeGeneratorConfig)
 run_cmake_presets(EmptyCacheKey)
 run_cmake_presets(EmptyEnvKey)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 0)
+run_cmake_presets(UnclosedMacro)
+run_cmake_presets(NoSuchMacro)
+run_cmake_presets(EnvCycle)
+run_cmake_presets(EmptyEnv)
+run_cmake_presets(EmptyPenv)
+set(CMakePresets_SCHEMA_EXPECTED_RESULT 1)
 
 # Test cmakeMinimumRequired field
 run_cmake_presets(MinimumRequiredInvalid)
@@ -170,13 +177,8 @@ run_cmake_presets(GoodInheritanceMultiSecond)
 run_cmake_presets(GoodInheritanceMacro)
 
 # Test bad preset arguments
-run_cmake_presets(UnclosedMacro)
-run_cmake_presets(NoSuchMacro)
 run_cmake_presets(VendorMacro)
 run_cmake_presets(InvalidGenerator)
-run_cmake_presets(EnvCycle)
-run_cmake_presets(EmptyEnv)
-run_cmake_presets(EmptyPenv)
 
 # Test Visual Studio-specific stuff
 if(RunCMake_GENERATOR MATCHES "^Visual Studio ")
diff --git a/Tests/RunCMake/CMakePresets/UnclosedMacro-stderr.txt b/Tests/RunCMake/CMakePresets/UnclosedMacro-stderr.txt
index 248510d..f9481f0 100644
--- a/Tests/RunCMake/CMakePresets/UnclosedMacro-stderr.txt
+++ b/Tests/RunCMake/CMakePresets/UnclosedMacro-stderr.txt
@@ -1 +1,2 @@
-^CMake Error: Could not evaluate preset "UnclosedMacro": Invalid macro expansion$
+^CMake Error: Could not read presets from [^
+]*/Tests/RunCMake/CMakePresets/UnclosedMacro: Invalid macro expansion$
diff --git a/Tests/RunCMake/CMakePresets/UnclosedMacro.json.in b/Tests/RunCMake/CMakePresets/UnclosedMacro.json.in
new file mode 100644
index 0000000..ad6cf7d
--- /dev/null
+++ b/Tests/RunCMake/CMakePresets/UnclosedMacro.json.in
@@ -0,0 +1,10 @@
+{
+  "version": 1,
+  "configurePresets": [
+    {
+      "name": "UnclosedMacro",
+      "generator": "@RunCMake_GENERATOR@",
+      "binaryDir": "${sourceDir"
+    }
+  ]
+}