fileapi: Add protocol v1 infrastructure with support for shared query files

Add a file-based API that clients may use to get semantic information
about the buildsystem that CMake generates.  Clients will write query
files under a designated location in the build tree, and CMake will
write reply files for clients to read.

Start with support for shared stateless query files.  These allow
clients to share requests for major object versions and get all those
recognized by CMake.  Once any client has written a shared request to a
build tree it will persist.  Other clients will not need to overwrite
the request (since it is stateless) and should not remove it either.

For now we add only an undocumented object kind to use for testing the
query and reply infrastructure.  Object kinds providing real semantic
information will be added later.

Issue: #18398

5 files changed, 423 insertions, 0 deletions

diff --git a/Source/CMakeLists.txt b/Source/CMakeLists.txt
index 9aebfa7..ec71fe0 100644
--- a/Source/CMakeLists.txt
+++ b/Source/CMakeLists.txt
@@ -207,6 +207,8 @@ set(SRCS
   cmExtraKateGenerator.h
   cmExtraSublimeTextGenerator.cxx
   cmExtraSublimeTextGenerator.h
+  cmFileAPI.cxx
+  cmFileAPI.h
   cmFileLock.cxx
   cmFileLock.h
   cmFileLockPool.cxx
diff --git a/Source/cmFileAPI.cxx b/Source/cmFileAPI.cxx
new file mode 100644
index 0000000..23e0ced
--- /dev/null
+++ b/Source/cmFileAPI.cxx
@@ -0,0 +1,299 @@
+/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+   file Copyright.txt or https://cmake.org/licensing for details.  */
+#include "cmFileAPI.h"
+
+#include "cmCryptoHash.h"
+#include "cmSystemTools.h"
+#include "cmTimestamp.h"
+#include "cmake.h"
+#include "cmsys/Directory.hxx"
+#include "cmsys/FStream.hxx"
+
+#include <algorithm>
+#include <cassert>
+#include <chrono>
+#include <ctime>
+#include <iomanip>
+#include <sstream>
+#include <utility>
+
+cmFileAPI::cmFileAPI(cmake* cm)
+  : CMakeInstance(cm)
+{
+  this->APIv1 =
+    this->CMakeInstance->GetHomeOutputDirectory() + "/.cmake/api/v1";
+
+  Json::StreamWriterBuilder wbuilder;
+  wbuilder["indentation"] = "\t";
+  this->JsonWriter =
+    std::unique_ptr<Json::StreamWriter>(wbuilder.newStreamWriter());
+}
+
+void cmFileAPI::ReadQueries()
+{
+  std::string const query_dir = this->APIv1 + "/query";
+  this->QueryExists = cmSystemTools::FileIsDirectory(query_dir);
+  if (!this->QueryExists) {
+    return;
+  }
+
+  // Load queries at the top level.
+  std::vector<std::string> queries = cmFileAPI::LoadDir(query_dir);
+
+  // Read the queries and save for later.
+  for (std::string& query : queries) {
+    if (!cmFileAPI::ReadQuery(query, this->TopQuery.Known)) {
+      this->TopQuery.Unknown.push_back(std::move(query));
+    }
+  }
+}
+
+void cmFileAPI::WriteReplies()
+{
+  if (this->QueryExists) {
+    cmSystemTools::MakeDirectory(this->APIv1 + "/reply");
+    this->WriteJsonFile(this->BuildReplyIndex(), "index", ComputeSuffixTime);
+  }
+
+  this->RemoveOldReplyFiles();
+}
+
+std::vector<std::string> cmFileAPI::LoadDir(std::string const& dir)
+{
+  std::vector<std::string> files;
+  cmsys::Directory d;
+  d.Load(dir);
+  for (unsigned long i = 0; i < d.GetNumberOfFiles(); ++i) {
+    std::string f = d.GetFile(i);
+    if (f != "." && f != "..") {
+      files.push_back(std::move(f));
+    }
+  }
+  std::sort(files.begin(), files.end());
+  return files;
+}
+
+void cmFileAPI::RemoveOldReplyFiles()
+{
+  std::string const reply_dir = this->APIv1 + "/reply";
+  std::vector<std::string> files = this->LoadDir(reply_dir);
+  for (std::string const& f : files) {
+    if (this->ReplyFiles.find(f) == this->ReplyFiles.end()) {
+      std::string file = reply_dir + "/" + f;
+      cmSystemTools::RemoveFile(file);
+    }
+  }
+}
+
+std::string cmFileAPI::WriteJsonFile(
+  Json::Value const& value, std::string const& prefix,
+  std::string (*computeSuffix)(std::string const&))
+{
+  std::string fileName;
+
+  // Write the json file with a temporary name.
+  std::string const& tmpFile = this->APIv1 + "/tmp.json";
+  cmsys::ofstream ftmp(tmpFile.c_str());
+  this->JsonWriter->write(value, &ftmp);
+  ftmp << "\n";
+  ftmp.close();
+  if (!ftmp) {
+    cmSystemTools::RemoveFile(tmpFile);
+    return fileName;
+  }
+
+  // Compute the final name for the file.
+  fileName = prefix + "-" + computeSuffix(tmpFile) + ".json";
+
+  // Create the destination.
+  std::string file = this->APIv1 + "/reply";
+  cmSystemTools::MakeDirectory(file);
+  file += "/";
+  file += fileName;
+
+  // If the final name already exists then assume it has proper content.
+  // Otherwise, atomically place the reply file at its final name
+  if (cmSystemTools::FileExists(file, true) ||
+      !cmSystemTools::RenameFile(tmpFile.c_str(), file.c_str())) {
+    cmSystemTools::RemoveFile(tmpFile);
+  }
+
+  // Record this among files we have just written.
+  this->ReplyFiles.insert(fileName);
+
+  return fileName;
+}
+
+std::string cmFileAPI::ComputeSuffixHash(std::string const& file)
+{
+  cmCryptoHash hasher(cmCryptoHash::AlgoSHA3_256);
+  std::string hash = hasher.HashFile(file);
+  hash.resize(20, '0');
+  return hash;
+}
+
+std::string cmFileAPI::ComputeSuffixTime(std::string const&)
+{
+  std::chrono::milliseconds ms =
+    std::chrono::duration_cast<std::chrono::milliseconds>(
+      std::chrono::system_clock::now().time_since_epoch());
+  std::chrono::seconds s =
+    std::chrono::duration_cast<std::chrono::seconds>(ms);
+
+  std::time_t ts = s.count();
+  std::size_t tms = ms.count() % 1000;
+
+  cmTimestamp cmts;
+  std::ostringstream ss;
+  ss << cmts.CreateTimestampFromTimeT(ts, "%Y-%m-%dT%H-%M-%S", true) << '-'
+     << std::setfill('0') << std::setw(4) << tms;
+  return ss.str();
+}
+
+bool cmFileAPI::ReadQuery(std::string const& query,
+                          std::vector<Object>& objects)
+{
+  // Parse the "<kind>-" syntax.
+  std::string::size_type sep_pos = query.find('-');
+  if (sep_pos == std::string::npos) {
+    return false;
+  }
+  std::string kindName = query.substr(0, sep_pos);
+  std::string verStr = query.substr(sep_pos + 1);
+  if (kindName == ObjectKindName(ObjectKind::InternalTest)) {
+    Object o;
+    o.Kind = ObjectKind::InternalTest;
+    if (verStr == "v1") {
+      o.Version = 1;
+    } else if (verStr == "v2") {
+      o.Version = 2;
+    } else {
+      return false;
+    }
+    objects.push_back(o);
+    return true;
+  }
+  return false;
+}
+
+Json::Value cmFileAPI::BuildReplyIndex()
+{
+  Json::Value index(Json::objectValue);
+
+  // Report information about this version of CMake.
+  index["cmake"] = this->BuildCMake();
+
+  // Reply to all queries that we loaded.
+  index["reply"] = this->BuildReply(this->TopQuery);
+
+  // Move our index of generated objects into its field.
+  Json::Value& objects = index["objects"] = Json::arrayValue;
+  for (auto& entry : this->ReplyIndexObjects) {
+    objects.append(std::move(entry.second)); // NOLINT(*)
+  }
+
+  return index;
+}
+
+Json::Value cmFileAPI::BuildCMake()
+{
+  Json::Value cmake = Json::objectValue;
+  cmake["version"] = this->CMakeInstance->ReportVersionJson();
+  Json::Value& cmake_paths = cmake["paths"] = Json::objectValue;
+  cmake_paths["cmake"] = cmSystemTools::GetCMakeCommand();
+  cmake_paths["ctest"] = cmSystemTools::GetCTestCommand();
+  cmake_paths["cpack"] = cmSystemTools::GetCPackCommand();
+  cmake_paths["root"] = cmSystemTools::GetCMakeRoot();
+  return cmake;
+}
+
+Json::Value cmFileAPI::BuildReply(Query const& q)
+{
+  Json::Value reply = Json::objectValue;
+  for (Object const& o : q.Known) {
+    std::string const& name = ObjectName(o);
+    reply[name] = this->AddReplyIndexObject(o);
+  }
+
+  for (std::string const& name : q.Unknown) {
+    reply[name] = cmFileAPI::BuildReplyError("unknown query file");
+  }
+  return reply;
+}
+
+Json::Value cmFileAPI::BuildReplyError(std::string const& error)
+{
+  Json::Value e = Json::objectValue;
+  e["error"] = error;
+  return e;
+}
+
+Json::Value const& cmFileAPI::AddReplyIndexObject(Object const& o)
+{
+  Json::Value& indexEntry = this->ReplyIndexObjects[o];
+  if (!indexEntry.isNull()) {
+    // The reply object has already been generated.
+    return indexEntry;
+  }
+
+  // Generate this reply object.
+  Json::Value const& object = this->BuildObject(o);
+  assert(object.isObject());
+
+  // Populate this index entry.
+  indexEntry = Json::objectValue;
+  indexEntry["kind"] = object["kind"];
+  indexEntry["version"] = object["version"];
+  indexEntry["jsonFile"] = this->WriteJsonFile(object, ObjectName(o));
+  return indexEntry;
+}
+
+const char* cmFileAPI::ObjectKindName(ObjectKind kind)
+{
+  // Keep in sync with ObjectKind enum.
+  static const char* objectKindNames[] = {
+    "__test" //
+  };
+  return objectKindNames[size_t(kind)];
+}
+
+std::string cmFileAPI::ObjectName(Object const& o)
+{
+  std::string name = ObjectKindName(o.Kind);
+  name += "-v";
+  name += std::to_string(o.Version);
+  return name;
+}
+
+Json::Value cmFileAPI::BuildObject(Object const& object)
+{
+  Json::Value value;
+
+  switch (object.Kind) {
+    case ObjectKind::InternalTest:
+      value = this->BuildInternalTest(object);
+      break;
+  }
+
+  return value;
+}
+
+// The "__test" object kind is for internal testing of CMake.
+
+static unsigned int const InternalTestV1Minor = 3;
+static unsigned int const InternalTestV2Minor = 0;
+
+Json::Value cmFileAPI::BuildInternalTest(Object const& object)
+{
+  Json::Value test = Json::objectValue;
+  test["kind"] = this->ObjectKindName(object.Kind);
+  Json::Value& version = test["version"] = Json::objectValue;
+  if (object.Version == 2) {
+    version["major"] = 2;
+    version["minor"] = InternalTestV2Minor;
+  } else {
+    version["major"] = 1;
+    version["minor"] = InternalTestV1Minor;
+  }
+  return test;
+}
diff --git a/Source/cmFileAPI.h b/Source/cmFileAPI.h
new file mode 100644
index 0000000..39b054d
--- /dev/null
+++ b/Source/cmFileAPI.h
@@ -0,0 +1,109 @@
+/* Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+   file Copyright.txt or https://cmake.org/licensing for details.  */
+#ifndef cmFileAPI_h
+#define cmFileAPI_h
+
+#include "cmConfigure.h" // IWYU pragma: keep
+
+#include "cm_jsoncpp_value.h"
+#include "cm_jsoncpp_writer.h"
+
+#include <map>
+#include <memory> // IWYU pragma: keep
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+class cmake;
+
+class cmFileAPI
+{
+public:
+  cmFileAPI(cmake* cm);
+
+  /** Read fileapi queries from disk.  */
+  void ReadQueries();
+
+  /** Write fileapi replies to disk.  */
+  void WriteReplies();
+
+  /** Get the "cmake" instance with which this was constructed.  */
+  cmake* GetCMakeInstance() const { return this->CMakeInstance; }
+
+private:
+  cmake* CMakeInstance;
+
+  /** The api/v1 directory location.  */
+  std::string APIv1;
+
+  /** The set of files we have just written to the reply directory.  */
+  std::unordered_set<std::string> ReplyFiles;
+
+  static std::vector<std::string> LoadDir(std::string const& dir);
+  void RemoveOldReplyFiles();
+
+  // Keep in sync with ObjectKindName.
+  enum class ObjectKind
+  {
+    InternalTest
+  };
+
+  /** Identify one object kind and major version.  */
+  struct Object
+  {
+    ObjectKind Kind;
+    unsigned long Version = 0;
+    friend bool operator<(Object const& l, Object const& r)
+    {
+      if (l.Kind != r.Kind) {
+        return l.Kind < r.Kind;
+      }
+      return l.Version < r.Version;
+    }
+  };
+
+  /** Represent content of a query directory.  */
+  struct Query
+  {
+    /** Known object kind-version pairs.  */
+    std::vector<Object> Known;
+    /** Unknown object kind names.  */
+    std::vector<std::string> Unknown;
+  };
+
+  /** Whether the top-level query directory exists at all.  */
+  bool QueryExists = false;
+
+  /** The content of the top-level query directory.  */
+  Query TopQuery;
+
+  /** Reply index object generated for object kind/version.
+      This populates the "objects" field of the reply index.  */
+  std::map<Object, Json::Value> ReplyIndexObjects;
+
+  std::unique_ptr<Json::StreamWriter> JsonWriter;
+
+  std::string WriteJsonFile(
+    Json::Value const& value, std::string const& prefix,
+    std::string (*computeSuffix)(std::string const&) = ComputeSuffixHash);
+  static std::string ComputeSuffixHash(std::string const&);
+  static std::string ComputeSuffixTime(std::string const&);
+
+  static bool ReadQuery(std::string const& query,
+                        std::vector<Object>& objects);
+
+  Json::Value BuildReplyIndex();
+  Json::Value BuildCMake();
+  Json::Value BuildReply(Query const& q);
+  static Json::Value BuildReplyError(std::string const& error);
+  Json::Value const& AddReplyIndexObject(Object const& o);
+
+  static const char* ObjectKindName(ObjectKind kind);
+  static std::string ObjectName(Object const& o);
+
+  Json::Value BuildObject(Object const& object);
+
+  Json::Value BuildInternalTest(Object const& object);
+};
+
+#endif
diff --git a/Source/cmake.cxx b/Source/cmake.cxx
index 2ac7f4d..e81d14b 100644
--- a/Source/cmake.cxx
+++ b/Source/cmake.cxx
@@ -30,6 +30,7 @@
 #if defined(CMAKE_BUILD_WITH_CMAKE)
 #  include "cm_jsoncpp_writer.h"
 
+#  include "cmFileAPI.h"
 #  include "cmGraphVizWriter.h"
 #  include "cmVariableWatch.h"
 #  include <unordered_map>
@@ -1443,6 +1444,11 @@ int cmake::ActualConfigure()
     this->TruncateOutputLog("CMakeError.log");
   }
 
+#if defined(CMAKE_BUILD_WITH_CMAKE)
+  this->FileAPI = cm::make_unique<cmFileAPI>(this);
+  this->FileAPI->ReadQueries();
+#endif
+
   // actually do the configure
   this->GlobalGenerator->Configure();
   // Before saving the cache
@@ -1682,6 +1688,10 @@ int cmake::Generate()
   // for the Visual Studio and Xcode generators.)
   this->SaveCache(this->GetHomeOutputDirectory());
 
+#if defined(CMAKE_BUILD_WITH_CMAKE)
+  this->FileAPI->WriteReplies();
+#endif
+
   return 0;
 }
 
diff --git a/Source/cmake.h b/Source/cmake.h
index d3d0e80..d00acc7 100644
--- a/Source/cmake.h
+++ b/Source/cmake.h
@@ -6,6 +6,7 @@
 #include "cmConfigure.h" // IWYU pragma: keep
 
 #include <map>
+#include <memory> // IWYU pragma: keep
 #include <set>
 #include <string>
 #include <unordered_set>
@@ -21,6 +22,7 @@
 #endif
 
 class cmExternalMakefileProjectGeneratorFactory;
+class cmFileAPI;
 class cmFileTimeComparison;
 class cmGlobalGenerator;
 class cmGlobalGeneratorFactory;
@@ -528,6 +530,7 @@ private:
 
 #if defined(CMAKE_BUILD_WITH_CMAKE)
   cmVariableWatch* VariableWatch;
+  std::unique_ptr<cmFileAPI> FileAPI;
 #endif
 
   cmState* State;