fileapi: Add protocol v1 infrastructure with support for shared query files

Add a file-based API that clients may use to get semantic information
about the buildsystem that CMake generates.  Clients will write query
files under a designated location in the build tree, and CMake will
write reply files for clients to read.

Start with support for shared stateless query files.  These allow
clients to share requests for major object versions and get all those
recognized by CMake.  Once any client has written a shared request to a
build tree it will persist.  Other clients will not need to overwrite
the request (since it is stateless) and should not remove it either.

For now we add only an undocumented object kind to use for testing the
query and reply infrastructure.  Object kinds providing real semantic
information will be added later.

Issue: #18398

18 files changed, 209 insertions, 0 deletions

diff --git a/Tests/RunCMake/CMakeLists.txt b/Tests/RunCMake/CMakeLists.txt
index a4d829b..d5aa0c3 100644
--- a/Tests/RunCMake/CMakeLists.txt
+++ b/Tests/RunCMake/CMakeLists.txt
@@ -66,6 +66,9 @@ function(add_RunCMake_test_group test types)
   endforeach()
 endfunction()
 
+# Some tests use python for extra checks.
+find_package(PythonInterp QUIET)
+
 if(XCODE_VERSION AND "${XCODE_VERSION}" VERSION_LESS 6.1)
   set(Swift_ARGS -DXCODE_BELOW_6_1=1)
 endif()
@@ -155,6 +158,7 @@ add_RunCMake_test(DisallowedCommands)
 add_RunCMake_test(ExternalData)
 add_RunCMake_test(FeatureSummary)
 add_RunCMake_test(FPHSA)
+add_RunCMake_test(FileAPI -DPYTHON_EXECUTABLE=${PYTHON_EXECUTABLE})
 add_RunCMake_test(FindBoost)
 add_RunCMake_test(FindLua)
 add_RunCMake_test(FindOpenGL)
diff --git a/Tests/RunCMake/FileAPI/CMakeLists.txt b/Tests/RunCMake/FileAPI/CMakeLists.txt
new file mode 100644
index 0000000..44025d3
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/CMakeLists.txt
@@ -0,0 +1,3 @@
+cmake_minimum_required(VERSION 3.12)
+project(${RunCMake_TEST} NONE)
+include(${RunCMake_TEST}.cmake)
diff --git a/Tests/RunCMake/FileAPI/Empty-check.cmake b/Tests/RunCMake/FileAPI/Empty-check.cmake
new file mode 100644
index 0000000..2764b42
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Empty-check.cmake
@@ -0,0 +1,8 @@
+set(expect
+  query
+  reply
+  reply/index-[0-9.T-]+.json
+  )
+check_api("^${expect}$")
+
+check_python(Empty)
diff --git a/Tests/RunCMake/FileAPI/Empty-check.py b/Tests/RunCMake/FileAPI/Empty-check.py
new file mode 100644
index 0000000..75bf096
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Empty-check.py
@@ -0,0 +1,15 @@
+from check_index import *
+
+def check_reply(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == []
+
+def check_objects(o):
+    assert is_list(o)
+    assert len(o) == 0
+
+assert is_dict(index)
+assert sorted(index.keys()) == ["cmake", "objects", "reply"]
+check_cmake(index["cmake"])
+check_reply(index["reply"])
+check_objects(index["objects"])
diff --git a/Tests/RunCMake/FileAPI/Empty-prep.cmake b/Tests/RunCMake/FileAPI/Empty-prep.cmake
new file mode 100644
index 0000000..1d1f69e
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Empty-prep.cmake
@@ -0,0 +1 @@
+file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query")
diff --git a/Tests/RunCMake/FileAPI/Empty.cmake b/Tests/RunCMake/FileAPI/Empty.cmake
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Empty.cmake
diff --git a/Tests/RunCMake/FileAPI/Nothing-check.cmake b/Tests/RunCMake/FileAPI/Nothing-check.cmake
new file mode 100644
index 0000000..cd4f42e
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Nothing-check.cmake
@@ -0,0 +1 @@
+check_api("^$")
diff --git a/Tests/RunCMake/FileAPI/Nothing-prep.cmake b/Tests/RunCMake/FileAPI/Nothing-prep.cmake
new file mode 100644
index 0000000..b850d47
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Nothing-prep.cmake
@@ -0,0 +1 @@
+file(MAKE_DIRECTORY "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1")
diff --git a/Tests/RunCMake/FileAPI/Nothing.cmake b/Tests/RunCMake/FileAPI/Nothing.cmake
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Nothing.cmake
diff --git a/Tests/RunCMake/FileAPI/RunCMakeTest.cmake b/Tests/RunCMake/FileAPI/RunCMakeTest.cmake
new file mode 100644
index 0000000..bb016ca
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/RunCMakeTest.cmake
@@ -0,0 +1,40 @@
+include(RunCMake)
+
+# Function called in *-check.cmake scripts to check api files.
+function(check_api expect)
+  file(GLOB_RECURSE actual
+    LIST_DIRECTORIES TRUE
+    RELATIVE ${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1
+    ${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/*
+    )
+  if(NOT "${actual}" MATCHES "${expect}")
+    set(RunCMake_TEST_FAILED "API files:
+  ${actual}
+do not match what we expected:
+  ${expect}
+in directory:
+  ${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1" PARENT_SCOPE)
+  endif()
+endfunction()
+
+function(check_python case)
+  if(RunCMake_TEST_FAILED OR NOT PYTHON_EXECUTABLE)
+    return()
+  endif()
+  file(GLOB index ${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/reply/index-*.json)
+  execute_process(
+    COMMAND ${PYTHON_EXECUTABLE} "${RunCMake_SOURCE_DIR}/${case}-check.py" "${index}"
+    RESULT_VARIABLE result
+    OUTPUT_VARIABLE output
+    ERROR_VARIABLE output
+    )
+  if(NOT result EQUAL 0)
+    string(REPLACE "\n" "\n  " output "  ${output}")
+    set(RunCMake_TEST_FAILED "Unexpected index:\n${output}" PARENT_SCOPE)
+  endif()
+endfunction()
+
+run_cmake(Nothing)
+run_cmake(Empty)
+run_cmake(Stale)
+run_cmake(SharedStateless)
diff --git a/Tests/RunCMake/FileAPI/SharedStateless-check.cmake b/Tests/RunCMake/FileAPI/SharedStateless-check.cmake
new file mode 100644
index 0000000..7f3bb23
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/SharedStateless-check.cmake
@@ -0,0 +1,15 @@
+set(expect
+  query
+  query/__test-v1
+  query/__test-v2
+  query/__test-v3
+  query/query.json
+  query/unknown
+  reply
+  reply/__test-v1-[0-9a-f]+.json
+  reply/__test-v2-[0-9a-f]+.json
+  reply/index-[0-9.T-]+.json
+  )
+check_api("^${expect}$")
+
+check_python(SharedStateless)
diff --git a/Tests/RunCMake/FileAPI/SharedStateless-check.py b/Tests/RunCMake/FileAPI/SharedStateless-check.py
new file mode 100644
index 0000000..79f52d7
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/SharedStateless-check.py
@@ -0,0 +1,22 @@
+from check_index import *
+
+def check_reply(r):
+    assert is_dict(r)
+    assert sorted(r.keys()) == ["__test-v1", "__test-v2", "__test-v3", "query.json", "unknown"]
+    check_index__test(r["__test-v1"], 1, 3)
+    check_index__test(r["__test-v2"], 2, 0)
+    check_error(r["__test-v3"], "unknown query file")
+    check_error(r["query.json"], "unknown query file")
+    check_error(r["unknown"], "unknown query file")
+
+def check_objects(o):
+    assert is_list(o)
+    assert len(o) == 2
+    check_index__test(o[0], 1, 3)
+    check_index__test(o[1], 2, 0)
+
+assert is_dict(index)
+assert sorted(index.keys()) == ["cmake", "objects", "reply"]
+check_cmake(index["cmake"])
+check_reply(index["reply"])
+check_objects(index["objects"])
diff --git a/Tests/RunCMake/FileAPI/SharedStateless-prep.cmake b/Tests/RunCMake/FileAPI/SharedStateless-prep.cmake
new file mode 100644
index 0000000..b280414
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/SharedStateless-prep.cmake
@@ -0,0 +1,6 @@
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/__test-v1" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/__test-v2" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/__test-v3" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/query.json" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/query/unknown" "")
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/reply/object-to-be-deleted.json" "")
diff --git a/Tests/RunCMake/FileAPI/SharedStateless.cmake b/Tests/RunCMake/FileAPI/SharedStateless.cmake
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/SharedStateless.cmake
diff --git a/Tests/RunCMake/FileAPI/Stale-check.cmake b/Tests/RunCMake/FileAPI/Stale-check.cmake
new file mode 100644
index 0000000..7ee2c9e
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Stale-check.cmake
@@ -0,0 +1,4 @@
+set(expect
+  reply
+  )
+check_api("^${expect}$")
diff --git a/Tests/RunCMake/FileAPI/Stale-prep.cmake b/Tests/RunCMake/FileAPI/Stale-prep.cmake
new file mode 100644
index 0000000..e920925
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Stale-prep.cmake
@@ -0,0 +1 @@
+file(WRITE "${RunCMake_TEST_BINARY_DIR}/.cmake/api/v1/reply/object-to-be-deleted.json" "")
diff --git a/Tests/RunCMake/FileAPI/Stale.cmake b/Tests/RunCMake/FileAPI/Stale.cmake
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/Stale.cmake
diff --git a/Tests/RunCMake/FileAPI/check_index.py b/Tests/RunCMake/FileAPI/check_index.py
new file mode 100644
index 0000000..6cc16a5
--- /dev/null
+++ b/Tests/RunCMake/FileAPI/check_index.py
@@ -0,0 +1,88 @@
+import sys
+import os
+import json
+import re
+
+if sys.version_info[0] >= 3:
+    unicode = str
+
+def is_bool(x):
+    return isinstance(x, bool)
+
+def is_dict(x):
+    return isinstance(x, dict)
+
+def is_list(x):
+    return isinstance(x, list)
+
+def is_int(x):
+    return isinstance(x, int) or isinstance(x, long)
+
+def is_string(x):
+    return isinstance(x, str) or isinstance(x, unicode)
+
+def check_cmake(cmake):
+    assert is_dict(cmake)
+    assert sorted(cmake.keys()) == ["paths", "version"]
+    check_cmake_version(cmake["version"])
+    check_cmake_paths(cmake["paths"])
+
+def check_cmake_version(v):
+    assert is_dict(v)
+    assert sorted(v.keys()) == ["isDirty", "major", "minor", "patch", "string", "suffix"]
+    assert is_string(v["string"])
+    assert is_int(v["major"])
+    assert is_int(v["minor"])
+    assert is_int(v["patch"])
+    assert is_string(v["suffix"])
+    assert is_bool(v["isDirty"])
+
+def check_cmake_paths(v):
+    assert is_dict(v)
+    assert sorted(v.keys()) == ["cmake", "cpack", "ctest", "root"]
+    assert is_string(v["cmake"])
+    assert is_string(v["cpack"])
+    assert is_string(v["ctest"])
+    assert is_string(v["root"])
+
+def check_index_object(indexEntry, kind, major, minor, check):
+    assert is_dict(indexEntry)
+    assert sorted(indexEntry.keys()) == ["jsonFile", "kind", "version"]
+    assert is_string(indexEntry["kind"])
+    assert indexEntry["kind"] == kind
+    assert is_dict(indexEntry["version"])
+    assert sorted(indexEntry["version"].keys()) == ["major", "minor"]
+    assert indexEntry["version"]["major"] == major
+    assert indexEntry["version"]["minor"] == minor
+    assert is_string(indexEntry["jsonFile"])
+    filepath = os.path.join(reply_dir, indexEntry["jsonFile"])
+    with open(filepath) as f:
+        object = json.load(f)
+    assert is_dict(object)
+    assert "kind" in object
+    assert is_string(object["kind"])
+    assert object["kind"] == kind
+    assert "version" in object
+    assert is_dict(object["version"])
+    assert sorted(object["version"].keys()) == ["major", "minor"]
+    assert object["version"]["major"] == major
+    assert object["version"]["minor"] == minor
+    if check:
+        check(object)
+
+def check_index__test(indexEntry, major, minor):
+    def check(object):
+        assert sorted(object.keys()) == ["kind", "version"]
+    check_index_object(indexEntry, "__test", major, minor, check)
+
+def check_error(value, error):
+    assert is_dict(value)
+    assert sorted(value.keys()) == ["error"]
+    assert is_string(value["error"])
+    assert value["error"] == error
+
+reply_index = sys.argv[1]
+reply_dir = os.path.dirname(reply_index)
+
+with open(reply_index) as f:
+    index = json.load(f)