Merge topic 'server-refactor'

39c2feaf misc: Added utility method to allow working with stacks
f5d2988e server: Swapped to cm_thread impl
2636d86c utility: Added minimal std::thread drop-in
d46b4ba8 server: Updated server tests to try various communication channels
08dca583 Tests: reworked server tests to allow other operation modes

Acked-by: Kitware Robot <kwrobot@kitware.com>
Merge-request: !1230

3 files changed, 224 insertions, 122 deletions

diff --git a/Tests/Server/CMakeLists.txt b/Tests/Server/CMakeLists.txt
index 08bef0c..41d1131 100644
--- a/Tests/Server/CMakeLists.txt
+++ b/Tests/Server/CMakeLists.txt
@@ -3,10 +3,10 @@ project(Server CXX)
 
 find_package(PythonInterp REQUIRED)
 
-macro(do_test bsname file)
+macro(do_test bsname file type)
   execute_process(COMMAND ${PYTHON_EXECUTABLE}
     -B # no .pyc files
-    "${CMAKE_SOURCE_DIR}/server-test.py"
+    "${CMAKE_SOURCE_DIR}/${type}-test.py"
     "${CMAKE_COMMAND}"
     "${CMAKE_SOURCE_DIR}/${file}"
     "${CMAKE_SOURCE_DIR}"
@@ -20,9 +20,9 @@ macro(do_test bsname file)
   endif()
 endmacro()
 
-do_test("test_cache" "tc_cache.json")
-do_test("test_handshake" "tc_handshake.json")
-do_test("test_globalSettings" "tc_globalSettings.json")
-do_test("test_buildsystem1" "tc_buildsystem1.json")
+do_test("test_cache" "tc_cache.json" "server")
+do_test("test_handshake" "tc_handshake.json" "server")
+do_test("test_globalSettings" "tc_globalSettings.json" "server")
+do_test("test_buildsystem1" "tc_buildsystem1.json" "server")
 
 add_executable(Server empty.cpp)
diff --git a/Tests/Server/cmakelib.py b/Tests/Server/cmakelib.py
index 2218e02..39e3618 100644
--- a/Tests/Server/cmakelib.py
+++ b/Tests/Server/cmakelib.py
@@ -1,5 +1,5 @@
 from __future__ import print_function
-import sys, subprocess, json
+import sys, subprocess, json, os, select, shutil, time, socket
 
 termwidth = 150
 
@@ -38,11 +38,50 @@ def col_print(title, array):
   for index in range(numRows):
     print(indent + pad.join(item.ljust(maxitemwidth) for item in array[index::numRows]))
 
+filterPacket = lambda x: x
+
+STDIN = 0
+PIPE = 1
+
+communicationMethods = [STDIN]
+
+if hasattr(socket, 'AF_UNIX'):
+  communicationMethods.append(PIPE)
+
+def defaultExitWithError(proc):
+  data = ""
+  try:
+    while select.select([proc.outPipe], [], [], 3.)[0]:
+      data = data + proc.outPipe.read(1)
+    if len(data):
+      print("Rest of raw buffer from server:")
+      printServer(data)
+  except:
+    pass
+  proc.outPipe.close()
+  proc.inPipe.close()
+  proc.kill()
+  sys.exit(1)
+
+exitWithError = lambda proc: defaultExitWithError(proc)
+
+serverTag = "SERVER"
+
+def printServer(*args):
+    print(serverTag + ">", *args)
+    print()
+    sys.stdout.flush()
+
+def printClient(*args):
+    print("CLIENT>", *args)
+    print()
+    sys.stdout.flush()
+
 def waitForRawMessage(cmakeCommand):
   stdoutdata = ""
   payload = ""
   while not cmakeCommand.poll():
-    stdoutdataLine = cmakeCommand.stdout.readline()
+    stdoutdataLine = cmakeCommand.outPipe.readline()
     if stdoutdataLine:
       stdoutdata += stdoutdataLine.decode('utf-8')
     else:
@@ -50,12 +89,16 @@ def waitForRawMessage(cmakeCommand):
     begin = stdoutdata.find('[== "CMake Server" ==[\n')
     end = stdoutdata.find(']== "CMake Server" ==]')
 
-    if (begin != -1 and end != -1):
+    if begin != -1 and end != -1:
       begin += len('[== "CMake Server" ==[\n')
       payload = stdoutdata[begin:end]
-      if print_communication:
-        print("\nSERVER>", json.loads(payload), "\n")
-      return json.loads(payload)
+      jsonPayload = json.loads(payload)
+      filteredPayload = filterPacket(jsonPayload)
+      if print_communication and filteredPayload:
+        printServer(filteredPayload)
+      if filteredPayload is not None or jsonPayload is None:
+          return jsonPayload
+      stdoutdata = stdoutdata[(end+len(']== "CMake Server" ==]')):]
 
 def writeRawData(cmakeCommand, content):
   writeRawData.counter += 1
@@ -71,27 +114,53 @@ def writeRawData(cmakeCommand, content):
     payload = payload.replace('\n', '\r\n')
 
   if print_communication:
-    print("\nCLIENT>", content, "(Use \\r\\n:", rn, ")\n")
-  cmakeCommand.stdin.write(payload.encode('utf-8'))
-  cmakeCommand.stdin.flush()
+    printClient(content, "(Use \\r\\n:", rn, ")")
+
+  cmakeCommand.write(payload.encode('utf-8'))
+
 writeRawData.counter = 0
 
 def writePayload(cmakeCommand, obj):
   writeRawData(cmakeCommand, json.dumps(obj))
 
-def initProc(cmakeCommand):
-  cmakeCommand = subprocess.Popen([cmakeCommand, "-E", "server", "--experimental", "--debug"],
-                                  stdin=subprocess.PIPE,
-                                  stdout=subprocess.PIPE)
+def getPipeName():
+  return "/tmp/server-test-socket"
+
+def attachPipe(cmakeCommand, pipeName):
+  time.sleep(1)
+  sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+  sock.connect(pipeName)
+  global serverTag
+  serverTag = "SERVER(PIPE)"
+  cmakeCommand.outPipe = sock.makefile()
+  cmakeCommand.inPipe = sock
+  cmakeCommand.write = cmakeCommand.inPipe.sendall
+
+def writeAndFlush(pipe, val):
+  pipe.write(val)
+  pipe.flush()
+
+def initServerProc(cmakeCommand, comm):
+  if comm == PIPE:
+    pipeName = getPipeName()
+    cmakeCommand = subprocess.Popen([cmakeCommand, "-E", "server", "--experimental", "--pipe=" + pipeName])
+    attachPipe(cmakeCommand, pipeName)
+  else:
+    cmakeCommand = subprocess.Popen([cmakeCommand, "-E", "server", "--experimental", "--debug"],
+                                    stdin=subprocess.PIPE,
+                                    stdout=subprocess.PIPE)
+    cmakeCommand.outPipe = cmakeCommand.stdout
+    cmakeCommand.inPipe = cmakeCommand.stdin
+    cmakeCommand.write = lambda val: writeAndFlush(cmakeCommand.inPipe, val)
 
   packet = waitForRawMessage(cmakeCommand)
   if packet == None:
     print("Not in server mode")
-    sys.exit(1)
+    sys.exit(2)
 
   if packet['type'] != 'hello':
     print("No hello message")
-    sys.exit(1)
+    sys.exit(3)
 
   return cmakeCommand
 
@@ -115,7 +184,8 @@ def waitForMessage(cmakeCommand, expected):
   packet = ordered(waitForRawMessage(cmakeCommand))
 
   if packet != data:
-    sys.exit(-1)
+    print ("Received unexpected message; test failed")
+    exitWithError(cmakeCommand)
   return packet
 
 def waitForReply(cmakeCommand, originalType, cookie, skipProgress):
@@ -124,25 +194,27 @@ def waitForReply(cmakeCommand, originalType, cookie, skipProgress):
     packet = waitForRawMessage(cmakeCommand)
     t = packet['type']
     if packet['cookie'] != cookie or packet['inReplyTo'] != originalType:
-      sys.exit(1)
+      print("cookie or inReplyTo mismatch")
+      sys.exit(4)
     if t == 'message' or t == 'progress':
       if skipProgress:
         continue
     if t == 'reply':
         break
-    sys.exit(1)
+    print("Unrecognized message", packet)
+    sys.exit(5)
 
   return packet
 
 def waitForError(cmakeCommand, originalType, cookie, message):
   packet = waitForRawMessage(cmakeCommand)
   if packet['cookie'] != cookie or packet['type'] != 'error' or packet['inReplyTo'] != originalType or packet['errorMessage'] != message:
-    sys.exit(1)
+    sys.exit(6)
 
 def waitForProgress(cmakeCommand, originalType, cookie, current, message):
   packet = waitForRawMessage(cmakeCommand)
   if packet['cookie'] != cookie or packet['type'] != 'progress' or packet['inReplyTo'] != originalType or packet['progressCurrent'] != current or packet['progressMessage'] != message:
-    sys.exit(1)
+    sys.exit(7)
 
 def handshake(cmakeCommand, major, minor, source, build, generator, extraGenerator):
   version = { 'major': major }
@@ -167,9 +239,9 @@ def validateGlobalSettings(cmakeCommand, cmakeCommandPath, data):
   versionString = version['string']
   vs = str(version['major']) + '.' + str(version['minor']) + '.' + str(version['patch'])
   if (versionString != vs and not versionString.startswith(vs + '-')):
-    sys.exit(1)
+    sys.exit(8)
   if (versionString != cmakeVersion):
-    sys.exit(1)
+    sys.exit(9)
 
   # validate generators:
   generatorObjects = capabilities['generators']
@@ -202,16 +274,16 @@ def validateGlobalSettings(cmakeCommand, cmakeCommandPath, data):
 
   for gen in cmakeGenerators:
     if (not gen in generators):
-        sys.exit(1)
+        sys.exit(10)
 
   gen = packet['generator']
   if (gen != '' and not (gen in generators)):
-    sys.exit(1)
+    sys.exit(11)
 
   for i in data:
     print("Validating", i)
     if (packet[i] != data[i]):
-      sys.exit(1)
+      sys.exit(12)
 
 def validateCache(cmakeCommand, data):
   packet = waitForReply(cmakeCommand, 'cache', '', False)
@@ -236,3 +308,43 @@ def validateCache(cmakeCommand, data):
   if (not hadHomeDir):
     print('No CMAKE_HOME_DIRECTORY found in cache.')
     sys.exit(1)
+
+def handleBasicMessage(proc, obj, debug):
+  if 'sendRaw' in obj:
+    data = obj['sendRaw']
+    if debug: print("Sending raw:", data)
+    writeRawData(proc, data)
+    return True
+  elif 'send' in obj:
+    data = obj['send']
+    if debug: print("Sending:", json.dumps(data))
+    writePayload(proc, data)
+    return True
+  elif 'recv' in obj:
+    data = obj['recv']
+    if debug: print("Waiting for:", json.dumps(data))
+    waitForMessage(proc, data)
+    return True
+  elif 'message' in obj:
+    print("MESSAGE:", obj["message"])
+    sys.stdout.flush()
+    return True
+  return False
+
+def shutdownProc(proc):
+  # Tell the server to exit.
+  proc.inPipe.close()
+  proc.outPipe.close()
+
+  # Wait for the server to exit.
+  # If this version of python supports it, terminate the server after a timeout.
+  try:
+    proc.wait(timeout=5)
+  except TypeError:
+    proc.wait()
+  except:
+    proc.terminate()
+    raise
+
+  print('cmake-server exited: %d' % proc.returncode)
+  sys.exit(proc.returncode)
diff --git a/Tests/Server/server-test.py b/Tests/Server/server-test.py
index 9380910..701c6e9 100644
--- a/Tests/Server/server-test.py
+++ b/Tests/Server/server-test.py
@@ -9,7 +9,7 @@ sourceDir = sys.argv[3]
 buildDir = sys.argv[4] + "/" + os.path.splitext(os.path.basename(testFile))[0]
 cmakeGenerator = sys.argv[5]
 
-print("Test:", testFile,
+print("Server Test:", testFile,
       "\n-- SourceDir:", sourceDir,
       "\n-- BuildDir:", buildDir,
       "\n-- Generator:", cmakeGenerator)
@@ -17,99 +17,89 @@ print("Test:", testFile,
 if os.path.exists(buildDir):
     shutil.rmtree(buildDir)
 
-proc = cmakelib.initProc(cmakeCommand)
+cmakelib.filterBase = sourceDir
 
 with open(testFile) as f:
     testData = json.loads(f.read())
 
-for obj in testData:
-    if 'sendRaw' in obj:
-        data = obj['sendRaw']
-        if debug: print("Sending raw:", data)
-        cmakelib.writeRawData(proc, data)
-    elif 'send' in obj:
-        data = obj['send']
-        if debug: print("Sending:", json.dumps(data))
-        cmakelib.writePayload(proc, data)
-    elif 'recv' in obj:
-        data = obj['recv']
-        if debug: print("Waiting for:", json.dumps(data))
-        cmakelib.waitForMessage(proc, data)
-    elif 'reply' in obj:
-        data = obj['reply']
-        if debug: print("Waiting for reply:", json.dumps(data))
-        originalType = ""
-        cookie = ""
-        skipProgress = False;
-        if 'cookie' in data: cookie = data['cookie']
-        if 'type' in data: originalType = data['type']
-        if 'skipProgress' in data: skipProgress = data['skipProgress']
-        cmakelib.waitForReply(proc, originalType, cookie, skipProgress)
-    elif 'error' in obj:
-        data = obj['error']
-        if debug: print("Waiting for error:", json.dumps(data))
-        originalType = ""
-        cookie = ""
-        message = ""
-        if 'cookie' in data: cookie = data['cookie']
-        if 'type' in data: originalType = data['type']
-        if 'message' in data: message = data['message']
-        cmakelib.waitForError(proc, originalType, cookie, message)
-    elif 'progress' in obj:
-        data = obj['progress']
-        if debug: print("Waiting for progress:", json.dumps(data))
-        originalType = ''
-        cookie = ""
-        current = 0
-        message = ""
-        if 'cookie' in data: cookie = data['cookie']
-        if 'type' in data: originalType = data['type']
-        if 'current' in data: current = data['current']
-        if 'message' in data: message = data['message']
-        cmakelib.waitForProgress(proc, originalType, cookie, current, message)
-    elif 'handshake' in obj:
-        data = obj['handshake']
-        if debug: print("Doing handshake:", json.dumps(data))
-        major = -1
-        minor = -1
-        generator = cmakeGenerator
-        extraGenerator = ''
-        sourceDirectory = sourceDir
-        buildDirectory = buildDir
-        if 'major' in data: major = data['major']
-        if 'minor' in data: minor = data['minor']
-        if 'buildDirectory' in data: buildDirectory = data['buildDirectory']
-        if 'sourceDirectory' in data: sourceDirectory = data['sourceDirectory']
-        if 'generator' in data: generator = data['generator']
-        if 'extraGenerator' in data: extraGenerator = data['extraGenerator']
-        if not os.path.isabs(buildDirectory):
-            buildDirectory = buildDir + "/" + buildDirectory
-        if sourceDirectory != '' and not os.path.isabs(sourceDirectory):
-            sourceDirectory = sourceDir + "/" + sourceDirectory
-        cmakelib.handshake(proc, major, minor, sourceDirectory, buildDirectory,
-          generator, extraGenerator)
-    elif 'validateGlobalSettings' in obj:
-        data = obj['validateGlobalSettings']
-        if not 'buildDirectory' in data: data['buildDirectory'] = buildDir
-        if not 'sourceDirectory' in data: data['sourceDirectory'] = sourceDir
-        if not 'generator' in data: data['generator'] = cmakeGenerator
-        if not 'extraGenerator' in data: data['extraGenerator'] = ''
-        cmakelib.validateGlobalSettings(proc, cmakeCommand, data)
-    elif 'validateCache' in obj:
-        data = obj['validateCache']
-        if not 'isEmpty' in data: data['isEmpty'] = false
-        cmakelib.validateCache(proc, data)
-    elif 'message' in obj:
-        print("MESSAGE:", obj["message"])
-    elif 'reconnect' in obj:
-        cmakelib.exitProc(proc)
-        proc = cmakelib.initProc(cmakeCommand)
-    else:
-        print("Unknown command:", json.dumps(obj))
-        sys.exit(2)
+for communicationMethod in cmakelib.communicationMethods:
+    proc = cmakelib.initServerProc(cmakeCommand, communicationMethod)
+    if proc is None:
+        continue
 
-    print("Completed")
+    for obj in testData:
+        if cmakelib.handleBasicMessage(proc, obj, debug):
+            pass
+        elif 'reply' in obj:
+            data = obj['reply']
+            if debug: print("Waiting for reply:", json.dumps(data))
+            originalType = ""
+            cookie = ""
+            skipProgress = False;
+            if 'cookie' in data: cookie = data['cookie']
+            if 'type' in data: originalType = data['type']
+            if 'skipProgress' in data: skipProgress = data['skipProgress']
+            cmakelib.waitForReply(proc, originalType, cookie, skipProgress)
+        elif 'error' in obj:
+            data = obj['error']
+            if debug: print("Waiting for error:", json.dumps(data))
+            originalType = ""
+            cookie = ""
+            message = ""
+            if 'cookie' in data: cookie = data['cookie']
+            if 'type' in data: originalType = data['type']
+            if 'message' in data: message = data['message']
+            cmakelib.waitForError(proc, originalType, cookie, message)
+        elif 'progress' in obj:
+            data = obj['progress']
+            if debug: print("Waiting for progress:", json.dumps(data))
+            originalType = ''
+            cookie = ""
+            current = 0
+            message = ""
+            if 'cookie' in data: cookie = data['cookie']
+            if 'type' in data: originalType = data['type']
+            if 'current' in data: current = data['current']
+            if 'message' in data: message = data['message']
+            cmakelib.waitForProgress(proc, originalType, cookie, current, message)
+        elif 'handshake' in obj:
+            data = obj['handshake']
+            if debug: print("Doing handshake:", json.dumps(data))
+            major = -1
+            minor = -1
+            generator = cmakeGenerator
+            extraGenerator = ''
+            sourceDirectory = sourceDir
+            buildDirectory = buildDir
+            if 'major' in data: major = data['major']
+            if 'minor' in data: minor = data['minor']
+            if 'buildDirectory' in data: buildDirectory = data['buildDirectory']
+            if 'sourceDirectory' in data: sourceDirectory = data['sourceDirectory']
+            if 'generator' in data: generator = data['generator']
+            if 'extraGenerator' in data: extraGenerator = data['extraGenerator']
 
-cmakelib.exitProc(proc)
-print('cmake-server exited: %d' % proc.returncode)
-sys.exit(proc.returncode)
+            if not os.path.isabs(buildDirectory):
+                buildDirectory = buildDir + "/" + buildDirectory
+            if sourceDirectory != '' and not os.path.isabs(sourceDirectory):
+                sourceDirectory = sourceDir + "/" + sourceDirectory
+            cmakelib.handshake(proc, major, minor, sourceDirectory, buildDirectory,
+                               generator, extraGenerator)
+        elif 'validateGlobalSettings' in obj:
+            data = obj['validateGlobalSettings']
+            if not 'buildDirectory' in data: data['buildDirectory'] = buildDir
+            if not 'sourceDirectory' in data: data['sourceDirectory'] = sourceDir
+            if not 'generator' in data: data['generator'] = cmakeGenerator
+            if not 'extraGenerator' in data: data['extraGenerator'] = ''
+            cmakelib.validateGlobalSettings(proc, cmakeCommand, data)
+        elif 'validateCache' in obj:
+            data = obj['validateCache']
+            if not 'isEmpty' in data: data['isEmpty'] = false
+            cmakelib.validateCache(proc, data)
+        elif 'reconnect' in obj:
+            cmakelib.exitProc(proc)
+            proc = cmakelib.initServerProc(cmakeCommand, communicationMethod)
+        else:
+            print("Unknown command:", json.dumps(obj))
+            sys.exit(2)
+    cmakelib.shutdownProc(proc)
+    print("Completed")