Avoid hangs on POSIX systems when reading a lot of output from a piped commnd. (Christoph Wiedemann)

2 files changed, 74 insertions, 35 deletions

diff --git a/src/engine/SCons/ActionTests.py b/src/engine/SCons/ActionTests.py
index f980fb7..d14edfd 100644
--- a/src/engine/SCons/ActionTests.py
+++ b/src/engine/SCons/ActionTests.py
@@ -63,8 +63,16 @@ except:
     pass
 f.close()
 if os.environ.has_key( 'ACTPY_PIPE' ):
-    sys.stdout.write( 'act.py: stdout: executed act.py\\n' )
-    sys.stderr.write( 'act.py: stderr: executed act.py\\n' ) 
+    if os.environ.has_key( 'PIPE_STDOUT_MSG' ):
+         stdout_msg = os.environ['PIPE_STDOUT_MSG']
+    else:
+         stdout_msg = "act.py: stdout: executed act.py\\n"
+    sys.stdout.write( stdout_msg )
+    if os.environ.has_key( 'PIPE_STDERR_MSG' ):
+         stderr_msg = os.environ['PIPE_STDERR_MSG']
+    else:
+         stderr_msg = "act.py: stderr: executed act.py\\n"
+    sys.stderr.write( stderr_msg ) 
 sys.exit(0)
 """)
 
@@ -528,39 +536,59 @@ class CommandActionTestCase(unittest.TestCase):
         self.test_execute()
         self.env['PSTDOUT'].close()
         pipe_out = test.read( pipe_file )
+
         if sys.platform == 'win32':
             cr = '\r'
         else:
             cr = ''
-        act_out = "act.py: stdout: executed act.py"
-        act_err = "act.py: stderr: executed act.py"
-        found = re.findall( "%s%s\n%s%s\n" % (act_out, cr, act_err, cr),
-                            pipe_out )
-        assert len(found) == 8, found
+        act_out = "act.py: stdout: executed act.py\n"
+        act_err = "act.py: stderr: executed act.py\n"
+
+        # Since we are now using select(), stdout and stderr can be
+        # intermixed, so count the lines separately.
+        outlines = re.findall(act_out + cr, pipe_out)
+        errlines = re.findall(act_err + cr, pipe_out)
+        assert len(outlines) == 8, outlines
+        assert len(errlines) == 8, errlines
 
         # test redirection operators
-        def test_redirect(self, redir):
+        def test_redirect(self, redir, stdout_msg, stderr_msg):
             cmd = r'%s %s %s xyzzy %s' % (python, act_py, outfile, redir)
             pipe = open( pipe_file, "w" )
             act = SCons.Action.CommandAction(cmd)
-            r = act([], [], self.env.Copy(PSTDOUT = pipe, PSTDERR = pipe))
+            env = Environment( ENV = {'ACTPY_PIPE' : '1',
+                                      'PIPE_STDOUT_MSG' : stdout_msg,
+                                      'PIPE_STDERR_MSG' : stderr_msg},
+                               PIPE_BUILD = 1,
+                               PSTDOUT = pipe, PSTDERR = pipe )
+            r = act([], [], env)
             pipe.close()
             assert r == 0
             return (test.read(outfile2, 'r'), test.read(pipe_file, 'r'))
-        
-        (redirected, pipe_out) = test_redirect(self,'> %s' % outfile2 )
-        assert redirected == "%s\n" % act_out
-        assert pipe_out == "%s\n" % act_err
-
-        (redirected, pipe_out) = test_redirect(self,'2> %s' % outfile2 )
-        assert redirected == "%s\n" % act_err
-        assert pipe_out == "%s\n" % act_out
-
-        (redirected, pipe_out) = test_redirect(self,'> %s 2>&1' % outfile2 )
-        assert (redirected == "%s\n%s\n" % (act_out, act_err) or
-                redirected == "%s\n%s\n" % (act_err, act_out))
-        assert pipe_out == "" 
-        
+
+        (redirected, pipe_out) = test_redirect(self,'> %s' % outfile2,
+                                               act_out, act_err)
+        assert redirected == act_out
+        assert pipe_out == act_err
+
+        (redirected, pipe_out) = test_redirect(self,'2> %s' % outfile2,
+                                               act_out, act_err)
+        assert redirected == act_err
+        assert pipe_out == act_out
+
+        (redirected, pipe_out) = test_redirect(self,'> %s 2>&1' % outfile2,
+                                               act_out, act_err)
+        assert (redirected == act_out + act_err or
+                redirected == act_err + act_out)
+        assert pipe_out == ""
+
+        act_err = "Long Command Output\n"*3000
+        # the size of the string should exceed the system's default block size
+        act_out = ""
+        (redirected, pipe_out) = test_redirect(self,'> %s' % outfile2,
+                                               act_out, act_err)
+        assert (redirected == act_out)
+        assert (pipe_out == act_err)
 
     def test_set_handler(self):
         """Test setting the command handler...
diff --git a/src/engine/SCons/Platform/posix.py b/src/engine/SCons/Platform/posix.py
index 3546a17..9b7e11c 100644
--- a/src/engine/SCons/Platform/posix.py
+++ b/src/engine/SCons/Platform/posix.py
@@ -37,6 +37,7 @@ import os.path
 import popen2
 import string
 import sys
+import select
 
 import SCons.Util
 
@@ -93,18 +94,33 @@ def fork_spawn(sh, escape, cmd, args, env):
             return stat | 0x80
         return stat >> 8
 
+def process_cmd_output(cmd_stdout, cmd_stderr, stdout, stderr):
+    stdout_eof = stderr_eof = 0
+    while not (stdout_eof and stderr_eof):
+        (i,o,e) = select.select([cmd_stdout, cmd_stderr], [], [])
+        if cmd_stdout in i:
+            str = cmd_stdout.read()
+            if len(str) == 0:
+                stdout_eof = 1
+            elif stdout != None:
+                stdout.write(str)
+        if cmd_stderr in i:
+            str = cmd_stderr.read()
+            if len(str) == 0:
+                #sys.__stderr__.write( "stderr_eof=1\n" )
+                stderr_eof = 1
+            else:
+                #sys.__stderr__.write( "str(stderr) = %s\n" % str )
+                stderr.write(str)
+    
+
 def piped_env_spawn(sh, escape, cmd, args, env, stdout, stderr):
     # spawn using Popen3 combined with the env command
     # the command name and the command's stdout is written to stdout
     # the command's stderr is written to stderr
     s = _get_env_command( sh, escape, cmd, args, env)
     proc = popen2.Popen3(s, 1)
-    # process stdout
-    if stdout != None:
-        stdout.write(proc.fromchild.read())
-    # process stderr
-    if stderr != None:
-        stderr.write(proc.childerr.read())
+    process_cmd_output(proc.fromchild, proc.childerr, stdout, stderr)
     stat = proc.wait()
     if stat & 0xff:
         return stat | 0x80
@@ -151,12 +167,7 @@ def piped_fork_spawn(sh, escape, cmd, args, env, stdout, stderr):
             childErr = os.fdopen( rFdErr )
         else:
             childErr = childOut
-        # process stdout
-        if stdout != None:
-            stdout.write( childOut.read() )
-        # process stderr
-        if stderr != None:
-            stderr.write( childErr.read() )
+        process_cmd_output(childOut, childErr, stdout, stderr)
         os.close( rFdOut )
         if stdout != stderr:
             os.close( rFdErr )