[PR #3345] fix more PY3.8 fails

File closes in msvs tool
Context manager in rpm tool
Dummy compiler, assembler scripts can be called in unexpected
  ways (namely by gcc tool init), which passes --version flag.
  don't take exception on the getopt call.
Try again to undo my breakage of _subproc, which was failing on Win+PY27

Signed-off-by: Mats Wichmann <mats@linux.com>

10 files changed, 65 insertions, 41 deletions

diff --git a/src/engine/SCons/Action.py b/src/engine/SCons/Action.py
index fd859ad..f1d4d28 100644
--- a/src/engine/SCons/Action.py
+++ b/src/engine/SCons/Action.py
@@ -784,10 +784,7 @@ def _subproc(scons_env, cmd, error = 'ignore', **kw):
             if DEVNULL:
                 kw[stream] = DEVNULL
             else:
-                if sys.platform == 'win32':
-                    kw[stream] = msvcrt.get_osfhandle(os.open(os.devnull, os.O_RDWR))
-                else:
-                    kw[stream] = open(os.devnull, "r+")
+                kw[stream] = open(os.devnull, "r+")
 
     # Figure out what shell environment to use
     ENV = kw.get('env', None)
diff --git a/src/engine/SCons/Platform/PlatformTests.py b/src/engine/SCons/Platform/PlatformTests.py
index ae070f8..c89610f 100644
--- a/src/engine/SCons/Platform/PlatformTests.py
+++ b/src/engine/SCons/Platform/PlatformTests.py
@@ -186,7 +186,8 @@ class TempFileMungeTestCase(unittest.TestCase):
         cmd = t(None, None, env, 0)
         # print("CMD is:%s"%cmd)
 
-        file_content = open(cmd[-1],'rb').read()
+        with open(cmd[-1],'rb') as f:
+            file_content = f.read()
         # print("Content is:[%s]"%file_content)
         # ...and restoring its setting.
         SCons.Action.print_actions = old_actions
diff --git a/src/engine/SCons/Tool/msvs.py b/src/engine/SCons/Tool/msvs.py
index 8ec33e6..1f55a04 100644
--- a/src/engine/SCons/Tool/msvs.py
+++ b/src/engine/SCons/Tool/msvs.py
@@ -695,6 +695,7 @@ class _GenerateV6DSP(_DSPGenerator):
         while line and line != '\n':
             line = dspfile.readline()
             datas = datas + line
+        dspfile.close()
 
         # OK, we've found our little pickled cache of data.
         try:
@@ -713,6 +714,7 @@ class _GenerateV6DSP(_DSPGenerator):
         while line and line != '\n':
             line = dspfile.readline()
             datas = datas + line
+        dspfile.close()
 
         # OK, we've found our little pickled cache of data.
         # it has a "# " in front of it, so we strip that.
@@ -1008,6 +1010,7 @@ class _GenerateV7DSP(_DSPGenerator, _GenerateV7User):
         while line and line != '\n':
             line = dspfile.readline()
             datas = datas + line
+        dspfile.close()
 
         # OK, we've found our little pickled cache of data.
         try:
@@ -1491,6 +1494,7 @@ class _GenerateV7DSW(_DSWGenerator):
         while line:
             line = dswfile.readline()
             datas = datas + line
+        dspfile.close()
 
         # OK, we've found our little pickled cache of data.
         try:
@@ -1738,6 +1742,7 @@ def GenerateProject(target, source, env):
             raise
 
         bdsp.write("This is just a placeholder file.\nThe real project file is here:\n%s\n" % dspfile.get_abspath())
+        bdsp.close()
 
     GenerateDSP(dspfile, source, env)
 
@@ -1754,6 +1759,7 @@ def GenerateProject(target, source, env):
                 raise
 
             bdsw.write("This is just a placeholder file.\nThe real workspace file is here:\n%s\n" % dswfile.get_abspath())
+            bdsw.close()
 
         GenerateDSW(dswfile, source, env)
 
diff --git a/src/engine/SCons/Tool/rpm.py b/src/engine/SCons/Tool/rpm.py
index 5198f84..2fd802a 100644
--- a/src/engine/SCons/Tool/rpm.py
+++ b/src/engine/SCons/Tool/rpm.py
@@ -51,43 +51,44 @@ def get_cmd(source, env):
     if SCons.Util.is_List(source):
         tar_file_with_included_specfile = source[0]
     return "%s %s %s"%(env['RPM'], env['RPMFLAGS'],
-                       tar_file_with_included_specfile.get_abspath() )
+                       tar_file_with_included_specfile.get_abspath())
 
 def build_rpm(target, source, env):
     # create a temporary rpm build root.
-    tmpdir = os.path.join( os.path.dirname( target[0].get_abspath() ), 'rpmtemp' )
+    tmpdir = os.path.join(os.path.dirname(target[0].get_abspath()), 'rpmtemp')
     if os.path.exists(tmpdir):
         shutil.rmtree(tmpdir)
 
     # now create the mandatory rpm directory structure.
     for d in ['RPMS', 'SRPMS', 'SPECS', 'BUILD']:
-        os.makedirs( os.path.join( tmpdir, d ) )
+        os.makedirs(os.path.join(tmpdir, d))
 
     # set the topdir as an rpmflag.
-    env.Prepend( RPMFLAGS = '--define \'_topdir %s\'' % tmpdir )
+    env.Prepend(RPMFLAGS = '--define \'_topdir %s\'' % tmpdir)
 
     # now call rpmbuild to create the rpm package.
     handle  = subprocess.Popen(get_cmd(source, env),
                                stdout=subprocess.PIPE,
                                stderr=subprocess.STDOUT,
                                shell=True)
-    output = SCons.Util.to_str(handle.stdout.read())
+    with handle.stdout:
+        output = SCons.Util.to_str(handle.stdout.read())
     status = handle.wait()
 
     if status:
-        raise SCons.Errors.BuildError( node=target[0],
-                                       errstr=output,
-                                       filename=str(target[0]) )
+        raise SCons.Errors.BuildError(node=target[0],
+                                      errstr=output,
+                                      filename=str(target[0]))
     else:
         # XXX: assume that LC_ALL=C is set while running rpmbuild
-        output_files = re.compile( 'Wrote: (.*)' ).findall( output )
+        output_files = re.compile('Wrote: (.*)').findall(output)
 
-        for output, input in zip( output_files, target ):
+        for output, input in zip(output_files, target):
             rpm_output = os.path.basename(output)
             expected   = os.path.basename(input.get_path())
 
             assert expected == rpm_output, "got %s but expected %s" % (rpm_output, expected)
-            shutil.copy( output, input.get_abspath() )
+            shutil.copy(output, input.get_abspath())
 
 
     # cleanup before leaving.
diff --git a/test/AS/fixture/myas.py b/test/AS/fixture/myas.py
index 4f5d9ac..3d05c79 100644
--- a/test/AS/fixture/myas.py
+++ b/test/AS/fixture/myas.py
@@ -23,11 +23,16 @@ if sys.platform == 'win32':
 
 else:
     import getopt
-    opts, args = getopt.getopt(sys.argv[1:], 'co:')
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], 'co:')
+    except getopt.GetoptError:
+        # we may be called with --version, just quit if so
+        sys.exit(0)
     for opt, arg in opts:
         if opt == '-o': out = arg
-    with open(args[0], 'rb') as ifp, open(out, 'wb') as ofp:
-        for l in ifp.readlines():
-            if l[:3] != b'#as':
-                ofp.write(l)
+    if args:
+        with open(args[0], 'rb') as ifp, open(out, 'wb') as ofp:
+            for l in ifp.readlines():
+                if l[:3] != b'#as':
+                    ofp.write(l)
     sys.exit(0)
diff --git a/test/Fortran/link-with-cxx.py b/test/Fortran/link-with-cxx.py
index f559b0a..22d9081 100644
--- a/test/Fortran/link-with-cxx.py
+++ b/test/Fortran/link-with-cxx.py
@@ -39,23 +39,23 @@ _python_ = TestSCons._python_
 test = TestSCons.TestSCons(match = TestSCons.match_re)
 
 
-test.write('test_linker.py', r"""
+test.write('test_linker.py', """\
 import sys
 if sys.argv[1] == '-o':
-    ofp = open(sys.argv[2], 'wb')
-    infiles = sys.argv[3:]
+    with open(sys.argv[2], 'wb') as ofp:
+        for infile in sys.argv[3:]:
+            with open(infile, 'rb') as ifp:
+                ofp.write(ifp.read())
 elif sys.argv[1][:5] == '/OUT:':
-    ofp = open(sys.argv[1][5:], 'wb')
-    infiles = sys.argv[2:]
-for infile in infiles:
-    with open(infile, 'rb') as ifp:
-        ofp.write(ifp.read())
-ofp.close()
+    with open(sys.argv[1][5:], 'wb') as ofp:
+        for infile in sys.argv[2:]:
+            with open(infile, 'rb') as ifp:
+                ofp.write(ifp.read())
 sys.exit(0)
 """)
 
 
-test.write('test_fortran.py', r"""
+test.write('test_fortran.py', """\
 import sys
 with open(sys.argv[2], 'wb') as ofp:
     for infile in sys.argv[4:]:
diff --git a/test/Parallel/failed-build.py b/test/Parallel/failed-build.py
index bfdc01d..12a8f04 100644
--- a/test/Parallel/failed-build.py
+++ b/test/Parallel/failed-build.py
@@ -77,7 +77,14 @@ test.write('mycopy.py', r"""\
 import os
 import sys
 import time
-os.mkdir('mycopy.started')
+try:
+    os.makedirs('mycopy.started', exist_ok=True)
+except TypeError:  # Python 2 has no exist_ok
+    try:
+        os.mkdir('mycopy.started')
+    except FileExistsError:
+        pass
+
 with open(sys.argv[1], 'wb') as ofp, open(sys.argv[2], 'rb') as ifp:
     ofp.write(ifp.read())
 for i in [1, 2, 3, 4, 5]:
diff --git a/test/gnutools.py b/test/gnutools.py
index bd8e366..040de2a 100644
--- a/test/gnutools.py
+++ b/test/gnutools.py
@@ -45,7 +45,11 @@ test.subdir('gnutools')
 test.write(['gnutools','mygcc.py'], """
 import getopt
 import sys
-cmd_opts, args = getopt.getopt(sys.argv[1:], 'f:s:co:', [])
+try:
+    cmd_opts, args = getopt.getopt(sys.argv[1:], 'f:s:co:', [])
+except getopt.GetoptError:
+    # we may be called with --version, just quit if so
+    sys.exit(0)
 out = None
 opt_string = ''
 for opt, arg in cmd_opts:
@@ -62,7 +66,11 @@ sys.exit(0)
 test.write(['gnutools','myg++.py'], """
 import getopt
 import sys
-cmd_opts, args = getopt.getopt(sys.argv[1:], 'f:s:co:', [])
+try:
+    cmd_opts, args = getopt.getopt(sys.argv[1:], 'f:s:co:', [])
+except getopt.GetoptError:
+    # we may be called with --version, just quit if so
+    sys.exit(0)
 out = None
 opt_string = ''
 for opt, arg in cmd_opts:
diff --git a/test/leaky-handles.py b/test/leaky-handles.py
index 3787ee2..e64053c 100644
--- a/test/leaky-handles.py
+++ b/test/leaky-handles.py
@@ -41,11 +41,10 @@ if os.name != 'posix' or sys.platform == 'darwin':
 
 test.write('SConstruct', """
 
-#Leak a file handle
-open('/dev/null')
-
-#Check it gets closed
-test2 = Command('test2', [], '@ls /proc/$$$$/fd|wc -l')
+# Leave a file handle open when invoking a command
+with open('/dev/null'):
+    # Check it gets closed when an external command is forked off:
+    test2 = Command('test2', [], '@ls /proc/$$$$/fd|wc -l')
 """)
 
 # In theory that should have 3 lines (handles 0/1/2). This is v. unix specific
diff --git a/test/packaging/rpm/explicit-target.py b/test/packaging/rpm/explicit-target.py
index e8fbd39..553ce27 100644
--- a/test/packaging/rpm/explicit-target.py
+++ b/test/packaging/rpm/explicit-target.py
@@ -48,7 +48,7 @@ test.subdir('src')
 mainpath = os.path.join('src', 'main.c')
 test.file_fixture(mainpath, mainpath)
 
-test.write('SConstruct', """
+test.write('SConstruct', """\
 import os
 
 env=Environment(tools=['default', 'packaging'])
@@ -77,7 +77,7 @@ env.Package( NAME           = 'foo',
 
 expect = """
 scons: *** Setting target is not supported for rpm.
-""" + test.python_file_line(test.workpath('SConstruct'), 24)
+""" + test.python_file_line(test.workpath('SConstruct'), 12)
 
 test.run(arguments='', status=2, stderr=expect)