[WIP] [PY 3.8] fix more warnings

Several locations with simple usage of deprecated "imp" module
changed to use "importlib". These match with work in #3159,
but this is not a complete implementation of #3159.

More regex patterns are changed to be raw strings.

Some strings which did not seem appropriate to change to raw
strings (e.g. contain embedded tabs, which Python should honor)
had backslashes escaped to avoid accidental Python interpretation.
Example:
   '\t<Import Project="$(VCTargetsPath)\\Microsoft.Cpp.targets" />\n'
Python 3.8 was Warning \M was an unknown escape.

More open().write(), open().read() style usage changed to use
context managers so the file is closed.

WIP part: even with Python 3.7, the tests which call sconsign.py
fail; oddly they do not fail without the patch to compat.py.
sconsign.py does an import using imp module (which is what
generates the errors) so needs to be updated anyway.  It does not
quite fit the "simple usage" pattern - can't do a simple relative
import since sconsign is normally located elsewhere in the tree than
the main scons code body.

With this version of the patch, 700 tests now pass with 3.8, and
Warning messages reduced to 2800 (current master has 200 pass,
9000 warns)

Signed-off-by: Mats Wichmann <mats@linux.com>

5 files changed, 77 insertions, 58 deletions

diff --git a/testing/framework/TestCmd.py b/testing/framework/TestCmd.py
index 24f2350..e24371c 100644
--- a/testing/framework/TestCmd.py
+++ b/testing/framework/TestCmd.py
@@ -370,7 +370,7 @@ testprefix = 'testcmd.'
 if os.name in ('posix', 'nt'):
     testprefix += "%s." % str(os.getpid())
 
-re_space = re.compile('\s')
+re_space = re.compile(r'\s')
 
 
 def _caller(tblist, skip):
diff --git a/testing/framework/TestCmdTests.py b/testing/framework/TestCmdTests.py
index b9226fd..bb9e3f9 100644
--- a/testing/framework/TestCmdTests.py
+++ b/testing/framework/TestCmdTests.py
@@ -280,8 +280,10 @@ class chmod_TestCase(TestCmdTestCase):
         wdir_file1 = os.path.join(test.workdir, 'file1')
         wdir_sub_file2 = os.path.join(test.workdir, 'sub', 'file2')
 
-        open(wdir_file1, 'w').write("")
-        open(wdir_sub_file2, 'w').write("")
+        with open(wdir_file1, 'w') as f:
+            f.write("")
+        with open(wdir_sub_file2, 'w') as f:
+            f.write("")
 
         if sys.platform == 'win32':
 
@@ -1576,11 +1578,16 @@ class read_TestCase(TestCmdTestCase):
         wdir_file4 = os.path.join(test.workdir, 'file4')
         wdir_file5 = os.path.join(test.workdir, 'file5')
 
-        open(wdir_file1, 'wb').write("")
-        open(wdir_file2, 'wb').write("Test\nfile\n#2.\n")
-        open(wdir_foo_file3, 'wb').write("Test\nfile\n#3.\n")
-        open(wdir_file4, 'wb').write("Test\nfile\n#4.\n")
-        open(wdir_file5, 'wb').write("Test\r\nfile\r\n#5.\r\n")
+        with open(wdir_file1, 'wb'):
+            f.write("")
+        with open(wdir_file2, 'wb'):
+            f.write("Test\nfile\n#2.\n")
+        with open(wdir_foo_file3, 'wb'):
+            f.write("Test\nfile\n#3.\n")
+        with open(wdir_file4, 'wb'):
+            f.write("Test\nfile\n#4.\n")
+        with open(wdir_file5, 'wb'):
+            f.write("Test\r\nfile\r\n#5.\r\n")
 
         try:
             contents = test.read('no_file')
@@ -2343,16 +2350,15 @@ sys.stderr = Unbuffered(sys.stderr)
 
 sys.stdout.write('script_recv:  STDOUT\\n')
 sys.stderr.write('script_recv:  STDERR\\n')
-logfp = open(r'%s', 'wb')
-while 1:
-    line = sys.stdin.readline()
-    if not line:
-        break
-    logfp.write('script_recv:  ' + line)
-    sys.stdout.write('script_recv:  STDOUT:  ' + line)
-    sys.stderr.write('script_recv:  STDERR:  ' + line)
-logfp.close()
-        """ % t.recv_out_path
+with open(r'%s', 'wb') as logfp:
+    while 1:
+        line = sys.stdin.readline()
+        if not line:
+            break
+        logfp.write('script_recv:  ' + line)
+        sys.stdout.write('script_recv:  STDOUT:  ' + line)
+        sys.stderr.write('script_recv:  STDERR:  ' + line)
+""" % t.recv_out_path
         t.run_env.write(t.recv_script_path, text)
         os.chmod(t.recv_script_path, 0o644)  # XXX UNIX-specific
         return t
@@ -2621,7 +2627,8 @@ script_recv:  STDERR:  input
             p.stdin.write(input)
             p.stdin.close()
             p.wait()
-            result = open(t.recv_out_path, 'rb').read()
+            with open(t.recv_out_path, 'rb') as f:
+                result = f.read()
             expect = 'script_recv:  ' + input
             assert result == expect, repr(result)
 
@@ -2630,7 +2637,8 @@ script_recv:  STDERR:  input
             p.send(input)
             p.stdin.close()
             p.wait()
-            result = open(t.recv_out_path, 'rb').read()
+            with open(t.recv_out_path, 'rb') as f:
+                result = f.read()
             expect = 'script_recv:  ' + input
             assert result == expect, repr(result)
 
@@ -2689,7 +2697,8 @@ script_recv:  STDERR:  input to the receive script
 """
             assert stdout == expect_stdout, stdout
             assert stderr == expect_stderr, stderr
-            result = open(t.recv_out_path, 'rb').read()
+            with open(t.recv_out_path, 'rb') as f:
+                result = f.read()
             expect = ('script_recv:  ' + input) * 2
             assert result == expect, (result, stdout, stderr)
 
@@ -2807,15 +2816,18 @@ class symlink_TestCase(TestCmdTestCase):
         test.symlink('target1', 'file1')
         assert os.path.islink(wdir_file1)
         assert not os.path.exists(wdir_file1)
-        open(wdir_target1, 'w').write("")
+        with open(wdir_target1, 'w'):
+            f.write("")
         assert os.path.exists(wdir_file1)
 
         test.symlink('target2', ['foo', 'file2'])
         assert os.path.islink(wdir_foo_file2)
         assert not os.path.exists(wdir_foo_file2)
-        open(wdir_target2, 'w').write("")
+        with open(wdir_target2, 'w'):
+            f.write("")
         assert not os.path.exists(wdir_foo_file2)
-        open(wdir_foo_target2, 'w').write("")
+        with open(wdir_foo_target2, 'w'):
+            f.write("")
         assert os.path.exists(wdir_foo_file2)
 
 
@@ -2947,12 +2959,18 @@ class unlink_TestCase(TestCmdTestCase):
         wdir_foo_file4 = os.path.join(test.workdir, 'foo', 'file4')
         wdir_file5 = os.path.join(test.workdir, 'file5')
 
-        open(wdir_file1, 'w').write("")
-        open(wdir_file2, 'w').write("")
-        open(wdir_foo_file3a, 'w').write("")
-        open(wdir_foo_file3b, 'w').write("")
-        open(wdir_foo_file4, 'w').write("")
-        open(wdir_file5, 'w').write("")
+        with open(wdir_file1, 'w'):
+            f.write("")
+        with open(wdir_file2, 'w'):
+            f.write("")
+        with open(wdir_foo_file3a, 'w'):
+            f.write("")
+        with open(wdir_foo_file3b, 'w'):
+            f.write("")
+        with open(wdir_foo_file4, 'w'):
+            f.write("")
+        with open(wdir_file5, 'w'):
+            f.write("")
 
         try:
             contents = test.unlink('no_file')
@@ -2981,20 +2999,17 @@ class unlink_TestCase(TestCmdTestCase):
         # For Windows, open the file.
         os.chmod(test.workdir, 0o500)
         os.chmod(wdir_file5, 0o400)
-        f = open(wdir_file5, 'r')
-
-        try:
+        with open(wdir_file5, 'r') as f:
             try:
-                test.unlink('file5')
-            except OSError: # expect "Permission denied"
-                pass
-            except:
-                raise
-        finally:
-            os.chmod(test.workdir, 0o700)
-            os.chmod(wdir_file5, 0o600)
-            f.close()
-
+                try:
+                    test.unlink('file5')
+                except OSError: # expect "Permission denied"
+                    pass
+                except:
+                    raise
+            finally:
+                os.chmod(test.workdir, 0o700)
+                os.chmod(wdir_file5, 0o600)
 
 
 class touch_TestCase(TestCmdTestCase):
@@ -3005,8 +3020,10 @@ class touch_TestCase(TestCmdTestCase):
         wdir_file1 = os.path.join(test.workdir, 'file1')
         wdir_sub_file2 = os.path.join(test.workdir, 'sub', 'file2')
 
-        open(wdir_file1, 'w').write("")
-        open(wdir_sub_file2, 'w').write("")
+        with open(wdir_file1, 'w'):
+            f.write("")
+        with open(wdir_sub_file2, 'w'):
+            f.write("")
 
         file1_old_time = os.path.getmtime(wdir_file1)
         file2_old_time = os.path.getmtime(wdir_sub_file2)
@@ -3312,9 +3329,10 @@ class write_TestCase(TestCmdTestCase):
         if os.name != "nt":
             assert not os.path.exists(test.workpath('file10'))
 
-        assert open(test.workpath('file8'), 'r').read() == "Test file #8.\n"
-        assert open(test.workpath('file9'), 'rb').read() == "Test file #9.\r\n"
-
+        with open(test.workpath('file8'), 'r') as f:
+            assert f.read() == "Test file #8.\n"
+        with open(test.workpath('file9'), 'rb') as f:
+        assert f.read() == "Test file #9.\r\n"
 
 
 class variables_TestCase(TestCmdTestCase):
diff --git a/testing/framework/TestRuntest.py b/testing/framework/TestRuntest.py
index faf5d60..a752161 100644
--- a/testing/framework/TestRuntest.py
+++ b/testing/framework/TestRuntest.py
@@ -30,7 +30,7 @@ __all__.extend([ 'TestRuntest',
                  'pythonflags',
                ])
 
-if re.search('\s', python):
+if re.search(r'\s', python):
     pythonstring = _python_
 else:
     pythonstring = python
diff --git a/testing/framework/TestSConsMSVS.py b/testing/framework/TestSConsMSVS.py
index b975ebe..4c99bd4 100644
--- a/testing/framework/TestSConsMSVS.py
+++ b/testing/framework/TestSConsMSVS.py
@@ -773,7 +773,7 @@ expected_vcprojfile_10_0 = """\
 \t\t<NMakeForcedUsingAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(NMakeForcedUsingAssemblies)</NMakeForcedUsingAssemblies>
 \t</PropertyGroup>
 \t<ItemGroup>
-\t\t<ClInclude Include="sdk_dir\sdk.h" />
+\t\t<ClInclude Include="sdk_dir\\sdk.h" />
 \t</ItemGroup>
 \t<ItemGroup>
 \t\t<ClInclude Include="test.h" />
@@ -838,7 +838,7 @@ expected_vcprojfile_11_0 = """\
 \t\t<NMakeForcedUsingAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(NMakeForcedUsingAssemblies)</NMakeForcedUsingAssemblies>
 \t</PropertyGroup>
 \t<ItemGroup>
-\t\t<ClInclude Include="sdk_dir\sdk.h" />
+\t\t<ClInclude Include="sdk_dir\\sdk.h" />
 \t</ItemGroup>
 \t<ItemGroup>
 \t\t<ClInclude Include="test.h" />
@@ -903,7 +903,7 @@ expected_vcprojfile_14_0 = """\
 \t\t<NMakeForcedUsingAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(NMakeForcedUsingAssemblies)</NMakeForcedUsingAssemblies>
 \t</PropertyGroup>
 \t<ItemGroup>
-\t\t<ClInclude Include="sdk_dir\sdk.h" />
+\t\t<ClInclude Include="sdk_dir\\sdk.h" />
 \t</ItemGroup>
 \t<ItemGroup>
 \t\t<ClInclude Include="test.h" />
@@ -968,7 +968,7 @@ expected_vcprojfile_14_1 = """\
 \t\t<NMakeForcedUsingAssemblies Condition="'$(Configuration)|$(Platform)'=='Release|Win32'">$(NMakeForcedUsingAssemblies)</NMakeForcedUsingAssemblies>
 \t</PropertyGroup>
 \t<ItemGroup>
-\t\t<ClInclude Include="sdk_dir\sdk.h" />
+\t\t<ClInclude Include="sdk_dir\\sdk.h" />
 \t</ItemGroup>
 \t<ItemGroup>
 \t\t<ClInclude Include="test.h" />
@@ -1045,7 +1045,7 @@ env=Environment(platform='win32', tools=['msvs'], MSVS_VERSION='10.0',
                 HOST_ARCH='%(HOST_ARCH)s')
 
 testsrc = ['test1.cpp', 'test2.cpp']
-testincs = ['sdk_dir\sdk.h']
+testincs = ['sdk_dir\\sdk.h']
 testlocalincs = ['test.h']
 testresources = ['test.rc']
 testmisc = ['readme.txt']
@@ -1068,7 +1068,7 @@ env=Environment(platform='win32', tools=['msvs'], MSVS_VERSION='11.0',
                 HOST_ARCH='%(HOST_ARCH)s')
 
 testsrc = ['test1.cpp', 'test2.cpp']
-testincs = ['sdk_dir\sdk.h']
+testincs = ['sdk_dir\\sdk.h']
 testlocalincs = ['test.h']
 testresources = ['test.rc']
 testmisc = ['readme.txt']
@@ -1091,7 +1091,7 @@ env=Environment(platform='win32', tools=['msvs'], MSVS_VERSION='14.0',
                 HOST_ARCH='%(HOST_ARCH)s')
 
 testsrc = ['test1.cpp', 'test2.cpp']
-testincs = ['sdk_dir\sdk.h']
+testincs = ['sdk_dir\\sdk.h']
 testlocalincs = ['test.h']
 testresources = ['test.rc']
 testmisc = ['readme.txt']
@@ -1114,7 +1114,7 @@ env=Environment(platform='win32', tools=['msvs'], MSVS_VERSION='14.1',
                 HOST_ARCH='%(HOST_ARCH)s')
 
 testsrc = ['test1.cpp', 'test2.cpp']
-testincs = ['sdk_dir\sdk.h']
+testincs = ['sdk_dir\\sdk.h']
 testlocalincs = ['test.h']
 testresources = ['test.rc']
 testmisc = ['readme.txt']
diff --git a/testing/framework/TestSCons_time.py b/testing/framework/TestSCons_time.py
index 55248a4..8363e8d 100644
--- a/testing/framework/TestSCons_time.py
+++ b/testing/framework/TestSCons_time.py
@@ -272,7 +272,8 @@ class TestSCons_time(TestCommon):
             d, f = os.path.split(path)
             if not os.path.isdir(d):
                 os.makedirs(d)
-            open(path, 'w').write(content)
+            with open(path, 'w') as f:
+                f.write(content)
         return dir
 
     def write_sample_tarfile(self, archive, dir, files):