Merge pull request #3327 from mwichmann/open-and-regex-cleanups

Clean up some file opens, regex strings

16 files changed, 70 insertions, 52 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 6d8125f..eef7313 100755
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -34,6 +34,7 @@ RELEASE 3.0.5.alpha.yyyymmdd - NEW DATE WILL BE INSERTED HERE
     - Add the textfile tool to the default tool list
     - Fix syntax on is/is not clauses: should not use with a literal
     - Properly retrieve exit code when catching SystemExit
+    - Fix regex patterns that were not specified as raw strings
 
   From Bernhard M. Wiedemann:
     - Do not store build host+user name if reproducible builds are wanted
diff --git a/src/engine/SCons/BuilderTests.py b/src/engine/SCons/BuilderTests.py
index c9068ed..847e30a 100644
--- a/src/engine/SCons/BuilderTests.py
+++ b/src/engine/SCons/BuilderTests.py
@@ -680,7 +680,9 @@ class BuilderTestCase(unittest.TestCase):
     def test_single_source(self):
         """Test Builder with single_source flag set"""
         def func(target, source, env):
-            open(str(target[0]), "w")   # TODO: this just a throwaway?
+            """create the file"""
+            with open(str(target[0]), "w"):
+                pass
             if (len(source) == 1 and len(target) == 1):
                 env['CNT'][0] = env['CNT'][0] + 1
                 
diff --git a/src/engine/SCons/Defaults.py b/src/engine/SCons/Defaults.py
index 9ca9ccd..479ef7e 100644
--- a/src/engine/SCons/Defaults.py
+++ b/src/engine/SCons/Defaults.py
@@ -333,8 +333,8 @@ def touch_func(dest):
         if os.path.exists(file):
             atime = os.path.getatime(file)
         else:
-            open(file, 'w')
-            atime = mtime
+            with open(file, 'w'):
+                atime = mtime
         os.utime(file, (atime, mtime))
 
 Touch = ActionFactory(touch_func,
diff --git a/src/engine/SCons/Environment.py b/src/engine/SCons/Environment.py
index eea8aed..88fbc3b 100644
--- a/src/engine/SCons/Environment.py
+++ b/src/engine/SCons/Environment.py
@@ -1568,12 +1568,12 @@ class Base(SubstitutionEnvironment):
         """
         filename = self.subst(filename)
         try:
-            fp = open(filename, 'r')
+            with open(filename, 'r') as fp:
+                lines = SCons.Util.LogicalLines(fp).readlines()
         except IOError:
             if must_exist:
                 raise
             return
-        lines = SCons.Util.LogicalLines(fp).readlines()
         lines = [l for l in lines if l[0] != '#']
         tdlist = []
         for line in lines:
diff --git a/src/engine/SCons/Node/NodeTests.py b/src/engine/SCons/Node/NodeTests.py
index 7dc5f5d..7d347ac 100644
--- a/src/engine/SCons/Node/NodeTests.py
+++ b/src/engine/SCons/Node/NodeTests.py
@@ -1335,9 +1335,9 @@ class NodeListTestCase(unittest.TestCase):
         assert s == "['n3', 'n2', 'n1']", s
 
         r = repr(nl)
-        r = re.sub('at (0[xX])?[0-9a-fA-F]+', 'at 0x', r)
+        r = re.sub(r'at (0[xX])?[0-9a-fA-F]+', 'at 0x', r)
         # Don't care about ancestry: just leaf value of MyNode
-        r = re.sub('<.*?\.MyNode', '<MyNode', r)
+        r = re.sub(r'<.*?\.MyNode', '<MyNode', r)
         # New-style classes report as "object"; classic classes report
         # as "instance"...
         r = re.sub("object", "instance", r)
diff --git a/src/engine/SCons/SConsign.py b/src/engine/SCons/SConsign.py
index 5042aa1..dfafdc9 100644
--- a/src/engine/SCons/SConsign.py
+++ b/src/engine/SCons/SConsign.py
@@ -394,7 +394,8 @@ class DirFile(Dir):
                 # here, or in any of the following calls, would get
                 # raised, indicating something like a potentially
                 # serious disk or network issue.
-                open(self.sconsign, 'wb').write(open(fname, 'rb').read())
+                with open(self.sconsign, 'wb') as f, open(fname, 'rb') as f2:
+                    f.write(f2.read())
                 os.chmod(self.sconsign, mode)
         try:
             os.unlink(temp)
diff --git a/src/engine/SCons/Scanner/LaTeXTests.py b/src/engine/SCons/Scanner/LaTeXTests.py
index 0114d45..6dd7dac 100644
--- a/src/engine/SCons/Scanner/LaTeXTests.py
+++ b/src/engine/SCons/Scanner/LaTeXTests.py
@@ -37,7 +37,7 @@ import SCons.Scanner.LaTeX
 
 test = TestCmd.TestCmd(workdir = '')
 
-test.write('test1.latex',"""
+test.write('test1.latex',r"""
 \include{inc1}
 \input{inc2}
 include{incNO}
@@ -51,12 +51,12 @@ xyzzy \include{inc6}
 \subinputfrom{subdir}{inc3e}
 """)
 
-test.write('test2.latex',"""
+test.write('test2.latex',r"""
 \include{inc1}
 \include{inc3}
 """)
 
-test.write('test3.latex',"""
+test.write('test3.latex',r"""
 \includegraphics{inc4.eps}
 \includegraphics[width=60mm]{inc5.xyz}
 """)
diff --git a/src/engine/SCons/Scanner/ScannerTests.py b/src/engine/SCons/Scanner/ScannerTests.py
index 5cdd5b1..64a2345 100644
--- a/src/engine/SCons/Scanner/ScannerTests.py
+++ b/src/engine/SCons/Scanner/ScannerTests.py
@@ -461,7 +461,7 @@ class ClassicTestCase(unittest.TestCase):
     def test_find_include(self):
         """Test the Scanner.Classic find_include() method"""
         env = DummyEnvironment()
-        s = SCons.Scanner.Classic("t", ['.suf'], 'MYPATH', '^my_inc (\S+)')
+        s = SCons.Scanner.Classic("t", ['.suf'], 'MYPATH', r'^my_inc (\S+)')
 
         def _find_file(filename, paths):
             return paths[0]+'/'+filename
@@ -479,7 +479,7 @@ class ClassicTestCase(unittest.TestCase):
 
     def test_name(self):
         """Test setting the Scanner.Classic name"""
-        s = SCons.Scanner.Classic("my_name", ['.s'], 'MYPATH', '^my_inc (\S+)')
+        s = SCons.Scanner.Classic("my_name", ['.s'], 'MYPATH', r'^my_inc (\S+)')
         assert s.name == "my_name", s.name
 
     def test_scan(self):
@@ -505,7 +505,7 @@ class ClassicTestCase(unittest.TestCase):
                 return include, include
 
         env = DummyEnvironment()
-        s = MyScanner("t", ['.suf'], 'MYPATH', '^my_inc (\S+)')
+        s = MyScanner("t", ['.suf'], 'MYPATH', r'^my_inc (\S+)')
 
         # This set of tests is intended to test the scanning operation
         # of the Classic scanner.
diff --git a/src/engine/SCons/Script/SConscript.py b/src/engine/SCons/Script/SConscript.py
index 560402c..eabaddb 100644
--- a/src/engine/SCons/Script/SConscript.py
+++ b/src/engine/SCons/Script/SConscript.py
@@ -278,11 +278,12 @@ def _SConscript(fs, *files, **kw):
                         pass
                     try:
                         try:
-#                            _file_ = SCons.Util.to_str(_file_)
                             if Main.print_time:
                                 time1 = time.time()
-                            exec(compile(_file_.read(), _file_.name, 'exec'),
-                                 call_stack[-1].globals)
+                            scriptdata = _file_.read()
+                            scriptname = _file_.name
+                            _file_.close()
+                            exec(compile(scriptdata, scriptname, 'exec'), call_stack[-1].globals)
                         except SConscriptReturn:
                             pass
                     finally:
@@ -397,9 +398,9 @@ class SConsEnvironment(SCons.Environment.Base):
         something like 3.2b1."""
         version = version_string.split(' ')[0].split('.')
         v_major = int(version[0])
-        v_minor = int(re.match('\d+', version[1]).group())
+        v_minor = int(re.match(r'\d+', version[1]).group())
         if len(version) >= 3:
-            v_revision = int(re.match('\d+', version[2]).group())
+            v_revision = int(re.match(r'\d+', version[2]).group())
         else:
             v_revision = 0
         return v_major, v_minor, v_revision
diff --git a/src/engine/SCons/Tool/JavaCommon.py b/src/engine/SCons/Tool/JavaCommon.py
index 0d4c95a..853f7f2 100644
--- a/src/engine/SCons/Tool/JavaCommon.py
+++ b/src/engine/SCons/Tool/JavaCommon.py
@@ -358,7 +358,9 @@ if java_parsing:
             return self.outer_state
 
     def parse_java_file(fn, version=default_java_version):
-        return parse_java(open(fn, 'r').read(), version)
+        with open(fn, 'r') as f:
+            data = f.read()
+        return parse_java(data, version)
 
     def parse_java(contents, version=default_java_version, trace=None):
         """Parse a .java file and return a double of package directory,
diff --git a/src/engine/SCons/Tool/msvs.py b/src/engine/SCons/Tool/msvs.py
index 60ba278..f4439ba 100644
--- a/src/engine/SCons/Tool/msvs.py
+++ b/src/engine/SCons/Tool/msvs.py
@@ -520,7 +520,7 @@ class _DSPGenerator(object):
             config.cmdargs = cmdargs
             config.runfile = runfile
 
-            match = re.match('(.*)\|(.*)', variant)
+            match = re.match(r'(.*)\|(.*)', variant)
             if match:
                 config.variant = match.group(1)
                 config.platform = match.group(2)
@@ -1428,7 +1428,7 @@ class _GenerateV7DSW(_DSWGenerator):
         def AddConfig(self, variant, dswfile=dswfile):
             config = Config()
 
-            match = re.match('(.*)\|(.*)', variant)
+            match = re.match(r'(.*)\|(.*)', variant)
             if match:
                 config.variant = match.group(1)
                 config.platform = match.group(2)
diff --git a/src/engine/SCons/Tool/msvsTests.py b/src/engine/SCons/Tool/msvsTests.py
index 6adc598..477694a 100644
--- a/src/engine/SCons/Tool/msvsTests.py
+++ b/src/engine/SCons/Tool/msvsTests.py
@@ -482,8 +482,8 @@ class DummyRegistry(object):
     def parse(self, data):
         parents = [None, None]
         parents[0] = self.root
-        keymatch = re.compile('^\[(.*)\]$')
-        valmatch = re.compile('^(?:"(.*)"|[@])="(.*)"$')
+        keymatch = re.compile(r'^\[(.*)\]$')
+        valmatch = re.compile(r'^(?:"(.*)"|[@])="(.*)"$')
         for line in data:
             m1 = keymatch.match(line)
             if m1:
diff --git a/src/engine/SCons/Tool/swig.py b/src/engine/SCons/Tool/swig.py
index 77cfe1d..c6abefb 100644
--- a/src/engine/SCons/Tool/swig.py
+++ b/src/engine/SCons/Tool/swig.py
@@ -70,7 +70,9 @@ def _find_modules(src):
     directors = 0
     mnames = []
     try:
-        matches = _reModule.findall(open(src).read())
+        with open(src) as f:
+            data = f.read()
+        matches = _reModule.findall(data)
     except IOError:
         # If the file's not yet generated, guess the module name from the file stem
         matches = []
@@ -150,7 +152,7 @@ def _get_swig_version(env, swig):
     with pipe.stdout:
         out = SCons.Util.to_str(pipe.stdout.read())
 
-    match = re.search('SWIG Version\s+(\S+).*', out, re.MULTILINE)
+    match = re.search(r'SWIG Version\s+(\S+).*', out, re.MULTILINE)
     if match:
         version = match.group(1)
         if verbose:
diff --git a/src/engine/SCons/Tool/tex.py b/src/engine/SCons/Tool/tex.py
index 2dfa229..d361dac 100644
--- a/src/engine/SCons/Tool/tex.py
+++ b/src/engine/SCons/Tool/tex.py
@@ -74,15 +74,15 @@ openout_bcf_re = re.compile(r"OUTPUT *(.*\.bcf)")
 #printglossary_re = re.compile(r"^[^%]*\\printglossary", re.MULTILINE)
 
 # search to find rerun warnings
-warning_rerun_str = '(^LaTeX Warning:.*Rerun)|(^Package \w+ Warning:.*Rerun)'
+warning_rerun_str = r'(^LaTeX Warning:.*Rerun)|(^Package \w+ Warning:.*Rerun)'
 warning_rerun_re = re.compile(warning_rerun_str, re.MULTILINE)
 
 # search to find citation rerun warnings
-rerun_citations_str = "^LaTeX Warning:.*\n.*Rerun to get citations correct"
+rerun_citations_str = r"^LaTeX Warning:.*\n.*Rerun to get citations correct"
 rerun_citations_re = re.compile(rerun_citations_str, re.MULTILINE)
 
 # search to find undefined references or citations warnings
-undefined_references_str = '(^LaTeX Warning:.*undefined references)|(^Package \w+ Warning:.*undefined citations)'
+undefined_references_str = r'(^LaTeX Warning:.*undefined references)|(^Package \w+ Warning:.*undefined citations)'
 undefined_references_re = re.compile(undefined_references_str, re.MULTILINE)
 
 # used by the emitter
@@ -297,7 +297,8 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None
         logfilename = targetbase + '.log'
         logContent = ''
         if os.path.isfile(logfilename):
-            logContent = open(logfilename, "r").read()
+            with open(logfilename, "r") as f:
+                logContent = f.read()
 
 
         # Read the fls file to find all .aux files
@@ -305,7 +306,8 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None
         flsContent = ''
         auxfiles = []
         if os.path.isfile(flsfilename):
-            flsContent = open(flsfilename, "r").read()
+            with open(flsfilename, "r") as f:
+                flsContent = f.read()
             auxfiles = openout_aux_re.findall(flsContent)
             # remove duplicates
             dups = {}
@@ -315,7 +317,8 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None
 
         bcffiles = []
         if os.path.isfile(flsfilename):
-            flsContent = open(flsfilename, "r").read()
+            with open(flsfilename, "r") as f:
+                flsContent = f.read()
             bcffiles = openout_bcf_re.findall(flsContent)
             # remove duplicates
             dups = {}
@@ -338,7 +341,8 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None
                 already_bibtexed.append(auxfilename)
                 target_aux = os.path.join(targetdir, auxfilename)
                 if os.path.isfile(target_aux):
-                    content = open(target_aux, "r").read()
+                    with open(target_aux, "r") as f:
+                        content = f.read()
                     if content.find("bibdata") != -1:
                         if Verbose:
                             print("Need to run bibtex on ",auxfilename)
@@ -361,7 +365,8 @@ def InternalLaTeXAuxAction(XXXLaTeXAction, target = None, source= None, env=None
                 already_bibtexed.append(bcffilename)
                 target_bcf = os.path.join(targetdir, bcffilename)
                 if os.path.isfile(target_bcf):
-                    content = open(target_bcf, "r").read()
+                    with open(target_bcf, "r") as f:
+                        content = f.read()
                     if content.find("bibdata") != -1:
                         if Verbose:
                             print("Need to run biber on ",bcffilename)
@@ -647,7 +652,7 @@ def ScanFiles(theFile, target, paths, file_tests, file_tests_search, env, graphi
     if incResult:
         aux_files.append(os.path.join(targetdir, incResult.group(1)))
     if Verbose:
-        print("\include file names : ", aux_files)
+        print(r"\include file names : ", aux_files)
     # recursively call this on each of the included files
     inc_files = [ ]
     inc_files.extend( include_re.findall(content) )
@@ -813,7 +818,8 @@ def tex_emitter_core(target, source, env, graphics_extensions):
     # read fls file to get all other files that latex creates and will read on the next pass
     # remove files from list that we explicitly dealt with above
     if os.path.isfile(flsfilename):
-        content = open(flsfilename, "r").read()
+        with open(flsfilename, "r") as f:
+            content = f.read()
         out_files = openout_re.findall(content)
         myfiles = [auxfilename, logfilename, flsfilename, targetbase+'.dvi',targetbase+'.pdf']
         for filename in out_files[:]:
diff --git a/src/engine/SCons/cpp.py b/src/engine/SCons/cpp.py
index cdd3923..a413690 100644
--- a/src/engine/SCons/cpp.py
+++ b/src/engine/SCons/cpp.py
@@ -45,16 +45,16 @@ import re
 cpp_lines_dict = {
     # Fetch the rest of a #if/#elif as one argument,
     # with white space optional.
-    ('if', 'elif')      : '\s*(.+)',
+    ('if', 'elif')      : r'\s*(.+)',
 
     # Fetch the rest of a #ifdef/#ifndef as one argument,
     # separated from the keyword by white space.
-    ('ifdef', 'ifndef',): '\s+(.+)',
+    ('ifdef', 'ifndef',): r'\s+(.+)',
 
     # Fetch the rest of a #import/#include/#include_next line as one
     # argument, with white space optional.
     ('import', 'include', 'include_next',)
-                        : '\s*(.+)',
+                        : r'\s*(.+)',
 
     # We don't care what comes after a #else or #endif line.
     ('else', 'endif',)  : '',
@@ -64,10 +64,10 @@ cpp_lines_dict = {
     #   2) The optional parentheses and arguments (if it's a function-like
     #      macro, '' if it's not).
     #   3) The expansion value.
-    ('define',)         : '\s+([_A-Za-z][_A-Za-z0-9_]*)(\([^)]*\))?\s*(.*)',
+    ('define',)         : r'\s+([_A-Za-z][_A-Za-z0-9_]*)(\([^)]*\))?\s*(.*)',
 
     # Fetch the #undefed keyword from a #undef line.
-    ('undef',)          : '\s+([_A-Za-z][A-Za-z0-9_]*)',
+    ('undef',)          : r'\s+([_A-Za-z][A-Za-z0-9_]*)',
 }
 
 # Create a table that maps each individual C preprocessor directive to
@@ -97,7 +97,7 @@ l = [override.get(x, x) for x in list(Table.keys())]
 # a list of tuples, one for each preprocessor line.  The preprocessor
 # directive will be the first element in each tuple, and the rest of
 # the line will be the second element.
-e = '^\s*#\s*(' + '|'.join(l) + ')(.*)$'
+e = r'^\s*#\s*(' + '|'.join(l) + ')(.*)$'
 
 # And last but not least, compile the expression.
 CPP_Expression = re.compile(e, re.M)
@@ -144,12 +144,12 @@ CPP_to_Python_Ops_Expression = re.compile(expr)
 # A separate list of expressions to be evaluated and substituted
 # sequentially, not all at once.
 CPP_to_Python_Eval_List = [
-    ['defined\s+(\w+)',         '"\\1" in __dict__'],
-    ['defined\s*\((\w+)\)',     '"\\1" in __dict__'],
-    ['/\*.*\*/',                ''],
-    ['/\*.*',                   ''],
-    ['//.*',                    ''],
-    ['(0x[0-9A-Fa-f]*)[UL]+',   '\\1'],
+    [r'defined\s+(\w+)',         '"\\1" in __dict__'],
+    [r'defined\s*\((\w+)\)',     '"\\1" in __dict__'],
+    [r'/\*.*\*/',                ''],
+    [r'/\*.*',                   ''],
+    [r'//.*',                    ''],
+    [r'(0x[0-9A-Fa-f]*)[UL]+',   '\\1'],
 ]
 
 # Replace the string representations of the regular expressions in the
@@ -225,11 +225,11 @@ line_continuations = re.compile('\\\\\r?\n')
 # Search for a "function call" macro on an expansion.  Returns the
 # two-tuple of the "function" name itself, and a string containing the
 # arguments within the call parentheses.
-function_name = re.compile('(\S+)\(([^)]*)\)')
+function_name = re.compile(r'(\S+)\(([^)]*)\)')
 
 # Split a string containing comma-separated function call arguments into
 # the separate arguments.
-function_arg_separator = re.compile(',\s*')
+function_arg_separator = re.compile(r',\s*')
 
 
 
diff --git a/src/engine/SCons/dblite.py b/src/engine/SCons/dblite.py
index 628182b..14bd93d 100644
--- a/src/engine/SCons/dblite.py
+++ b/src/engine/SCons/dblite.py
@@ -108,16 +108,19 @@ class dblite(object):
             self._chgrp_to = -1  # don't chgrp
 
         if self._flag == "n":
-            self._open(self._file_name, "wb", self._mode)
+            with self._open(self._file_name, "wb", self._mode):
+                pass  # just make sure it exists
         else:
             try:
                 f = self._open(self._file_name, "rb")
             except IOError as e:
                 if self._flag != "c":
                     raise e
-                self._open(self._file_name, "wb", self._mode)
+                with self._open(self._file_name, "wb", self._mode):
+                    pass  # just make sure it exists
             else:
                 p = f.read()
+                f.close()
                 if len(p) > 0:
                     try:
                         if bytes is not str: