Merge pull request #4191 from mwichmann/maint/no-load-module

Stop using deprecated load_module

9 files changed, 140 insertions, 102 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 66d6e6c..438ff85 100755
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -202,6 +202,12 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
       but was not applied to other dialects, and e2e tests explicitly checked
       that FORTRANFLAGS did not propagate outside the FORTRAN dialect,
       so the conclusion is that behavior is intentional (issue #2257)
+    - SCons programmatic importing (tool modules and platform modules)
+      no longer uses the deprecated (since Py 3.10) importlib.load_module
+      routine, shifting to the preferred exec_module.  Old Python 2 compatible
+      import fallback (using the imp module) in tool module loading is dropped.
+      Tool module loading no longer special-cases Jython, which is a dead
+      project as far as SCons (no timeline in sight for Python 3 support).
     - Improvements to lex and yacc tools: better documentation of
       extra-file options, add test for extra-file behavior.
 
diff --git a/RELEASE.txt b/RELEASE.txt
index c86c1d5..6f9c4a8 100755
--- a/RELEASE.txt
+++ b/RELEASE.txt
@@ -113,6 +113,11 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY
   if scons can determine the ninja file doesnot need to be regenerated, which will also
   skip restarting the scons daemon. Note this option is could result in incorrect rebuilds
   if scons Glob or scons generated files are used in ninja build target's command lines.
+- Tool loading used to have a special case for Jython, it no longer does. This effectively
+  means SCons doesn't work with Jython, which has in reality been the case ever since
+  SCons dropped Python 2 support - there is still no timeline for Jython switching to
+  Python 3 compatibility.
+
 
 FIXES
 -----
diff --git a/SCons/Platform/PlatformTests.py b/SCons/Platform/PlatformTests.py
index bdb8996..ee0ab75 100644
--- a/SCons/Platform/PlatformTests.py
+++ b/SCons/Platform/PlatformTests.py
@@ -159,10 +159,6 @@ class PlatformTestCase(unittest.TestCase):
         assert p.synonyms != '', 'SCons.Platform.win32.get_architecture() not setting synonyms'
 
 
-
-
-
-
 class TempFileMungeTestCase(unittest.TestCase):
     def test_MAXLINELENGTH(self):
         """ Test different values for MAXLINELENGTH with the same
diff --git a/SCons/Platform/__init__.py b/SCons/Platform/__init__.py
index 93c5cd3..3fa5a75 100644
--- a/SCons/Platform/__init__.py
+++ b/SCons/Platform/__init__.py
@@ -83,7 +83,7 @@ def platform_default():
         return sys.platform
 
 
-def platform_module(name = platform_default()):
+def platform_module(name=platform_default()):
     """Return the imported module for the platform.
 
     This looks for a module name that matches the specified argument.
@@ -91,27 +91,41 @@ def platform_module(name = platform_default()):
     our execution environment.
     """
     full_name = 'SCons.Platform.' + name
-    if full_name not in sys.modules:
-        if os.name == 'java':
-            eval(full_name)
-        else:
+    try:
+        return sys.modules[full_name]
+    except KeyError:
+        try:
+            # the specific platform module is a relative import
+            mod = importlib.import_module("." + name, __name__)
+        except ModuleNotFoundError:
             try:
-                # the specific platform module is a relative import
-                mod = importlib.import_module("." + name, __name__)
-            except ImportError:
-                try:
-                    import zipimport
-                    importer = zipimport.zipimporter( sys.modules['SCons.Platform'].__path__[0] )
+                # This support was added to enable running inside
+                # a py2exe bundle a long time ago - unclear if it's
+                # still needed. It is *not* intended to load individual
+                # platform modules stored in a zipfile.
+                import zipimport
+
+                platform = sys.modules['SCons.Platform'].__path__[0]
+                importer = zipimport.zipimporter(platform)
+                if not hasattr(importer, 'find_spec'):
+                    # zipimport only added find_spec, exec_module in 3.10,
+                    # unlike importlib, where they've been around since 3.4.
+                    # If we don't have 'em, use the old way.
                     mod = importer.load_module(full_name)
-                except ImportError:
-                    raise SCons.Errors.UserError("No platform named '%s'" % name)
-            setattr(SCons.Platform, name, mod)
-    return sys.modules[full_name]
+                else:
+                    spec = importer.find_spec(full_name)
+                    mod = importlib.util.module_from_spec(spec)
+                    importer.exec_module(mod)
+                sys.modules[full_name] = mod
+            except zipimport.ZipImportError:
+                raise SCons.Errors.UserError("No platform named '%s'" % name)
+
+        setattr(SCons.Platform, name, mod)
+        return mod
 
 
 def DefaultToolList(platform, env):
-    """Select a default tool list for the specified platform.
-    """
+    """Select a default tool list for the specified platform."""
     return SCons.Tool.tool_list(platform, env)
 
 
@@ -328,8 +342,8 @@ class TempFileMunge:
 
 
 def Platform(name = platform_default()):
-    """Select a canned Platform specification.
-    """
+    """Select a canned Platform specification."""
+
     module = platform_module(name)
     spec = PlatformSpec(name, module.generate)
     return spec
diff --git a/SCons/Tool/__init__.py b/SCons/Tool/__init__.py
index afc579f..202b244 100644
--- a/SCons/Tool/__init__.py
+++ b/SCons/Tool/__init__.py
@@ -121,38 +121,28 @@ class Tool:
         if hasattr(module, 'options'):
             self.options = module.options
 
-    def _load_dotted_module_py2(self, short_name, full_name, searchpaths=None):
-        import imp
-
-        splitname = short_name.split('.')
-        index = 0
-        srchpths = searchpaths
-        for item in splitname:
-            file, path, desc = imp.find_module(item, srchpths)
-            mod = imp.load_module(full_name, file, path, desc)
-            srchpths = [path]
-        return mod, file
-
     def _tool_module(self):
+        """Try to load a tool module.
+
+        This will hunt in the toolpath for both a Python file (toolname.py)
+        and a Python module (toolname directory), then try the regular
+        import machinery, then fallback to try a zipfile.
+        """
         oldpythonpath = sys.path
         sys.path = self.toolpath + sys.path
-        # sys.stderr.write("Tool:%s\nPATH:%s\n"%(self.name,sys.path))
-
-        # From: http://stackoverflow.com/questions/67631/how-to-import-a-module-given-the-full-path/67692#67692
-        # import importlib.util
-        # spec = importlib.util.spec_from_file_location("module.name", "/path/to/file.py")
-        # foo = importlib.util.module_from_spec(spec)
-        # spec.loader.exec_module(foo)
-        # foo.MyClass()
-        # Py 3 code
-
-
+        # sys.stderr.write("Tool:%s\nPATH:%s\n" % (self.name,sys.path))
         # sys.stderr.write("toolpath:%s\n" % self.toolpath)
         # sys.stderr.write("SCONS.TOOL path:%s\n" % sys.modules['SCons.Tool'].__path__)
         debug = False
         spec = None
         found_name = self.name
         add_to_scons_tools_namespace = False
+
+
+        # Search for the tool module, but don't import it, yet.
+        #
+        # First look in the toolpath: these take priority.
+        # TODO: any reason to not just use find_spec here?
         for path in self.toolpath:
             sepname = self.name.replace('.', os.path.sep)
             file_path = os.path.join(path, "%s.py" % sepname)
@@ -173,6 +163,7 @@ class Tool:
             else:
                 continue
 
+        # Now look in the builtin tools (SCons.Tool package)
         if spec is None:
             if debug: sys.stderr.write("NO SPEC :%s\n" % self.name)
             spec = importlib.util.find_spec("." + self.name, package='SCons.Tool')
@@ -182,12 +173,14 @@ class Tool:
             if debug: sys.stderr.write("Spec Found? .%s :%s\n" % (self.name, spec))
 
         if spec is None:
+            # we are going to bail out here, format up stuff for the msg
             sconstools = os.path.normpath(sys.modules['SCons.Tool'].__path__[0])
             if self.toolpath:
                 sconstools = ", ".join(self.toolpath) + ", " + sconstools
             error_string = "No tool module '%s' found in %s" % (self.name, sconstools)
             raise SCons.Errors.UserError(error_string)
 
+        # We have a module spec, so we're good to go.
         module = importlib.util.module_from_spec(spec)
         if module is None:
             if debug: print("MODULE IS NONE:%s" % self.name)
@@ -204,13 +197,11 @@ class Tool:
             # Not sure what to do in the case that there already
             # exists sys.modules[self.name] but the source file is
             # different.. ?
-            module = spec.loader.load_module(spec.name)
-
             sys.modules[found_name] = module
+            spec.loader.exec_module(module)
             if add_to_scons_tools_namespace:
                 # If we found it in SCons.Tool, then add it to the module
                 setattr(SCons.Tool, self.name, module)
-
             found_module = module
 
         if found_module is not None:
@@ -219,31 +210,34 @@ class Tool:
 
         sys.path = oldpythonpath
 
+        # We try some other things here, but this is essentially dead code,
+        # because we bailed out above if we didn't find a module spec.
         full_name = 'SCons.Tool.' + self.name
         try:
             return sys.modules[full_name]
         except KeyError:
             try:
-                smpath = sys.modules['SCons.Tool'].__path__
-                try:
-                    module, file = self._load_dotted_module_py2(self.name, full_name, smpath)
-                    setattr(SCons.Tool, self.name, module)
-                    if file:
-                        file.close()
-                    return module
-                except ImportError as e:
-                    if str(e) != "No module named %s" % self.name:
-                        raise SCons.Errors.SConsEnvironmentError(e)
-                    try:
-                        import zipimport
-                        importer = zipimport.zipimporter(sys.modules['SCons.Tool'].__path__[0])
-                        module = importer.load_module(full_name)
-                        setattr(SCons.Tool, self.name, module)
-                        return module
-                    except ImportError as e:
-                        m = "No tool named '%s': %s" % (self.name, e)
-                        raise SCons.Errors.SConsEnvironmentError(m)
-            except ImportError as e:
+                # This support was added to enable running inside
+                # a py2exe bundle a long time ago - unclear if it's
+                # still needed. It is *not* intended to load individual
+                # tool modules stored in a zipfile.
+                import zipimport
+
+                tooldir = sys.modules['SCons.Tool'].__path__[0]
+                importer = zipimport.zipimporter(tooldir)
+                if not hasattr(importer, 'find_spec'):
+                    # zipimport only added find_spec, exec_module in 3.10,
+                    # unlike importlib, where they've been around since 3.4.
+                    # If we don't have 'em, use the old way.
+                    module = importer.load_module(full_name)
+                else:
+                    spec = importer.find_spec(full_name)
+                    module = importlib.util.module_from_spec(spec)
+                    importer.exec_module(module)
+                sys.modules[full_name] = module
+                setattr(SCons.Tool, self.name, module)
+                return module
+            except zipimport.ZipImportError as e:
                 m = "No tool named '%s': %s" % (self.name, e)
                 raise SCons.Errors.SConsEnvironmentError(m)
 
diff --git a/SCons/Utilities/sconsign.py b/SCons/Utilities/sconsign.py
index e595b2d..ae69196 100644
--- a/SCons/Utilities/sconsign.py
+++ b/SCons/Utilities/sconsign.py
@@ -28,6 +28,7 @@
 """Utility script to dump information from SCons signature database."""
 
 import getopt
+import importlib
 import os
 import sys
 from dbm import whichdb
@@ -51,13 +52,18 @@ def my_whichdb(filename):
 
 
 def my_import(mname):
+    """Import database module.
+
+    This was used if the module was *not* SCons.dblite, to allow
+    for programmatic importing.  It is no longer used, in favor of
+    importlib.import_module, and will be removed eventually.
+    """
     import imp
 
     if '.' in mname:
         i = mname.rfind('.')
         parent = my_import(mname[:i])
-        fp, pathname, description = imp.find_module(mname[i+1:],
-                                                    parent.__path__)
+        fp, pathname, description = imp.find_module(mname[i+1:], parent.__path__)
     else:
         fp, pathname, description = imp.find_module(mname)
     return imp.load_module(mname, fp, pathname, description)
@@ -371,36 +377,50 @@ def Do_SConsignDir(name):
 
 ##############################################################################
 def main():
-    global  Do_Call
+    global Do_Call
     global nodeinfo_string
     global args
     global Verbose
     global Readable
 
     helpstr = """\
-    Usage: sconsign [OPTIONS] [FILE ...]
-    Options:
-      -a, --act, --action         Print build action information.
-      -c, --csig                  Print content signature information.
-      -d DIR, --dir=DIR           Print only info about DIR.
-      -e ENTRY, --entry=ENTRY     Print only info about ENTRY.
-      -f FORMAT, --format=FORMAT  FILE is in the specified FORMAT.
-      -h, --help                  Print this message and exit.
-      -i, --implicit              Print implicit dependency information.
-      -r, --readable              Print timestamps in human-readable form.
-      --raw                       Print raw Python object representations.
-      -s, --size                  Print file sizes.
-      -t, --timestamp             Print timestamp information.
-      -v, --verbose               Verbose, describe each field.
-    """
+Usage: sconsign [OPTIONS] [FILE ...]
+
+Options:
+  -a, --act, --action         Print build action information.
+  -c, --csig                  Print content signature information.
+  -d DIR, --dir=DIR           Print only info about DIR.
+  -e ENTRY, --entry=ENTRY     Print only info about ENTRY.
+  -f FORMAT, --format=FORMAT  FILE is in the specified FORMAT.
+  -h, --help                  Print this message and exit.
+  -i, --implicit              Print implicit dependency information.
+  -r, --readable              Print timestamps in human-readable form.
+  --raw                       Print raw Python object representations.
+  -s, --size                  Print file sizes.
+  -t, --timestamp             Print timestamp information.
+  -v, --verbose               Verbose, describe each field.
+"""
 
     try:
-        opts, args = getopt.getopt(sys.argv[1:], "acd:e:f:hirstv",
-                                   ['act', 'action',
-                                    'csig', 'dir=', 'entry=',
-                                    'format=', 'help', 'implicit',
-                                    'raw', 'readable',
-                                    'size', 'timestamp', 'verbose'])
+        opts, args = getopt.getopt(
+            sys.argv[1:],
+            'acd:e:f:hirstv',
+            [
+                'act',
+                'action',
+                'csig',
+                'dir=',
+                'entry=',
+                'format=',
+                'help',
+                'implicit',
+                'raw',
+                'readable',
+                'size',
+                'timestamp',
+                'verbose',
+            ],
+        )
     except getopt.GetoptError as err:
         sys.stderr.write(str(err) + '\n')
         print(helpstr)
@@ -423,7 +443,7 @@ def main():
             if dbm_name:
                 try:
                     if dbm_name != "SCons.dblite":
-                        dbm = my_import(dbm_name)
+                        dbm = importlib.import_module(dbm_name)
                     else:
                         import SCons.dblite
 
@@ -466,7 +486,7 @@ def main():
             if dbm_name:
                 Map_Module = {'SCons.dblite': 'dblite'}
                 if dbm_name != "SCons.dblite":
-                    dbm = my_import(dbm_name)
+                    dbm = importlib.import_module(dbm_name)
                 else:
                     import SCons.dblite
 
diff --git a/scripts/scons-configure-cache.py b/scripts/scons-configure-cache.py
index da0e848..8f81d8a 100644
--- a/scripts/scons-configure-cache.py
+++ b/scripts/scons-configure-cache.py
@@ -2,7 +2,9 @@
 #
 # SCons - a Software Constructor
 #
-# __COPYRIGHT__
+# MIT License
+#
+# Copyright The SCons Foundation
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -47,7 +49,7 @@ import os
 import sys
 
 # python compatibility check
-if sys.version_info < (3, 5, 0):
+if sys.version_info < (3, 6, 0):
     msg = "scons: *** SCons version %s does not run under Python version %s.\n\
 Python >= 3.5 is required.\n"
     sys.stderr.write(msg % (__version__, sys.version.split()[0]))
@@ -92,4 +94,4 @@ sys.path = libs + sys.path
 from SCons.Utilities.ConfigureCache import main
 
 if __name__ == "__main__":
-    main()
\ No newline at end of file
+    main()
diff --git a/scripts/scons.py b/scripts/scons.py
index 86a6bf6..6a36ef7 100755
--- a/scripts/scons.py
+++ b/scripts/scons.py
@@ -41,9 +41,8 @@ __developer__ = "__DEVELOPER__"
 import os
 import sys
 
-
 # Python compatibility check
-if sys.version_info < (3, 5, 0):
+if sys.version_info < (3, 6, 0):
     msg = "scons: *** SCons version %s does not run under Python version %s.\n\
 Python >= 3.5 is required.\n"
     sys.stderr.write(msg % (__version__, sys.version.split()[0]))
diff --git a/scripts/sconsign.py b/scripts/sconsign.py
index 0edd05c..19e0c82 100644
--- a/scripts/sconsign.py
+++ b/scripts/sconsign.py
@@ -2,7 +2,9 @@
 #
 # SCons - a Software Constructor
 #
-# __COPYRIGHT__
+# MIT License
+#
+# Copyright The SCons Foundation
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -39,7 +41,7 @@ import os
 import sys
 
 # python compatibility check
-if sys.version_info < (3, 5, 0):
+if sys.version_info < (3, 6, 0):
     msg = "scons: *** SCons version %s does not run under Python version %s.\n\
 Python >= 3.5 is required.\n"
     sys.stderr.write(msg % (__version__, sys.version.split()[0]))