Update Script/Main.py so it uses importlib

instead of deprecated imp module.

import site_init.py file directly for Py3

Find spec for the site file by path rather than by fiddling
sys.path to restrict the search.

Py3: address the problem that we might get a "site_init.py" from
somewhere else in sys.path.

Clean up excess indentation / try nesting; adjust comment.

Signed-off-by: Mats Wichmann <mats@linux.com>

2 files changed, 86 insertions, 59 deletions

diff --git a/src/CHANGES.txt b/src/CHANGES.txt
index 407a83b..5093462 100755
--- a/src/CHANGES.txt
+++ b/src/CHANGES.txt
@@ -150,6 +150,9 @@ RELEASE 3.1.1 - Mon, 07 Aug 2019 20:09:12 -0500
         - JSON encoding errors for CacheDir config
         - JSON decoding errors for CacheDir config
 
+  From Mats Wichmann:
+    - Script/Main.py now uses importlib in the PY3 case.
+
 
 RELEASE 3.1.0 - Mon, 20 Jul 2019 16:59:23 -0700
 
diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py
index a0d7f4c..24bb94d 100644
--- a/src/engine/SCons/Script/Main.py
+++ b/src/engine/SCons/Script/Main.py
@@ -64,8 +64,8 @@ import SCons.SConf
 import SCons.Script
 import SCons.Taskmaster
 import SCons.Util
+from SCons.Util import PY3
 import SCons.Warnings
-
 import SCons.Script.Interactive
 
 # Global variables
@@ -698,80 +698,104 @@ def _create_path(plist):
     return path
 
 def _load_site_scons_dir(topdir, site_dir_name=None):
-    """Load the site_scons dir under topdir.
-    Prepends site_scons to sys.path, imports site_scons/site_init.py,
-    and prepends site_scons/site_tools to default toolpath."""
+    """Load the site directory under topdir.
+
+    If a site dir name is supplied use it, else use default "site_scons"
+    Prepend site dir to sys.path.
+    If a "site_tools" subdir exists, prepend to toolpath.
+    Import "site_init.py" from site dir if it exists.
+    """
     if site_dir_name:
         err_if_not_found = True       # user specified: err if missing
     else:
         site_dir_name = "site_scons"
-        err_if_not_found = False
-
+        err_if_not_found = False      # scons default: okay to be missing
     site_dir = os.path.join(topdir, site_dir_name)
+
     if not os.path.exists(site_dir):
         if err_if_not_found:
-            raise SCons.Errors.UserError("site dir %s not found."%site_dir)
+            raise SCons.Errors.UserError("site dir %s not found." % site_dir)
         return
+    sys.path.insert(0, os.path.abspath(site_dir))
 
     site_init_filename = "site_init.py"
     site_init_modname = "site_init"
     site_tools_dirname = "site_tools"
-    # prepend to sys.path
-    sys.path = [os.path.abspath(site_dir)] + sys.path
     site_init_file = os.path.join(site_dir, site_init_filename)
     site_tools_dir = os.path.join(site_dir, site_tools_dirname)
-    if os.path.exists(site_init_file):
-        import imp, re
-        try:
-            try:
-                fp, pathname, description = imp.find_module(site_init_modname,
-                                                            [site_dir])
-                # Load the file into SCons.Script namespace.  This is
-                # opaque and clever; m is the module object for the
-                # SCons.Script module, and the exec ... in call executes a
-                # file (or string containing code) in the context of the
-                # module's dictionary, so anything that code defines ends
-                # up adding to that module.  This is really short, but all
-                # the error checking makes it longer.
-                try:
-                    m = sys.modules['SCons.Script']
-                except Exception as e:
-                    fmt = 'cannot import site_init.py: missing SCons.Script module %s'
-                    raise SCons.Errors.InternalError(fmt % repr(e))
-                try:
-                    sfx = description[0]
-                    modname = os.path.basename(pathname)[:-len(sfx)]
-                    site_m = {"__file__": pathname, "__name__": modname, "__doc__": None}
-                    re_special = re.compile("__[^_]+__")
-                    for k, v in m.__dict__.items():
-                        if not re_special.match(k):
-                            site_m[k] = v
-
-                    # This is the magic.
-                    exec(compile(fp.read(), fp.name, 'exec'), site_m)
-                except KeyboardInterrupt:
-                    raise
-                except Exception as e:
-                    fmt = '*** Error loading site_init file %s:\n'
-                    sys.stderr.write(fmt % repr(site_init_file))
-                    raise
-                else:
-                    for k in site_m:
-                        if not re_special.match(k):
-                            m.__dict__[k] = site_m[k]
-            except KeyboardInterrupt:
-                raise
-            except ImportError as e:
-                fmt = '*** cannot import site init file %s:\n'
-                sys.stderr.write(fmt % repr(site_init_file))
-                raise
-        finally:
-            if fp:
-                fp.close()
+
     if os.path.exists(site_tools_dir):
-        # prepend to DefaultToolpath
         SCons.Tool.DefaultToolpath.insert(0, os.path.abspath(site_tools_dir))
 
+    if not os.path.exists(site_init_file):
+        return
+
+    if PY3:
+        import importlib.util
+    else:
+        import imp
+    import re
+
+    # "import" the site_init.py file into the SCons.Script namespace.
+    # This is a variant on the basic Python import flow in that the globals
+    # dict for the compile step is prepopulated from the SCons.Script
+    # module object; on success the SCons.Script globals are refilled
+    # from the site_init globals so it all appears in SCons.Script
+    # instead of as a separate module.
+    try:
+        try:
+            m = sys.modules['SCons.Script']
+        except KeyError:
+            fmt = 'cannot import {}: missing SCons.Script module'
+            raise SCons.Errors.InternalError(fmt.format(site_init_file))
+
+        if PY3:
+            spec = importlib.util.spec_from_file_location(site_init_modname,
+                                                          site_init_file)
+            fp = open(spec.origin, 'r')
+            site_m = {"__file__": spec.origin,
+                      "__name__": spec.name,
+                      "__doc__": None}
+        else:
+            fp, pathname, description = imp.find_module(site_init_modname,
+                                                        [site_dir])
+            sfx = description[0]
+            modname = os.path.basename(pathname)[:-len(sfx)]
+            site_m = {"__file__": pathname,
+                      "__name__": modname,
+                      "__doc__": None}
+
+        re_dunder = re.compile(r"__[^_]+__")
+        for k, v in m.__dict__.items():
+            if not re_dunder.match(k):
+                site_m[k] = v
+
+        try:
+            codeobj = compile(fp.read(), fp.name, 'exec')
+            exec(codeobj, site_m)
+        except KeyboardInterrupt:
+            raise
+        except Exception:
+            fmt = '*** Error loading site_init file %s:\n'
+            sys.stderr.write(fmt % repr(site_init_file))
+            raise
+        else:
+            for k, v in site_m.items():
+                if not re_dunder.match(k):
+                    m.__dict__[k] = v
+
+    except KeyboardInterrupt:
+        raise
+    except Exception:
+        fmt = '*** cannot import site init file %s:\n'
+        sys.stderr.write(fmt % repr(site_init_file))
+        raise
+    finally:
+        try:
+            fp.close()
+        except Exception:
+            pass
+
 def _load_all_site_scons_dirs(topdir, verbose=None):
     """Load all of the predefined site_scons dir.
     Order is significant; we load them in order from most generic
@@ -810,7 +834,7 @@ def _load_all_site_scons_dirs(topdir, verbose=None):
         sysdirs=['/usr/share/scons',
                  homedir('.scons')]
 
-    dirs=sysdirs + [topdir]
+    dirs = sysdirs + [topdir]
     for d in dirs:
         if verbose:    # this is used by unit tests.
             print("Loading site dir ", d)