From 4f5bb72605c6163c1bb32c9ceceee5bd168728b8 Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 11 Feb 2020 08:52:35 -0700 Subject: Update Script/Main.py so it uses importlib instead of deprecated imp module. import site_init.py file directly for Py3 Find spec for the site file by path rather than by fiddling sys.path to restrict the search. Py3: address the problem that we might get a "site_init.py" from somewhere else in sys.path. Clean up excess indentation / try nesting; adjust comment. Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 3 + src/engine/SCons/Script/Main.py | 142 +++++++++++++++++++++++----------------- 2 files changed, 86 insertions(+), 59 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 407a83b..5093462 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -150,6 +150,9 @@ RELEASE 3.1.1 - Mon, 07 Aug 2019 20:09:12 -0500 - JSON encoding errors for CacheDir config - JSON decoding errors for CacheDir config + From Mats Wichmann: + - Script/Main.py now uses importlib in the PY3 case. + RELEASE 3.1.0 - Mon, 20 Jul 2019 16:59:23 -0700 diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index a0d7f4c..24bb94d 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -64,8 +64,8 @@ import SCons.SConf import SCons.Script import SCons.Taskmaster import SCons.Util +from SCons.Util import PY3 import SCons.Warnings - import SCons.Script.Interactive # Global variables @@ -698,80 +698,104 @@ def _create_path(plist): return path def _load_site_scons_dir(topdir, site_dir_name=None): - """Load the site_scons dir under topdir. - Prepends site_scons to sys.path, imports site_scons/site_init.py, - and prepends site_scons/site_tools to default toolpath.""" + """Load the site directory under topdir. + + If a site dir name is supplied use it, else use default "site_scons" + Prepend site dir to sys.path. + If a "site_tools" subdir exists, prepend to toolpath. + Import "site_init.py" from site dir if it exists. + """ if site_dir_name: err_if_not_found = True # user specified: err if missing else: site_dir_name = "site_scons" - err_if_not_found = False - + err_if_not_found = False # scons default: okay to be missing site_dir = os.path.join(topdir, site_dir_name) + if not os.path.exists(site_dir): if err_if_not_found: - raise SCons.Errors.UserError("site dir %s not found."%site_dir) + raise SCons.Errors.UserError("site dir %s not found." % site_dir) return + sys.path.insert(0, os.path.abspath(site_dir)) site_init_filename = "site_init.py" site_init_modname = "site_init" site_tools_dirname = "site_tools" - # prepend to sys.path - sys.path = [os.path.abspath(site_dir)] + sys.path site_init_file = os.path.join(site_dir, site_init_filename) site_tools_dir = os.path.join(site_dir, site_tools_dirname) - if os.path.exists(site_init_file): - import imp, re - try: - try: - fp, pathname, description = imp.find_module(site_init_modname, - [site_dir]) - # Load the file into SCons.Script namespace. This is - # opaque and clever; m is the module object for the - # SCons.Script module, and the exec ... in call executes a - # file (or string containing code) in the context of the - # module's dictionary, so anything that code defines ends - # up adding to that module. This is really short, but all - # the error checking makes it longer. - try: - m = sys.modules['SCons.Script'] - except Exception as e: - fmt = 'cannot import site_init.py: missing SCons.Script module %s' - raise SCons.Errors.InternalError(fmt % repr(e)) - try: - sfx = description[0] - modname = os.path.basename(pathname)[:-len(sfx)] - site_m = {"__file__": pathname, "__name__": modname, "__doc__": None} - re_special = re.compile("__[^_]+__") - for k, v in m.__dict__.items(): - if not re_special.match(k): - site_m[k] = v - - # This is the magic. - exec(compile(fp.read(), fp.name, 'exec'), site_m) - except KeyboardInterrupt: - raise - except Exception as e: - fmt = '*** Error loading site_init file %s:\n' - sys.stderr.write(fmt % repr(site_init_file)) - raise - else: - for k in site_m: - if not re_special.match(k): - m.__dict__[k] = site_m[k] - except KeyboardInterrupt: - raise - except ImportError as e: - fmt = '*** cannot import site init file %s:\n' - sys.stderr.write(fmt % repr(site_init_file)) - raise - finally: - if fp: - fp.close() + if os.path.exists(site_tools_dir): - # prepend to DefaultToolpath SCons.Tool.DefaultToolpath.insert(0, os.path.abspath(site_tools_dir)) + if not os.path.exists(site_init_file): + return + + if PY3: + import importlib.util + else: + import imp + import re + + # "import" the site_init.py file into the SCons.Script namespace. + # This is a variant on the basic Python import flow in that the globals + # dict for the compile step is prepopulated from the SCons.Script + # module object; on success the SCons.Script globals are refilled + # from the site_init globals so it all appears in SCons.Script + # instead of as a separate module. + try: + try: + m = sys.modules['SCons.Script'] + except KeyError: + fmt = 'cannot import {}: missing SCons.Script module' + raise SCons.Errors.InternalError(fmt.format(site_init_file)) + + if PY3: + spec = importlib.util.spec_from_file_location(site_init_modname, + site_init_file) + fp = open(spec.origin, 'r') + site_m = {"__file__": spec.origin, + "__name__": spec.name, + "__doc__": None} + else: + fp, pathname, description = imp.find_module(site_init_modname, + [site_dir]) + sfx = description[0] + modname = os.path.basename(pathname)[:-len(sfx)] + site_m = {"__file__": pathname, + "__name__": modname, + "__doc__": None} + + re_dunder = re.compile(r"__[^_]+__") + for k, v in m.__dict__.items(): + if not re_dunder.match(k): + site_m[k] = v + + try: + codeobj = compile(fp.read(), fp.name, 'exec') + exec(codeobj, site_m) + except KeyboardInterrupt: + raise + except Exception: + fmt = '*** Error loading site_init file %s:\n' + sys.stderr.write(fmt % repr(site_init_file)) + raise + else: + for k, v in site_m.items(): + if not re_dunder.match(k): + m.__dict__[k] = v + + except KeyboardInterrupt: + raise + except Exception: + fmt = '*** cannot import site init file %s:\n' + sys.stderr.write(fmt % repr(site_init_file)) + raise + finally: + try: + fp.close() + except Exception: + pass + def _load_all_site_scons_dirs(topdir, verbose=None): """Load all of the predefined site_scons dir. Order is significant; we load them in order from most generic @@ -810,7 +834,7 @@ def _load_all_site_scons_dirs(topdir, verbose=None): sysdirs=['/usr/share/scons', homedir('.scons')] - dirs=sysdirs + [topdir] + dirs = sysdirs + [topdir] for d in dirs: if verbose: # this is used by unit tests. print("Loading site dir ", d) -- cgit v0.12 From 82df07778ae85895266897bac7ea5f434183eb4f Mon Sep 17 00:00:00 2001 From: Mats Wichmann Date: Tue, 11 Feb 2020 09:23:06 -0700 Subject: Drop Python2 support from Main.py site_dir code Also simplify cleanup a bit - we don't need to leave the site file open, can use a context manager to read the code and let it close there. Signed-off-by: Mats Wichmann --- src/CHANGES.txt | 4 +--- src/engine/SCons/Script/Main.py | 52 ++++++++++++++--------------------------- 2 files changed, 19 insertions(+), 37 deletions(-) diff --git a/src/CHANGES.txt b/src/CHANGES.txt index 5093462..d1c5f3b 100755 --- a/src/CHANGES.txt +++ b/src/CHANGES.txt @@ -62,6 +62,7 @@ RELEASE VERSION/DATE TO BE FILLED IN LATER - Accommodate VS 2017 Express - it's got a more liberal license then VS Community, so some people prefer it (from 2019, no more Express) - vswhere call should also now work even if programs aren't on the C: drive. + - Script/Main.py now uses importlib instead of imp module. RELEASE 3.1.2 - Mon, 17 Dec 2019 02:06:27 +0000 @@ -150,9 +151,6 @@ RELEASE 3.1.1 - Mon, 07 Aug 2019 20:09:12 -0500 - JSON encoding errors for CacheDir config - JSON decoding errors for CacheDir config - From Mats Wichmann: - - Script/Main.py now uses importlib in the PY3 case. - RELEASE 3.1.0 - Mon, 20 Jul 2019 16:59:23 -0700 diff --git a/src/engine/SCons/Script/Main.py b/src/engine/SCons/Script/Main.py index 24bb94d..59401cc 100644 --- a/src/engine/SCons/Script/Main.py +++ b/src/engine/SCons/Script/Main.py @@ -43,7 +43,9 @@ __revision__ = "__FILE__ __REVISION__ __DATE__ __DEVELOPER__" import SCons.compat +import importlib.util import os +import re import sys import time import traceback @@ -64,7 +66,6 @@ import SCons.SConf import SCons.Script import SCons.Taskmaster import SCons.Util -from SCons.Util import PY3 import SCons.Warnings import SCons.Script.Interactive @@ -730,12 +731,6 @@ def _load_site_scons_dir(topdir, site_dir_name=None): if not os.path.exists(site_init_file): return - if PY3: - import importlib.util - else: - import imp - import re - # "import" the site_init.py file into the SCons.Script namespace. # This is a variant on the basic Python import flow in that the globals # dict for the compile step is prepopulated from the SCons.Script @@ -749,52 +744,41 @@ def _load_site_scons_dir(topdir, site_dir_name=None): fmt = 'cannot import {}: missing SCons.Script module' raise SCons.Errors.InternalError(fmt.format(site_init_file)) - if PY3: - spec = importlib.util.spec_from_file_location(site_init_modname, - site_init_file) - fp = open(spec.origin, 'r') - site_m = {"__file__": spec.origin, - "__name__": spec.name, - "__doc__": None} - else: - fp, pathname, description = imp.find_module(site_init_modname, - [site_dir]) - sfx = description[0] - modname = os.path.basename(pathname)[:-len(sfx)] - site_m = {"__file__": pathname, - "__name__": modname, - "__doc__": None} - + spec = importlib.util.spec_from_file_location(site_init_modname, site_init_file) + site_m = { + "__file__": spec.origin, + "__name__": spec.name, + "__doc__": None, + } re_dunder = re.compile(r"__[^_]+__") + # update site dict with all but magic (dunder) methods for k, v in m.__dict__.items(): if not re_dunder.match(k): site_m[k] = v + with open(spec.origin, 'r') as f: + code = f.read() try: - codeobj = compile(fp.read(), fp.name, 'exec') + codeobj = compile(code, spec.name, "exec") exec(codeobj, site_m) except KeyboardInterrupt: raise except Exception: - fmt = '*** Error loading site_init file %s:\n' - sys.stderr.write(fmt % repr(site_init_file)) + fmt = "*** Error loading site_init file {}:\n" + sys.stderr.write(fmt.format(site_init_file)) raise else: + # now refill globals with site_init's symbols for k, v in site_m.items(): if not re_dunder.match(k): m.__dict__[k] = v - except KeyboardInterrupt: raise except Exception: - fmt = '*** cannot import site init file %s:\n' - sys.stderr.write(fmt % repr(site_init_file)) + fmt = "*** cannot import site init file {}:\n" + sys.stderr.write(fmt.format(site_init_file)) raise - finally: - try: - fp.close() - except Exception: - pass + def _load_all_site_scons_dirs(topdir, verbose=None): """Load all of the predefined site_scons dir. -- cgit v0.12