From fc2a0a8e3cb1d40fd965576060c28c8bd2ea1ad5 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Fri, 27 Oct 2006 23:06:01 +0000 Subject: Thomas Heller fixed modulefinder and added a test. Thanks! --- Lib/modulefinder.py | 142 +++++++++++++++++------ Lib/test/test_modulefinder.py | 259 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 364 insertions(+), 37 deletions(-) create mode 100644 Lib/test/test_modulefinder.py diff --git a/Lib/modulefinder.py b/Lib/modulefinder.py index 2a31dbf..9c90d8d 100644 --- a/Lib/modulefinder.py +++ b/Lib/modulefinder.py @@ -1,13 +1,14 @@ """Find modules used by a script, using introspection.""" - # This module should be kept compatible with Python 2.2, see PEP 291. +from __future__ import generators import dis import imp import marshal import os import sys import new +import struct if hasattr(sys.__stdout__, "newlines"): READ_MODE = "U" # universal line endings @@ -15,11 +16,12 @@ else: # remain compatible with Python < 2.3 READ_MODE = "r" -LOAD_CONST = dis.opname.index('LOAD_CONST') -IMPORT_NAME = dis.opname.index('IMPORT_NAME') -STORE_NAME = dis.opname.index('STORE_NAME') -STORE_GLOBAL = dis.opname.index('STORE_GLOBAL') +LOAD_CONST = chr(dis.opname.index('LOAD_CONST')) +IMPORT_NAME = chr(dis.opname.index('IMPORT_NAME')) +STORE_NAME = chr(dis.opname.index('STORE_NAME')) +STORE_GLOBAL = chr(dis.opname.index('STORE_GLOBAL')) STORE_OPS = [STORE_NAME, STORE_GLOBAL] +HAVE_ARGUMENT = chr(dis.HAVE_ARGUMENT) # Modulefinder does a good job at simulating Python's, but it can not # handle __path__ modifications packages make at runtime. Therefore there @@ -118,9 +120,9 @@ class ModuleFinder: stuff = (ext, "r", imp.PY_SOURCE) self.load_module(name, fp, pathname, stuff) - def import_hook(self, name, caller=None, fromlist=None): - self.msg(3, "import_hook", name, caller, fromlist) - parent = self.determine_parent(caller) + def import_hook(self, name, caller=None, fromlist=None, level=-1): + self.msg(3, "import_hook", name, caller, fromlist, level) + parent = self.determine_parent(caller, level=level) q, tail = self.find_head_package(parent, name) m = self.load_tail(q, tail) if not fromlist: @@ -129,12 +131,26 @@ class ModuleFinder: self.ensure_fromlist(m, fromlist) return None - def determine_parent(self, caller): - self.msgin(4, "determine_parent", caller) - if not caller: + def determine_parent(self, caller, level=-1): + self.msgin(4, "determine_parent", caller, level) + if not caller or level == 0: self.msgout(4, "determine_parent -> None") return None pname = caller.__name__ + if level >= 1: # relative import + if caller.__path__: + level -= 1 + if level == 0: + parent = self.modules[pname] + assert parent is caller + self.msgout(4, "determine_parent ->", parent) + return parent + if pname.count(".") < level: + raise ImportError, "relative importpath too deep" + pname = ".".join(pname.split(".")[:-level]) + parent = self.modules[pname] + self.msgout(4, "determine_parent ->", parent) + return parent if caller.__path__: parent = self.modules[pname] assert caller is parent @@ -294,13 +310,13 @@ class ModuleFinder: self.badmodules[name] = {} self.badmodules[name][caller.__name__] = 1 - def _safe_import_hook(self, name, caller, fromlist): + def _safe_import_hook(self, name, caller, fromlist, level=-1): # wrapper for self.import_hook() that won't raise ImportError if name in self.badmodules: self._add_badmodule(name, caller) return try: - self.import_hook(name, caller) + self.import_hook(name, caller, level=level) except ImportError, msg: self.msg(2, "ImportError:", str(msg)) self._add_badmodule(name, caller) @@ -311,38 +327,83 @@ class ModuleFinder: self._add_badmodule(sub, caller) continue try: - self.import_hook(name, caller, [sub]) + self.import_hook(name, caller, [sub], level=level) except ImportError, msg: self.msg(2, "ImportError:", str(msg)) fullname = name + "." + sub self._add_badmodule(fullname, caller) + def scan_opcodes(self, co, + unpack = struct.unpack): + # Scan the code, and yield 'interesting' opcode combinations + # Version for Python 2.4 and older + code = co.co_code + names = co.co_names + consts = co.co_consts + while code: + c = code[0] + if c in STORE_OPS: + oparg, = unpack('= HAVE_ARGUMENT: + code = code[3:] + else: + code = code[1:] + + def scan_opcodes_25(self, co, + unpack = struct.unpack): + # Scan the code, and yield 'interesting' opcode combinations + # Python 2.5 version (has absolute and relative imports) + code = co.co_code + names = co.co_names + consts = co.co_consts + LOAD_LOAD_AND_IMPORT = LOAD_CONST + LOAD_CONST + IMPORT_NAME + while code: + c = code[0] + if c in STORE_OPS: + oparg, = unpack('= HAVE_ARGUMENT: + code = code[3:] + else: + code = code[1:] + def scan_code(self, co, m): code = co.co_code - n = len(code) - i = 0 - fromlist = None - while i < n: - c = code[i] - i = i+1 - op = ord(c) - if op >= dis.HAVE_ARGUMENT: - oparg = ord(code[i]) + ord(code[i+1])*256 - i = i+2 - if op == LOAD_CONST: - # An IMPORT_NAME is always preceded by a LOAD_CONST, it's - # a tuple of "from" names, or None for a regular import. - # The tuple may contain "*" for "from import *" - fromlist = co.co_consts[oparg] - elif op == IMPORT_NAME: - assert fromlist is None or type(fromlist) is tuple - name = co.co_names[oparg] + if sys.version_info >= (2, 5): + scanner = self.scan_opcodes_25 + else: + scanner = self.scan_opcodes + for what, args in scanner(co): + if what == "store": + name, = args + m.globalnames[name] = 1 + elif what == "absolute_import": + fromlist, name = args have_star = 0 if fromlist is not None: if "*" in fromlist: have_star = 1 fromlist = [f for f in fromlist if f != "*"] - self._safe_import_hook(name, m, fromlist) + self._safe_import_hook(name, m, fromlist, level=0) if have_star: # We've encountered an "import *". If it is a Python module, # the code has already been parsed and we can suck out the @@ -362,10 +423,17 @@ class ModuleFinder: m.starimports[name] = 1 else: m.starimports[name] = 1 - elif op in STORE_OPS: - # keep track of all global names that are assigned to - name = co.co_names[oparg] - m.globalnames[name] = 1 + elif what == "relative_import": + level, fromlist, name = args + if name: + self._safe_import_hook(name, m, fromlist, level=level) + else: + parent = self.determine_parent(m, level=level) + self._safe_import_hook(parent.__name__, None, fromlist, level=0) + else: + # We don't expect anything else from the generator. + raise RuntimeError(what) + for c in co.co_consts: if isinstance(c, type(co)): self.scan_code(c, m) diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py new file mode 100644 index 0000000..0b8e451 --- /dev/null +++ b/Lib/test/test_modulefinder.py @@ -0,0 +1,259 @@ +import __future__ +import sys, os +import unittest +import distutils.dir_util +import tempfile + +from test import test_support + +try: set +except NameError: from sets import Set as set + +import modulefinder + +# Note: To test modulefinder with Python 2.2, sets.py and +# modulefinder.py must be available - they are not in the standard +# library. + +TEST_DIR = tempfile.mkdtemp() +TEST_PATH = [TEST_DIR, os.path.dirname(__future__.__file__)] + +# Each test description is a list of 5 items: +# +# 1. a module name that will be imported by modulefinder +# 2. a list of module names that modulefinder is required to find +# 3. a list of module names that modulefinder should complain +# about because they are not found +# 4. a list of module names that modulefinder should complain +# about because they MAY be not found +# 5. a string specifying packages to create; the format is obvious imo. +# +# Each package will be created in TEST_DIR, and TEST_DIR will be +# removed after the tests again. +# Modulefinder searches in a path that contains TEST_DIR, plus +# the standard Lib directory. + +maybe_test = [ + "a.module", + ["a", "a.module", "sys", + "b"], + ["c"], ["b.something"], + """\ +a/__init__.py +a/module.py + from b import something + from c import something +b/__init__.py + from sys import * +"""] + +maybe_test_new = [ + "a.module", + ["a", "a.module", "sys", + "b"], + ["c"], ["b.something"], + """\ +a/__init__.py +a/module.py + from b import something + from c import something +b/__init__.py + from sys import * +"""] + +package_test = [ + "a.module", + ["a", "a.b", "a.c", "a.module", "mymodule", "sys"], + ["blahblah", "c"], [], + """\ +mymodule.py +a/__init__.py + import blahblah + from a import b + import c +a/module.py + import sys + from a import b as x + from a.c import sillyname +a/b.py +a/c.py + from a.module import x + import mymodule as sillyname + from sys import version_info +"""] + +absolute_import_test = [ + "a.module", + ["a", "a.module", + "b", "b.x", "b.y", "b.z", + "sys", "exceptions"], + ["blahblah", "z"], [], + """\ +mymodule.py +a/__init__.py +a/module.py + import sys # sys + import blahblah # fails + import exceptions # exceptions + import b.x # b.x + from b import y # b.y + from b.z import * # b.z.* +a/exceptions.py +a/sys.py + import mymodule +a/b/__init__.py +a/b/x.py +a/b/y.py +a/b/z.py +b/__init__.py + import z +b/unused.py +b/x.py +b/y.py +b/z.py +"""] + +relative_import_test = [ + "a.module", + ["a", "a.module", + "a.b", "a.b.y", "a.b.z", + "a.b.c", "a.b.c.moduleC", + "a.b.c.d", "a.b.c.e", + "a.b.x", + "exceptions"], + [], [], + """\ +mymodule.py +a/__init__.py + from .b import y, z # a.b.y, a.b.z +a/module.py + import exceptions # exceptions +a/exceptions.py +a/sys.py +a/b/__init__.py + from ..b import x # a.b.x + #from a.b.c import moduleC + from .c import moduleC # a.b.moduleC +a/b/x.py +a/b/y.py +a/b/z.py +a/b/g.py +a/b/c/__init__.py + from ..c import e # a.b.c.e +a/b/c/moduleC.py + from ..c import d # a.b.c.d +a/b/c/d.py +a/b/c/e.py +a/b/c/x.py +"""] + +relative_import_test_2 = [ + "a.module", + ["a", "a.module", + "a.sys", + "a.b", "a.b.y", "a.b.z", + "a.b.c", "a.b.c.d", + "a.b.c.e", + "a.b.c.moduleC", + "a.b.c.f", + "a.b.x", + "a.another"], + [], [], + """\ +mymodule.py +a/__init__.py + from . import sys # a.sys +a/another.py +a/module.py + from .b import y, z # a.b.y, a.b.z +a/exceptions.py +a/sys.py +a/b/__init__.py + from .c import moduleC # a.b.c.moduleC + from .c import d # a.b.c.d +a/b/x.py +a/b/y.py +a/b/z.py +a/b/c/__init__.py + from . import e # a.b.c.e +a/b/c/moduleC.py + # + from . import f # a.b.c.f + from .. import x # a.b.x + from ... import another # a.another +a/b/c/d.py +a/b/c/e.py +a/b/c/f.py +"""] + +def open_file(path): + ##print "#", os.path.abspath(path) + dirname = os.path.dirname(path) + distutils.dir_util.mkpath(dirname) + return open(path, "w") + +def create_package(source): + ofi = None + for line in source.splitlines(): + if line.startswith(" ") or line.startswith("\t"): + ofi.write(line.strip() + "\n") + else: + ofi = open_file(os.path.join(TEST_DIR, line.strip())) + +class ModuleFinderTest(unittest.TestCase): + def _do_test(self, info, report=False): + import_this, modules, missing, maybe_missing, source = info + create_package(source) + try: + mf = modulefinder.ModuleFinder(path=TEST_PATH) + mf.import_hook(import_this) + if report: + mf.report() +## # This wouldn't work in general when executed several times: +## opath = sys.path[:] +## sys.path = TEST_PATH +## try: +## __import__(import_this) +## except: +## import traceback; traceback.print_exc() +## sys.path = opath +## return + modules = set(modules) + found = set(mf.modules.keys()) + more = list(found - modules) + less = list(modules - found) + # check if we found what we expected, not more, not less + self.failUnlessEqual((more, less), ([], [])) + + # check for missing and maybe missing modules + bad, maybe = mf.any_missing_maybe() + self.failUnlessEqual(bad, missing) + self.failUnlessEqual(maybe, maybe_missing) + finally: + distutils.dir_util.remove_tree(TEST_DIR) + + def test_package(self): + self._do_test(package_test) + + def test_maybe(self): + self._do_test(maybe_test) + + if getattr(__future__, "absolute_import", None): + + def test_maybe_new(self): + self._do_test(maybe_test_new) + + def test_absolute_imports(self): + self._do_test(absolute_import_test) + + def test_relative_imports(self): + self._do_test(relative_import_test) + + def test_relative_imports_2(self): + self._do_test(relative_import_test_2) + +def test_main(): + test_support.run_unittest(ModuleFinderTest) + +if __name__ == "__main__": + unittest.main() -- cgit v0.12