From 52e14d640be3a7fa2c17f5a2a6bc9626d622aa40 Mon Sep 17 00:00:00 2001 From: Just van Rossum Date: Mon, 30 Dec 2002 22:08:05 +0000 Subject: PEP 302 + zipimport: - new import hooks in import.c, exposed in the sys module - new module called 'zipimport' - various changes to allow bootstrapping from zip files I hope I didn't break the Windows build (or anything else for that matter), but then again, it's been sitting on sf long enough... Regarding the latest discussions on python-dev: zipimport sets pkg.__path__ as specified in PEP 273, and likewise, sys.path item such as /path/to/Archive.zip/subdir/ are supported again. --- Include/pythonrun.h | 1 + Lib/site.py | 13 +- Lib/test/test_importhooks.py | 204 ++++++++ Lib/test/test_zipimport.py | 180 +++++++ Modules/Setup.dist | 4 + Modules/getpath.c | 18 + Modules/zipimport.c | 1187 ++++++++++++++++++++++++++++++++++++++++++ PC/config.c | 2 + PC/getpathp.c | 49 +- PCbuild/pythoncore.dsp | 4 + Python/import.c | 255 ++++++++- Python/importdl.h | 3 +- Python/pythonrun.c | 2 + 13 files changed, 1887 insertions(+), 35 deletions(-) create mode 100644 Lib/test/test_importhooks.py create mode 100644 Lib/test/test_zipimport.py create mode 100644 Modules/zipimport.c diff --git a/Include/pythonrun.h b/Include/pythonrun.h index 01e0e32..b888193 100644 --- a/Include/pythonrun.h +++ b/Include/pythonrun.h @@ -99,6 +99,7 @@ PyAPI_FUNC(PyObject *) _PyBuiltin_Init(void); PyAPI_FUNC(PyObject *) _PySys_Init(void); PyAPI_FUNC(void) _PyImport_Init(void); PyAPI_FUNC(void) _PyExc_Init(void); +PyAPI_FUNC(void) _PyImportHooks_Init(void); /* Various internal finalizers */ PyAPI_FUNC(void) _PyExc_Fini(void); diff --git a/Lib/site.py b/Lib/site.py index a672765..0ab7a0f 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -73,16 +73,11 @@ del m # only absolute pathnames, even if we're running from the build directory. L = [] _dirs_in_sys_path = {} +dir = dircase = None # sys.path may be empty at this point for dir in sys.path: - # Filter out paths that don't exist, but leave in the empty string - # since it's a special case. We also need to special-case the Mac, - # as file names are allowed on sys.path there. - if sys.platform != 'mac': - if dir and not os.path.isdir(dir): - continue - else: - if dir and not os.path.exists(dir): - continue + # Filter out duplicate paths (on case-insensitive file systems also + # if they only differ in case); turn relative paths into absolute + # paths. dir, dircase = makepath(dir) if not dircase in _dirs_in_sys_path: L.append(dir) diff --git a/Lib/test/test_importhooks.py b/Lib/test/test_importhooks.py new file mode 100644 index 0000000..070b578 --- /dev/null +++ b/Lib/test/test_importhooks.py @@ -0,0 +1,204 @@ +import sys +import imp +import os +import unittest +from test import test_support + + +test_src = """\ +def get_name(): + return __name__ +def get_file(): + return __file__ +""" + +test_co = compile(test_src, "", "exec") +test_path = "!!!_test_!!!" + + +class ImportTracker: + """Importer that only tracks attempted imports.""" + def __init__(self): + self.imports = [] + def find_module(self, fullname, path=None): + self.imports.append(fullname) + return None + + +class TestImporter: + + modules = { + "hooktestmodule": (False, test_co), + "hooktestpackage": (True, test_co), + "hooktestpackage.sub": (True, test_co), + "hooktestpackage.sub.subber": (False, test_co), + } + + def __init__(self, path=test_path): + if path != test_path: + # if out class is on sys.path_hooks, we must raise + # ImportError for any path item that we can't handle. + raise ImportError + self.path = path + + def _get__path__(self): + raise NotImplementedError + + def find_module(self, fullname, path=None): + if fullname in self.modules: + return self + else: + return None + + def load_module(self, fullname): + ispkg, code = self.modules[fullname] + mod = imp.new_module(fullname) + sys.modules[fullname] = mod + mod.__file__ = "<%s>" % self.__class__.__name__ + mod.__loader__ = self + if ispkg: + mod.__path__ = self._get__path__() + exec code in mod.__dict__ + return mod + + +class MetaImporter(TestImporter): + def _get__path__(self): + return [] + +class PathImporter(TestImporter): + def _get__path__(self): + return [self.path] + + +class ImportBlocker: + """Place an ImportBlocker instance on sys.meta_path and you + can be sure the modules you specified can't be imported, even + if it's a builtin.""" + def __init__(self, *namestoblock): + self.namestoblock = dict.fromkeys(namestoblock) + def find_module(self, fullname, path=None): + if fullname in self.namestoblock: + return self + return None + def load_module(self, fullname): + raise ImportError, "I dare you" + + +class ImpWrapper: + + def __init__(self, path=None): + if path is not None and not os.path.isdir(path): + raise ImportError + self.path = path + + def find_module(self, fullname, path=None): + subname = fullname.split(".")[-1] + if subname != fullname and self.path is None: + return None + if self.path is None: + path = None + else: + path = [self.path] + try: + file, filename, stuff = imp.find_module(subname, path) + except ImportError: + return None + return ImpLoader(file, filename, stuff) + + +class ImpLoader: + + def __init__(self, file, filename, stuff): + self.file = file + self.filename = filename + self.stuff = stuff + + def load_module(self, fullname): + mod = imp.load_module(fullname, self.file, self.filename, self.stuff) + if self.file: + self.file.close() + mod.__loader__ = self # for introspection + return mod + + +class ImportHooksBaseTestCase(unittest.TestCase): + + def setUp(self): + self.path = sys.path[:] + self.meta_path = sys.meta_path[:] + self.path_hooks = sys.path_hooks[:] + sys.path_importer_cache.clear() + self.tracker = ImportTracker() + sys.meta_path.insert(0, self.tracker) + + def tearDown(self): + sys.path[:] = self.path + sys.meta_path[:] = self.meta_path + sys.path_hooks[:] = self.path_hooks + sys.path_importer_cache.clear() + for fullname in self.tracker.imports: + if fullname in sys.modules: + del sys.modules[fullname] + + +class ImportHooksTestCase(ImportHooksBaseTestCase): + + def doTestImports(self, importer=None): + import hooktestmodule + import hooktestpackage + import hooktestpackage.sub + import hooktestpackage.sub.subber + self.assertEqual(hooktestmodule.get_name(), + "hooktestmodule") + self.assertEqual(hooktestpackage.get_name(), + "hooktestpackage") + self.assertEqual(hooktestpackage.sub.get_name(), + "hooktestpackage.sub") + self.assertEqual(hooktestpackage.sub.subber.get_name(), + "hooktestpackage.sub.subber") + if importer: + self.assertEqual(hooktestmodule.__loader__, importer) + self.assertEqual(hooktestpackage.__loader__, importer) + self.assertEqual(hooktestpackage.sub.__loader__, importer) + self.assertEqual(hooktestpackage.sub.subber.__loader__, importer) + + def testMetaPath(self): + i = MetaImporter() + sys.meta_path.append(i) + self.doTestImports(i) + + def testPathHook(self): + sys.path_hooks.append(PathImporter) + sys.path.append(test_path) + self.doTestImports() + + def testBlocker(self): + mname = "exceptions" # an arbitrary harmless builtin module + if mname in sys.modules: + del sys.modules[mname] + sys.meta_path.append(ImportBlocker(mname)) + try: + __import__(mname) + except ImportError: + pass + else: + self.fail("'%s' was not supposed to be importable" % mname) + + def testImpWrapper(self): + i = ImpWrapper() + sys.meta_path.append(i) + sys.path_hooks.append(ImpWrapper) + mnames = ("colorsys", "urlparse", "distutils.core", "compiler.misc") + for mname in mnames: + parent = mname.split(".")[0] + for n in sys.modules.keys(): + if n.startswith(parent): + del sys.modules[n] + for mname in mnames: + m = __import__(mname, globals(), locals(), ["__dummy__"]) + m.__loader__ # to make sure we actually handled the import + + +if __name__ == "__main__": + test_support.run_unittest(ImportHooksTestCase) diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py new file mode 100644 index 0000000..07520a7 --- /dev/null +++ b/Lib/test/test_zipimport.py @@ -0,0 +1,180 @@ +import sys +import os +import marshal +import imp +import struct +import time + +import zlib # implied prerequisite +from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED +from test import test_support +from test.test_importhooks import ImportHooksBaseTestCase, test_src, test_co + +import zipimport + + +def make_pyc(co, mtime): + data = marshal.dumps(co) + pyc = imp.get_magic() + struct.pack("", "exec"), NOW) + files = {TESTMOD + pyc_ext: (NOW, pyc), + "some.data": (NOW, "some data")} + self.doTest(pyc_ext, files, TESTMOD) + + +class CompressedZipImportTestCase(UncompressedZipImportTestCase): + compression = ZIP_DEFLATED + + +if __name__ == "__main__": + test_support.run_unittest(UncompressedZipImportTestCase) + test_support.run_unittest(CompressedZipImportTestCase) diff --git a/Modules/Setup.dist b/Modules/Setup.dist index 6df361d..5388086 100644 --- a/Modules/Setup.dist +++ b/Modules/Setup.dist @@ -113,6 +113,10 @@ errno errnomodule.c # posix (UNIX) errno values _sre _sre.c # Fredrik Lundh's new regular expressions _codecs _codecsmodule.c # access to the builtin codecs and codec registry +# The zipimport module is always imported at startup. Having it as a +# builtin module avoids some bootstrapping problems and reduces overhead. +zipimport zipimport.c + # The rest of the modules listed in this file are all commented out by # default. Usually they can be detected and built as dynamically # loaded modules by the new setup.py script added in Python 2.1. If diff --git a/Modules/getpath.c b/Modules/getpath.c index 54b57c7..03646a5 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -365,6 +365,7 @@ calculate_path(void) char *path = getenv("PATH"); char *prog = Py_GetProgramName(); char argv0_path[MAXPATHLEN+1]; + char zip_path[MAXPATHLEN+1]; int pfound, efound; /* 1 if found; -1 if found build directory */ char *buf; size_t bufsz; @@ -483,6 +484,18 @@ calculate_path(void) else reduce(prefix); + strncpy(zip_path, prefix, MAXPATHLEN); + if (pfound > 0) { /* Use the reduced prefix returned by Py_GetPrefix() */ + reduce(zip_path); + reduce(zip_path); + } + else + strncpy(zip_path, PREFIX, MAXPATHLEN); + joinpath(zip_path, "lib/python00.zip"); + bufsz = strlen(zip_path); /* Replace "00" with version */ + zip_path[bufsz - 6] = VERSION[0]; + zip_path[bufsz - 5] = VERSION[2]; + if (!(efound = search_for_exec_prefix(argv0_path, home))) { if (!Py_FrozenFlag) fprintf(stderr, @@ -521,6 +534,7 @@ calculate_path(void) defpath = delim + 1; } + bufsz += strlen(zip_path) + 1; bufsz += strlen(exec_prefix) + 1; /* This is the only malloc call in this file */ @@ -541,6 +555,10 @@ calculate_path(void) else buf[0] = '\0'; + /* Next is the default zip path */ + strcat(buf, zip_path); + strcat(buf, delimiter); + /* Next goes merge of compile-time $PYTHONPATH with * dynamically located prefix. */ diff --git a/Modules/zipimport.c b/Modules/zipimport.c new file mode 100644 index 0000000..593c6e0 --- /dev/null +++ b/Modules/zipimport.c @@ -0,0 +1,1187 @@ +#include "Python.h" +#include "structmember.h" +#include "osdefs.h" +#include "marshal.h" +#include "compile.h" +#include + + +#define IS_SOURCE 0x0 +#define IS_BYTECODE 0x1 +#define IS_PACKAGE 0x2 + +struct st_zip_searchorder { + char suffix[14]; + int type; +}; + +/* zip_searchorder defines how we search for a module in the Zip + archive: we first search for a package __init__, then for + non-package .pyc, .pyo and .py entries. The .pyc and .pyo entries + are swapped by initzipimport() if we run in optimized mode. Also, + '/' is replaced by SEP there. */ +struct st_zip_searchorder zip_searchorder[] = { + {"/__init__.pyc", IS_PACKAGE | IS_BYTECODE}, + {"/__init__.pyo", IS_PACKAGE | IS_BYTECODE}, + {"/__init__.py", IS_PACKAGE | IS_SOURCE}, + {".pyc", IS_BYTECODE}, + {".pyo", IS_BYTECODE}, + {".py", IS_SOURCE}, + {"", 0} +}; + +/* zipimporter object definition and support */ + +typedef struct _zipimporter ZipImporter; + +struct _zipimporter { + PyObject_HEAD + PyObject *archive; /* pathname of the Zip archive */ + PyObject *prefix; /* file prefix: "a/sub/directory/" */ + PyObject *files; /* dict with file info {path: toc_entry} */ +}; + +static PyTypeObject ZipImporter_Type; +static PyObject *ZipImportError; +static PyObject *zip_directory_cache = NULL; + +/* forward decls */ +static PyObject *read_directory(char *archive); +static PyObject *get_data(char *archive, PyObject *toc_entry); +static PyObject *get_module_code(ZipImporter *self, char *fullname, + int *p_ispackage, char **p_modpath); + + +#define ZipImporter_Check(op) PyObject_TypeCheck(op, &ZipImporter_Type) + + +/* zipimporter.__init__ + Split the "subdirectory" from the Zip archive path, lookup a matching + entry in sys.path_importer_cache, fetch the file directory from there + if found, or else read it from the archive. */ +static int +zipimporter_init(ZipImporter *self, PyObject *args, PyObject *kwds) +{ + char *path, *p, *prefix, buf[MAXPATHLEN+2]; + int len; + + if (!PyArg_ParseTuple(args, "s:zipimporter", + &path)) + return -1; + + len = strlen(path); + if (len == 0) { + PyErr_SetString(ZipImportError, "archive path is empty"); + return -1; + } + if (len >= MAXPATHLEN) { + PyErr_SetString(ZipImportError, + "archive path too long"); + return -1; + } + strcpy(buf, path); + +#ifdef ALTSEP + for (p = buf; *p; p++) { + if (*p == ALTSEP) + *p = SEP; + } +#endif + + path = NULL; + prefix = NULL; + for (;;) { + struct stat statbuf; + int rv; + + rv = stat(buf, &statbuf); + if (rv == 0) { + /* it exists */ + if (S_ISREG(statbuf.st_mode)) + /* it's a file */ + path = buf; + break; + } + /* back up one path element */ + p = strchr(buf, SEP); + if (prefix != NULL) + *prefix = SEP; + if (p == NULL) + break; + *p = '\0'; + prefix = p; + } + if (path != NULL) { + PyObject *files; + files = PyDict_GetItemString(zip_directory_cache, path); + if (files == NULL) { + files = read_directory(buf); + if (files == NULL) + return -1; + if (PyDict_SetItemString(zip_directory_cache, path, + files) != 0) + return -1; + } + else + Py_INCREF(files); + self->files = files; + } + else { + PyErr_SetString(ZipImportError, "not a Zip file"); + return -1; + } + + if (prefix == NULL) + prefix = ""; + else { + prefix++; + len = strlen(prefix); + if (prefix[len-1] != SEP) { + /* add trailing SEP */ + prefix[len] = SEP; + prefix[len + 1] = '\0'; + } + } + + self->archive = PyString_FromString(buf); + if (self->archive == NULL) + return -1; + + self->prefix = PyString_FromString(prefix); + if (self->prefix == NULL) + return -1; + + return 0; +} + +/* GC support. */ +static int +zipimporter_traverse(PyObject *obj, visitproc visit, void *arg) +{ + ZipImporter *self = (ZipImporter *)obj; + int err; + + if (self->files != NULL) { + err = visit(self->files, arg); + if (err) + return err; + } + return 0; +} + +static void +zipimporter_dealloc(ZipImporter *self) +{ + PyObject_GC_UnTrack(self); + Py_XDECREF(self->archive); + Py_XDECREF(self->files); + self->ob_type->tp_free((PyObject *)self); +} + +static PyObject * +zipimporter_repr(ZipImporter *self) +{ + char buf[500]; + char *archive = "???"; + char *prefix = ""; + + if (self->archive != NULL && PyString_Check(self->archive)) + archive = PyString_AsString(self->archive); + if (self->prefix != NULL && PyString_Check(self->prefix)) + prefix = PyString_AsString(self->prefix); + if (prefix != NULL && *prefix) + PyOS_snprintf(buf, sizeof(buf), + "", + archive, SEP, prefix); + else + PyOS_snprintf(buf, sizeof(buf), + "", + archive); + return PyString_FromString(buf); +} + +/* return fullname.split(".")[-1] */ +static char * +get_subname(char *fullname) +{ + char *subname = strrchr(fullname, '.'); + if (subname == NULL) + subname = fullname; + else + subname++; + return subname; +} + +/* Given a (sub)modulename, write the potential file path in the + archive (without extension) to the path buffer. Return the + length of the resulting string. */ +static int +make_filename(char *prefix, char *name, char *path) +{ + int len; + char *p; + + len = strlen(prefix); + + /* self.prefix + name [+ SEP + "__init__"] + ".py[co]" */ + if (len + strlen(name) + 13 >= MAXPATHLEN) { + PyErr_SetString(ZipImportError, "path too long"); + return -1; + } + + strcpy(path, prefix); + strcpy(path + len, name); + for (p = path + len; *p; p++) { + if (*p == '.') + *p = SEP; + } + len += strlen(name); + return len; +} + +enum module_info { + MI_ERROR, + MI_NOT_FOUND, + MI_MODULE, + MI_PACKAGE +}; + +/* Return some information about a module. */ +static enum module_info +get_module_info(ZipImporter *self, char *fullname) +{ + char *subname, path[MAXPATHLEN + 1]; + int len; + struct st_zip_searchorder *zso; + + subname = get_subname(fullname); + + len = make_filename(PyString_AsString(self->prefix), subname, path); + if (len < 0) + return MI_ERROR; + + for (zso = zip_searchorder; *zso->suffix; zso++) { + strcpy(path + len, zso->suffix); + if (PyDict_GetItemString(self->files, path) != NULL) { + if (zso->type & IS_PACKAGE) + return MI_PACKAGE; + else + return MI_MODULE; + } + } + return MI_NOT_FOUND; +} + +/* Check whether we can satisfy the import of the module named by + 'fullname'. Return self if we can, None if we can't. */ +static PyObject * +zipimporter_find_module(PyObject *obj, PyObject *args) +{ + ZipImporter *self = (ZipImporter *)obj; + PyObject *path = NULL; + char *fullname; + enum module_info mi; + + if (!PyArg_ParseTuple(args, "s|O:zipimporter.find_module", + &fullname, &path)) + return NULL; + + mi = get_module_info(self, fullname); + if (mi == MI_ERROR) + return NULL; + if (mi == MI_NOT_FOUND) { + Py_INCREF(Py_None); + return Py_None; + } + Py_INCREF(self); + return (PyObject *)self; +} + +/* Load and return the module named by 'fullname'. */ +static PyObject * +zipimporter_load_module(PyObject *obj, PyObject *args) +{ + ZipImporter *self = (ZipImporter *)obj; + PyObject *code, *mod, *dict; + char *fullname, *modpath; + int ispackage; + + if (!PyArg_ParseTuple(args, "s:zipimporter.load_module", + &fullname)) + return NULL; + + code = get_module_code(self, fullname, &ispackage, &modpath); + if (code == NULL) + return NULL; + + mod = PyImport_AddModule(fullname); + if (mod == NULL) { + Py_DECREF(code); + return NULL; + } + dict = PyModule_GetDict(mod); + + /* mod.__loader__ = self */ + if (PyDict_SetItemString(dict, "__loader__", (PyObject *)self) != 0) + goto error; + + if (ispackage) { + /* add __path__ to the module *before* the code gets + executed */ + PyObject *pkgpath, *fullpath; + char *prefix = PyString_AsString(self->prefix); + char *subname = get_subname(fullname); + int err; + + fullpath = PyString_FromFormat("%s%c%s%s", + PyString_AsString(self->archive), + SEP, + *prefix ? prefix : "", + subname); + if (fullpath == NULL) + goto error; + + pkgpath = Py_BuildValue("[O]", fullpath); + Py_DECREF(fullpath); + if (pkgpath == NULL) + goto error; + err = PyDict_SetItemString(dict, "__path__", pkgpath); + Py_DECREF(pkgpath); + if (err != 0) + goto error; + } + mod = PyImport_ExecCodeModuleEx(fullname, code, modpath); + Py_DECREF(code); + if (Py_VerboseFlag) + PySys_WriteStderr("import %s # loaded from Zip %s\n", + fullname, modpath); + return mod; +error: + Py_DECREF(code); + Py_DECREF(mod); + return NULL; +} + +/* Return a bool signifying whether the module is a package or not. */ +static PyObject * +zipimporter_is_package(PyObject *obj, PyObject *args) +{ + ZipImporter *self = (ZipImporter *)obj; + char *fullname; + enum module_info mi; + + if (!PyArg_ParseTuple(args, "s:zipimporter.find_module", + &fullname)) + return NULL; + + mi = get_module_info(self, fullname); + if (mi == MI_ERROR) + return NULL; + if (mi == MI_NOT_FOUND) { + PyErr_Format(ZipImportError, "can't find module '%.200s'", + fullname); + return NULL; + } + return PyBool_FromLong(mi == MI_PACKAGE); +} + +static PyObject * +zipimporter_get_data(PyObject *obj, PyObject *args) +{ + ZipImporter *self = (ZipImporter *)obj; + char *path; +#ifdef ALTSEP + char *p, buf[MAXPATHLEN + 1];; +#endif + PyObject *toc_entry; + int len; + + if (!PyArg_ParseTuple(args, "s:zipimporter.get_data", &path)) + return NULL; + +#ifdef ALTSEP + if (strlen(path) >= MAXPATHLEN) { + PyErr_SetString(ZipImportError, "path too long"); + return NULL; + } + strcpy(buf, path); + for (p = buf; *p; p++) { + if (*p == ALTSEP) + *p = SEP; + } + path = buf; +#endif + len = PyString_Size(self->archive); + if (len < strlen(path) && + strncmp(path, PyString_AsString(self->archive), len) == 0 && + path[len] == SEP) { + path = path + len + 1; + } + + toc_entry = PyDict_GetItemString(self->files, path); + if (toc_entry == NULL) { + PyErr_Format(PyExc_IOError, "file not found [%.200s]", + path); + return NULL; + } + return get_data(PyString_AsString(self->archive), toc_entry); +} + +static PyObject * +zipimporter_get_code(PyObject *obj, PyObject *args) +{ + ZipImporter *self = (ZipImporter *)obj; + char *fullname; + + if (!PyArg_ParseTuple(args, "s:zipimporter.get_code", &fullname)) + return NULL; + + return get_module_code(self, fullname, NULL, NULL); +} + +static PyObject * +zipimporter_get_source(PyObject *obj, PyObject *args) +{ + ZipImporter *self = (ZipImporter *)obj; + PyObject *toc_entry; + char *fullname, *subname, path[MAXPATHLEN+1]; + int len; + enum module_info mi; + + if (!PyArg_ParseTuple(args, "s:zipimporter.get_source", &fullname)) + return NULL; + + mi = get_module_info(self, fullname); + if (mi == MI_ERROR) + return NULL; + if (mi == MI_NOT_FOUND) { + PyErr_Format(ZipImportError, "can't find module '%.200s'", + fullname); + return NULL; + } + subname = get_subname(fullname); + + len = make_filename(PyString_AsString(self->prefix), subname, path); + if (len < 0) + return NULL; + + if (mi == MI_PACKAGE) { + path[len] = SEP; + strcpy(path + len + 1, "__init__.py"); + } + else + strcpy(path + len, ".py"); + + toc_entry = PyDict_GetItemString(self->files, path); + if (toc_entry != NULL) + return get_data(PyString_AsString(self->archive), toc_entry); + + /* we have the module, but no source */ + Py_INCREF(Py_None); + return Py_None; +} + +PyDoc_STRVAR(doc_find_module, +"find_module(fullname, path=None) -> self or None.\n\ +\n\ +Search for a module specified by 'fullname'. 'fullname' must be the\n\ +fully qualified (dotted) module name. It returns the zipimporter\n\ +instance itself if the module was found, or None if it wasn't.\n\ +The optional 'path' argument is ignored -- it's there for compatibility\n\ +with the importer protocol."); + +PyDoc_STRVAR(doc_load_module, +"load_module(fullname) -> module.\n\ +\n\ +Load the module specified by 'fullname'. 'fullname' must be the\n\ +fully qualified (dotted) module name. It returns the imported\n\ +module, or raises ZipImportError if it wasn't found."); + +PyDoc_STRVAR(doc_get_data, +"get_data(pathname) -> string with file data.\n\ +\n\ +Return the data associated with 'pathname'. Raise IOError if\n\ +the file wasn't found."); + +PyDoc_STRVAR(doc_is_package, +"is_package(fullname) -> bool.\n\ +\n\ +Return True if the module specified by fullname is a package.\n\ +Raise ZipImportError is the module couldn't be found."); + +PyDoc_STRVAR(doc_get_code, +"get_code(fullname) -> code object.\n\ +\n\ +Return the code object for the specified module. Raise ZipImportError\n\ +is the module couldn't be found."); + +PyDoc_STRVAR(doc_get_source, +"get_source(fullname) -> source string.\n\ +\n\ +Return the source code for the specified module. Raise ZipImportError\n\ +is the module couldn't be found, return None if the archive does\n\ +contain the module, but has no source for it."); + +static PyMethodDef zipimporter_methods[] = { + {"find_module", zipimporter_find_module, METH_VARARGS, + doc_find_module}, + {"load_module", zipimporter_load_module, METH_VARARGS, + doc_load_module}, + {"get_data", zipimporter_get_data, METH_VARARGS, + doc_get_data}, + {"get_code", zipimporter_get_code, METH_VARARGS, + doc_get_code}, + {"get_source", zipimporter_get_source, METH_VARARGS, + doc_get_source}, + {"is_package", zipimporter_is_package, METH_VARARGS, + doc_is_package}, + {NULL, NULL} /* sentinel */ +}; + +static PyMemberDef zipimporter_members[] = { + {"archive", T_OBJECT, offsetof(ZipImporter, archive), READONLY}, + {"prefix", T_OBJECT, offsetof(ZipImporter, prefix), READONLY}, + {"_files", T_OBJECT, offsetof(ZipImporter, files), READONLY}, + {NULL} +}; + +PyDoc_STRVAR(zipimporter_doc, +"zipimporter(archivepath) -> zipimporter object\n\ +\n\ +Create a new zipimporter instance. 'archivepath' must be a path to\n\ +a zipfile. ZipImportError is raised if 'archivepath' doesn't point to\n\ +a valid Zip archive."); + +#define DEFERRED_ADDRESS(ADDR) 0 + +static PyTypeObject ZipImporter_Type = { + PyObject_HEAD_INIT(DEFERRED_ADDRESS(&PyType_Type)) + 0, + "zipimport.zipimporter", + sizeof(ZipImporter), + 0, /* tp_itemsize */ + (destructor)zipimporter_dealloc, /* tp_dealloc */ + 0, /* tp_print */ + 0, /* tp_getattr */ + 0, /* tp_setattr */ + 0, /* tp_compare */ + (reprfunc)zipimporter_repr, /* tp_repr */ + 0, /* tp_as_number */ + 0, /* tp_as_sequence */ + 0, /* tp_as_mapping */ + 0, /* tp_hash */ + 0, /* tp_call */ + 0, /* tp_str */ + PyObject_GenericGetAttr, /* tp_getattro */ + 0, /* tp_setattro */ + 0, /* tp_as_buffer */ + Py_TPFLAGS_DEFAULT | Py_TPFLAGS_BASETYPE | + Py_TPFLAGS_HAVE_GC, /* tp_flags */ + zipimporter_doc, /* tp_doc */ + zipimporter_traverse, /* tp_traverse */ + 0, /* tp_clear */ + 0, /* tp_richcompare */ + 0, /* tp_weaklistoffset */ + 0, /* tp_iter */ + 0, /* tp_iternext */ + zipimporter_methods, /* tp_methods */ + zipimporter_members, /* tp_members */ + 0, /* tp_getset */ + 0, /* tp_base */ + 0, /* tp_dict */ + 0, /* tp_descr_get */ + 0, /* tp_descr_set */ + 0, /* tp_dictoffset */ + (initproc)zipimporter_init, /* tp_init */ + PyType_GenericAlloc, /* tp_alloc */ + PyType_GenericNew, /* tp_new */ + PyObject_GC_Del, /* tp_free */ +}; + + +/* implementation */ + +/* Given a buffer, return the short that is represented by the first + 2 bytes, encoded as little endian. This partially reimplements + marshal.c:r_short(). */ +static int +get_short(unsigned char *buf) +{ + short x; + x = buf[0]; + x |= buf[1] << 8; + /* Sign-extension, in case short greater than 16 bits */ + x |= -(x & 0x8000); + return x; +} + +/* Given a buffer, return the long that is represented by the first + 4 bytes, encoded as little endian. This partially reimplements + marshal.c:r_long() */ +static long +get_long(unsigned char *buf) { + long x; + x = buf[0]; + x |= (long)buf[1] << 8; + x |= (long)buf[2] << 16; + x |= (long)buf[3] << 24; +#if SIZEOF_LONG > 4 + /* Sign extension for 64-bit machines */ + x |= -(x & 0x80000000L); +#endif + return x; +} + +/* + read_directory(archive) -> files dict (new reference) + + Given a path to a Zip archive, build a dict, mapping file names + (local to the archive, using SEP as a separator) to toc entries. + + A toc_entry is a tuple: + + (compress, # compression kind; 0 for uncompressed + data_size, # size of compressed data on disk + file_size, # size of decompressed data + file_offset, # offset of file header from start of archive + time, # mod time of file (in dos format) + date, # mod data of file (in dos format) + crc, # crc checksum of the data + ) + + Directories can be recognized by the trailing SEP in the name, + data_size and file_offset are 0. +*/ +static PyObject * +read_directory(char *archive) +{ + PyObject *files = NULL; + FILE *fp; + long compress, crc, data_size, file_size, file_offset, date, time; + long header_offset, name_size, header_size, header_end; + long i, l, length, count; + char path[MAXPATHLEN + 5]; + char name[MAXPATHLEN + 5]; + char *p, endof_central_dir[22]; + + if (strlen(archive) > MAXPATHLEN) { + PyErr_SetString(PyExc_OverflowError, + "Zip path name is too long"); + return NULL; + } + strcpy(path, archive); + + fp = fopen(archive, "rb"); + if (fp == NULL) { + PyErr_Format(ZipImportError, "can't open Zip file: " + "'%.200s'", archive); + return NULL; + } + fseek(fp, -22, 2); /* Seek from end of file */ + header_end = ftell(fp); + if (fread(endof_central_dir, 1, 22, fp) != 22) { + fclose(fp); + PyErr_Format(ZipImportError, "can't read Zip file: " + "'%.200s'", archive); + return NULL; + } + if (get_long(endof_central_dir) != 0x06054B50) { + /* Bad: End of Central Dir signature */ + fclose(fp); + PyErr_Format(ZipImportError, "not a Zip file: " + "'%.200s'", archive); + return NULL; + } + + header_offset = get_long(endof_central_dir + 16); + + files = PyDict_New(); + if (files == NULL) + goto error; + + length = (long)strlen(path); + path[length] = SEP; + + /* Start of Central Directory */ + count = 0; + for (;;) { + PyObject *t; + int err; + + fseek(fp, header_offset, 0); /* Start of file header */ + l = PyMarshal_ReadLongFromFile(fp); + if (l != 0x02014B50) + break; /* Bad: Central Dir File Header */ + fseek(fp, header_offset + 10, 0); + compress = PyMarshal_ReadShortFromFile(fp); + time = PyMarshal_ReadShortFromFile(fp); + date = PyMarshal_ReadShortFromFile(fp); + crc = PyMarshal_ReadLongFromFile(fp); + data_size = PyMarshal_ReadLongFromFile(fp); + file_size = PyMarshal_ReadLongFromFile(fp); + name_size = PyMarshal_ReadShortFromFile(fp); + header_size = 46 + name_size + + PyMarshal_ReadShortFromFile(fp) + + PyMarshal_ReadShortFromFile(fp); + fseek(fp, header_offset + 42, 0); + file_offset = PyMarshal_ReadLongFromFile(fp); + if (name_size > MAXPATHLEN) + name_size = MAXPATHLEN; + + p = name; + for (i = 0; i < name_size; i++) { + *p = (char)getc(fp); + if (*p == '/') + *p = SEP; + p++; + } + *p = 0; /* Add terminating null byte */ + header_offset += header_size; + + strncpy(path + length + 1, name, MAXPATHLEN - length - 1); + + t = Py_BuildValue("siiiiiii", path, compress, data_size, + file_size, file_offset, time, date, crc); + if (t == NULL) + goto error; + err = PyDict_SetItemString(files, name, t); + Py_DECREF(t); + if (err != 0) + goto error; + count++; + } + fclose(fp); + if (Py_VerboseFlag) + PySys_WriteStderr("# zipimport: found %ld names in %s\n", + count, archive); + return files; +error: + fclose(fp); + Py_XDECREF(files); + return NULL; +} + +/* Return the zlib.decompress function object, or NULL if zlib couldn't + be imported. The function is cached when found, so subsequent calls + don't import zlib again. Returns a *borrowed* reference. + XXX This makes zlib.decompress immortal. */ +static PyObject * +get_decompress_func(void) +{ + static PyObject *decompress = NULL; + + if (decompress == NULL) { + PyObject *zlib; + static int importing_zlib = 0; + + if (importing_zlib != 0) + /* Someone has a zlib.py[co] in their Zip file; + let's avoid a stack overflow. */ + return NULL; + importing_zlib = 1; + zlib = PyImport_ImportModule("zlib"); /* import zlib */ + importing_zlib = 0; + if (zlib != NULL) { + decompress = PyObject_GetAttrString(zlib, + "decompress"); + Py_DECREF(zlib); + } + else + PyErr_Clear(); + if (Py_VerboseFlag) + PySys_WriteStderr("# zipimport: zlib %s\n", + zlib != NULL ? "available": "UNAVAILABLE"); + } + return decompress; +} + +/* Given a path to a Zip file and a toc_entry, return the (uncompressed) + data as a new reference. */ +static PyObject * +get_data(char *archive, PyObject *toc_entry) +{ + PyObject *raw_data, *data = NULL, *decompress; + char *buf; + FILE *fp; + int err, bytes_read = 0; + long l; + char *datapath; + long compress, data_size, file_size, file_offset; + long time, date, crc; + + if (!PyArg_ParseTuple(toc_entry, "siiiiiii", &datapath, &compress, + &data_size, &file_size, &file_offset, &time, + &date, &crc)) { + return NULL; + } + + fp = fopen(archive, "rb"); + if (!fp) { + PyErr_Format(PyExc_IOError, + "zipimport: can not open file %s", archive); + return NULL; + } + + /* Check to make sure the local file header is correct */ + fseek(fp, file_offset, 0); + l = PyMarshal_ReadLongFromFile(fp); + if (l != 0x04034B50) { + /* Bad: Local File Header */ + PyErr_Format(ZipImportError, + "bad local file header in %s", + archive); + fclose(fp); + return NULL; + } + fseek(fp, file_offset + 26, 0); + l = 30 + PyMarshal_ReadShortFromFile(fp) + + PyMarshal_ReadShortFromFile(fp); /* local header size */ + file_offset += l; /* Start of file data */ + + raw_data = PyString_FromStringAndSize((char *)NULL, compress == 0 ? + data_size : data_size + 1); + if (raw_data == NULL) { + fclose(fp); + return NULL; + } + buf = PyString_AsString(raw_data); + + err = fseek(fp, file_offset, 0); + if (err == 0) + bytes_read = fread(buf, 1, data_size, fp); + fclose(fp); + if (err || bytes_read != data_size) { + PyErr_SetString(PyExc_IOError, + "zipimport: can't read data"); + Py_DECREF(raw_data); + return NULL; + } + + if (compress != 0) { + buf[data_size] = 'Z'; /* saw this in zipfile.py */ + data_size++; + } + buf[data_size] = '\0'; + + if (compress == 0) /* data is not compressed */ + return raw_data; + + /* Decompress with zlib */ + decompress = get_decompress_func(); + if (decompress == NULL) { + PyErr_SetString(ZipImportError, + "can't decompress data; " + "zlib not available"); + goto error; + } + data = PyObject_CallFunction(decompress, "Ol", raw_data, -15); +error: + Py_DECREF(raw_data); + return data; +} + +/* Lenient date/time comparison function. The precision of the mtime + in the archive is lower than the mtime stored in a .pyc: we + must allow a difference of at most one second. */ +static int +eq_mtime(time_t t1, time_t t2) +{ + time_t d = t1 - t2; + if (d < 0) + d = -d; + /* dostime only stores even seconds, so be lenient */ + return d <= 1; +} + +/* Given the contents of a .py[co] file in a buffer, unmarshal the data + and return the code object. Return None if it the magic word doesn't + match (we do this instead of raising an exception as we fall back + to .py if available and we don't want to mask other errors). + Returns a new reference. */ +static PyObject * +unmarshal_code(char *pathname, PyObject *data, time_t mtime) +{ + PyObject *code; + char *buf = PyString_AsString(data); + int size = PyString_Size(data); + + if (size <= 9) { + PyErr_SetString(ZipImportError, + "bad pyc data"); + return NULL; + } + + if (get_long(buf) != PyImport_GetMagicNumber()) { + if (Py_VerboseFlag) + PySys_WriteStderr("# %s has bad magic\n", + pathname); + Py_INCREF(Py_None); + return Py_None; /* signal caller to try alternative */ + } + + if (mtime != 0 && !eq_mtime(get_long(buf + 4), mtime)) { + if (Py_VerboseFlag) + PySys_WriteStderr("# %s has bad mtime\n", + pathname); + Py_INCREF(Py_None); + return Py_None; /* signal caller to try alternative */ + } + + code = PyMarshal_ReadObjectFromString(buf + 8, size - 8); + if (code == NULL) + return NULL; + if (!PyCode_Check(code)) { + Py_DECREF(code); + PyErr_Format(PyExc_TypeError, + "compiled module %.200s is not a code object", + pathname); + return NULL; + } + return code; +} + +/* Replace any occurances of "\r\n?" in the input string with "\n". + This converts DOS and Mac line endings to Unix line endings. + Also append a trailing "\n" to be compatible with + PyParser_SimpleParseFile(). Returns a new reference. */ +static PyObject * +normalize_line_endings(PyObject *source) +{ + char *q, *p = PyString_AsString(source); + int length = PyString_Size(source) + 1; + PyObject *fixed_source; + + fixed_source = PyString_FromStringAndSize(p, length); + if (fixed_source == NULL) + return NULL; + + q = PyString_AsString(fixed_source); + /* replace "\r\n?" by "\n" */ + for (;;) { + if (*p == '\r') { + *q++ = '\n'; + if (*(p + 1) == '\n') { + p++; + length--; + } + } + else + *q++ = *p; + if (*p == '\0') + break; + p++; + } + *q++ = '\n'; /* add trailing \n */ + *q = '\0'; + _PyString_Resize(&fixed_source, length); + return fixed_source; +} + +/* Given a string buffer containing Python source code, compile it + return and return a code object as a new reference. */ +static PyObject * +compile_source(char *pathname, PyObject *source) +{ + PyObject *code, *fixed_source; + + fixed_source = normalize_line_endings(source); + if (fixed_source == NULL) + return NULL; + + code = Py_CompileString(PyString_AsString(fixed_source), pathname, + Py_file_input); + Py_DECREF(fixed_source); + return code; +} + +/* Convert the date/time values found in the Zip archive to a value + that's compatible with the time stamp stored in .pyc files. */ +time_t parse_dostime(int dostime, int dosdate) +{ + struct tm stm; + + stm.tm_sec = (dostime & 0x1f) * 2; + stm.tm_min = (dostime >> 5) & 0x3f; + stm.tm_hour = (dostime >> 11) & 0x1f; + stm.tm_mday = dosdate & 0x1f; + stm.tm_mon = ((dosdate >> 5) & 0x0f) - 1; + stm.tm_year = ((dosdate >> 9) & 0x7f) + 80; + stm.tm_isdst = 0; /* wday/yday is ignored */ + + return mktime(&stm); +} + +/* Given a path to a .pyc or .pyo file in the archive, return the + modifictaion time of the matching .py file, or 0 if no source + is available. */ +static time_t +get_mtime_of_source(ZipImporter *self, char *path) +{ + PyObject *toc_entry; + time_t mtime = 0; + int lastchar = strlen(path) - 1; + char savechar = path[lastchar]; + path[lastchar] = '\0'; /* strip 'c' or 'o' from *.py[co] */ + toc_entry = PyDict_GetItemString(self->files, path); + if (toc_entry != NULL && PyTuple_Check(toc_entry) && + PyTuple_Size(toc_entry) == 8) { + /* fetch the time stamp of the .py file for comparison + with an embedded pyc time stamp */ + int time, date; + time = PyInt_AsLong(PyTuple_GetItem(toc_entry, 5)); + date = PyInt_AsLong(PyTuple_GetItem(toc_entry, 6)); + mtime = parse_dostime(time, date); + } + path[lastchar] = savechar; + return mtime; +} + +/* Return the code object for the module named by 'fullname' from the + Zip archive as a new reference. */ +static PyObject * +get_code_from_data(ZipImporter *self, int ispackage, int isbytecode, + time_t mtime, PyObject *toc_entry) +{ + PyObject *data, *code; + char *modpath; + char *archive = PyString_AsString(self->archive); + + if (archive == NULL) + return NULL; + + data = get_data(archive, toc_entry); + if (data == NULL) + return NULL; + + modpath = PyString_AsString(PyTuple_GetItem(toc_entry, 0)); + + if (isbytecode) { + code = unmarshal_code(modpath, data, mtime); + } + else { + code = compile_source(modpath, data); + } + Py_DECREF(data); + return code; +} + +/* Get the code object assoiciated with the module specified by + 'fullname'. */ +static PyObject * +get_module_code(ZipImporter *self, char *fullname, + int *p_ispackage, char **p_modpath) +{ + PyObject *toc_entry; + char *subname, path[MAXPATHLEN + 1]; + int len; + struct st_zip_searchorder *zso; + + subname = get_subname(fullname); + + len = make_filename(PyString_AsString(self->prefix), subname, path); + if (len < 0) + return NULL; + + for (zso = zip_searchorder; *zso->suffix; zso++) { + PyObject *code = NULL; + + strcpy(path + len, zso->suffix); + if (Py_VerboseFlag > 1) + PySys_WriteStderr("# trying %s%c%s\n", + PyString_AsString(self->archive), + SEP, path); + toc_entry = PyDict_GetItemString(self->files, path); + if (toc_entry != NULL) { + time_t mtime = 0; + int ispackage = zso->type & IS_PACKAGE; + int isbytecode = zso->type & IS_BYTECODE; + + if (isbytecode) + mtime = get_mtime_of_source(self, path); + if (p_ispackage != NULL) + *p_ispackage = ispackage; + code = get_code_from_data(self, ispackage, + isbytecode, mtime, + toc_entry); + if (code == Py_None) { + /* bad magic number or non-matching mtime + in byte code, try next */ + Py_DECREF(code); + continue; + } + if (code != NULL && p_modpath != NULL) + *p_modpath = PyString_AsString( + PyTuple_GetItem(toc_entry, 0)); + return code; + } + } + PyErr_Format(ZipImportError, "can't find module '%.200s'", fullname); + return NULL; +} + + +/* Module init */ + +PyDoc_STRVAR(zipimport_doc, +"zipimport provides support for importing Python modules from Zip archives.\n\ +\n\ +This module exports three objects:\n\ +- zipimporter: a class; its constructor takes a path to a Zip archive.\n\ +- ZipImporterError: exception raised by zipimporter objects. It's a\n\ + subclass of ImportError, so it can be caught as ImportError, too.\n\ +- _zip_directory_cache: a dict, mapping archive paths to zip directory\n\ + info dicts, as used in zipimporter._files.\n\ +\n\ +It is usually not needed to use the zipimport module explicitly; it is\n\ +used by the builtin import mechanism for sys.path items that are paths\n\ +to Zip archives."); + +PyMODINIT_FUNC +initzipimport(void) +{ + PyObject *mod; + + if (PyType_Ready(&ZipImporter_Type) < 0) + return; + + /* Correct directory separator */ + zip_searchorder[0].suffix[0] = SEP; + zip_searchorder[1].suffix[0] = SEP; + zip_searchorder[2].suffix[0] = SEP; + if (Py_OptimizeFlag) { + /* Reverse *.pyc and *.pyo */ + struct st_zip_searchorder tmp; + tmp = zip_searchorder[0]; + zip_searchorder[0] = zip_searchorder[1]; + zip_searchorder[1] = tmp; + tmp = zip_searchorder[3]; + zip_searchorder[3] = zip_searchorder[4]; + zip_searchorder[4] = tmp; + } + + mod = Py_InitModule4("zipimport", NULL, zipimport_doc, + NULL, PYTHON_API_VERSION); + + ZipImportError = PyErr_NewException("zipimport.ZipImportError", + PyExc_ImportError, NULL); + if (ZipImportError == NULL) + return; + + Py_INCREF(ZipImportError); + if (PyModule_AddObject(mod, "ZipImportError", + ZipImportError) < 0) + return; + + Py_INCREF(&ZipImporter_Type); + if (PyModule_AddObject(mod, "zipimporter", + (PyObject *)&ZipImporter_Type) < 0) + return; + + zip_directory_cache = PyDict_New(); + if (zip_directory_cache == NULL) + return; + Py_INCREF(zip_directory_cache); + if (PyModule_AddObject(mod, "_zip_directory_cache", + zip_directory_cache) < 0) + return; +} diff --git a/PC/config.c b/PC/config.c index af023bf..b20831d 100644 --- a/PC/config.c +++ b/PC/config.c @@ -43,6 +43,7 @@ extern void initxreadlines(void); extern void init_weakref(void); extern void init_hotshot(void); extern void initxxsubtype(void); +extern void initzipimport(void); extern void init_random(void); /* XXX tim: what's the purpose of ADDMODULE MARKER? */ @@ -98,6 +99,7 @@ struct _inittab _PyImport_Inittab[] = { {"_random", init_random}, {"xxsubtype", initxxsubtype}, + {"zipimport", initzipimport}, /* XXX tim: what's the purpose of ADDMODULE MARKER? */ /* -- ADDMODULE MARKER 2 -- */ diff --git a/PC/getpathp.c b/PC/getpathp.c index a38224a..4930ad8 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -81,6 +81,7 @@ static char prefix[MAXPATHLEN+1]; static char progpath[MAXPATHLEN+1]; +static char dllpath[MAXPATHLEN+1]; static char *module_search_path = NULL; @@ -350,6 +351,7 @@ get_progpath(void) char *prog = Py_GetProgramName(); #ifdef MS_WINDOWS + extern HANDLE PyWin_DLLhModule; #ifdef UNICODE WCHAR wprogpath[MAXPATHLEN+1]; /* Windows documents that GetModuleFileName() will "truncate", @@ -357,6 +359,14 @@ get_progpath(void) PLUS Windows itself defines MAX_PATH as the same, but anyway... */ wprogpath[MAXPATHLEN]=_T('\0'); + if (PyWin_DLLhModule && + GetModuleFileName(PyWin_DLLhModule, wprogpath, MAXPATHLEN)) { + WideCharToMultiByte(CP_ACP, 0, + wprogpath, -1, + dllpath, MAXPATHLEN+1, + NULL, NULL); + } + wprogpath[MAXPATHLEN]=_T('\0')'; if (GetModuleFileName(NULL, wprogpath, MAXPATHLEN)) { WideCharToMultiByte(CP_ACP, 0, wprogpath, -1, @@ -366,6 +376,9 @@ get_progpath(void) } #else /* static init of progpath ensures final char remains \0 */ + if (PyWin_DLLhModule) + if (!GetModuleFileName(PyWin_DLLhModule, dllpath, MAXPATHLEN)) + dllpath[0] = 0; if (GetModuleFileName(NULL, progpath, MAXPATHLEN)) return; #endif @@ -427,6 +440,8 @@ calculate_path(void) int skiphome, skipdefault; char *machinepath = NULL; char *userpath = NULL; + char zip_path[MAXPATHLEN+1]; + size_t len; #endif get_progpath(); @@ -447,6 +462,21 @@ calculate_path(void) #ifdef MS_WINDOWS + /* Calculate zip archive path */ + if (dllpath[0]) /* use name of python DLL */ + strncpy(zip_path, dllpath, MAXPATHLEN); + else /* use name of executable program */ + strncpy(zip_path, progpath, MAXPATHLEN); + len = strlen(zip_path); + if (len > 4) { + zip_path[len-3] = 'z'; /* change ending to "zip" */ + zip_path[len-2] = 'i'; + zip_path[len-1] = 'p'; + } + else { + zip_path[0] = 0; + } + skiphome = pythonhome==NULL ? 0 : 1; machinepath = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); userpath = getpythonregpath(HKEY_CURRENT_USER, skiphome); @@ -458,14 +488,15 @@ calculate_path(void) /* We need to construct a path from the following parts. (1) the PYTHONPATH environment variable, if set; - (2) for Win32, the machinepath and userpath, if set; - (3) the PYTHONPATH config macro, with the leading "." + (2) for Win32, the zip archive file path; + (3) for Win32, the machinepath and userpath, if set; + (4) the PYTHONPATH config macro, with the leading "." of each component replaced with pythonhome, if set; - (4) the directory containing the executable (argv0_path). - The length calculation calculates #3 first. + (5) the directory containing the executable (argv0_path). + The length calculation calculates #4 first. Extra rules: - - If PYTHONHOME is set (in any way) item (2) is ignored. - - If registry values are used, (3) and (4) are ignored. + - If PYTHONHOME is set (in any way) item (3) is ignored. + - If registry values are used, (4) and (5) are ignored. */ /* Calculate size of return buffer */ @@ -487,6 +518,7 @@ calculate_path(void) bufsz += strlen(userpath) + 1; if (machinepath) bufsz += strlen(machinepath) + 1; + bufsz += strlen(zip_path) + 1; #endif if (envpath != NULL) bufsz += strlen(envpath) + 1; @@ -518,6 +550,11 @@ calculate_path(void) *buf++ = DELIM; } #ifdef MS_WINDOWS + if (zip_path[0]) { + strcpy(buf, zip_path); + buf = strchr(buf, '\0'); + *buf++ = DELIM; + } if (userpath) { strcpy(buf, userpath); buf = strchr(buf, '\0'); diff --git a/PCbuild/pythoncore.dsp b/PCbuild/pythoncore.dsp index 5e98a7c..0c5d43d 100644 --- a/PCbuild/pythoncore.dsp +++ b/PCbuild/pythoncore.dsp @@ -549,5 +549,9 @@ SOURCE=..\Modules\xxsubtype.c SOURCE=..\Modules\yuvconvert.c # End Source File +# Begin Source File + +SOURCE=..\Modules\zipimport.c +# End Source File # End Target # End Project diff --git a/Python/import.c b/Python/import.c index 9b3944a..5df1f0a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -153,6 +153,72 @@ _PyImport_Init(void) } void +_PyImportHooks_Init(void) +{ + PyObject *v, *path_hooks = NULL, *zimpimport; + int err = 0; + + /* adding sys.path_hooks and sys.path_importer_cache, setting up + zipimport */ + + if (Py_VerboseFlag) + PySys_WriteStderr("# installing zipimport hook\n"); + + v = PyList_New(0); + if (v == NULL) + goto error; + err = PySys_SetObject("meta_path", v); + Py_DECREF(v); + if (err) + goto error; + v = PyDict_New(); + if (v == NULL) + goto error; + err = PySys_SetObject("path_importer_cache", v); + Py_DECREF(v); + if (err) + goto error; + path_hooks = PyList_New(0); + if (path_hooks == NULL) + goto error; + err = PySys_SetObject("path_hooks", path_hooks); + if (err) { + error: + PyErr_Print(); + Py_FatalError("initializing sys.meta_path, sys.path_hooks or " + "path_importer_cache failed"); + } + zimpimport = PyImport_ImportModule("zipimport"); + if (zimpimport == NULL) { + PyErr_Clear(); /* No zip import module -- okay */ + if (Py_VerboseFlag) + PySys_WriteStderr("# can't import zipimport\n"); + } + else { + PyObject *zipimporter = PyObject_GetAttrString(zimpimport, + "zipimporter"); + Py_DECREF(zimpimport); + if (zipimporter == NULL) { + PyErr_Clear(); /* No zipimporter object -- okay */ + if (Py_VerboseFlag) + PySys_WriteStderr( + "# can't import zipimport.zimimporter\n"); + } + else { + /* sys.path_hooks.append(zipimporter) */ + err = PyList_Append(path_hooks, zipimporter); + Py_DECREF(zipimporter); + if (err) + goto error; + if (Py_VerboseFlag) + PySys_WriteStderr( + "# installed zipimport hook\n"); + } + } + Py_DECREF(path_hooks); +} + +void _PyImport_Fini(void) { Py_XDECREF(extensions); @@ -246,6 +312,7 @@ static char* sys_deletes[] = { "path", "argv", "ps1", "ps2", "exitfunc", "exc_type", "exc_value", "exc_traceback", "last_type", "last_value", "last_traceback", + "path_hooks", "path_importer_cache", "meta_path", NULL }; @@ -808,9 +875,9 @@ load_source_module(char *name, char *pathname, FILE *fp) /* Forward */ -static PyObject *load_module(char *, FILE *, char *, int); -static struct filedescr *find_module(char *, PyObject *, - char *, size_t, FILE **); +static PyObject *load_module(char *, FILE *, char *, int, PyObject *); +static struct filedescr *find_module(char *, char *, PyObject *, + char *, size_t, FILE **, PyObject **); static struct _frozen *find_frozen(char *name); /* Load a package and return its module object WITH INCREMENTED @@ -848,7 +915,7 @@ load_package(char *name, char *pathname) goto cleanup; } buf[0] = '\0'; - fdp = find_module("__init__", path, buf, sizeof(buf), &fp); + fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { PyErr_Clear(); @@ -857,7 +924,7 @@ load_package(char *name, char *pathname) m = NULL; goto cleanup; } - m = load_module(name, fp, buf, fdp->type); + m = load_module(name, fp, buf, fdp->type, NULL); if (fp != NULL) fclose(fp); cleanup: @@ -885,6 +952,61 @@ is_builtin(char *name) } +/* Return an importer object for a sys.path/pkg.__path__ item 'p', + possibly by fetching it from the path_importer_cache dict. If it + wasn't yet cached, traverse path_hooks until it a hook is found + that can handle the path item. Return None if no hook could; + this tells our caller it should fall back to the builtin + import mechanism. Cache the result in path_importer_cache. + Returns a borrowed reference. */ + +static PyObject * +get_path_importer(PyObject *path_importer_cache, PyObject *path_hooks, + PyObject *p) +{ + PyObject *importer; + int j, nhooks; + + /* These conditions are the caller's responsibility: */ + assert(PyList_Check(path_hooks)); + assert(PyDict_Check(path_importer_cache)); + + nhooks = PyList_Size(path_hooks); + if (nhooks < 0) + return NULL; /* Shouldn't happen */ + + importer = PyDict_GetItem(path_importer_cache, p); + if (importer != NULL) + return importer; + + /* set path_importer_cache[p] to None to avoid recursion */ + if (PyDict_SetItem(path_importer_cache, p, Py_None) != 0) + return NULL; + + for (j = 0; j < nhooks; j++) { + PyObject *hook = PyList_GetItem(path_hooks, j); + if (hook == NULL) + return NULL; + importer = PyObject_CallFunction(hook, "O", p); + if (importer != NULL) + break; + + if (!PyErr_ExceptionMatches(PyExc_ImportError)) { + return NULL; + } + PyErr_Clear(); + } + if (importer == NULL) + importer = Py_None; + else if (importer != Py_None) { + int err = PyDict_SetItem(path_importer_cache, p, importer); + Py_DECREF(importer); + if (err != 0) + return NULL; + } + return importer; +} + /* Search the path (default sys.path) for a module. Return the corresponding filedescr struct, and (via return arguments) the pathname and an open file. Return NULL if the module is not found. */ @@ -896,16 +1018,18 @@ extern FILE *PyWin_FindRegisteredModule(const char *, struct filedescr **, static int case_ok(char *, int, int, char *); static int find_init_module(char *); /* Forward */ +static struct filedescr importhookdescr = {"", "", IMP_HOOK}; static struct filedescr * -find_module(char *realname, PyObject *path, char *buf, size_t buflen, - FILE **p_fp) +find_module(char *fullname, char *subname, PyObject *path, char *buf, + size_t buflen, FILE **p_fp, PyObject **p_loader) { int i, npath; size_t len, namelen; struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; + PyObject *path_hooks, *path_importer_cache; #ifndef RISCOS struct stat statbuf; #endif @@ -918,13 +1042,50 @@ find_module(char *realname, PyObject *path, char *buf, size_t buflen, size_t saved_namelen; char *saved_buf = NULL; #endif + if (p_loader != NULL) + *p_loader = NULL; - if (strlen(realname) > MAXPATHLEN) { + if (strlen(subname) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "module name is too long"); return NULL; } - strcpy(name, realname); + strcpy(name, subname); + + /* sys.meta_path import hook */ + if (p_loader != NULL) { + PyObject *meta_path; + + meta_path = PySys_GetObject("meta_path"); + if (meta_path == NULL || !PyList_Check(meta_path)) { + PyErr_SetString(PyExc_ImportError, + "sys.meta_path must be a list of " + "import hooks"); + return NULL; + } + Py_INCREF(meta_path); /* zap guard */ + npath = PyList_Size(meta_path); + for (i = 0; i < npath; i++) { + PyObject *loader; + PyObject *hook = PyList_GetItem(meta_path, i); + loader = PyObject_CallMethod(hook, "find_module", + "sO", fullname, + path != NULL ? + path : Py_None); + if (loader == NULL) { + Py_DECREF(meta_path); + return NULL; /* true error */ + } + if (loader != Py_None) { + /* a loader was found */ + *p_loader = loader; + Py_DECREF(meta_path); + return &importhookdescr; + } + Py_DECREF(loader); + } + Py_DECREF(meta_path); + } if (path != NULL && PyString_Check(path)) { /* The only type of submodule allowed inside a "frozen" @@ -978,6 +1139,22 @@ find_module(char *realname, PyObject *path, char *buf, size_t buflen, "sys.path must be a list of directory names"); return NULL; } + + path_hooks = PySys_GetObject("path_hooks"); + if (path_hooks == NULL || !PyList_Check(path_hooks)) { + PyErr_SetString(PyExc_ImportError, + "sys.path_hooks must be a list of " + "import hooks"); + return NULL; + } + path_importer_cache = PySys_GetObject("path_importer_cache"); + if (path_importer_cache == NULL || + !PyDict_Check(path_importer_cache)) { + PyErr_SetString(PyExc_ImportError, + "sys.path_importer_cache must be a dict"); + return NULL; + } + npath = PyList_Size(path); namelen = strlen(name); for (i = 0; i < npath; i++) { @@ -1005,6 +1182,33 @@ find_module(char *realname, PyObject *path, char *buf, size_t buflen, Py_XDECREF(copy); continue; /* v contains '\0' */ } + + /* sys.path_hooks import hook */ + if (p_loader != NULL) { + PyObject *importer; + + importer = get_path_importer(path_importer_cache, + path_hooks, v); + if (importer == NULL) + return NULL; + /* Note: importer is a borrowed reference */ + if (importer != Py_None) { + PyObject *loader; + loader = PyObject_CallMethod(importer, + "find_module", + "s", fullname); + if (loader == NULL) + return NULL; /* error */ + if (loader != Py_None) { + /* a loader was found */ + *p_loader = loader; + return &importhookdescr; + } + Py_DECREF(loader); + } + /* no hook was successful, use builtin import */ + } + #ifdef macintosh /* ** Speedup: each sys.path item is interned, and @@ -1079,7 +1283,7 @@ find_module(char *realname, PyObject *path, char *buf, size_t buflen, * dynamically loaded module we're going to try, * truncate the name before trying */ - if (strlen(realname) > 8) { + if (strlen(subname) > 8) { /* is this an attempt to load a C extension? */ const struct filedescr *scan; scan = _PyImport_DynLoadFiletab; @@ -1092,7 +1296,7 @@ find_module(char *realname, PyObject *path, char *buf, size_t buflen, if (scan->suffix != NULL) { /* yes, so truncate the name */ namelen = 8; - len -= strlen(realname) - namelen; + len -= strlen(subname) - namelen; buf[len] = '\0'; } } @@ -1444,7 +1648,7 @@ static int init_builtin(char *); /* Forward */ its module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_module(char *name, FILE *fp, char *buf, int type) +load_module(char *name, FILE *fp, char *buf, int type, PyObject *loader) { PyObject *modules; PyObject *m; @@ -1523,6 +1727,16 @@ load_module(char *name, FILE *fp, char *buf, int type) Py_INCREF(m); break; + case IMP_HOOK: { + if (loader == NULL) { + PyErr_SetString(PyExc_ImportError, + "import hook without loader"); + return NULL; + } + m = PyObject_CallMethod(loader, "load_module", "s", name); + break; + } + default: PyErr_Format(PyExc_ImportError, "Don't know how to import %.200s (type code %d)", @@ -1978,7 +2192,7 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(m); } else { - PyObject *path; + PyObject *path, *loader = NULL; char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp = NULL; @@ -1995,7 +2209,8 @@ import_submodule(PyObject *mod, char *subname, char *fullname) } buf[0] = '\0'; - fdp = find_module(subname, path, buf, MAXPATHLEN+1, &fp); + fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1, + &fp, &loader); Py_XDECREF(path); if (fdp == NULL) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) @@ -2004,7 +2219,8 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(Py_None); return Py_None; } - m = load_module(fullname, fp, buf, fdp->type); + m = load_module(fullname, fp, buf, fdp->type, loader); + Py_XDECREF(loader); if (fp) fclose(fp); if (mod != Py_None) { @@ -2080,11 +2296,11 @@ PyImport_ReloadModule(PyObject *m) PyErr_Clear(); } buf[0] = '\0'; - fdp = find_module(subname, path, buf, MAXPATHLEN+1, &fp); + fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, NULL); Py_XDECREF(path); if (fdp == NULL) return NULL; - m = load_module(name, fp, buf, fdp->type); + m = load_module(name, fp, buf, fdp->type, NULL); if (fp) fclose(fp); return m; @@ -2228,7 +2444,7 @@ call_find_module(char *name, PyObject *path) pathname[0] = '\0'; if (path == Py_None) path = NULL; - fdp = find_module(name, path, pathname, MAXPATHLEN+1, &fp); + fdp = find_module(NULL, name, path, pathname, MAXPATHLEN+1, &fp, NULL); if (fdp == NULL) return NULL; if (fp != NULL) { @@ -2465,7 +2681,7 @@ imp_load_module(PyObject *self, PyObject *args) if (fp == NULL) return NULL; } - return load_module(name, fp, pathname, type); + return load_module(name, fp, pathname, type, NULL); } static PyObject * @@ -2579,6 +2795,7 @@ initimp(void) if (setint(d, "C_BUILTIN", C_BUILTIN) < 0) goto failure; if (setint(d, "PY_FROZEN", PY_FROZEN) < 0) goto failure; if (setint(d, "PY_CODERESOURCE", PY_CODERESOURCE) < 0) goto failure; + if (setint(d, "IMP_HOOK", IMP_HOOK) < 0) goto failure; failure: ; diff --git a/Python/importdl.h b/Python/importdl.h index 8e04f28..5a2d45c 100644 --- a/Python/importdl.h +++ b/Python/importdl.h @@ -16,7 +16,8 @@ enum filetype { PKG_DIRECTORY, C_BUILTIN, PY_FROZEN, - PY_CODERESOURCE /* Mac only */ + PY_CODERESOURCE, /* Mac only */ + IMP_HOOK }; struct filedescr { diff --git a/Python/pythonrun.c b/Python/pythonrun.c index a31c2d9..7469cb8 100644 --- a/Python/pythonrun.c +++ b/Python/pythonrun.c @@ -161,6 +161,8 @@ Py_Initialize(void) /* phase 2 of builtins */ _PyImport_FixupExtension("__builtin__", "__builtin__"); + _PyImportHooks_Init(); + initsigs(); /* Signal handling stuff, including initintr() */ initmain(); /* Module __main__ */ -- cgit v0.12