Need for speed: Patch #921466 : sys.path_importer_cache is now used to cache valid and

  invalid file paths for the built-in import machinery which leads to
  fewer open calls on startup.

  Also fix issue with PEP 302 style import hooks which lead to more open()
  calls than necessary.

3 files changed, 38 insertions, 4 deletions

diff --git a/Lib/pkgutil.py b/Lib/pkgutil.py
index ddf2a72..26c797f 100644
--- a/Lib/pkgutil.py
+++ b/Lib/pkgutil.py
@@ -340,11 +340,13 @@ def get_importer(path_item):
             importer = None
         sys.path_importer_cache.setdefault(path_item, importer)
 
-    if importer is None:
+    # The boolean values are used for caching valid and invalid
+    # file paths for the built-in import machinery
+    if importer in (None, True, False):
         try:
             importer = ImpImporter(path_item)
         except ImportError:
-            pass
+            importer = None
     return importer
 
 
diff --git a/Misc/NEWS b/Misc/NEWS
index be5e4ea..a18e9e7 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -12,6 +12,10 @@ What's New in Python 2.5 alpha 3?
 Core and builtins
 -----------------
 
+- Patch #921466: sys.path_importer_cache is now used to cache valid and
+  invalid file paths for the built-in import machinery which leads to
+  fewer open calls on startup.
+
 - Patch #1442927: ``long(str, base)`` is now up to 6x faster for non-power-
   of-2 bases.  The largest speedup is for inputs with about 1000 decimal
   digits.  Conversion from non-power-of-2 bases remains quadratic-time in
diff --git a/Python/import.c b/Python/import.c
index 862f33c..e09365b 100644
--- a/Python/import.c
+++ b/Python/import.c
@@ -1240,7 +1240,33 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
 			if (importer == NULL)
 				return NULL;
 			/* Note: importer is a borrowed reference */
-			if (importer != Py_None) {
+			if (importer == Py_False) {
+				/* Cached as not being a valid dir. */
+				Py_XDECREF(copy);
+				continue;
+			}
+			else if (importer == Py_True) {
+				/* Cached as being a valid dir, so just
+				 * continue below. */
+			}
+			else if (importer == Py_None) {
+				/* No importer was found, so it has to be a file.
+				 * Check if the directory is valid. */
+#ifdef HAVE_STAT
+				if (stat(buf, &statbuf) != 0) {
+					/* Directory does not exist. */
+					PyDict_SetItem(path_importer_cache,
+					               v, Py_False);
+					Py_XDECREF(copy);
+					continue;
+				} else {
+					PyDict_SetItem(path_importer_cache,
+					               v, Py_True);
+				}
+#endif
+			}
+			else {
+				/* A real import hook importer was found. */
 				PyObject *loader;
 				loader = PyObject_CallMethod(importer,
 							     "find_module",
@@ -1253,9 +1279,11 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf,
 					return &importhookdescr;
 				}
 				Py_DECREF(loader);
+				Py_XDECREF(copy);
+				continue;
 			}
-			/* no hook was successful, use builtin import */
 		}
+		/* no hook was found, use builtin import */
 
 		if (len > 0 && buf[len-1] != SEP
 #ifdef ALTSEP