Merge part of the trunk changes into the p3yk branch. This merges from 43030

(branch-creation time) up to 43067. 43068 and 43069 contain a little
swapping action between re.py and sre.py, and this mightily confuses svn
merge, so later changes are going in separately.

This merge should break no additional tests.

The last-merged revision is going in a 'last_merge' property on '.' (the
branch directory.) Arbitrarily chosen, really; if there's a BCP for this, I
couldn't find it, but we can easily change it afterwards ;)

6 files changed, 240 insertions, 13 deletions

diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py
index b4c229a..b850912 100755
--- a/Lib/test/regrtest.py
+++ b/Lib/test/regrtest.py
@@ -1136,9 +1136,6 @@ class _ExpectedSkips:
             s = _expectations[sys.platform]
             self.expected = set(s.split())
 
-            # this isn't a regularly run unit test, it is always skipped
-            self.expected.add('test_hashlib_speed')
-
             if not os.path.supports_unicode_filenames:
                 self.expected.add('test_pep277')
 
diff --git a/Lib/test/test_StringIO.py b/Lib/test/test_StringIO.py
index c61f7cc..cc3367f 100644
--- a/Lib/test/test_StringIO.py
+++ b/Lib/test/test_StringIO.py
@@ -75,6 +75,13 @@ class TestGenericStringIO(unittest.TestCase):
         f.close()
         self.assertEqual(f.closed, True)
 
+    def test_isatty(self):
+        f = self.MODULE.StringIO()
+        self.assertRaises(TypeError, f.isatty, None)
+        self.assertEqual(f.isatty(), False)
+        f.close()
+        self.assertRaises(ValueError, f.isatty)
+
     def test_iterator(self):
         eq = self.assertEqual
         unless = self.failUnless
@@ -87,6 +94,8 @@ class TestGenericStringIO(unittest.TestCase):
             eq(line, self._line + '\n')
             i += 1
         eq(i, 5)
+        self._fp.close()
+        self.assertRaises(ValueError, self._fp.next)
 
 class TestStringIO(TestGenericStringIO):
     MODULE = StringIO
diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py
index 7bf5314..74a9cdf 100644
--- a/Lib/test/test___all__.py
+++ b/Lib/test/test___all__.py
@@ -5,8 +5,6 @@ from test.test_support import verify, verbose
 import sys
 import warnings
 
-warnings.filterwarnings("ignore", ".* 'pre' .*", DeprecationWarning,
-                        r'pre$')
 warnings.filterwarnings("ignore", ".* regsub .*", DeprecationWarning,
                         r'^regsub$')
 warnings.filterwarnings("ignore",
@@ -122,7 +120,6 @@ class AllTest(unittest.TestCase):
         self.check_all("poplib")
         self.check_all("posixpath")
         self.check_all("pprint")
-        self.check_all("pre")  # deprecated
         self.check_all("profile")
         self.check_all("pstats")
         self.check_all("pty")
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index 3944d65..913aa91 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -41,6 +41,33 @@ class ReadTest(unittest.TestCase):
         self.assertEqual(r.bytebuffer, "")
         self.assertEqual(r.charbuffer, u"")
 
+        # do the check again, this time using a incremental decoder
+        d = codecs.getincrementaldecoder(self.encoding)()
+        result = u""
+        for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
+            result += d.decode(c)
+            self.assertEqual(result, partialresult)
+        # check that there's nothing left in the buffers
+        self.assertEqual(d.decode("", True), u"")
+        self.assertEqual(d.buffer, "")
+
+        # Check whether the rest method works properly
+        d.reset()
+        result = u""
+        for (c, partialresult) in zip(input.encode(self.encoding), partialresults):
+            result += d.decode(c)
+            self.assertEqual(result, partialresult)
+        # check that there's nothing left in the buffers
+        self.assertEqual(d.decode("", True), u"")
+        self.assertEqual(d.buffer, "")
+
+        # check iterdecode()
+        encoded = input.encode(self.encoding)
+        self.assertEqual(
+            input,
+            u"".join(codecs.iterdecode(encoded, self.encoding))
+        )
+
     def test_readline(self):
         def getreader(input):
             stream = StringIO.StringIO(input.encode(self.encoding))
@@ -977,6 +1004,12 @@ class BasicUnicodeTest(unittest.TestCase):
     def test_basics(self):
         s = u"abc123" # all codecs should be able to encode these
         for encoding in all_unicode_encodings:
+            name = codecs.lookup(encoding).name
+            if encoding.endswith("_codec"):
+                name += "_codec"
+            elif encoding == "latin_1":
+                name = "latin_1"
+            self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-"))
             (bytes, size) = codecs.getencoder(encoding)(s)
             if encoding != "unicode_internal":
                 self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding))
@@ -999,6 +1032,30 @@ class BasicUnicodeTest(unittest.TestCase):
                     decodedresult += reader.read()
                 self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
 
+                # check incremental decoder/encoder and iterencode()/iterdecode()
+                try:
+                    encoder = codecs.getincrementalencoder(encoding)()
+                except LookupError: # no IncrementalEncoder
+                    pass
+                else:
+                    # check incremental decoder/encoder
+                    encodedresult = ""
+                    for c in s:
+                        encodedresult += encoder.encode(c)
+                    decoder = codecs.getincrementaldecoder(encoding)()
+                    decodedresult = u""
+                    for c in encodedresult:
+                        decodedresult += decoder.decode(c)
+                    self.assertEqual(decodedresult, s, "%r != %r (encoding=%r)" % (decodedresult, s, encoding))
+
+                    # check iterencode()/iterdecode()
+                    result = u"".join(codecs.iterdecode(codecs.iterencode(s, encoding), encoding))
+                    self.assertEqual(result, s, "%r != %r (encoding=%r)" % (result, s, encoding))
+
+                    # check iterencode()/iterdecode() with empty string
+                    result = u"".join(codecs.iterdecode(codecs.iterencode(u"", encoding), encoding))
+                    self.assertEqual(result, u"")
+
     def test_seek(self):
         # all codecs should be able to encode these
         s = u"%s\n%s\n" % (100*u"abc123", 100*u"def456")
diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py
new file mode 100644
index 0000000..88e9900
--- /dev/null
+++ b/Lib/test/test_runpy.py
@@ -0,0 +1,172 @@
+# Test the runpy module
+import unittest
+import os
+import os.path
+import sys
+import tempfile
+from test.test_support import verbose, run_unittest
+from runpy import _run_module_code, run_module
+
+# Set up the test code and expected results
+
+class RunModuleCodeTest(unittest.TestCase):
+
+    expected_result = ["Top level assignment", "Lower level reference"]
+    test_source = (
+        "# Check basic code execution\n"
+        "result = ['Top level assignment']\n"
+        "def f():\n"
+        "    result.append('Lower level reference')\n"
+        "f()\n"
+        "# Check the sys module\n"
+        "import sys\n"
+        "run_argv0 = sys.argv[0]\n"
+        "if __name__ in sys.modules:\n"
+        "    run_name = sys.modules[__name__].__name__\n"
+        "# Check nested operation\n"
+        "import runpy\n"
+        "nested = runpy._run_module_code('x=1\\n', mod_name='<run>',\n"
+        "                                          alter_sys=True)\n"
+    )
+
+
+    def test_run_module_code(self):
+        initial = object()
+        name = "<Nonsense>"
+        file = "Some other nonsense"
+        loader = "Now you're just being silly"
+        d1 = dict(initial=initial)
+        saved_argv0 = sys.argv[0]
+        d2 = _run_module_code(self.test_source,
+                              d1,
+                              name,
+                              file,
+                              loader,
+                              True)
+        self.failUnless("result" not in d1)
+        self.failUnless(d2["initial"] is initial)
+        self.failUnless(d2["result"] == self.expected_result)
+        self.failUnless(d2["nested"]["x"] == 1)
+        self.failUnless(d2["__name__"] is name)
+        self.failUnless(d2["run_name"] is name)
+        self.failUnless(d2["__file__"] is file)
+        self.failUnless(d2["run_argv0"] is file)
+        self.failUnless(d2["__loader__"] is loader)
+        self.failUnless(sys.argv[0] is saved_argv0)
+        self.failUnless(name not in sys.modules)
+
+    def test_run_module_code_defaults(self):
+        saved_argv0 = sys.argv[0]
+        d = _run_module_code(self.test_source)
+        self.failUnless(d["result"] == self.expected_result)
+        self.failUnless(d["__name__"] is None)
+        self.failUnless(d["__file__"] is None)
+        self.failUnless(d["__loader__"] is None)
+        self.failUnless(d["run_argv0"] is saved_argv0)
+        self.failUnless("run_name" not in d)
+        self.failUnless(sys.argv[0] is saved_argv0)
+
+class RunModuleTest(unittest.TestCase):
+
+    def expect_import_error(self, mod_name):
+        try:
+            run_module(mod_name)
+        except ImportError:
+            pass
+        else:
+            self.fail("Expected import error for " + mod_name)
+
+    def test_invalid_names(self):
+        self.expect_import_error("sys")
+        self.expect_import_error("sys.imp.eric")
+        self.expect_import_error("os.path.half")
+        self.expect_import_error("a.bee")
+        self.expect_import_error(".howard")
+        self.expect_import_error("..eaten")
+
+    def test_library_module(self):
+        run_module("runpy")
+
+    def _make_pkg(self, source, depth):
+        pkg_name = "__runpy_pkg__"
+        init_fname = "__init__"+os.extsep+"py"
+        test_fname = "runpy_test"+os.extsep+"py"
+        pkg_dir = sub_dir = tempfile.mkdtemp()
+        if verbose: print "  Package tree in:", sub_dir
+        sys.path.insert(0, pkg_dir)
+        if verbose: print "  Updated sys.path:", sys.path[0]
+        for i in range(depth):
+            sub_dir = os.path.join(sub_dir, pkg_name)
+            os.mkdir(sub_dir)
+            if verbose: print "  Next level in:", sub_dir
+            pkg_fname = os.path.join(sub_dir, init_fname)
+            pkg_file = open(pkg_fname, "w")
+            pkg_file.close()
+            if verbose: print "  Created:", pkg_fname
+        mod_fname = os.path.join(sub_dir, test_fname)
+        mod_file = open(mod_fname, "w")
+        mod_file.write(source)
+        mod_file.close()
+        if verbose: print "  Created:", mod_fname
+        mod_name = (pkg_name+".")*depth + "runpy_test"
+        return pkg_dir, mod_fname, mod_name
+
+    def _del_pkg(self, top, depth, mod_name):
+        for i in range(depth+1): # Don't forget the module itself
+            parts = mod_name.rsplit(".", i)
+            entry = parts[0]
+            try:
+                del sys.modules[entry]
+            except KeyError, ex:
+                if verbose: print ex # Persist with cleaning up
+        if verbose: print "  Removed sys.modules entries"
+        del sys.path[0]
+        if verbose: print "  Removed sys.path entry"
+        for root, dirs, files in os.walk(top, topdown=False):
+            for name in files:
+                try:
+                    os.remove(os.path.join(root, name))
+                except OSError, ex:
+                    if verbose: print ex # Persist with cleaning up
+            for name in dirs:
+                fullname = os.path.join(root, name)
+                try:
+                    os.rmdir(fullname)
+                except OSError, ex:
+                    if verbose: print ex # Persist with cleaning up
+        try:
+            os.rmdir(top)
+            if verbose: print "  Removed package tree"
+        except OSError, ex:
+            if verbose: print ex # Persist with cleaning up
+
+    def _check_module(self, depth):
+        pkg_dir, mod_fname, mod_name = (
+               self._make_pkg("x=1\n", depth))
+        try:
+            if verbose: print "Running from source:", mod_name
+            d1 = run_module(mod_name) # Read from source
+            self.failUnless(d1["x"] == 1)
+            del d1 # Ensure __loader__ entry doesn't keep file open
+            __import__(mod_name)
+            os.remove(mod_fname)
+            if verbose: print "Running from compiled:", mod_name
+            d2 = run_module(mod_name) # Read from bytecode
+            self.failUnless(d2["x"] == 1)
+            del d2 # Ensure __loader__ entry doesn't keep file open
+        finally:
+            self._del_pkg(pkg_dir, depth, mod_name)
+        if verbose: print "Module executed successfully"
+
+    def test_run_module(self):
+        for depth in range(4):
+            if verbose: print "Testing package depth:", depth
+            self._check_module(depth)
+
+
+def test_main():
+    run_unittest(RunModuleCodeTest)
+    run_unittest(RunModuleTest)
+
+if __name__ == "__main__":
+    test_main()
diff --git a/Lib/test/test_hashlib_speed.py b/Lib/test/time_hashlib.py
index c84a62b..1bf707d 100644
--- a/Lib/test/test_hashlib_speed.py
+++ b/Lib/test/time_hashlib.py
@@ -1,15 +1,14 @@
+# It's intended that this script be run by hand.  It runs speed tests on
+# hashlib functions; it does not test for correctness.
 
 import sys, time
 import hashlib
-from test import test_support
 
 
 def creatorFunc():
     raise RuntimeError, "eek, creatorFunc not overridden"
 
-
 def test_scaled_msg(scale, name):
-
     iterations = 106201/scale * 20
     longStr = 'Z'*scale
 
@@ -39,10 +38,6 @@ def test_zero():
 
 
 
-### this 'test' is not normally run.  skip it if the test runner finds it
-if __name__ != '__main__':
-    raise test_support.TestSkipped, "not a unit test (stand alone benchmark)"
-
 hName = sys.argv[1]
 
 #