Merge with hg.python.org.

12 files changed, 197 insertions, 64 deletions

diff --git a/Lib/dbm/__init__.py b/Lib/dbm/__init__.py
index b2a254a..6e890f3 100644
--- a/Lib/dbm/__init__.py
+++ b/Lib/dbm/__init__.py
@@ -67,10 +67,10 @@ def open(file, flag = 'r', mode = 0o666):
         if not _defaultmod:
             raise ImportError("no dbm clone found; tried %s" % _names)
 
-    # guess the type of an existing database
-    result = whichdb(file)
+    # guess the type of an existing database, if not creating a new one
+    result = whichdb(file) if 'n' not in flag else None
     if result is None:
-        # db doesn't exist
+        # db doesn't exist or 'n' flag was specified to create a new db
         if 'c' in flag or 'n' in flag:
             # file doesn't exist and the new flag was used so use default type
             mod = _defaultmod
diff --git a/Lib/email/generator.py b/Lib/email/generator.py
index 531fa9a..f0e7a95 100644
--- a/Lib/email/generator.py
+++ b/Lib/email/generator.py
@@ -59,7 +59,7 @@ class Generator:
         self._fp.write(s)
 
     def flatten(self, msg, unixfrom=False, linesep='\n'):
-        """Print the message object tree rooted at msg to the output file
+        r"""Print the message object tree rooted at msg to the output file
         specified when the Generator instance was created.
 
         unixfrom is a flag that forces the printing of a Unix From_ delimiter
@@ -70,7 +70,10 @@ class Generator:
         Note that for subobjects, no From_ line is printed.
 
         linesep specifies the characters used to indicate a new line in
-        the output.
+        the output.  The default value is the most useful for typical
+        Python applications, but it can be set to \r\n to produce RFC-compliant
+        line separators when needed.
+
         """
         # We use the _XXX constants for operating on data that comes directly
         # from the msg, and _encoded_XXX constants for operating on data that
diff --git a/Lib/email/header.py b/Lib/email/header.py
index 35cdb2b..e171617 100644
--- a/Lib/email/header.py
+++ b/Lib/email/header.py
@@ -276,7 +276,7 @@ class Header:
         self._chunks.append((s, charset))
 
     def encode(self, splitchars=';, \t', maxlinelen=None, linesep='\n'):
-        """Encode a message header into an RFC-compliant format.
+        r"""Encode a message header into an RFC-compliant format.
 
         There are many issues involved in converting a given string for use in
         an email header.  Only certain character sets are readable in most
diff --git a/Lib/subprocess.py b/Lib/subprocess.py
index c8af5d1..69b769b 100644
--- a/Lib/subprocess.py
+++ b/Lib/subprocess.py
@@ -191,8 +191,10 @@ should prepare for OSErrors.
 
 A ValueError will be raised if Popen is called with invalid arguments.
 
-check_call() and check_output() will raise CalledProcessError, if the
-called process returns a non-zero return code.
+Exceptions defined within this module inherit from SubprocessError.
+check_call() and check_output() will raise CalledProcessError if the
+called process returns a non-zero return code.  TimeoutExpired
+be raised if a timeout was specified and expired.
 
 
 Security
@@ -348,7 +350,10 @@ import builtins
 import warnings
 
 # Exception classes used by this module.
-class CalledProcessError(Exception):
+class SubprocessError(Exception): pass
+
+
+class CalledProcessError(SubprocessError):
     """This exception is raised when a process run by check_call() or
     check_output() returns a non-zero exit status.
     The exit status will be stored in the returncode attribute;
@@ -362,7 +367,7 @@ class CalledProcessError(Exception):
         return "Command '%s' returned non-zero exit status %d" % (self.cmd, self.returncode)
 
 
-class TimeoutExpired(Exception):
+class TimeoutExpired(SubprocessError):
     """This exception is raised when the timeout expires while waiting for a
     child process.
     """
diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py
index 74c9c44..26d4c14 100644
--- a/Lib/test/test_dbm.py
+++ b/Lib/test/test_dbm.py
@@ -70,6 +70,14 @@ class AnyDBMTestCase(unittest.TestCase):
         self.read_helper(f)
         f.close()
 
+    def test_anydbm_creation_n_file_exists_with_invalid_contents(self):
+        with open(_fname, "w") as w:
+            pass # create an empty file
+
+        f = dbm.open(_fname, 'n')
+        self.addCleanup(f.close)
+        self.assertEqual(len(f), 0)
+
     def test_anydbm_modification(self):
         self.init_db()
         f = dbm.open(_fname, 'c')
diff --git a/Lib/test/test_pep292.py b/Lib/test/test_pep292.py
index a967649..a1e52e9 100644
--- a/Lib/test/test_pep292.py
+++ b/Lib/test/test_pep292.py
@@ -42,6 +42,19 @@ class TestTemplate(unittest.TestCase):
         s = Template('$who likes $$')
         eq(s.substitute(dict(who='tim', what='ham')), 'tim likes $')
 
+    def test_invalid(self):
+        class MyPattern(Template):
+            pattern = r"""
+            (?:
+            (?P<invalid>)            |
+            (?P<escaped>%(delim)s)   |
+            @(?P<named>%(id)s)       |
+            @{(?P<braced>%(id)s)}
+            )
+            """
+        s = MyPattern('$')
+        self.assertRaises(ValueError, s.substitute, dict())
+
     def test_percents(self):
         eq = self.assertEqual
         s = Template('%(foo)s $foo ${foo}')
diff --git a/Lib/test/test_readline.py b/Lib/test/test_readline.py
index 5f5a90a..5483dd3 100644
--- a/Lib/test/test_readline.py
+++ b/Lib/test/test_readline.py
@@ -12,6 +12,10 @@ from test.support import run_unittest, import_module
 readline = import_module('readline')
 
 class TestHistoryManipulation (unittest.TestCase):
+
+    @unittest.skipIf(not hasattr(readline, 'clear_history'),
+                     "The history update test cannot be run because the "
+                     "clear_history method is not available.")
     def testHistoryUpdates(self):
         readline.clear_history()
 
diff --git a/Lib/test/test_string.py b/Lib/test/test_string.py
index b495d69..c4b186a 100644
--- a/Lib/test/test_string.py
+++ b/Lib/test/test_string.py
@@ -112,6 +112,30 @@ class ModuleTest(unittest.TestCase):
         self.assertRaises(ValueError, fmt.format, "{0}", 10, 20, i=100)
         self.assertRaises(ValueError, fmt.format, "{i}", 10, 20, i=100)
 
+    def test_vformat_assert(self):
+        cls = string.Formatter()
+        kwargs = {
+            "i": 100
+        }
+        self.assertRaises(ValueError, cls._vformat,
+                cls.format, "{0}", kwargs, set(), -2)
+
+    def test_convert_field(self):
+        cls = string.Formatter()
+        self.assertEqual(cls.format("{0!s}", 'foo'), 'foo')
+        self.assertRaises(ValueError, cls.format, "{0!h}", 'foo')
+
+    def test_get_field(self):
+        cls = string.Formatter()
+        class MyClass:
+            name = 'lumberjack'
+        x = MyClass()
+        self.assertEqual(cls.format("{0.name}", x), 'lumberjack')
+
+        lookup = ["eggs", "and", "spam"]
+        self.assertEqual(cls.format("{0[2]}", lookup), 'spam')
+
+
 def test_main():
     support.run_unittest(ModuleTest)
 
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index 4ddbe3f..1d0d98c 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -7,7 +7,9 @@ import socket
 import array
 
 import urllib.request
-from urllib.request import Request, OpenerDirector
+# The proxy bypass method imported below has logic specific to the OSX
+# proxy config data structure but is testable on all platforms.
+from urllib.request import Request, OpenerDirector, _proxy_bypass_macosx_sysconf
 
 # XXX
 # Request
@@ -1076,6 +1078,17 @@ class HandlerTests(unittest.TestCase):
         self.assertEqual(req.get_host(), "www.python.org")
         del os.environ['no_proxy']
 
+    def test_proxy_no_proxy_all(self):
+        os.environ['no_proxy'] = '*'
+        o = OpenerDirector()
+        ph = urllib.request.ProxyHandler(dict(http="proxy.example.com"))
+        o.add_handler(ph)
+        req = Request("http://www.python.org")
+        self.assertEqual(req.get_host(), "www.python.org")
+        r = o.open(req)
+        self.assertEqual(req.get_host(), "www.python.org")
+        del os.environ['no_proxy']
+
 
     def test_proxy_https(self):
         o = OpenerDirector()
@@ -1116,6 +1129,26 @@ class HandlerTests(unittest.TestCase):
         self.assertEqual(req.get_host(), "proxy.example.com:3128")
         self.assertEqual(req.get_header("Proxy-authorization"),"FooBar")
 
+    def test_osx_proxy_bypass(self):
+        bypass = {
+            'exclude_simple': False,
+            'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.10',
+                           '10.0/16']
+        }
+        # Check hosts that should trigger the proxy bypass
+        for host in ('foo.bar', 'www.bar.com', '127.0.0.1', '10.10.0.1',
+                     '10.0.0.1'):
+            self.assertTrue(_proxy_bypass_macosx_sysconf(host, bypass),
+                            'expected bypass of %s to be True' % host)
+        # Check hosts that should not trigger the proxy bypass
+        for host in ('abc.foo.bar', 'bar.com', '127.0.0.2', '10.11.0.1', 'test'):
+            self.assertFalse(_proxy_bypass_macosx_sysconf(host, bypass),
+                             'expected bypass of %s to be False' % host)
+
+        # Check the exclude_simple flag
+        bypass = {'exclude_simple': True, 'exceptions': []}
+        self.assertTrue(_proxy_bypass_macosx_sysconf('test', bypass))
+
     def test_basic_auth(self, quote_char='"'):
         opener = OpenerDirector()
         password_manager = MockPasswordManager()
diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py
index 270e5e8..4e47707 100644
--- a/Lib/unittest/case.py
+++ b/Lib/unittest/case.py
@@ -469,7 +469,7 @@ class TestCase(object):
                         warnings.warn("TestResult has no addExpectedFailure method, reporting as passes",
                                       RuntimeWarning)
                         result.addSuccess(self)
-
+            return result
         finally:
             result.stopTest(self)
             if orig_result is None:
diff --git a/Lib/unittest/test/test_case.py b/Lib/unittest/test/test_case.py
index 351b6f8..852ac86 100644
--- a/Lib/unittest/test/test_case.py
+++ b/Lib/unittest/test/test_case.py
@@ -386,27 +386,62 @@ class Test_TestCase(unittest.TestCase, TestEquality, TestHashing):
         self.assertIsInstance(Foo().id(), str)
 
 
-    # "If result is omitted or None, a temporary result object is created
-    # and used, but is not made available to the caller. As TestCase owns the
+    # "If result is omitted or None, a temporary result object is created,
+    # used, and is made available to the caller. As TestCase owns the
     # temporary result startTestRun and stopTestRun are called.
 
     def test_run__uses_defaultTestResult(self):
         events = []
+        defaultResult = LoggingResult(events)
 
         class Foo(unittest.TestCase):
             def test(self):
                 events.append('test')
 
             def defaultTestResult(self):
-                return LoggingResult(events)
+                return defaultResult
 
         # Make run() find a result object on its own
-        Foo('test').run()
+        result = Foo('test').run()
 
+        self.assertIs(result, defaultResult)
         expected = ['startTestRun', 'startTest', 'test', 'addSuccess',
             'stopTest', 'stopTestRun']
         self.assertEqual(events, expected)
 
+
+    # "The result object is returned to run's caller"
+    def test_run__returns_given_result(self):
+
+        class Foo(unittest.TestCase):
+            def test(self):
+                pass
+
+        result = unittest.TestResult()
+
+        retval = Foo('test').run(result)
+        self.assertIs(retval, result)
+
+
+    # "The same effect [as method run] may be had by simply calling the
+    # TestCase instance."
+    def test_call__invoking_an_instance_delegates_to_run(self):
+        resultIn = unittest.TestResult()
+        resultOut = unittest.TestResult()
+
+        class Foo(unittest.TestCase):
+            def test(self):
+                pass
+
+            def run(self, result):
+                self.assertIs(result, resultIn)
+                return resultOut
+
+        retval = Foo('test')(resultIn)
+
+        self.assertIs(retval, resultOut)
+
+
     def testShortDescriptionWithoutDocstring(self):
         self.assertIsNone(self.shortDescription())
 
diff --git a/Lib/urllib/request.py b/Lib/urllib/request.py
index d583a82..96eeb78 100644
--- a/Lib/urllib/request.py
+++ b/Lib/urllib/request.py
@@ -2208,68 +2208,76 @@ def proxy_bypass_environment(host):
     return 0
 
 
-if sys.platform == 'darwin':
-    from _scproxy import _get_proxy_settings, _get_proxies
-
-    def proxy_bypass_macosx_sysconf(host):
-        """
-        Return True iff this host shouldn't be accessed using a proxy
+# This code tests an OSX specific data structure but is testable on all
+# platforms
+def _proxy_bypass_macosx_sysconf(host, proxy_settings):
+    """
+    Return True iff this host shouldn't be accessed using a proxy
 
-        This function uses the MacOSX framework SystemConfiguration
-        to fetch the proxy information.
-        """
-        import re
-        import socket
-        from fnmatch import fnmatch
+    This function uses the MacOSX framework SystemConfiguration
+    to fetch the proxy information.
 
-        hostonly, port = splitport(host)
+    proxy_settings come from _scproxy._get_proxy_settings or get mocked ie:
+    { 'exclude_simple': bool,
+      'exceptions': ['foo.bar', '*.bar.com', '127.0.0.1', '10.1', '10.0/16']
+    }
+    """
+    import re
+    import socket
+    from fnmatch import fnmatch
 
-        def ip2num(ipAddr):
-            parts = ipAddr.split('.')
-            parts = list(map(int, parts))
-            if len(parts) != 4:
-                parts = (parts + [0, 0, 0, 0])[:4]
-            return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
+    hostonly, port = splitport(host)
 
-        proxy_settings = _get_proxy_settings()
+    def ip2num(ipAddr):
+        parts = ipAddr.split('.')
+        parts = list(map(int, parts))
+        if len(parts) != 4:
+            parts = (parts + [0, 0, 0, 0])[:4]
+        return (parts[0] << 24) | (parts[1] << 16) | (parts[2] << 8) | parts[3]
 
-        # Check for simple host names:
-        if '.' not in host:
-            if proxy_settings['exclude_simple']:
-                return True
+    # Check for simple host names:
+    if '.' not in host:
+        if proxy_settings['exclude_simple']:
+            return True
 
-        hostIP = None
+    hostIP = None
 
-        for value in proxy_settings.get('exceptions', ()):
-            # Items in the list are strings like these: *.local, 169.254/16
-            if not value: continue
+    for value in proxy_settings.get('exceptions', ()):
+        # Items in the list are strings like these: *.local, 169.254/16
+        if not value: continue
 
-            m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
-            if m is not None:
-                if hostIP is None:
-                    try:
-                        hostIP = socket.gethostbyname(hostonly)
-                        hostIP = ip2num(hostIP)
-                    except socket.error:
-                        continue
+        m = re.match(r"(\d+(?:\.\d+)*)(/\d+)?", value)
+        if m is not None:
+            if hostIP is None:
+                try:
+                    hostIP = socket.gethostbyname(hostonly)
+                    hostIP = ip2num(hostIP)
+                except socket.error:
+                    continue
+
+            base = ip2num(m.group(1))
+            mask = m.group(2)
+            if mask is None:
+                mask = 8 * (m.group(1).count('.') + 1)
+            else:
+                mask = int(mask[1:])
+            mask = 32 - mask
 
-                base = ip2num(m.group(1))
-                mask = m.group(2)
-                if mask is None:
-                    mask = 8 * (m.group(1).count('.') + 1)
+            if (hostIP >> mask) == (base >> mask):
+                return True
 
-                else:
-                    mask = int(mask[1:])
-                    mask = 32 - mask
+        elif fnmatch(host, value):
+            return True
 
-                if (hostIP >> mask) == (base >> mask):
-                    return True
+    return False
 
-            elif fnmatch(host, value):
-                return True
 
-        return False
+if sys.platform == 'darwin':
+    from _scproxy import _get_proxy_settings, _get_proxies
 
+    def proxy_bypass_macosx_sysconf(host):
+        proxy_settings = _get_proxy_settings()
+        return _proxy_bypass_macosx_sysconf(host, proxy_settings)
 
     def getproxies_macosx_sysconf():
         """Return a dictionary of scheme -> proxy server URL mappings.