Merge.

24 files changed, 249 insertions, 47 deletions

diff --git a/Lib/asyncio/base_subprocess.py b/Lib/asyncio/base_subprocess.py
index d15fb15..c5efda7 100644
--- a/Lib/asyncio/base_subprocess.py
+++ b/Lib/asyncio/base_subprocess.py
@@ -114,7 +114,6 @@ class BaseSubprocessTransport(transports.SubprocessTransport):
         assert returncode is not None, returncode
         assert self._returncode is None, self._returncode
         self._returncode = returncode
-        self._loop._subprocess_closed(self)
         self._call(self._protocol.process_exited)
         self._try_finish()
 
diff --git a/Lib/asyncio/unix_events.py b/Lib/asyncio/unix_events.py
index 2418642..219c88a 100644
--- a/Lib/asyncio/unix_events.py
+++ b/Lib/asyncio/unix_events.py
@@ -169,9 +169,6 @@ class _UnixSelectorEventLoop(selector_events.BaseSelectorEventLoop):
     def _child_watcher_callback(self, pid, returncode, transp):
         self.call_soon_threadsafe(transp._process_exited, returncode)
 
-    def _subprocess_closed(self, transp):
-        pass
-
 
 def _set_nonblocking(fd):
     flags = fcntl.fcntl(fd, fcntl.F_GETFL)
diff --git a/Lib/asyncio/windows_events.py b/Lib/asyncio/windows_events.py
index 2e9ec69..3c21e43 100644
--- a/Lib/asyncio/windows_events.py
+++ b/Lib/asyncio/windows_events.py
@@ -178,9 +178,6 @@ class ProactorEventLoop(proactor_events.BaseProactorEventLoop):
         yield from transp._post_init()
         return transp
 
-    def _subprocess_closed(self, transport):
-        pass
-
 
 class IocpProactor:
     """Proactor implementation using IOCP."""
diff --git a/Lib/codecs.py b/Lib/codecs.py
index 2e2e755..c2065da 100644
--- a/Lib/codecs.py
+++ b/Lib/codecs.py
@@ -475,15 +475,12 @@ class StreamReader(Codec):
         # read until we get the required number of characters (if available)
         while True:
             # can the request be satisfied from the character buffer?
-            if chars < 0:
-                if size < 0:
-                    if self.charbuffer:
-                        break
-                elif len(self.charbuffer) >= size:
-                    break
-            else:
+            if chars >= 0:
                 if len(self.charbuffer) >= chars:
                     break
+            elif size >= 0:
+                if len(self.charbuffer) >= size:
+                    break
             # we need more data
             if size < 0:
                 newdata = self.stream.read()
@@ -491,6 +488,8 @@ class StreamReader(Codec):
                 newdata = self.stream.read(size)
             # decode bytes (those remaining from the last call included)
             data = self.bytebuffer + newdata
+            if not data:
+                break
             try:
                 newchars, decodedbytes = self.decode(data, self.errors)
             except UnicodeDecodeError as exc:
diff --git a/Lib/concurrent/futures/_base.py b/Lib/concurrent/futures/_base.py
index 3d03280..c3b1f01 100644
--- a/Lib/concurrent/futures/_base.py
+++ b/Lib/concurrent/futures/_base.py
@@ -181,7 +181,8 @@ def as_completed(fs, timeout=None):
 
     Returns:
         An iterator that yields the given Futures as they complete (finished or
-        cancelled).
+        cancelled). If any given Futures are duplicated, they will be returned
+        once.
 
     Raises:
         TimeoutError: If the entire result iterator could not be generated
@@ -190,11 +191,12 @@ def as_completed(fs, timeout=None):
     if timeout is not None:
         end_time = timeout + time.time()
 
+    fs = set(fs)
     with _AcquireFutures(fs):
         finished = set(
                 f for f in fs
                 if f._state in [CANCELLED_AND_NOTIFIED, FINISHED])
-        pending = set(fs) - finished
+        pending = fs - finished
         waiter = _create_and_install_waiters(fs, _AS_COMPLETED)
 
     try:
diff --git a/Lib/idlelib/CallTips.py b/Lib/idlelib/CallTips.py
index 4f88da6..81bd5f1 100644
--- a/Lib/idlelib/CallTips.py
+++ b/Lib/idlelib/CallTips.py
@@ -5,16 +5,16 @@ parameter and docstring information when you type an opening parenthesis, and
 which disappear when you type a closing parenthesis.
 
 """
+import __main__
+import inspect
 import re
 import sys
+import textwrap
 import types
-import inspect
 
 from idlelib import CallTipWindow
 from idlelib.HyperParser import HyperParser
 
-import __main__
-
 class CallTips:
 
     menudefs = [
@@ -117,8 +117,9 @@ def get_entity(expression):
             return None
 
 # The following are used in get_argspec and some in tests
-_MAX_COLS = 79
+_MAX_COLS = 85
 _MAX_LINES = 5  # enough for bytes
+_INDENT = ' '*4  # for wrapped signatures
 _first_param = re.compile('(?<=\()\w*\,?\s*')
 _default_callable_argspec = "See source or doc"
 
@@ -149,13 +150,15 @@ def get_argspec(ob):
                 isinstance(ob_call, types.MethodType)):
             argspec = _first_param.sub("", argspec)
 
+    lines = (textwrap.wrap(argspec, _MAX_COLS, subsequent_indent=_INDENT)
+            if len(argspec) > _MAX_COLS else [argspec] if argspec else [])
+
     if isinstance(ob_call, types.MethodType):
         doc = ob_call.__doc__
     else:
         doc = getattr(ob, "__doc__", "")
     if doc:
-        lines = [argspec] if argspec else []
-        for line in doc.split('\n', 5)[:_MAX_LINES]:
+        for line in doc.split('\n', _MAX_LINES)[:_MAX_LINES]:
             line = line.strip()
             if not line:
                 break
diff --git a/Lib/idlelib/configDialog.py b/Lib/idlelib/configDialog.py
index 1f4a3a5..efe5c43 100644
--- a/Lib/idlelib/configDialog.py
+++ b/Lib/idlelib/configDialog.py
@@ -82,9 +82,10 @@ class ConfigDialog(Toplevel):
         else:
             extraKwds=dict(padx=6, pady=3)
 
-        self.buttonHelp = Button(frameActionButtons,text='Help',
-                command=self.Help,takefocus=FALSE,
-                **extraKwds)
+# Comment out button creation and packing until implement self.Help
+##        self.buttonHelp = Button(frameActionButtons,text='Help',
+##                command=self.Help,takefocus=FALSE,
+##                **extraKwds)
         self.buttonOk = Button(frameActionButtons,text='Ok',
                 command=self.Ok,takefocus=FALSE,
                 **extraKwds)
@@ -98,7 +99,7 @@ class ConfigDialog(Toplevel):
         self.CreatePageHighlight()
         self.CreatePageKeys()
         self.CreatePageGeneral()
-        self.buttonHelp.pack(side=RIGHT,padx=5)
+##        self.buttonHelp.pack(side=RIGHT,padx=5)
         self.buttonOk.pack(side=LEFT,padx=5)
         self.buttonApply.pack(side=LEFT,padx=5)
         self.buttonCancel.pack(side=LEFT,padx=5)
diff --git a/Lib/idlelib/idle_test/test_calltips.py b/Lib/idlelib/idle_test/test_calltips.py
index 5b51732..ab69bd0 100644
--- a/Lib/idlelib/idle_test/test_calltips.py
+++ b/Lib/idlelib/idle_test/test_calltips.py
@@ -1,5 +1,6 @@
 import unittest
 import idlelib.CallTips as ct
+import textwrap
 import types
 
 default_tip = ct._default_callable_argspec
@@ -55,32 +56,45 @@ class Get_signatureTest(unittest.TestCase):
         gtest(list.__new__,
                'T.__new__(S, ...) -> a new object with type S, a subtype of T')
         gtest(list.__init__,
-               'Initializes self.  See help(type(self)) for accurate signature.')
+               'x.__init__(...) initializes x; see help(type(x)) for signature')
         append_doc =  "L.append(object) -> None -- append object to end"
         gtest(list.append, append_doc)
         gtest([].append, append_doc)
         gtest(List.append, append_doc)
 
-        gtest(types.MethodType, "Create a bound instance method object.")
+        gtest(types.MethodType, "method(function, instance)")
         gtest(SB(), default_tip)
 
+    def test_signature_wrap(self):
+        self.assertEqual(signature(textwrap.TextWrapper), '''\
+(width=70, initial_indent='', subsequent_indent='', expand_tabs=True,
+    replace_whitespace=True, fix_sentence_endings=False, break_long_words=True,
+    drop_whitespace=True, break_on_hyphens=True, tabsize=8, *, max_lines=None,
+    placeholder=' [...]')''')
+
+    def test_docline_truncation(self):
+        def f(): pass
+        f.__doc__ = 'a'*300
+        self.assertEqual(signature(f), '()\n' + 'a' * (ct._MAX_COLS-3) + '...')
+
     def test_multiline_docstring(self):
         # Test fewer lines than max.
-        self.assertEqual(signature(dict),
-                "dict(mapping) -> new dictionary initialized from a mapping object's\n"
-                "(key, value) pairs\n"
-                "dict(iterable) -> new dictionary initialized as if via:\n"
-                "d = {}\n"
-                "for k, v in iterable:"
-                )
-
-        # Test max lines and line (currently) too long.
-        self.assertEqual(signature(bytes),
-"bytes(string, encoding[, errors]) -> bytes\n"
-"bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer\n"
-#bytes(int) -> bytes object of size given by the parameter initialized with null bytes
-"bytes(int) -> bytes object of size given by the parameter initialized with n...\n"
-"bytes() -> empty bytes object")
+        self.assertEqual(signature(list),
+                "list() -> new empty list\n"
+                "list(iterable) -> new list initialized from iterable's items")
+
+        # Test max lines
+        self.assertEqual(signature(bytes), '''\
+bytes(iterable_of_ints) -> bytes
+bytes(string, encoding[, errors]) -> bytes
+bytes(bytes_or_buffer) -> immutable copy of bytes_or_buffer
+bytes(int) -> bytes object of size given by the parameter initialized with null bytes
+bytes() -> empty bytes object''')
+
+        # Test more than max lines
+        def f(): pass
+        f.__doc__ = 'a\n' * 15
+        self.assertEqual(signature(f), '()' + '\na' * ct._MAX_LINES)
 
     def test_functions(self):
         def t1(): 'doc'
@@ -109,6 +123,16 @@ class Get_signatureTest(unittest.TestCase):
                             (tc.__call__, '(ci)'), (tc, '(ci)'), (TC.cm, "(a)"),):
             self.assertEqual(signature(meth), mtip + "\ndoc")
 
+    def test_starred_parameter(self):
+        # test that starred first parameter is *not* removed from argspec
+        class C:
+            def m1(*args): pass
+            def m2(**kwds): pass
+        c = C()
+        for meth, mtip  in ((C.m1, '(*args)'), (c.m1, "(*args)"),
+                                      (C.m2, "(**kwds)"), (c.m2, "(**kwds)"),):
+            self.assertEqual(signature(meth), mtip)
+
     def test_non_ascii_name(self):
         # test that re works to delete a first parameter name that
         # includes non-ascii chars, such as various forms of A.
diff --git a/Lib/test/imghdrdata/python.bmp b/Lib/test/imghdrdata/python.bmp
new file mode 100644
index 0000000..675f951
--- /dev/null
+++ b/Lib/test/imghdrdata/python.bmp
diff --git a/Lib/test/imghdrdata/python.gif b/Lib/test/imghdrdata/python.gif
new file mode 100644
index 0000000..96fd9fe
--- /dev/null
+++ b/Lib/test/imghdrdata/python.gif
diff --git a/Lib/test/imghdrdata/python.jpg b/Lib/test/imghdrdata/python.jpg
new file mode 100644
index 0000000..21222c0
--- /dev/null
+++ b/Lib/test/imghdrdata/python.jpg
diff --git a/Lib/test/imghdrdata/python.pbm b/Lib/test/imghdrdata/python.pbm
new file mode 100644
index 0000000..1848ba7
--- /dev/null
+++ b/Lib/test/imghdrdata/python.pbm
@@ -0,0 +1,3 @@
+P4
+16 16
+ûñ¿úßÕ­±[ñ¥a_ÁX
°
°
ðððð?ÿÿ
\ No newline at end of file
diff --git a/Lib/test/imghdrdata/python.pgm b/Lib/test/imghdrdata/python.pgm
new file mode 100644
index 0000000..8349f2a
--- /dev/null
+++ b/Lib/test/imghdrdata/python.pgm
diff --git a/Lib/test/imghdrdata/python.png b/Lib/test/imghdrdata/python.png
new file mode 100644
index 0000000..1a987f7
--- /dev/null
+++ b/Lib/test/imghdrdata/python.png
diff --git a/Lib/test/imghdrdata/python.ppm b/Lib/test/imghdrdata/python.ppm
new file mode 100644
index 0000000..7d9cdb3
--- /dev/null
+++ b/Lib/test/imghdrdata/python.ppm
diff --git a/Lib/test/imghdrdata/python.ras b/Lib/test/imghdrdata/python.ras
new file mode 100644
index 0000000..130e96f
--- /dev/null
+++ b/Lib/test/imghdrdata/python.ras
diff --git a/Lib/test/imghdrdata/python.sgi b/Lib/test/imghdrdata/python.sgi
new file mode 100644
index 0000000..ffe9081
--- /dev/null
+++ b/Lib/test/imghdrdata/python.sgi
diff --git a/Lib/test/imghdrdata/python.tiff b/Lib/test/imghdrdata/python.tiff
new file mode 100644
index 0000000..39d0bfc
--- /dev/null
+++ b/Lib/test/imghdrdata/python.tiff
diff --git a/Lib/test/imghdrdata/python.xbm b/Lib/test/imghdrdata/python.xbm
new file mode 100644
index 0000000..cfbee2e
--- /dev/null
+++ b/Lib/test/imghdrdata/python.xbm
@@ -0,0 +1,6 @@
+#define python_width 16
+#define python_height 16
+static char python_bits[] = {
+  0xDF, 0xFE, 0x8F, 0xFD, 0x5F, 0xFB, 0xAB, 0xFE, 0xB5, 0x8D, 0xDA, 0x8F, 
+  0xA5, 0x86, 0xFA, 0x83, 0x1A, 0x80, 0x0D, 0x80, 0x0D, 0x80, 0x0F, 0xE0, 
+  0x0F, 0xF8, 0x0F, 0xF8, 0x0F, 0xFC, 0xFF, 0xFF, };
diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py
index f9c9e69..3950c3b 100644
--- a/Lib/test/test_codecs.py
+++ b/Lib/test/test_codecs.py
@@ -175,6 +175,40 @@ class ReadTest(MixInCheckStateHandling):
                         size*"a",
                     )
 
+    def test_mixed_readline_and_read(self):
+        lines = ["Humpty Dumpty sat on a wall,\n",
+                 "Humpty Dumpty had a great fall.\r\n",
+                 "All the king's horses and all the king's men\r",
+                 "Couldn't put Humpty together again."]
+        data = ''.join(lines)
+        def getreader():
+            stream = io.BytesIO(data.encode(self.encoding))
+            return codecs.getreader(self.encoding)(stream)
+
+        # Issue #8260: Test readline() followed by read()
+        f = getreader()
+        self.assertEqual(f.readline(), lines[0])
+        self.assertEqual(f.read(), ''.join(lines[1:]))
+        self.assertEqual(f.read(), '')
+
+        # Issue #16636: Test readline() followed by readlines()
+        f = getreader()
+        self.assertEqual(f.readline(), lines[0])
+        self.assertEqual(f.readlines(), lines[1:])
+        self.assertEqual(f.read(), '')
+
+        # Test read() followed by read()
+        f = getreader()
+        self.assertEqual(f.read(size=40, chars=5), data[:5])
+        self.assertEqual(f.read(), data[5:])
+        self.assertEqual(f.read(), '')
+
+        # Issue #12446: Test read() followed by readlines()
+        f = getreader()
+        self.assertEqual(f.read(size=40, chars=5), data[:5])
+        self.assertEqual(f.readlines(), [lines[0][5:]] + lines[1:])
+        self.assertEqual(f.read(), '')
+
     def test_bug1175396(self):
         s = [
             '<%!--===================================================\r\n',
@@ -2370,8 +2404,6 @@ class TransformCodecTest(unittest.TestCase):
 
     def test_readline(self):
         for encoding in bytes_transform_encodings:
-            if encoding in ['uu_codec', 'zlib_codec']:
-                continue
             with self.subTest(encoding=encoding):
                 sin = codecs.encode(b"\x80", encoding)
                 reader = codecs.getreader(encoding)(io.BytesIO(sin))
@@ -2522,6 +2554,7 @@ class ExceptionChainingTest(unittest.TestCase):
         with self.assertRaisesRegex(exc_type, full_msg) as caught:
             yield caught
         self.assertIsInstance(caught.exception.__cause__, exc_type)
+        self.assertIsNotNone(caught.exception.__cause__.__traceback__)
 
     def raise_obj(self, *args, **kwds):
         # Helper to dynamically change the object raised by a test codec
diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py
index a7e945c..c74b2ca 100644
--- a/Lib/test/test_concurrent_futures.py
+++ b/Lib/test/test_concurrent_futures.py
@@ -350,6 +350,13 @@ class AsCompletedTests:
                               SUCCESSFUL_FUTURE]),
                          completed_futures)
 
+    def test_duplicate_futures(self):
+        # Issue 20367. Duplicate futures should not raise exceptions or give
+        # duplicate responses.
+        future1 = self.executor.submit(time.sleep, 2)
+        completed = [f for f in futures.as_completed([future1,future1])]
+        self.assertEqual(len(completed), 1)
+
 
 class ThreadPoolAsCompletedTests(ThreadPoolMixin, AsCompletedTests, unittest.TestCase):
     pass
diff --git a/Lib/test/test_imghdr.py b/Lib/test/test_imghdr.py
new file mode 100644
index 0000000..0ad4343
--- /dev/null
+++ b/Lib/test/test_imghdr.py
@@ -0,0 +1,131 @@
+import imghdr
+import io
+import os
+import unittest
+import warnings
+from test.support import findfile, TESTFN, unlink
+
+TEST_FILES = (
+    ('python.png', 'png'),
+    ('python.gif', 'gif'),
+    ('python.bmp', 'bmp'),
+    ('python.ppm', 'ppm'),
+    ('python.pgm', 'pgm'),
+    ('python.pbm', 'pbm'),
+    ('python.jpg', 'jpeg'),
+    ('python.ras', 'rast'),
+    ('python.sgi', 'rgb'),
+    ('python.tiff', 'tiff'),
+    ('python.xbm', 'xbm')
+)
+
+class UnseekableIO(io.FileIO):
+    def tell(self):
+        raise io.UnsupportedOperation
+
+    def seek(self, *args, **kwargs):
+        raise io.UnsupportedOperation
+
+class TestImghdr(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.testfile = findfile('python.png', subdir='imghdrdata')
+        with open(cls.testfile, 'rb') as stream:
+            cls.testdata = stream.read()
+
+    def tearDown(self):
+        unlink(TESTFN)
+
+    def test_data(self):
+        for filename, expected in TEST_FILES:
+            filename = findfile(filename, subdir='imghdrdata')
+            self.assertEqual(imghdr.what(filename), expected)
+            with open(filename, 'rb') as stream:
+                self.assertEqual(imghdr.what(stream), expected)
+            with open(filename, 'rb') as stream:
+                data = stream.read()
+            self.assertEqual(imghdr.what(None, data), expected)
+            self.assertEqual(imghdr.what(None, bytearray(data)), expected)
+
+    def test_register_test(self):
+        def test_jumbo(h, file):
+            if h.startswith(b'eggs'):
+                return 'ham'
+        imghdr.tests.append(test_jumbo)
+        self.addCleanup(imghdr.tests.pop)
+        self.assertEqual(imghdr.what(None, b'eggs'), 'ham')
+
+    def test_file_pos(self):
+        with open(TESTFN, 'wb') as stream:
+            stream.write(b'ababagalamaga')
+            pos = stream.tell()
+            stream.write(self.testdata)
+        with open(TESTFN, 'rb') as stream:
+            stream.seek(pos)
+            self.assertEqual(imghdr.what(stream), 'png')
+            self.assertEqual(stream.tell(), pos)
+
+    def test_bad_args(self):
+        with self.assertRaises(TypeError):
+            imghdr.what()
+        with self.assertRaises(AttributeError):
+            imghdr.what(None)
+        with self.assertRaises(TypeError):
+            imghdr.what(self.testfile, 1)
+        with self.assertRaises(AttributeError):
+            imghdr.what(os.fsencode(self.testfile))
+        with open(self.testfile, 'rb') as f:
+            with self.assertRaises(AttributeError):
+                imghdr.what(f.fileno())
+
+    def test_invalid_headers(self):
+        for header in (b'\211PN\r\n',
+                       b'\001\331',
+                       b'\x59\xA6',
+                       b'cutecat',
+                       b'000000JFI',
+                       b'GIF80'):
+            self.assertIsNone(imghdr.what(None, header))
+
+    def test_string_data(self):
+        with warnings.catch_warnings():
+            warnings.simplefilter("ignore", BytesWarning)
+            for filename, _ in TEST_FILES:
+                filename = findfile(filename, subdir='imghdrdata')
+                with open(filename, 'rb') as stream:
+                    data = stream.read().decode('latin1')
+                with self.assertRaises(TypeError):
+                    imghdr.what(io.StringIO(data))
+                with self.assertRaises(TypeError):
+                    imghdr.what(None, data)
+
+    def test_missing_file(self):
+        with self.assertRaises(FileNotFoundError):
+            imghdr.what('missing')
+
+    def test_closed_file(self):
+        stream = open(self.testfile, 'rb')
+        stream.close()
+        with self.assertRaises(ValueError) as cm:
+            imghdr.what(stream)
+        stream = io.BytesIO(self.testdata)
+        stream.close()
+        with self.assertRaises(ValueError) as cm:
+            imghdr.what(stream)
+
+    def test_unseekable(self):
+        with open(TESTFN, 'wb') as stream:
+            stream.write(self.testdata)
+        with UnseekableIO(TESTFN, 'rb') as stream:
+            with self.assertRaises(io.UnsupportedOperation):
+                imghdr.what(stream)
+
+    def test_output_stream(self):
+        with open(TESTFN, 'wb') as stream:
+            stream.write(self.testdata)
+            stream.seek(0)
+            with self.assertRaises(OSError) as cm:
+                imghdr.what(stream)
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py
index 2da3ac0..e99ca9e 100644
--- a/Lib/test/test_sundry.py
+++ b/Lib/test/test_sundry.py
@@ -6,7 +6,7 @@ import unittest
 
 class TestUntestedModules(unittest.TestCase):
     def test_untested_modules_can_be_imported(self):
-        untested = ('bdb', 'encodings', 'formatter', 'imghdr',
+        untested = ('bdb', 'encodings', 'formatter',
                     'nturl2path', 'tabnanny')
         with support.check_warnings(quiet=True):
             for name in untested:
diff --git a/Lib/wsgiref/__init__.py b/Lib/wsgiref/__init__.py
index 46c579f..1efbba0 100644
--- a/Lib/wsgiref/__init__.py
+++ b/Lib/wsgiref/__init__.py
@@ -1,4 +1,4 @@
-"""wsgiref -- a WSGI (PEP 333) Reference Library
+"""wsgiref -- a WSGI (PEP 3333) Reference Library
 
 Current Contents: