diff options
Diffstat (limited to 'Lib/test')
242 files changed, 21220 insertions, 2344 deletions
diff --git a/Lib/test/buffer_tests.py b/Lib/test/buffer_tests.py index 6d20f7d..cf54c28 100644 --- a/Lib/test/buffer_tests.py +++ b/Lib/test/buffer_tests.py @@ -200,7 +200,13 @@ class MixinBytesBufferCommonTests(object): self.marshal(b'abc\ndef\r\nghi\n\r').splitlines()) self.assertEqual([b'', b'abc', b'def', b'ghi', b''], self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines()) + self.assertEqual([b'', b'abc', b'def', b'ghi', b''], + self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(False)) + self.assertEqual([b'\n', b'abc\n', b'def\r\n', b'ghi\n', b'\r'], + self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(True)) + self.assertEqual([b'', b'abc', b'def', b'ghi', b''], + self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(keepends=False)) self.assertEqual([b'\n', b'abc\n', b'def\r\n', b'ghi\n', b'\r'], - self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(1)) + self.marshal(b'\nabc\ndef\r\nghi\n\r').splitlines(keepends=True)) self.assertRaises(TypeError, self.marshal(b'abc').splitlines, 42, 42) diff --git a/Lib/test/crashers/README b/Lib/test/crashers/README index 2a73e1b..0259a06 100644 --- a/Lib/test/crashers/README +++ b/Lib/test/crashers/README @@ -14,3 +14,7 @@ note if the cause is system or environment dependent and what the variables are. Once the crash is fixed, the test case should be moved into an appropriate test (even if it was originally from the test suite). This ensures the regression doesn't happen again. And if it does, it should be easier to track down. + +Also see Lib/test_crashers.py which exercises the crashers in this directory. +In particular, make sure to add any new infinite loop crashers to the black +list so it doesn't try to run them. diff --git a/Lib/test/crashers/compiler_recursion.py b/Lib/test/crashers/compiler_recursion.py index 4954bdd..31f28a9 100644 --- a/Lib/test/crashers/compiler_recursion.py +++ b/Lib/test/crashers/compiler_recursion.py @@ -1,5 +1,13 @@ """ -The compiler (>= 2.5) recurses happily. +The compiler (>= 2.5) recurses happily until it blows the stack. + +Recorded on the tracker as http://bugs.python.org/issue11383 """ -compile('()'*9**5, '?', 'exec') +# The variant below blows up in compiler_call, but there are assorted +# other variations that blow up in other functions +# e.g. '1*'*10**5+'1' will die in compiler_visit_expr + +# The exact limit to destroy the stack will vary by platform +# but 10M should do the trick even with huge stack allocations +compile('()'*10**7, '?', 'exec') diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 38f3b8f..85036aa 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -1289,12 +1289,18 @@ class TestDate(HarmlessMixedComparison, unittest.TestCase): self.assertTrue(self.theclass.min) self.assertTrue(self.theclass.max) - def test_strftime_out_of_range(self): - # For nasty technical reasons, we can't handle years before 1000. - cls = self.theclass - self.assertEqual(cls(1000, 1, 1).strftime("%Y"), "1000") - for y in 1, 49, 51, 99, 100, 999: - self.assertRaises(ValueError, cls(y, 1, 1).strftime, "%Y") + def test_strftime_y2k(self): + for y in (1, 49, 70, 99, 100, 999, 1000, 1970): + d = self.theclass(y, 1, 1) + # Issue 13305: For years < 1000, the value is not always + # padded to 4 digits across platforms. The C standard + # assumes year >= 1900, so it does not specify the number + # of digits. + if d.strftime("%Y") != '%04d' % y: + # Year 42 returns '42', not padded + self.assertEqual(d.strftime("%Y"), '%d' % y) + # '0042' is obtained anyway + self.assertEqual(d.strftime("%4Y"), '%04d' % y) def test_replace(self): cls = self.theclass diff --git a/Lib/test/decimaltestdata/extra.decTest b/Lib/test/decimaltestdata/extra.decTest index fe8b77a..b630d8e 100644 --- a/Lib/test/decimaltestdata/extra.decTest +++ b/Lib/test/decimaltestdata/extra.decTest @@ -222,12 +222,25 @@ extr1700 power 10 1e-999999999 -> 1.000000000000000 Inexact Rounded extr1701 power 100.0 -557.71e-742888888 -> 1.000000000000000 Inexact Rounded extr1702 power 10 1e-100 -> 1.000000000000000 Inexact Rounded +-- Another one (see issue #12080). Thanks again to Stefan Krah. +extr1703 power 4 -1.2e-999999999 -> 1.000000000000000 Inexact Rounded + -- A couple of interesting exact cases for power. Note that the specification -- requires these to be reported as Inexact. extr1710 power 1e375 56e-3 -> 1.000000000000000E+21 Inexact Rounded extr1711 power 10000 0.75 -> 1000.000000000000 Inexact Rounded extr1712 power 1e-24 0.875 -> 1.000000000000000E-21 Inexact Rounded +-- Some more exact cases, exercising power with negative second argument. +extr1720 power 400 -0.5 -> 0.05000000000000000 Inexact Rounded +extr1721 power 4096 -0.75 -> 0.001953125000000000 Inexact Rounded +extr1722 power 625e4 -0.25 -> 0.02000000000000000 Inexact Rounded + +-- Nonexact cases, to exercise some of the early exit conditions from +-- _power_exact. +extr1730 power 2048 -0.75 -> 0.003284751622084822 Inexact Rounded + + -- Tests for the is_* boolean operations precision: 9 maxExponent: 999 diff --git a/Lib/test/exception_hierarchy.txt b/Lib/test/exception_hierarchy.txt index 5037b33..1c1f69f 100644 --- a/Lib/test/exception_hierarchy.txt +++ b/Lib/test/exception_hierarchy.txt @@ -11,11 +11,6 @@ BaseException +-- AssertionError +-- AttributeError +-- BufferError - +-- EnvironmentError - | +-- IOError - | +-- OSError - | +-- WindowsError (Windows) - | +-- VMSError (VMS) +-- EOFError +-- ImportError +-- LookupError @@ -24,6 +19,22 @@ BaseException +-- MemoryError +-- NameError | +-- UnboundLocalError + +-- OSError + | +-- BlockingIOError + | +-- ChildProcessError + | +-- ConnectionError + | | +-- BrokenPipeError + | | +-- ConnectionAbortedError + | | +-- ConnectionRefusedError + | | +-- ConnectionResetError + | +-- FileExistsError + | +-- FileNotFoundError + | +-- InterruptedError + | +-- IsADirectoryError + | +-- NotADirectoryError + | +-- PermissionError + | +-- ProcessLookupError + | +-- TimeoutError +-- ReferenceError +-- RuntimeError | +-- NotImplementedError diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py index 1caab1c..88527df 100644 --- a/Lib/test/fork_wait.py +++ b/Lib/test/fork_wait.py @@ -43,6 +43,7 @@ class ForkWait(unittest.TestCase): self.assertEqual(spid, cpid) self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) + @support.reap_threads def test_wait(self): for i in range(NUM_THREADS): _thread.start_new(self.f, (i,)) @@ -69,7 +70,8 @@ class ForkWait(unittest.TestCase): os._exit(n) else: # Parent - self.wait_impl(cpid) - # Tell threads to die - self.stop = 1 - time.sleep(2*SHORTSLEEP) # Wait for threads to die + try: + self.wait_impl(cpid) + finally: + # Tell threads to die + self.stop = 1 diff --git a/Lib/test/test_future1.py b/Lib/test/future_test1.py index 297c2e0..297c2e0 100644 --- a/Lib/test/test_future1.py +++ b/Lib/test/future_test1.py diff --git a/Lib/test/test_future2.py b/Lib/test/future_test2.py index 3d7fc86..3d7fc86 100644 --- a/Lib/test/test_future2.py +++ b/Lib/test/future_test2.py diff --git a/Lib/test/json_tests/test_dump.py b/Lib/test/json_tests/test_dump.py index 083c11f..4b3386f 100644 --- a/Lib/test/json_tests/test_dump.py +++ b/Lib/test/json_tests/test_dump.py @@ -1,6 +1,7 @@ from io import StringIO from test.json_tests import PyTest, CTest +from test.support import bigmemtest, _1G class TestDump: def test_dump(self): @@ -21,4 +22,20 @@ class TestDump: class TestPyDump(TestDump, PyTest): pass -class TestCDump(TestDump, CTest): pass + +class TestCDump(TestDump, CTest): + + # The size requirement here is hopefully over-estimated (actual + # memory consumption depending on implementation details, and also + # system memory management, since this may allocate a lot of + # small objects). + + @bigmemtest(size=_1G, memuse=1) + def test_large_list(self, size): + N = int(30 * 1024 * 1024 * (size / _1G)) + l = [1] * N + encoded = self.dumps(l) + self.assertEqual(len(encoded), N * 3) + self.assertEqual(encoded[:1], "[") + self.assertEqual(encoded[-2:], "1]") + self.assertEqual(encoded[1:-2], "1, " * (N - 1)) diff --git a/Lib/test/json_tests/test_scanstring.py b/Lib/test/json_tests/test_scanstring.py index f82cdee..426c8dd 100644 --- a/Lib/test/json_tests/test_scanstring.py +++ b/Lib/test/json_tests/test_scanstring.py @@ -9,14 +9,9 @@ class TestScanstring: scanstring('"z\\ud834\\udd20x"', 1, True), ('z\U0001d120x', 16)) - if sys.maxunicode == 65535: - self.assertEqual( - scanstring('"z\U0001d120x"', 1, True), - ('z\U0001d120x', 6)) - else: - self.assertEqual( - scanstring('"z\U0001d120x"', 1, True), - ('z\U0001d120x', 5)) + self.assertEqual( + scanstring('"z\U0001d120x"', 1, True), + ('z\U0001d120x', 5)) self.assertEqual( scanstring('"\\u007b"', 1, True), diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem new file mode 100644 index 0000000..e905748 --- /dev/null +++ b/Lib/test/keycert.passwd.pem @@ -0,0 +1,33 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICVDCCAb2gAwIBAgIJANfHOBkZr8JOMA0GCSqGSIb3DQEBBQUAMF8xCzAJBgNV +BAYTAlhZMRcwFQYDVQQHEw5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9u +IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDAeFw0xMDEw +MDgyMzAxNTZaFw0yMDEwMDUyMzAxNTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH +Ew5DYXN0bGUgQW50aHJheDEjMCEGA1UEChMaUHl0aG9uIFNvZnR3YXJlIEZvdW5k +YXRpb24xEjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAw +gYkCgYEA21vT5isq7F68amYuuNpSFlKDPrMUCa4YWYqZRt2OZ+/3NKaZ2xAiSwr7 +6MrQF70t5nLbSPpqE5+5VrS58SY+g/sXLiFd6AplH1wJZwh78DofbFYXUggktFMt +pTyiX8jtP66bkcPkDADA089RI1TQR6Ca+n7HFa7c1fabVV6i3zkCAwEAAaMYMBYw +FAYDVR0RBA0wC4IJbG9jYWxob3N0MA0GCSqGSIb3DQEBBQUAA4GBAHPctQBEQ4wd +BJ6+JcpIraopLn8BGhbjNWj40mmRqWB/NAWF6M5ne7KpGAu7tLeG4hb1zLaldK8G +lxy2GPSRF6LFS48dpEj2HbMv2nvv6xxalDMJ9+DicWgAKTQ6bcX2j3GUkCR0g/T1 +CRlNBAAlvhKzO7Clpf9l0YKBEfraJByX +-----END CERTIFICATE----- diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index be054ea..42e118b 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -418,6 +418,47 @@ class CommonTest(seq_tests.CommonTest): self.assertRaises(TypeError, u.reverse, 42) + def test_clear(self): + u = self.type2test([2, 3, 4]) + u.clear() + self.assertEqual(u, []) + + u = self.type2test([]) + u.clear() + self.assertEqual(u, []) + + u = self.type2test([]) + u.append(1) + u.clear() + u.append(2) + self.assertEqual(u, [2]) + + self.assertRaises(TypeError, u.clear, None) + + def test_copy(self): + u = self.type2test([1, 2, 3]) + v = u.copy() + self.assertEqual(v, [1, 2, 3]) + + u = self.type2test([]) + v = u.copy() + self.assertEqual(v, []) + + # test that it's indeed a copy and not a reference + u = self.type2test(['a', 'b']) + v = u.copy() + v.append('i') + self.assertEqual(u, ['a', 'b']) + self.assertEqual(v, u + ['i']) + + # test that it's a shallow, not a deep copy + u = self.type2test([1, 2, [3, 4], 5]) + v = u.copy() + self.assertEqual(u, v) + self.assertIs(v[3], u[3]) + + self.assertRaises(TypeError, u.copy, None) + def test_sort(self): u = self.type2test([1, 0]) u.sort() diff --git a/Lib/test/lock_tests.py b/Lib/test/lock_tests.py index 30148e6..12871c1 100644 --- a/Lib/test/lock_tests.py +++ b/Lib/test/lock_tests.py @@ -4,7 +4,7 @@ Various tests for synchronization primitives. import sys import time -from _thread import start_new_thread, get_ident, TIMEOUT_MAX +from _thread import start_new_thread, TIMEOUT_MAX import threading import unittest @@ -31,7 +31,7 @@ class Bunch(object): self.finished = [] self._can_exit = not wait_before_exit def task(): - tid = get_ident() + tid = threading.get_ident() self.started.append(tid) try: f() @@ -247,6 +247,7 @@ class RLockTests(BaseLockTests): # Cannot release an unacquired lock lock = self.locktype() self.assertRaises(RuntimeError, lock.release) + self.assertRaises(RuntimeError, lock._release_save) lock.acquire() lock.acquire() lock.release() @@ -254,6 +255,7 @@ class RLockTests(BaseLockTests): lock.release() lock.release() self.assertRaises(RuntimeError, lock.release) + self.assertRaises(RuntimeError, lock._release_save) def test_different_thread(self): # Cannot release from a different thread diff --git a/Lib/test/mailcap.txt b/Lib/test/mailcap.txt new file mode 100644 index 0000000..f61135d --- /dev/null +++ b/Lib/test/mailcap.txt @@ -0,0 +1,39 @@ +# Mailcap file for test_mailcap; based on RFC 1524 +# Referred to by test_mailcap.py + +# +# This is a comment. +# + +application/frame; showframe %s; print="cat %s | lp" +application/postscript; ps-to-terminal %s;\ + needsterminal +application/postscript; ps-to-terminal %s; \ + compose=idraw %s +application/x-dvi; xdvi %s +application/x-movie; movieplayer %s; compose=moviemaker %s; \ + description="Movie"; \ + x11-bitmap="/usr/lib/Zmail/bitmaps/movie.xbm" +application/*; echo "This is \"%t\" but \ + is 50 \% Greek to me" \; cat %s; copiousoutput + +audio/basic; showaudio %s; compose=audiocompose %s; edit=audiocompose %s;\ +description="An audio fragment" +audio/* ; /usr/local/bin/showaudio %t + +image/rgb; display %s +#image/gif; display %s +image/x-xwindowdump; display %s + +# The continuation char shouldn't \ +# make a difference in a comment. + +message/external-body; showexternal %s %{access-type} %{name} %{site} \ + %{directory} %{mode} %{server}; needsterminal; composetyped = extcompose %s; \ + description="A reference to data stored in an external location" + +text/richtext; shownonascii iso-8859-8 -e richtext -p %s; test=test "`echo \ + %{charset} | tr '[A-Z]' '[a-z]'`" = iso-8859-8; copiousoutput + +video/mpeg; mpeg_play %s +video/*; animate %s diff --git a/Lib/test/math_testcases.txt b/Lib/test/math_testcases.txt index 5e24335..9585188 100644 --- a/Lib/test/math_testcases.txt +++ b/Lib/test/math_testcases.txt @@ -517,3 +517,117 @@ expm10306 expm1 1.79e308 -> inf overflow -- weaker version of expm10302 expm10307 expm1 709.5 -> 1.3549863193146328e+308 + +------------------------- +-- log2: log to base 2 -- +------------------------- + +-- special values +log20000 log2 0.0 -> -inf divide-by-zero +log20001 log2 -0.0 -> -inf divide-by-zero +log20002 log2 inf -> inf +log20003 log2 -inf -> nan invalid +log20004 log2 nan -> nan + +-- exact value at 1.0 +log20010 log2 1.0 -> 0.0 + +-- negatives +log20020 log2 -5e-324 -> nan invalid +log20021 log2 -1.0 -> nan invalid +log20022 log2 -1.7e-308 -> nan invalid + +-- exact values at powers of 2 +log20100 log2 2.0 -> 1.0 +log20101 log2 4.0 -> 2.0 +log20102 log2 8.0 -> 3.0 +log20103 log2 16.0 -> 4.0 +log20104 log2 32.0 -> 5.0 +log20105 log2 64.0 -> 6.0 +log20106 log2 128.0 -> 7.0 +log20107 log2 256.0 -> 8.0 +log20108 log2 512.0 -> 9.0 +log20109 log2 1024.0 -> 10.0 +log20110 log2 2048.0 -> 11.0 + +log20200 log2 0.5 -> -1.0 +log20201 log2 0.25 -> -2.0 +log20202 log2 0.125 -> -3.0 +log20203 log2 0.0625 -> -4.0 + +-- values close to 1.0 +log20300 log2 1.0000000000000002 -> 3.2034265038149171e-16 +log20301 log2 1.0000000001 -> 1.4426951601859516e-10 +log20302 log2 1.00001 -> 1.4426878274712997e-5 + +log20310 log2 0.9999999999999999 -> -1.6017132519074588e-16 +log20311 log2 0.9999999999 -> -1.4426951603302210e-10 +log20312 log2 0.99999 -> -1.4427022544056922e-5 + +-- tiny values +log20400 log2 5e-324 -> -1074.0 +log20401 log2 1e-323 -> -1073.0 +log20402 log2 1.5e-323 -> -1072.4150374992789 +log20403 log2 2e-323 -> -1072.0 + +log20410 log2 1e-308 -> -1023.1538532253076 +log20411 log2 2.2250738585072014e-308 -> -1022.0 +log20412 log2 4.4501477170144028e-308 -> -1021.0 +log20413 log2 1e-307 -> -1019.8319251304202 + +-- huge values +log20500 log2 1.7976931348623157e+308 -> 1024.0 +log20501 log2 1.7e+308 -> 1023.9193879716706 +log20502 log2 8.9884656743115795e+307 -> 1023.0 + +-- selection of random values +log20600 log2 -7.2174324841039838e+289 -> nan invalid +log20601 log2 -2.861319734089617e+265 -> nan invalid +log20602 log2 -4.3507646894008962e+257 -> nan invalid +log20603 log2 -6.6717265307520224e+234 -> nan invalid +log20604 log2 -3.9118023786619294e+229 -> nan invalid +log20605 log2 -1.5478221302505161e+206 -> nan invalid +log20606 log2 -1.4380485131364602e+200 -> nan invalid +log20607 log2 -3.7235198730382645e+185 -> nan invalid +log20608 log2 -1.0472242235095724e+184 -> nan invalid +log20609 log2 -5.0141781956163884e+160 -> nan invalid +log20610 log2 -2.1157958031160324e+124 -> nan invalid +log20611 log2 -7.9677558612567718e+90 -> nan invalid +log20612 log2 -5.5553906194063732e+45 -> nan invalid +log20613 log2 -16573900952607.953 -> nan invalid +log20614 log2 -37198371019.888618 -> nan invalid +log20615 log2 -6.0727115121422674e-32 -> nan invalid +log20616 log2 -2.5406841656526057e-38 -> nan invalid +log20617 log2 -4.9056766703267657e-43 -> nan invalid +log20618 log2 -2.1646786075228305e-71 -> nan invalid +log20619 log2 -2.470826790488573e-78 -> nan invalid +log20620 log2 -3.8661709303489064e-165 -> nan invalid +log20621 log2 -1.0516496976649986e-182 -> nan invalid +log20622 log2 -1.5935458614317996e-255 -> nan invalid +log20623 log2 -2.8750977267336654e-293 -> nan invalid +log20624 log2 -7.6079466794732585e-296 -> nan invalid +log20625 log2 3.2073253539988545e-307 -> -1018.1505544209213 +log20626 log2 1.674937885472249e-244 -> -809.80634755783126 +log20627 log2 1.0911259044931283e-214 -> -710.76679472274213 +log20628 log2 2.0275372624809709e-154 -> -510.55719818383272 +log20629 log2 7.3926087369631841e-115 -> -379.13564735312292 +log20630 log2 1.3480198206342423e-86 -> -285.25497445094436 +log20631 log2 8.9927384655719947e-83 -> -272.55127136401637 +log20632 log2 3.1452398713597487e-60 -> -197.66251564496875 +log20633 log2 7.0706573215457351e-55 -> -179.88420087782217 +log20634 log2 3.1258285390731669e-49 -> -161.13023800505653 +log20635 log2 8.2253046627829942e-41 -> -133.15898277355879 +log20636 log2 7.8691367397519897e+49 -> 165.75068202732419 +log20637 log2 2.9920561983925013e+64 -> 214.18453534573757 +log20638 log2 4.7827254553946841e+77 -> 258.04629628445673 +log20639 log2 3.1903566496481868e+105 -> 350.47616767491166 +log20640 log2 5.6195082449502419e+113 -> 377.86831861008250 +log20641 log2 9.9625658250651047e+125 -> 418.55752921228753 +log20642 log2 2.7358945220961532e+145 -> 483.13158636923413 +log20643 log2 2.785842387926931e+174 -> 579.49360214860280 +log20644 log2 2.4169172507252751e+193 -> 642.40529039289652 +log20645 log2 3.1689091206395632e+205 -> 682.65924573798395 +log20646 log2 2.535995592365391e+208 -> 692.30359597460460 +log20647 log2 6.2011236566089916e+233 -> 776.64177576730913 +log20648 log2 2.1843274820677632e+253 -> 841.57499717289647 +log20649 log2 8.7493931063474791e+297 -> 989.74182713073981 diff --git a/Lib/test/mock_socket.py b/Lib/test/mock_socket.py index 8036932..d09e78c 100644 --- a/Lib/test/mock_socket.py +++ b/Lib/test/mock_socket.py @@ -106,7 +106,8 @@ def socket(family=None, type=None, proto=None): return MockSocket() -def create_connection(address, timeout=socket_module._GLOBAL_DEFAULT_TIMEOUT): +def create_connection(address, timeout=socket_module._GLOBAL_DEFAULT_TIMEOUT, + source_address=None): try: int_port = int(address[1]) except ValueError: diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 49be720..828225d 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -4,10 +4,11 @@ import pickle import pickletools import sys import copyreg +import weakref from http.cookies import SimpleCookie from test.support import ( - TestFailed, TESTFN, run_with_locale, + TestFailed, TESTFN, run_with_locale, no_tracing, _2G, _4G, bigmemtest, ) @@ -18,7 +19,7 @@ from pickle import bytes_types # kind of outer loop. protocols = range(pickle.HIGHEST_PROTOCOL + 1) -character_size = 4 if sys.maxunicode > 0xFFFF else 2 +ascii_char_size = 1 # Return True if opcode code appears in the pickle, else False. @@ -869,6 +870,25 @@ class AbstractPickleTests(unittest.TestCase): self.assertEqual(B(x), B(y), detail) self.assertEqual(x.__dict__, y.__dict__, detail) + def test_newobj_proxies(self): + # NEWOBJ should use the __class__ rather than the raw type + classes = myclasses[:] + # Cannot create weakproxies to these classes + for c in (MyInt, MyTuple): + classes.remove(c) + for proto in protocols: + for C in classes: + B = C.__base__ + x = C(C.sample) + x.foo = 42 + p = weakref.proxy(x) + s = self.dumps(p, proto) + y = self.loads(s) + self.assertEqual(type(y), type(x)) # rather than type(p) + detail = (proto, C, B, x, y, type(y)) + self.assertEqual(B(x), B(y), detail) + self.assertEqual(x.__dict__, y.__dict__, detail) + # Register a type with copyreg, with extension code extcode. Pickle # an object of that type. Check that the resulting pickle uses opcode # (EXT[124]) under proto 2, and not in proto 1. @@ -1029,13 +1049,13 @@ class AbstractPickleTests(unittest.TestCase): y = self.loads(s) self.assertEqual(y._reduce_called, 1) + @no_tracing def test_bad_getattr(self): x = BadGetattr() for proto in 0, 1: self.assertRaises(RuntimeError, self.dumps, x, proto) # protocol 2 don't raise a RuntimeError. d = self.dumps(x, 2) - self.assertRaises(RuntimeError, self.loads, d) def test_reduce_bad_iterator(self): # Issue4176: crash when 4th and 5th items of __reduce__() @@ -1125,6 +1145,15 @@ class AbstractPickleTests(unittest.TestCase): empty = self.loads(b'\x80\x03U\x00q\x00.', encoding='koi8-r') self.assertEqual(empty, '') + def test_int_pickling_efficiency(self): + # Test compacity of int representation (see issue #12744) + for proto in protocols: + sizes = [len(self.dumps(2**n, proto)) for n in range(70)] + # the size function is monotonic + self.assertEqual(sorted(sizes), sizes) + if proto >= 2: + self.assertLessEqual(sizes[-1], 14) + def check_negative_32b_binXXX(self, dumped): if sys.maxsize > 2**32: self.skipTest("test is only meaningful on 32-bit builds") @@ -1206,7 +1235,7 @@ class BigmemPickleTests(unittest.TestCase): # All protocols use 1-byte per printable ASCII character; we add another # byte because the encoded form has to be copied into the internal buffer. - @bigmemtest(size=_2G, memuse=2 + character_size, dry_run=False) + @bigmemtest(size=_2G, memuse=2 + ascii_char_size, dry_run=False) def test_huge_str_32b(self, size): data = "abcd" * (size // 4) try: @@ -1223,7 +1252,7 @@ class BigmemPickleTests(unittest.TestCase): # BINUNICODE (protocols 1, 2 and 3) cannot carry more than # 2**32 - 1 bytes of utf-8 encoded unicode. - @bigmemtest(size=_4G, memuse=1 + character_size, dry_run=False) + @bigmemtest(size=_4G, memuse=1 + ascii_char_size, dry_run=False) def test_huge_str_64b(self, size): data = "a" * size try: diff --git a/Lib/test/regrtest.py b/Lib/test/regrtest.py index 10b515b..f859c51 100755 --- a/Lib/test/regrtest.py +++ b/Lib/test/regrtest.py @@ -20,6 +20,11 @@ python -E -Wd -m test [options] [test_name1 ...] Options: -h/--help -- print this text and exit +--timeout TIMEOUT + -- dump the traceback and exit if a test takes more + than TIMEOUT seconds; disabled if TIMEOUT is negative + or equals to zero +--wait -- wait for user input, e.g., allow a debugger to be attached Verbosity @@ -44,6 +49,9 @@ Selecting tests -- specify which special resource intensive tests to run -M/--memlimit LIMIT -- run very large memory-consuming tests + --testdir DIR + -- execute test files in the specified directory (instead + of the Python stdlib test suite) Special runs @@ -125,6 +133,8 @@ resources to test. Currently only the following are defined: all - Enable all special resources. + none - Disable all special resources (this is the default). + audio - Tests that use the audio device. (There are known cases of broken audio drivers that can crash Python or even the Linux kernel.) @@ -156,15 +166,18 @@ option '-uall,-gui'. """ import builtins -import errno +import faulthandler import getopt import io import json import logging import os +import packaging.command +import packaging.database import platform import random import re +import signal import sys import sysconfig import tempfile @@ -224,6 +237,7 @@ ENV_CHANGED = -1 SKIPPED = -2 RESOURCE_DENIED = -3 INTERRUPTED = -4 +CHILD_ERROR = -5 # error in a child process from test import support @@ -267,6 +281,18 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, on the command line. """ + # Display the Python traceback on fatal errors (e.g. segfault) + faulthandler.enable(all_threads=True) + + # Display the Python traceback on SIGALRM or SIGUSR1 signal + signals = [] + if hasattr(signal, 'SIGALRM'): + signals.append(signal.SIGALRM) + if hasattr(signal, 'SIGUSR1'): + signals.append(signal.SIGUSR1) + for signum in signals: + faulthandler.register(signum, chain=True) + replace_stdout() support.record_original_stdout(sys.stdout) @@ -277,7 +303,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, 'use=', 'threshold=', 'trace', 'coverdir=', 'nocoverdir', 'runleaks', 'huntrleaks=', 'memlimit=', 'randseed=', 'multiprocess=', 'coverage', 'slaveargs=', 'forever', 'debug', - 'start=', 'nowindows', 'header', 'failfast', 'match']) + 'start=', 'nowindows', 'header', 'testdir=', 'timeout=', 'wait', + 'failfast', 'match']) except getopt.error as msg: usage(msg) @@ -288,6 +315,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, use_resources = [] debug = False start = None + timeout = None for o, a in opts: if o in ('-h', '--help'): print(__doc__) @@ -331,7 +359,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, elif o in ('-T', '--coverage'): trace = True elif o in ('-D', '--coverdir'): - coverdir = os.path.join(os.getcwd(), a) + # CWD is replaced with a temporary dir before calling main(), so we + # need join it with the saved CWD so it goes where the user expects. + coverdir = os.path.join(support.SAVEDCWD, a) elif o in ('-N', '--nocoverdir'): coverdir = None elif o in ('-R', '--huntrleaks'): @@ -360,6 +390,9 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if r == 'all': use_resources[:] = RESOURCE_NAMES continue + if r == 'none': + del use_resources[:] + continue remove = False if r[0] == '-': remove = True @@ -390,6 +423,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, forever = True elif o in ('-j', '--multiprocess'): use_mp = int(a) + if use_mp <= 0: + try: + import multiprocessing + # Use all cores + extras for tests that like to sleep + use_mp = 2 + multiprocessing.cpu_count() + except (ImportError, NotImplementedError): + use_mp = 3 + if use_mp == 1: + use_mp = None elif o == '--header': header = True elif o == '--slaveargs': @@ -402,6 +444,21 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, print() # Force a newline (just in case) print(json.dumps(result)) sys.exit(0) + elif o == '--testdir': + # CWD is replaced with a temporary dir before calling main(), so we + # join it with the saved CWD so it ends up where the user expects. + testdir = os.path.join(support.SAVEDCWD, a) + elif o == '--timeout': + if hasattr(faulthandler, 'dump_tracebacks_later'): + timeout = float(a) + if timeout <= 0: + timeout = None + else: + print("Warning: The timeout option requires " + "faulthandler.dump_tracebacks_later") + timeout = None + elif o == '--wait': + input("Press any key to continue...") else: print(("No handler for option {}. Please report this as a bug " "at http://bugs.python.org.").format(o), file=sys.stderr) @@ -480,7 +537,13 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, print("== ", os.getcwd()) print("Testing with flags:", sys.flags) - alltests = findtests(testdir, stdtests, nottests) + # if testdir is set, then we are not running the python tests suite, so + # don't add default tests to be executed or skipped (pass empty values) + if testdir: + alltests = findtests(testdir, list(), set()) + else: + alltests = findtests(testdir, stdtests, nottests) + selected = tests or args or alltests if single: selected = selected[:1] @@ -555,7 +618,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, (test, verbose, quiet), dict(huntrleaks=huntrleaks, use_resources=use_resources, debug=debug, output_on_failure=verbose3, - failfast=failfast, match_tests=match_tests) + timeout=timeout, failfast=failfast, + match_tests=match_tests) ) yield (test, args_tuple) pending = tests_and_args() @@ -576,10 +640,15 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, universal_newlines=True, close_fds=(os.name != 'nt')) stdout, stderr = popen.communicate() + retcode = popen.wait() # Strip last refcount output line if it exists, since it # comes from the shutdown of the interpreter in the subcommand. stderr = debug_output_pat.sub("", stderr) stdout, _, result = stdout.strip().rpartition("\n") + if retcode != 0: + result = (CHILD_ERROR, "Exit code %s" % retcode) + output.put((test, stdout.rstrip(), stderr.rstrip(), result)) + return if not result: output.put((None, None, None, None)) return @@ -612,6 +681,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if result[0] == INTERRUPTED: assert result[1] == 'KeyboardInterrupt' raise KeyboardInterrupt # What else? + if result[0] == CHILD_ERROR: + raise Exception("Child error on {}: {}".format(test, result[1])) test_index += 1 except KeyboardInterrupt: interrupted = True @@ -628,13 +699,14 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, if trace: # If we're tracing code coverage, then we don't exit with status # if on a false return value from main. - tracer.runctx('runtest(test, verbose, quiet)', + tracer.runctx('runtest(test, verbose, quiet, timeout=timeout)', globals=globals(), locals=vars()) else: try: result = runtest(test, verbose, quiet, huntrleaks, debug, output_on_failure=verbose3, - failfast=failfast, match_tests=match_tests) + timeout=timeout, failfast=failfast, + match_tests=match_tests) accumulate_result(test, result) except KeyboardInterrupt: interrupted = True @@ -705,7 +777,7 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, sys.stdout.flush() try: verbose = True - ok = runtest(test, True, quiet, huntrleaks, debug) + ok = runtest(test, True, quiet, huntrleaks, debug, timeout=timeout) except KeyboardInterrupt: # print a newline separate from the ^C print() @@ -730,6 +802,8 @@ def main(tests=None, testdir=None, verbose=0, quiet=False, sys.exit(len(bad) > 0 or interrupted) +# small set of tests to determine if we have a basically functioning interpreter +# (i.e. if any of these fail, then anything else is likely to follow) STDTESTS = [ 'test_grammar', 'test_opcodes', @@ -740,12 +814,11 @@ STDTESTS = [ 'test_unittest', 'test_doctest', 'test_doctest2', + 'test_support' ] -NOTTESTS = { - 'test_future1', - 'test_future2', -} +# set of tests that we don't want to be executed when using regrtest +NOTTESTS = set() def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): """Return a list of all applicable test modules.""" @@ -754,25 +827,22 @@ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS): tests = [] others = set(stdtests) | nottests for name in names: - modname, ext = os.path.splitext(name) - if modname[:5] == "test_" and ext == ".py" and modname not in others: - tests.append(modname) + mod, ext = os.path.splitext(name) + if mod[:5] == "test_" and ext in (".py", "") and mod not in others: + tests.append(mod) return stdtests + sorted(tests) def replace_stdout(): """Set stdout encoder error handler to backslashreplace (as stderr error handler) to avoid UnicodeEncodeError when printing a traceback""" - if os.name == "nt": - # Replace sys.stdout breaks the stdout newlines on Windows: issue #8533 - return - import atexit stdout = sys.stdout sys.stdout = open(stdout.fileno(), 'w', encoding=stdout.encoding, errors="backslashreplace", - closefd=False) + closefd=False, + newline='\n') def restore_stdout(): sys.stdout.close() @@ -781,7 +851,8 @@ def replace_stdout(): def runtest(test, verbose, quiet, huntrleaks=False, debug=False, use_resources=None, - output_on_failure=False, failfast=False, match_tests=None): + output_on_failure=False, failfast=False, match_tests=None, + timeout=None): """Run a single test. test -- the name of the test @@ -791,6 +862,8 @@ def runtest(test, verbose, quiet, huntrleaks -- run multiple times to test for leaks; requires a debug build; a triple corresponding to -R's three arguments output_on_failure -- if true, display test output on failure + timeout -- dump the traceback and exit if a test takes more than + timeout seconds Returns one of the test result constants: INTERRUPTED KeyboardInterrupt when run under -j @@ -803,6 +876,9 @@ def runtest(test, verbose, quiet, if use_resources is not None: support.use_resources = use_resources + use_timeout = (timeout is not None) + if use_timeout: + faulthandler.dump_tracebacks_later(timeout, exit=True) try: support.match_tests = match_tests if failfast: @@ -841,6 +917,8 @@ def runtest(test, verbose, quiet, display_failure=not verbose) return result finally: + if use_timeout: + faulthandler.cancel_dump_tracebacks_later() cleanup_test_droppings(test, verbose) runtest.stringio = None @@ -886,9 +964,12 @@ class saved_test_environment: resources = ('sys.argv', 'cwd', 'sys.stdin', 'sys.stdout', 'sys.stderr', 'os.environ', 'sys.path', 'sys.path_hooks', '__import__', 'warnings.filters', 'asyncore.socket_map', - 'logging._handlers', 'logging._handlerList', + 'logging._handlers', 'logging._handlerList', 'sys.gettrace', 'sys.warnoptions', 'threading._dangling', - 'multiprocessing.process._dangling') + 'multiprocessing.process._dangling', + 'sysconfig._CONFIG_VARS', 'sysconfig._SCHEMES', + 'packaging.command._COMMANDS', 'packaging.database_caches', + ) def get_sys_argv(self): return id(sys.argv), sys.argv, sys.argv[:] @@ -935,6 +1016,11 @@ class saved_test_environment: sys.path_hooks = saved_hooks[1] sys.path_hooks[:] = saved_hooks[2] + def get_sys_gettrace(self): + return sys.gettrace() + def restore_sys_gettrace(self, trace_fxn): + sys.settrace(trace_fxn) + def get___import__(self): return builtins.__import__ def restore___import__(self, import_): @@ -970,6 +1056,44 @@ class saved_test_environment: # Can't easily revert the logging state pass + def get_packaging_command__COMMANDS(self): + # registry mapping command names to full dotted path or to the actual + # class (resolved on demand); this check only looks at the names, not + # the types of the values (IOW, if a value changes from a string + # (dotted path) to a class it's okay but if a key (i.e. command class) + # is added we complain) + id_ = id(packaging.command._COMMANDS) + keys = set(packaging.command._COMMANDS) + return id_, keys + def restore_packaging_command__COMMANDS(self, saved): + # if command._COMMANDS was bound to another dict object, we can't + # restore the previous object and contents, because the get_ method + # above does not return the dict object (to ignore changes in values) + for key in packaging.command._COMMANDS.keys() - saved[1]: + del packaging.command._COMMANDS[key] + + def get_packaging_database_caches(self): + # caching system used by the PEP 376 implementation + # we have one boolean and four dictionaries, initially empty + switch = packaging.database._cache_enabled + saved = [] + for name in ('_cache_name', '_cache_name_egg', + '_cache_path', '_cache_path_egg'): + cache = getattr(packaging.database, name) + saved.append((id(cache), cache, cache.copy())) + return switch, saved + def restore_packaging_database_caches(self, saved): + switch, saved_caches = saved + packaging.database._cache_enabled = switch + for offset, name in enumerate(('_cache_name', '_cache_name_egg', + '_cache_path', '_cache_path_egg')): + _, cache, items = saved_caches[offset] + # put back the same object in place + setattr(packaging.database, name, cache) + # now restore its items + cache.clear() + cache.update(items) + def get_sys_warnoptions(self): return id(sys.warnoptions), sys.warnoptions, sys.warnoptions[:] def restore_sys_warnoptions(self, saved_options): @@ -1001,6 +1125,27 @@ class saved_test_environment: multiprocessing.process._dangling.clear() multiprocessing.process._dangling.update(saved) + def get_sysconfig__CONFIG_VARS(self): + # make sure the dict is initialized + sysconfig.get_config_var('prefix') + return (id(sysconfig._CONFIG_VARS), sysconfig._CONFIG_VARS, + dict(sysconfig._CONFIG_VARS)) + def restore_sysconfig__CONFIG_VARS(self, saved): + sysconfig._CONFIG_VARS = saved[1] + sysconfig._CONFIG_VARS.clear() + sysconfig._CONFIG_VARS.update(saved[2]) + + def get_sysconfig__SCHEMES(self): + # it's mildly evil to look at the internal attribute, but it's easier + # than copying a RawConfigParser object + return (id(sysconfig._SCHEMES), sysconfig._SCHEMES._sections, + sysconfig._SCHEMES._sections.copy()) + def restore_sysconfig__SCHEMES(self, saved): + sysconfig._SCHEMES._sections = saved[1] + sysconfig._SCHEMES._sections.clear() + sysconfig._SCHEMES._sections.update(saved[2]) + + def resource_info(self): for name in self.resources: method_suffix = name.replace('.', '_') @@ -1140,7 +1285,8 @@ def dash_R(the_module, test, indirect_test, huntrleaks): False if the test didn't leak references; True if we detected refleaks. """ # This code is hackish and inelegant, but it seems to do the job. - import copyreg, _abcoll + import copyreg + import collections.abc if not hasattr(sys, 'gettotalrefcount'): raise Exception("Tracking reference leaks requires a debug build " @@ -1157,7 +1303,7 @@ def dash_R(the_module, test, indirect_test, huntrleaks): else: zdc = zipimport._zip_directory_cache.copy() abcs = {} - for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: @@ -1203,7 +1349,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): import gc, copyreg import _strptime, linecache import urllib.parse, urllib.request, mimetypes, doctest - import struct, filecmp, _abcoll + import struct, filecmp, collections.abc from distutils.dir_util import _path_created from weakref import WeakSet @@ -1230,7 +1376,7 @@ def dash_R_cleanup(fs, ps, pic, zdc, abcs): sys._clear_type_cache() # Clear ABC registries, restoring previously saved ABC registries. - for abc in [getattr(_abcoll, a) for a in _abcoll.__all__]: + for abc in [getattr(collections.abc, a) for a in collections.abc.__all__]: if not isabstract(abc): continue for obj in abc.__subclasses__() + [abc]: @@ -1309,13 +1455,14 @@ def printlist(x, width=70, indent=4): # Tests that are expected to be skipped everywhere except on one platform # are also handled separately. -_expectations = { - 'win32': +_expectations = ( + ('win32', """ test__locale test_crypt test_curses test_dbm + test_devpoll test_fcntl test_fork1 test_epoll @@ -1338,15 +1485,16 @@ _expectations = { test_threadsignals test_wait3 test_wait4 - """, - 'linux2': + """), + ('linux', """ test_curses + test_devpoll test_largefile test_kqueue test_ossaudiodev - """, - 'unixware7': + """), + ('unixware', """ test_epoll test_largefile @@ -1356,8 +1504,8 @@ _expectations = { test_pyexpat test_sax test_sundry - """, - 'openunix8': + """), + ('openunix', """ test_epoll test_largefile @@ -1367,8 +1515,8 @@ _expectations = { test_pyexpat test_sax test_sundry - """, - 'sco_sv3': + """), + ('sco_sv', """ test_asynchat test_fork1 @@ -1387,11 +1535,12 @@ _expectations = { test_threaded_import test_threadedtempfile test_threading - """, - 'darwin': + """), + ('darwin', """ test__locale test_curses + test_devpoll test_epoll test_dbm_gnu test_gdb @@ -1400,8 +1549,8 @@ _expectations = { test_minidom test_ossaudiodev test_poll - """, - 'sunos5': + """), + ('sunos', """ test_curses test_dbm @@ -1412,8 +1561,8 @@ _expectations = { test_openpty test_zipfile test_zlib - """, - 'hp-ux11': + """), + ('hp-ux', """ test_curses test_epoll @@ -1428,11 +1577,12 @@ _expectations = { test_sax test_zipfile test_zlib - """, - 'cygwin': + """), + ('cygwin', """ test_curses test_dbm + test_devpoll test_epoll test_ioctl test_kqueue @@ -1440,8 +1590,8 @@ _expectations = { test_locale test_ossaudiodev test_socketserver - """, - 'os2emx': + """), + ('os2emx', """ test_audioop test_curses @@ -1454,9 +1604,10 @@ _expectations = { test_pty test_resource test_signal - """, - 'freebsd4': + """), + ('freebsd', """ + test_devpoll test_epoll test_dbm_gnu test_locale @@ -1471,8 +1622,8 @@ _expectations = { test_timeout test_urllibnet test_multiprocessing - """, - 'aix5': + """), + ('aix', """ test_bz2 test_epoll @@ -1486,10 +1637,11 @@ _expectations = { test_ttk_textonly test_zipimport test_zlib - """, - 'openbsd3': + """), + ('openbsd', """ test_ctypes + test_devpoll test_epoll test_dbm_gnu test_locale @@ -1501,11 +1653,12 @@ _expectations = { test_ttk_guionly test_ttk_textonly test_multiprocessing - """, - 'netbsd3': + """), + ('netbsd', """ test_ctypes test_curses + test_devpoll test_epoll test_dbm_gnu test_locale @@ -1516,12 +1669,8 @@ _expectations = { test_ttk_guionly test_ttk_textonly test_multiprocessing - """, -} -_expectations['freebsd5'] = _expectations['freebsd4'] -_expectations['freebsd6'] = _expectations['freebsd4'] -_expectations['freebsd7'] = _expectations['freebsd4'] -_expectations['freebsd8'] = _expectations['freebsd4'] + """), +) class _ExpectedSkips: def __init__(self): @@ -1529,9 +1678,13 @@ class _ExpectedSkips: from test import test_timeout self.valid = False - if sys.platform in _expectations: - s = _expectations[sys.platform] - self.expected = set(s.split()) + expected = None + for item in _expectations: + if sys.platform.startswith(item[0]): + expected = item[1] + break + if expected is not None: + self.expected = set(expected.split()) # These are broken tests, for now skipped on every platform. # XXX Fix these! @@ -1591,9 +1744,8 @@ def _make_temp_dir_for_build(TEMPDIR): TEMPDIR = os.path.abspath(TEMPDIR) try: os.mkdir(TEMPDIR) - except OSError as e: - if e.errno != errno.EEXIST: - raise + except FileExistsError: + pass # Define a writable temp dir that will be used as cwd while running # the tests. The name of the dir includes the pid to allow parallel diff --git a/Lib/test/script_helper.py b/Lib/test/script_helper.py index 371c33d..e556eca 100644 --- a/Lib/test/script_helper.py +++ b/Lib/test/script_helper.py @@ -56,11 +56,12 @@ def assert_python_failure(*args, **env_vars): """ return _assert_python(False, *args, **env_vars) -def spawn_python(*args): +def spawn_python(*args, **kw): cmd_line = [sys.executable, '-E'] cmd_line.extend(args) return subprocess.Popen(cmd_line, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, + **kw) def kill_python(p): p.stdin.close() diff --git a/Lib/test/ssl_key.passwd.pem b/Lib/test/ssl_key.passwd.pem new file mode 100644 index 0000000..2524672 --- /dev/null +++ b/Lib/test/ssl_key.passwd.pem @@ -0,0 +1,18 @@ +-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: DES-EDE3-CBC,1A8D9D2A02EC698A + +kJYbfZ8L0sfe9Oty3gw0aloNnY5E8fegRfQLZlNoxTl6jNt0nIwI8kDJ36CZgR9c +u3FDJm/KqrfUoz8vW+qEnWhSG7QPX2wWGPHd4K94Yz/FgrRzZ0DoK7XxXq9gOtVA +AVGQhnz32p+6WhfGsCr9ArXEwRZrTk/FvzEPaU5fHcoSkrNVAGX8IpSVkSDwEDQr +Gv17+cfk99UV1OCza6yKHoFkTtrC+PZU71LomBabivS2Oc4B9hYuSR2hF01wTHP+ +YlWNagZOOVtNz4oKK9x9eNQpmfQXQvPPTfusexKIbKfZrMvJoxcm1gfcZ0H/wK6P +6wmXSG35qMOOztCZNtperjs1wzEBXznyK8QmLcAJBjkfarABJX9vBEzZV0OUKhy+ +noORFwHTllphbmydLhu6ehLUZMHPhzAS5UN7srtpSN81eerDMy0RMUAwA7/PofX1 +94Me85Q8jP0PC9ETdsJcPqLzAPETEYu0ELewKRcrdyWi+tlLFrpE5KT/s5ecbl9l +7B61U4Kfd1PIXc/siINhU3A3bYK+845YyUArUOnKf1kEox7p1RpD7yFqVT04lRTo +cibNKATBusXSuBrp2G6GNuhWEOSafWCKJQAzgCYIp6ZTV2khhMUGppc/2H3CF6cO +zX0KtlPVZC7hLkB6HT8SxYUwF1zqWY7+/XPPdc37MeEZ87Q3UuZwqORLY+Z0hpgt +L5JXBCoklZhCAaN2GqwFLXtGiRSRFGY7xXIhbDTlE65Wv1WGGgDLMKGE1gOz3yAo +2jjG1+yAHJUdE69XTFHSqSkvaloA1W03LdMXZ9VuQJ/ySXCie6ABAQ== +-----END RSA PRIVATE KEY----- diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index d792529..43ecf2a 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -28,6 +28,11 @@ class BaseTest(unittest.TestCase): # Change in subclasses to change the behaviour of fixtesttype() type2test = None + # Whether the "contained items" of the container are integers in + # range(0, 256) (i.e. bytes, bytearray) or strings of length 1 + # (str) + contains_bytes = False + # All tests pass their arguments to the testing methods # as str objects. fixtesttype() can be used to propagate # these arguments to the appropriate type @@ -47,11 +52,12 @@ class BaseTest(unittest.TestCase): return obj # check that obj.method(*args) returns result - def checkequal(self, result, obj, methodname, *args): + def checkequal(self, result, obj, methodname, *args, **kwargs): result = self.fixtype(result) obj = self.fixtype(obj) args = self.fixtype(args) - realresult = getattr(obj, methodname)(*args) + kwargs = self.fixtype(kwargs) + realresult = getattr(obj, methodname)(*args, **kwargs) self.assertEqual( result, realresult @@ -116,7 +122,11 @@ class BaseTest(unittest.TestCase): self.checkequal(0, '', 'count', 'xx', sys.maxsize, 0) self.checkraises(TypeError, 'hello', 'count') - self.checkraises(TypeError, 'hello', 'count', 42) + + if self.contains_bytes: + self.checkequal(0, 'hello', 'count', 42) + else: + self.checkraises(TypeError, 'hello', 'count', 42) # For a variety of combinations, # verify that str.count() matches an equivalent function @@ -162,7 +172,11 @@ class BaseTest(unittest.TestCase): self.checkequal( 2, 'rrarrrrrrrrra', 'find', 'a', None, 6) self.checkraises(TypeError, 'hello', 'find') - self.checkraises(TypeError, 'hello', 'find', 42) + + if self.contains_bytes: + self.checkequal(-1, 'hello', 'find', 42) + else: + self.checkraises(TypeError, 'hello', 'find', 42) self.checkequal(0, '', 'find', '') self.checkequal(-1, '', 'find', '', 1, 1) @@ -216,7 +230,11 @@ class BaseTest(unittest.TestCase): self.checkequal( 2, 'rrarrrrrrrrra', 'rfind', 'a', None, 6) self.checkraises(TypeError, 'hello', 'rfind') - self.checkraises(TypeError, 'hello', 'rfind', 42) + + if self.contains_bytes: + self.checkequal(-1, 'hello', 'rfind', 42) + else: + self.checkraises(TypeError, 'hello', 'rfind', 42) # For a variety of combinations, # verify that str.rfind() matches __contains__ @@ -263,7 +281,11 @@ class BaseTest(unittest.TestCase): self.checkequal( 2, 'rrarrrrrrrrra', 'index', 'a', None, 6) self.checkraises(TypeError, 'hello', 'index') - self.checkraises(TypeError, 'hello', 'index', 42) + + if self.contains_bytes: + self.checkraises(ValueError, 'hello', 'index', 42) + else: + self.checkraises(TypeError, 'hello', 'index', 42) def test_rindex(self): self.checkequal(12, 'abcdefghiabc', 'rindex', '') @@ -285,7 +307,11 @@ class BaseTest(unittest.TestCase): self.checkequal( 2, 'rrarrrrrrrrra', 'rindex', 'a', None, 6) self.checkraises(TypeError, 'hello', 'rindex') - self.checkraises(TypeError, 'hello', 'rindex', 42) + + if self.contains_bytes: + self.checkraises(ValueError, 'hello', 'rindex', 42) + else: + self.checkraises(TypeError, 'hello', 'rindex', 42) def test_lower(self): self.checkequal('hello', 'HeLLo', 'lower') @@ -908,7 +934,14 @@ class MixinStrUnicodeUserStringTest: self.checkequal(['abc', 'def', 'ghi'], "abc\ndef\r\nghi\n", 'splitlines') self.checkequal(['abc', 'def', 'ghi', ''], "abc\ndef\r\nghi\n\r", 'splitlines') self.checkequal(['', 'abc', 'def', 'ghi', ''], "\nabc\ndef\r\nghi\n\r", 'splitlines') - self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'], "\nabc\ndef\r\nghi\n\r", 'splitlines', 1) + self.checkequal(['', 'abc', 'def', 'ghi', ''], + "\nabc\ndef\r\nghi\n\r", 'splitlines', False) + self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'], + "\nabc\ndef\r\nghi\n\r", 'splitlines', True) + self.checkequal(['', 'abc', 'def', 'ghi', ''], "\nabc\ndef\r\nghi\n\r", + 'splitlines', keepends=False) + self.checkequal(['\n', 'abc\n', 'def\r\n', 'ghi\n', '\r'], + "\nabc\ndef\r\nghi\n\r", 'splitlines', keepends=True) self.checkraises(TypeError, 'abc', 'splitlines', 42, 42) diff --git a/Lib/test/support.py b/Lib/test/support.py index ab71d47..556e82f4 100644 --- a/Lib/test/support.py +++ b/Lib/test/support.py @@ -15,7 +15,7 @@ import shutil import warnings import unittest import importlib -import collections +import collections.abc import re import subprocess import imp @@ -23,6 +23,7 @@ import time import sysconfig import fnmatch import logging.handlers +import struct try: import _thread, threading @@ -34,26 +35,34 @@ try: except ImportError: multiprocessing = None +try: + import faulthandler +except ImportError: + faulthandler = None + +try: + import zlib +except ImportError: + zlib = None __all__ = [ "Error", "TestFailed", "ResourceDenied", "import_module", "verbose", "use_resources", "max_memuse", "record_original_stdout", "get_original_stdout", "unload", "unlink", "rmtree", "forget", - "is_resource_enabled", "requires", "requires_mac_ver", - "find_unused_port", "bind_port", - "fcmp", "is_jython", "TESTFN", "HOST", "FUZZ", "SAVEDCWD", "temp_cwd", - "findfile", "sortdict", "check_syntax_error", "open_urlresource", - "check_warnings", "CleanImport", "EnvironmentVarGuard", - "TransientResource", "captured_output", "captured_stdout", - "captured_stdin", "captured_stderr", - "time_out", "socket_peer_reset", "ioerror_peer_reset", - "run_with_locale", 'temp_umask', "transient_internet", - "set_memlimit", "bigmemtest", "bigaddrspacetest", "BasicTestRunner", - "run_unittest", "run_doctest", "threading_setup", "threading_cleanup", - "reap_children", "cpython_only", "check_impl_detail", "get_attribute", - "swap_item", "swap_attr", "requires_IEEE_754", + "is_resource_enabled", "requires", "requires_freebsd_version", + "requires_linux_version", "requires_mac_ver", "find_unused_port", "bind_port", + "IPV6_ENABLED", "is_jython", "TESTFN", "HOST", "SAVEDCWD", "temp_cwd", + "findfile", "create_empty_file", "sortdict", "check_syntax_error", "open_urlresource", + "check_warnings", "CleanImport", "EnvironmentVarGuard", "TransientResource", + "captured_stdout", "captured_stdin", "captured_stderr", "time_out", + "socket_peer_reset", "ioerror_peer_reset", "run_with_locale", 'temp_umask', + "transient_internet", "set_memlimit", "bigmemtest", "bigaddrspacetest", + "BasicTestRunner", "run_unittest", "run_doctest", "threading_setup", + "threading_cleanup", "reap_children", "cpython_only", "check_impl_detail", + "get_attribute", "swap_item", "swap_attr", "requires_IEEE_754", "TestHandler", "Matcher", "can_symlink", "skip_unless_symlink", - "import_fresh_module", "failfast", + "import_fresh_module", "requires_zlib", "PIPE_MAX_SIZE", "failfast", + "anticipate_failure" ] class Error(Exception): @@ -124,6 +133,17 @@ def _save_and_block_module(name, orig_modules): return saved +def anticipate_failure(condition): + """Decorator to mark a test that is known to be broken in some cases + + Any use of this decorator should have a comment identifying the + associated tracker issue. + """ + if condition: + return unittest.expectedFailure + return lambda f: f + + def import_fresh_module(name, fresh=(), blocked=(), deprecated=False): """Imports and returns a module, deliberately bypassing the sys.modules cache and importing a fresh copy of the module. Once the import is complete, @@ -167,8 +187,7 @@ def get_attribute(obj, name): try: attribute = getattr(obj, name) except AttributeError: - raise unittest.SkipTest("module %s has no attribute %s" % ( - obj.__name__, name)) + raise unittest.SkipTest("object %r has no attribute %r" % (obj, name)) else: return attribute @@ -209,8 +228,7 @@ def rmtree(path): try: shutil.rmtree(path) except OSError as error: - # Unix returns ENOENT, Windows returns ESRCH. - if error.errno not in (errno.ENOENT, errno.ESRCH): + if error.errno != errno.ENOENT: raise def make_legacy_pyc(source): @@ -295,9 +313,52 @@ def requires(resource, msg=None): return if not is_resource_enabled(resource): if msg is None: - msg = "Use of the `%s' resource not enabled" % resource + msg = "Use of the %r resource not enabled" % resource raise ResourceDenied(msg) +def _requires_unix_version(sysname, min_version): + """Decorator raising SkipTest if the OS is `sysname` and the version is less + than `min_version`. + + For example, @_requires_unix_version('FreeBSD', (7, 2)) raises SkipTest if + the FreeBSD version is less than 7.2. + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kw): + if platform.system() == sysname: + version_txt = platform.release().split('-', 1)[0] + try: + version = tuple(map(int, version_txt.split('.'))) + except ValueError: + pass + else: + if version < min_version: + min_version_txt = '.'.join(map(str, min_version)) + raise unittest.SkipTest( + "%s version %s or higher required, not %s" + % (sysname, min_version_txt, version_txt)) + return wrapper + return decorator + +def requires_freebsd_version(*min_version): + """Decorator raising SkipTest if the OS is FreeBSD and the FreeBSD version is + less than `min_version`. + + For example, @requires_freebsd_version(7, 2) raises SkipTest if the FreeBSD + version is less than 7.2. + """ + return _requires_unix_version('FreeBSD', min_version) + +def requires_linux_version(*min_version): + """Decorator raising SkipTest if the OS is Linux and the Linux version is + less than `min_version`. + + For example, @requires_linux_version(2, 6, 32) raises SkipTest if the Linux + version is less than 2.6.32. + """ + return _requires_unix_version('Linux', min_version) + def requires_mac_ver(*min_version): """Decorator raising SkipTest if the OS is Mac OS X and the OS X version if less than min_version. @@ -325,6 +386,7 @@ def requires_mac_ver(*min_version): return wrapper return decorator + HOST = 'localhost' def find_unused_port(family=socket.AF_INET, socktype=socket.SOCK_STREAM): @@ -420,29 +482,35 @@ def bind_port(sock, host=HOST): port = sock.getsockname()[1] return port -FUZZ = 1e-6 - -def fcmp(x, y): # fuzzy comparison function - if isinstance(x, float) or isinstance(y, float): +def _is_ipv6_enabled(): + """Check whether IPv6 is enabled on this host.""" + if socket.has_ipv6: try: - fuzz = (abs(x) + abs(y)) * FUZZ - if abs(x-y) <= fuzz: - return 0 - except: + sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + sock.bind(('::1', 0)) + except (socket.error, socket.gaierror): pass - elif type(x) == type(y) and isinstance(x, (tuple, list)): - for i in range(min(len(x), len(y))): - outcome = fcmp(x[i], y[i]) - if outcome != 0: - return outcome - return (len(x) > len(y)) - (len(x) < len(y)) - return (x > y) - (x < y) + else: + sock.close() + return True + return False + +IPV6_ENABLED = _is_ipv6_enabled() + + +# A constant likely larger than the underlying OS pipe buffer size. +# Windows limit seems to be around 512B, and most Unix kernels have a 64K pipe +# buffer size: take 1M to be sure. +PIPE_MAX_SIZE = 1024 * 1024 + # decorator for skipping tests on non-IEEE 754 platforms requires_IEEE_754 = unittest.skipUnless( float.__getformat__("double").startswith("IEEE"), "test requires IEEE 754 doubles") +requires_zlib = unittest.skipUnless(zlib, 'requires zlib') + is_jython = sys.platform.startswith('java') # Filename used for testing @@ -543,14 +611,15 @@ def temp_cwd(name='tempcwd', quiet=False, path=None): rmtree(name) -@contextlib.contextmanager -def temp_umask(umask): - """Context manager that temporarily sets the process umask.""" - oldmask = os.umask(umask) - try: - yield - finally: - os.umask(oldmask) +if hasattr(os, "umask"): + @contextlib.contextmanager + def temp_umask(umask): + """Context manager that temporarily sets the process umask.""" + oldmask = os.umask(umask) + try: + yield + finally: + os.umask(oldmask) def findfile(file, here=__file__, subdir=None): @@ -568,6 +637,11 @@ def findfile(file, here=__file__, subdir=None): if os.path.exists(fn): return fn return file +def create_empty_file(filename): + """Create an empty file. If the file already exists, truncate it.""" + fd = os.open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + os.close(fd) + def sortdict(dict): "Like repr(dict), but in sorted order." items = sorted(dict.items()) @@ -632,7 +706,7 @@ def open_urlresource(url, *args, **kw): f = check_valid_file(fn) if f is not None: return f - raise TestFailed('invalid resource "%s"' % fn) + raise TestFailed('invalid resource %r' % fn) class WarningsRecorder(object): @@ -753,7 +827,7 @@ class CleanImport(object): sys.modules.update(self.original_modules) -class EnvironmentVarGuard(collections.MutableMapping): +class EnvironmentVarGuard(collections.abc.MutableMapping): """Class to help protect the environment variable properly. Can be used as a context manager.""" @@ -883,7 +957,7 @@ def transient_internet(resource_name, *, timeout=30.0, errnos=()): ('WSANO_DATA', 11004), ] - denied = ResourceDenied("Resource '%s' is not available" % resource_name) + denied = ResourceDenied("Resource %r is not available" % resource_name) captured_errnos = errnos gai_errnos = [] if not captured_errnos: @@ -972,6 +1046,16 @@ def gc_collect(): gc.collect() gc.collect() +@contextlib.contextmanager +def disable_gc(): + have_gc = gc.isenabled() + gc.disable() + try: + yield + finally: + if have_gc: + gc.enable() + def python_is_optimized(): """Find if Python was built with optimizations.""" @@ -980,7 +1064,7 @@ def python_is_optimized(): for opt in cflags.split(): if opt.startswith('-O'): final_opt = opt - return final_opt and final_opt != '-O0' + return final_opt != '' and final_opt != '-O0' #======================================================================= @@ -1053,41 +1137,66 @@ def set_memlimit(limit): raise ValueError('Memory limit %r too low to be useful' % (limit,)) max_memuse = memlimit -def _memory_watchdog(start_evt, finish_evt, period=10.0): - """A function which periodically watches the process' memory consumption +class _MemoryWatchdog: + """An object which periodically watches the process' memory consumption and prints it out. """ - # XXX: because of the GIL, and because the very long operations tested - # in most bigmem tests are uninterruptible, the loop below gets woken up - # much less often than expected. - # The polling code should be rewritten in raw C, without holding the GIL, - # and push results onto an anonymous pipe. - try: - page_size = os.sysconf('SC_PAGESIZE') - except (ValueError, AttributeError): + + def __init__(self): + self.procfile = '/proc/{pid}/statm'.format(pid=os.getpid()) + self.started = False + self.thread = None try: - page_size = os.sysconf('SC_PAGE_SIZE') + self.page_size = os.sysconf('SC_PAGESIZE') except (ValueError, AttributeError): - page_size = 4096 - procfile = '/proc/{pid}/statm'.format(pid=os.getpid()) - try: - f = open(procfile, 'rb') - except IOError as e: - warnings.warn('/proc not available for stats: {}'.format(e), - RuntimeWarning) - sys.stderr.flush() - return - with f: - start_evt.set() - old_data = -1 - while not finish_evt.wait(period): - f.seek(0) - statm = f.read().decode('ascii') - data = int(statm.split()[5]) - if data != old_data: - old_data = data + try: + self.page_size = os.sysconf('SC_PAGE_SIZE') + except (ValueError, AttributeError): + self.page_size = 4096 + + def consumer(self, fd): + HEADER = "l" + header_size = struct.calcsize(HEADER) + try: + while True: + header = os.read(fd, header_size) + if len(header) < header_size: + # Pipe closed on other end + break + data_len, = struct.unpack(HEADER, header) + data = os.read(fd, data_len) + statm = data.decode('ascii') + data = int(statm.split()[5]) print(" ... process data size: {data:.1f}G" - .format(data=data * page_size / (1024 ** 3))) + .format(data=data * self.page_size / (1024 ** 3))) + finally: + os.close(fd) + + def start(self): + if not faulthandler or not hasattr(faulthandler, '_file_watchdog'): + return + try: + rfd = os.open(self.procfile, os.O_RDONLY) + except OSError as e: + warnings.warn('/proc not available for stats: {}'.format(e), + RuntimeWarning) + sys.stderr.flush() + return + pipe_fd, wfd = os.pipe() + # _file_watchdog() doesn't take the GIL in its child thread, and + # therefore collects statistics timely + faulthandler._file_watchdog(rfd, wfd, 1.0) + self.started = True + self.thread = threading.Thread(target=self.consumer, args=(pipe_fd,)) + self.thread.daemon = True + self.thread.start() + + def stop(self): + if not self.started: + return + faulthandler._cancel_file_watchdog() + self.thread.join() + def bigmemtest(size, memuse, dry_run=True): """Decorator for bigmem tests. @@ -1114,27 +1223,20 @@ def bigmemtest(size, memuse, dry_run=True): "not enough memory: %.1fG minimum needed" % (size * memuse / (1024 ** 3))) - if real_max_memuse and verbose and threading: + if real_max_memuse and verbose and faulthandler and threading: print() print(" ... expected peak memory use: {peak:.1f}G" .format(peak=size * memuse / (1024 ** 3))) - sys.stdout.flush() - start_evt = threading.Event() - finish_evt = threading.Event() - t = threading.Thread(target=_memory_watchdog, - args=(start_evt, finish_evt, 0.5)) - t.daemon = True - t.start() - start_evt.set() + watchdog = _MemoryWatchdog() + watchdog.start() else: - t = None + watchdog = None try: return f(self, maxsize) finally: - if t: - finish_evt.set() - t.join() + if watchdog: + watchdog.stop() wrapper.size = size wrapper.memuse = memuse @@ -1216,6 +1318,33 @@ def check_impl_detail(**guards): return guards.get(platform.python_implementation().lower(), default) +def no_tracing(func): + """Decorator to temporarily turn off tracing for the duration of a test.""" + if not hasattr(sys, 'gettrace'): + return func + else: + @functools.wraps(func) + def wrapper(*args, **kwargs): + original_trace = sys.gettrace() + try: + sys.settrace(None) + return func(*args, **kwargs) + finally: + sys.settrace(original_trace) + return wrapper + + +def refcount_test(test): + """Decorator for tests which involve reference counting. + + To start, the decorator does not run the test if is not run by CPython. + After that, any trace function is unset during the test to prevent + unexpected refcounts caused by the trace function. + + """ + return no_tracing(cpython_only(test)) + + def _filter_suite(suite, pred): """Recursively filter test cases in a suite based on a predicate.""" newtests = [] @@ -1228,7 +1357,6 @@ def _filter_suite(suite, pred): newtests.append(test) suite._tests = newtests - def _run_suite(suite): """Run tests from a unittest.TestSuite-derived class.""" if verbose: @@ -1455,7 +1583,7 @@ def strip_python_stderr(stderr): def args_from_interpreter_flags(): """Return a list of command-line arguments reproducing the current - settings in sys.flags.""" + settings in sys.flags and sys.warnoptions.""" flag_opt_map = { 'bytes_warning': 'b', 'dont_write_bytecode': 'B', @@ -1470,6 +1598,8 @@ def args_from_interpreter_flags(): v = getattr(sys.flags, flag) if v > 0: args.append('-' + opt * v) + for opt in sys.warnoptions: + args.append('-W' + opt) return args #============================================================ diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index d86f97c..1319a64 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -121,11 +121,32 @@ class TestABC(unittest.TestCase): self.assertFalse(issubclass(B, (A,))) self.assertNotIsInstance(b, A) self.assertNotIsInstance(b, (A,)) - A.register(B) + B1 = A.register(B) + self.assertTrue(issubclass(B, A)) + self.assertTrue(issubclass(B, (A,))) + self.assertIsInstance(b, A) + self.assertIsInstance(b, (A,)) + self.assertIs(B1, B) + class C(B): + pass + c = C() + self.assertTrue(issubclass(C, A)) + self.assertTrue(issubclass(C, (A,))) + self.assertIsInstance(c, A) + self.assertIsInstance(c, (A,)) + + def test_register_as_class_deco(self): + class A(metaclass=abc.ABCMeta): + pass + @A.register + class B(object): + pass + b = B() self.assertTrue(issubclass(B, A)) self.assertTrue(issubclass(B, (A,))) self.assertIsInstance(b, A) self.assertIsInstance(b, (A,)) + @A.register class C(B): pass c = C() @@ -133,6 +154,7 @@ class TestABC(unittest.TestCase): self.assertTrue(issubclass(C, (A,))) self.assertIsInstance(c, A) self.assertIsInstance(c, (A,)) + self.assertIs(C, A.register(C)) def test_isinstance_invalidation(self): class A(metaclass=abc.ABCMeta): diff --git a/Lib/test/test_abstract_numbers.py b/Lib/test/test_abstract_numbers.py index 2a396cd..253e6f0 100644 --- a/Lib/test/test_abstract_numbers.py +++ b/Lib/test/test_abstract_numbers.py @@ -14,6 +14,7 @@ class TestNumbers(unittest.TestCase): self.assertEqual(7, int(7).real) self.assertEqual(0, int(7).imag) self.assertEqual(7, int(7).conjugate()) + self.assertEqual(-7, int(-7).conjugate()) self.assertEqual(7, int(7).numerator) self.assertEqual(1, int(7).denominator) @@ -24,6 +25,7 @@ class TestNumbers(unittest.TestCase): self.assertEqual(7.3, float(7.3).real) self.assertEqual(0, float(7.3).imag) self.assertEqual(7.3, float(7.3).conjugate()) + self.assertEqual(-7.3, float(-7.3).conjugate()) def test_complex(self): self.assertFalse(issubclass(complex, Real)) diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index 5d213c4..5a40919 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -4019,6 +4019,37 @@ class TestHelpSubparsersWithHelpOrdering(HelpTestCase): ''' + +class TestHelpMetavarTypeFormatter(HelpTestCase): + """""" + + def custom_type(string): + return string + + parser_signature = Sig(prog='PROG', description='description', + formatter_class=argparse.MetavarTypeHelpFormatter) + argument_signatures = [Sig('a', type=int), + Sig('-b', type=custom_type), + Sig('-c', type=float, metavar='SOME FLOAT')] + argument_group_signatures = [] + usage = '''\ + usage: PROG [-h] [-b custom_type] [-c SOME FLOAT] int + ''' + help = usage + '''\ + + description + + positional arguments: + int + + optional arguments: + -h, --help show this help message and exit + -b custom_type + -c SOME FLOAT + ''' + version = '' + + # ===================================== # Optional/Positional constructor tests # ===================================== @@ -4409,7 +4440,7 @@ class TestEncoding(TestCase): def _test_module_encoding(self, path): path, _ = os.path.splitext(path) path += ".py" - with codecs.open(path, 'r', 'utf8') as f: + with codecs.open(path, 'r', 'utf-8') as f: f.read() def test_argparse_module_encoding(self): @@ -4451,6 +4482,67 @@ class TestArgumentTypeError(TestCase): else: self.fail() +# ========================= +# MessageContentError tests +# ========================= + +class TestMessageContentError(TestCase): + + def test_missing_argument_name_in_message(self): + parser = ErrorRaisingArgumentParser(prog='PROG', usage='') + parser.add_argument('req_pos', type=str) + parser.add_argument('-req_opt', type=int, required=True) + parser.add_argument('need_one', type=str, nargs='+') + + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args([]) + msg = str(cm.exception) + self.assertRegex(msg, 'req_pos') + self.assertRegex(msg, 'req_opt') + self.assertRegex(msg, 'need_one') + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(['myXargument']) + msg = str(cm.exception) + self.assertNotIn(msg, 'req_pos') + self.assertRegex(msg, 'req_opt') + self.assertRegex(msg, 'need_one') + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(['myXargument', '-req_opt=1']) + msg = str(cm.exception) + self.assertNotIn(msg, 'req_pos') + self.assertNotIn(msg, 'req_opt') + self.assertRegex(msg, 'need_one') + + def test_optional_optional_not_in_message(self): + parser = ErrorRaisingArgumentParser(prog='PROG', usage='') + parser.add_argument('req_pos', type=str) + parser.add_argument('--req_opt', type=int, required=True) + parser.add_argument('--opt_opt', type=bool, nargs='?', + default=True) + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args([]) + msg = str(cm.exception) + self.assertRegex(msg, 'req_pos') + self.assertRegex(msg, 'req_opt') + self.assertNotIn(msg, 'opt_opt') + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args(['--req_opt=1']) + msg = str(cm.exception) + self.assertRegex(msg, 'req_pos') + self.assertNotIn(msg, 'req_opt') + self.assertNotIn(msg, 'opt_opt') + + def test_optional_positional_not_in_message(self): + parser = ErrorRaisingArgumentParser(prog='PROG', usage='') + parser.add_argument('req_pos') + parser.add_argument('optional_positional', nargs='?', default='eggs') + with self.assertRaises(ArgumentParserError) as cm: + parser.parse_args([]) + msg = str(cm.exception) + self.assertRegex(msg, 'req_pos') + self.assertNotIn(msg, 'optional_positional') + + # ====================== # parse_known_args tests # ====================== diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index 5190c35..434e495 100755 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -16,6 +16,13 @@ import warnings import array from array import _array_reconstructor as array_reconstructor +try: + # Try to determine availability of long long independently + # of the array module under test + struct.calcsize('@q') + have_long_long = True +except struct.error: + have_long_long = False class ArraySubclass(array.array): pass @@ -26,6 +33,8 @@ class ArraySubclassWithKwargs(array.array): tests = [] # list to accumulate all tests typecodes = "ubBhHiIlLfd" +if have_long_long: + typecodes += 'qQ' class BadConstructorTest(unittest.TestCase): @@ -209,10 +218,14 @@ class BaseTest(unittest.TestCase): self.assertEqual(bi[1], len(a)) def test_byteswap(self): - a = array.array(self.typecode, self.example) + if self.typecode == 'u': + example = '\U00100100' + else: + example = self.example + a = array.array(self.typecode, example) self.assertRaises(TypeError, a.byteswap, 42) if a.itemsize in (1, 2, 4, 8): - b = array.array(self.typecode, self.example) + b = array.array(self.typecode, example) b.byteswap() if a.itemsize==1: self.assertEqual(a, b) @@ -1002,7 +1015,7 @@ class UnicodeTest(StringTest): smallerexample = '\x01\u263a\x00\ufefe' biggerexample = '\x01\u263a\x01\ufeff' outside = str('\x33') - minitemsize = 2 + minitemsize = 4 def test_unicode(self): self.assertRaises(TypeError, array.array, 'b', 'foo') @@ -1014,6 +1027,7 @@ class UnicodeTest(StringTest): a.fromunicode('\x11abc\xff\u1234') s = a.tounicode() self.assertEqual(s, '\xa0\xc2\u1234 \x11abc\xff\u1234') + self.assertEqual(a.itemsize, 4) s = '\x00="\'a\\b\x80\xff\u0000\u0001\u1234' a = array.array('u', s) @@ -1205,6 +1219,18 @@ class UnsignedLongTest(UnsignedNumberTest): minitemsize = 4 tests.append(UnsignedLongTest) +@unittest.skipIf(not have_long_long, 'need long long support') +class LongLongTest(SignedNumberTest): + typecode = 'q' + minitemsize = 8 +tests.append(LongLongTest) + +@unittest.skipIf(not have_long_long, 'need long long support') +class UnsignedLongLongTest(UnsignedNumberTest): + typecode = 'Q' + minitemsize = 8 +tests.append(UnsignedLongLongTest) + class FPTest(NumberTest): example = [-42.0, 0, 42, 1e5, -1e10] smallerexample = [-42.0, 0, 42, 1e5, -2e10] diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py index 2ba3ea5..4fbe5b4 100644 --- a/Lib/test/test_ast.py +++ b/Lib/test/test_ast.py @@ -1,4 +1,6 @@ -import sys, unittest +import os +import sys +import unittest from test import support import ast @@ -52,6 +54,9 @@ exec_tests = [ "while v:pass", # If "if v:pass", + # With + "with x as y: pass", + "with x as y, z as q: pass", # Raise "raise Exception('string')", # TryExcept @@ -487,8 +492,412 @@ class ASTHelpers_Test(unittest.TestCase): self.assertEqual(ast.literal_eval('1.5 - 2j'), 1.5 - 2j) +class ASTValidatorTests(unittest.TestCase): + + def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): + mod.lineno = mod.col_offset = 0 + ast.fix_missing_locations(mod) + with self.assertRaises(exc) as cm: + compile(mod, "<test>", mode) + if msg is not None: + self.assertIn(msg, str(cm.exception)) + + def expr(self, node, msg=None, *, exc=ValueError): + mod = ast.Module([ast.Expr(node)]) + self.mod(mod, msg, exc=exc) + + def stmt(self, stmt, msg=None): + mod = ast.Module([stmt]) + self.mod(mod, msg) + + def test_module(self): + m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) + self.mod(m, "must have Load context", "single") + m = ast.Expression(ast.Name("x", ast.Store())) + self.mod(m, "must have Load context", "eval") + + def _check_arguments(self, fac, check): + def arguments(args=None, vararg=None, varargannotation=None, + kwonlyargs=None, kwarg=None, kwargannotation=None, + defaults=None, kw_defaults=None): + if args is None: + args = [] + if kwonlyargs is None: + kwonlyargs = [] + if defaults is None: + defaults = [] + if kw_defaults is None: + kw_defaults = [] + args = ast.arguments(args, vararg, varargannotation, kwonlyargs, + kwarg, kwargannotation, defaults, kw_defaults) + return fac(args) + args = [ast.arg("x", ast.Name("x", ast.Store()))] + check(arguments(args=args), "must have Load context") + check(arguments(varargannotation=ast.Num(3)), + "varargannotation but no vararg") + check(arguments(varargannotation=ast.Name("x", ast.Store()), vararg="x"), + "must have Load context") + check(arguments(kwonlyargs=args), "must have Load context") + check(arguments(kwargannotation=ast.Num(42)), + "kwargannotation but no kwarg") + check(arguments(kwargannotation=ast.Name("x", ast.Store()), + kwarg="x"), "must have Load context") + check(arguments(defaults=[ast.Num(3)]), + "more positional defaults than args") + check(arguments(kw_defaults=[ast.Num(4)]), + "length of kwonlyargs is not the same as kw_defaults") + args = [ast.arg("x", ast.Name("x", ast.Load()))] + check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), + "must have Load context") + args = [ast.arg("a", ast.Name("x", ast.Load())), + ast.arg("b", ast.Name("y", ast.Load()))] + check(arguments(kwonlyargs=args, + kw_defaults=[None, ast.Name("x", ast.Store())]), + "must have Load context") + + def test_funcdef(self): + a = ast.arguments([], None, None, [], None, None, [], []) + f = ast.FunctionDef("x", a, [], [], None) + self.stmt(f, "empty body on FunctionDef") + f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())], + None) + self.stmt(f, "must have Load context") + f = ast.FunctionDef("x", a, [ast.Pass()], [], + ast.Name("x", ast.Store())) + self.stmt(f, "must have Load context") + def fac(args): + return ast.FunctionDef("x", args, [ast.Pass()], [], None) + self._check_arguments(fac, self.stmt) + + def test_classdef(self): + def cls(bases=None, keywords=None, starargs=None, kwargs=None, + body=None, decorator_list=None): + if bases is None: + bases = [] + if keywords is None: + keywords = [] + if body is None: + body = [ast.Pass()] + if decorator_list is None: + decorator_list = [] + return ast.ClassDef("myclass", bases, keywords, starargs, + kwargs, body, decorator_list) + self.stmt(cls(bases=[ast.Name("x", ast.Store())]), + "must have Load context") + self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), + "must have Load context") + self.stmt(cls(starargs=ast.Name("x", ast.Store())), + "must have Load context") + self.stmt(cls(kwargs=ast.Name("x", ast.Store())), + "must have Load context") + self.stmt(cls(body=[]), "empty body on ClassDef") + self.stmt(cls(body=[None]), "None disallowed") + self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]), + "must have Load context") + + def test_delete(self): + self.stmt(ast.Delete([]), "empty targets on Delete") + self.stmt(ast.Delete([None]), "None disallowed") + self.stmt(ast.Delete([ast.Name("x", ast.Load())]), + "must have Del context") + + def test_assign(self): + self.stmt(ast.Assign([], ast.Num(3)), "empty targets on Assign") + self.stmt(ast.Assign([None], ast.Num(3)), "None disallowed") + self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Num(3)), + "must have Store context") + self.stmt(ast.Assign([ast.Name("x", ast.Store())], + ast.Name("y", ast.Store())), + "must have Load context") + + def test_augassign(self): + aug = ast.AugAssign(ast.Name("x", ast.Load()), ast.Add(), + ast.Name("y", ast.Load())) + self.stmt(aug, "must have Store context") + aug = ast.AugAssign(ast.Name("x", ast.Store()), ast.Add(), + ast.Name("y", ast.Store())) + self.stmt(aug, "must have Load context") + + def test_for(self): + x = ast.Name("x", ast.Store()) + y = ast.Name("y", ast.Load()) + p = ast.Pass() + self.stmt(ast.For(x, y, [], []), "empty body on For") + self.stmt(ast.For(ast.Name("x", ast.Load()), y, [p], []), + "must have Store context") + self.stmt(ast.For(x, ast.Name("y", ast.Store()), [p], []), + "must have Load context") + e = ast.Expr(ast.Name("x", ast.Store())) + self.stmt(ast.For(x, y, [e], []), "must have Load context") + self.stmt(ast.For(x, y, [p], [e]), "must have Load context") + + def test_while(self): + self.stmt(ast.While(ast.Num(3), [], []), "empty body on While") + self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), + "must have Load context") + self.stmt(ast.While(ast.Num(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]), + "must have Load context") + + def test_if(self): + self.stmt(ast.If(ast.Num(3), [], []), "empty body on If") + i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) + self.stmt(i, "must have Load context") + i = ast.If(ast.Num(3), [ast.Expr(ast.Name("x", ast.Store()))], []) + self.stmt(i, "must have Load context") + i = ast.If(ast.Num(3), [ast.Pass()], + [ast.Expr(ast.Name("x", ast.Store()))]) + self.stmt(i, "must have Load context") + + def test_with(self): + p = ast.Pass() + self.stmt(ast.With([], [p]), "empty items on With") + i = ast.withitem(ast.Num(3), None) + self.stmt(ast.With([i], []), "empty body on With") + i = ast.withitem(ast.Name("x", ast.Store()), None) + self.stmt(ast.With([i], [p]), "must have Load context") + i = ast.withitem(ast.Num(3), ast.Name("x", ast.Load())) + self.stmt(ast.With([i], [p]), "must have Store context") + + def test_raise(self): + r = ast.Raise(None, ast.Num(3)) + self.stmt(r, "Raise with cause but no exception") + r = ast.Raise(ast.Name("x", ast.Store()), None) + self.stmt(r, "must have Load context") + r = ast.Raise(ast.Num(4), ast.Name("x", ast.Store())) + self.stmt(r, "must have Load context") + + def test_try(self): + p = ast.Pass() + t = ast.Try([], [], [], [p]) + self.stmt(t, "empty body on Try") + t = ast.Try([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) + self.stmt(t, "must have Load context") + t = ast.Try([p], [], [], []) + self.stmt(t, "Try has neither except handlers nor finalbody") + t = ast.Try([p], [], [p], [p]) + self.stmt(t, "Try has orelse but no except handlers") + t = ast.Try([p], [ast.ExceptHandler(None, "x", [])], [], []) + self.stmt(t, "empty body on ExceptHandler") + e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] + self.stmt(ast.Try([p], e, [], []), "must have Load context") + e = [ast.ExceptHandler(None, "x", [p])] + t = ast.Try([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) + self.stmt(t, "must have Load context") + t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) + self.stmt(t, "must have Load context") + + def test_assert(self): + self.stmt(ast.Assert(ast.Name("x", ast.Store()), None), + "must have Load context") + assrt = ast.Assert(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store())) + self.stmt(assrt, "must have Load context") + + def test_import(self): + self.stmt(ast.Import([]), "empty names on Import") + + def test_importfrom(self): + imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) + self.stmt(imp, "level less than -1") + self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") + + def test_global(self): + self.stmt(ast.Global([]), "empty names on Global") + + def test_nonlocal(self): + self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") + + def test_expr(self): + e = ast.Expr(ast.Name("x", ast.Store())) + self.stmt(e, "must have Load context") + + def test_boolop(self): + b = ast.BoolOp(ast.And(), []) + self.expr(b, "less than 2 values") + b = ast.BoolOp(ast.And(), [ast.Num(3)]) + self.expr(b, "less than 2 values") + b = ast.BoolOp(ast.And(), [ast.Num(4), None]) + self.expr(b, "None disallowed") + b = ast.BoolOp(ast.And(), [ast.Num(4), ast.Name("x", ast.Store())]) + self.expr(b, "must have Load context") + + def test_unaryop(self): + u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) + self.expr(u, "must have Load context") + + def test_lambda(self): + a = ast.arguments([], None, None, [], None, None, [], []) + self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), + "must have Load context") + def fac(args): + return ast.Lambda(args, ast.Name("x", ast.Load())) + self._check_arguments(fac, self.expr) + + def test_ifexp(self): + l = ast.Name("x", ast.Load()) + s = ast.Name("y", ast.Store()) + for args in (s, l, l), (l, s, l), (l, l, s): + self.expr(ast.IfExp(*args), "must have Load context") + + def test_dict(self): + d = ast.Dict([], [ast.Name("x", ast.Load())]) + self.expr(d, "same number of keys as values") + d = ast.Dict([None], [ast.Name("x", ast.Load())]) + self.expr(d, "None disallowed") + d = ast.Dict([ast.Name("x", ast.Load())], [None]) + self.expr(d, "None disallowed") + + def test_set(self): + self.expr(ast.Set([None]), "None disallowed") + s = ast.Set([ast.Name("x", ast.Store())]) + self.expr(s, "must have Load context") + + def _check_comprehension(self, fac): + self.expr(fac([]), "comprehension with no generators") + g = ast.comprehension(ast.Name("x", ast.Load()), + ast.Name("x", ast.Load()), []) + self.expr(fac([g]), "must have Store context") + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Store()), []) + self.expr(fac([g]), "must have Load context") + x = ast.Name("x", ast.Store()) + y = ast.Name("y", ast.Load()) + g = ast.comprehension(x, y, [None]) + self.expr(fac([g]), "None disallowed") + g = ast.comprehension(x, y, [ast.Name("x", ast.Store())]) + self.expr(fac([g]), "must have Load context") + + def _simple_comp(self, fac): + g = ast.comprehension(ast.Name("x", ast.Store()), + ast.Name("x", ast.Load()), []) + self.expr(fac(ast.Name("x", ast.Store()), [g]), + "must have Load context") + def wrap(gens): + return fac(ast.Name("x", ast.Store()), gens) + self._check_comprehension(wrap) + + def test_listcomp(self): + self._simple_comp(ast.ListComp) + + def test_setcomp(self): + self._simple_comp(ast.SetComp) + + def test_generatorexp(self): + self._simple_comp(ast.GeneratorExp) + + def test_dictcomp(self): + g = ast.comprehension(ast.Name("y", ast.Store()), + ast.Name("p", ast.Load()), []) + c = ast.DictComp(ast.Name("x", ast.Store()), + ast.Name("y", ast.Load()), [g]) + self.expr(c, "must have Load context") + c = ast.DictComp(ast.Name("x", ast.Load()), + ast.Name("y", ast.Store()), [g]) + self.expr(c, "must have Load context") + def factory(comps): + k = ast.Name("x", ast.Load()) + v = ast.Name("y", ast.Load()) + return ast.DictComp(k, v, comps) + self._check_comprehension(factory) + + def test_yield(self): + self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") + + def test_compare(self): + left = ast.Name("x", ast.Load()) + comp = ast.Compare(left, [ast.In()], []) + self.expr(comp, "no comparators") + comp = ast.Compare(left, [ast.In()], [ast.Num(4), ast.Num(5)]) + self.expr(comp, "different number of comparators and operands") + comp = ast.Compare(ast.Num("blah"), [ast.In()], [left]) + self.expr(comp, "non-numeric", exc=TypeError) + comp = ast.Compare(left, [ast.In()], [ast.Num("blah")]) + self.expr(comp, "non-numeric", exc=TypeError) + + def test_call(self): + func = ast.Name("x", ast.Load()) + args = [ast.Name("y", ast.Load())] + keywords = [ast.keyword("w", ast.Name("z", ast.Load()))] + stararg = ast.Name("p", ast.Load()) + kwarg = ast.Name("q", ast.Load()) + call = ast.Call(ast.Name("x", ast.Store()), args, keywords, stararg, + kwarg) + self.expr(call, "must have Load context") + call = ast.Call(func, [None], keywords, stararg, kwarg) + self.expr(call, "None disallowed") + bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))] + call = ast.Call(func, args, bad_keywords, stararg, kwarg) + self.expr(call, "must have Load context") + call = ast.Call(func, args, keywords, ast.Name("z", ast.Store()), kwarg) + self.expr(call, "must have Load context") + call = ast.Call(func, args, keywords, stararg, + ast.Name("w", ast.Store())) + self.expr(call, "must have Load context") + + def test_num(self): + class subint(int): + pass + class subfloat(float): + pass + class subcomplex(complex): + pass + for obj in "0", "hello", subint(), subfloat(), subcomplex(): + self.expr(ast.Num(obj), "non-numeric", exc=TypeError) + + def test_attribute(self): + attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) + self.expr(attr, "must have Load context") + + def test_subscript(self): + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Index(ast.Num(3)), + ast.Load()) + self.expr(sub, "must have Load context") + x = ast.Name("x", ast.Load()) + sub = ast.Subscript(x, ast.Index(ast.Name("y", ast.Store())), + ast.Load()) + self.expr(sub, "must have Load context") + s = ast.Name("x", ast.Store()) + for args in (s, None, None), (None, s, None), (None, None, s): + sl = ast.Slice(*args) + self.expr(ast.Subscript(x, sl, ast.Load()), + "must have Load context") + sl = ast.ExtSlice([]) + self.expr(ast.Subscript(x, sl, ast.Load()), "empty dims on ExtSlice") + sl = ast.ExtSlice([ast.Index(s)]) + self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") + + def test_starred(self): + left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], + ast.Store()) + assign = ast.Assign([left], ast.Num(4)) + self.stmt(assign, "must have Store context") + + def _sequence(self, fac): + self.expr(fac([None], ast.Load()), "None disallowed") + self.expr(fac([ast.Name("x", ast.Store())], ast.Load()), + "must have Load context") + + def test_list(self): + self._sequence(ast.List) + + def test_tuple(self): + self._sequence(ast.Tuple) + + def test_stdlib_validates(self): + stdlib = os.path.dirname(ast.__file__) + tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] + tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) + for module in tests: + fn = os.path.join(stdlib, module) + with open(fn, "r", encoding="utf-8") as fp: + source = fp.read() + mod = ast.parse(source) + compile(mod, fn, "exec") + + def test_main(): - support.run_unittest(AST_Tests, ASTHelpers_Test) + support.run_unittest(AST_Tests, ASTHelpers_Test, ASTValidatorTests) def main(): if __name__ != '__main__': @@ -522,9 +931,11 @@ exec_results = [ ('Module', [('For', (1, 0), ('Name', (1, 4), 'v', ('Store',)), ('Name', (1, 9), 'v', ('Load',)), [('Pass', (1, 11))], [])]), ('Module', [('While', (1, 0), ('Name', (1, 6), 'v', ('Load',)), [('Pass', (1, 8))], [])]), ('Module', [('If', (1, 0), ('Name', (1, 3), 'v', ('Load',)), [('Pass', (1, 5))], [])]), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',)))], [('Pass', (1, 13))])]), +('Module', [('With', (1, 0), [('withitem', ('Name', (1, 5), 'x', ('Load',)), ('Name', (1, 10), 'y', ('Store',))), ('withitem', ('Name', (1, 13), 'z', ('Load',)), ('Name', (1, 18), 'q', ('Store',)))], [('Pass', (1, 21))])]), ('Module', [('Raise', (1, 0), ('Call', (1, 6), ('Name', (1, 6), 'Exception', ('Load',)), [('Str', (1, 16), 'string')], [], None, None), None)]), -('Module', [('TryExcept', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [])]), -('Module', [('TryFinally', (1, 0), [('Pass', (2, 2))], [('Pass', (4, 2))])]), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [('ExceptHandler', (3, 0), ('Name', (3, 7), 'Exception', ('Load',)), None, [('Pass', (4, 2))])], [], [])]), +('Module', [('Try', (1, 0), [('Pass', (2, 2))], [], [], [('Pass', (4, 2))])]), ('Module', [('Assert', (1, 0), ('Name', (1, 7), 'v', ('Load',)), None)]), ('Module', [('Import', (1, 0), [('alias', 'sys', None)])]), ('Module', [('ImportFrom', (1, 0), 'sys', [('alias', 'v', None)], 0)]), diff --git a/Lib/test/test_asyncore.py b/Lib/test/test_asyncore.py index 53c49a8..52dff0f 100644 --- a/Lib/test/test_asyncore.py +++ b/Lib/test/test_asyncore.py @@ -20,6 +20,8 @@ except ImportError: HOST = support.HOST +HAS_UNIX_SOCKETS = hasattr(socket, 'AF_UNIX') + class dummysocket: def __init__(self): self.closed = False @@ -87,6 +89,13 @@ def capture_server(evt, buf, serv): serv.close() evt.set() +def bind_af_aware(sock, addr): + """Helper function to bind a socket according to its family.""" + if HAS_UNIX_SOCKETS and sock.family == socket.AF_UNIX: + # Make sure the path doesn't exist. + unlink(addr) + sock.bind(addr) + class HelperFunctionTests(unittest.TestCase): def test_readwriteexc(self): @@ -352,7 +361,7 @@ class DispatcherWithSendTests(unittest.TestCase): @support.reap_threads def test_send(self): evt = threading.Event() - sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock = socket.socket() sock.settimeout(3) port = support.bind_port(sock) @@ -367,7 +376,7 @@ class DispatcherWithSendTests(unittest.TestCase): data = b"Suppose there isn't a 16-ton weight?" d = dispatcherwithsend_noread() - d.create_socket(socket.AF_INET, socket.SOCK_STREAM) + d.create_socket() d.connect((HOST, port)) # give time for socket to connect @@ -467,22 +476,22 @@ class BaseTestHandler(asyncore.dispatcher): raise -class TCPServer(asyncore.dispatcher): +class BaseServer(asyncore.dispatcher): """A server which listens on an address and dispatches the connection to a handler. """ - def __init__(self, handler=BaseTestHandler, host=HOST, port=0): + def __init__(self, family, addr, handler=BaseTestHandler): asyncore.dispatcher.__init__(self) - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.create_socket(family) self.set_reuse_addr() - self.bind((host, port)) + bind_af_aware(self.socket, addr) self.listen(5) self.handler = handler @property def address(self): - return self.socket.getsockname()[:2] + return self.socket.getsockname() def handle_accepted(self, sock, addr): self.handler(sock) @@ -493,9 +502,9 @@ class TCPServer(asyncore.dispatcher): class BaseClient(BaseTestHandler): - def __init__(self, address): + def __init__(self, family, address): BaseTestHandler.__init__(self) - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.create_socket(family) self.connect(address) def handle_connect(self): @@ -525,8 +534,8 @@ class BaseTestAPI(unittest.TestCase): def handle_connect(self): self.flag = True - server = TCPServer() - client = TestClient(server.address) + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_handle_accept(self): @@ -534,18 +543,18 @@ class BaseTestAPI(unittest.TestCase): class TestListener(BaseTestHandler): - def __init__(self): + def __init__(self, family, addr): BaseTestHandler.__init__(self) - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.bind((HOST, 0)) + self.create_socket(family) + bind_af_aware(self.socket, addr) self.listen(5) - self.address = self.socket.getsockname()[:2] + self.address = self.socket.getsockname() def handle_accept(self): self.flag = True - server = TestListener() - client = BaseClient(server.address) + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) self.loop_waiting_for_flag(server) def test_handle_accepted(self): @@ -553,12 +562,12 @@ class BaseTestAPI(unittest.TestCase): class TestListener(BaseTestHandler): - def __init__(self): + def __init__(self, family, addr): BaseTestHandler.__init__(self) - self.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.bind((HOST, 0)) + self.create_socket(family) + bind_af_aware(self.socket, addr) self.listen(5) - self.address = self.socket.getsockname()[:2] + self.address = self.socket.getsockname() def handle_accept(self): asyncore.dispatcher.handle_accept(self) @@ -567,8 +576,8 @@ class BaseTestAPI(unittest.TestCase): sock.close() self.flag = True - server = TestListener() - client = BaseClient(server.address) + server = TestListener(self.family, self.addr) + client = BaseClient(self.family, server.address) self.loop_waiting_for_flag(server) @@ -584,8 +593,8 @@ class BaseTestAPI(unittest.TestCase): BaseTestHandler.__init__(self, conn) self.send(b'x' * 1024) - server = TCPServer(TestHandler) - client = TestClient(server.address) + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_handle_write(self): @@ -595,8 +604,8 @@ class BaseTestAPI(unittest.TestCase): def handle_write(self): self.flag = True - server = TCPServer() - client = TestClient(server.address) + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_handle_close(self): @@ -619,8 +628,40 @@ class BaseTestAPI(unittest.TestCase): BaseTestHandler.__init__(self, conn) self.close() - server = TCPServer(TestHandler) - client = TestClient(server.address) + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) + self.loop_waiting_for_flag(client) + + def test_handle_close_after_conn_broken(self): + # Check that ECONNRESET/EPIPE is correctly handled (issues #5661 and + # #11265). + + data = b'\0' * 128 + + class TestClient(BaseClient): + + def handle_write(self): + self.send(data) + + def handle_close(self): + self.flag = True + self.close() + + def handle_expt(self): + self.flag = True + self.close() + + class TestHandler(BaseTestHandler): + + def handle_read(self): + self.recv(len(data)) + self.close() + + def writable(self): + return False + + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) @unittest.skipIf(sys.platform.startswith("sunos"), @@ -629,9 +670,12 @@ class BaseTestAPI(unittest.TestCase): # Make sure handle_expt is called on OOB data received. # Note: this might fail on some platforms as OOB data is # tenuously supported and rarely used. + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") class TestClient(BaseClient): def handle_expt(self): + self.socket.recv(1024, socket.MSG_OOB) self.flag = True class TestHandler(BaseTestHandler): @@ -639,8 +683,8 @@ class BaseTestAPI(unittest.TestCase): BaseTestHandler.__init__(self, conn) self.socket.send(bytes(chr(244), 'latin-1'), socket.MSG_OOB) - server = TCPServer(TestHandler) - client = TestClient(server.address) + server = BaseServer(self.family, self.addr, TestHandler) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_handle_error(self): @@ -657,13 +701,13 @@ class BaseTestAPI(unittest.TestCase): else: raise Exception("exception not raised") - server = TCPServer() - client = TestClient(server.address) + server = BaseServer(self.family, self.addr) + client = TestClient(self.family, server.address) self.loop_waiting_for_flag(client) def test_connection_attributes(self): - server = TCPServer() - client = BaseClient(server.address) + server = BaseServer(self.family, self.addr) + client = BaseClient(self.family, server.address) # we start disconnected self.assertFalse(server.connected) @@ -693,25 +737,29 @@ class BaseTestAPI(unittest.TestCase): def test_create_socket(self): s = asyncore.dispatcher() - s.create_socket(socket.AF_INET, socket.SOCK_STREAM) - self.assertEqual(s.socket.family, socket.AF_INET) + s.create_socket(self.family) + self.assertEqual(s.socket.family, self.family) SOCK_NONBLOCK = getattr(socket, 'SOCK_NONBLOCK', 0) self.assertEqual(s.socket.type, socket.SOCK_STREAM | SOCK_NONBLOCK) def test_bind(self): + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") s1 = asyncore.dispatcher() - s1.create_socket(socket.AF_INET, socket.SOCK_STREAM) - s1.bind((HOST, 0)) + s1.create_socket(self.family) + s1.bind(self.addr) s1.listen(5) port = s1.socket.getsockname()[1] s2 = asyncore.dispatcher() - s2.create_socket(socket.AF_INET, socket.SOCK_STREAM) + s2.create_socket(self.family) # EADDRINUSE indicates the socket was correctly bound - self.assertRaises(socket.error, s2.bind, (HOST, port)) + self.assertRaises(socket.error, s2.bind, (self.addr[0], port)) def test_set_reuse_addr(self): - sock = socket.socket() + if HAS_UNIX_SOCKETS and self.family == socket.AF_UNIX: + self.skipTest("Not applicable to AF_UNIX sockets.") + sock = socket.socket(self.family) try: sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) except socket.error: @@ -719,11 +767,11 @@ class BaseTestAPI(unittest.TestCase): else: # if SO_REUSEADDR succeeded for sock we expect asyncore # to do the same - s = asyncore.dispatcher(socket.socket()) + s = asyncore.dispatcher(socket.socket(self.family)) self.assertFalse(s.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)) s.socket.close() - s.create_socket(socket.AF_INET, socket.SOCK_STREAM) + s.create_socket(self.family) s.set_reuse_addr() self.assertTrue(s.socket.getsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR)) @@ -731,18 +779,52 @@ class BaseTestAPI(unittest.TestCase): sock.close() -class TestAPI_UseSelect(BaseTestAPI): +class TestAPI_UseIPv4Sockets(BaseTestAPI): + family = socket.AF_INET + addr = (HOST, 0) + +@unittest.skipUnless(support.IPV6_ENABLED, 'IPv6 support required') +class TestAPI_UseIPv6Sockets(BaseTestAPI): + family = socket.AF_INET6 + addr = ('::1', 0) + +@unittest.skipUnless(HAS_UNIX_SOCKETS, 'Unix sockets required') +class TestAPI_UseUnixSockets(BaseTestAPI): + if HAS_UNIX_SOCKETS: + family = socket.AF_UNIX + addr = support.TESTFN + + def tearDown(self): + unlink(self.addr) + BaseTestAPI.tearDown(self) + +class TestAPI_UseIPv4Select(TestAPI_UseIPv4Sockets): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseIPv4Poll(TestAPI_UseIPv4Sockets): + use_poll = True + +class TestAPI_UseIPv6Select(TestAPI_UseIPv6Sockets): use_poll = False @unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') -class TestAPI_UsePoll(BaseTestAPI): +class TestAPI_UseIPv6Poll(TestAPI_UseIPv6Sockets): use_poll = True +class TestAPI_UseUnixSocketsSelect(TestAPI_UseUnixSockets): + use_poll = False + +@unittest.skipUnless(hasattr(select, 'poll'), 'select.poll required') +class TestAPI_UseUnixSocketsPoll(TestAPI_UseUnixSockets): + use_poll = True def test_main(): tests = [HelperFunctionTests, DispatcherTests, DispatcherWithSendTests, - DispatcherWithSendTests_UsePoll, TestAPI_UseSelect, - TestAPI_UsePoll, FileWrapperTest] + DispatcherWithSendTests_UsePoll, FileWrapperTest, + TestAPI_UseIPv4Select, TestAPI_UseIPv4Poll, TestAPI_UseIPv6Select, + TestAPI_UseIPv6Poll, TestAPI_UseUnixSocketsSelect, + TestAPI_UseUnixSocketsPoll] run_unittest(*tests) if __name__ == "__main__": diff --git a/Lib/test/test_bigmem.py b/Lib/test/test_bigmem.py index f3c6ebb..28e69a0 100644 --- a/Lib/test/test_bigmem.py +++ b/Lib/test/test_bigmem.py @@ -1,3 +1,13 @@ +"""Bigmem tests - tests for the 32-bit boundary in containers. + +These tests try to exercise the 32-bit boundary that is sometimes, if +rarely, exceeded in practice, but almost never tested. They are really only +meaningful on 64-bit builds on machines with a *lot* of memory, but the +tests are always run, usually with very low memory limits to make sure the +tests themselves don't suffer from bitrot. To run them for real, pass a +high memory limit to regrtest, with the -M option. +""" + from test import support from test.support import bigmemtest, _1G, _2G, _4G @@ -6,20 +16,35 @@ import operator import sys import functools +# These tests all use one of the bigmemtest decorators to indicate how much +# memory they use and how much memory they need to be even meaningful. The +# decorators take two arguments: a 'memuse' indicator declaring +# (approximate) bytes per size-unit the test will use (at peak usage), and a +# 'minsize' indicator declaring a minimum *useful* size. A test that +# allocates a bytestring to test various operations near the end will have a +# minsize of at least 2Gb (or it wouldn't reach the 32-bit limit, so the +# test wouldn't be very useful) and a memuse of 1 (one byte per size-unit, +# if it allocates only one big string at a time.) +# +# When run with a memory limit set, both decorators skip tests that need +# more memory than available to be meaningful. The precisionbigmemtest will +# always pass minsize as size, even if there is much more memory available. +# The bigmemtest decorator will scale size upward to fill available memory. +# # Bigmem testing houserules: # # - Try not to allocate too many large objects. It's okay to rely on -# refcounting semantics, but don't forget that 's = create_largestring()' +# refcounting semantics, and don't forget that 's = create_largestring()' # doesn't release the old 's' (if it exists) until well after its new # value has been created. Use 'del s' before the create_largestring call. # -# - Do *not* compare large objects using assertEqual or similar. It's a -# lengthy operation and the errormessage will be utterly useless due to -# its size. To make sure whether a result has the right contents, better -# to use the strip or count methods, or compare meaningful slices. +# - Do *not* compare large objects using assertEqual, assertIn or similar. +# It's a lengthy operation and the errormessage will be utterly useless +# due to its size. To make sure whether a result has the right contents, +# better to use the strip or count methods, or compare meaningful slices. # # - Don't forget to test for large indices, offsets and results and such, -# in addition to large sizes. +# in addition to large sizes. Anything that probes the 32-bit boundary. # # - When repeating an object (say, a substring, or a small list) to create # a large object, make the subobject of a length that is not a power of @@ -37,7 +62,9 @@ import functools # fail as well. I do not know whether it is due to memory fragmentation # issues, or other specifics of the platform malloc() routine. -character_size = 4 if sys.maxunicode > 0xFFFF else 2 +ascii_char_size = 1 +ucs2_char_size = 2 +ucs4_char_size = 4 class BaseStrTest: @@ -92,7 +119,7 @@ class BaseStrTest: _ = self.from_latin1 s = _('-') * size tabsize = 8 - self.assertEqual(s.expandtabs(), s) + self.assertTrue(s.expandtabs() == s) del s slen, remainder = divmod(size, tabsize) s = _(' \t') * slen @@ -347,7 +374,7 @@ class BaseStrTest: # suffer for the list size. (Otherwise, it'd cost another 48 times # size in bytes!) Nevertheless, a list of size takes # 8*size bytes. - @bigmemtest(size=_2G + 5, memuse=10) + @bigmemtest(size=_2G + 5, memuse=2 * ascii_char_size + 8) def test_split_large(self, size): _ = self.from_latin1 s = _(' a') * size + _(' ') @@ -366,9 +393,9 @@ class BaseStrTest: # take up an inordinate amount of memory chunksize = int(size ** 0.5 + 2) // 2 SUBSTR = _(' ') * chunksize + _('\n') + _(' ') * chunksize + _('\r\n') - s = SUBSTR * chunksize + s = SUBSTR * (chunksize * 2) l = s.splitlines() - self.assertEqual(len(l), chunksize * 2) + self.assertEqual(len(l), chunksize * 4) expected = _(' ') * chunksize for item in l: self.assertEqual(item, expected) @@ -419,14 +446,7 @@ class BaseStrTest: def test_translate(self, size): _ = self.from_latin1 SUBSTR = _('aZz.z.Aaz.') - if isinstance(SUBSTR, str): - trans = { - ord(_('.')): _('-'), - ord(_('a')): _('!'), - ord(_('Z')): _('$'), - } - else: - trans = bytes.maketrans(b'.aZ', b'-!$') + trans = bytes.maketrans(b'.aZ', b'-!$') sublen = len(SUBSTR) repeats = size // sublen + 2 s = SUBSTR * repeats @@ -519,19 +539,19 @@ class BaseStrTest: edge = _('-') * (size // 2) s = _('').join([edge, SUBSTR, edge]) del edge - self.assertIn(SUBSTR, s) - self.assertNotIn(SUBSTR * 2, s) - self.assertIn(_('-'), s) - self.assertNotIn(_('a'), s) + self.assertTrue(SUBSTR in s) + self.assertFalse(SUBSTR * 2 in s) + self.assertTrue(_('-') in s) + self.assertFalse(_('a') in s) s += _('a') - self.assertIn(_('a'), s) + self.assertTrue(_('a') in s) @bigmemtest(size=_2G + 10, memuse=2) def test_compare(self, size): _ = self.from_latin1 s1 = _('-') * size s2 = _('-') * size - self.assertEqual(s1, s2) + self.assertTrue(s1 == s2) del s2 s2 = s1 + _('a') self.assertFalse(s1 == s2) @@ -552,7 +572,7 @@ class BaseStrTest: h1 = hash(s) del s s = _('\x00') * (size + 1) - self.assertFalse(h1 == hash(s)) + self.assertNotEqual(h1, hash(s)) class StrTest(unittest.TestCase, BaseStrTest): @@ -563,7 +583,6 @@ class StrTest(unittest.TestCase, BaseStrTest): def basic_encode_test(self, size, enc, c='.', expectedsize=None): if expectedsize is None: expectedsize = size - try: s = c * size self.assertEqual(len(s.encode(enc)), expectedsize) @@ -582,48 +601,52 @@ class StrTest(unittest.TestCase, BaseStrTest): memuse = meth.memuse except AttributeError: continue - meth.memuse = character_size * memuse + meth.memuse = ascii_char_size * memuse self._adjusted[name] = memuse def tearDown(self): for name, memuse in self._adjusted.items(): getattr(type(self), name).memuse = memuse - # the utf8 encoder preallocates big time (4x the number of characters) - @bigmemtest(size=_2G + 2, memuse=character_size + 4) + # Many codecs convert to the legacy representation first, explaining + # why we add 'ucs4_char_size' to the 'memuse' below. + + @bigmemtest(size=_2G + 2, memuse=ascii_char_size + 1) def test_encode(self, size): return self.basic_encode_test(size, 'utf-8') - @bigmemtest(size=_4G // 6 + 2, memuse=character_size + 1) + @bigmemtest(size=_4G // 6 + 2, memuse=ascii_char_size + ucs4_char_size + 1) def test_encode_raw_unicode_escape(self, size): try: return self.basic_encode_test(size, 'raw_unicode_escape') except MemoryError: pass # acceptable on 32-bit - @bigmemtest(size=_4G // 5 + 70, memuse=character_size + 1) + @bigmemtest(size=_4G // 5 + 70, memuse=ascii_char_size + ucs4_char_size + 1) def test_encode_utf7(self, size): try: return self.basic_encode_test(size, 'utf7') except MemoryError: pass # acceptable on 32-bit - @bigmemtest(size=_4G // 4 + 5, memuse=character_size + 4) + @bigmemtest(size=_4G // 4 + 5, memuse=ascii_char_size + ucs4_char_size + 4) def test_encode_utf32(self, size): try: - return self.basic_encode_test(size, 'utf32', expectedsize=4*size+4) + return self.basic_encode_test(size, 'utf32', expectedsize=4 * size + 4) except MemoryError: pass # acceptable on 32-bit - @bigmemtest(size=_2G - 1, memuse=character_size + 1) + @bigmemtest(size=_2G - 1, memuse=ascii_char_size + 1) def test_encode_ascii(self, size): return self.basic_encode_test(size, 'ascii', c='A') - @bigmemtest(size=_2G + 10, memuse=character_size * 2) + # str % (...) uses a Py_UCS4 intermediate representation + + @bigmemtest(size=_2G + 10, memuse=ascii_char_size * 2 + ucs4_char_size) def test_format(self, size): s = '-' * size sf = '%s' % (s,) - self.assertEqual(s, sf) + self.assertTrue(s == sf) del sf sf = '..%s..' % (s,) self.assertEqual(len(sf), len(s) + 4) @@ -640,7 +663,7 @@ class StrTest(unittest.TestCase, BaseStrTest): self.assertEqual(s.count('.'), 3) self.assertEqual(s.count('-'), size * 2) - @bigmemtest(size=_2G + 10, memuse=character_size * 2) + @bigmemtest(size=_2G + 10, memuse=ascii_char_size * 2) def test_repr_small(self, size): s = '-' * size s = repr(s) @@ -661,7 +684,7 @@ class StrTest(unittest.TestCase, BaseStrTest): self.assertEqual(s.count('\\'), size) self.assertEqual(s.count('0'), size * 2) - @bigmemtest(size=_2G + 10, memuse=character_size * 5) + @bigmemtest(size=_2G + 10, memuse=ascii_char_size * 5) def test_repr_large(self, size): s = '\x00' * size s = repr(s) @@ -671,7 +694,13 @@ class StrTest(unittest.TestCase, BaseStrTest): self.assertEqual(s.count('\\'), size) self.assertEqual(s.count('0'), size * 2) - @bigmemtest(size=_2G // 5 + 1, memuse=character_size * 7) + # ascii() calls encode('ascii', 'backslashreplace'), which itself + # creates a temporary Py_UNICODE representation in addition to the + # original (Py_UCS2) one + # There's also some overallocation when resizing the ascii() result + # that isn't taken into account here. + @bigmemtest(size=_2G // 5 + 1, memuse=ucs2_char_size + + ucs4_char_size + ascii_char_size * 6) def test_unicode_repr(self, size): # Use an assigned, but not printable code point. # It is in the range of the low surrogates \uDC00-\uDFFF. @@ -686,9 +715,7 @@ class StrTest(unittest.TestCase, BaseStrTest): finally: r = s = None - # The character takes 4 bytes even in UCS-2 builds because it will - # be decomposed into surrogates. - @bigmemtest(size=_2G // 5 + 1, memuse=4 + character_size * 9) + @bigmemtest(size=_2G // 5 + 1, memuse=ucs4_char_size * 2 + ascii_char_size * 10) def test_unicode_repr_wide(self, size): char = "\U0001DCBA" s = char * size @@ -701,24 +728,37 @@ class StrTest(unittest.TestCase, BaseStrTest): finally: r = s = None - @bigmemtest(size=_4G // 5, memuse=character_size * (6 + 1)) - def _test_unicode_repr_overflow(self, size): - # XXX not sure what this test is about - char = "\uDCBA" - s = char * size - try: - r = repr(s) - self.assertTrue(s == eval(r)) - finally: - r = s = None + # The original test_translate is overriden here, so as to get the + # correct size estimate: str.translate() uses an intermediate Py_UCS4 + # representation. + + @bigmemtest(size=_2G, memuse=ascii_char_size * 2 + ucs4_char_size) + def test_translate(self, size): + _ = self.from_latin1 + SUBSTR = _('aZz.z.Aaz.') + trans = { + ord(_('.')): _('-'), + ord(_('a')): _('!'), + ord(_('Z')): _('$'), + } + sublen = len(SUBSTR) + repeats = size // sublen + 2 + s = SUBSTR * repeats + s = s.translate(trans) + self.assertEqual(len(s), repeats * sublen) + self.assertEqual(s[:sublen], SUBSTR.translate(trans)) + self.assertEqual(s[-sublen:], SUBSTR.translate(trans)) + self.assertEqual(s.count(_('.')), 0) + self.assertEqual(s.count(_('!')), repeats * 2) + self.assertEqual(s.count(_('z')), repeats * 3) class BytesTest(unittest.TestCase, BaseStrTest): def from_latin1(self, s): - return s.encode("latin1") + return s.encode("latin-1") - @bigmemtest(size=_2G + 2, memuse=1 + character_size) + @bigmemtest(size=_2G + 2, memuse=1 + ascii_char_size) def test_decode(self, size): s = self.from_latin1('.') * size self.assertEqual(len(s.decode('utf-8')), size) @@ -727,9 +767,9 @@ class BytesTest(unittest.TestCase, BaseStrTest): class BytearrayTest(unittest.TestCase, BaseStrTest): def from_latin1(self, s): - return bytearray(s.encode("latin1")) + return bytearray(s.encode("latin-1")) - @bigmemtest(size=_2G + 2, memuse=1 + character_size) + @bigmemtest(size=_2G + 2, memuse=1 + ascii_char_size) def test_decode(self, size): s = self.from_latin1('.') * size self.assertEqual(len(s.decode('utf-8')), size) @@ -752,7 +792,7 @@ class TupleTest(unittest.TestCase): def test_compare(self, size): t1 = ('',) * size t2 = ('',) * size - self.assertEqual(t1, t2) + self.assertTrue(t1 == t2) del t2 t2 = ('',) * (size + 1) self.assertFalse(t1 == t2) @@ -783,9 +823,9 @@ class TupleTest(unittest.TestCase): def test_contains(self, size): t = (1, 2, 3, 4, 5) * size self.assertEqual(len(t), size * 5) - self.assertIn(5, t) - self.assertNotIn((1, 2, 3, 4, 5), t) - self.assertNotIn(0, t) + self.assertTrue(5 in t) + self.assertFalse((1, 2, 3, 4, 5) in t) + self.assertFalse(0 in t) @bigmemtest(size=_2G + 10, memuse=8) def test_hash(self, size): @@ -869,11 +909,11 @@ class TupleTest(unittest.TestCase): self.assertEqual(s[-5:], '0, 0)') self.assertEqual(s.count('0'), size) - @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size) def test_repr_large(self, size): return self.basic_test_repr(size) @@ -888,7 +928,7 @@ class ListTest(unittest.TestCase): def test_compare(self, size): l1 = [''] * size l2 = [''] * size - self.assertEqual(l1, l2) + self.assertTrue(l1 == l2) del l2 l2 = [''] * (size + 1) self.assertFalse(l1 == l2) @@ -934,9 +974,9 @@ class ListTest(unittest.TestCase): def test_contains(self, size): l = [1, 2, 3, 4, 5] * size self.assertEqual(len(l), size * 5) - self.assertIn(5, l) - self.assertNotIn([1, 2, 3, 4, 5], l) - self.assertNotIn(0, l) + self.assertTrue(5 in l) + self.assertFalse([1, 2, 3, 4, 5] in l) + self.assertFalse(0 in l) @bigmemtest(size=_2G + 10, memuse=8) def test_hash(self, size): @@ -1044,11 +1084,11 @@ class ListTest(unittest.TestCase): self.assertEqual(s[-5:], '0, 0]') self.assertEqual(s.count('0'), size) - @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G // 3 + 2, memuse=8 + 3 * ascii_char_size) def test_repr_small(self, size): return self.basic_test_repr(size) - @bigmemtest(size=_2G + 2, memuse=8 + 3 * character_size) + @bigmemtest(size=_2G + 2, memuse=8 + 3 * ascii_char_size) def test_repr_large(self, size): return self.basic_test_repr(size) diff --git a/Lib/test/test_bool.py b/Lib/test/test_bool.py index b296870..4bab28b 100644 --- a/Lib/test/test_bool.py +++ b/Lib/test/test_bool.py @@ -330,6 +330,16 @@ class BoolTest(unittest.TestCase): except (Exception) as e_len: self.assertEqual(str(e_bool), str(e_len)) + def test_real_and_imag(self): + self.assertEqual(True.real, 1) + self.assertEqual(True.imag, 0) + self.assertIs(type(True.real), int) + self.assertIs(type(True.imag), int) + self.assertEqual(False.real, 0) + self.assertEqual(False.imag, 0) + self.assertIs(type(False.real), int) + self.assertIs(type(False.imag), int) + def test_main(): support.run_unittest(BoolTest) diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index 55fb63a..bfa5ee7 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -12,7 +12,7 @@ import types import builtins import random import traceback -from test.support import fcmp, TESTFN, unlink, run_unittest, check_warnings +from test.support import TESTFN, unlink, run_unittest, check_warnings from operator import neg try: import pty, signal @@ -255,8 +255,7 @@ class BuiltinTest(unittest.TestCase): self.assertEqual(chr(0xff), '\xff') self.assertRaises(ValueError, chr, 1<<24) self.assertEqual(chr(sys.maxunicode), - str(('\\U%08x' % (sys.maxunicode)).encode("ascii"), - 'unicode-escape')) + str('\\U0010ffff'.encode("ascii"), 'unicode-escape')) self.assertRaises(TypeError, chr) self.assertEqual(chr(0x0000FFFF), "\U0000FFFF") self.assertEqual(chr(0x00010000), "\U00010000") @@ -378,7 +377,15 @@ class BuiltinTest(unittest.TestCase): f = Foo() self.assertTrue(dir(f) == ["ga", "kan", "roo"]) - # dir(obj__dir__not_list) + # dir(obj__dir__tuple) + class Foo(object): + def __dir__(self): + return ("b", "c", "a") + res = dir(Foo()) + self.assertIsInstance(res, list) + self.assertTrue(res == ["a", "b", "c"]) + + # dir(obj__dir__not_sequence) class Foo(object): def __dir__(self): return 7 @@ -391,6 +398,8 @@ class BuiltinTest(unittest.TestCase): except: self.assertEqual(len(dir(sys.exc_info()[2])), 4) + # test that object has a __dir__() + self.assertEqual(sorted([].__dir__()), dir([])) def test_divmod(self): self.assertEqual(divmod(12, 7), (1, 5)) @@ -400,10 +409,13 @@ class BuiltinTest(unittest.TestCase): self.assertEqual(divmod(-sys.maxsize-1, -1), (sys.maxsize+1, 0)) - self.assertTrue(not fcmp(divmod(3.25, 1.0), (3.0, 0.25))) - self.assertTrue(not fcmp(divmod(-3.25, 1.0), (-4.0, 0.75))) - self.assertTrue(not fcmp(divmod(3.25, -1.0), (-4.0, -0.75))) - self.assertTrue(not fcmp(divmod(-3.25, -1.0), (3.0, -0.25))) + for num, denom, exp_result in [ (3.25, 1.0, (3.0, 0.25)), + (-3.25, 1.0, (-4.0, 0.75)), + (3.25, -1.0, (-4.0, -0.75)), + (-3.25, -1.0, (3.0, -0.25))]: + result = divmod(num, denom) + self.assertAlmostEqual(result[0], exp_result[0]) + self.assertAlmostEqual(result[1], exp_result[1]) self.assertRaises(TypeError, divmod) @@ -1197,6 +1209,9 @@ class BuiltinTest(unittest.TestCase): self.assertRaises(TypeError, sum, 42) self.assertRaises(TypeError, sum, ['a', 'b', 'c']) self.assertRaises(TypeError, sum, ['a', 'b', 'c'], '') + self.assertRaises(TypeError, sum, [b'a', b'c'], b'') + values = [bytearray(b'a'), bytearray(b'b')] + self.assertRaises(TypeError, sum, values, bytearray(b'')) self.assertRaises(TypeError, sum, [[1], [2], [3]]) self.assertRaises(TypeError, sum, [{2:3}]) self.assertRaises(TypeError, sum, [{2:3}]*2, {2:3}) @@ -1358,14 +1373,14 @@ class BuiltinTest(unittest.TestCase): # -------------------------------------------------------------------- # Issue #7994: object.__format__ with a non-empty format string is - # pending deprecated + # deprecated def test_deprecated_format_string(obj, fmt_str, should_raise_warning): with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always", PendingDeprecationWarning) + warnings.simplefilter("always", DeprecationWarning) format(obj, fmt_str) if should_raise_warning: self.assertEqual(len(w), 1) - self.assertIsInstance(w[0].message, PendingDeprecationWarning) + self.assertIsInstance(w[0].message, DeprecationWarning) self.assertIn('object.__format__ with a non-empty format ' 'string', str(w[0].message)) else: @@ -1409,6 +1424,13 @@ class BuiltinTest(unittest.TestCase): self.assertRaises(ValueError, x.translate, b"1", 1) self.assertRaises(TypeError, x.translate, b"1"*256, 1) + def test_construct_singletons(self): + for const in None, Ellipsis, NotImplemented: + tp = type(const) + self.assertIs(tp(), const) + self.assertRaises(TypeError, tp, 1, 2) + self.assertRaises(TypeError, tp, a=1, b=2) + class TestSorted(unittest.TestCase): def test_basic(self): diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 5eab8f5..7acfde5 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -188,24 +188,26 @@ class BaseBytesTest(unittest.TestCase): def test_encoding(self): sample = "Hello world\n\u1234\u5678\u9abc" - for enc in ("utf8", "utf16"): + for enc in ("utf-8", "utf-16"): b = self.type2test(sample, enc) self.assertEqual(b, self.type2test(sample.encode(enc))) - self.assertRaises(UnicodeEncodeError, self.type2test, sample, "latin1") - b = self.type2test(sample, "latin1", "ignore") + self.assertRaises(UnicodeEncodeError, self.type2test, sample, "latin-1") + b = self.type2test(sample, "latin-1", "ignore") self.assertEqual(b, self.type2test(sample[:-3], "utf-8")) def test_decode(self): sample = "Hello world\n\u1234\u5678\u9abc\def0\def0" - for enc in ("utf8", "utf16"): + for enc in ("utf-8", "utf-16"): b = self.type2test(sample, enc) self.assertEqual(b.decode(enc), sample) sample = "Hello world\n\x80\x81\xfe\xff" - b = self.type2test(sample, "latin1") - self.assertRaises(UnicodeDecodeError, b.decode, "utf8") - self.assertEqual(b.decode("utf8", "ignore"), "Hello world\n") - self.assertEqual(b.decode(errors="ignore", encoding="utf8"), + b = self.type2test(sample, "latin-1") + self.assertRaises(UnicodeDecodeError, b.decode, "utf-8") + self.assertEqual(b.decode("utf-8", "ignore"), "Hello world\n") + self.assertEqual(b.decode(errors="ignore", encoding="utf-8"), "Hello world\n") + # Default encoding is utf-8 + self.assertEqual(self.type2test(b'\xe2\x98\x83').decode(), '\u2603') def test_from_int(self): b = self.type2test(0) @@ -291,10 +293,27 @@ class BaseBytesTest(unittest.TestCase): def test_count(self): b = self.type2test(b'mississippi') + i = 105 + p = 112 + w = 119 + self.assertEqual(b.count(b'i'), 4) self.assertEqual(b.count(b'ss'), 2) self.assertEqual(b.count(b'w'), 0) + self.assertEqual(b.count(i), 4) + self.assertEqual(b.count(w), 0) + + self.assertEqual(b.count(b'i', 6), 2) + self.assertEqual(b.count(b'p', 6), 2) + self.assertEqual(b.count(b'i', 1, 3), 1) + self.assertEqual(b.count(b'p', 7, 9), 1) + + self.assertEqual(b.count(i, 6), 2) + self.assertEqual(b.count(p, 6), 2) + self.assertEqual(b.count(i, 1, 3), 1) + self.assertEqual(b.count(p, 7, 9), 1) + def test_startswith(self): b = self.type2test(b'hello') self.assertFalse(self.type2test().startswith(b"anything")) @@ -325,35 +344,81 @@ class BaseBytesTest(unittest.TestCase): def test_find(self): b = self.type2test(b'mississippi') + i = 105 + w = 119 + self.assertEqual(b.find(b'ss'), 2) + self.assertEqual(b.find(b'w'), -1) + self.assertEqual(b.find(b'mississippian'), -1) + + self.assertEqual(b.find(i), 1) + self.assertEqual(b.find(w), -1) + self.assertEqual(b.find(b'ss', 3), 5) self.assertEqual(b.find(b'ss', 1, 7), 2) self.assertEqual(b.find(b'ss', 1, 3), -1) - self.assertEqual(b.find(b'w'), -1) - self.assertEqual(b.find(b'mississippian'), -1) + + self.assertEqual(b.find(i, 6), 7) + self.assertEqual(b.find(i, 1, 3), 1) + self.assertEqual(b.find(w, 1, 3), -1) def test_rfind(self): b = self.type2test(b'mississippi') + i = 105 + w = 119 + self.assertEqual(b.rfind(b'ss'), 5) - self.assertEqual(b.rfind(b'ss', 3), 5) - self.assertEqual(b.rfind(b'ss', 0, 6), 2) self.assertEqual(b.rfind(b'w'), -1) self.assertEqual(b.rfind(b'mississippian'), -1) + self.assertEqual(b.rfind(i), 10) + self.assertEqual(b.rfind(w), -1) + + self.assertEqual(b.rfind(b'ss', 3), 5) + self.assertEqual(b.rfind(b'ss', 0, 6), 2) + + self.assertEqual(b.rfind(i, 1, 3), 1) + self.assertEqual(b.rfind(i, 3, 9), 7) + self.assertEqual(b.rfind(w, 1, 3), -1) + def test_index(self): - b = self.type2test(b'world') - self.assertEqual(b.index(b'w'), 0) - self.assertEqual(b.index(b'orl'), 1) - self.assertRaises(ValueError, b.index, b'worm') - self.assertRaises(ValueError, b.index, b'ldo') + b = self.type2test(b'mississippi') + i = 105 + w = 119 + + self.assertEqual(b.index(b'ss'), 2) + self.assertRaises(ValueError, b.index, b'w') + self.assertRaises(ValueError, b.index, b'mississippian') + + self.assertEqual(b.index(i), 1) + self.assertRaises(ValueError, b.index, w) + + self.assertEqual(b.index(b'ss', 3), 5) + self.assertEqual(b.index(b'ss', 1, 7), 2) + self.assertRaises(ValueError, b.index, b'ss', 1, 3) + + self.assertEqual(b.index(i, 6), 7) + self.assertEqual(b.index(i, 1, 3), 1) + self.assertRaises(ValueError, b.index, w, 1, 3) def test_rindex(self): - # XXX could be more rigorous - b = self.type2test(b'world') - self.assertEqual(b.rindex(b'w'), 0) - self.assertEqual(b.rindex(b'orl'), 1) - self.assertRaises(ValueError, b.rindex, b'worm') - self.assertRaises(ValueError, b.rindex, b'ldo') + b = self.type2test(b'mississippi') + i = 105 + w = 119 + + self.assertEqual(b.rindex(b'ss'), 5) + self.assertRaises(ValueError, b.rindex, b'w') + self.assertRaises(ValueError, b.rindex, b'mississippian') + + self.assertEqual(b.rindex(i), 10) + self.assertRaises(ValueError, b.rindex, w) + + self.assertEqual(b.rindex(b'ss', 3), 5) + self.assertEqual(b.rindex(b'ss', 0, 6), 2) + + self.assertEqual(b.rindex(i, 1, 3), 1) + self.assertEqual(b.rindex(i, 3, 9), 7) + self.assertRaises(ValueError, b.rindex, w, 1, 3) def test_replace(self): b = self.type2test(b'mississippi') @@ -473,6 +538,27 @@ class BaseBytesTest(unittest.TestCase): self.assertRaises(TypeError, self.type2test(b'abc').lstrip, 'b') self.assertRaises(TypeError, self.type2test(b'abc').rstrip, 'b') + def test_center(self): + # Fill character can be either bytes or bytearray (issue 12380) + b = self.type2test(b'abc') + for fill_type in (bytes, bytearray): + self.assertEqual(b.center(7, fill_type(b'-')), + self.type2test(b'--abc--')) + + def test_ljust(self): + # Fill character can be either bytes or bytearray (issue 12380) + b = self.type2test(b'abc') + for fill_type in (bytes, bytearray): + self.assertEqual(b.ljust(7, fill_type(b'-')), + self.type2test(b'abc----')) + + def test_rjust(self): + # Fill character can be either bytes or bytearray (issue 12380) + b = self.type2test(b'abc') + for fill_type in (bytes, bytearray): + self.assertEqual(b.rjust(7, fill_type(b'-')), + self.type2test(b'----abc')) + def test_ord(self): b = self.type2test(b'\0A\x7f\x80\xff') self.assertEqual([ord(b[i:i+1]) for i in range(len(b))], @@ -529,6 +615,14 @@ class BaseBytesTest(unittest.TestCase): self.assertEqual(True, b.startswith(h, None, -2)) self.assertEqual(False, b.startswith(x, None, None)) + def test_integer_arguments_out_of_byte_range(self): + b = self.type2test(b'hello') + + for method in (b.count, b.find, b.index, b.rfind, b.rindex): + self.assertRaises(ValueError, method, -1) + self.assertRaises(ValueError, method, 256) + self.assertRaises(ValueError, method, 9999) + def test_find_etc_raise_correct_error_messages(self): # issue 11828 b = self.type2test(b'hello') @@ -634,6 +728,39 @@ class ByteArrayTest(BaseBytesTest): b.reverse() self.assertFalse(b) + def test_clear(self): + b = bytearray(b'python') + b.clear() + self.assertEqual(b, b'') + + b = bytearray(b'') + b.clear() + self.assertEqual(b, b'') + + b = bytearray(b'') + b.append(ord('r')) + b.clear() + b.append(ord('p')) + self.assertEqual(b, b'p') + + def test_copy(self): + b = bytearray(b'abc') + bb = b.copy() + self.assertEqual(bb, b'abc') + + b = bytearray(b'') + bb = b.copy() + self.assertEqual(bb, b'') + + # test that it's indeed a copy and not a reference + b = bytearray(b'abc') + bb = b.copy() + self.assertEqual(b, bb) + self.assertIsNot(b, bb) + bb.append(ord('d')) + self.assertEqual(bb, b'abcd') + self.assertEqual(b, b'abc') + def test_regexps(self): def by(s): return bytearray(map(ord, s)) @@ -1105,9 +1232,11 @@ class FixedStringTest(test.string_tests.BaseTest): class ByteArrayAsStringTest(FixedStringTest): type2test = bytearray + contains_bytes = True class BytesAsStringTest(FixedStringTest): type2test = bytes + contains_bytes = True class SubclassTest(unittest.TestCase): diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index be35580..c324fb1 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -1,10 +1,11 @@ #!/usr/bin/env python3 from test import support -from test.support import TESTFN +from test.support import TESTFN, bigmemtest, _4G import unittest from io import BytesIO import os +import random import subprocess import sys @@ -21,9 +22,31 @@ has_cmdline_bunzip2 = sys.platform not in ("win32", "os2emx") class BaseTest(unittest.TestCase): "Base for other testcases." - TEXT = b'root:x:0:0:root:/root:/bin/bash\nbin:x:1:1:bin:/bin:\ndaemon:x:2:2:daemon:/sbin:\nadm:x:3:4:adm:/var/adm:\nlp:x:4:7:lp:/var/spool/lpd:\nsync:x:5:0:sync:/sbin:/bin/sync\nshutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\nhalt:x:7:0:halt:/sbin:/sbin/halt\nmail:x:8:12:mail:/var/spool/mail:\nnews:x:9:13:news:/var/spool/news:\nuucp:x:10:14:uucp:/var/spool/uucp:\noperator:x:11:0:operator:/root:\ngames:x:12:100:games:/usr/games:\ngopher:x:13:30:gopher:/usr/lib/gopher-data:\nftp:x:14:50:FTP User:/var/ftp:/bin/bash\nnobody:x:65534:65534:Nobody:/home:\npostfix:x:100:101:postfix:/var/spool/postfix:\nniemeyer:x:500:500::/home/niemeyer:/bin/bash\npostgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\nmysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\nwww:x:103:104::/var/www:/bin/false\n' + TEXT_LINES = [ + b'root:x:0:0:root:/root:/bin/bash\n', + b'bin:x:1:1:bin:/bin:\n', + b'daemon:x:2:2:daemon:/sbin:\n', + b'adm:x:3:4:adm:/var/adm:\n', + b'lp:x:4:7:lp:/var/spool/lpd:\n', + b'sync:x:5:0:sync:/sbin:/bin/sync\n', + b'shutdown:x:6:0:shutdown:/sbin:/sbin/shutdown\n', + b'halt:x:7:0:halt:/sbin:/sbin/halt\n', + b'mail:x:8:12:mail:/var/spool/mail:\n', + b'news:x:9:13:news:/var/spool/news:\n', + b'uucp:x:10:14:uucp:/var/spool/uucp:\n', + b'operator:x:11:0:operator:/root:\n', + b'games:x:12:100:games:/usr/games:\n', + b'gopher:x:13:30:gopher:/usr/lib/gopher-data:\n', + b'ftp:x:14:50:FTP User:/var/ftp:/bin/bash\n', + b'nobody:x:65534:65534:Nobody:/home:\n', + b'postfix:x:100:101:postfix:/var/spool/postfix:\n', + b'niemeyer:x:500:500::/home/niemeyer:/bin/bash\n', + b'postgres:x:101:102:PostgreSQL Server:/var/lib/pgsql:/bin/bash\n', + b'mysql:x:102:103:MySQL server:/var/lib/mysql:/bin/bash\n', + b'www:x:103:104::/var/www:/bin/false\n', + ] + TEXT = b''.join(TEXT_LINES) DATA = b'BZh91AY&SY.\xc8N\x18\x00\x01>_\x80\x00\x10@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe00\x01\x99\xaa\x00\xc0\x03F\x86\x8c#&\x83F\x9a\x03\x06\xa6\xd0\xa6\x93M\x0fQ\xa7\xa8\x06\x804hh\x12$\x11\xa4i4\xf14S\xd2<Q\xb5\x0fH\xd3\xd4\xdd\xd5\x87\xbb\xf8\x94\r\x8f\xafI\x12\xe1\xc9\xf8/E\x00pu\x89\x12]\xc9\xbbDL\nQ\x0e\t1\x12\xdf\xa0\xc0\x97\xac2O9\x89\x13\x94\x0e\x1c7\x0ed\x95I\x0c\xaaJ\xa4\x18L\x10\x05#\x9c\xaf\xba\xbc/\x97\x8a#C\xc8\xe1\x8cW\xf9\xe2\xd0\xd6M\xa7\x8bXa<e\x84t\xcbL\xb3\xa7\xd9\xcd\xd1\xcb\x84.\xaf\xb3\xab\xab\xad`n}\xa0lh\tE,\x8eZ\x15\x17VH>\x88\xe5\xcd9gd6\x0b\n\xe9\x9b\xd5\x8a\x99\xf7\x08.K\x8ev\xfb\xf7xw\xbb\xdf\xa1\x92\xf1\xdd|/";\xa2\xba\x9f\xd5\xb1#A\xb6\xf6\xb3o\xc9\xc5y\\\xebO\xe7\x85\x9a\xbc\xb6f8\x952\xd5\xd7"%\x89>V,\xf7\xa6z\xe2\x9f\xa3\xdf\x11\x11"\xd6E)I\xa9\x13^\xca\xf3r\xd0\x03U\x922\xf26\xec\xb6\xed\x8b\xc3U\x13\x9d\xc5\x170\xa4\xfa^\x92\xacDF\x8a\x97\xd6\x19\xfe\xdd\xb8\xbd\x1a\x9a\x19\xa3\x80ankR\x8b\xe5\xd83]\xa9\xc6\x08\x82f\xf6\xb9"6l$\xb8j@\xc0\x8a\xb0l1..\xbak\x83ls\x15\xbc\xf4\xc1\x13\xbe\xf8E\xb8\x9d\r\xa8\x9dk\x84\xd3n\xfa\xacQ\x07\xb1%y\xaav\xb4\x08\xe0z\x1b\x16\xf5\x04\xe9\xcc\xb9\x08z\x1en7.G\xfc]\xc9\x14\xe1B@\xbb!8`' - DATA_CRLF = b'BZh91AY&SY\xaez\xbbN\x00\x01H\xdf\x80\x00\x12@\x02\xff\xf0\x01\x07n\x00?\xe7\xff\xe0@\x01\xbc\xc6`\x86*\x8d=M\xa9\x9a\x86\xd0L@\x0fI\xa6!\xa1\x13\xc8\x88jdi\x8d@\x03@\x1a\x1a\x0c\x0c\x83 \x00\xc4h2\x19\x01\x82D\x84e\t\xe8\x99\x89\x19\x1ah\x00\r\x1a\x11\xaf\x9b\x0fG\xf5(\x1b\x1f?\t\x12\xcf\xb5\xfc\x95E\x00ps\x89\x12^\xa4\xdd\xa2&\x05(\x87\x04\x98\x89u\xe40%\xb6\x19\'\x8c\xc4\x89\xca\x07\x0e\x1b!\x91UIFU%C\x994!DI\xd2\xfa\xf0\xf1N8W\xde\x13A\xf5\x9cr%?\x9f3;I45A\xd1\x8bT\xb1<l\xba\xcb_\xc00xY\x17r\x17\x88\x08\x08@\xa0\ry@\x10\x04$)`\xf2\xce\x89z\xb0s\xec\x9b.iW\x9d\x81\xb5-+t\x9f\x1a\'\x97dB\xf5x\xb5\xbe.[.\xd7\x0e\x81\xe7\x08\x1cN`\x88\x10\xca\x87\xc3!"\x80\x92R\xa1/\xd1\xc0\xe6mf\xac\xbd\x99\xcca\xb3\x8780>\xa4\xc7\x8d\x1a\\"\xad\xa1\xabyBg\x15\xb9l\x88\x88\x91k"\x94\xa4\xd4\x89\xae*\xa6\x0b\x10\x0c\xd6\xd4m\xe86\xec\xb5j\x8a\x86j\';\xca.\x01I\xf2\xaaJ\xe8\x88\x8cU+t3\xfb\x0c\n\xa33\x13r2\r\x16\xe0\xb3(\xbf\x1d\x83r\xe7M\xf0D\x1365\xd8\x88\xd3\xa4\x92\xcb2\x06\x04\\\xc1\xb0\xea//\xbek&\xd8\xe6+t\xe5\xa1\x13\xada\x16\xder5"w]\xa2i\xb7[\x97R \xe2IT\xcd;Z\x04dk4\xad\x8a\t\xd3\x81z\x10\xf1:^`\xab\x1f\xc5\xdc\x91N\x14$+\x9e\xae\xd3\x80' if has_cmdline_bunzip2: def decompress(self, data): @@ -54,83 +77,135 @@ class BZ2FileTest(BaseTest): if os.path.isfile(self.filename): os.unlink(self.filename) - def createTempFile(self, crlf=0): + def createTempFile(self, streams=1): with open(self.filename, "wb") as f: - if crlf: - data = self.DATA_CRLF - else: - data = self.DATA - f.write(data) + f.write(self.DATA * streams) def testRead(self): - # "Test BZ2File.read()" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.read, None) self.assertEqual(bz2f.read(), self.TEXT) + def testReadMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.read, None) + self.assertEqual(bz2f.read(), self.TEXT * 5) + + def testReadMonkeyMultiStream(self): + # Test BZ2File.read() on a multi-stream archive where a stream + # boundary coincides with the end of the raw read buffer. + buffer_size = bz2._BUFFER_SIZE + bz2._BUFFER_SIZE = len(self.DATA) + try: + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.read, None) + self.assertEqual(bz2f.read(), self.TEXT * 5) + finally: + bz2._BUFFER_SIZE = buffer_size + def testRead0(self): - # Test BBZ2File.read(0)" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.read, None) self.assertEqual(bz2f.read(0), b"") def testReadChunk10(self): - # "Test BZ2File.read() in chunks of 10 bytes" self.createTempFile() with BZ2File(self.filename) as bz2f: text = b'' - while 1: + while True: str = bz2f.read(10) if not str: break text += str self.assertEqual(text, self.TEXT) + def testReadChunk10MultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + text = b'' + while True: + str = bz2f.read(10) + if not str: + break + text += str + self.assertEqual(text, self.TEXT * 5) + def testRead100(self): - # "Test BZ2File.read(100)" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(100), self.TEXT[:100]) + def testPeek(self): + self.createTempFile() + with BZ2File(self.filename) as bz2f: + pdata = bz2f.peek() + self.assertNotEqual(len(pdata), 0) + self.assertTrue(self.TEXT.startswith(pdata)) + self.assertEqual(bz2f.read(), self.TEXT) + + def testReadInto(self): + self.createTempFile() + with BZ2File(self.filename) as bz2f: + n = 128 + b = bytearray(n) + self.assertEqual(bz2f.readinto(b), n) + self.assertEqual(b, self.TEXT[:n]) + n = len(self.TEXT) - n + b = bytearray(len(self.TEXT)) + self.assertEqual(bz2f.readinto(b), n) + self.assertEqual(b[:n], self.TEXT[-n:]) + def testReadLine(self): - # "Test BZ2File.readline()" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readline, None) - sio = BytesIO(self.TEXT) - for line in sio.readlines(): + for line in self.TEXT_LINES: + self.assertEqual(bz2f.readline(), line) + + def testReadLineMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.readline, None) + for line in self.TEXT_LINES * 5: self.assertEqual(bz2f.readline(), line) def testReadLines(self): - # "Test BZ2File.readlines()" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.readlines, None) - sio = BytesIO(self.TEXT) - self.assertEqual(bz2f.readlines(), sio.readlines()) + self.assertEqual(bz2f.readlines(), self.TEXT_LINES) + + def testReadLinesMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.readlines, None) + self.assertEqual(bz2f.readlines(), self.TEXT_LINES * 5) def testIterator(self): - # "Test iter(BZ2File)" self.createTempFile() with BZ2File(self.filename) as bz2f: - sio = BytesIO(self.TEXT) - self.assertEqual(list(iter(bz2f)), sio.readlines()) + self.assertEqual(list(iter(bz2f)), self.TEXT_LINES) + + def testIteratorMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + self.assertEqual(list(iter(bz2f)), self.TEXT_LINES * 5) def testClosedIteratorDeadlock(self): - # "Test that iteration on a closed bz2file releases the lock." - # http://bugs.python.org/issue3309 + # Issue #3309: Iteration on a closed BZ2File should release the lock. self.createTempFile() bz2f = BZ2File(self.filename) bz2f.close() self.assertRaises(ValueError, bz2f.__next__) - # This call will deadlock of the above .__next__ call failed to + # This call will deadlock if the above .__next__ call failed to # release the lock. self.assertRaises(ValueError, bz2f.readlines) def testWrite(self): - # "Test BZ2File.write()" with BZ2File(self.filename, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) @@ -138,10 +213,9 @@ class BZ2FileTest(BaseTest): self.assertEqual(self.decompress(f.read()), self.TEXT) def testWriteChunks10(self): - # "Test BZ2File.write() with chunks of 10 bytes" with BZ2File(self.filename, "w") as bz2f: n = 0 - while 1: + while True: str = self.TEXT[n*10:(n+1)*10] if not str: break @@ -150,13 +224,19 @@ class BZ2FileTest(BaseTest): with open(self.filename, 'rb') as f: self.assertEqual(self.decompress(f.read()), self.TEXT) + def testWriteNonDefaultCompressLevel(self): + expected = bz2.compress(self.TEXT, compresslevel=5) + with BZ2File(self.filename, "w", compresslevel=5) as bz2f: + bz2f.write(self.TEXT) + with open(self.filename, "rb") as f: + self.assertEqual(f.read(), expected) + def testWriteLines(self): - # "Test BZ2File.writelines()" with BZ2File(self.filename, "w") as bz2f: self.assertRaises(TypeError, bz2f.writelines) - sio = BytesIO(self.TEXT) - bz2f.writelines(sio.readlines()) - # patch #1535500 + bz2f.writelines(self.TEXT_LINES) + # Issue #1535500: Calling writelines() on a closed BZ2File + # should raise an exception. self.assertRaises(ValueError, bz2f.writelines, ["a"]) with open(self.filename, 'rb') as f: self.assertEqual(self.decompress(f.read()), self.TEXT) @@ -169,39 +249,73 @@ class BZ2FileTest(BaseTest): self.assertRaises(IOError, bz2f.write, b"a") self.assertRaises(IOError, bz2f.writelines, [b"a"]) + def testAppend(self): + with BZ2File(self.filename, "w") as bz2f: + self.assertRaises(TypeError, bz2f.write) + bz2f.write(self.TEXT) + with BZ2File(self.filename, "a") as bz2f: + self.assertRaises(TypeError, bz2f.write) + bz2f.write(self.TEXT) + with open(self.filename, 'rb') as f: + self.assertEqual(self.decompress(f.read()), self.TEXT * 2) + def testSeekForward(self): - # "Test BZ2File.seek(150, 0)" self.createTempFile() with BZ2File(self.filename) as bz2f: self.assertRaises(TypeError, bz2f.seek) bz2f.seek(150) self.assertEqual(bz2f.read(), self.TEXT[150:]) + def testSeekForwardAcrossStreams(self): + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + self.assertRaises(TypeError, bz2f.seek) + bz2f.seek(len(self.TEXT) + 150) + self.assertEqual(bz2f.read(), self.TEXT[150:]) + def testSeekBackwards(self): - # "Test BZ2File.seek(-150, 1)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.read(500) bz2f.seek(-150, 1) self.assertEqual(bz2f.read(), self.TEXT[500-150:]) + def testSeekBackwardsAcrossStreams(self): + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + readto = len(self.TEXT) + 100 + while readto > 0: + readto -= len(bz2f.read(readto)) + bz2f.seek(-150, 1) + self.assertEqual(bz2f.read(), self.TEXT[100-150:] + self.TEXT) + def testSeekBackwardsFromEnd(self): - # "Test BZ2File.seek(-150, 2)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(-150, 2) self.assertEqual(bz2f.read(), self.TEXT[len(self.TEXT)-150:]) + def testSeekBackwardsFromEndAcrossStreams(self): + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + bz2f.seek(-1000, 2) + self.assertEqual(bz2f.read(), (self.TEXT * 2)[-1000:]) + def testSeekPostEnd(self): - # "Test BZ2File.seek(150000)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(150000) self.assertEqual(bz2f.tell(), len(self.TEXT)) self.assertEqual(bz2f.read(), b"") + def testSeekPostEndMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + bz2f.seek(150000) + self.assertEqual(bz2f.tell(), len(self.TEXT) * 5) + self.assertEqual(bz2f.read(), b"") + def testSeekPostEndTwice(self): - # "Test BZ2File.seek(150000) twice" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(150000) @@ -209,27 +323,45 @@ class BZ2FileTest(BaseTest): self.assertEqual(bz2f.tell(), len(self.TEXT)) self.assertEqual(bz2f.read(), b"") + def testSeekPostEndTwiceMultiStream(self): + self.createTempFile(streams=5) + with BZ2File(self.filename) as bz2f: + bz2f.seek(150000) + bz2f.seek(150000) + self.assertEqual(bz2f.tell(), len(self.TEXT) * 5) + self.assertEqual(bz2f.read(), b"") + def testSeekPreStart(self): - # "Test BZ2File.seek(-150, 0)" self.createTempFile() with BZ2File(self.filename) as bz2f: bz2f.seek(-150) self.assertEqual(bz2f.tell(), 0) self.assertEqual(bz2f.read(), self.TEXT) + def testSeekPreStartMultiStream(self): + self.createTempFile(streams=2) + with BZ2File(self.filename) as bz2f: + bz2f.seek(-150) + self.assertEqual(bz2f.tell(), 0) + self.assertEqual(bz2f.read(), self.TEXT * 2) + + def testFileno(self): + self.createTempFile() + with open(self.filename, 'rb') as rawf: + with BZ2File(fileobj=rawf) as bz2f: + self.assertEqual(bz2f.fileno(), rawf.fileno()) + def testOpenDel(self): - # "Test opening and deleting a file many times" self.createTempFile() for i in range(10000): o = BZ2File(self.filename) del o def testOpenNonexistent(self): - # "Test opening a nonexistent file" self.assertRaises(IOError, BZ2File, "/non/existent") - def testBug1191043(self): - # readlines() for files containing no newline + def testReadlinesNoNewline(self): + # Issue #1191043: readlines() fails on a file containing no newline. data = b'BZh91AY&SY\xd9b\x89]\x00\x00\x00\x03\x80\x04\x00\x02\x00\x0c\x00 \x00!\x9ah3M\x13<]\xc9\x14\xe1BCe\x8a%t' with open(self.filename, "wb") as f: f.write(data) @@ -241,7 +373,6 @@ class BZ2FileTest(BaseTest): self.assertEqual(xlines, [b'Test']) def testContextProtocol(self): - # BZ2File supports the context management protocol f = None with BZ2File(self.filename, "wb") as f: f.write(b"xxx") @@ -264,7 +395,7 @@ class BZ2FileTest(BaseTest): @unittest.skipUnless(threading, 'Threading required for this test.') def testThreading(self): - # Using a BZ2File from several threads doesn't deadlock (issue #7205). + # Issue #7205: Using a BZ2File from several threads shouldn't deadlock. data = b"1" * 2**20 nthreads = 10 with bz2.BZ2File(self.filename, 'wb') as f: @@ -277,22 +408,76 @@ class BZ2FileTest(BaseTest): for t in threads: t.join() - def testMixedIterationReads(self): - # Issue #8397: mixed iteration and reads should be forbidden. - with bz2.BZ2File(self.filename, 'wb') as f: - # The internal buffer size is hard-wired to 8192 bytes, we must - # write out more than that for the test to stop half through - # the buffer. - f.write(self.TEXT * 100) - with bz2.BZ2File(self.filename, 'rb') as f: - next(f) - self.assertRaises(ValueError, f.read) - self.assertRaises(ValueError, f.readline) - self.assertRaises(ValueError, f.readlines) + def testMixedIterationAndReads(self): + self.createTempFile() + linelen = len(self.TEXT_LINES[0]) + halflen = linelen // 2 + with bz2.BZ2File(self.filename) as bz2f: + bz2f.read(halflen) + self.assertEqual(next(bz2f), self.TEXT_LINES[0][halflen:]) + self.assertEqual(bz2f.read(), self.TEXT[linelen:]) + with bz2.BZ2File(self.filename) as bz2f: + bz2f.readline() + self.assertEqual(next(bz2f), self.TEXT_LINES[1]) + self.assertEqual(bz2f.readline(), self.TEXT_LINES[2]) + with bz2.BZ2File(self.filename) as bz2f: + bz2f.readlines() + with self.assertRaises(StopIteration): + next(bz2f) + self.assertEqual(bz2f.readlines(), []) + + def testMultiStreamOrdering(self): + # Test the ordering of streams when reading a multi-stream archive. + data1 = b"foo" * 1000 + data2 = b"bar" * 1000 + with BZ2File(self.filename, "w") as bz2f: + bz2f.write(data1) + with BZ2File(self.filename, "a") as bz2f: + bz2f.write(data2) + with BZ2File(self.filename) as bz2f: + self.assertEqual(bz2f.read(), data1 + data2) + + # Tests for a BZ2File wrapping another file object: + + def testReadBytesIO(self): + with BytesIO(self.DATA) as bio: + with BZ2File(fileobj=bio) as bz2f: + self.assertRaises(TypeError, bz2f.read, None) + self.assertEqual(bz2f.read(), self.TEXT) + self.assertFalse(bio.closed) + + def testPeekBytesIO(self): + with BytesIO(self.DATA) as bio: + with BZ2File(fileobj=bio) as bz2f: + pdata = bz2f.peek() + self.assertNotEqual(len(pdata), 0) + self.assertTrue(self.TEXT.startswith(pdata)) + self.assertEqual(bz2f.read(), self.TEXT) + + def testWriteBytesIO(self): + with BytesIO() as bio: + with BZ2File(fileobj=bio, mode="w") as bz2f: + self.assertRaises(TypeError, bz2f.write) + bz2f.write(self.TEXT) + self.assertEqual(self.decompress(bio.getvalue()), self.TEXT) + self.assertFalse(bio.closed) + + def testSeekForwardBytesIO(self): + with BytesIO(self.DATA) as bio: + with BZ2File(fileobj=bio) as bz2f: + self.assertRaises(TypeError, bz2f.seek) + bz2f.seek(150) + self.assertEqual(bz2f.read(), self.TEXT[150:]) + + def testSeekBackwardsBytesIO(self): + with BytesIO(self.DATA) as bio: + with BZ2File(fileobj=bio) as bz2f: + bz2f.read(500) + bz2f.seek(-150, 1) + self.assertEqual(bz2f.read(), self.TEXT[500-150:]) class BZ2CompressorTest(BaseTest): def testCompress(self): - # "Test BZ2Compressor.compress()/flush()" bz2c = BZ2Compressor() self.assertRaises(TypeError, bz2c.compress) data = bz2c.compress(self.TEXT) @@ -300,11 +485,10 @@ class BZ2CompressorTest(BaseTest): self.assertEqual(self.decompress(data), self.TEXT) def testCompressChunks10(self): - # "Test BZ2Compressor.compress()/flush() with chunks of 10 bytes" bz2c = BZ2Compressor() n = 0 data = b'' - while 1: + while True: str = self.TEXT[n*10:(n+1)*10] if not str: break @@ -313,23 +497,38 @@ class BZ2CompressorTest(BaseTest): data += bz2c.flush() self.assertEqual(self.decompress(data), self.TEXT) + @bigmemtest(size=_4G + 100, memuse=2) + def testCompress4G(self, size): + # "Test BZ2Compressor.compress()/flush() with >4GiB input" + bz2c = BZ2Compressor() + data = b"x" * size + try: + compressed = bz2c.compress(data) + compressed += bz2c.flush() + finally: + data = None # Release memory + data = bz2.decompress(compressed) + try: + self.assertEqual(len(data), size) + self.assertEqual(len(data.strip(b"x")), 0) + finally: + data = None + class BZ2DecompressorTest(BaseTest): def test_Constructor(self): self.assertRaises(TypeError, BZ2Decompressor, 42) def testDecompress(self): - # "Test BZ2Decompressor.decompress()" bz2d = BZ2Decompressor() self.assertRaises(TypeError, bz2d.decompress) text = bz2d.decompress(self.DATA) self.assertEqual(text, self.TEXT) def testDecompressChunks10(self): - # "Test BZ2Decompressor.decompress() with chunks of 10 bytes" bz2d = BZ2Decompressor() text = b'' n = 0 - while 1: + while True: str = self.DATA[n*10:(n+1)*10] if not str: break @@ -338,7 +537,6 @@ class BZ2DecompressorTest(BaseTest): self.assertEqual(text, self.TEXT) def testDecompressUnusedData(self): - # "Test BZ2Decompressor.decompress() with unused data" bz2d = BZ2Decompressor() unused_data = b"this is unused data" text = bz2d.decompress(self.DATA+unused_data) @@ -346,34 +544,49 @@ class BZ2DecompressorTest(BaseTest): self.assertEqual(bz2d.unused_data, unused_data) def testEOFError(self): - # "Calling BZ2Decompressor.decompress() after EOS must raise EOFError" bz2d = BZ2Decompressor() text = bz2d.decompress(self.DATA) self.assertRaises(EOFError, bz2d.decompress, b"anything") + @bigmemtest(size=_4G + 100, memuse=3) + def testDecompress4G(self, size): + # "Test BZ2Decompressor.decompress() with >4GiB input" + blocksize = 10 * 1024 * 1024 + block = random.getrandbits(blocksize * 8).to_bytes(blocksize, 'little') + try: + data = block * (size // blocksize + 1) + compressed = bz2.compress(data) + bz2d = BZ2Decompressor() + decompressed = bz2d.decompress(compressed) + self.assertTrue(decompressed == data) + finally: + data = None + compressed = None + decompressed = None + class FuncTest(BaseTest): "Test module functions" def testCompress(self): - # "Test compress() function" data = bz2.compress(self.TEXT) self.assertEqual(self.decompress(data), self.TEXT) def testDecompress(self): - # "Test decompress() function" text = bz2.decompress(self.DATA) self.assertEqual(text, self.TEXT) def testDecompressEmpty(self): - # "Test decompress() function with empty string" text = bz2.decompress(b"") self.assertEqual(text, b"") def testDecompressIncomplete(self): - # "Test decompress() function with incomplete data" self.assertRaises(ValueError, bz2.decompress, self.DATA[:-10]) + def testDecompressMultiStream(self): + text = bz2.decompress(self.DATA * 5) + self.assertEqual(text, self.TEXT * 5) + def test_main(): support.run_unittest( BZ2FileTest, @@ -385,5 +598,3 @@ def test_main(): if __name__ == '__main__': test_main() - -# vim:ts=4:sw=4 diff --git a/Lib/test/test_calendar.py b/Lib/test/test_calendar.py index d3093ac..7180afe 100644 --- a/Lib/test/test_calendar.py +++ b/Lib/test/test_calendar.py @@ -177,7 +177,7 @@ class OutputTestCase(unittest.TestCase): return not c.isspace() and not c.isdigit() lines = [] - for line in s.splitlines(False): + for line in s.splitlines(keepends=False): # Drop texts, as they are locale dependent if line and not filter(neitherspacenordigit, line): lines.append(line) diff --git a/Lib/test/test_capi.py b/Lib/test/test_capi.py index ada393d..c649d3e 100644 --- a/Lib/test/test_capi.py +++ b/Lib/test/test_capi.py @@ -193,18 +193,17 @@ def test_main(): idents = [] def callback(): - idents.append(_thread.get_ident()) + idents.append(threading.get_ident()) _testcapi._test_thread_state(callback) a = b = callback time.sleep(1) # Check our main thread is in the list exactly 3 times. - if idents.count(_thread.get_ident()) != 3: + if idents.count(threading.get_ident()) != 3: raise support.TestFailed( "Couldn't find main thread correctly in the list") if threading: - import _thread import time TestThreadState() t = threading.Thread(target=TestThreadState) diff --git a/Lib/test/test_cgi.py b/Lib/test/test_cgi.py index 9951e93..cb59008 100644 --- a/Lib/test/test_cgi.py +++ b/Lib/test/test_cgi.py @@ -155,13 +155,7 @@ class CgiTests(unittest.TestCase): cgi.logfp = None cgi.logfile = "/dev/null" cgi.initlog("%s", "Testing log 3") - def log_cleanup(): - """Restore the global state of the log vars.""" - cgi.logfile = '' - cgi.logfp.close() - cgi.logfp = None - cgi.log = cgi.initlog - self.addCleanup(log_cleanup) + self.addCleanup(cgi.closelog) cgi.log("Testing log 4") def test_fieldstorage_readline(self): diff --git a/Lib/test/test_cgitb.py b/Lib/test/test_cgitb.py new file mode 100644 index 0000000..16a4b1a --- /dev/null +++ b/Lib/test/test_cgitb.py @@ -0,0 +1,55 @@ +from test.support import run_unittest +import unittest +import sys +import subprocess +import cgitb + +class TestCgitb(unittest.TestCase): + + def test_fonts(self): + text = "Hello Robbie!" + self.assertEqual(cgitb.small(text), "<small>{}</small>".format(text)) + self.assertEqual(cgitb.strong(text), "<strong>{}</strong>".format(text)) + self.assertEqual(cgitb.grey(text), + '<font color="#909090">{}</font>'.format(text)) + + def test_blanks(self): + self.assertEqual(cgitb.small(""), "") + self.assertEqual(cgitb.strong(""), "") + self.assertEqual(cgitb.grey(""), "") + + def test_html(self): + try: + raise ValueError("Hello World") + except ValueError as err: + # If the html was templated we could do a bit more here. + # At least check that we get details on what we just raised. + html = cgitb.html(sys.exc_info()) + self.assertIn("ValueError", html) + self.assertIn(str(err), html) + + def test_text(self): + try: + raise ValueError("Hello World") + except ValueError as err: + text = cgitb.text(sys.exc_info()) + self.assertIn("ValueError", text) + self.assertIn("Hello World", text) + + def test_hook(self): + proc = subprocess.Popen([sys.executable, '-c', + ('import cgitb;' + 'cgitb.enable();' + 'raise ValueError("Hello World")')], + stdout=subprocess.PIPE) + out = proc.stdout.read().decode(sys.getfilesystemencoding()) + self.addCleanup(proc.stdout.close) + self.assertIn("ValueError", out) + self.assertIn("Hello World", out) + + +def test_main(): + run_unittest(TestCgitb) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index 56a8e39..9738691 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -31,12 +31,6 @@ class CmdLineTest(unittest.TestCase): self.verify_valid_flag('-O') self.verify_valid_flag('-OO') - def test_q(self): - self.verify_valid_flag('-Qold') - self.verify_valid_flag('-Qnew') - self.verify_valid_flag('-Qwarn') - self.verify_valid_flag('-Qwarnall') - def test_site_flag(self): self.verify_valid_flag('-S') @@ -151,7 +145,7 @@ class CmdLineTest(unittest.TestCase): @unittest.skipUnless(sys.platform == 'darwin', 'test specific to Mac OS X') def test_osx_utf8(self): def check_output(text): - decoded = text.decode('utf8', 'surrogateescape') + decoded = text.decode('utf-8', 'surrogateescape') expected = ascii(decoded).encode('ascii') + b'\n' env = os.environ.copy() @@ -223,7 +217,7 @@ class CmdLineTest(unittest.TestCase): self.assertIn(path2.encode('ascii'), out) def test_displayhook_unencodable(self): - for encoding in ('ascii', 'latin1', 'utf8'): + for encoding in ('ascii', 'latin-1', 'utf-8'): env = os.environ.copy() env['PYTHONIOENCODING'] = encoding p = subprocess.Popen( diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index c5b1e25..81bf80d 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -1,5 +1,18 @@ -import test.support, unittest -import sys, codecs, html.entities, unicodedata +import codecs +import html.entities +import sys +import test.support +import unicodedata +import unittest +import warnings + +try: + import ctypes +except ImportError: + ctypes = None + SIZEOF_WCHAR_T = -1 +else: + SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar) class PosReturn: # this can be used for configurable callbacks @@ -135,22 +148,14 @@ class CodecCallbackTest(unittest.TestCase): def test_backslashescape(self): # Does the same as the "unicode-escape" encoding, but with different # base encodings. - sin = "a\xac\u1234\u20ac\u8000" - if sys.maxunicode > 0xffff: - sin += chr(sys.maxunicode) - sout = b"a\\xac\\u1234\\u20ac\\u8000" - if sys.maxunicode > 0xffff: - sout += bytes("\\U%08x" % sys.maxunicode, "ascii") + sin = "a\xac\u1234\u20ac\u8000\U0010ffff" + sout = b"a\\xac\\u1234\\u20ac\\u8000\\U0010ffff" self.assertEqual(sin.encode("ascii", "backslashreplace"), sout) - sout = b"a\xac\\u1234\\u20ac\\u8000" - if sys.maxunicode > 0xffff: - sout += bytes("\\U%08x" % sys.maxunicode, "ascii") + sout = b"a\xac\\u1234\\u20ac\\u8000\\U0010ffff" self.assertEqual(sin.encode("latin-1", "backslashreplace"), sout) - sout = b"a\xac\\u1234\xa4\\u8000" - if sys.maxunicode > 0xffff: - sout += bytes("\\U%08x" % sys.maxunicode, "ascii") + sout = b"a\xac\\u1234\xa4\\u8000\\U0010ffff" self.assertEqual(sin.encode("iso-8859-15", "backslashreplace"), sout) def test_decoding_callbacks(self): @@ -200,33 +205,37 @@ class CodecCallbackTest(unittest.TestCase): self.assertRaises(TypeError, codecs.charmap_encode, sin, "replace", charmap) def test_decodeunicodeinternal(self): - self.assertRaises( - UnicodeDecodeError, - b"\x00\x00\x00\x00\x00".decode, - "unicode-internal", - ) - if sys.maxunicode > 0xffff: + with test.support.check_warnings(('unicode_internal codec has been ' + 'deprecated', DeprecationWarning)): + self.assertRaises( + UnicodeDecodeError, + b"\x00\x00\x00\x00\x00".decode, + "unicode-internal", + ) + if SIZEOF_WCHAR_T == 4: def handler_unicodeinternal(exc): if not isinstance(exc, UnicodeDecodeError): raise TypeError("don't know how to handle %r" % exc) return ("\x01", 1) - self.assertEqual( - b"\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"), - "\u0000" - ) + with test.support.check_warnings(('unicode_internal codec has been ' + 'deprecated', DeprecationWarning)): + self.assertEqual( + b"\x00\x00\x00\x00\x00".decode("unicode-internal", "ignore"), + "\u0000" + ) - self.assertEqual( - b"\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"), - "\u0000\ufffd" - ) + self.assertEqual( + b"\x00\x00\x00\x00\x00".decode("unicode-internal", "replace"), + "\u0000\ufffd" + ) - codecs.register_error("test.hui", handler_unicodeinternal) + codecs.register_error("test.hui", handler_unicodeinternal) - self.assertEqual( - b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"), - "\u0000\u0001\u0000" - ) + self.assertEqual( + b"\x00\x00\x00\x00\x00".decode("unicode-internal", "test.hui"), + "\u0000\u0001\u0000" + ) def test_callbacks(self): def handler1(exc): @@ -355,7 +364,7 @@ class CodecCallbackTest(unittest.TestCase): ["ascii", "\uffffx", 0, 1, "ouch"], "'ascii' codec can't encode character '\\uffff' in position 0: ouch" ) - if sys.maxunicode > 0xffff: + if SIZEOF_WCHAR_T == 4: self.check_exceptionobjectargs( UnicodeEncodeError, ["ascii", "\U00010000x", 0, 1, "ouch"], @@ -390,7 +399,7 @@ class CodecCallbackTest(unittest.TestCase): ["g\uffffrk", 1, 2, "ouch"], "can't translate character '\\uffff' in position 1: ouch" ) - if sys.maxunicode > 0xffff: + if SIZEOF_WCHAR_T == 4: self.check_exceptionobjectargs( UnicodeTranslateError, ["g\U00010000rk", 1, 2, "ouch"], @@ -577,31 +586,30 @@ class CodecCallbackTest(unittest.TestCase): UnicodeEncodeError("ascii", "\uffff", 0, 1, "ouch")), ("\\uffff", 1) ) - # 1 on UCS-4 builds, 2 on UCS-2 - len_wide = len("\U00010000") - self.assertEqual( - codecs.backslashreplace_errors( - UnicodeEncodeError("ascii", "\U00010000", - 0, len_wide, "ouch")), - ("\\U00010000", len_wide) - ) - self.assertEqual( - codecs.backslashreplace_errors( - UnicodeEncodeError("ascii", "\U0010ffff", - 0, len_wide, "ouch")), - ("\\U0010ffff", len_wide) - ) - # Lone surrogates (regardless of unicode width) - self.assertEqual( - codecs.backslashreplace_errors( - UnicodeEncodeError("ascii", "\ud800", 0, 1, "ouch")), - ("\\ud800", 1) - ) - self.assertEqual( - codecs.backslashreplace_errors( - UnicodeEncodeError("ascii", "\udfff", 0, 1, "ouch")), - ("\\udfff", 1) - ) + if SIZEOF_WCHAR_T > 0: + self.assertEqual( + codecs.backslashreplace_errors( + UnicodeEncodeError("ascii", "\U00010000", + 0, 1, "ouch")), + ("\\U00010000", 1) + ) + self.assertEqual( + codecs.backslashreplace_errors( + UnicodeEncodeError("ascii", "\U0010ffff", + 0, 1, "ouch")), + ("\\U0010ffff", 1) + ) + # Lone surrogates (regardless of unicode width) + self.assertEqual( + codecs.backslashreplace_errors( + UnicodeEncodeError("ascii", "\ud800", 0, 1, "ouch")), + ("\\ud800", 1) + ) + self.assertEqual( + codecs.backslashreplace_errors( + UnicodeEncodeError("ascii", "\udfff", 0, 1, "ouch")), + ("\\udfff", 1) + ) def test_badhandlerresults(self): results = ( 42, "foo", (1,2,3), ("foo", 1, 3), ("foo", None), ("foo",), ("foo", 1, 3), ("foo", None), ("foo",) ) @@ -622,12 +630,14 @@ class CodecCallbackTest(unittest.TestCase): ("utf-7", b"+x-"), ("unicode-internal", b"\x00"), ): - self.assertRaises( - TypeError, - bytes.decode, - enc, - "test.badhandler" - ) + with test.support.check_warnings(): + # unicode-internal has been deprecated + self.assertRaises( + TypeError, + bytes.decode, + enc, + "test.badhandler" + ) def test_lookup(self): self.assertEqual(codecs.strict_errors, codecs.lookup_error("strict")) @@ -679,7 +689,7 @@ class CodecCallbackTest(unittest.TestCase): # Python/codecs.c::PyCodec_XMLCharRefReplaceErrors() # and inline implementations v = (1, 5, 10, 50, 100, 500, 1000, 5000, 10000, 50000) - if sys.maxunicode>=100000: + if SIZEOF_WCHAR_T == 4: v += (100000, 500000, 1000000) s = "".join([chr(x) for x in v]) codecs.register_error("test.xmlcharrefreplace", codecs.xmlcharrefreplace_errors) @@ -843,8 +853,12 @@ class CodecCallbackTest(unittest.TestCase): else: raise TypeError("don't know how to handle %r" % exc) codecs.register_error("test.replacing", replacing) - for (encoding, data) in baddata: - self.assertRaises(TypeError, data.decode, encoding, "test.replacing") + + with test.support.check_warnings(): + # unicode-internal has been deprecated + for (encoding, data) in baddata: + with self.assertRaises(TypeError): + data.decode(encoding, "test.replacing") def mutating(exc): if isinstance(exc, UnicodeDecodeError): @@ -855,8 +869,11 @@ class CodecCallbackTest(unittest.TestCase): codecs.register_error("test.mutating", mutating) # If the decoder doesn't pick up the modified input the following # will lead to an endless loop - for (encoding, data) in baddata: - self.assertRaises(TypeError, data.decode, encoding, "test.replacing") + with test.support.check_warnings(): + # unicode-internal has been deprecated + for (encoding, data) in baddata: + with self.assertRaises(TypeError): + data.decode(encoding, "test.replacing") def test_main(): test.support.run_unittest(CodecCallbackTest) diff --git a/Lib/test/test_codecencodings_cn.py b/Lib/test/test_codecencodings_cn.py index dca9f10..ee3d165 100644 --- a/Lib/test/test_codecencodings_cn.py +++ b/Lib/test/test_codecencodings_cn.py @@ -15,8 +15,8 @@ class Test_GB2312(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x81\x81\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x81\x81\xc1\xc4", "replace", "abc\ufffd\u804a"), - (b"abc\x81\x81\xc1\xc4\xc8", "replace", "abc\ufffd\u804a\ufffd"), + (b"abc\x81\x81\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x81\x81\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), (b"abc\x81\x81\xc1\xc4", "ignore", "abc\u804a"), (b"\xc1\x64", "strict", None), ) @@ -28,8 +28,8 @@ class Test_GBK(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u804a"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), (b"\x83\x34\x83\x31", "strict", None), ("\u30fb", "strict", None), @@ -42,11 +42,14 @@ class Test_GB18030(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u804a"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u804a\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u804a"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u804a\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u804a"), - (b"abc\x84\x39\x84\x39\xc1\xc4", "replace", "abc\ufffd\u804a"), + (b"abc\x84\x39\x84\x39\xc1\xc4", "replace", "abc\ufffd9\ufffd9\u804a"), ("\u30fb", "strict", b"\x819\xa79"), + (b"abc\x84\x32\x80\x80def", "replace", 'abc\ufffd2\ufffd\ufffddef'), + (b"abc\x81\x30\x81\x30def", "strict", 'abc\x80def'), + (b"abc\x86\x30\x81\x30def", "replace", 'abc\ufffd0\ufffd0def'), ) has_iso10646 = True @@ -74,9 +77,11 @@ class Test_HZ(test_multibytecodec_support.TestBase, unittest.TestCase): '\u5df1\u6240\u4e0d\u6b32\uff0c\u52ff\u65bd\u65bc\u4eba\u3002' 'Bye.\n'), # invalid bytes - (b'ab~cd', 'replace', 'ab\uFFFDd'), + (b'ab~cd', 'replace', 'ab\uFFFDcd'), (b'ab\xffcd', 'replace', 'ab\uFFFDcd'), (b'ab~{\x81\x81\x41\x44~}cd', 'replace', 'ab\uFFFD\uFFFD\u804Acd'), + (b'ab~{\x41\x44~}cd', 'replace', 'ab\u804Acd'), + (b"ab~{\x79\x79\x41\x44~}cd", "replace", "ab\ufffd\ufffd\u804acd"), ) def test_main(): diff --git a/Lib/test/test_codecencodings_hk.py b/Lib/test/test_codecencodings_hk.py index ccdc0b4..520df43 100644 --- a/Lib/test/test_codecencodings_hk.py +++ b/Lib/test/test_codecencodings_hk.py @@ -15,8 +15,8 @@ class Test_Big5HKSCS(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u8b10"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), ) diff --git a/Lib/test/test_codecencodings_jp.py b/Lib/test/test_codecencodings_jp.py index f56a373..87e4812 100644 --- a/Lib/test/test_codecencodings_jp.py +++ b/Lib/test/test_codecencodings_jp.py @@ -15,50 +15,57 @@ class Test_CP932(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x81\x00\x81\x00\x82\x84", "strict", None), (b"abc\xf8", "strict", None), - (b"abc\x81\x00\x82\x84", "replace", "abc\ufffd\uff44"), - (b"abc\x81\x00\x82\x84\x88", "replace", "abc\ufffd\uff44\ufffd"), - (b"abc\x81\x00\x82\x84", "ignore", "abc\uff44"), + (b"abc\x81\x00\x82\x84", "replace", "abc\ufffd\x00\uff44"), + (b"abc\x81\x00\x82\x84\x88", "replace", "abc\ufffd\x00\uff44\ufffd"), + (b"abc\x81\x00\x82\x84", "ignore", "abc\x00\uff44"), + (b"ab\xEBxy", "replace", "ab\uFFFDxy"), + (b"ab\xF0\x39xy", "replace", "ab\uFFFD9xy"), + (b"ab\xEA\xF0xy", "replace", 'ab\ufffd\ue038y'), # sjis vs cp932 (b"\\\x7e", "replace", "\\\x7e"), (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\uff3c\u2225\uff0d"), ) +euc_commontests = ( + # invalid bytes + (b"abc\x80\x80\xc1\xc4", "strict", None), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u7956"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u7956\ufffd"), + (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u7956"), + (b"abc\xc8", "strict", None), + (b"abc\x8f\x83\x83", "replace", "abc\ufffd\ufffd\ufffd"), + (b"\x82\xFCxy", "replace", "\ufffd\ufffdxy"), + (b"\xc1\x64", "strict", None), + (b"\xa1\xc0", "strict", "\uff3c"), + (b"\xa1\xc0\\", "strict", "\uff3c\\"), + (b"\x8eXY", "replace", "\ufffdXY"), +) + +class Test_EUC_JIS_2004(test_multibytecodec_support.TestBase, + unittest.TestCase): + encoding = 'euc_jis_2004' + tstring = test_multibytecodec_support.load_teststring('euc_jisx0213') + codectests = euc_commontests + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" + ) + class Test_EUC_JISX0213(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'euc_jisx0213' tstring = test_multibytecodec_support.load_teststring('euc_jisx0213') - codectests = ( - # invalid bytes - (b"abc\x80\x80\xc1\xc4", "strict", None), - (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u7956"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u7956\ufffd"), - (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u7956"), - (b"abc\x8f\x83\x83", "replace", "abc\ufffd"), - (b"\xc1\x64", "strict", None), - (b"\xa1\xc0", "strict", "\uff3c"), - ) + codectests = euc_commontests xmlcharnametest = ( "\xab\u211c\xbb = \u2329\u1234\u232a", b"\xa9\xa8ℜ\xa9\xb2 = ⟨ሴ⟩" ) -eucjp_commontests = ( - (b"abc\x80\x80\xc1\xc4", "strict", None), - (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u7956"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u7956\ufffd"), - (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u7956"), - (b"abc\x8f\x83\x83", "replace", "abc\ufffd"), - (b"\xc1\x64", "strict", None), -) - class Test_EUC_JP_COMPAT(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'euc_jp' tstring = test_multibytecodec_support.load_teststring('euc_jp') - codectests = eucjp_commontests + ( - (b"\xa1\xc0\\", "strict", "\uff3c\\"), + codectests = euc_commontests + ( ("\xa5", "strict", b"\x5c"), ("\u203e", "strict", b"\x7e"), ) @@ -66,8 +73,6 @@ class Test_EUC_JP_COMPAT(test_multibytecodec_support.TestBase, shiftjis_commonenctests = ( (b"abc\x80\x80\x82\x84", "strict", None), (b"abc\xf8", "strict", None), - (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\uff44"), - (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\uff44\ufffd"), (b"abc\x80\x80\x82\x84def", "ignore", "abc\uff44def"), ) @@ -75,20 +80,41 @@ class Test_SJIS_COMPAT(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'shift_jis' tstring = test_multibytecodec_support.load_teststring('shift_jis') codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + (b"\\\x7e", "strict", "\\\x7e"), (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\uff3c\u2016\u2212"), + (b"abc\x81\x39", "replace", "abc\ufffd9"), + (b"abc\xEA\xFC", "replace", "abc\ufffd\ufffd"), + (b"abc\xFF\x58", "replace", "abc\ufffdX"), + ) + +class Test_SJIS_2004(test_multibytecodec_support.TestBase, unittest.TestCase): + encoding = 'shift_jis_2004' + tstring = test_multibytecodec_support.load_teststring('shift_jis') + codectests = shiftjis_commonenctests + ( + (b"\\\x7e", "strict", "\xa5\u203e"), + (b"\x81\x5f\x81\x61\x81\x7c", "strict", "\\\u2016\u2212"), + (b"abc\xEA\xFC", "strict", "abc\u64bf"), + (b"\x81\x39xy", "replace", "\ufffd9xy"), + (b"\xFF\x58xy", "replace", "\ufffdXxy"), + (b"\x80\x80\x82\x84xy", "replace", "\ufffd\ufffd\uff44xy"), + (b"\x80\x80\x82\x84\x88xy", "replace", "\ufffd\ufffd\uff44\u5864y"), + (b"\xFC\xFBxy", "replace", '\ufffd\u95b4y'), + ) + xmlcharnametest = ( + "\xab\u211c\xbb = \u2329\u1234\u232a", + b"\x85Gℜ\x85Q = ⟨ሴ⟩" ) class Test_SJISX0213(test_multibytecodec_support.TestBase, unittest.TestCase): encoding = 'shift_jisx0213' tstring = test_multibytecodec_support.load_teststring('shift_jisx0213') - codectests = ( - # invalid bytes - (b"abc\x80\x80\x82\x84", "strict", None), - (b"abc\xf8", "strict", None), - (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\uff44"), - (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\uff44\ufffd"), - (b"abc\x80\x80\x82\x84def", "ignore", "abc\uff44def"), + codectests = shiftjis_commonenctests + ( + (b"abc\x80\x80\x82\x84", "replace", "abc\ufffd\ufffd\uff44"), + (b"abc\x80\x80\x82\x84\x88", "replace", "abc\ufffd\ufffd\uff44\ufffd"), + # sjis vs cp932 (b"\\\x7e", "replace", "\xa5\u203e"), (b"\x81\x5f\x81\x61\x81\x7c", "replace", "\x5c\u2016\u2212"), diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py index de4da7f..4997e83 100644 --- a/Lib/test/test_codecencodings_kr.py +++ b/Lib/test/test_codecencodings_kr.py @@ -15,8 +15,8 @@ class Test_CP949(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\uc894"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\uc894"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), ) @@ -27,8 +27,8 @@ class Test_EUCKR(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\uc894"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\uc894\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", 'abc\ufffd\ufffd\uc894'), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\uc894\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\uc894"), # composed make-up sequence errors @@ -40,13 +40,14 @@ class Test_EUCKR(test_multibytecodec_support.TestBase, unittest.TestCase): (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4", "strict", None), (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "strict", "\uc4d4"), (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4x", "strict", "\uc4d4x"), - (b"a\xa4\xd4\xa4\xb6\xa4", "replace", "a\ufffd"), + (b"a\xa4\xd4\xa4\xb6\xa4", "replace", 'a\ufffd'), (b"\xa4\xd4\xa3\xb6\xa4\xd0\xa4\xd4", "strict", None), (b"\xa4\xd4\xa4\xb6\xa3\xd0\xa4\xd4", "strict", None), (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa3\xd4", "strict", None), - (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", "\ufffd"), - (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", "\ufffd"), - (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", "\ufffd"), + (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", '\ufffd\u6e21\ufffd\u3160\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", '\ufffd\u6e21\ub544\ufffd\ufffd'), + (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", '\ufffd\u6e21\ub544\u572d\ufffd'), + (b"\xa4\xd4\xff\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "replace", '\ufffd\ufffd\ufffd\uc4d4'), (b"\xc1\xc4", "strict", "\uc894"), ) @@ -57,9 +58,13 @@ class Test_JOHAB(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ucd27"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ucd27\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\ucd27"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\ucd27\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\ucd27"), + (b"\xD8abc", "replace", "\uFFFDabc"), + (b"\xD8\xFFabc", "replace", "\uFFFD\uFFFDabc"), + (b"\x84bxy", "replace", "\uFFFDbxy"), + (b"\x8CBxy", "replace", "\uFFFDBxy"), ) def test_main(): diff --git a/Lib/test/test_codecencodings_tw.py b/Lib/test/test_codecencodings_tw.py index 12d3c9f..f2f3c18 100644 --- a/Lib/test/test_codecencodings_tw.py +++ b/Lib/test/test_codecencodings_tw.py @@ -15,8 +15,8 @@ class Test_Big5(test_multibytecodec_support.TestBase, unittest.TestCase): # invalid bytes (b"abc\x80\x80\xc1\xc4", "strict", None), (b"abc\xc8", "strict", None), - (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\u8b10"), - (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\u8b10\ufffd"), + (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\ufffd\u8b10"), + (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\ufffd\u8b10\ufffd"), (b"abc\x80\x80\xc1\xc4", "ignore", "abc\u8b10"), ) diff --git a/Lib/test/test_codecmaps_tw.py b/Lib/test/test_codecmaps_tw.py index 6db5091..412b9de 100644 --- a/Lib/test/test_codecmaps_tw.py +++ b/Lib/test/test_codecmaps_tw.py @@ -23,6 +23,9 @@ class TestCP950Map(test_multibytecodec_support.TestBase_Mapping, (b'\xa2\xcc', '\u5341'), (b'\xa2\xce', '\u5345'), ] + codectests = ( + (b"\xFFxy", "replace", "\ufffdxy"), + ) def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index 4899a59..e885a5a 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -1,8 +1,25 @@ -from test import support -import unittest +import _testcapi import codecs +import io import locale -import sys, _testcapi, io +import sys +import unittest +import warnings + +from test import support + +if sys.platform == 'win32': + VISTA_OR_LATER = (sys.getwindowsversion().major >= 6) +else: + VISTA_OR_LATER = False + +try: + import ctypes +except ImportError: + ctypes = None + SIZEOF_WCHAR_T = -1 +else: + SIZEOF_WCHAR_T = ctypes.sizeof(ctypes.c_wchar) class Queue(object): """ @@ -622,8 +639,113 @@ class UTF8Test(ReadTest): b"abc\xed\xa0\x80def") self.assertEqual(b"abc\xed\xa0\x80def".decode("utf-8", "surrogatepass"), "abc\ud800def") + self.assertEqual("\U00010fff\uD800".encode("utf-8", "surrogatepass"), + b"\xf0\x90\xbf\xbf\xed\xa0\x80") + self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("utf-8", "surrogatepass"), + "\U00010fff\uD800") self.assertTrue(codecs.lookup_error("surrogatepass")) +@unittest.skipUnless(sys.platform == 'win32', + 'cp65001 is a Windows-only codec') +class CP65001Test(ReadTest): + encoding = "cp65001" + + def test_encode(self): + tests = [ + ('abc', 'strict', b'abc'), + ('\xe9\u20ac', 'strict', b'\xc3\xa9\xe2\x82\xac'), + ('\U0010ffff', 'strict', b'\xf4\x8f\xbf\xbf'), + ] + if VISTA_OR_LATER: + tests.extend(( + ('\udc80', 'strict', None), + ('\udc80', 'ignore', b''), + ('\udc80', 'replace', b'?'), + ('\udc80', 'backslashreplace', b'\\udc80'), + ('\udc80', 'surrogatepass', b'\xed\xb2\x80'), + )) + else: + tests.append(('\udc80', 'strict', b'\xed\xb2\x80')) + for text, errors, expected in tests: + if expected is not None: + try: + encoded = text.encode('cp65001', errors) + except UnicodeEncodeError as err: + self.fail('Unable to encode %a to cp65001 with ' + 'errors=%r: %s' % (text, errors, err)) + self.assertEqual(encoded, expected, + '%a.encode("cp65001", %r)=%a != %a' + % (text, errors, encoded, expected)) + else: + self.assertRaises(UnicodeEncodeError, + text.encode, "cp65001", errors) + + def test_decode(self): + tests = [ + (b'abc', 'strict', 'abc'), + (b'\xc3\xa9\xe2\x82\xac', 'strict', '\xe9\u20ac'), + (b'\xf4\x8f\xbf\xbf', 'strict', '\U0010ffff'), + (b'\xef\xbf\xbd', 'strict', '\ufffd'), + (b'[\xc3\xa9]', 'strict', '[\xe9]'), + # invalid bytes + (b'[\xff]', 'strict', None), + (b'[\xff]', 'ignore', '[]'), + (b'[\xff]', 'replace', '[\ufffd]'), + (b'[\xff]', 'surrogateescape', '[\udcff]'), + ] + if VISTA_OR_LATER: + tests.extend(( + (b'[\xed\xb2\x80]', 'strict', None), + (b'[\xed\xb2\x80]', 'ignore', '[]'), + (b'[\xed\xb2\x80]', 'replace', '[\ufffd\ufffd\ufffd]'), + )) + else: + tests.extend(( + (b'[\xed\xb2\x80]', 'strict', '[\udc80]'), + )) + for raw, errors, expected in tests: + if expected is not None: + try: + decoded = raw.decode('cp65001', errors) + except UnicodeDecodeError as err: + self.fail('Unable to decode %a from cp65001 with ' + 'errors=%r: %s' % (raw, errors, err)) + self.assertEqual(decoded, expected, + '%a.decode("cp65001", %r)=%a != %a' + % (raw, errors, decoded, expected)) + else: + self.assertRaises(UnicodeDecodeError, + raw.decode, 'cp65001', errors) + + @unittest.skipUnless(VISTA_OR_LATER, 'require Windows Vista or later') + def test_lone_surrogates(self): + self.assertRaises(UnicodeEncodeError, "\ud800".encode, "cp65001") + self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "cp65001") + self.assertEqual("[\uDC80]".encode("cp65001", "backslashreplace"), + b'[\\udc80]') + self.assertEqual("[\uDC80]".encode("cp65001", "xmlcharrefreplace"), + b'[�]') + self.assertEqual("[\uDC80]".encode("cp65001", "surrogateescape"), + b'[\x80]') + self.assertEqual("[\uDC80]".encode("cp65001", "ignore"), + b'[]') + self.assertEqual("[\uDC80]".encode("cp65001", "replace"), + b'[?]') + + @unittest.skipUnless(VISTA_OR_LATER, 'require Windows Vista or later') + def test_surrogatepass_handler(self): + self.assertEqual("abc\ud800def".encode("cp65001", "surrogatepass"), + b"abc\xed\xa0\x80def") + self.assertEqual(b"abc\xed\xa0\x80def".decode("cp65001", "surrogatepass"), + "abc\ud800def") + self.assertEqual("\U00010fff\uD800".encode("cp65001", "surrogatepass"), + b"\xf0\x90\xbf\xbf\xed\xa0\x80") + self.assertEqual(b"\xf0\x90\xbf\xbf\xed\xa0\x80".decode("cp65001", "surrogatepass"), + "\U00010fff\uD800") + self.assertTrue(codecs.lookup_error("surrogatepass")) + + + class UTF7Test(ReadTest): encoding = "utf-7" @@ -884,61 +1006,70 @@ class PunycodeTest(unittest.TestCase): self.assertEqual(uni, puny.decode("punycode")) class UnicodeInternalTest(unittest.TestCase): + @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_bug1251300(self): # Decoding with unicode_internal used to not correctly handle "code # points" above 0x10ffff on UCS-4 builds. - if sys.maxunicode > 0xffff: - ok = [ - (b"\x00\x10\xff\xff", "\U0010ffff"), - (b"\x00\x00\x01\x01", "\U00000101"), - (b"", ""), - ] - not_ok = [ - b"\x7f\xff\xff\xff", - b"\x80\x00\x00\x00", - b"\x81\x00\x00\x00", - b"\x00", - b"\x00\x00\x00\x00\x00", - ] - for internal, uni in ok: - if sys.byteorder == "little": - internal = bytes(reversed(internal)) + ok = [ + (b"\x00\x10\xff\xff", "\U0010ffff"), + (b"\x00\x00\x01\x01", "\U00000101"), + (b"", ""), + ] + not_ok = [ + b"\x7f\xff\xff\xff", + b"\x80\x00\x00\x00", + b"\x81\x00\x00\x00", + b"\x00", + b"\x00\x00\x00\x00\x00", + ] + for internal, uni in ok: + if sys.byteorder == "little": + internal = bytes(reversed(internal)) + with support.check_warnings(): self.assertEqual(uni, internal.decode("unicode_internal")) - for internal in not_ok: - if sys.byteorder == "little": - internal = bytes(reversed(internal)) + for internal in not_ok: + if sys.byteorder == "little": + internal = bytes(reversed(internal)) + with support.check_warnings(('unicode_internal codec has been ' + 'deprecated', DeprecationWarning)): self.assertRaises(UnicodeDecodeError, internal.decode, - "unicode_internal") + "unicode_internal") + @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_decode_error_attributes(self): - if sys.maxunicode > 0xffff: - try: + try: + with support.check_warnings(('unicode_internal codec has been ' + 'deprecated', DeprecationWarning)): b"\x00\x00\x00\x00\x00\x11\x11\x00".decode("unicode_internal") - except UnicodeDecodeError as ex: - self.assertEqual("unicode_internal", ex.encoding) - self.assertEqual(b"\x00\x00\x00\x00\x00\x11\x11\x00", ex.object) - self.assertEqual(4, ex.start) - self.assertEqual(8, ex.end) - else: - self.fail() + except UnicodeDecodeError as ex: + self.assertEqual("unicode_internal", ex.encoding) + self.assertEqual(b"\x00\x00\x00\x00\x00\x11\x11\x00", ex.object) + self.assertEqual(4, ex.start) + self.assertEqual(8, ex.end) + else: + self.fail() + @unittest.skipUnless(SIZEOF_WCHAR_T == 4, 'specific to 32-bit wchar_t') def test_decode_callback(self): - if sys.maxunicode > 0xffff: - codecs.register_error("UnicodeInternalTest", codecs.ignore_errors) - decoder = codecs.getdecoder("unicode_internal") + codecs.register_error("UnicodeInternalTest", codecs.ignore_errors) + decoder = codecs.getdecoder("unicode_internal") + with support.check_warnings(('unicode_internal codec has been ' + 'deprecated', DeprecationWarning)): ab = "ab".encode("unicode_internal").decode() ignored = decoder(bytes("%s\x22\x22\x22\x22%s" % (ab[:4], ab[4:]), "ascii"), "UnicodeInternalTest") - self.assertEqual(("ab", 12), ignored) + self.assertEqual(("ab", 12), ignored) def test_encode_length(self): - # Issue 3739 - encoder = codecs.getencoder("unicode_internal") - self.assertEqual(encoder("a")[1], 1) - self.assertEqual(encoder("\xe9\u0142")[1], 2) + with support.check_warnings(('unicode_internal codec has been ' + 'deprecated', DeprecationWarning)): + # Issue 3739 + encoder = codecs.getencoder("unicode_internal") + self.assertEqual(encoder("a")[1], 1) + self.assertEqual(encoder("\xe9\u0142")[1], 2) - self.assertEqual(codecs.escape_encode(br'\x00')[1], 4) + self.assertEqual(codecs.escape_encode(br'\x00')[1], 4) # From http://www.gnu.org/software/libidn/draft-josefsson-idn-test-vectors.html nameprep_tests = [ @@ -1262,7 +1393,7 @@ class EncodedFileTest(unittest.TestCase): self.assertEqual(ef.read(), b'\\\xd5\n\x00\x00\xae') f = io.BytesIO() - ef = codecs.EncodedFile(f, 'utf-8', 'latin1') + ef = codecs.EncodedFile(f, 'utf-8', 'latin-1') ef.write(b'\xc3\xbc') self.assertEqual(f.getvalue(), b'\xfc') @@ -1394,10 +1525,13 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): elif encoding == "latin_1": name = "latin_1" self.assertEqual(encoding.replace("_", "-"), name.replace("_", "-")) - (b, size) = codecs.getencoder(encoding)(s) - self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding)) - (chars, size) = codecs.getdecoder(encoding)(b) - self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding)) + + with support.check_warnings(): + # unicode-internal has been deprecated + (b, size) = codecs.getencoder(encoding)(s) + self.assertEqual(size, len(s), "%r != %r (encoding=%r)" % (size, len(s), encoding)) + (chars, size) = codecs.getdecoder(encoding)(b) + self.assertEqual(chars, s, "%r != %r (encoding=%r)" % (chars, s, encoding)) if encoding not in broken_unicode_with_streams: # check stream reader/writer @@ -1501,7 +1635,9 @@ class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): def test_bad_encode_args(self): for encoding in all_unicode_encodings: encoder = codecs.getencoder(encoding) - self.assertRaises(TypeError, encoder) + with support.check_warnings(): + # unicode-internal has been deprecated + self.assertRaises(TypeError, encoder) def test_encoding_map_type_initialized(self): from encodings import cp1140 @@ -1623,7 +1759,7 @@ class SurrogateEscapeTest(unittest.TestCase): def test_latin1(self): # Issue6373 - self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin1", "surrogateescape"), + self.assertEqual("\udce4\udceb\udcef\udcf6\udcfc".encode("latin-1", "surrogateescape"), b"\xe4\xeb\xef\xf6\xfc") @@ -1732,6 +1868,155 @@ class TransformCodecTest(unittest.TestCase): self.assertEqual(sout, b"\x80") +@unittest.skipUnless(sys.platform == 'win32', + 'code pages are specific to Windows') +class CodePageTest(unittest.TestCase): + # CP_UTF8 is already tested by CP65001Test + CP_UTF8 = 65001 + + def test_invalid_code_page(self): + self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a') + self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a') + self.assertRaises(WindowsError, codecs.code_page_encode, 123, 'a') + self.assertRaises(WindowsError, codecs.code_page_decode, 123, b'a') + + def test_code_page_name(self): + self.assertRaisesRegex(UnicodeEncodeError, 'cp932', + codecs.code_page_encode, 932, '\xff') + self.assertRaisesRegex(UnicodeDecodeError, 'cp932', + codecs.code_page_decode, 932, b'\x81\x00') + self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8', + codecs.code_page_decode, self.CP_UTF8, b'\xff') + + def check_decode(self, cp, tests): + for raw, errors, expected in tests: + if expected is not None: + try: + decoded = codecs.code_page_decode(cp, raw, errors) + except UnicodeDecodeError as err: + self.fail('Unable to decode %a from "cp%s" with ' + 'errors=%r: %s' % (raw, cp, errors, err)) + self.assertEqual(decoded[0], expected, + '%a.decode("cp%s", %r)=%a != %a' + % (raw, cp, errors, decoded[0], expected)) + # assert 0 <= decoded[1] <= len(raw) + self.assertGreaterEqual(decoded[1], 0) + self.assertLessEqual(decoded[1], len(raw)) + else: + self.assertRaises(UnicodeDecodeError, + codecs.code_page_decode, cp, raw, errors) + + def check_encode(self, cp, tests): + for text, errors, expected in tests: + if expected is not None: + try: + encoded = codecs.code_page_encode(cp, text, errors) + except UnicodeEncodeError as err: + self.fail('Unable to encode %a to "cp%s" with ' + 'errors=%r: %s' % (text, cp, errors, err)) + self.assertEqual(encoded[0], expected, + '%a.encode("cp%s", %r)=%a != %a' + % (text, cp, errors, encoded[0], expected)) + self.assertEqual(encoded[1], len(text)) + else: + self.assertRaises(UnicodeEncodeError, + codecs.code_page_encode, cp, text, errors) + + def test_cp932(self): + self.check_encode(932, ( + ('abc', 'strict', b'abc'), + ('\uff44\u9a3e', 'strict', b'\x82\x84\xe9\x80'), + # test error handlers + ('\xff', 'strict', None), + ('[\xff]', 'ignore', b'[]'), + ('[\xff]', 'replace', b'[y]'), + ('[\u20ac]', 'replace', b'[?]'), + ('[\xff]', 'backslashreplace', b'[\\xff]'), + ('[\xff]', 'xmlcharrefreplace', b'[ÿ]'), + )) + self.check_decode(932, ( + (b'abc', 'strict', 'abc'), + (b'\x82\x84\xe9\x80', 'strict', '\uff44\u9a3e'), + # invalid bytes + (b'[\xff]', 'strict', None), + (b'[\xff]', 'ignore', '[]'), + (b'[\xff]', 'replace', '[\ufffd]'), + (b'[\xff]', 'surrogateescape', '[\udcff]'), + (b'\x81\x00abc', 'strict', None), + (b'\x81\x00abc', 'ignore', '\x00abc'), + (b'\x81\x00abc', 'replace', '\ufffd\x00abc'), + )) + + def test_cp1252(self): + self.check_encode(1252, ( + ('abc', 'strict', b'abc'), + ('\xe9\u20ac', 'strict', b'\xe9\x80'), + ('\xff', 'strict', b'\xff'), + ('\u0141', 'strict', None), + ('\u0141', 'ignore', b''), + ('\u0141', 'replace', b'L'), + )) + self.check_decode(1252, ( + (b'abc', 'strict', 'abc'), + (b'\xe9\x80', 'strict', '\xe9\u20ac'), + (b'\xff', 'strict', '\xff'), + )) + + def test_cp_utf7(self): + cp = 65000 + self.check_encode(cp, ( + ('abc', 'strict', b'abc'), + ('\xe9\u20ac', 'strict', b'+AOkgrA-'), + ('\U0010ffff', 'strict', b'+2//f/w-'), + ('\udc80', 'strict', b'+3IA-'), + ('\ufffd', 'strict', b'+//0-'), + )) + self.check_decode(cp, ( + (b'abc', 'strict', 'abc'), + (b'+AOkgrA-', 'strict', '\xe9\u20ac'), + (b'+2//f/w-', 'strict', '\U0010ffff'), + (b'+3IA-', 'strict', '\udc80'), + (b'+//0-', 'strict', '\ufffd'), + # invalid bytes + (b'[+/]', 'strict', '[]'), + (b'[\xff]', 'strict', '[\xff]'), + )) + + def test_multibyte_encoding(self): + self.check_decode(932, ( + (b'\x84\xe9\x80', 'ignore', '\u9a3e'), + (b'\x84\xe9\x80', 'replace', '\ufffd\u9a3e'), + )) + self.check_decode(self.CP_UTF8, ( + (b'\xff\xf4\x8f\xbf\xbf', 'ignore', '\U0010ffff'), + (b'\xff\xf4\x8f\xbf\xbf', 'replace', '\ufffd\U0010ffff'), + )) + if VISTA_OR_LATER: + self.check_encode(self.CP_UTF8, ( + ('[\U0010ffff\uDC80]', 'ignore', b'[\xf4\x8f\xbf\xbf]'), + ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'), + )) + + def test_incremental(self): + decoded = codecs.code_page_decode(932, b'\x82', 'strict', False) + self.assertEqual(decoded, ('', 0)) + + decoded = codecs.code_page_decode(932, + b'\xe9\x80\xe9', 'strict', + False) + self.assertEqual(decoded, ('\u9a3e', 2)) + + decoded = codecs.code_page_decode(932, + b'\xe9\x80\xe9\x80', 'strict', + False) + self.assertEqual(decoded, ('\u9a3e\u9a3e', 4)) + + decoded = codecs.code_page_decode(932, + b'abc', 'strict', + False) + self.assertEqual(decoded, ('abc', 3)) + + def test_main(): support.run_unittest( UTF32Test, @@ -1742,6 +2027,7 @@ def test_main(): UTF16BETest, UTF8Test, UTF8SigTest, + CP65001Test, UTF7Test, UTF16ExTest, ReadBufferTest, @@ -1760,6 +2046,7 @@ def test_main(): SurrogateEscapeTest, BomTest, TransformCodecTest, + CodePageTest, ) diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 8dc5559..88c3129 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1,6 +1,7 @@ """Unit tests for collections.py.""" import unittest, doctest, operator +from test.support import TESTFN, forget, unlink import inspect from test import support from collections import namedtuple, Counter, OrderedDict, _count_elements @@ -10,21 +11,20 @@ from random import randrange, shuffle import keyword import re import sys -from collections import _ChainMap -from collections import Hashable, Iterable, Iterator -from collections import Sized, Container, Callable -from collections import Set, MutableSet -from collections import Mapping, MutableMapping, KeysView, ItemsView, UserDict -from collections import Sequence, MutableSequence -from collections import ByteString +from collections import UserDict +from collections import ChainMap +from collections.abc import Hashable, Iterable, Iterator +from collections.abc import Sized, Container, Callable +from collections.abc import Set, MutableSet +from collections.abc import Mapping, MutableMapping, KeysView, ItemsView +from collections.abc import Sequence, MutableSequence +from collections.abc import ByteString ################################################################################ -### _ChainMap (helper class for configparser) +### ChainMap (helper class for configparser and the string module) ################################################################################ -ChainMap = _ChainMap # rename to keep test code in sync with 3.3 version - class TestChainMap(unittest.TestCase): def test_basics(self): @@ -128,6 +128,7 @@ class TestNamedTuple(unittest.TestCase): self.assertEqual(Point.__module__, __name__) self.assertEqual(Point.__getitem__, tuple.__getitem__) self.assertEqual(Point._fields, ('x', 'y')) + self.assertIn('class Point(tuple)', Point._source) self.assertRaises(ValueError, namedtuple, 'abc%', 'efg ghi') # type has non-alpha char self.assertRaises(ValueError, namedtuple, 'class', 'efg ghi') # type has keyword @@ -327,6 +328,17 @@ class TestNamedTuple(unittest.TestCase): pass self.assertEqual(repr(B(1)), 'B(x=1)') + def test_source(self): + # verify that _source can be run through exec() + tmp = namedtuple('NTColor', 'red green blue') + globals().pop('NTColor', None) # remove artifacts from other tests + exec(tmp._source, globals()) + self.assertIn('NTColor', globals()) + c = NTColor(10, 20, 30) + self.assertEqual((c.red, c.green, c.blue), (10, 20, 30)) + self.assertEqual(NTColor._fields, ('red', 'green', 'blue')) + globals().pop('NTColor', None) # clean-up after this test + ################################################################################ ### Abstract Base Classes @@ -729,6 +741,44 @@ class TestCollectionABCs(ABCTestCase): self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') + def test_MutableSequence_mixins(self): + # Test the mixins of MutableSequence by creating a miminal concrete + # class inherited from it. + class MutableSequenceSubclass(MutableSequence): + def __init__(self): + self.lst = [] + + def __setitem__(self, index, value): + self.lst[index] = value + + def __getitem__(self, index): + return self.lst[index] + + def __len__(self): + return len(self.lst) + + def __delitem__(self, index): + del self.lst[index] + + def insert(self, index, value): + self.lst.insert(index, value) + + mss = MutableSequenceSubclass() + mss.append(0) + mss.extend((1, 2, 3, 4)) + self.assertEqual(len(mss), 5) + self.assertEqual(mss[3], 3) + mss.reverse() + self.assertEqual(mss[3], 1) + mss.pop() + self.assertEqual(len(mss), 4) + mss.remove(3) + self.assertEqual(len(mss), 3) + mss += (10, 20, 30) + self.assertEqual(len(mss), 6) + self.assertEqual(mss[-1], 30) + mss.clear() + self.assertEqual(len(mss), 0) ################################################################################ ### Counter @@ -882,6 +932,27 @@ class TestCounter(unittest.TestCase): set_result = setop(set(p.elements()), set(q.elements())) self.assertEqual(counter_result, dict.fromkeys(set_result, 1)) + def test_inplace_operations(self): + elements = 'abcd' + for i in range(1000): + # test random pairs of multisets + p = Counter(dict((elem, randrange(-2,4)) for elem in elements)) + p.update(e=1, f=-1, g=0) + q = Counter(dict((elem, randrange(-2,4)) for elem in elements)) + q.update(h=1, i=-1, j=0) + for inplace_op, regular_op in [ + (Counter.__iadd__, Counter.__add__), + (Counter.__isub__, Counter.__sub__), + (Counter.__ior__, Counter.__or__), + (Counter.__iand__, Counter.__and__), + ]: + c = p.copy() + c_id = id(c) + regular_result = regular_op(c, q) + inplace_result = inplace_op(c, q) + self.assertEqual(inplace_result, regular_result) + self.assertEqual(id(inplace_result), c_id) + def test_subtract(self): c = Counter(a=-5, b=0, c=5, d=10, e=15,g=40) c.subtract(a=1, b=2, c=-3, d=10, e=20, f=30, h=-50) @@ -893,6 +964,11 @@ class TestCounter(unittest.TestCase): c.subtract('aaaabbcce') self.assertEqual(c, Counter(a=-1, b=0, c=-1, d=1, e=-1)) + def test_unary(self): + c = Counter(a=-5, b=0, c=5, d=10, e=15,g=40) + self.assertEqual(dict(+c), dict(c=5, d=10, e=15, g=40)) + self.assertEqual(dict(-c), dict(a=5)) + def test_repr_nonsortable(self): c = Counter(a=2, b=None) r = repr(c) diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 58ef297..c5f9189 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -1,6 +1,7 @@ import unittest import sys import _ast +import types from test import support class TestSpecifics(unittest.TestCase): @@ -433,6 +434,14 @@ if 1: ast.body = [_ast.BoolOp()] self.assertRaises(TypeError, compile, ast, '<ast>', 'exec') + @support.cpython_only + def test_same_filename_used(self): + s = """def f(): pass\ndef g(): pass""" + c = compile(s, "myfile", "exec") + for obj in c.co_consts: + if isinstance(obj, types.CodeType): + self.assertIs(obj.co_filename, c.co_filename) + def test_main(): support.run_unittest(TestSpecifics) diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 6cc57f8..7522a54 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -19,7 +19,7 @@ import unittest from concurrent import futures from concurrent.futures._base import ( PENDING, RUNNING, CANCELLED, CANCELLED_AND_NOTIFIED, FINISHED, Future) -import concurrent.futures.process +from concurrent.futures.process import BrokenProcessPool def create_future(state=PENDING, exception=None, result=None): @@ -34,7 +34,7 @@ PENDING_FUTURE = create_future(state=PENDING) RUNNING_FUTURE = create_future(state=RUNNING) CANCELLED_FUTURE = create_future(state=CANCELLED) CANCELLED_AND_NOTIFIED_FUTURE = create_future(state=CANCELLED_AND_NOTIFIED) -EXCEPTION_FUTURE = create_future(state=FINISHED, exception=IOError()) +EXCEPTION_FUTURE = create_future(state=FINISHED, exception=OSError()) SUCCESSFUL_FUTURE = create_future(state=FINISHED, result=42) @@ -154,7 +154,7 @@ class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest): processes = self.executor._processes self.executor.shutdown() - for p in processes: + for p in processes.values(): p.join() def test_context_manager_shutdown(self): @@ -163,7 +163,7 @@ class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest): self.assertEqual(list(e.map(abs, range(-5, 5))), [5, 4, 3, 2, 1, 0, 1, 2, 3, 4]) - for p in processes: + for p in processes.values(): p.join() def test_del_shutdown(self): @@ -174,7 +174,7 @@ class ProcessPoolShutdownTest(ProcessPoolMixin, ExecutorShutdownTest): del executor queue_management_thread.join() - for p in processes: + for p in processes.values(): p.join() class WaitTests(unittest.TestCase): @@ -260,14 +260,14 @@ class WaitTests(unittest.TestCase): def test_timeout(self): future1 = self.executor.submit(mul, 6, 7) - future2 = self.executor.submit(time.sleep, 3) + future2 = self.executor.submit(time.sleep, 6) finished, pending = futures.wait( [CANCELLED_AND_NOTIFIED_FUTURE, EXCEPTION_FUTURE, SUCCESSFUL_FUTURE, future1, future2], - timeout=1.5, + timeout=5, return_when=futures.ALL_COMPLETED) self.assertEqual(set([CANCELLED_AND_NOTIFIED_FUTURE, @@ -357,8 +357,8 @@ class ExecutorTest(unittest.TestCase): results = [] try: for i in self.executor.map(time.sleep, - [0, 0, 3], - timeout=1.5): + [0, 0, 6], + timeout=5): results.append(i) except futures.TimeoutError: pass @@ -367,13 +367,38 @@ class ExecutorTest(unittest.TestCase): self.assertEqual([None, None], results) + def test_shutdown_race_issue12456(self): + # Issue #12456: race condition at shutdown where trying to post a + # sentinel in the call queue blocks (the queue is full while processes + # have exited). + self.executor.map(str, [2] * (self.worker_count + 1)) + self.executor.shutdown() + class ThreadPoolExecutorTest(ThreadPoolMixin, ExecutorTest): - pass + def test_map_submits_without_iteration(self): + """Tests verifying issue 11777.""" + finished = [] + def record_finished(n): + finished.append(n) + + self.executor.map(record_finished, range(10)) + self.executor.shutdown(wait=True) + self.assertCountEqual(finished, range(10)) class ProcessPoolExecutorTest(ProcessPoolMixin, ExecutorTest): - pass + def test_killed_child(self): + # When a child process is abruptly terminated, the whole pool gets + # "broken". + futures = [self.executor.submit(time.sleep, 3)] + # Get one of the processes, and terminate (kill) it + p = next(iter(self.executor._processes.values())) + p.terminate() + for fut in futures: + self.assertRaises(BrokenProcessPool, fut.result) + # Submitting other jobs fails as well. + self.assertRaises(BrokenProcessPool, self.executor.submit, pow, 2, 8) class FutureTests(unittest.TestCase): @@ -476,7 +501,7 @@ class FutureTests(unittest.TestCase): '<Future at 0x[0-9a-f]+ state=cancelled>') self.assertRegex( repr(EXCEPTION_FUTURE), - '<Future at 0x[0-9a-f]+ state=finished raised IOError>') + '<Future at 0x[0-9a-f]+ state=finished raised OSError>') self.assertRegex( repr(SUCCESSFUL_FUTURE), '<Future at 0x[0-9a-f]+ state=finished returned int>') @@ -487,7 +512,7 @@ class FutureTests(unittest.TestCase): f2 = create_future(state=RUNNING) f3 = create_future(state=CANCELLED) f4 = create_future(state=CANCELLED_AND_NOTIFIED) - f5 = create_future(state=FINISHED, exception=IOError()) + f5 = create_future(state=FINISHED, exception=OSError()) f6 = create_future(state=FINISHED, result=5) self.assertTrue(f1.cancel()) @@ -541,7 +566,7 @@ class FutureTests(unittest.TestCase): CANCELLED_FUTURE.result, timeout=0) self.assertRaises(futures.CancelledError, CANCELLED_AND_NOTIFIED_FUTURE.result, timeout=0) - self.assertRaises(IOError, EXCEPTION_FUTURE.result, timeout=0) + self.assertRaises(OSError, EXCEPTION_FUTURE.result, timeout=0) self.assertEqual(SUCCESSFUL_FUTURE.result(timeout=0), 42) def test_result_with_success(self): @@ -580,7 +605,7 @@ class FutureTests(unittest.TestCase): self.assertRaises(futures.CancelledError, CANCELLED_AND_NOTIFIED_FUTURE.exception, timeout=0) self.assertTrue(isinstance(EXCEPTION_FUTURE.exception(timeout=0), - IOError)) + OSError)) self.assertEqual(SUCCESSFUL_FUTURE.exception(timeout=0), None) def test_exception_with_success(self): @@ -589,14 +614,14 @@ class FutureTests(unittest.TestCase): time.sleep(1) with f1._condition: f1._state = FINISHED - f1._exception = IOError() + f1._exception = OSError() f1._condition.notify_all() f1 = create_future(state=PENDING) t = threading.Thread(target=notification) t.start() - self.assertTrue(isinstance(f1.exception(timeout=5), IOError)) + self.assertTrue(isinstance(f1.exception(timeout=5), OSError)) @test.support.reap_threads def test_main(): @@ -609,7 +634,8 @@ def test_main(): ThreadPoolAsCompletedTests, FutureTests, ProcessPoolShutdownTest, - ThreadPoolShutdownTest) + ThreadPoolShutdownTest, + ) finally: test.support.reap_children() diff --git a/Lib/test/test_cfgparser.py b/Lib/test/test_configparser.py index 1db9217..1db9217 100644 --- a/Lib/test/test_cfgparser.py +++ b/Lib/test/test_configparser.py diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index a84c109..c4baae4 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -17,7 +17,7 @@ class TestCopy(unittest.TestCase): # Attempt full line coverage of copy.py from top to bottom def test_exceptions(self): - self.assertTrue(copy.Error is copy.error) + self.assertIs(copy.Error, copy.error) self.assertTrue(issubclass(copy.Error, Exception)) # The copy() method @@ -54,20 +54,26 @@ class TestCopy(unittest.TestCase): def test_copy_reduce_ex(self): class C(object): def __reduce_ex__(self, proto): + c.append(1) return "" def __reduce__(self): - raise support.TestFailed("shouldn't call this") + self.fail("shouldn't call this") + c = [] x = C() y = copy.copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) + self.assertEqual(c, [1]) def test_copy_reduce(self): class C(object): def __reduce__(self): + c.append(1) return "" + c = [] x = C() y = copy.copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) + self.assertEqual(c, [1]) def test_copy_cant(self): class C(object): @@ -91,7 +97,7 @@ class TestCopy(unittest.TestCase): "hello", "hello\u1234", f.__code__, NewStyle, range(10), Classic, max] for x in tests: - self.assertTrue(copy.copy(x) is x, repr(x)) + self.assertIs(copy.copy(x), x) def test_copy_list(self): x = [1, 2, 3] @@ -185,9 +191,9 @@ class TestCopy(unittest.TestCase): x = [x, x] y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y[0] is not x[0]) - self.assertTrue(y[0] is y[1]) + self.assertIsNot(y, x) + self.assertIsNot(y[0], x[0]) + self.assertIs(y[0], y[1]) def test_deepcopy_issubclass(self): # XXX Note: there's no way to test the TypeError coming out of @@ -227,20 +233,26 @@ class TestCopy(unittest.TestCase): def test_deepcopy_reduce_ex(self): class C(object): def __reduce_ex__(self, proto): + c.append(1) return "" def __reduce__(self): - raise support.TestFailed("shouldn't call this") + self.fail("shouldn't call this") + c = [] x = C() y = copy.deepcopy(x) - self.assertTrue(y is x) + self.assertIs(y, x) + self.assertEqual(c, [1]) def test_deepcopy_reduce(self): class C(object): def __reduce__(self): + c.append(1) return "" + c = [] x = C() y = copy.deepcopy(x) - self.assertTrue(y is x) + self.assertIs(y, x) + self.assertEqual(c, [1]) def test_deepcopy_cant(self): class C(object): @@ -264,14 +276,14 @@ class TestCopy(unittest.TestCase): "hello", "hello\u1234", f.__code__, NewStyle, range(10), Classic, max] for x in tests: - self.assertTrue(copy.deepcopy(x) is x, repr(x)) + self.assertIs(copy.deepcopy(x), x) def test_deepcopy_list(self): x = [[1, 2], 3] y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(x is not y) - self.assertTrue(x[0] is not y[0]) + self.assertIsNot(x, y) + self.assertIsNot(x[0], y[0]) def test_deepcopy_reflexive_list(self): x = [] @@ -279,16 +291,26 @@ class TestCopy(unittest.TestCase): y = copy.deepcopy(x) for op in comparisons: self.assertRaises(RuntimeError, op, y, x) - self.assertTrue(y is not x) - self.assertTrue(y[0] is y) + self.assertIsNot(y, x) + self.assertIs(y[0], y) self.assertEqual(len(y), 1) + def test_deepcopy_empty_tuple(self): + x = () + y = copy.deepcopy(x) + self.assertIs(x, y) + def test_deepcopy_tuple(self): x = ([1, 2], 3) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(x is not y) - self.assertTrue(x[0] is not y[0]) + self.assertIsNot(x, y) + self.assertIsNot(x[0], y[0]) + + def test_deepcopy_tuple_of_immutables(self): + x = ((1, 2), 3) + y = copy.deepcopy(x) + self.assertIs(x, y) def test_deepcopy_reflexive_tuple(self): x = ([],) @@ -296,16 +318,16 @@ class TestCopy(unittest.TestCase): y = copy.deepcopy(x) for op in comparisons: self.assertRaises(RuntimeError, op, y, x) - self.assertTrue(y is not x) - self.assertTrue(y[0] is not x[0]) - self.assertTrue(y[0][0] is y) + self.assertIsNot(y, x) + self.assertIsNot(y[0], x[0]) + self.assertIs(y[0][0], y) def test_deepcopy_dict(self): x = {"foo": [1, 2], "bar": 3} y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(x is not y) - self.assertTrue(x["foo"] is not y["foo"]) + self.assertIsNot(x, y) + self.assertIsNot(x["foo"], y["foo"]) def test_deepcopy_reflexive_dict(self): x = {} @@ -315,15 +337,30 @@ class TestCopy(unittest.TestCase): self.assertRaises(TypeError, op, y, x) for op in equality_comparisons: self.assertRaises(RuntimeError, op, y, x) - self.assertTrue(y is not x) - self.assertTrue(y['foo'] is y) + self.assertIsNot(y, x) + self.assertIs(y['foo'], y) self.assertEqual(len(y), 1) def test_deepcopy_keepalive(self): memo = {} - x = 42 + x = [] + y = copy.deepcopy(x, memo) + self.assertIs(memo[id(memo)][0], x) + + def test_deepcopy_dont_memo_immutable(self): + memo = {} + x = [1, 2, 3, 4] y = copy.deepcopy(x, memo) - self.assertTrue(memo[id(x)] is x) + self.assertEqual(y, x) + # There's the entry for the new list, and the keep alive. + self.assertEqual(len(memo), 2) + + memo = {} + x = [(1, 2)] + y = copy.deepcopy(x, memo) + self.assertEqual(y, x) + # Tuples with immutable contents are immutable for deepcopy. + self.assertEqual(len(memo), 2) def test_deepcopy_inst_vanilla(self): class C: @@ -334,7 +371,7 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_deepcopy(self): class C: @@ -347,8 +384,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_getinitargs(self): class C: @@ -361,8 +398,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_getstate(self): class C: @@ -375,8 +412,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_setstate(self): class C: @@ -389,8 +426,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_inst_getstate_setstate(self): class C: @@ -405,8 +442,8 @@ class TestCopy(unittest.TestCase): x = C([42]) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y, x) + self.assertIsNot(y.foo, x.foo) def test_deepcopy_reflexive_inst(self): class C: @@ -414,8 +451,8 @@ class TestCopy(unittest.TestCase): x = C() x.foo = x y = copy.deepcopy(x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is y) + self.assertIsNot(y, x) + self.assertIs(y.foo, y) # _reconstruct() @@ -425,9 +462,9 @@ class TestCopy(unittest.TestCase): return "" x = C() y = copy.copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) y = copy.deepcopy(x) - self.assertTrue(y is x) + self.assertIs(y, x) def test_reconstruct_nostate(self): class C(object): @@ -436,9 +473,9 @@ class TestCopy(unittest.TestCase): x = C() x.foo = 42 y = copy.copy(x) - self.assertTrue(y.__class__ is x.__class__) + self.assertIs(y.__class__, x.__class__) y = copy.deepcopy(x) - self.assertTrue(y.__class__ is x.__class__) + self.assertIs(y.__class__, x.__class__) def test_reconstruct_state(self): class C(object): @@ -452,7 +489,7 @@ class TestCopy(unittest.TestCase): self.assertEqual(y, x) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y.foo, x.foo) def test_reconstruct_state_setstate(self): class C(object): @@ -468,7 +505,7 @@ class TestCopy(unittest.TestCase): self.assertEqual(y, x) y = copy.deepcopy(x) self.assertEqual(y, x) - self.assertTrue(y.foo is not x.foo) + self.assertIsNot(y.foo, x.foo) def test_reconstruct_reflexive(self): class C(object): @@ -476,8 +513,8 @@ class TestCopy(unittest.TestCase): x = C() x.foo = x y = copy.deepcopy(x) - self.assertTrue(y is not x) - self.assertTrue(y.foo is y) + self.assertIsNot(y, x) + self.assertIs(y.foo, y) # Additions for Python 2.3 and pickle protocol 2 @@ -491,12 +528,12 @@ class TestCopy(unittest.TestCase): x = C([[1, 2], 3]) y = copy.copy(x) self.assertEqual(x, y) - self.assertTrue(x is not y) - self.assertTrue(x[0] is y[0]) + self.assertIsNot(x, y) + self.assertIs(x[0], y[0]) y = copy.deepcopy(x) self.assertEqual(x, y) - self.assertTrue(x is not y) - self.assertTrue(x[0] is not y[0]) + self.assertIsNot(x, y) + self.assertIsNot(x[0], y[0]) def test_reduce_5tuple(self): class C(dict): @@ -508,12 +545,12 @@ class TestCopy(unittest.TestCase): x = C([("foo", [1, 2]), ("bar", 3)]) y = copy.copy(x) self.assertEqual(x, y) - self.assertTrue(x is not y) - self.assertTrue(x["foo"] is y["foo"]) + self.assertIsNot(x, y) + self.assertIs(x["foo"], y["foo"]) y = copy.deepcopy(x) self.assertEqual(x, y) - self.assertTrue(x is not y) - self.assertTrue(x["foo"] is not y["foo"]) + self.assertIsNot(x, y) + self.assertIsNot(x["foo"], y["foo"]) def test_copy_slots(self): class C(object): @@ -521,7 +558,7 @@ class TestCopy(unittest.TestCase): x = C() x.foo = [42] y = copy.copy(x) - self.assertTrue(x.foo is y.foo) + self.assertIs(x.foo, y.foo) def test_deepcopy_slots(self): class C(object): @@ -530,7 +567,7 @@ class TestCopy(unittest.TestCase): x.foo = [42] y = copy.deepcopy(x) self.assertEqual(x.foo, y.foo) - self.assertTrue(x.foo is not y.foo) + self.assertIsNot(x.foo, y.foo) def test_deepcopy_dict_subclass(self): class C(dict): @@ -547,7 +584,7 @@ class TestCopy(unittest.TestCase): y = copy.deepcopy(x) self.assertEqual(x, y) self.assertEqual(x._keys, y._keys) - self.assertTrue(x is not y) + self.assertIsNot(x, y) x['bar'] = 1 self.assertNotEqual(x, y) self.assertNotEqual(x._keys, y._keys) @@ -560,8 +597,8 @@ class TestCopy(unittest.TestCase): y = copy.copy(x) self.assertEqual(list(x), list(y)) self.assertEqual(x.foo, y.foo) - self.assertTrue(x[0] is y[0]) - self.assertTrue(x.foo is y.foo) + self.assertIs(x[0], y[0]) + self.assertIs(x.foo, y.foo) def test_deepcopy_list_subclass(self): class C(list): @@ -571,8 +608,8 @@ class TestCopy(unittest.TestCase): y = copy.deepcopy(x) self.assertEqual(list(x), list(y)) self.assertEqual(x.foo, y.foo) - self.assertTrue(x[0] is not y[0]) - self.assertTrue(x.foo is not y.foo) + self.assertIsNot(x[0], y[0]) + self.assertIsNot(x.foo, y.foo) def test_copy_tuple_subclass(self): class C(tuple): @@ -589,8 +626,8 @@ class TestCopy(unittest.TestCase): self.assertEqual(tuple(x), ([1, 2], 3)) y = copy.deepcopy(x) self.assertEqual(tuple(y), ([1, 2], 3)) - self.assertTrue(x is not y) - self.assertTrue(x[0] is not y[0]) + self.assertIsNot(x, y) + self.assertIsNot(x[0], y[0]) def test_getstate_exc(self): class EvilState(object): @@ -618,10 +655,10 @@ class TestCopy(unittest.TestCase): obj = C() x = weakref.ref(obj) y = _copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) del obj y = _copy(x) - self.assertTrue(y is x) + self.assertIs(y, x) def test_copy_weakref(self): self._check_weakref(copy.copy) @@ -637,7 +674,7 @@ class TestCopy(unittest.TestCase): u[a] = b u[c] = d v = copy.copy(u) - self.assertFalse(v is u) + self.assertIsNot(v, u) self.assertEqual(v, u) self.assertEqual(v[a], b) self.assertEqual(v[c], d) @@ -667,8 +704,8 @@ class TestCopy(unittest.TestCase): v = copy.deepcopy(u) self.assertNotEqual(v, u) self.assertEqual(len(v), 2) - self.assertFalse(v[a] is b) - self.assertFalse(v[c] is d) + self.assertIsNot(v[a], b) + self.assertIsNot(v[c], d) self.assertEqual(v[a].i, b.i) self.assertEqual(v[c].i, d.i) del c @@ -687,12 +724,12 @@ class TestCopy(unittest.TestCase): self.assertNotEqual(v, u) self.assertEqual(len(v), 2) (x, y), (z, t) = sorted(v.items(), key=lambda pair: pair[0].i) - self.assertFalse(x is a) + self.assertIsNot(x, a) self.assertEqual(x.i, a.i) - self.assertTrue(y is b) - self.assertFalse(z is c) + self.assertIs(y, b) + self.assertIsNot(z, c) self.assertEqual(z.i, c.i) - self.assertTrue(t is d) + self.assertIs(t, d) del x, y, z, t del d self.assertEqual(len(v), 1) @@ -705,7 +742,7 @@ class TestCopy(unittest.TestCase): f.b = f.m g = copy.deepcopy(f) self.assertEqual(g.m, g.b) - self.assertTrue(g.b.__self__ is g) + self.assertIs(g.b.__self__, g) g.b() diff --git a/Lib/test/test_cprofile.py b/Lib/test/test_cprofile.py index ae17c2b..5676668 100644 --- a/Lib/test/test_cprofile.py +++ b/Lib/test/test_cprofile.py @@ -18,16 +18,19 @@ class CProfileTest(ProfileTest): def test_bad_counter_during_dealloc(self): import _lsprof # Must use a file as StringIO doesn't trigger the bug. - with open(TESTFN, 'w') as file: - sys.stderr = file - try: - obj = _lsprof.Profiler(lambda: int) - obj.enable() - obj = _lsprof.Profiler(1) - obj.disable() - finally: - sys.stderr = sys.__stderr__ - unlink(TESTFN) + orig_stderr = sys.stderr + try: + with open(TESTFN, 'w') as file: + sys.stderr = file + try: + obj = _lsprof.Profiler(lambda: int) + obj.enable() + obj = _lsprof.Profiler(1) + obj.disable() + finally: + sys.stderr = orig_stderr + finally: + unlink(TESTFN) def test_main(): diff --git a/Lib/test/test_crashers.py b/Lib/test/test_crashers.py new file mode 100644 index 0000000..336ccbe --- /dev/null +++ b/Lib/test/test_crashers.py @@ -0,0 +1,38 @@ +# Tests that the crashers in the Lib/test/crashers directory actually +# do crash the interpreter as expected +# +# If a crasher is fixed, it should be moved elsewhere in the test suite to +# ensure it continues to work correctly. + +import unittest +import glob +import os.path +import test.support +from test.script_helper import assert_python_failure + +CRASHER_DIR = os.path.join(os.path.dirname(__file__), "crashers") +CRASHER_FILES = os.path.join(CRASHER_DIR, "*.py") + +infinite_loops = ["infinite_loop_re.py", "nasty_eq_vs_dict.py"] + +class CrasherTest(unittest.TestCase): + + @unittest.skip("these tests are too fragile") + @test.support.cpython_only + def test_crashers_crash(self): + for fname in glob.glob(CRASHER_FILES): + if os.path.basename(fname) in infinite_loops: + continue + # Some "crashers" only trigger an exception rather than a + # segfault. Consider that an acceptable outcome. + if test.support.verbose: + print("Checking crasher:", fname) + assert_python_failure(fname) + + +def test_main(): + test.support.run_unittest(CrasherTest) + test.support.reap_children() + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_crypt.py b/Lib/test/test_crypt.py index 2adb28d..dc107d8 100644 --- a/Lib/test/test_crypt.py +++ b/Lib/test/test_crypt.py @@ -10,6 +10,25 @@ class CryptTestCase(unittest.TestCase): if support.verbose: print('Test encryption: ', c) + def test_salt(self): + self.assertEqual(len(crypt._saltchars), 64) + for method in crypt.methods: + salt = crypt.mksalt(method) + self.assertEqual(len(salt), + method.salt_chars + (3 if method.ident else 0)) + + def test_saltedcrypt(self): + for method in crypt.methods: + pw = crypt.crypt('assword', method) + self.assertEqual(len(pw), method.total_size) + pw = crypt.crypt('assword', crypt.mksalt(method)) + self.assertEqual(len(pw), method.total_size) + + def test_methods(self): + # Gurantee that METHOD_CRYPT is the last method in crypt.methods. + self.assertTrue(len(crypt.methods) >= 1) + self.assertEqual(crypt.METHOD_CRYPT, crypt.methods[-1]) + def test_main(): support.run_unittest(CryptTestCase) diff --git a/Lib/test/test_curses.py b/Lib/test/test_curses.py index 5812147..72be3e7 100644 --- a/Lib/test/test_curses.py +++ b/Lib/test/test_curses.py @@ -264,6 +264,22 @@ def test_issue6243(stdscr): curses.ungetch(1025) stdscr.getkey() +def test_unget_wch(stdscr): + if not hasattr(curses, 'unget_wch'): + return + ch = 'a' + curses.unget_wch(ch) + read = stdscr.get_wch() + read = chr(read) + if read != ch: + raise AssertionError("%r != %r" % (read, ch)) + + ch = ord('a') + curses.unget_wch(ch) + read = stdscr.get_wch() + if read != ch: + raise AssertionError("%r != %r" % (read, ch)) + def test_issue10570(): b = curses.tparm(curses.tigetstr("cup"), 5, 3) assert type(b) is bytes @@ -277,16 +293,17 @@ def main(stdscr): test_userptr_without_set(stdscr) test_resize_term(stdscr) test_issue6243(stdscr) + test_unget_wch(stdscr) test_issue10570() finally: curses.resetty() def test_main(): - if not sys.stdout.isatty(): - raise unittest.SkipTest("sys.stdout is not a tty") + if not sys.__stdout__.isatty(): + raise unittest.SkipTest("sys.__stdout__ is not a tty") # testing setupterm() inside initscr/endwin # causes terminal breakage - curses.setupterm(fd=sys.stdout.fileno()) + curses.setupterm(fd=sys.__stdout__.fileno()) try: stdscr = curses.initscr() main(stdscr) diff --git a/Lib/test/test_dbm.py b/Lib/test/test_dbm.py index 26d4c14..02df7e3 100644 --- a/Lib/test/test_dbm.py +++ b/Lib/test/test_dbm.py @@ -71,8 +71,8 @@ class AnyDBMTestCase(unittest.TestCase): f.close() def test_anydbm_creation_n_file_exists_with_invalid_contents(self): - with open(_fname, "w") as w: - pass # create an empty file + # create an empty file + test.support.create_empty_file(_fname) f = dbm.open(_fname, 'n') self.addCleanup(f.close) diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index e46cd91..014e9c7 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -1834,18 +1834,9 @@ class ContextAPItests(unittest.TestCase): # only, the attribute should be gettable/settable via both # `clamp` and `_clamp`; in Python 3.3, `_clamp` should be # removed. - c = Context(clamp = 0) - self.assertEqual(c.clamp, 0) - - with check_warnings(("", DeprecationWarning)): - c._clamp = 1 - self.assertEqual(c.clamp, 1) - with check_warnings(("", DeprecationWarning)): - self.assertEqual(c._clamp, 1) - c.clamp = 0 - self.assertEqual(c.clamp, 0) - with check_warnings(("", DeprecationWarning)): - self.assertEqual(c._clamp, 0) + c = Context() + with self.assertRaises(AttributeError): + clamp_value = c._clamp def test_abs(self): c = Context() @@ -1970,6 +1961,17 @@ class ContextAPItests(unittest.TestCase): self.assertRaises(TypeError, c.fma, 2, '3', 4) self.assertRaises(TypeError, c.fma, 2, 3, '4') + # Issue 12079 for Context.fma ... + self.assertRaises(TypeError, c.fma, + Decimal('Infinity'), Decimal(0), "not a decimal") + self.assertRaises(TypeError, c.fma, + Decimal(1), Decimal('snan'), 1.222) + # ... and for Decimal.fma. + self.assertRaises(TypeError, Decimal('Infinity').fma, + Decimal(0), "not a decimal") + self.assertRaises(TypeError, Decimal(1).fma, + Decimal('snan'), 1.222) + def test_is_finite(self): c = Context() d = c.is_finite(Decimal(10)) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index b214996..15219db 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -652,19 +652,19 @@ class ClassPropertiesAndMethods(unittest.TestCase): class A(metaclass=AMeta): pass self.assertEqual(['AMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() class B(metaclass=BMeta): pass # BMeta.__new__ calls AMeta.__new__ with super: self.assertEqual(['BMeta', 'AMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() class C(A, B): pass # The most derived metaclass is BMeta: self.assertEqual(['BMeta', 'AMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() # BMeta.__prepare__ should've been called: self.assertIn('BMeta_was_here', C.__dict__) @@ -672,20 +672,20 @@ class ClassPropertiesAndMethods(unittest.TestCase): class C2(B, A): pass self.assertEqual(['BMeta', 'AMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() self.assertIn('BMeta_was_here', C2.__dict__) # Check correct metaclass calculation when a metaclass is declared: class D(C, metaclass=type): pass self.assertEqual(['BMeta', 'AMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() self.assertIn('BMeta_was_here', D.__dict__) class E(C, metaclass=AMeta): pass self.assertEqual(['BMeta', 'AMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() self.assertIn('BMeta_was_here', E.__dict__) # Special case: the given metaclass isn't a class, @@ -727,33 +727,33 @@ class ClassPropertiesAndMethods(unittest.TestCase): pass self.assertIs(ANotMeta, type(A)) self.assertEqual(['ANotMeta'], prepare_calls) - prepare_calls[:] = [] + prepare_calls.clear() self.assertEqual(['ANotMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() class B(metaclass=BNotMeta): pass self.assertIs(BNotMeta, type(B)) self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - prepare_calls[:] = [] + prepare_calls.clear() self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() class C(A, B): pass self.assertIs(BNotMeta, type(C)) self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - prepare_calls[:] = [] + prepare_calls.clear() class C2(B, A): pass self.assertIs(BNotMeta, type(C2)) self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - prepare_calls[:] = [] + prepare_calls.clear() # This is a TypeError, because of a metaclass conflict: # BNotMeta is neither a subclass, nor a superclass of type @@ -765,25 +765,25 @@ class ClassPropertiesAndMethods(unittest.TestCase): pass self.assertIs(BNotMeta, type(E)) self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - prepare_calls[:] = [] + prepare_calls.clear() class F(object(), C): pass self.assertIs(BNotMeta, type(F)) self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - prepare_calls[:] = [] + prepare_calls.clear() class F2(C, object()): pass self.assertIs(BNotMeta, type(F2)) self.assertEqual(['BNotMeta', 'ANotMeta'], new_calls) - new_calls[:] = [] + new_calls.clear() self.assertEqual(['BNotMeta', 'ANotMeta'], prepare_calls) - prepare_calls[:] = [] + prepare_calls.clear() # TypeError: BNotMeta is neither a # subclass, nor a superclass of int @@ -1799,12 +1799,7 @@ order (MRO) for bases """ for attr, obj in env.items(): setattr(X, attr, obj) setattr(X, name, ErrDescr()) - try: - runner(X()) - except MyException: - pass - else: - self.fail("{0!r} didn't raise".format(name)) + self.assertRaises(MyException, runner, X()) def test_specials(self): # Testing special operators... @@ -2241,9 +2236,6 @@ order (MRO) for bases """ # Two essentially featureless objects, just inheriting stuff from # object. self.assertEqual(dir(NotImplemented), dir(Ellipsis)) - if support.check_impl_detail(): - # None differs in PyPy: it has a __nonzero__ - self.assertEqual(dir(None), dir(Ellipsis)) # Nasty test case for proxied objects class Wrapper(object): @@ -4429,6 +4421,27 @@ order (MRO) for bases """ foo = Foo() str(foo) + def test_slot_shadows_class_variable(self): + with self.assertRaises(ValueError) as cm: + class X: + __slots__ = ["foo"] + foo = None + m = str(cm.exception) + self.assertEqual("'foo' in __slots__ conflicts with class variable", m) + + def test_set_doc(self): + class X: + "elephant" + X.__doc__ = "banana" + self.assertEqual(X.__doc__, "banana") + with self.assertRaises(TypeError) as cm: + type(list).__dict__["__doc__"].__set__(list, "blah") + self.assertIn("can't set list.__doc__", str(cm.exception)) + with self.assertRaises(TypeError) as cm: + type(X).__dict__["__doc__"].__delete__(X) + self.assertIn("can't delete X.__doc__", str(cm.exception)) + self.assertEqual(X.__doc__, "banana") + class DictProxyTests(unittest.TestCase): def setUp(self): class C(object): @@ -4436,6 +4449,8 @@ class DictProxyTests(unittest.TestCase): pass self.C = C + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __local__') def test_iter_keys(self): # Testing dict-proxy keys... it = self.C.__dict__.keys() @@ -4445,6 +4460,8 @@ class DictProxyTests(unittest.TestCase): self.assertEqual(keys, ['__dict__', '__doc__', '__module__', '__weakref__', 'meth']) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __local__') def test_iter_values(self): # Testing dict-proxy values... it = self.C.__dict__.values() @@ -4452,6 +4469,8 @@ class DictProxyTests(unittest.TestCase): values = list(it) self.assertEqual(len(values), 5) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __local__') def test_iter_items(self): # Testing dict-proxy iteritems... it = self.C.__dict__.items() diff --git a/Lib/test/test_descrtut.py b/Lib/test/test_descrtut.py index 2db3d33..f495e18 100644 --- a/Lib/test/test_descrtut.py +++ b/Lib/test/test_descrtut.py @@ -170,6 +170,7 @@ You can get the information from the list type: '__contains__', '__delattr__', '__delitem__', + '__dir__', '__doc__', '__eq__', '__format__', @@ -199,6 +200,8 @@ You can get the information from the list type: '__str__', '__subclasshook__', 'append', + 'clear', + 'copy', 'count', 'extend', 'index', diff --git a/Lib/test/test_devpoll.py b/Lib/test/test_devpoll.py new file mode 100644 index 0000000..bef4e18 --- /dev/null +++ b/Lib/test/test_devpoll.py @@ -0,0 +1,94 @@ +# Test case for the select.devpoll() function + +# Initial tests are copied as is from "test_poll.py" + +import os, select, random, unittest, sys +from test.support import TESTFN, run_unittest + +try: + select.devpoll +except AttributeError: + raise unittest.SkipTest("select.devpoll not defined -- skipping test_devpoll") + + +def find_ready_matching(ready, flag): + match = [] + for fd, mode in ready: + if mode & flag: + match.append(fd) + return match + +class DevPollTests(unittest.TestCase): + + def test_devpoll1(self): + # Basic functional test of poll object + # Create a bunch of pipe and test that poll works with them. + + p = select.devpoll() + + NUM_PIPES = 12 + MSG = b" This is a test." + MSG_LEN = len(MSG) + readers = [] + writers = [] + r2w = {} + w2r = {} + + for i in range(NUM_PIPES): + rd, wr = os.pipe() + p.register(rd) + p.modify(rd, select.POLLIN) + p.register(wr, select.POLLOUT) + readers.append(rd) + writers.append(wr) + r2w[rd] = wr + w2r[wr] = rd + + bufs = [] + + while writers: + ready = p.poll() + ready_writers = find_ready_matching(ready, select.POLLOUT) + if not ready_writers: + self.fail("no pipes ready for writing") + wr = random.choice(ready_writers) + os.write(wr, MSG) + + ready = p.poll() + ready_readers = find_ready_matching(ready, select.POLLIN) + if not ready_readers: + self.fail("no pipes ready for reading") + self.assertEqual([w2r[wr]], ready_readers) + rd = ready_readers[0] + buf = os.read(rd, MSG_LEN) + self.assertEqual(len(buf), MSG_LEN) + bufs.append(buf) + os.close(r2w[rd]) ; os.close(rd) + p.unregister(r2w[rd]) + p.unregister(rd) + writers.remove(r2w[rd]) + + self.assertEqual(bufs, [MSG] * NUM_PIPES) + + def test_timeout_overflow(self): + pollster = select.devpoll() + w, r = os.pipe() + pollster.register(w) + + pollster.poll(-1) + self.assertRaises(OverflowError, pollster.poll, -2) + self.assertRaises(OverflowError, pollster.poll, -1 << 31) + self.assertRaises(OverflowError, pollster.poll, -1 << 64) + + pollster.poll(0) + pollster.poll(1) + pollster.poll(1 << 30) + self.assertRaises(OverflowError, pollster.poll, 1 << 31) + self.assertRaises(OverflowError, pollster.poll, 1 << 63) + self.assertRaises(OverflowError, pollster.poll, 1 << 64) + +def test_main(): + run_unittest(DevPollTests) + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 7a61493..4b0b02a 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -1,11 +1,35 @@ # Minimal tests for dis module from test.support import run_unittest, captured_stdout +import difflib import unittest import sys import dis import io +class _C: + def __init__(self, x): + self.x = x == 1 + +dis_c_instance_method = """\ + %-4d 0 LOAD_FAST 1 (x) + 3 LOAD_CONST 1 (1) + 6 COMPARE_OP 2 (==) + 9 LOAD_FAST 0 (self) + 12 STORE_ATTR 0 (x) + 15 LOAD_CONST 0 (None) + 18 RETURN_VALUE +""" % (_C.__init__.__code__.co_firstlineno + 1,) + +dis_c_instance_method_bytes = """\ + 0 LOAD_FAST 1 (1) + 3 LOAD_CONST 1 (1) + 6 COMPARE_OP 2 (==) + 9 LOAD_FAST 0 (0) + 12 STORE_ATTR 0 (0) + 15 LOAD_CONST 0 (0) + 18 RETURN_VALUE +""" def _f(a): print(a) @@ -23,6 +47,16 @@ dis_f = """\ _f.__code__.co_firstlineno + 2) +dis_f_co_code = """\ + 0 LOAD_GLOBAL 0 (0) + 3 LOAD_FAST 0 (0) + 6 CALL_FUNCTION 1 + 9 POP_TOP + 10 LOAD_CONST 1 (1) + 13 RETURN_VALUE +""" + + def bug708901(): for res in range(1, 10): @@ -138,18 +172,27 @@ dis_compound_stmt_str = """\ """ class DisTests(unittest.TestCase): - def do_disassembly_test(self, func, expected): + + def get_disassembly(self, func, lasti=-1, wrapper=True): s = io.StringIO() save_stdout = sys.stdout sys.stdout = s - dis.dis(func) - sys.stdout = save_stdout - got = s.getvalue() + try: + if wrapper: + dis.dis(func) + else: + dis.disassemble(func, lasti) + finally: + sys.stdout = save_stdout # Trim trailing blanks (if any). - lines = got.split('\n') - lines = [line.rstrip() for line in lines] - expected = expected.split("\n") - import difflib + return [line.rstrip() for line in s.getvalue().splitlines()] + + def get_disassemble_as_string(self, func, lasti=-1): + return '\n'.join(self.get_disassembly(func, lasti, False)) + + def do_disassembly_test(self, func, expected): + lines = self.get_disassembly(func) + expected = expected.splitlines() if expected != lines: self.fail( "events did not match expectation:\n" + @@ -157,7 +200,7 @@ class DisTests(unittest.TestCase): lines))) def test_opmap(self): - self.assertEqual(dis.opmap["STOP_CODE"], 0) + self.assertEqual(dis.opmap["NOP"], 9) self.assertIn(dis.opmap["LOAD_CONST"], dis.hasconst) self.assertIn(dis.opmap["STORE_NAME"], dis.hasname) @@ -211,6 +254,44 @@ class DisTests(unittest.TestCase): self.do_disassembly_test(simple_stmt_str, dis_simple_stmt_str) self.do_disassembly_test(compound_stmt_str, dis_compound_stmt_str) + def test_disassemble_bytes(self): + self.do_disassembly_test(_f.__code__.co_code, dis_f_co_code) + + def test_disassemble_method(self): + self.do_disassembly_test(_C(1).__init__, dis_c_instance_method) + + def test_disassemble_method_bytes(self): + method_bytecode = _C(1).__init__.__code__.co_code + self.do_disassembly_test(method_bytecode, dis_c_instance_method_bytes) + + def test_dis_none(self): + try: + del sys.last_traceback + except AttributeError: + pass + self.assertRaises(RuntimeError, dis.dis, None) + + def test_dis_object(self): + self.assertRaises(TypeError, dis.dis, object()) + + def test_dis_traceback(self): + try: + del sys.last_traceback + except AttributeError: + pass + + try: + 1/0 + except Exception as e: + tb = e.__traceback__ + sys.last_traceback = tb + + tb_dis = self.get_disassemble_as_string(tb.tb_frame.f_code, tb.tb_lasti) + self.do_disassembly_test(None, tb_dis) + + def test_dis_object(self): + self.assertRaises(TypeError, dis.dis, object()) + code_info_code_info = """\ Name: code_info Filename: (.*) @@ -363,6 +444,13 @@ class CodeInfoTests(unittest.TestCase): dis.show_code(x) self.assertRegex(output.getvalue(), expected+"\n") + def test_code_info_object(self): + self.assertRaises(TypeError, dis.code_info, object()) + + def test_pretty_flags_no_flags(self): + self.assertEqual(dis.pretty_flags(0), '0x0') + + def test_main(): run_unittest(DisTests, CodeInfoTests) diff --git a/Lib/test/test_doctest.py b/Lib/test/test_doctest.py index 13836ba..cd87179 100644 --- a/Lib/test/test_doctest.py +++ b/Lib/test/test_doctest.py @@ -5,6 +5,7 @@ Test script for doctest. from test import support import doctest import os +import sys # NOTE: There are some additional tests relating to interaction with @@ -373,7 +374,7 @@ We'll simulate a __file__ attr that ends in pyc: >>> tests = finder.find(sample_func) >>> print(tests) # doctest: +ELLIPSIS - [<DocTest sample_func from ...:17 (1 example)>] + [<DocTest sample_func from ...:18 (1 example)>] The exact name depends on how test_doctest was invoked, so allow for leading path components. @@ -1686,226 +1687,227 @@ Run the debugger on the docstring, and then restore sys.stdin. """ -def test_pdb_set_trace(): - """Using pdb.set_trace from a doctest. - - You can use pdb.set_trace from a doctest. To do so, you must - retrieve the set_trace function from the pdb module at the time - you use it. The doctest module changes sys.stdout so that it can - capture program output. It also temporarily replaces pdb.set_trace - with a version that restores stdout. This is necessary for you to - see debugger output. - - >>> doc = ''' - ... >>> x = 42 - ... >>> raise Exception('clé') - ... Traceback (most recent call last): - ... Exception: clé - ... >>> import pdb; pdb.set_trace() - ... ''' - >>> parser = doctest.DocTestParser() - >>> test = parser.get_doctest(doc, {}, "foo-bar@baz", "foo-bar@baz.py", 0) - >>> runner = doctest.DocTestRunner(verbose=False) - - To demonstrate this, we'll create a fake standard input that - captures our debugger input: - - >>> import tempfile - >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ - ... 'print(x)', # print data defined by the example - ... 'continue', # stop debugging - ... '']) - - >>> try: runner.run(test) - ... finally: sys.stdin = real_stdin - --Return-- - > <doctest foo-bar@baz[2]>(1)<module>()->None - -> import pdb; pdb.set_trace() - (Pdb) print(x) - 42 - (Pdb) continue - TestResults(failed=0, attempted=3) - - You can also put pdb.set_trace in a function called from a test: - - >>> def calls_set_trace(): - ... y=2 - ... import pdb; pdb.set_trace() - - >>> doc = ''' - ... >>> x=1 - ... >>> calls_set_trace() - ... ''' - >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) - >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ - ... 'print(y)', # print data defined in the function - ... 'up', # out of function - ... 'print(x)', # print data defined by the example - ... 'continue', # stop debugging - ... '']) - - >>> try: - ... runner.run(test) - ... finally: - ... sys.stdin = real_stdin - --Return-- - > <doctest test.test_doctest.test_pdb_set_trace[8]>(3)calls_set_trace()->None - -> import pdb; pdb.set_trace() - (Pdb) print(y) - 2 - (Pdb) up - > <doctest foo-bar@baz[1]>(1)<module>() - -> calls_set_trace() - (Pdb) print(x) - 1 - (Pdb) continue - TestResults(failed=0, attempted=2) - - During interactive debugging, source code is shown, even for - doctest examples: - - >>> doc = ''' - ... >>> def f(x): - ... ... g(x*2) - ... >>> def g(x): - ... ... print(x+3) - ... ... import pdb; pdb.set_trace() - ... >>> f(3) - ... ''' - >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) - >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ - ... 'list', # list source from example 2 - ... 'next', # return from g() - ... 'list', # list source from example 1 - ... 'next', # return from f() - ... 'list', # list source from example 3 - ... 'continue', # stop debugging - ... '']) - >>> try: runner.run(test) - ... finally: sys.stdin = real_stdin - ... # doctest: +NORMALIZE_WHITESPACE - --Return-- - > <doctest foo-bar@baz[1]>(3)g()->None - -> import pdb; pdb.set_trace() - (Pdb) list - 1 def g(x): - 2 print(x+3) - 3 -> import pdb; pdb.set_trace() - [EOF] - (Pdb) next - --Return-- - > <doctest foo-bar@baz[0]>(2)f()->None - -> g(x*2) - (Pdb) list - 1 def f(x): - 2 -> g(x*2) - [EOF] - (Pdb) next - --Return-- - > <doctest foo-bar@baz[2]>(1)<module>()->None - -> f(3) - (Pdb) list - 1 -> f(3) - [EOF] - (Pdb) continue - ********************************************************************** - File "foo-bar@baz.py", line 7, in foo-bar@baz - Failed example: - f(3) - Expected nothing - Got: - 9 - TestResults(failed=1, attempted=3) - """ - -def test_pdb_set_trace_nested(): - """This illustrates more-demanding use of set_trace with nested functions. - - >>> class C(object): - ... def calls_set_trace(self): - ... y = 1 - ... import pdb; pdb.set_trace() - ... self.f1() - ... y = 2 - ... def f1(self): - ... x = 1 - ... self.f2() - ... x = 2 - ... def f2(self): - ... z = 1 - ... z = 2 - - >>> calls_set_trace = C().calls_set_trace - - >>> doc = ''' - ... >>> a = 1 - ... >>> calls_set_trace() - ... ''' - >>> parser = doctest.DocTestParser() - >>> runner = doctest.DocTestRunner(verbose=False) - >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) - >>> real_stdin = sys.stdin - >>> sys.stdin = _FakeInput([ - ... 'print(y)', # print data defined in the function - ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', - ... 'up', 'print(x)', - ... 'up', 'print(y)', - ... 'up', 'print(foo)', - ... 'continue', # stop debugging - ... '']) - - >>> try: - ... runner.run(test) - ... finally: - ... sys.stdin = real_stdin - ... # doctest: +REPORT_NDIFF - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace() - -> self.f1() - (Pdb) print(y) - 1 - (Pdb) step - --Call-- - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(7)f1() - -> def f1(self): - (Pdb) step - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(8)f1() - -> x = 1 - (Pdb) step - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1() - -> self.f2() - (Pdb) step - --Call-- - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(11)f2() - -> def f2(self): - (Pdb) step - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(12)f2() - -> z = 1 - (Pdb) step - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(13)f2() - -> z = 2 - (Pdb) print(z) - 1 - (Pdb) up - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1() - -> self.f2() - (Pdb) print(x) - 1 - (Pdb) up - > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace() - -> self.f1() - (Pdb) print(y) - 1 - (Pdb) up - > <doctest foo-bar@baz[1]>(1)<module>() - -> calls_set_trace() - (Pdb) print(foo) - *** NameError: name 'foo' is not defined - (Pdb) continue - TestResults(failed=0, attempted=2) -""" +if not hasattr(sys, 'gettrace') or not sys.gettrace(): + def test_pdb_set_trace(): + """Using pdb.set_trace from a doctest. + + You can use pdb.set_trace from a doctest. To do so, you must + retrieve the set_trace function from the pdb module at the time + you use it. The doctest module changes sys.stdout so that it can + capture program output. It also temporarily replaces pdb.set_trace + with a version that restores stdout. This is necessary for you to + see debugger output. + + >>> doc = ''' + ... >>> x = 42 + ... >>> raise Exception('clé') + ... Traceback (most recent call last): + ... Exception: clé + ... >>> import pdb; pdb.set_trace() + ... ''' + >>> parser = doctest.DocTestParser() + >>> test = parser.get_doctest(doc, {}, "foo-bar@baz", "foo-bar@baz.py", 0) + >>> runner = doctest.DocTestRunner(verbose=False) + + To demonstrate this, we'll create a fake standard input that + captures our debugger input: + + >>> import tempfile + >>> real_stdin = sys.stdin + >>> sys.stdin = _FakeInput([ + ... 'print(x)', # print data defined by the example + ... 'continue', # stop debugging + ... '']) + + >>> try: runner.run(test) + ... finally: sys.stdin = real_stdin + --Return-- + > <doctest foo-bar@baz[2]>(1)<module>()->None + -> import pdb; pdb.set_trace() + (Pdb) print(x) + 42 + (Pdb) continue + TestResults(failed=0, attempted=3) + + You can also put pdb.set_trace in a function called from a test: + + >>> def calls_set_trace(): + ... y=2 + ... import pdb; pdb.set_trace() + + >>> doc = ''' + ... >>> x=1 + ... >>> calls_set_trace() + ... ''' + >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> real_stdin = sys.stdin + >>> sys.stdin = _FakeInput([ + ... 'print(y)', # print data defined in the function + ... 'up', # out of function + ... 'print(x)', # print data defined by the example + ... 'continue', # stop debugging + ... '']) + + >>> try: + ... runner.run(test) + ... finally: + ... sys.stdin = real_stdin + --Return-- + > <doctest test.test_doctest.test_pdb_set_trace[8]>(3)calls_set_trace()->None + -> import pdb; pdb.set_trace() + (Pdb) print(y) + 2 + (Pdb) up + > <doctest foo-bar@baz[1]>(1)<module>() + -> calls_set_trace() + (Pdb) print(x) + 1 + (Pdb) continue + TestResults(failed=0, attempted=2) + + During interactive debugging, source code is shown, even for + doctest examples: + + >>> doc = ''' + ... >>> def f(x): + ... ... g(x*2) + ... >>> def g(x): + ... ... print(x+3) + ... ... import pdb; pdb.set_trace() + ... >>> f(3) + ... ''' + >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> real_stdin = sys.stdin + >>> sys.stdin = _FakeInput([ + ... 'list', # list source from example 2 + ... 'next', # return from g() + ... 'list', # list source from example 1 + ... 'next', # return from f() + ... 'list', # list source from example 3 + ... 'continue', # stop debugging + ... '']) + >>> try: runner.run(test) + ... finally: sys.stdin = real_stdin + ... # doctest: +NORMALIZE_WHITESPACE + --Return-- + > <doctest foo-bar@baz[1]>(3)g()->None + -> import pdb; pdb.set_trace() + (Pdb) list + 1 def g(x): + 2 print(x+3) + 3 -> import pdb; pdb.set_trace() + [EOF] + (Pdb) next + --Return-- + > <doctest foo-bar@baz[0]>(2)f()->None + -> g(x*2) + (Pdb) list + 1 def f(x): + 2 -> g(x*2) + [EOF] + (Pdb) next + --Return-- + > <doctest foo-bar@baz[2]>(1)<module>()->None + -> f(3) + (Pdb) list + 1 -> f(3) + [EOF] + (Pdb) continue + ********************************************************************** + File "foo-bar@baz.py", line 7, in foo-bar@baz + Failed example: + f(3) + Expected nothing + Got: + 9 + TestResults(failed=1, attempted=3) + """ + + def test_pdb_set_trace_nested(): + """This illustrates more-demanding use of set_trace with nested functions. + + >>> class C(object): + ... def calls_set_trace(self): + ... y = 1 + ... import pdb; pdb.set_trace() + ... self.f1() + ... y = 2 + ... def f1(self): + ... x = 1 + ... self.f2() + ... x = 2 + ... def f2(self): + ... z = 1 + ... z = 2 + + >>> calls_set_trace = C().calls_set_trace + + >>> doc = ''' + ... >>> a = 1 + ... >>> calls_set_trace() + ... ''' + >>> parser = doctest.DocTestParser() + >>> runner = doctest.DocTestRunner(verbose=False) + >>> test = parser.get_doctest(doc, globals(), "foo-bar@baz", "foo-bar@baz.py", 0) + >>> real_stdin = sys.stdin + >>> sys.stdin = _FakeInput([ + ... 'print(y)', # print data defined in the function + ... 'step', 'step', 'step', 'step', 'step', 'step', 'print(z)', + ... 'up', 'print(x)', + ... 'up', 'print(y)', + ... 'up', 'print(foo)', + ... 'continue', # stop debugging + ... '']) + + >>> try: + ... runner.run(test) + ... finally: + ... sys.stdin = real_stdin + ... # doctest: +REPORT_NDIFF + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace() + -> self.f1() + (Pdb) print(y) + 1 + (Pdb) step + --Call-- + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(7)f1() + -> def f1(self): + (Pdb) step + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(8)f1() + -> x = 1 + (Pdb) step + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1() + -> self.f2() + (Pdb) step + --Call-- + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(11)f2() + -> def f2(self): + (Pdb) step + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(12)f2() + -> z = 1 + (Pdb) step + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(13)f2() + -> z = 2 + (Pdb) print(z) + 1 + (Pdb) up + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(9)f1() + -> self.f2() + (Pdb) print(x) + 1 + (Pdb) up + > <doctest test.test_doctest.test_pdb_set_trace_nested[0]>(5)calls_set_trace() + -> self.f1() + (Pdb) print(y) + 1 + (Pdb) up + > <doctest foo-bar@baz[1]>(1)<module>() + -> calls_set_trace() + (Pdb) print(foo) + *** NameError: name 'foo' is not defined + (Pdb) continue + TestResults(failed=0, attempted=2) + """ def test_DocTestSuite(): """DocTestSuite creates a unittest test suite from a doctest. diff --git a/Lib/test/test_dummy_thread.py b/Lib/test/test_dummy_thread.py index c61078d..2fafe1d 100644 --- a/Lib/test/test_dummy_thread.py +++ b/Lib/test/test_dummy_thread.py @@ -35,8 +35,8 @@ class LockTests(unittest.TestCase): "Lock object did not release properly.") def test_improper_release(self): - #Make sure release of an unlocked thread raises _thread.error - self.assertRaises(_thread.error, self.lock.release) + #Make sure release of an unlocked thread raises RuntimeError + self.assertRaises(RuntimeError, self.lock.release) def test_cond_acquire_success(self): #Make sure the conditional acquiring of the lock works. diff --git a/Lib/test/test_email.py b/Lib/test/test_email.py deleted file mode 100644 index 5eebba5..0000000 --- a/Lib/test/test_email.py +++ /dev/null @@ -1,14 +0,0 @@ -# Copyright (C) 2001-2007 Python Software Foundation -# email package unit tests - -# The specific tests now live in Lib/email/test -from email.test.test_email import suite -from email.test.test_email_codecs import suite as codecs_suite -from test import support - -def test_main(): - support.run_unittest(suite()) - support.run_unittest(codecs_suite()) - -if __name__ == '__main__': - test_main() diff --git a/Lib/test/test_email/__init__.py b/Lib/test/test_email/__init__.py new file mode 100644 index 0000000..280afbd --- /dev/null +++ b/Lib/test/test_email/__init__.py @@ -0,0 +1,45 @@ +import os +import sys +import unittest +import test.support +import email +from test.test_email import __file__ as landmark + +# used by regrtest and __main__. +def test_main(): + here = os.path.dirname(__file__) + # Unittest mucks with the path, so we have to save and restore + # it to keep regrtest happy. + savepath = sys.path[:] + test.support._run_suite(unittest.defaultTestLoader.discover(here)) + sys.path[:] = savepath + + +# helper code used by a number of test modules. + +def openfile(filename, *args, **kws): + path = os.path.join(os.path.dirname(landmark), 'data', filename) + return open(path, *args, **kws) + + +# Base test class +class TestEmailBase(unittest.TestCase): + + maxDiff = None + + def __init__(self, *args, **kw): + super().__init__(*args, **kw) + self.addTypeEqualityFunc(bytes, self.assertBytesEqual) + + ndiffAssertEqual = unittest.TestCase.assertEqual + + def _msgobj(self, filename): + with openfile(filename) as fp: + return email.message_from_file(fp) + + def _bytes_repr(self, b): + return [repr(x) for x in b.splitlines(keepends=True)] + + def assertBytesEqual(self, first, second, msg): + """Our byte strings are really encoded strings; improve diff output""" + self.assertEqual(self._bytes_repr(first), self._bytes_repr(second)) diff --git a/Lib/test/test_email/__main__.py b/Lib/test/test_email/__main__.py new file mode 100644 index 0000000..98af9ec --- /dev/null +++ b/Lib/test/test_email/__main__.py @@ -0,0 +1,3 @@ +from test.test_email import test_main + +test_main() diff --git a/Lib/test/test_email/data/PyBanner048.gif b/Lib/test/test_email/data/PyBanner048.gif Binary files differnew file mode 100644 index 0000000..1a5c87f --- /dev/null +++ b/Lib/test/test_email/data/PyBanner048.gif diff --git a/Lib/test/test_email/data/audiotest.au b/Lib/test/test_email/data/audiotest.au Binary files differnew file mode 100644 index 0000000..f76b050 --- /dev/null +++ b/Lib/test/test_email/data/audiotest.au diff --git a/Lib/test/test_email/data/msg_01.txt b/Lib/test/test_email/data/msg_01.txt new file mode 100644 index 0000000..7e33bcf --- /dev/null +++ b/Lib/test/test_email/data/msg_01.txt @@ -0,0 +1,19 @@ +Return-Path: <bbb@zzz.org> +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +Do you like this message? + +-Me diff --git a/Lib/test/test_email/data/msg_02.txt b/Lib/test/test_email/data/msg_02.txt new file mode 100644 index 0000000..43f2480 --- /dev/null +++ b/Lib/test/test_email/data/msg_02.txt @@ -0,0 +1,135 @@ +MIME-version: 1.0 +From: ppp-request@zzz.org +Sender: ppp-admin@zzz.org +To: ppp@zzz.org +Subject: Ppp digest, Vol 1 #2 - 5 msgs +Date: Fri, 20 Apr 2001 20:18:00 -0400 (EDT) +X-Mailer: Mailman v2.0.4 +X-Mailman-Version: 2.0.4 +Content-Type: multipart/mixed; boundary="192.168.1.2.889.32614.987812255.500.21814" + +--192.168.1.2.889.32614.987812255.500.21814 +Content-type: text/plain; charset=us-ascii +Content-description: Masthead (Ppp digest, Vol 1 #2) + +Send Ppp mailing list submissions to + ppp@zzz.org + +To subscribe or unsubscribe via the World Wide Web, visit + http://www.zzz.org/mailman/listinfo/ppp +or, via email, send a message with subject or body 'help' to + ppp-request@zzz.org + +You can reach the person managing the list at + ppp-admin@zzz.org + +When replying, please edit your Subject line so it is more specific +than "Re: Contents of Ppp digest..." + + +--192.168.1.2.889.32614.987812255.500.21814 +Content-type: text/plain; charset=us-ascii +Content-description: Today's Topics (5 msgs) + +Today's Topics: + + 1. testing #1 (Barry A. Warsaw) + 2. testing #2 (Barry A. Warsaw) + 3. testing #3 (Barry A. Warsaw) + 4. testing #4 (Barry A. Warsaw) + 5. testing #5 (Barry A. Warsaw) + +--192.168.1.2.889.32614.987812255.500.21814 +Content-Type: multipart/digest; boundary="__--__--" + +--__--__-- + +Message: 1 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Date: Fri, 20 Apr 2001 20:16:13 -0400 +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Subject: [Ppp] testing #1 +Precedence: bulk + + +hello + + +--__--__-- + +Message: 2 +Date: Fri, 20 Apr 2001 20:16:21 -0400 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Precedence: bulk + + +hello + + +--__--__-- + +Message: 3 +Date: Fri, 20 Apr 2001 20:16:25 -0400 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Subject: [Ppp] testing #3 +Precedence: bulk + + +hello + + +--__--__-- + +Message: 4 +Date: Fri, 20 Apr 2001 20:16:28 -0400 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Subject: [Ppp] testing #4 +Precedence: bulk + + +hello + + +--__--__-- + +Message: 5 +Date: Fri, 20 Apr 2001 20:16:32 -0400 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +To: ppp@zzz.org +From: barry@digicool.com (Barry A. Warsaw) +Subject: [Ppp] testing #5 +Precedence: bulk + + +hello + + + + +--__--__---- +--192.168.1.2.889.32614.987812255.500.21814 +Content-type: text/plain; charset=us-ascii +Content-description: Digest Footer + +_______________________________________________ +Ppp mailing list +Ppp@zzz.org +http://www.zzz.org/mailman/listinfo/ppp + + +--192.168.1.2.889.32614.987812255.500.21814-- + +End of Ppp Digest + diff --git a/Lib/test/test_email/data/msg_03.txt b/Lib/test/test_email/data/msg_03.txt new file mode 100644 index 0000000..c748ebf --- /dev/null +++ b/Lib/test/test_email/data/msg_03.txt @@ -0,0 +1,16 @@ +Return-Path: <bbb@zzz.org> +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +Do you like this message? + +-Me diff --git a/Lib/test/test_email/data/msg_04.txt b/Lib/test/test_email/data/msg_04.txt new file mode 100644 index 0000000..1f633c4 --- /dev/null +++ b/Lib/test/test_email/data/msg_04.txt @@ -0,0 +1,37 @@ +Return-Path: <barry@python.org> +Delivered-To: barry@python.org +Received: by mail.python.org (Postfix, from userid 889) + id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="h90VIIIKmx" +Content-Transfer-Encoding: 7bit +Message-ID: <15261.36209.358846.118674@anthem.python.org> +From: barry@python.org (Barry A. Warsaw) +To: barry@python.org +Subject: a simple multipart +Date: Tue, 11 Sep 2001 00:05:05 -0400 +X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid +X-Attribution: BAW +X-Oblique-Strategy: Make a door into a window + + +--h90VIIIKmx +Content-Type: text/plain +Content-Disposition: inline; + filename="msg.txt" +Content-Transfer-Encoding: 7bit + +a simple kind of mirror +to reflect upon our own + +--h90VIIIKmx +Content-Type: text/plain +Content-Disposition: inline; + filename="msg.txt" +Content-Transfer-Encoding: 7bit + +a simple kind of mirror +to reflect upon our own + +--h90VIIIKmx-- + diff --git a/Lib/test/test_email/data/msg_05.txt b/Lib/test/test_email/data/msg_05.txt new file mode 100644 index 0000000..87d5e9c --- /dev/null +++ b/Lib/test/test_email/data/msg_05.txt @@ -0,0 +1,28 @@ +From: foo +Subject: bar +To: baz +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + boundary="D1690A7AC1.996856090/mail.example.com" +Message-Id: <20010803162810.0CA8AA7ACC@mail.example.com> + +This is a MIME-encapsulated message. + +--D1690A7AC1.996856090/mail.example.com +Content-Type: text/plain + +Yadda yadda yadda + +--D1690A7AC1.996856090/mail.example.com + +Yadda yadda yadda + +--D1690A7AC1.996856090/mail.example.com +Content-Type: message/rfc822 + +From: nobody@python.org + +Yadda yadda yadda + +--D1690A7AC1.996856090/mail.example.com-- + diff --git a/Lib/test/test_email/data/msg_06.txt b/Lib/test/test_email/data/msg_06.txt new file mode 100644 index 0000000..69f3a47 --- /dev/null +++ b/Lib/test/test_email/data/msg_06.txt @@ -0,0 +1,33 @@ +Return-Path: <barry@python.org> +Delivered-To: barry@python.org +MIME-Version: 1.0 +Content-Type: message/rfc822 +Content-Description: forwarded message +Content-Transfer-Encoding: 7bit +Message-ID: <15265.9482.641338.555352@python.org> +From: barry@zope.com (Barry A. Warsaw) +Sender: barry@python.org +To: barry@python.org +Subject: forwarded message from Barry A. Warsaw +Date: Thu, 13 Sep 2001 17:28:42 -0400 +X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid +X-Attribution: BAW +X-Oblique-Strategy: Be dirty +X-Url: http://barry.wooz.org + +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Return-Path: <barry@python.org> +Delivered-To: barry@python.org +Message-ID: <15265.9468.713530.98441@python.org> +From: barry@zope.com (Barry A. Warsaw) +Sender: barry@python.org +To: barry@python.org +Subject: testing +Date: Thu, 13 Sep 2001 17:28:28 -0400 +X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid +X-Attribution: BAW +X-Oblique-Strategy: Spectrum analysis +X-Url: http://barry.wooz.org + + diff --git a/Lib/test/test_email/data/msg_07.txt b/Lib/test/test_email/data/msg_07.txt new file mode 100644 index 0000000..721f3a0 --- /dev/null +++ b/Lib/test/test_email/data/msg_07.txt @@ -0,0 +1,83 @@ +MIME-Version: 1.0 +From: Barry <barry@digicool.com> +To: Dingus Lovers <cravindogs@cravindogs.com> +Subject: Here is your dingus fish +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + +Hi there, + +This is the dingus fish. + +--BOUNDARY +Content-Type: image/gif; name="dingusfish.gif" +Content-Transfer-Encoding: base64 +content-disposition: attachment; filename="dingusfish.gif" + +R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 +7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f +qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI +oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR +RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA +/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ +YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb +6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC +M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA +B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D +z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX +JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 +ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR +n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX +eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ +qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe +wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j +Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 +doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x +lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A +E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq +Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic +isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD +ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv +Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn +I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff +BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG +pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n +B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ +1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 +YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW +FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 +ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu +zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s +ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D +oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 +cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ +7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz +GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ +FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt +pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb +6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS +kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe +UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 +9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y +xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI +fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV +SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH +VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR +fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE +J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN +l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq +I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi +yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w +qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi +MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky +uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR +WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM +YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I +MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 +cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa +BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_08.txt b/Lib/test/test_email/data/msg_08.txt new file mode 100644 index 0000000..b563083 --- /dev/null +++ b/Lib/test/test_email/data/msg_08.txt @@ -0,0 +1,24 @@ +MIME-Version: 1.0 +From: Barry Warsaw <barry@zope.com> +To: Dingus Lovers <cravindogs@cravindogs.com> +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" + + +--BOUNDARY +Content-Type: text/plain; charset="iso-8859-2" + + +--BOUNDARY +Content-Type: text/plain; charset="koi8-r" + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_09.txt b/Lib/test/test_email/data/msg_09.txt new file mode 100644 index 0000000..575c4c2 --- /dev/null +++ b/Lib/test/test_email/data/msg_09.txt @@ -0,0 +1,24 @@ +MIME-Version: 1.0 +From: Barry Warsaw <barry@zope.com> +To: Dingus Lovers <cravindogs@cravindogs.com> +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" + + +--BOUNDARY +Content-Type: text/plain + + +--BOUNDARY +Content-Type: text/plain; charset="koi8-r" + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_10.txt b/Lib/test/test_email/data/msg_10.txt new file mode 100644 index 0000000..0790396 --- /dev/null +++ b/Lib/test/test_email/data/msg_10.txt @@ -0,0 +1,39 @@ +MIME-Version: 1.0 +From: Barry Warsaw <barry@zope.com> +To: Dingus Lovers <cravindogs@cravindogs.com> +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +Content-Transfer-Encoding: 7bit + +This is a 7bit encoded message. + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" +Content-Transfer-Encoding: Quoted-Printable + +=A1This is a Quoted Printable encoded message! + +--BOUNDARY +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: Base64 + +VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2Uu + + +--BOUNDARY +Content-Type: text/plain; charset="iso-8859-1" +Content-Transfer-Encoding: Base64 + +VGhpcyBpcyBhIEJhc2U2NCBlbmNvZGVkIG1lc3NhZ2UuCg== + + +--BOUNDARY +Content-Type: text/plain; charset="iso-8859-1" + +This has no Content-Transfer-Encoding: header. + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_11.txt b/Lib/test/test_email/data/msg_11.txt new file mode 100644 index 0000000..8f7f199 --- /dev/null +++ b/Lib/test/test_email/data/msg_11.txt @@ -0,0 +1,7 @@ +Content-Type: message/rfc822 +MIME-Version: 1.0 +Subject: The enclosing message + +Subject: An enclosed message + +Here is the body of the message. diff --git a/Lib/test/test_email/data/msg_12.txt b/Lib/test/test_email/data/msg_12.txt new file mode 100644 index 0000000..4bec8d9 --- /dev/null +++ b/Lib/test/test_email/data/msg_12.txt @@ -0,0 +1,36 @@ +MIME-Version: 1.0 +From: Barry Warsaw <barry@zope.com> +To: Dingus Lovers <cravindogs@cravindogs.com> +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" + + +--BOUNDARY +Content-Type: multipart/mixed; boundary="ANOTHER" + +--ANOTHER +Content-Type: text/plain; charset="iso-8859-2" + + +--ANOTHER +Content-Type: text/plain; charset="iso-8859-3" + +--ANOTHER-- + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/plain; charset="koi8-r" + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_12a.txt b/Lib/test/test_email/data/msg_12a.txt new file mode 100644 index 0000000..e94224e --- /dev/null +++ b/Lib/test/test_email/data/msg_12a.txt @@ -0,0 +1,38 @@ +MIME-Version: 1.0 +From: Barry Warsaw <barry@zope.com> +To: Dingus Lovers <cravindogs@cravindogs.com> +Subject: Lyrics +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/html; charset="iso-8859-1" + + +--BOUNDARY +Content-Type: multipart/mixed; boundary="ANOTHER" + +--ANOTHER +Content-Type: text/plain; charset="iso-8859-2" + + +--ANOTHER +Content-Type: text/plain; charset="iso-8859-3" + + +--ANOTHER-- + + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + + +--BOUNDARY +Content-Type: text/plain; charset="koi8-r" + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_13.txt b/Lib/test/test_email/data/msg_13.txt new file mode 100644 index 0000000..8e6d52d --- /dev/null +++ b/Lib/test/test_email/data/msg_13.txt @@ -0,0 +1,94 @@ +MIME-Version: 1.0 +From: Barry <barry@digicool.com> +To: Dingus Lovers <cravindogs@cravindogs.com> +Subject: Here is your dingus fish +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="OUTER" + +--OUTER +Content-Type: text/plain; charset="us-ascii" + +A text/plain part + +--OUTER +Content-Type: multipart/mixed; boundary=BOUNDARY + + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" + +Hi there, + +This is the dingus fish. + +--BOUNDARY +Content-Type: image/gif; name="dingusfish.gif" +Content-Transfer-Encoding: base64 +content-disposition: attachment; filename="dingusfish.gif" + +R0lGODdhAAEAAfAAAP///wAAACwAAAAAAAEAAQAC/oSPqcvtD6OctNqLs968+w+G4kiW5omm6sq2 +7gvH8kzX9o3n+s73/g8MCofEovGITGICTKbyCV0FDNOo9SqpQqpOrJfXzTQj2vD3TGtqL+NtGQ2f +qTXmxzuOd7WXdcc9DyjU53ewFni4s0fGhdiYaEhGBelICTNoV1j5NUnFcrmUqemjNifJVWpaOqaI +oFq3SspZsSraE7sHq3jr1MZqWvi662vxV4tD+pvKW6aLDOCLyur8PDwbanyDeq0N3DctbQYeLDvR +RY6t95m6UB0d3mwIrV7e2VGNvjjffukeJp4w7F65KecGFsTHQGAygOrgrWs1jt28Rc88KESYcGLA +/obvTkH6p+CinWJiJmIMqXGQwH/y4qk0SYjgQTczT3ajKZGfuI0uJ4kkVI/DT5s3/ejkxI0aT4Y+ +YTYgWbImUaXk9nlLmnSh1qJiJFl0OpUqRK4oOy7NyRQtHWofhoYVxkwWXKUSn0YsS+fUV6lhqfYb +6ayd3Z5qQdG1B7bvQzaJjwUV2lixMUZ7JVsOlfjWVr/3NB/uFvnySBN6Dcb6rGwaRM3wsormw5cC +M9NxWy/bWdufudCvy8bOAjXjVVwta/uO21sE5RHBCzNFXtgq9ORtH4eYjVP4Yryo026nvkFmCeyA +B29efV6ravCMK5JwWd5897Qrx7ll38o6iHDZ/rXPR//feevhF4l7wjUGX3xq1eeRfM4RSJGBIV1D +z1gKPkfWag3mVBVvva1RlX5bAJTPR/2YqNtw/FkIYYEi/pIZiAdpcxpoHtmnYYoZtvhUftzdx5ZX +JSKDW405zkGcZzzGZ6KEv4FI224oDmijlEf+xp6MJK5ojY/ASeVUR+wsKRuJ+XFZ5o7ZeEime8t1 +ouUsU6YjF5ZtUihhkGfCdFQLWQFJ3UXxmElfhQnR+eCdcDbkFZp6vTRmj56ApCihn5QGpaToNZmR +n3NVSpZcQpZ2KEONusaiCsKAug0wkQbJSFO+PTSjneGxOuFjPlUk3ovWvdIerjUg9ZGIOtGq/qeX +eCYrrCX+1UPsgTKGGRSbzd5q156d/gpfbJxe66eD5iQKrXj7RGgruGxs62qebBHUKS32CKluCiqZ +qh+pmehmEb71noAUoe5e9Zm17S7773V10pjrtG4CmuurCV/n6zLK5turWNhqOvFXbjhZrMD0YhKe +wR0zOyuvsh6MWrGoIuzvyWu5y1WIFAqmJselypxXh6dKLNOKEB98L88bS2rkNqqlKzCNJp9c0G0j +Gzh0iRrCbHSXmPR643QS+4rWhgFmnSbSuXCjS0xAOWkU2UdLqyuUNfHSFdUouy3bm5i5GnDM3tG8 +doJ4r5tqu3pPbRSVfvs8uJzeNXhp3n4j/tZ42SwH7eaWUUOjc3qFV9453UHTXZfcLH+OeNs5g36x +lBnHvTm7EbMbLeuaLncao8vWCXimfo1o+843Ak6y4ChNeGntvAYvfLK4ezmoyNIbNCLTCXO9ZV3A +E8/s88RczPzDwI4Ob7XZyl7+9Miban29h+tJZPrE21wgvBphDfrrfPdCTPKJD/y98L1rZwHcV6Jq +Zab0metpuNIX/qAFPoz171WUaUb4HAhBSzHuHfjzHb3kha/2Cctis/ORArVHNYfFyYRH2pYIRzic +isVOfPWD1b6mRTqpCRBozzof6UZVvFXRxWIr3GGrEviGYgyPMfahheiSaLs/9QeFu7oZ/ndSY8DD +ya9x+uPed+7mxN2IzIISBOMLFYWVqC3Pew1T2nFuuCiwZS5/v6II10i4t1OJcUH2U9zxKodHsGGv +Oa+zkvNUYUOa/TCCRutF9MzDwdlUMJADTCGSbDQ5OV4PTamDoPEi6Ecc/RF5RWwkcdSXvSOaDWSn +I9LlvubFTQpuc6JKXLcKeb+xdbKRBnwREemXyjg6ME65aJiOuBgrktzykfPLJBKR9ClMavJ62/Ff +BlNIyod9yX9wcSXexnXFpvkrbXk64xsx5Db7wXKP5fSgsvwIMM/9631VLBfkmtbHRXpqmtei52hG +pUwSlo+BASQoeILDOBgREECxBBh5/iYmNsQ9dIv5+OI++QkqdsJPc3uykz5fkM+OraeekcQF7X4n +B5S67za5U967PmooGQhUXfF7afXyCD7ONdRe17QogYjVx38uLwtrS6nhTnm15LQUnu9E2uK6CNI/ +1HOABj0ESwOjut4FEpFQpdNAm4K2LHnDWHNcmKB2ioKBogysVZtMO2nSxUdZ8Yk2kJc7URioLVI0 +YgmtIwZj4LoeKemgnOnbUdGnzZ4Oa6scqiolBGqS6RgWNLu0RMhcaE6rhhU4hiuqFXPAG8fGwTPW +FKeLMtdVmXLSs5YJGF/YeVm7rREMlY3UYE+yCxbaMXX8y15m5zVHq6GOKDMynzII/jdUHdyVqIy0 +ifX2+r/EgtZcvRzSb72gU9ui87M2VecjKildW/aFqaYhKoryUjfB/g4qtyVuc60xFDGmCxwjW+qu +zjuwl2GkOWn66+3QiiEctvd04OVvcCVzjgT7lrkvjVGKKHmmlDUKowSeikb5kK/mJReuWOxONx+s +ULsl+Lqb0CVn0SrVyJ6wt4t6yTeSCafhPhAf0OXn6L60UMxiLolFAtmN35S2Ob1lZpQ1r/n0Qb5D +oQ1zJiRVDgF8N3Q8TYfbi3DyWCy3lT1nxyBs6FT3S2GOzWRlxwKvlRP0RPJA9SjxEy0UoEnkA+M4 +cnzLMJrBGWLFEaaUb5lvpqbq/loOaU5+DFuHPxo82/OZuM8FXG3oVNZhtWpMpb/0Xu5m/LfLhHZQ +7yuVI0MqZ7NE43imC8jH3IwGZlbPm0xkJYs7+2U48hXTsFSMqgGDvai0kLxyynKNT/waj+q1c1tz +GjOpPBgdCSq3UKZxCSsqFIY+O6JbAWGWcV1pwqLyj5sGqCF1xb1F3varUWqrJv6cN3PrUXzijtfZ +FshpBL3Xwr4GIPvU2N8EjrJgS1zl21rbXQMXeXc5jjFyrhpCzijSv/RQtyPSzHCFMhlME95fHglt +pRsX+dfSQjUeHAlpWzJ5iOo79Ldnaxai6bXTcGO3fp07ri7HLEmXXPlYi8bv/qVxvNcdra6m7Rlb +6JBTb5fd66VhFRjGArh2n7R1rDW4P5NOT9K0I183T2scYkeZ3q/VFyLb09U9ajzXBS8Kgkhc4mBS +kYY9cy3Vy9lUnuNJH8HGIclUilwnBtjUOH0gteGOZ4c/XNrhXLSYDyxfnD8z1pDy7rYRvDolhnbe +UMzxCZUs40s6s7UIvBnLgc0+vKuOkIXeOrDymlp+Zxra4MZLBbVrqD/jTJ597pDmnw5c4+DbyB88 +9Cg9DodYcSuMZT/114pptqc/EuTjRPvH/z5slzI3tluOEBBLqOXLOX+0I5929tO97wkvl/atCz+y +xJrdwteW2FNW/NSmBP+f/maYtVs/bYyBC7Ox3jsYZHL05CIrBa/nS+b3bHfiYm4Ueil1YZZSgAUI +fFZ1dxUmeA2oQRQ3RuGXNGLFV9/XbGFGPV6kfzk1TBBCd+izc7q1H+OHMJwmaBX2IQNYVAKHYepV +SSGCe6CnbYHHETKGNe43EDvFgZr0gB/nVHPHZ80VV1ojOiI3XDvYIkl4ayo4bxQIgrFXWTvBI0nH +VElWMuw2aLUWCRHHf8ymVCHjFlJnOSojfevCYyyyZDH0IcvHhrsnQ5O1OsWzONuVVKIxSxiFZ/tR +fKDAf6xFTnw4O9Qig2VCfW2hJQrmMOuHW0W3dLQmCMO2ccdUd/xyfflH/olTiHZVdGwb8nIwRzSE +J15jFlOJuBZBZ4CiyHyd2IFylFlB+HgHhYabhWOGwYO1ZH/Og1dtQlFMk352CGRSIFTapnWQEUtN +l4zv8S0aaCFDyGCBqDUxZYpxGHX01y/JuH1xhn7TOCnNCI4eKDs5WGX4R425F4vF1o3BJ4vO0otq +I3rimI7jJY1jISqnBxknCIvruF83mF5wN4X7qGLIhR8A2Vg0yFERSIXn9Vv3GHy3Vj/WIkKddlYi +yIMv2I/VMjTLpW7pt05SWIZR0RPyxpB4SIUM9lBPGBl0GC7oSEEwRYLe4pJpZY2P0zbI1n+Oc44w +qY3PUnmF0ixjVpDD/mJ9wpOBGTVgXlaCaZiPcIWK5NiKBIiPdGaQ0TWGvAiG7nMchdZb7Vgf8zNi +MuMyzRdy/lePe9iC4TRx7WhhOQI/QiSVNAmAa2lT/piFbuh7ofJoYSZzrSZ1bvmWw3eN2nKUPVky +uPN5/VRfohRd0VYZoqhKIlU6TXYhJxmPUIloAwc1bPmHEpaZYZORHNlXUJM07hATwHR8MJYqkwWR +WaIezFhxSFlc8/Fq82hEnpeRozg3ULhhr9lAGtVEkCg5ZNRuuVleBPaZadhG0ZgkyPmDOTOKzViM +YgOcpukKqQcbjAWS0IleQ2ROjdh6A+md1qWdBRSX7iSYgFRTtRmBpJioieXJiHfJiMGIR9fJOn8I +MSfXYhspn4ooSa2mSAj4n+8Bmg03fBJZoPOJgsVZRxu1oOMRPXYYjdqjihFaEoZpXBREanuJoRI6 +cibFinq4ngUKh/wQd/H5ofYCZ0HJXR62opZFaAT0iFIZo4DIiUojkjeqKiuoZirKo5Y1a7AWckGa +BkuYoD5lpDK6eUs6CkDqpETwl1EqpfhJpVeKpVl6EgUAADs= + +--BOUNDARY-- + +--OUTER-- diff --git a/Lib/test/test_email/data/msg_14.txt b/Lib/test/test_email/data/msg_14.txt new file mode 100644 index 0000000..5d98d2f --- /dev/null +++ b/Lib/test/test_email/data/msg_14.txt @@ -0,0 +1,23 @@ +Return-Path: <bbb@zzz.org> +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: text; charset=us-ascii +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +I'm sorry but I'm using a drainbread ISP, which although big and +wealthy can't seem to generate standard compliant email. :( + +This message has a Content-Type: header with no subtype. I hope you +can still read it. + +-Me diff --git a/Lib/test/test_email/data/msg_15.txt b/Lib/test/test_email/data/msg_15.txt new file mode 100644 index 0000000..0025624 --- /dev/null +++ b/Lib/test/test_email/data/msg_15.txt @@ -0,0 +1,52 @@ +Return-Path: <xx@xx.dk> +Received: from fepD.post.tele.dk (195.41.46.149) by mail.groupcare.dk (LSMTP for Windows NT v1.1b) with SMTP id <0.0014F8A2@mail.groupcare.dk>; Mon, 30 Apr 2001 12:17:50 +0200 +User-Agent: Microsoft-Outlook-Express-Macintosh-Edition/5.02.2106 +Subject: XX +From: xx@xx.dk +To: XX +Message-ID: <xxxx> +Mime-version: 1.0 +Content-type: multipart/mixed; + boundary="MS_Mac_OE_3071477847_720252_MIME_Part" + +> Denne meddelelse er i MIME-format. Da dit postl + +--MS_Mac_OE_3071477847_720252_MIME_Part +Content-type: multipart/alternative; + boundary="MS_Mac_OE_3071477847_720252_MIME_Part" + + +--MS_Mac_OE_3071477847_720252_MIME_Part +Content-type: text/plain; charset="ISO-8859-1" +Content-transfer-encoding: quoted-printable + +Some removed test. + +--MS_Mac_OE_3071477847_720252_MIME_Part +Content-type: text/html; charset="ISO-8859-1" +Content-transfer-encoding: quoted-printable + +<HTML> +<HEAD> +<TITLE>Some removed HTML</TITLE> +</HEAD> +<BODY> +Some removed text. +</BODY> +</HTML> + + +--MS_Mac_OE_3071477847_720252_MIME_Part-- + + +--MS_Mac_OE_3071477847_720252_MIME_Part +Content-type: image/gif; name="xx.gif"; + x-mac-creator="6F676C65"; + x-mac-type="47494666" +Content-disposition: attachment +Content-transfer-encoding: base64 + +Some removed base64 encoded chars. + +--MS_Mac_OE_3071477847_720252_MIME_Part-- + diff --git a/Lib/test/test_email/data/msg_16.txt b/Lib/test/test_email/data/msg_16.txt new file mode 100644 index 0000000..56167e9 --- /dev/null +++ b/Lib/test/test_email/data/msg_16.txt @@ -0,0 +1,123 @@ +Return-Path: <> +Delivered-To: scr-admin@socal-raves.org +Received: from cougar.noc.ucla.edu (cougar.noc.ucla.edu [169.232.10.18]) + by babylon.socal-raves.org (Postfix) with ESMTP id CCC2C51B84 + for <scr-admin@socal-raves.org>; Sun, 23 Sep 2001 20:13:54 -0700 (PDT) +Received: from sims-ms-daemon by cougar.noc.ucla.edu + (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) + id <0GK500B01D0B8Y@cougar.noc.ucla.edu> for scr-admin@socal-raves.org; Sun, + 23 Sep 2001 20:14:35 -0700 (PDT) +Received: from cougar.noc.ucla.edu + (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) + id <0GK500B01D0B8X@cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) +Date: Sun, 23 Sep 2001 20:14:35 -0700 (PDT) +From: Internet Mail Delivery <postmaster@ucla.edu> +Subject: Delivery Notification: Delivery has failed +To: scr-admin@socal-raves.org +Message-id: <0GK500B04D0B8X@cougar.noc.ucla.edu> +MIME-version: 1.0 +Sender: scr-owner@socal-raves.org +Errors-To: scr-owner@socal-raves.org +X-BeenThere: scr@socal-raves.org +X-Mailman-Version: 2.1a3 +Precedence: bulk +List-Help: <mailto:scr-request@socal-raves.org?subject=help> +List-Post: <mailto:scr@socal-raves.org> +List-Subscribe: <http://socal-raves.org/mailman/listinfo/scr>, + <mailto:scr-request@socal-raves.org?subject=subscribe> +List-Id: SoCal-Raves <scr.socal-raves.org> +List-Unsubscribe: <http://socal-raves.org/mailman/listinfo/scr>, + <mailto:scr-request@socal-raves.org?subject=unsubscribe> +List-Archive: <http://socal-raves.org/mailman/private/scr/> +Content-Type: multipart/report; boundary="Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)" + + +--Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) +Content-type: text/plain; charset=ISO-8859-1 + +This report relates to a message you sent with the following header fields: + + Message-id: <002001c144a6$8752e060$56104586@oxy.edu> + Date: Sun, 23 Sep 2001 20:10:55 -0700 + From: "Ian T. Henry" <henryi@oxy.edu> + To: SoCal Raves <scr@socal-raves.org> + Subject: [scr] yeah for Ians!! + +Your message cannot be delivered to the following recipients: + + Recipient address: jangel1@cougar.noc.ucla.edu + Reason: recipient reached disk quota + + +--Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) +Content-type: message/DELIVERY-STATUS + +Original-envelope-id: 0GK500B4HD0888@cougar.noc.ucla.edu +Reporting-MTA: dns; cougar.noc.ucla.edu + +Action: failed +Status: 5.0.0 (recipient reached disk quota) +Original-recipient: rfc822;jangel1@cougar.noc.ucla.edu +Final-recipient: rfc822;jangel1@cougar.noc.ucla.edu + +--Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA) +Content-type: MESSAGE/RFC822 + +Return-path: scr-admin@socal-raves.org +Received: from sims-ms-daemon by cougar.noc.ucla.edu + (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) + id <0GK500B01D0B8X@cougar.noc.ucla.edu>; Sun, 23 Sep 2001 20:14:35 -0700 (PDT) +Received: from panther.noc.ucla.edu by cougar.noc.ucla.edu + (Sun Internet Mail Server sims.3.5.2000.03.23.18.03.p10) + with ESMTP id <0GK500B4GD0888@cougar.noc.ucla.edu> for jangel1@sims-ms-daemon; + Sun, 23 Sep 2001 20:14:33 -0700 (PDT) +Received: from babylon.socal-raves.org + (ip-209-85-222-117.dreamhost.com [209.85.222.117]) + by panther.noc.ucla.edu (8.9.1a/8.9.1) with ESMTP id UAA09793 for + <jangel1@ucla.edu>; Sun, 23 Sep 2001 20:14:32 -0700 (PDT) +Received: from babylon (localhost [127.0.0.1]) by babylon.socal-raves.org + (Postfix) with ESMTP id D3B2951B70; Sun, 23 Sep 2001 20:13:47 -0700 (PDT) +Received: by babylon.socal-raves.org (Postfix, from userid 60001) + id A611F51B82; Sun, 23 Sep 2001 20:13:46 -0700 (PDT) +Received: from tiger.cc.oxy.edu (tiger.cc.oxy.edu [134.69.3.112]) + by babylon.socal-raves.org (Postfix) with ESMTP id ADA7351B70 for + <scr@socal-raves.org>; Sun, 23 Sep 2001 20:13:44 -0700 (PDT) +Received: from ent (n16h86.dhcp.oxy.edu [134.69.16.86]) + by tiger.cc.oxy.edu (8.8.8/8.8.8) with SMTP id UAA08100 for + <scr@socal-raves.org>; Sun, 23 Sep 2001 20:14:24 -0700 (PDT) +Date: Sun, 23 Sep 2001 20:10:55 -0700 +From: "Ian T. Henry" <henryi@oxy.edu> +Subject: [scr] yeah for Ians!! +Sender: scr-admin@socal-raves.org +To: SoCal Raves <scr@socal-raves.org> +Errors-to: scr-admin@socal-raves.org +Message-id: <002001c144a6$8752e060$56104586@oxy.edu> +MIME-version: 1.0 +X-Mailer: Microsoft Outlook Express 5.50.4522.1200 +Content-type: text/plain; charset=us-ascii +Precedence: bulk +Delivered-to: scr-post@babylon.socal-raves.org +Delivered-to: scr@socal-raves.org +X-Converted-To-Plain-Text: from multipart/alternative by demime 0.98e +X-Converted-To-Plain-Text: Alternative section used was text/plain +X-BeenThere: scr@socal-raves.org +X-Mailman-Version: 2.1a3 +List-Help: <mailto:scr-request@socal-raves.org?subject=help> +List-Post: <mailto:scr@socal-raves.org> +List-Subscribe: <http://socal-raves.org/mailman/listinfo/scr>, + <mailto:scr-request@socal-raves.org?subject=subscribe> +List-Id: SoCal-Raves <scr.socal-raves.org> +List-Unsubscribe: <http://socal-raves.org/mailman/listinfo/scr>, + <mailto:scr-request@socal-raves.org?subject=unsubscribe> +List-Archive: <http://socal-raves.org/mailman/private/scr/> + +I always love to find more Ian's that are over 3 years old!! + +Ian +_______________________________________________ +For event info, list questions, or to unsubscribe, see http://www.socal-raves.org/ + + + +--Boundary_(ID_PGS2F2a+z+/jL7hupKgRhA)-- + diff --git a/Lib/test/test_email/data/msg_17.txt b/Lib/test/test_email/data/msg_17.txt new file mode 100644 index 0000000..8d86e41 --- /dev/null +++ b/Lib/test/test_email/data/msg_17.txt @@ -0,0 +1,12 @@ +MIME-Version: 1.0 +From: Barry <barry@digicool.com> +To: Dingus Lovers <cravindogs@cravindogs.com> +Subject: Here is your dingus fish +Date: Fri, 20 Apr 2001 19:35:02 -0400 +Content-Type: multipart/mixed; boundary="BOUNDARY" + +Hi there, + +This is the dingus fish. + +[Non-text (image/gif) part of message omitted, filename dingusfish.gif] diff --git a/Lib/test/test_email/data/msg_18.txt b/Lib/test/test_email/data/msg_18.txt new file mode 100644 index 0000000..f9f4904 --- /dev/null +++ b/Lib/test/test_email/data/msg_18.txt @@ -0,0 +1,6 @@ +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; + spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" + diff --git a/Lib/test/test_email/data/msg_19.txt b/Lib/test/test_email/data/msg_19.txt new file mode 100644 index 0000000..49bf7fc --- /dev/null +++ b/Lib/test/test_email/data/msg_19.txt @@ -0,0 +1,43 @@ +Send Ppp mailing list submissions to + ppp@zzz.org + +To subscribe or unsubscribe via the World Wide Web, visit + http://www.zzz.org/mailman/listinfo/ppp +or, via email, send a message with subject or body 'help' to + ppp-request@zzz.org + +You can reach the person managing the list at + ppp-admin@zzz.org + +When replying, please edit your Subject line so it is more specific +than "Re: Contents of Ppp digest..." + +Today's Topics: + + 1. testing #1 (Barry A. Warsaw) + 2. testing #2 (Barry A. Warsaw) + 3. testing #3 (Barry A. Warsaw) + 4. testing #4 (Barry A. Warsaw) + 5. testing #5 (Barry A. Warsaw) + +hello + + +hello + + +hello + + +hello + + +hello + + + +_______________________________________________ +Ppp mailing list +Ppp@zzz.org +http://www.zzz.org/mailman/listinfo/ppp + diff --git a/Lib/test/test_email/data/msg_20.txt b/Lib/test/test_email/data/msg_20.txt new file mode 100644 index 0000000..1a6a887 --- /dev/null +++ b/Lib/test/test_email/data/msg_20.txt @@ -0,0 +1,22 @@ +Return-Path: <bbb@zzz.org> +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Cc: ccc@zzz.org +CC: ddd@zzz.org +cc: eee@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +Do you like this message? + +-Me diff --git a/Lib/test/test_email/data/msg_21.txt b/Lib/test/test_email/data/msg_21.txt new file mode 100644 index 0000000..23590b2 --- /dev/null +++ b/Lib/test/test_email/data/msg_21.txt @@ -0,0 +1,20 @@ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: Test +Content-Type: multipart/mixed; boundary="BOUNDARY" + +MIME message +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +One +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Two +--BOUNDARY-- +End of MIME message diff --git a/Lib/test/test_email/data/msg_22.txt b/Lib/test/test_email/data/msg_22.txt new file mode 100644 index 0000000..af9de5f --- /dev/null +++ b/Lib/test/test_email/data/msg_22.txt @@ -0,0 +1,46 @@ +Mime-Version: 1.0 +Message-Id: <a05001902b7f1c33773e9@[134.84.183.138]> +Date: Tue, 16 Oct 2001 13:59:25 +0300 +To: a@example.com +From: b@example.com +Content-Type: multipart/mixed; boundary="============_-1208892523==_============" + +--============_-1208892523==_============ +Content-Type: text/plain; charset="us-ascii" ; format="flowed" + +Text text text. +--============_-1208892523==_============ +Content-Id: <a05001902b7f1c33773e9@[134.84.183.138].0.0> +Content-Type: image/jpeg; name="wibble.JPG" + ; x-mac-type="4A504547" + ; x-mac-creator="474B4F4E" +Content-Disposition: attachment; filename="wibble.JPG" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB +AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABIEBAREA +g6bCjjw/pIZSjO6FWFpldjySOmCNrO7DBZibUXhTwtCixw+GtAijVdqxxaPp0aKvmGXa +qrbBQvms0mAMeYS/3iTV1dG0hHaRNK01XblnWxtVdjkHLMIgTyqnk9VB7CrP2KzIINpa +4O7I+zxYO9WV8jZg71Zlb+8rMDkEirAVQFAUAKAFAAAUAYAUDgADgY6DjpRtXj5RxjHA +4wQRj0wQCMdCAewpaKKK/9k= +--============_-1208892523==_============ +Content-Id: <a05001902b7f1c33773e9@[134.84.183.138].0.1> +Content-Type: image/jpeg; name="wibble2.JPG" + ; x-mac-type="4A504547" + ; x-mac-creator="474B4F4E" +Content-Disposition: attachment; filename="wibble2.JPG" +Content-Transfer-Encoding: base64 + +/9j/4AAQSkZJRgABAQAAAQABAAD/2wBDAAEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEB +AQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQEBAQH/wAALCAXABJ0BAREA +/8QAHwAAAQUBAQEBAQEAAAAAAAAAAAECAwQFBgcICQoL/8QAtRAAAgEDAwIEAwUFBAQA +W6NFJJBEkU10kKGTcWMDwxuU+0JHvk8qAtOpNwqSR0n8c3BlDyXHlqsUltHEiTvdXLxR +7vMiGDNJAJWkAMk8ZkCFp5G2oo5W++INrbQtNfTQxJAuXlupz9oS4d5Y1W+E2XlWZJJE +Y7LWYQxTLE1zuMbfBPxw8X2fibVdIbSbI6nLZxX635t9TjtYreWR7WGKJTLJFFKSlozO +0ShxIXM43uC3/9k= +--============_-1208892523==_============ +Content-Type: text/plain; charset="us-ascii" ; format="flowed" + +Text text text. +--============_-1208892523==_============-- + diff --git a/Lib/test/test_email/data/msg_23.txt b/Lib/test/test_email/data/msg_23.txt new file mode 100644 index 0000000..bb2e8ec --- /dev/null +++ b/Lib/test/test_email/data/msg_23.txt @@ -0,0 +1,8 @@ +From: aperson@dom.ain +Content-Type: multipart/mixed; boundary="BOUNDARY" + +--BOUNDARY +Content-Type: text/plain + +A message part +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_24.txt b/Lib/test/test_email/data/msg_24.txt new file mode 100644 index 0000000..4e52339 --- /dev/null +++ b/Lib/test/test_email/data/msg_24.txt @@ -0,0 +1,10 @@ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY + + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_25.txt b/Lib/test/test_email/data/msg_25.txt new file mode 100644 index 0000000..9e35275 --- /dev/null +++ b/Lib/test/test_email/data/msg_25.txt @@ -0,0 +1,117 @@ +From MAILER-DAEMON Fri Apr 06 16:46:09 2001 +Received: from [204.245.199.98] (helo=zinfandel.lacita.com) + by www.linux.org.uk with esmtp (Exim 3.13 #1) + id 14lYR6-0008Iv-00 + for linuxuser-admin@www.linux.org.uk; Fri, 06 Apr 2001 16:46:09 +0100 +Received: from localhost (localhost) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with internal id JAB03225; Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) +Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) +From: Mail Delivery Subsystem <MAILER-DAEMON@zinfandel.lacita.com> +Subject: Returned mail: Too many hops 19 (17 max): from <linuxuser-admin@www.linux.org.uk> via [199.164.235.226], to <scoffman@wellpartner.com> +Message-Id: <200104061723.JAB03225@zinfandel.lacita.com> +To: <linuxuser-admin@www.linux.org.uk> +To: postmaster@zinfandel.lacita.com +MIME-Version: 1.0 +Content-Type: multipart/report; report-type=delivery-status; + bo +Auto-Submitted: auto-generated (failure) + +This is a MIME-encapsulated message + +--JAB03225.986577786/zinfandel.lacita.com + +The original message was received at Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) +from [199.164.235.226] + + ----- The following addresses have delivery notifications ----- +<scoffman@wellpartner.com> (unrecoverable error) + + ----- Transcript of session follows ----- +554 Too many hops 19 (17 max): from <linuxuser-admin@www.linux.org.uk> via [199.164.235.226], to <scoffman@wellpartner.com> + +--JAB03225.986577786/zinfandel.lacita.com +Content-Type: message/delivery-status + +Reporting-MTA: dns; zinfandel.lacita.com +Received-From-MTA: dns; [199.164.235.226] +Arrival-Date: Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) + +Final-Recipient: rfc822; scoffman@wellpartner.com +Action: failed +Status: 5.4.6 +Last-Attempt-Date: Fri, 6 Apr 2001 09:23:06 -0800 (GMT-0800) + +--JAB03225.986577786/zinfandel.lacita.com +Content-Type: text/rfc822-headers + +Return-Path: linuxuser-admin@www.linux.org.uk +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03225 for <scoffman@wellpartner.com>; Fri, 6 Apr 2001 09:23:03 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03221 for <scoffman@wellpartner.com>; Fri, 6 Apr 2001 09:22:18 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03217 for <scoffman@wellpartner.com>; Fri, 6 Apr 2001 09:21:37 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03213 for <scoffman@wellpartner.com>; Fri, 6 Apr 2001 09:20:56 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03209 for <scoffman@wellpartner.com>; Fri, 6 Apr 2001 09:20:15 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03205 for <scoffman@wellpartner.com>; Fri, 6 Apr 2001 09:19:33 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03201 for <scoffman@wellpartner.com>; Fri, 6 Apr 2001 09:18:52 -0800 (GMT-0800) +Received: from zinfandel.lacita.com ([204.245.199.98]) + by + fo +Received: from ns1.wellpartner.net ([199.164.235.226]) by zinfandel.lacita.com (8.7.3/8.6.10-MT4.00) with ESMTP id JAA03197 for <scoffman@wellpartner.com>; Fri, 6 Apr 2001 09:17:54 -0800 (GMT-0800) +Received: from www.linux.org.uk (parcelfarce.linux.theplanet.co.uk [195.92.249.252]) + by + fo +Received: from localhost.localdomain + ([ + by + id +Received: from [212.1.130.11] (helo=s1.uklinux.net ident=root) + by + id + fo +Received: from server (ppp-2-22.cvx4.telinco.net [212.1.149.22]) + by + fo +From: Daniel James <daniel@linuxuser.co.uk> +Organization: LinuxUser +To: linuxuser@www.linux.org.uk +X-Mailer: KMail [version 1.1.99] +Content-Type: text/plain; + c +MIME-Version: 1.0 +Message-Id: <01040616033903.00962@server> +Content-Transfer-Encoding: 8bit +Subject: [LinuxUser] bulletin no. 45 +Sender: linuxuser-admin@www.linux.org.uk +Errors-To: linuxuser-admin@www.linux.org.uk +X-BeenThere: linuxuser@www.linux.org.uk +X-Mailman-Version: 2.0.3 +Precedence: bulk +List-Help: <mailto:linuxuser-request@www.linux.org.uk?subject=help> +List-Post: <mailto:linuxuser@www.linux.org.uk> +List-Subscribe: <http://www.linux.org.uk/mailman/listinfo/linuxuser>, + <m +List-Id: bulletins from LinuxUser magazine <linuxuser.www.linux.org.uk> +List-Unsubscribe: <http://www.linux.org.uk/mailman/listinfo/linuxuser>, + <m +List-Archive: <http://www.linux.org.uk/pipermail/linuxuser/> +Date: Fri, 6 Apr 2001 16:03:39 +0100 + +--JAB03225.986577786/zinfandel.lacita.com-- + + diff --git a/Lib/test/test_email/data/msg_26.txt b/Lib/test/test_email/data/msg_26.txt new file mode 100644 index 0000000..58efaa9 --- /dev/null +++ b/Lib/test/test_email/data/msg_26.txt @@ -0,0 +1,46 @@ +Received: from xcar [192.168.0.2] by jeeves.wooster.local
+ (SMTPD32-7.07 EVAL) id AFF92F0214; Sun, 12 May 2002 08:55:37 +0100
+Date: Sun, 12 May 2002 08:56:15 +0100
+From: Father Time <father.time@xcar.wooster.local>
+To: timbo@jeeves.wooster.local
+Subject: IMAP file test
+Message-ID: <6df65d354b.father.time@rpc.wooster.local>
+X-Organization: Home
+User-Agent: Messenger-Pro/2.50a (MsgServe/1.50) (RISC-OS/4.02) POPstar/2.03
+MIME-Version: 1.0
+Content-Type: multipart/mixed; boundary="1618492860--2051301190--113853680"
+Status: R
+X-UIDL: 319998302
+
+This message is in MIME format which your mailer apparently does not support.
+You either require a newer version of your software which supports MIME, or
+a separate MIME decoding utility. Alternatively, ask the sender of this
+message to resend it in a different format.
+
+--1618492860--2051301190--113853680
+Content-Type: text/plain; charset=us-ascii
+
+Simple email with attachment.
+
+
+--1618492860--2051301190--113853680
+Content-Type: application/riscos; name="clock.bmp,69c"; type=BMP;
+ load=&fff69c4b; exec=&355dd4d1; access=&03
+Content-Disposition: attachment; filename="clock.bmp"
+Content-Transfer-Encoding: base64
+
+Qk12AgAAAAAAAHYAAAAoAAAAIAAAACAAAAABAAQAAAAAAAAAAADXDQAA1w0AAAAAAAAA
+AAAAAAAAAAAAiAAAiAAAAIiIAIgAAACIAIgAiIgAALu7uwCIiIgAERHdACLuIgAz//8A
+zAAAAN0R3QDu7iIA////AAAAAAAAAAAAAAAAAAAAAAAAAAi3AAAAAAAAADeAAAAAAAAA
+C3ADMzMzMANwAAAAAAAAAAAHMAAAAANwAAAAAAAAAACAMAd3zPfwAwgAAAAAAAAIAwd/
+f8x/f3AwgAAAAAAAgDB0x/f3//zPAwgAAAAAAAcHfM9////8z/AwAAAAAAiwd/f3////
+////A4AAAAAAcEx/f///////zAMAAAAAiwfM9////3///8zwOAAAAAcHf3////B/////
+8DAAAAALB/f3///wd3d3//AwAAAABwTPf//wCQAAD/zAMAAAAAsEx/f///B////8wDAA
+AAAHB39////wf/////AwAAAACwf39///8H/////wMAAAAIcHfM9///B////M8DgAAAAA
+sHTH///wf///xAMAAAAACHB3f3//8H////cDgAAAAAALB3zH//D//M9wMAAAAAAAgLB0
+z39///xHAwgAAAAAAAgLB3d3RHd3cDCAAAAAAAAAgLAHd0R3cAMIAAAAAAAAgAgLcAAA
+AAMwgAgAAAAACDAAAAu7t7cwAAgDgAAAAABzcIAAAAAAAAgDMwAAAAAAN7uwgAAAAAgH
+MzMAAAAACH97tzAAAAALu3c3gAAAAAAL+7tzDABAu7f7cAAAAAAACA+3MA7EQAv/sIAA
+AAAAAAAIAAAAAAAAAIAAAAAA
+
+--1618492860--2051301190--113853680--
diff --git a/Lib/test/test_email/data/msg_27.txt b/Lib/test/test_email/data/msg_27.txt new file mode 100644 index 0000000..d019176 --- /dev/null +++ b/Lib/test/test_email/data/msg_27.txt @@ -0,0 +1,15 @@ +Return-Path: <aperson@dom.ain> +Received: by mail.dom.ain (Postfix, from userid 889) + id B9D0AD35DB; Tue, 4 Jun 2002 21:46:59 -0400 (EDT) +Message-ID: <15613.28051.707126.569693@dom.ain> +Date: Tue, 4 Jun 2002 21:46:59 -0400 +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii +Content-Transfer-Encoding: 7bit +Subject: bug demonstration + 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 + more text +From: aperson@dom.ain (Anne P. Erson) +To: bperson@dom.ain (Barney P. Erson) + +test diff --git a/Lib/test/test_email/data/msg_28.txt b/Lib/test/test_email/data/msg_28.txt new file mode 100644 index 0000000..1e4824c --- /dev/null +++ b/Lib/test/test_email/data/msg_28.txt @@ -0,0 +1,25 @@ +From: aperson@dom.ain +MIME-Version: 1.0 +Content-Type: multipart/digest; boundary=BOUNDARY + +--BOUNDARY +Content-Type: message/rfc822 + +Content-Type: text/plain; charset=us-ascii +To: aa@bb.org +From: cc@dd.org +Subject: ee + +message 1 + +--BOUNDARY +Content-Type: message/rfc822 + +Content-Type: text/plain; charset=us-ascii +To: aa@bb.org +From: cc@dd.org +Subject: ee + +message 2 + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_29.txt b/Lib/test/test_email/data/msg_29.txt new file mode 100644 index 0000000..1fab561 --- /dev/null +++ b/Lib/test/test_email/data/msg_29.txt @@ -0,0 +1,22 @@ +Return-Path: <bbb@zzz.org> +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) + id 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: text/plain; charset=us-ascii; + title*0*="us-ascii'en'This%20is%20even%20more%20"; + title*1*="%2A%2A%2Afun%2A%2A%2A%20"; + title*2="isn't it!" +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 + + +Hi, + +Do you like this message? + +-Me diff --git a/Lib/test/test_email/data/msg_30.txt b/Lib/test/test_email/data/msg_30.txt new file mode 100644 index 0000000..4334bb6 --- /dev/null +++ b/Lib/test/test_email/data/msg_30.txt @@ -0,0 +1,23 @@ +From: aperson@dom.ain +MIME-Version: 1.0 +Content-Type: multipart/digest; boundary=BOUNDARY + +--BOUNDARY + +Content-Type: text/plain; charset=us-ascii +To: aa@bb.org +From: cc@dd.org +Subject: ee + +message 1 + +--BOUNDARY + +Content-Type: text/plain; charset=us-ascii +To: aa@bb.org +From: cc@dd.org +Subject: ee + +message 2 + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_31.txt b/Lib/test/test_email/data/msg_31.txt new file mode 100644 index 0000000..1e58e56 --- /dev/null +++ b/Lib/test/test_email/data/msg_31.txt @@ -0,0 +1,15 @@ +From: aperson@dom.ain +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary=BOUNDARY_ + +--BOUNDARY +Content-Type: text/plain + +message 1 + +--BOUNDARY +Content-Type: text/plain + +message 2 + +--BOUNDARY-- diff --git a/Lib/test/test_email/data/msg_32.txt b/Lib/test/test_email/data/msg_32.txt new file mode 100644 index 0000000..07ec5af --- /dev/null +++ b/Lib/test/test_email/data/msg_32.txt @@ -0,0 +1,14 @@ +Delivered-To: freebsd-isp@freebsd.org +Date: Tue, 26 Sep 2000 12:23:03 -0500 +From: Anne Person <aperson@example.com> +To: Barney Dude <bdude@example.com> +Subject: Re: Limiting Perl CPU Utilization... +Mime-Version: 1.0 +Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii +Content-Disposition: inline +User-Agent: Mutt/1.3.8i +Sender: owner-freebsd-isp@FreeBSD.ORG +Precedence: bulk +X-Loop: FreeBSD.org + +Some message. diff --git a/Lib/test/test_email/data/msg_33.txt b/Lib/test/test_email/data/msg_33.txt new file mode 100644 index 0000000..042787a --- /dev/null +++ b/Lib/test/test_email/data/msg_33.txt @@ -0,0 +1,29 @@ +Delivered-To: freebsd-isp@freebsd.org +Date: Wed, 27 Sep 2000 11:11:09 -0500 +From: Anne Person <aperson@example.com> +To: Barney Dude <bdude@example.com> +Subject: Re: Limiting Perl CPU Utilization... +Mime-Version: 1.0 +Content-Type: multipart/signed; micalg*=ansi-x3.4-1968''pgp-md5; + protocol*=ansi-x3.4-1968''application%2Fpgp-signature; + boundary*="ansi-x3.4-1968''EeQfGwPcQSOJBaQU" +Content-Disposition: inline +Sender: owner-freebsd-isp@FreeBSD.ORG +Precedence: bulk +X-Loop: FreeBSD.org + + +--EeQfGwPcQSOJBaQU +Content-Type: text/plain; charset*=ansi-x3.4-1968''us-ascii +Content-Disposition: inline +Content-Transfer-Encoding: quoted-printable + +part 1 + +--EeQfGwPcQSOJBaQU +Content-Type: text/plain +Content-Disposition: inline + +part 2 + +--EeQfGwPcQSOJBaQU-- diff --git a/Lib/test/test_email/data/msg_34.txt b/Lib/test/test_email/data/msg_34.txt new file mode 100644 index 0000000..055dfea --- /dev/null +++ b/Lib/test/test_email/data/msg_34.txt @@ -0,0 +1,19 @@ +From: aperson@dom.ain +To: bperson@dom.ain +Content-Type: multipart/digest; boundary=XYZ + +--XYZ +Content-Type: text/plain + + +This is a text plain part that is counter to recommended practice in +RFC 2046, $5.1.5, but is not illegal + +--XYZ + +From: cperson@dom.ain +To: dperson@dom.ain + +A submessage + +--XYZ-- diff --git a/Lib/test/test_email/data/msg_35.txt b/Lib/test/test_email/data/msg_35.txt new file mode 100644 index 0000000..be7d5a2 --- /dev/null +++ b/Lib/test/test_email/data/msg_35.txt @@ -0,0 +1,4 @@ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: here's something interesting +counter to RFC 2822, there's no separating newline here diff --git a/Lib/test/test_email/data/msg_36.txt b/Lib/test/test_email/data/msg_36.txt new file mode 100644 index 0000000..5632c30 --- /dev/null +++ b/Lib/test/test_email/data/msg_36.txt @@ -0,0 +1,40 @@ +Mime-Version: 1.0 +Content-Type: Multipart/Mixed; Boundary="NextPart" +To: IETF-Announce:; +From: Internet-Drafts@ietf.org +Subject: I-D ACTION:draft-ietf-mboned-mix-00.txt +Date: Tue, 22 Dec 1998 16:55:06 -0500 + +--NextPart + +Blah blah blah + +--NextPart +Content-Type: Multipart/Alternative; Boundary="OtherAccess" + +--OtherAccess +Content-Type: Message/External-body; + access-type="mail-server"; + server="mailserv@ietf.org" + +Content-Type: text/plain +Content-ID: <19981222151406.I-D@ietf.org> + +ENCODING mime +FILE /internet-drafts/draft-ietf-mboned-mix-00.txt + +--OtherAccess +Content-Type: Message/External-body; + name="draft-ietf-mboned-mix-00.txt"; + site="ftp.ietf.org"; + access-type="anon-ftp"; + directory="internet-drafts" + +Content-Type: text/plain +Content-ID: <19981222151406.I-D@ietf.org> + + +--OtherAccess-- + +--NextPart-- + diff --git a/Lib/test/test_email/data/msg_37.txt b/Lib/test/test_email/data/msg_37.txt new file mode 100644 index 0000000..038d34a --- /dev/null +++ b/Lib/test/test_email/data/msg_37.txt @@ -0,0 +1,22 @@ +Content-Type: multipart/mixed; boundary=ABCDE + +--ABCDE +Content-Type: text/x-one + +Blah + +--ABCDE +--ABCDE +Content-Type: text/x-two + +Blah + +--ABCDE +--ABCDE +--ABCDE +--ABCDE +Content-Type: text/x-two + +Blah + +--ABCDE-- diff --git a/Lib/test/test_email/data/msg_38.txt b/Lib/test/test_email/data/msg_38.txt new file mode 100644 index 0000000..006df81 --- /dev/null +++ b/Lib/test/test_email/data/msg_38.txt @@ -0,0 +1,101 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" + +------- =_aaaaaaaaaa0 +Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.1@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa2" +Content-ID: <20592.1022586929.2@example.com> + +------- =_aaaaaaaaaa2 +Content-Type: text/plain +Content-ID: <20592.1022586929.3@example.com> +Content-Description: very tricky +Content-Transfer-Encoding: 7bit + + +Unlike the test test_nested-multiples-with-internal-boundary, this +piece of text not only contains the outer boundary tags +------- =_aaaaaaaaaa1 +and +------- =_aaaaaaaaaa0 +but puts them at the start of a line! And, to be even nastier, it +even includes a couple of end tags, such as this one: + +------- =_aaaaaaaaaa1-- + +and this one, which is from a multipart we haven't even seen yet! + +------- =_aaaaaaaaaa4-- + +This will, I'm sure, cause much breakage of MIME parsers. But, as +far as I can tell, it's perfectly legal. I have not yet ever seen +a case of this in the wild, but I've seen *similar* things. + + +------- =_aaaaaaaaaa2 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.4@example.com> +Content-Description: patch2 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa2-- + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa3" +Content-ID: <20592.1022586929.6@example.com> + +------- =_aaaaaaaaaa3 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.7@example.com> +Content-Description: patch3 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa3 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.8@example.com> +Content-Description: patch4 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa3-- + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa4" +Content-ID: <20592.1022586929.10@example.com> + +------- =_aaaaaaaaaa4 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.11@example.com> +Content-Description: patch5 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa4 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.12@example.com> +Content-Description: patch6 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa4-- + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa0 +Content-Type: text/plain; charset="us-ascii" +Content-ID: <20592.1022586929.15@example.com> + +-- +It's never too late to have a happy childhood. + +------- =_aaaaaaaaaa0-- diff --git a/Lib/test/test_email/data/msg_39.txt b/Lib/test/test_email/data/msg_39.txt new file mode 100644 index 0000000..124b269 --- /dev/null +++ b/Lib/test/test_email/data/msg_39.txt @@ -0,0 +1,83 @@ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa0" + +------- =_aaaaaaaaaa0 +Content-Type: multipart/mixed; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.1@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.2@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.3@example.com> +Content-Description: patch1 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.4@example.com> +Content-Description: patch2 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.6@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.7@example.com> +Content-Description: patch3 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.8@example.com> +Content-Description: patch4 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa1 +Content-Type: multipart/alternative; boundary="----- =_aaaaaaaaaa1" +Content-ID: <20592.1022586929.10@example.com> + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.11@example.com> +Content-Description: patch5 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1 +Content-Type: application/octet-stream +Content-ID: <20592.1022586929.12@example.com> +Content-Description: patch6 +Content-Transfer-Encoding: base64 + +XXX + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa1-- + +------- =_aaaaaaaaaa0 +Content-Type: text/plain; charset="us-ascii" +Content-ID: <20592.1022586929.15@example.com> + +-- +It's never too late to have a happy childhood. + +------- =_aaaaaaaaaa0-- diff --git a/Lib/test/test_email/data/msg_40.txt b/Lib/test/test_email/data/msg_40.txt new file mode 100644 index 0000000..1435fa1 --- /dev/null +++ b/Lib/test/test_email/data/msg_40.txt @@ -0,0 +1,10 @@ +MIME-Version: 1.0 +Content-Type: text/html; boundary="--961284236552522269" + +----961284236552522269 +Content-Type: text/html; +Content-Transfer-Encoding: 7Bit + +<html></html> + +----961284236552522269-- diff --git a/Lib/test/test_email/data/msg_41.txt b/Lib/test/test_email/data/msg_41.txt new file mode 100644 index 0000000..76cdd1c --- /dev/null +++ b/Lib/test/test_email/data/msg_41.txt @@ -0,0 +1,8 @@ +From: "Allison Dunlap" <xxx@example.com> +To: yyy@example.com +Subject: 64423 +Date: Sun, 11 Jul 2004 16:09:27 -0300 +MIME-Version: 1.0 +Content-Type: multipart/alternative; + +Blah blah blah diff --git a/Lib/test/test_email/data/msg_42.txt b/Lib/test/test_email/data/msg_42.txt new file mode 100644 index 0000000..a75f8f4 --- /dev/null +++ b/Lib/test/test_email/data/msg_42.txt @@ -0,0 +1,20 @@ +Content-Type: multipart/mixed; boundary="AAA" +From: Mail Delivery Subsystem <xxx@example.com> +To: yyy@example.com + +This is a MIME-encapsulated message + +--AAA + +Stuff + +--AAA +Content-Type: message/rfc822 + +From: webmaster@python.org +To: zzz@example.com +Content-Type: multipart/mixed; boundary="BBB" + +--BBB-- + +--AAA-- diff --git a/Lib/test/test_email/data/msg_43.txt b/Lib/test/test_email/data/msg_43.txt new file mode 100644 index 0000000..797d12c --- /dev/null +++ b/Lib/test/test_email/data/msg_43.txt @@ -0,0 +1,217 @@ +From SRS0=aO/p=ON=bag.python.org=None@bounce2.pobox.com Fri Nov 26 21:40:36 2004 +X-VM-v5-Data: ([nil nil nil nil nil nil nil nil nil] + [nil nil nil nil nil nil nil "MAILER DAEMON <>" "MAILER DAEMON <>" nil nil "Banned file: auto__mail.python.bat in mail from you" "^From:" nil nil nil nil "Banned file: auto__mail.python.bat in mail from you" nil nil nil nil nil nil nil] + nil) +MIME-Version: 1.0 +Message-Id: <edab.7804f5cb8070@python.org> +Content-Type: multipart/report; report-type=delivery-status; + charset=utf-8; + boundary="----------=_1101526904-1956-5" +X-Virus-Scanned: by XS4ALL Virus Scanner +X-UIDL: 4\G!!!<c"!UV["!M7C!! +From: MAILER DAEMON <> +To: <webmaster@python.org> +Subject: Banned file: auto__mail.python.bat in mail from you +Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +This is a multi-part message in MIME format... + +------------=_1101526904-1956-5 +Content-Type: text/plain; charset="utf-8" +Content-Disposition: inline +Content-Transfer-Encoding: 7bit + +BANNED FILENAME ALERT + +Your message to: xxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxxxxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxx@dot.ca.gov, xxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxx@dot.ca.gov, xxxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxx@dot.ca.gov, xxxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxx@dot.ca.gov, xxxxxxx@dot.ca.gov, xxxxxxxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxxx@dot.ca.gov, xxxx@dot.ca.gov, xxxxxxxx@dot.ca.gov, xxxxxxxxxx@dot.ca.gov, xxxxxxxxxxxxxxxxxx@dot.ca.gov +was blocked by our Spam Firewall. The email you sent with the following subject has NOT BEEN DELIVERED: + +Subject: Delivery_failure_notice + +An attachment in that mail was of a file type that the Spam Firewall is set to block. + + + +------------=_1101526904-1956-5 +Content-Type: message/delivery-status +Content-Disposition: inline +Content-Transfer-Encoding: 7bit +Content-Description: Delivery error report + +Reporting-MTA: dns; sacspam01.dot.ca.gov +Received-From-MTA: smtp; sacspam01.dot.ca.gov ([127.0.0.1]) +Arrival-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +Final-Recipient: rfc822; xxxxxxx@dot.ca.gov +Action: failed +Status: 5.7.1 +Diagnostic-Code: smtp; 550 5.7.1 Message content rejected, id=01956-02-2 - BANNED: auto__mail.python.bat +Last-Attempt-Date: Fri, 26 Nov 2004 19:41:44 -0800 (PST) + +------------=_1101526904-1956-5 +Content-Type: text/rfc822-headers +Content-Disposition: inline +Content-Transfer-Encoding: 7bit +Content-Description: Undelivered-message headers + +Received: from kgsav.org (ppp-70-242-162-63.dsl.spfdmo.swbell.net [70.242.162.63]) + by sacspam01.dot.ca.gov (Spam Firewall) with SMTP + id A232AD03DE3A; Fri, 26 Nov 2004 19:41:35 -0800 (PST) +From: webmaster@python.org +To: xxxxx@dot.ca.gov +Date: Sat, 27 Nov 2004 03:35:30 UTC +Subject: Delivery_failure_notice +Importance: Normal +X-Priority: 3 (Normal) +X-MSMail-Priority: Normal +Message-ID: <edab.7804f5cb8070@python.org> +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="====67bd2b7a5.f99f7" +Content-Transfer-Encoding: 7bit + +------------=_1101526904-1956-5-- + diff --git a/Lib/test/test_email/data/msg_44.txt b/Lib/test/test_email/data/msg_44.txt new file mode 100644 index 0000000..15a2252 --- /dev/null +++ b/Lib/test/test_email/data/msg_44.txt @@ -0,0 +1,33 @@ +Return-Path: <barry@python.org> +Delivered-To: barry@python.org +Received: by mail.python.org (Postfix, from userid 889) + id C2BF0D37C6; Tue, 11 Sep 2001 00:05:05 -0400 (EDT) +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary="h90VIIIKmx" +Content-Transfer-Encoding: 7bit +Message-ID: <15261.36209.358846.118674@anthem.python.org> +From: barry@python.org (Barry A. Warsaw) +To: barry@python.org +Subject: a simple multipart +Date: Tue, 11 Sep 2001 00:05:05 -0400 +X-Mailer: VM 6.95 under 21.4 (patch 4) "Artificial Intelligence" XEmacs Lucid +X-Attribution: BAW +X-Oblique-Strategy: Make a door into a window + + +--h90VIIIKmx +Content-Type: text/plain; name="msg.txt" +Content-Transfer-Encoding: 7bit + +a simple kind of mirror +to reflect upon our own + +--h90VIIIKmx +Content-Type: text/plain; name="msg.txt" +Content-Transfer-Encoding: 7bit + +a simple kind of mirror +to reflect upon our own + +--h90VIIIKmx-- + diff --git a/Lib/test/test_email/data/msg_45.txt b/Lib/test/test_email/data/msg_45.txt new file mode 100644 index 0000000..58fde95 --- /dev/null +++ b/Lib/test/test_email/data/msg_45.txt @@ -0,0 +1,33 @@ +From: <foo@bar.baz> +To: <baz@bar.foo> +Subject: test +X-Long-Line: Some really long line contains a lot of text and thus has to be rewrapped because it is some + really long + line +MIME-Version: 1.0 +Content-Type: multipart/signed; boundary="borderline"; + protocol="application/pgp-signature"; micalg=pgp-sha1 + +This is an OpenPGP/MIME signed message (RFC 2440 and 3156) +--borderline +Content-Type: text/plain +X-Long-Line: Another really long line contains a lot of text and thus has to be rewrapped because it is another + really long + line + +This is the signed contents. + +--borderline +Content-Type: application/pgp-signature; name="signature.asc" +Content-Description: OpenPGP digital signature +Content-Disposition: attachment; filename="signature.asc" + +-----BEGIN PGP SIGNATURE----- +Version: GnuPG v2.0.6 (GNU/Linux) + +iD8DBQFG03voRhp6o4m9dFsRApSZAKCCAN3IkJlVRg6NvAiMHlvvIuMGPQCeLZtj +FGwfnRHFBFO/S4/DKysm0lI= +=t7+s +-----END PGP SIGNATURE----- + +--borderline-- diff --git a/Lib/test/test_email/data/msg_46.txt b/Lib/test/test_email/data/msg_46.txt new file mode 100644 index 0000000..1e22c4f --- /dev/null +++ b/Lib/test/test_email/data/msg_46.txt @@ -0,0 +1,23 @@ +Return-Path: <sender@example.net> +Delivery-Date: Mon, 08 Feb 2010 14:05:16 +0100 +Received: from example.org (example.org [64.5.53.58]) + by example.net (node=mxbap2) with ESMTP (Nemesis) + id UNIQUE for someone@example.com; Mon, 08 Feb 2010 14:05:16 +0100 +Date: Mon, 01 Feb 2010 12:21:16 +0100 +From: "Sender" <sender@example.net> +To: <someone@example.com> +Subject: GroupwiseForwardingTest +Mime-Version: 1.0 +Content-Type: message/rfc822 + +Return-path: <sender@example.net> +Message-ID: <4B66B890.4070408@teconcept.de> +Date: Mon, 01 Feb 2010 12:18:40 +0100 +From: "Dr. Sender" <sender@example.net> +MIME-Version: 1.0 +To: "Recipient" <recipient@example.com> +Subject: GroupwiseForwardingTest +Content-Type: text/plain; charset=ISO-8859-15 +Content-Transfer-Encoding: 7bit + +Testing email forwarding with Groupwise 1.2.2010 diff --git a/Lib/test/test_email/test_asian_codecs.py b/Lib/test/test_email/test_asian_codecs.py new file mode 100644 index 0000000..a4dd9a9 --- /dev/null +++ b/Lib/test/test_email/test_asian_codecs.py @@ -0,0 +1,82 @@ +# Copyright (C) 2002-2006 Python Software Foundation +# Contact: email-sig@python.org +# email package unit tests for (optional) Asian codecs + +import unittest +from test.support import run_unittest + +from test.test_email.test_email import TestEmailBase +from email.charset import Charset +from email.header import Header, decode_header +from email.message import Message + +# We're compatible with Python 2.3, but it doesn't have the built-in Asian +# codecs, so we have to skip all these tests. +try: + str(b'foo', 'euc-jp') +except LookupError: + raise unittest.SkipTest + + + +class TestEmailAsianCodecs(TestEmailBase): + def test_japanese_codecs(self): + eq = self.ndiffAssertEqual + jcode = "euc-jp" + gcode = "iso-8859-1" + j = Charset(jcode) + g = Charset(gcode) + h = Header("Hello World!") + jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' + b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode) + ghello = str(b'Gr\xfc\xdf Gott!', gcode) + h.append(jhello, j) + h.append(ghello, g) + # BAW: This used to -- and maybe should -- fold the two iso-8859-1 + # chunks into a single encoded word. However it doesn't violate the + # standard to have them as two encoded chunks and maybe it's + # reasonable <wink> for each .append() call to result in a separate + # encoded word. + eq(h.encode(), """\ +Hello World! =?iso-2022-jp?b?GyRCJU8lbSE8JW8hPCVrJUkhKhsoQg==?= + =?iso-8859-1?q?Gr=FC=DF_Gott!?=""") + eq(decode_header(h.encode()), + [(b'Hello World!', None), + (b'\x1b$B%O%m!<%o!<%k%I!*\x1b(B', 'iso-2022-jp'), + (b'Gr\xfc\xdf Gott!', gcode)]) + subject_bytes = (b'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5' + b'\xa4\xec\xa4\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2' + b'\xf1\xbc\xd4\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3' + b'\xa4\xc6\xa4\xa4\xa4\xde\xa4\xb9') + subject = str(subject_bytes, jcode) + h = Header(subject, j, header_name="Subject") + # test a very long header + enc = h.encode() + # TK: splitting point may differ by codec design and/or Header encoding + eq(enc , """\ +=?iso-2022-jp?b?dGVzdC1qYSAbJEIkWEVqOUYkNSRsJD8lYSE8JWskTztKGyhC?= + =?iso-2022-jp?b?GyRCMnE8VCROPjVHJyRyQlQkQyRGJCQkXiQ5GyhC?=""") + # TK: full decode comparison + eq(str(h).encode(jcode), subject_bytes) + + def test_payload_encoding_utf8(self): + jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' + b'\xa5\xeb\xa5\xc9\xa1\xaa', 'euc-jp') + msg = Message() + msg.set_payload(jhello, 'utf-8') + ustr = msg.get_payload(decode=True).decode(msg.get_content_charset()) + self.assertEqual(jhello, ustr) + + def test_payload_encoding(self): + jcode = 'euc-jp' + jhello = str(b'\xa5\xcf\xa5\xed\xa1\xbc\xa5\xef\xa1\xbc' + b'\xa5\xeb\xa5\xc9\xa1\xaa', jcode) + msg = Message() + msg.set_payload(jhello, jcode) + ustr = msg.get_payload(decode=True).decode(msg.get_content_charset()) + self.assertEqual(jhello, ustr) + + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_email.py b/Lib/test/test_email/test_email.py new file mode 100644 index 0000000..121c939 --- /dev/null +++ b/Lib/test/test_email/test_email.py @@ -0,0 +1,4828 @@ +# Copyright (C) 2001-2010 Python Software Foundation +# Contact: email-sig@python.org +# email package unit tests + +import os +import re +import sys +import time +import base64 +import difflib +import unittest +import warnings +import textwrap + +from io import StringIO, BytesIO +from itertools import chain + +import email + +from email.charset import Charset +from email.header import Header, decode_header, make_header +from email.parser import Parser, HeaderParser +from email.generator import Generator, DecodedGenerator +from email.message import Message +from email.mime.application import MIMEApplication +from email.mime.audio import MIMEAudio +from email.mime.text import MIMEText +from email.mime.image import MIMEImage +from email.mime.base import MIMEBase +from email.mime.message import MIMEMessage +from email.mime.multipart import MIMEMultipart +from email import utils +from email import errors +from email import encoders +from email import iterators +from email import base64mime +from email import quoprimime + +from test.support import run_unittest, unlink +from test.test_email import openfile, TestEmailBase + +NL = '\n' +EMPTYSTRING = '' +SPACE = ' ' + + +# Test various aspects of the Message class's API +class TestMessageAPI(TestEmailBase): + def test_get_all(self): + eq = self.assertEqual + msg = self._msgobj('msg_20.txt') + eq(msg.get_all('cc'), ['ccc@zzz.org', 'ddd@zzz.org', 'eee@zzz.org']) + eq(msg.get_all('xx', 'n/a'), 'n/a') + + def test_getset_charset(self): + eq = self.assertEqual + msg = Message() + eq(msg.get_charset(), None) + charset = Charset('iso-8859-1') + msg.set_charset(charset) + eq(msg['mime-version'], '1.0') + eq(msg.get_content_type(), 'text/plain') + eq(msg['content-type'], 'text/plain; charset="iso-8859-1"') + eq(msg.get_param('charset'), 'iso-8859-1') + eq(msg['content-transfer-encoding'], 'quoted-printable') + eq(msg.get_charset().input_charset, 'iso-8859-1') + # Remove the charset + msg.set_charset(None) + eq(msg.get_charset(), None) + eq(msg['content-type'], 'text/plain') + # Try adding a charset when there's already MIME headers present + msg = Message() + msg['MIME-Version'] = '2.0' + msg['Content-Type'] = 'text/x-weird' + msg['Content-Transfer-Encoding'] = 'quinted-puntable' + msg.set_charset(charset) + eq(msg['mime-version'], '2.0') + eq(msg['content-type'], 'text/x-weird; charset="iso-8859-1"') + eq(msg['content-transfer-encoding'], 'quinted-puntable') + + def test_set_charset_from_string(self): + eq = self.assertEqual + msg = Message() + msg.set_charset('us-ascii') + eq(msg.get_charset().input_charset, 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + + def test_set_payload_with_charset(self): + msg = Message() + charset = Charset('iso-8859-1') + msg.set_payload('This is a string payload', charset) + self.assertEqual(msg.get_charset().input_charset, 'iso-8859-1') + + def test_get_charsets(self): + eq = self.assertEqual + + msg = self._msgobj('msg_08.txt') + charsets = msg.get_charsets() + eq(charsets, [None, 'us-ascii', 'iso-8859-1', 'iso-8859-2', 'koi8-r']) + + msg = self._msgobj('msg_09.txt') + charsets = msg.get_charsets('dingbat') + eq(charsets, ['dingbat', 'us-ascii', 'iso-8859-1', 'dingbat', + 'koi8-r']) + + msg = self._msgobj('msg_12.txt') + charsets = msg.get_charsets() + eq(charsets, [None, 'us-ascii', 'iso-8859-1', None, 'iso-8859-2', + 'iso-8859-3', 'us-ascii', 'koi8-r']) + + def test_get_filename(self): + eq = self.assertEqual + + msg = self._msgobj('msg_04.txt') + filenames = [p.get_filename() for p in msg.get_payload()] + eq(filenames, ['msg.txt', 'msg.txt']) + + msg = self._msgobj('msg_07.txt') + subpart = msg.get_payload(1) + eq(subpart.get_filename(), 'dingusfish.gif') + + def test_get_filename_with_name_parameter(self): + eq = self.assertEqual + + msg = self._msgobj('msg_44.txt') + filenames = [p.get_filename() for p in msg.get_payload()] + eq(filenames, ['msg.txt', 'msg.txt']) + + def test_get_boundary(self): + eq = self.assertEqual + msg = self._msgobj('msg_07.txt') + # No quotes! + eq(msg.get_boundary(), 'BOUNDARY') + + def test_set_boundary(self): + eq = self.assertEqual + # This one has no existing boundary parameter, but the Content-Type: + # header appears fifth. + msg = self._msgobj('msg_01.txt') + msg.set_boundary('BOUNDARY') + header, value = msg.items()[4] + eq(header.lower(), 'content-type') + eq(value, 'text/plain; charset="us-ascii"; boundary="BOUNDARY"') + # This one has a Content-Type: header, with a boundary, stuck in the + # middle of its headers. Make sure the order is preserved; it should + # be fifth. + msg = self._msgobj('msg_04.txt') + msg.set_boundary('BOUNDARY') + header, value = msg.items()[4] + eq(header.lower(), 'content-type') + eq(value, 'multipart/mixed; boundary="BOUNDARY"') + # And this one has no Content-Type: header at all. + msg = self._msgobj('msg_03.txt') + self.assertRaises(errors.HeaderParseError, + msg.set_boundary, 'BOUNDARY') + + def test_make_boundary(self): + msg = MIMEMultipart('form-data') + # Note that when the boundary gets created is an implementation + # detail and might change. + self.assertEqual(msg.items()[0][1], 'multipart/form-data') + # Trigger creation of boundary + msg.as_string() + self.assertEqual(msg.items()[0][1][:33], + 'multipart/form-data; boundary="==') + # XXX: there ought to be tests of the uniqueness of the boundary, too. + + def test_message_rfc822_only(self): + # Issue 7970: message/rfc822 not in multipart parsed by + # HeaderParser caused an exception when flattened. + with openfile('msg_46.txt') as fp: + msgdata = fp.read() + parser = HeaderParser() + msg = parser.parsestr(msgdata) + out = StringIO() + gen = Generator(out, True, 0) + gen.flatten(msg, False) + self.assertEqual(out.getvalue(), msgdata) + + def test_byte_message_rfc822_only(self): + # Make sure new bytes header parser also passes this. + with openfile('msg_46.txt', 'rb') as fp: + msgdata = fp.read() + parser = email.parser.BytesHeaderParser() + msg = parser.parsebytes(msgdata) + out = BytesIO() + gen = email.generator.BytesGenerator(out) + gen.flatten(msg) + self.assertEqual(out.getvalue(), msgdata) + + def test_get_decoded_payload(self): + eq = self.assertEqual + msg = self._msgobj('msg_10.txt') + # The outer message is a multipart + eq(msg.get_payload(decode=True), None) + # Subpart 1 is 7bit encoded + eq(msg.get_payload(0).get_payload(decode=True), + b'This is a 7bit encoded message.\n') + # Subpart 2 is quopri + eq(msg.get_payload(1).get_payload(decode=True), + b'\xa1This is a Quoted Printable encoded message!\n') + # Subpart 3 is base64 + eq(msg.get_payload(2).get_payload(decode=True), + b'This is a Base64 encoded message.') + # Subpart 4 is base64 with a trailing newline, which + # used to be stripped (issue 7143). + eq(msg.get_payload(3).get_payload(decode=True), + b'This is a Base64 encoded message.\n') + # Subpart 5 has no Content-Transfer-Encoding: header. + eq(msg.get_payload(4).get_payload(decode=True), + b'This has no Content-Transfer-Encoding: header.\n') + + def test_get_decoded_uu_payload(self): + eq = self.assertEqual + msg = Message() + msg.set_payload('begin 666 -\n+:&5L;&\\@=V]R;&0 \n \nend\n') + for cte in ('x-uuencode', 'uuencode', 'uue', 'x-uue'): + msg['content-transfer-encoding'] = cte + eq(msg.get_payload(decode=True), b'hello world') + # Now try some bogus data + msg.set_payload('foo') + eq(msg.get_payload(decode=True), b'foo') + + def test_get_payload_n_raises_on_non_multipart(self): + msg = Message() + self.assertRaises(TypeError, msg.get_payload, 1) + + def test_decoded_generator(self): + eq = self.assertEqual + msg = self._msgobj('msg_07.txt') + with openfile('msg_17.txt') as fp: + text = fp.read() + s = StringIO() + g = DecodedGenerator(s) + g.flatten(msg) + eq(s.getvalue(), text) + + def test__contains__(self): + msg = Message() + msg['From'] = 'Me' + msg['to'] = 'You' + # Check for case insensitivity + self.assertTrue('from' in msg) + self.assertTrue('From' in msg) + self.assertTrue('FROM' in msg) + self.assertTrue('to' in msg) + self.assertTrue('To' in msg) + self.assertTrue('TO' in msg) + + def test_as_string(self): + eq = self.ndiffAssertEqual + msg = self._msgobj('msg_01.txt') + with openfile('msg_01.txt') as fp: + text = fp.read() + eq(text, str(msg)) + fullrepr = msg.as_string(unixfrom=True) + lines = fullrepr.split('\n') + self.assertTrue(lines[0].startswith('From ')) + eq(text, NL.join(lines[1:])) + + def test_bad_param(self): + msg = email.message_from_string("Content-Type: blarg; baz; boo\n") + self.assertEqual(msg.get_param('baz'), '') + + def test_missing_filename(self): + msg = email.message_from_string("From: foo\n") + self.assertEqual(msg.get_filename(), None) + + def test_bogus_filename(self): + msg = email.message_from_string( + "Content-Disposition: blarg; filename\n") + self.assertEqual(msg.get_filename(), '') + + def test_missing_boundary(self): + msg = email.message_from_string("From: foo\n") + self.assertEqual(msg.get_boundary(), None) + + def test_get_params(self): + eq = self.assertEqual + msg = email.message_from_string( + 'X-Header: foo=one; bar=two; baz=three\n') + eq(msg.get_params(header='x-header'), + [('foo', 'one'), ('bar', 'two'), ('baz', 'three')]) + msg = email.message_from_string( + 'X-Header: foo; bar=one; baz=two\n') + eq(msg.get_params(header='x-header'), + [('foo', ''), ('bar', 'one'), ('baz', 'two')]) + eq(msg.get_params(), None) + msg = email.message_from_string( + 'X-Header: foo; bar="one"; baz=two\n') + eq(msg.get_params(header='x-header'), + [('foo', ''), ('bar', 'one'), ('baz', 'two')]) + + def test_get_param_liberal(self): + msg = Message() + msg['Content-Type'] = 'Content-Type: Multipart/mixed; boundary = "CPIMSSMTPC06p5f3tG"' + self.assertEqual(msg.get_param('boundary'), 'CPIMSSMTPC06p5f3tG') + + def test_get_param(self): + eq = self.assertEqual + msg = email.message_from_string( + "X-Header: foo=one; bar=two; baz=three\n") + eq(msg.get_param('bar', header='x-header'), 'two') + eq(msg.get_param('quuz', header='x-header'), None) + eq(msg.get_param('quuz'), None) + msg = email.message_from_string( + 'X-Header: foo; bar="one"; baz=two\n') + eq(msg.get_param('foo', header='x-header'), '') + eq(msg.get_param('bar', header='x-header'), 'one') + eq(msg.get_param('baz', header='x-header'), 'two') + # XXX: We are not RFC-2045 compliant! We cannot parse: + # msg["Content-Type"] = 'text/plain; weird="hey; dolly? [you] @ <\\"home\\">?"' + # msg.get_param("weird") + # yet. + + def test_get_param_funky_continuation_lines(self): + msg = self._msgobj('msg_22.txt') + self.assertEqual(msg.get_payload(1).get_param('name'), 'wibble.JPG') + + def test_get_param_with_semis_in_quotes(self): + msg = email.message_from_string( + 'Content-Type: image/pjpeg; name="Jim&&Jill"\n') + self.assertEqual(msg.get_param('name'), 'Jim&&Jill') + self.assertEqual(msg.get_param('name', unquote=False), + '"Jim&&Jill"') + + def test_get_param_with_quotes(self): + msg = email.message_from_string( + 'Content-Type: foo; bar*0="baz\\"foobar"; bar*1="\\"baz"') + self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') + msg = email.message_from_string( + "Content-Type: foo; bar*0=\"baz\\\"foobar\"; bar*1=\"\\\"baz\"") + self.assertEqual(msg.get_param('bar'), 'baz"foobar"baz') + + def test_field_containment(self): + unless = self.assertTrue + msg = email.message_from_string('Header: exists') + unless('header' in msg) + unless('Header' in msg) + unless('HEADER' in msg) + self.assertFalse('headerx' in msg) + + def test_set_param(self): + eq = self.assertEqual + msg = Message() + msg.set_param('charset', 'iso-2022-jp') + eq(msg.get_param('charset'), 'iso-2022-jp') + msg.set_param('importance', 'high value') + eq(msg.get_param('importance'), 'high value') + eq(msg.get_param('importance', unquote=False), '"high value"') + eq(msg.get_params(), [('text/plain', ''), + ('charset', 'iso-2022-jp'), + ('importance', 'high value')]) + eq(msg.get_params(unquote=False), [('text/plain', ''), + ('charset', '"iso-2022-jp"'), + ('importance', '"high value"')]) + msg.set_param('charset', 'iso-9999-xx', header='X-Jimmy') + eq(msg.get_param('charset', header='X-Jimmy'), 'iso-9999-xx') + + def test_del_param(self): + eq = self.assertEqual + msg = self._msgobj('msg_05.txt') + eq(msg.get_params(), + [('multipart/report', ''), ('report-type', 'delivery-status'), + ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) + old_val = msg.get_param("report-type") + msg.del_param("report-type") + eq(msg.get_params(), + [('multipart/report', ''), + ('boundary', 'D1690A7AC1.996856090/mail.example.com')]) + msg.set_param("report-type", old_val) + eq(msg.get_params(), + [('multipart/report', ''), + ('boundary', 'D1690A7AC1.996856090/mail.example.com'), + ('report-type', old_val)]) + + def test_del_param_on_other_header(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', filename='bud.gif') + msg.del_param('filename', 'content-disposition') + self.assertEqual(msg['content-disposition'], 'attachment') + + def test_del_param_on_nonexistent_header(self): + msg = Message() + msg.del_param('filename', 'content-disposition') + + def test_del_nonexistent_param(self): + msg = Message() + msg.add_header('Content-Type', 'text/plain', charset='utf-8') + existing_header = msg['Content-Type'] + msg.del_param('foobar', header='Content-Type') + self.assertEqual(msg['Content-Type'], 'text/plain; charset="utf-8"') + + def test_set_type(self): + eq = self.assertEqual + msg = Message() + self.assertRaises(ValueError, msg.set_type, 'text') + msg.set_type('text/plain') + eq(msg['content-type'], 'text/plain') + msg.set_param('charset', 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + msg.set_type('text/html') + eq(msg['content-type'], 'text/html; charset="us-ascii"') + + def test_set_type_on_other_header(self): + msg = Message() + msg['X-Content-Type'] = 'text/plain' + msg.set_type('application/octet-stream', 'X-Content-Type') + self.assertEqual(msg['x-content-type'], 'application/octet-stream') + + def test_get_content_type_missing(self): + msg = Message() + self.assertEqual(msg.get_content_type(), 'text/plain') + + def test_get_content_type_missing_with_default_type(self): + msg = Message() + msg.set_default_type('message/rfc822') + self.assertEqual(msg.get_content_type(), 'message/rfc822') + + def test_get_content_type_from_message_implicit(self): + msg = self._msgobj('msg_30.txt') + self.assertEqual(msg.get_payload(0).get_content_type(), + 'message/rfc822') + + def test_get_content_type_from_message_explicit(self): + msg = self._msgobj('msg_28.txt') + self.assertEqual(msg.get_payload(0).get_content_type(), + 'message/rfc822') + + def test_get_content_type_from_message_text_plain_implicit(self): + msg = self._msgobj('msg_03.txt') + self.assertEqual(msg.get_content_type(), 'text/plain') + + def test_get_content_type_from_message_text_plain_explicit(self): + msg = self._msgobj('msg_01.txt') + self.assertEqual(msg.get_content_type(), 'text/plain') + + def test_get_content_maintype_missing(self): + msg = Message() + self.assertEqual(msg.get_content_maintype(), 'text') + + def test_get_content_maintype_missing_with_default_type(self): + msg = Message() + msg.set_default_type('message/rfc822') + self.assertEqual(msg.get_content_maintype(), 'message') + + def test_get_content_maintype_from_message_implicit(self): + msg = self._msgobj('msg_30.txt') + self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') + + def test_get_content_maintype_from_message_explicit(self): + msg = self._msgobj('msg_28.txt') + self.assertEqual(msg.get_payload(0).get_content_maintype(), 'message') + + def test_get_content_maintype_from_message_text_plain_implicit(self): + msg = self._msgobj('msg_03.txt') + self.assertEqual(msg.get_content_maintype(), 'text') + + def test_get_content_maintype_from_message_text_plain_explicit(self): + msg = self._msgobj('msg_01.txt') + self.assertEqual(msg.get_content_maintype(), 'text') + + def test_get_content_subtype_missing(self): + msg = Message() + self.assertEqual(msg.get_content_subtype(), 'plain') + + def test_get_content_subtype_missing_with_default_type(self): + msg = Message() + msg.set_default_type('message/rfc822') + self.assertEqual(msg.get_content_subtype(), 'rfc822') + + def test_get_content_subtype_from_message_implicit(self): + msg = self._msgobj('msg_30.txt') + self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') + + def test_get_content_subtype_from_message_explicit(self): + msg = self._msgobj('msg_28.txt') + self.assertEqual(msg.get_payload(0).get_content_subtype(), 'rfc822') + + def test_get_content_subtype_from_message_text_plain_implicit(self): + msg = self._msgobj('msg_03.txt') + self.assertEqual(msg.get_content_subtype(), 'plain') + + def test_get_content_subtype_from_message_text_plain_explicit(self): + msg = self._msgobj('msg_01.txt') + self.assertEqual(msg.get_content_subtype(), 'plain') + + def test_get_content_maintype_error(self): + msg = Message() + msg['Content-Type'] = 'no-slash-in-this-string' + self.assertEqual(msg.get_content_maintype(), 'text') + + def test_get_content_subtype_error(self): + msg = Message() + msg['Content-Type'] = 'no-slash-in-this-string' + self.assertEqual(msg.get_content_subtype(), 'plain') + + def test_replace_header(self): + eq = self.assertEqual + msg = Message() + msg.add_header('First', 'One') + msg.add_header('Second', 'Two') + msg.add_header('Third', 'Three') + eq(msg.keys(), ['First', 'Second', 'Third']) + eq(msg.values(), ['One', 'Two', 'Three']) + msg.replace_header('Second', 'Twenty') + eq(msg.keys(), ['First', 'Second', 'Third']) + eq(msg.values(), ['One', 'Twenty', 'Three']) + msg.add_header('First', 'Eleven') + msg.replace_header('First', 'One Hundred') + eq(msg.keys(), ['First', 'Second', 'Third', 'First']) + eq(msg.values(), ['One Hundred', 'Twenty', 'Three', 'Eleven']) + self.assertRaises(KeyError, msg.replace_header, 'Fourth', 'Missing') + + def test_broken_base64_payload(self): + x = 'AwDp0P7//y6LwKEAcPa/6Q=9' + msg = Message() + msg['content-type'] = 'audio/x-midi' + msg['content-transfer-encoding'] = 'base64' + msg.set_payload(x) + self.assertEqual(msg.get_payload(decode=True), + bytes(x, 'raw-unicode-escape')) + + def test_broken_unicode_payload(self): + # This test improves coverage but is not a compliance test. + # The behavior in this situation is currently undefined by the API. + x = 'this is a br\xf6ken thing to do' + msg = Message() + msg['content-type'] = 'text/plain' + msg['content-transfer-encoding'] = '8bit' + msg.set_payload(x) + self.assertEqual(msg.get_payload(decode=True), + bytes(x, 'raw-unicode-escape')) + + def test_questionable_bytes_payload(self): + # This test improves coverage but is not a compliance test, + # since it involves poking inside the black box. + x = 'this is a quéstionable thing to do'.encode('utf-8') + msg = Message() + msg['content-type'] = 'text/plain; charset="utf-8"' + msg['content-transfer-encoding'] = '8bit' + msg._payload = x + self.assertEqual(msg.get_payload(decode=True), x) + + # Issue 1078919 + def test_ascii_add_header(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename='bud.gif') + self.assertEqual('attachment; filename="bud.gif"', + msg['Content-Disposition']) + + def test_noascii_add_header(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename="Fußballer.ppt") + self.assertEqual( + 'attachment; filename*=utf-8\'\'Fu%C3%9Fballer.ppt', + msg['Content-Disposition']) + + def test_nonascii_add_header_via_triple(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename=('iso-8859-1', '', 'Fußballer.ppt')) + self.assertEqual( + 'attachment; filename*=iso-8859-1\'\'Fu%DFballer.ppt', + msg['Content-Disposition']) + + def test_ascii_add_header_with_tspecial(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename="windows [filename].ppt") + self.assertEqual( + 'attachment; filename="windows [filename].ppt"', + msg['Content-Disposition']) + + def test_nonascii_add_header_with_tspecial(self): + msg = Message() + msg.add_header('Content-Disposition', 'attachment', + filename="Fußballer [filename].ppt") + self.assertEqual( + "attachment; filename*=utf-8''Fu%C3%9Fballer%20%5Bfilename%5D.ppt", + msg['Content-Disposition']) + + def test_add_header_with_name_only_param(self): + msg = Message() + msg.add_header('Content-Disposition', 'inline', foo_bar=None) + self.assertEqual("inline; foo-bar", msg['Content-Disposition']) + + def test_add_header_with_no_value(self): + msg = Message() + msg.add_header('X-Status', None) + self.assertEqual('', msg['X-Status']) + + # Issue 5871: reject an attempt to embed a header inside a header value + # (header injection attack). + def test_embeded_header_via_Header_rejected(self): + msg = Message() + msg['Dummy'] = Header('dummy\nX-Injected-Header: test') + self.assertRaises(errors.HeaderParseError, msg.as_string) + + def test_embeded_header_via_string_rejected(self): + msg = Message() + msg['Dummy'] = 'dummy\nX-Injected-Header: test' + self.assertRaises(errors.HeaderParseError, msg.as_string) + +# Test the email.encoders module +class TestEncoders(unittest.TestCase): + + def test_EncodersEncode_base64(self): + with openfile('PyBanner048.gif', 'rb') as fp: + bindata = fp.read() + mimed = email.mime.image.MIMEImage(bindata) + base64ed = mimed.get_payload() + # the transfer-encoded body lines should all be <=76 characters + lines = base64ed.split('\n') + self.assertLessEqual(max([ len(x) for x in lines ]), 76) + + def test_encode_empty_payload(self): + eq = self.assertEqual + msg = Message() + msg.set_charset('us-ascii') + eq(msg['content-transfer-encoding'], '7bit') + + def test_default_cte(self): + eq = self.assertEqual + # 7bit data and the default us-ascii _charset + msg = MIMEText('hello world') + eq(msg['content-transfer-encoding'], '7bit') + # Similar, but with 8bit data + msg = MIMEText('hello \xf8 world') + eq(msg['content-transfer-encoding'], '8bit') + # And now with a different charset + msg = MIMEText('hello \xf8 world', _charset='iso-8859-1') + eq(msg['content-transfer-encoding'], 'quoted-printable') + + def test_encode7or8bit(self): + # Make sure a charset whose input character set is 8bit but + # whose output character set is 7bit gets a transfer-encoding + # of 7bit. + eq = self.assertEqual + msg = MIMEText('文', _charset='euc-jp') + eq(msg['content-transfer-encoding'], '7bit') + + +# Test long header wrapping +class TestLongHeaders(TestEmailBase): + + maxDiff = None + + def test_split_long_continuation(self): + eq = self.ndiffAssertEqual + msg = email.message_from_string("""\ +Subject: bug demonstration +\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 +\tmore text + +test +""") + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +Subject: bug demonstration +\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 +\tmore text + +test +""") + + def test_another_long_almost_unsplittable_header(self): + eq = self.ndiffAssertEqual + hstr = """\ +bug demonstration +\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 +\tmore text""" + h = Header(hstr, continuation_ws='\t') + eq(h.encode(), """\ +bug demonstration +\t12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 +\tmore text""") + h = Header(hstr.replace('\t', ' ')) + eq(h.encode(), """\ +bug demonstration + 12345678911234567892123456789312345678941234567895123456789612345678971234567898112345678911234567892123456789112345678911234567892123456789 + more text""") + + def test_long_nonstring(self): + eq = self.ndiffAssertEqual + g = Charset("iso-8859-1") + cz = Charset("iso-8859-2") + utf8 = Charset("utf-8") + g_head = (b'Die Mieter treten hier ein werden mit einem Foerderband ' + b'komfortabel den Korridor entlang, an s\xfcdl\xfcndischen ' + b'Wandgem\xe4lden vorbei, gegen die rotierenden Klingen ' + b'bef\xf6rdert. ') + cz_head = (b'Finan\xe8ni metropole se hroutily pod tlakem jejich ' + b'd\xf9vtipu.. ') + utf8_head = ('\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f' + '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00' + '\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c' + '\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067' + '\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das ' + 'Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder ' + 'die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066' + '\u3044\u307e\u3059\u3002') + h = Header(g_head, g, header_name='Subject') + h.append(cz_head, cz) + h.append(utf8_head, utf8) + msg = Message() + msg['Subject'] = h + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +Subject: =?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerderb?= + =?iso-8859-1?q?and_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndischen?= + =?iso-8859-1?q?_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Klingen_bef?= + =?iso-8859-1?q?=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_metropole_se_hrouti?= + =?iso-8859-2?q?ly_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= =?utf-8?b?5q2j56K6?= + =?utf-8?b?44Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE44G+44Gb44KT44CC5LiA?= + =?utf-8?b?6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB44GC44Go44Gv44Gn44Gf44KJ?= + =?utf-8?b?44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CMV2VubiBpc3QgZGFzIE51bnN0dWNr?= + =?utf-8?b?IGdpdCB1bmQgU2xvdGVybWV5ZXI/IEphISBCZWloZXJodW5kIGRhcyBPZGVyIGRp?= + =?utf-8?b?ZSBGbGlwcGVyd2FsZHQgZ2Vyc3B1dC7jgI3jgajoqIDjgaPjgabjgYTjgb7jgZk=?= + =?utf-8?b?44CC?= + +""") + eq(h.encode(maxlinelen=76), """\ +=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerde?= + =?iso-8859-1?q?rband_komfortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndis?= + =?iso-8859-1?q?chen_Wandgem=E4lden_vorbei=2C_gegen_die_rotierenden_Klinge?= + =?iso-8859-1?q?n_bef=F6rdert=2E_?= =?iso-8859-2?q?Finan=E8ni_metropole_se?= + =?iso-8859-2?q?_hroutily_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= + =?utf-8?b?5q2j56K644Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE44G+44Gb?= + =?utf-8?b?44KT44CC5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB44GC44Go?= + =?utf-8?b?44Gv44Gn44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CMV2VubiBp?= + =?utf-8?b?c3QgZGFzIE51bnN0dWNrIGdpdCB1bmQgU2xvdGVybWV5ZXI/IEphISBCZWlo?= + =?utf-8?b?ZXJodW5kIGRhcyBPZGVyIGRpZSBGbGlwcGVyd2FsZHQgZ2Vyc3B1dC7jgI0=?= + =?utf-8?b?44Go6KiA44Gj44Gm44GE44G+44GZ44CC?=""") + + def test_long_header_encode(self): + eq = self.ndiffAssertEqual + h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' + 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', + header_name='X-Foobar-Spoink-Defrobnit') + eq(h.encode(), '''\ +wasnipoop; giraffes="very-long-necked-animals"; + spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') + + def test_long_header_encode_with_tab_continuation_is_just_a_hint(self): + eq = self.ndiffAssertEqual + h = Header('wasnipoop; giraffes="very-long-necked-animals"; ' + 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', + header_name='X-Foobar-Spoink-Defrobnit', + continuation_ws='\t') + eq(h.encode(), '''\ +wasnipoop; giraffes="very-long-necked-animals"; + spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') + + def test_long_header_encode_with_tab_continuation(self): + eq = self.ndiffAssertEqual + h = Header('wasnipoop; giraffes="very-long-necked-animals";\t' + 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"', + header_name='X-Foobar-Spoink-Defrobnit', + continuation_ws='\t') + eq(h.encode(), '''\ +wasnipoop; giraffes="very-long-necked-animals"; +\tspooge="yummy"; hippos="gargantuan"; marshmallows="gooey"''') + + def test_header_encode_with_different_output_charset(self): + h = Header('文', 'euc-jp') + self.assertEqual(h.encode(), "=?iso-2022-jp?b?GyRCSjgbKEI=?=") + + def test_long_header_encode_with_different_output_charset(self): + h = Header(b'test-ja \xa4\xd8\xc5\xea\xb9\xc6\xa4\xb5\xa4\xec\xa4' + b'\xbf\xa5\xe1\xa1\xbc\xa5\xeb\xa4\xcf\xbb\xca\xb2\xf1\xbc\xd4' + b'\xa4\xce\xbe\xb5\xc7\xa7\xa4\xf2\xc2\xd4\xa4\xc3\xa4\xc6\xa4' + b'\xa4\xa4\xde\xa4\xb9'.decode('euc-jp'), 'euc-jp') + res = """\ +=?iso-2022-jp?b?dGVzdC1qYSAbJEIkWEVqOUYkNSRsJD8lYSE8JWskTztKMnE8VCROPjUbKEI=?= + =?iso-2022-jp?b?GyRCRyckckJUJEMkRiQkJF4kORsoQg==?=""" + self.assertEqual(h.encode(), res) + + def test_header_splitter(self): + eq = self.ndiffAssertEqual + msg = MIMEText('') + # It'd be great if we could use add_header() here, but that doesn't + # guarantee an order of the parameters. + msg['X-Foobar-Spoink-Defrobnit'] = ( + 'wasnipoop; giraffes="very-long-necked-animals"; ' + 'spooge="yummy"; hippos="gargantuan"; marshmallows="gooey"') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), '''\ +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +X-Foobar-Spoink-Defrobnit: wasnipoop; giraffes="very-long-necked-animals"; + spooge="yummy"; hippos="gargantuan"; marshmallows="gooey" + +''') + + def test_no_semis_header_splitter(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'test@dom.ain' + msg['References'] = SPACE.join('<%d@dom.ain>' % i for i in range(10)) + msg.set_payload('Test') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +From: test@dom.ain +References: <0@dom.ain> <1@dom.ain> <2@dom.ain> <3@dom.ain> <4@dom.ain> + <5@dom.ain> <6@dom.ain> <7@dom.ain> <8@dom.ain> <9@dom.ain> + +Test""") + + def test_last_split_chunk_does_not_fit(self): + eq = self.ndiffAssertEqual + h = Header('Subject: the first part of this is short, but_the_second' + '_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line' + '_all_by_itself') + eq(h.encode(), """\ +Subject: the first part of this is short, + but_the_second_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself""") + + def test_splittable_leading_char_followed_by_overlong_unsplitable(self): + eq = self.ndiffAssertEqual + h = Header(', but_the_second' + '_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line' + '_all_by_itself') + eq(h.encode(), """\ +, + but_the_second_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself""") + + def test_multiple_splittable_leading_char_followed_by_overlong_unsplitable(self): + eq = self.ndiffAssertEqual + h = Header(', , but_the_second' + '_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line' + '_all_by_itself') + eq(h.encode(), """\ +, , + but_the_second_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself""") + + def test_trailing_splitable_on_overlong_unsplitable(self): + eq = self.ndiffAssertEqual + h = Header('this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself;') + eq(h.encode(), "this_part_does_not_fit_within_maxlinelen_and_thus_should_" + "be_on_a_line_all_by_itself;") + + def test_trailing_splitable_on_overlong_unsplitable_with_leading_splitable(self): + eq = self.ndiffAssertEqual + h = Header('; ' + 'this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself; ') + eq(h.encode(), """\ +; + this_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself; """) + + def test_long_header_with_multiple_sequential_split_chars(self): + eq = self.ndiffAssertEqual + h = Header('This is a long line that has two whitespaces in a row. ' + 'This used to cause truncation of the header when folded') + eq(h.encode(), """\ +This is a long line that has two whitespaces in a row. This used to cause + truncation of the header when folded""") + + def test_splitter_split_on_punctuation_only_if_fws(self): + eq = self.ndiffAssertEqual + h = Header('thisverylongheaderhas;semicolons;and,commas,but' + 'they;arenotlegal;fold,points') + eq(h.encode(), "thisverylongheaderhas;semicolons;and,commas,butthey;" + "arenotlegal;fold,points") + + def test_leading_splittable_in_the_middle_just_before_overlong_last_part(self): + eq = self.ndiffAssertEqual + h = Header('this is a test where we need to have more than one line ' + 'before; our final line that is just too big to fit;; ' + 'this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself;') + eq(h.encode(), """\ +this is a test where we need to have more than one line before; + our final line that is just too big to fit;; + this_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself;""") + + def test_overlong_last_part_followed_by_split_point(self): + eq = self.ndiffAssertEqual + h = Header('this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself ') + eq(h.encode(), "this_part_does_not_fit_within_maxlinelen_and_thus_" + "should_be_on_a_line_all_by_itself ") + + def test_multiline_with_overlong_parts_separated_by_two_split_points(self): + eq = self.ndiffAssertEqual + h = Header('this_is_a__test_where_we_need_to_have_more_than_one_line_' + 'before_our_final_line_; ; ' + 'this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself; ') + eq(h.encode(), """\ +this_is_a__test_where_we_need_to_have_more_than_one_line_before_our_final_line_; + ; + this_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself; """) + + def test_multiline_with_overlong_last_part_followed_by_split_point(self): + eq = self.ndiffAssertEqual + h = Header('this is a test where we need to have more than one line ' + 'before our final line; ; ' + 'this_part_does_not_fit_within_maxlinelen_and_thus_should_' + 'be_on_a_line_all_by_itself; ') + eq(h.encode(), """\ +this is a test where we need to have more than one line before our final line; + ; + this_part_does_not_fit_within_maxlinelen_and_thus_should_be_on_a_line_all_by_itself; """) + + def test_long_header_with_whitespace_runs(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'test@dom.ain' + msg['References'] = SPACE.join(['<foo@dom.ain> '] * 10) + msg.set_payload('Test') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +From: test@dom.ain +References: <foo@dom.ain> <foo@dom.ain> <foo@dom.ain> <foo@dom.ain> + <foo@dom.ain> <foo@dom.ain> <foo@dom.ain> <foo@dom.ain> + <foo@dom.ain> <foo@dom.ain>\x20\x20 + +Test""") + + def test_long_run_with_semi_header_splitter(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'test@dom.ain' + msg['References'] = SPACE.join(['<foo@dom.ain>'] * 10) + '; abc' + msg.set_payload('Test') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), """\ +From: test@dom.ain +References: <foo@dom.ain> <foo@dom.ain> <foo@dom.ain> <foo@dom.ain> + <foo@dom.ain> <foo@dom.ain> <foo@dom.ain> <foo@dom.ain> <foo@dom.ain> + <foo@dom.ain>; abc + +Test""") + + def test_splitter_split_on_punctuation_only_if_fws(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'test@dom.ain' + msg['References'] = ('thisverylongheaderhas;semicolons;and,commas,but' + 'they;arenotlegal;fold,points') + msg.set_payload('Test') + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + # XXX the space after the header should not be there. + eq(sfp.getvalue(), """\ +From: test@dom.ain +References:\x20 + thisverylongheaderhas;semicolons;and,commas,butthey;arenotlegal;fold,points + +Test""") + + def test_no_split_long_header(self): + eq = self.ndiffAssertEqual + hstr = 'References: ' + 'x' * 80 + h = Header(hstr) + # These come on two lines because Headers are really field value + # classes and don't really know about their field names. + eq(h.encode(), """\ +References: + xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx""") + h = Header('x' * 80) + eq(h.encode(), 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx') + + def test_splitting_multiple_long_lines(self): + eq = self.ndiffAssertEqual + hstr = """\ +from babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin@babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) +\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin@babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) +\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; for <mailman-admin@babylon.socal-raves.org>; Sat, 2 Feb 2002 17:00:06 -0800 (PST) +""" + h = Header(hstr, continuation_ws='\t') + eq(h.encode(), """\ +from babylon.socal-raves.org (localhost [127.0.0.1]); + by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; + for <mailman-admin@babylon.socal-raves.org>; + Sat, 2 Feb 2002 17:00:06 -0800 (PST) +\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); + by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; + for <mailman-admin@babylon.socal-raves.org>; + Sat, 2 Feb 2002 17:00:06 -0800 (PST) +\tfrom babylon.socal-raves.org (localhost [127.0.0.1]); + by babylon.socal-raves.org (Postfix) with ESMTP id B570E51B81; + for <mailman-admin@babylon.socal-raves.org>; + Sat, 2 Feb 2002 17:00:06 -0800 (PST)""") + + def test_splitting_first_line_only_is_long(self): + eq = self.ndiffAssertEqual + hstr = """\ +from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] helo=cthulhu.gerg.ca) +\tby kronos.mems-exchange.org with esmtp (Exim 4.05) +\tid 17k4h5-00034i-00 +\tfor test@mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""" + h = Header(hstr, maxlinelen=78, header_name='Received', + continuation_ws='\t') + eq(h.encode(), """\ +from modemcable093.139-201-24.que.mc.videotron.ca ([24.201.139.93] + helo=cthulhu.gerg.ca) +\tby kronos.mems-exchange.org with esmtp (Exim 4.05) +\tid 17k4h5-00034i-00 +\tfor test@mems-exchange.org; Wed, 28 Aug 2002 11:25:20 -0400""") + + def test_long_8bit_header(self): + eq = self.ndiffAssertEqual + msg = Message() + h = Header('Britische Regierung gibt', 'iso-8859-1', + header_name='Subject') + h.append('gr\xfcnes Licht f\xfcr Offshore-Windkraftprojekte') + eq(h.encode(maxlinelen=76), """\ +=?iso-8859-1?q?Britische_Regierung_gibt_gr=FCnes_Licht_f=FCr_Offs?= + =?iso-8859-1?q?hore-Windkraftprojekte?=""") + msg['Subject'] = h + eq(msg.as_string(maxheaderlen=76), """\ +Subject: =?iso-8859-1?q?Britische_Regierung_gibt_gr=FCnes_Licht_f=FCr_Offs?= + =?iso-8859-1?q?hore-Windkraftprojekte?= + +""") + eq(msg.as_string(maxheaderlen=0), """\ +Subject: =?iso-8859-1?q?Britische_Regierung_gibt_gr=FCnes_Licht_f=FCr_Offshore-Windkraftprojekte?= + +""") + + def test_long_8bit_header_no_charset(self): + eq = self.ndiffAssertEqual + msg = Message() + header_string = ('Britische Regierung gibt gr\xfcnes Licht ' + 'f\xfcr Offshore-Windkraftprojekte ' + '<a-very-long-address@example.com>') + msg['Reply-To'] = header_string + self.assertRaises(UnicodeEncodeError, msg.as_string) + msg = Message() + msg['Reply-To'] = Header(header_string, 'utf-8', + header_name='Reply-To') + eq(msg.as_string(maxheaderlen=78), """\ +Reply-To: =?utf-8?q?Britische_Regierung_gibt_gr=C3=BCnes_Licht_f=C3=BCr_Offs?= + =?utf-8?q?hore-Windkraftprojekte_=3Ca-very-long-address=40example=2Ecom=3E?= + +""") + + def test_long_to_header(self): + eq = self.ndiffAssertEqual + to = ('"Someone Test #A" <someone@eecs.umich.edu>,' + '<someone@eecs.umich.edu>, ' + '"Someone Test #B" <someone@umich.edu>, ' + '"Someone Test #C" <someone@eecs.umich.edu>, ' + '"Someone Test #D" <someone@eecs.umich.edu>') + msg = Message() + msg['To'] = to + eq(msg.as_string(maxheaderlen=78), '''\ +To: "Someone Test #A" <someone@eecs.umich.edu>,<someone@eecs.umich.edu>, + "Someone Test #B" <someone@umich.edu>, + "Someone Test #C" <someone@eecs.umich.edu>, + "Someone Test #D" <someone@eecs.umich.edu> + +''') + + def test_long_line_after_append(self): + eq = self.ndiffAssertEqual + s = 'This is an example of string which has almost the limit of header length.' + h = Header(s) + h.append('Add another line.') + eq(h.encode(maxlinelen=76), """\ +This is an example of string which has almost the limit of header length. + Add another line.""") + + def test_shorter_line_with_append(self): + eq = self.ndiffAssertEqual + s = 'This is a shorter line.' + h = Header(s) + h.append('Add another sentence. (Surprise?)') + eq(h.encode(), + 'This is a shorter line. Add another sentence. (Surprise?)') + + def test_long_field_name(self): + eq = self.ndiffAssertEqual + fn = 'X-Very-Very-Very-Long-Header-Name' + gs = ('Die Mieter treten hier ein werden mit einem Foerderband ' + 'komfortabel den Korridor entlang, an s\xfcdl\xfcndischen ' + 'Wandgem\xe4lden vorbei, gegen die rotierenden Klingen ' + 'bef\xf6rdert. ') + h = Header(gs, 'iso-8859-1', header_name=fn) + # BAW: this seems broken because the first line is too long + eq(h.encode(maxlinelen=76), """\ +=?iso-8859-1?q?Die_Mieter_treten_hier_e?= + =?iso-8859-1?q?in_werden_mit_einem_Foerderband_komfortabel_den_Korridor_e?= + =?iso-8859-1?q?ntlang=2C_an_s=FCdl=FCndischen_Wandgem=E4lden_vorbei=2C_ge?= + =?iso-8859-1?q?gen_die_rotierenden_Klingen_bef=F6rdert=2E_?=""") + + def test_long_received_header(self): + h = ('from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) ' + 'by hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; ' + 'Wed, 05 Mar 2003 18:10:18 -0700') + msg = Message() + msg['Received-1'] = Header(h, continuation_ws='\t') + msg['Received-2'] = h + # This should be splitting on spaces not semicolons. + self.ndiffAssertEqual(msg.as_string(maxheaderlen=78), """\ +Received-1: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by + hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; + Wed, 05 Mar 2003 18:10:18 -0700 +Received-2: from FOO.TLD (vizworld.acl.foo.tld [123.452.678.9]) by + hrothgar.la.mastaler.com (tmda-ofmipd) with ESMTP; + Wed, 05 Mar 2003 18:10:18 -0700 + +""") + + def test_string_headerinst_eq(self): + h = ('<15975.17901.207240.414604@sgigritzmann1.mathematik.' + 'tu-muenchen.de> (David Bremner\'s message of ' + '"Thu, 6 Mar 2003 13:58:21 +0100")') + msg = Message() + msg['Received-1'] = Header(h, header_name='Received-1', + continuation_ws='\t') + msg['Received-2'] = h + # XXX The space after the ':' should not be there. + self.ndiffAssertEqual(msg.as_string(maxheaderlen=78), """\ +Received-1:\x20 + <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de> (David + Bremner's message of \"Thu, 6 Mar 2003 13:58:21 +0100\") +Received-2:\x20 + <15975.17901.207240.414604@sgigritzmann1.mathematik.tu-muenchen.de> (David + Bremner's message of \"Thu, 6 Mar 2003 13:58:21 +0100\") + +""") + + def test_long_unbreakable_lines_with_continuation(self): + eq = self.ndiffAssertEqual + msg = Message() + t = """\ +iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 + locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp""" + msg['Face-1'] = t + msg['Face-2'] = Header(t, header_name='Face-2') + msg['Face-3'] = ' ' + t + # XXX This splitting is all wrong. It the first value line should be + # snug against the field name or the space after the header not there. + eq(msg.as_string(maxheaderlen=78), """\ +Face-1:\x20 + iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 + locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp +Face-2:\x20 + iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 + locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp +Face-3:\x20 + iVBORw0KGgoAAAANSUhEUgAAADAAAAAwBAMAAAClLOS0AAAAGFBMVEUAAAAkHiJeRUIcGBi9 + locQDQ4zJykFBAXJfWDjAAACYUlEQVR4nF2TQY/jIAyFc6lydlG5x8Nyp1Y69wj1PN2I5gzp + +""") + + def test_another_long_multiline_header(self): + eq = self.ndiffAssertEqual + m = ('Received: from siimage.com ' + '([172.25.1.3]) by zima.siliconimage.com with ' + 'Microsoft SMTPSVC(5.0.2195.4905); ' + 'Wed, 16 Oct 2002 07:41:11 -0700') + msg = email.message_from_string(m) + eq(msg.as_string(maxheaderlen=78), '''\ +Received: from siimage.com ([172.25.1.3]) by zima.siliconimage.com with + Microsoft SMTPSVC(5.0.2195.4905); Wed, 16 Oct 2002 07:41:11 -0700 + +''') + + def test_long_lines_with_different_header(self): + eq = self.ndiffAssertEqual + h = ('List-Unsubscribe: ' + '<http://lists.sourceforge.net/lists/listinfo/spamassassin-talk>,' + ' <mailto:spamassassin-talk-request@lists.sourceforge.net' + '?subject=unsubscribe>') + msg = Message() + msg['List'] = h + msg['List'] = Header(h, header_name='List') + eq(msg.as_string(maxheaderlen=78), """\ +List: List-Unsubscribe: + <http://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, + <mailto:spamassassin-talk-request@lists.sourceforge.net?subject=unsubscribe> +List: List-Unsubscribe: + <http://lists.sourceforge.net/lists/listinfo/spamassassin-talk>, + <mailto:spamassassin-talk-request@lists.sourceforge.net?subject=unsubscribe> + +""") + + def test_long_rfc2047_header_with_embedded_fws(self): + h = Header(textwrap.dedent("""\ + We're going to pretend this header is in a non-ascii character set + \tto see if line wrapping with encoded words and embedded + folding white space works"""), + charset='utf-8', + header_name='Test') + self.assertEqual(h.encode()+'\n', textwrap.dedent("""\ + =?utf-8?q?We=27re_going_to_pretend_this_header_is_in_a_non-ascii_chara?= + =?utf-8?q?cter_set?= + =?utf-8?q?_to_see_if_line_wrapping_with_encoded_words_and_embedded?= + =?utf-8?q?_folding_white_space_works?=""")+'\n') + + + +# Test mangling of "From " lines in the body of a message +class TestFromMangling(unittest.TestCase): + def setUp(self): + self.msg = Message() + self.msg['From'] = 'aaa@bbb.org' + self.msg.set_payload("""\ +From the desk of A.A.A.: +Blah blah blah +""") + + def test_mangled_from(self): + s = StringIO() + g = Generator(s, mangle_from_=True) + g.flatten(self.msg) + self.assertEqual(s.getvalue(), """\ +From: aaa@bbb.org + +>From the desk of A.A.A.: +Blah blah blah +""") + + def test_dont_mangle_from(self): + s = StringIO() + g = Generator(s, mangle_from_=False) + g.flatten(self.msg) + self.assertEqual(s.getvalue(), """\ +From: aaa@bbb.org + +From the desk of A.A.A.: +Blah blah blah +""") + + + +# Test the basic MIMEAudio class +class TestMIMEAudio(unittest.TestCase): + def setUp(self): + with openfile('audiotest.au', 'rb') as fp: + self._audiodata = fp.read() + self._au = MIMEAudio(self._audiodata) + + def test_guess_minor_type(self): + self.assertEqual(self._au.get_content_type(), 'audio/basic') + + def test_encoding(self): + payload = self._au.get_payload() + self.assertEqual(base64.decodebytes(bytes(payload, 'ascii')), + self._audiodata) + + def test_checkSetMinor(self): + au = MIMEAudio(self._audiodata, 'fish') + self.assertEqual(au.get_content_type(), 'audio/fish') + + def test_add_header(self): + eq = self.assertEqual + unless = self.assertTrue + self._au.add_header('Content-Disposition', 'attachment', + filename='audiotest.au') + eq(self._au['content-disposition'], + 'attachment; filename="audiotest.au"') + eq(self._au.get_params(header='content-disposition'), + [('attachment', ''), ('filename', 'audiotest.au')]) + eq(self._au.get_param('filename', header='content-disposition'), + 'audiotest.au') + missing = [] + eq(self._au.get_param('attachment', header='content-disposition'), '') + unless(self._au.get_param('foo', failobj=missing, + header='content-disposition') is missing) + # Try some missing stuff + unless(self._au.get_param('foobar', missing) is missing) + unless(self._au.get_param('attachment', missing, + header='foobar') is missing) + + + +# Test the basic MIMEImage class +class TestMIMEImage(unittest.TestCase): + def setUp(self): + with openfile('PyBanner048.gif', 'rb') as fp: + self._imgdata = fp.read() + self._im = MIMEImage(self._imgdata) + + def test_guess_minor_type(self): + self.assertEqual(self._im.get_content_type(), 'image/gif') + + def test_encoding(self): + payload = self._im.get_payload() + self.assertEqual(base64.decodebytes(bytes(payload, 'ascii')), + self._imgdata) + + def test_checkSetMinor(self): + im = MIMEImage(self._imgdata, 'fish') + self.assertEqual(im.get_content_type(), 'image/fish') + + def test_add_header(self): + eq = self.assertEqual + unless = self.assertTrue + self._im.add_header('Content-Disposition', 'attachment', + filename='dingusfish.gif') + eq(self._im['content-disposition'], + 'attachment; filename="dingusfish.gif"') + eq(self._im.get_params(header='content-disposition'), + [('attachment', ''), ('filename', 'dingusfish.gif')]) + eq(self._im.get_param('filename', header='content-disposition'), + 'dingusfish.gif') + missing = [] + eq(self._im.get_param('attachment', header='content-disposition'), '') + unless(self._im.get_param('foo', failobj=missing, + header='content-disposition') is missing) + # Try some missing stuff + unless(self._im.get_param('foobar', missing) is missing) + unless(self._im.get_param('attachment', missing, + header='foobar') is missing) + + + +# Test the basic MIMEApplication class +class TestMIMEApplication(unittest.TestCase): + def test_headers(self): + eq = self.assertEqual + msg = MIMEApplication(b'\xfa\xfb\xfc\xfd\xfe\xff') + eq(msg.get_content_type(), 'application/octet-stream') + eq(msg['content-transfer-encoding'], 'base64') + + def test_body(self): + eq = self.assertEqual + bytesdata = b'\xfa\xfb\xfc\xfd\xfe\xff' + msg = MIMEApplication(bytesdata) + # whitespace in the cte encoded block is RFC-irrelevant. + eq(msg.get_payload().strip(), '+vv8/f7/') + eq(msg.get_payload(decode=True), bytesdata) + + + +# Test the basic MIMEText class +class TestMIMEText(unittest.TestCase): + def setUp(self): + self._msg = MIMEText('hello there') + + def test_types(self): + eq = self.assertEqual + unless = self.assertTrue + eq(self._msg.get_content_type(), 'text/plain') + eq(self._msg.get_param('charset'), 'us-ascii') + missing = [] + unless(self._msg.get_param('foobar', missing) is missing) + unless(self._msg.get_param('charset', missing, header='foobar') + is missing) + + def test_payload(self): + self.assertEqual(self._msg.get_payload(), 'hello there') + self.assertTrue(not self._msg.is_multipart()) + + def test_charset(self): + eq = self.assertEqual + msg = MIMEText('hello there', _charset='us-ascii') + eq(msg.get_charset().input_charset, 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + + def test_7bit_input(self): + eq = self.assertEqual + msg = MIMEText('hello there', _charset='us-ascii') + eq(msg.get_charset().input_charset, 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + + def test_7bit_input_no_charset(self): + eq = self.assertEqual + msg = MIMEText('hello there') + eq(msg.get_charset(), 'us-ascii') + eq(msg['content-type'], 'text/plain; charset="us-ascii"') + self.assertTrue('hello there' in msg.as_string()) + + def test_utf8_input(self): + teststr = '\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' + eq = self.assertEqual + msg = MIMEText(teststr, _charset='utf-8') + eq(msg.get_charset().output_charset, 'utf-8') + eq(msg['content-type'], 'text/plain; charset="utf-8"') + eq(msg.get_payload(decode=True), teststr.encode('utf-8')) + + @unittest.skip("can't fix because of backward compat in email5, " + "will fix in email6") + def test_utf8_input_no_charset(self): + teststr = '\u043a\u0438\u0440\u0438\u043b\u0438\u0446\u0430' + self.assertRaises(UnicodeEncodeError, MIMEText, teststr) + + + +# Test complicated multipart/* messages +class TestMultipart(TestEmailBase): + def setUp(self): + with openfile('PyBanner048.gif', 'rb') as fp: + data = fp.read() + container = MIMEBase('multipart', 'mixed', boundary='BOUNDARY') + image = MIMEImage(data, name='dingusfish.gif') + image.add_header('content-disposition', 'attachment', + filename='dingusfish.gif') + intro = MIMEText('''\ +Hi there, + +This is the dingus fish. +''') + container.attach(intro) + container.attach(image) + container['From'] = 'Barry <barry@digicool.com>' + container['To'] = 'Dingus Lovers <cravindogs@cravindogs.com>' + container['Subject'] = 'Here is your dingus fish' + + now = 987809702.54848599 + timetuple = time.localtime(now) + if timetuple[-1] == 0: + tzsecs = time.timezone + else: + tzsecs = time.altzone + if tzsecs > 0: + sign = '-' + else: + sign = '+' + tzoffset = ' %s%04d' % (sign, tzsecs / 36) + container['Date'] = time.strftime( + '%a, %d %b %Y %H:%M:%S', + time.localtime(now)) + tzoffset + self._msg = container + self._im = image + self._txt = intro + + def test_hierarchy(self): + # convenience + eq = self.assertEqual + unless = self.assertTrue + raises = self.assertRaises + # tests + m = self._msg + unless(m.is_multipart()) + eq(m.get_content_type(), 'multipart/mixed') + eq(len(m.get_payload()), 2) + raises(IndexError, m.get_payload, 2) + m0 = m.get_payload(0) + m1 = m.get_payload(1) + unless(m0 is self._txt) + unless(m1 is self._im) + eq(m.get_payload(), [m0, m1]) + unless(not m0.is_multipart()) + unless(not m1.is_multipart()) + + def test_empty_multipart_idempotent(self): + text = """\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + + +--BOUNDARY + + +--BOUNDARY-- +""" + msg = Parser().parsestr(text) + self.ndiffAssertEqual(text, msg.as_string()) + + def test_no_parts_in_a_multipart_with_none_epilogue(self): + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.set_boundary('BOUNDARY') + self.ndiffAssertEqual(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY + +--BOUNDARY--''') + + def test_no_parts_in_a_multipart_with_empty_epilogue(self): + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.preamble = '' + outer.epilogue = '' + outer.set_boundary('BOUNDARY') + self.ndiffAssertEqual(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + + +--BOUNDARY + +--BOUNDARY-- +''') + + def test_one_part_in_a_multipart(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.set_boundary('BOUNDARY') + msg = MIMEText('hello world') + outer.attach(msg) + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY--''') + + def test_seq_parts_in_a_multipart_with_empty_preamble(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.preamble = '' + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY--''') + + + def test_seq_parts_in_a_multipart_with_none_preamble(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.preamble = None + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY--''') + + + def test_seq_parts_in_a_multipart_with_none_epilogue(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.epilogue = None + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY--''') + + + def test_seq_parts_in_a_multipart_with_empty_epilogue(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.epilogue = '' + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY-- +''') + + + def test_seq_parts_in_a_multipart_with_nl_epilogue(self): + eq = self.ndiffAssertEqual + outer = MIMEBase('multipart', 'mixed') + outer['Subject'] = 'A subject' + outer['To'] = 'aperson@dom.ain' + outer['From'] = 'bperson@dom.ain' + outer.epilogue = '\n' + msg = MIMEText('hello world') + outer.attach(msg) + outer.set_boundary('BOUNDARY') + eq(outer.as_string(), '''\ +Content-Type: multipart/mixed; boundary="BOUNDARY" +MIME-Version: 1.0 +Subject: A subject +To: aperson@dom.ain +From: bperson@dom.ain + +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +hello world +--BOUNDARY-- + +''') + + def test_message_external_body(self): + eq = self.assertEqual + msg = self._msgobj('msg_36.txt') + eq(len(msg.get_payload()), 2) + msg1 = msg.get_payload(1) + eq(msg1.get_content_type(), 'multipart/alternative') + eq(len(msg1.get_payload()), 2) + for subpart in msg1.get_payload(): + eq(subpart.get_content_type(), 'message/external-body') + eq(len(subpart.get_payload()), 1) + subsubpart = subpart.get_payload(0) + eq(subsubpart.get_content_type(), 'text/plain') + + def test_double_boundary(self): + # msg_37.txt is a multipart that contains two dash-boundary's in a + # row. Our interpretation of RFC 2046 calls for ignoring the second + # and subsequent boundaries. + msg = self._msgobj('msg_37.txt') + self.assertEqual(len(msg.get_payload()), 3) + + def test_nested_inner_contains_outer_boundary(self): + eq = self.ndiffAssertEqual + # msg_38.txt has an inner part that contains outer boundaries. My + # interpretation of RFC 2046 (based on sections 5.1 and 5.1.2) say + # these are illegal and should be interpreted as unterminated inner + # parts. + msg = self._msgobj('msg_38.txt') + sfp = StringIO() + iterators._structure(msg, sfp) + eq(sfp.getvalue(), """\ +multipart/mixed + multipart/mixed + multipart/alternative + text/plain + text/plain + text/plain + text/plain +""") + + def test_nested_with_same_boundary(self): + eq = self.ndiffAssertEqual + # msg 39.txt is similarly evil in that it's got inner parts that use + # the same boundary as outer parts. Again, I believe the way this is + # parsed is closest to the spirit of RFC 2046 + msg = self._msgobj('msg_39.txt') + sfp = StringIO() + iterators._structure(msg, sfp) + eq(sfp.getvalue(), """\ +multipart/mixed + multipart/mixed + multipart/alternative + application/octet-stream + application/octet-stream + text/plain +""") + + def test_boundary_in_non_multipart(self): + msg = self._msgobj('msg_40.txt') + self.assertEqual(msg.as_string(), '''\ +MIME-Version: 1.0 +Content-Type: text/html; boundary="--961284236552522269" + +----961284236552522269 +Content-Type: text/html; +Content-Transfer-Encoding: 7Bit + +<html></html> + +----961284236552522269-- +''') + + def test_boundary_with_leading_space(self): + eq = self.assertEqual + msg = email.message_from_string('''\ +MIME-Version: 1.0 +Content-Type: multipart/mixed; boundary=" XXXX" + +-- XXXX +Content-Type: text/plain + + +-- XXXX +Content-Type: text/plain + +-- XXXX-- +''') + self.assertTrue(msg.is_multipart()) + eq(msg.get_boundary(), ' XXXX') + eq(len(msg.get_payload()), 2) + + def test_boundary_without_trailing_newline(self): + m = Parser().parsestr("""\ +Content-Type: multipart/mixed; boundary="===============0012394164==" +MIME-Version: 1.0 + +--===============0012394164== +Content-Type: image/file1.jpg +MIME-Version: 1.0 +Content-Transfer-Encoding: base64 + +YXNkZg== +--===============0012394164==--""") + self.assertEqual(m.get_payload(0).get_payload(), 'YXNkZg==') + + + +# Test some badly formatted messages +class TestNonConformantBase: + + def _msgobj(self, filename): + with openfile(filename) as fp: + return email.message_from_file(fp, policy=self.policy) + + def test_parse_missing_minor_type(self): + eq = self.assertEqual + msg = self._msgobj('msg_14.txt') + eq(msg.get_content_type(), 'text/plain') + eq(msg.get_content_maintype(), 'text') + eq(msg.get_content_subtype(), 'plain') + + def test_same_boundary_inner_outer(self): + unless = self.assertTrue + msg = self._msgobj('msg_15.txt') + # XXX We can probably eventually do better + inner = msg.get_payload(0) + unless(hasattr(inner, 'defects')) + self.assertEqual(len(self.get_defects(inner)), 1) + unless(isinstance(self.get_defects(inner)[0], + errors.StartBoundaryNotFoundDefect)) + + def test_multipart_no_boundary(self): + unless = self.assertTrue + msg = self._msgobj('msg_25.txt') + unless(isinstance(msg.get_payload(), str)) + self.assertEqual(len(self.get_defects(msg)), 2) + unless(isinstance(self.get_defects(msg)[0], + errors.NoBoundaryInMultipartDefect)) + unless(isinstance(self.get_defects(msg)[1], + errors.MultipartInvariantViolationDefect)) + + multipart_msg = textwrap.dedent("""\ + Date: Wed, 14 Nov 2007 12:56:23 GMT + From: foo@bar.invalid + To: foo@bar.invalid + Subject: Content-Transfer-Encoding: base64 and multipart + MIME-Version: 1.0 + Content-Type: multipart/mixed; + boundary="===============3344438784458119861=="{} + + --===============3344438784458119861== + Content-Type: text/plain + + Test message + + --===============3344438784458119861== + Content-Type: application/octet-stream + Content-Transfer-Encoding: base64 + + YWJj + + --===============3344438784458119861==-- + """) + + def test_multipart_invalid_cte(self): + msg = email.message_from_string( + self.multipart_msg.format("\nContent-Transfer-Encoding: base64"), + policy = self.policy) + self.assertEqual(len(self.get_defects(msg)), 1) + self.assertIsInstance(self.get_defects(msg)[0], + errors.InvalidMultipartContentTransferEncodingDefect) + + def test_multipart_no_cte_no_defect(self): + msg = email.message_from_string( + self.multipart_msg.format(''), + policy = self.policy) + self.assertEqual(len(self.get_defects(msg)), 0) + + def test_multipart_valid_cte_no_defect(self): + for cte in ('7bit', '8bit', 'BINary'): + msg = email.message_from_string( + self.multipart_msg.format( + "\nContent-Transfer-Encoding: {}".format(cte)), + policy = self.policy) + self.assertEqual(len(self.get_defects(msg)), 0) + + def test_invalid_content_type(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + msg = Message() + # RFC 2045, $5.2 says invalid yields text/plain + msg['Content-Type'] = 'text' + eq(msg.get_content_maintype(), 'text') + eq(msg.get_content_subtype(), 'plain') + eq(msg.get_content_type(), 'text/plain') + # Clear the old value and try something /really/ invalid + del msg['content-type'] + msg['Content-Type'] = 'foo' + eq(msg.get_content_maintype(), 'text') + eq(msg.get_content_subtype(), 'plain') + eq(msg.get_content_type(), 'text/plain') + # Still, make sure that the message is idempotently generated + s = StringIO() + g = Generator(s) + g.flatten(msg) + neq(s.getvalue(), 'Content-Type: foo\n\n') + + def test_no_start_boundary(self): + eq = self.ndiffAssertEqual + msg = self._msgobj('msg_31.txt') + eq(msg.get_payload(), """\ +--BOUNDARY +Content-Type: text/plain + +message 1 + +--BOUNDARY +Content-Type: text/plain + +message 2 + +--BOUNDARY-- +""") + + def test_no_separating_blank_line(self): + eq = self.ndiffAssertEqual + msg = self._msgobj('msg_35.txt') + eq(msg.as_string(), """\ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: here's something interesting + +counter to RFC 2822, there's no separating newline here +""") + + def test_lying_multipart(self): + unless = self.assertTrue + msg = self._msgobj('msg_41.txt') + unless(hasattr(msg, 'defects')) + self.assertEqual(len(self.get_defects(msg)), 2) + unless(isinstance(self.get_defects(msg)[0], + errors.NoBoundaryInMultipartDefect)) + unless(isinstance(self.get_defects(msg)[1], + errors.MultipartInvariantViolationDefect)) + + def test_missing_start_boundary(self): + outer = self._msgobj('msg_42.txt') + # The message structure is: + # + # multipart/mixed + # text/plain + # message/rfc822 + # multipart/mixed [*] + # + # [*] This message is missing its start boundary + bad = outer.get_payload(1).get_payload(0) + self.assertEqual(len(self.get_defects(bad)), 1) + self.assertTrue(isinstance(self.get_defects(bad)[0], + errors.StartBoundaryNotFoundDefect)) + + def test_first_line_is_continuation_header(self): + eq = self.assertEqual + m = ' Line 1\nLine 2\nLine 3' + msg = email.message_from_string(m, policy=self.policy) + eq(msg.keys(), []) + eq(msg.get_payload(), 'Line 2\nLine 3') + eq(len(self.get_defects(msg)), 1) + self.assertTrue(isinstance(self.get_defects(msg)[0], + errors.FirstHeaderLineIsContinuationDefect)) + eq(self.get_defects(msg)[0].line, ' Line 1\n') + + +class TestNonConformant(TestNonConformantBase, TestEmailBase): + + policy=email.policy.default + + def get_defects(self, obj): + return obj.defects + + +class TestNonConformantCapture(TestNonConformantBase, TestEmailBase): + + class CapturePolicy(email.policy.Policy): + captured = None + def register_defect(self, obj, defect): + self.captured.append(defect) + + def setUp(self): + self.policy = self.CapturePolicy(captured=list()) + + def get_defects(self, obj): + return self.policy.captured + + +class TestRaisingDefects(TestEmailBase): + + def _msgobj(self, filename): + with openfile(filename) as fp: + return email.message_from_file(fp, policy=email.policy.strict) + + def test_same_boundary_inner_outer(self): + with self.assertRaises(errors.StartBoundaryNotFoundDefect): + self._msgobj('msg_15.txt') + + def test_multipart_no_boundary(self): + with self.assertRaises(errors.NoBoundaryInMultipartDefect): + self._msgobj('msg_25.txt') + + def test_lying_multipart(self): + with self.assertRaises(errors.NoBoundaryInMultipartDefect): + self._msgobj('msg_41.txt') + + + def test_missing_start_boundary(self): + with self.assertRaises(errors.StartBoundaryNotFoundDefect): + self._msgobj('msg_42.txt') + + def test_first_line_is_continuation_header(self): + m = ' Line 1\nLine 2\nLine 3' + with self.assertRaises(errors.FirstHeaderLineIsContinuationDefect): + msg = email.message_from_string(m, policy=email.policy.strict) + + +# Test RFC 2047 header encoding and decoding +class TestRFC2047(TestEmailBase): + def test_rfc2047_multiline(self): + eq = self.assertEqual + s = """Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz + foo bar =?mac-iceland?q?r=8Aksm=9Arg=8Cs?=""" + dh = decode_header(s) + eq(dh, [ + (b'Re:', None), + (b'r\x8aksm\x9arg\x8cs', 'mac-iceland'), + (b'baz foo bar', None), + (b'r\x8aksm\x9arg\x8cs', 'mac-iceland')]) + header = make_header(dh) + eq(str(header), + 'Re: r\xe4ksm\xf6rg\xe5s baz foo bar r\xe4ksm\xf6rg\xe5s') + self.ndiffAssertEqual(header.encode(maxlinelen=76), """\ +Re: =?mac-iceland?q?r=8Aksm=9Arg=8Cs?= baz foo bar =?mac-iceland?q?r=8Aksm?= + =?mac-iceland?q?=9Arg=8Cs?=""") + + def test_whitespace_eater_unicode(self): + eq = self.assertEqual + s = '=?ISO-8859-1?Q?Andr=E9?= Pirard <pirard@dom.ain>' + dh = decode_header(s) + eq(dh, [(b'Andr\xe9', 'iso-8859-1'), + (b'Pirard <pirard@dom.ain>', None)]) + header = str(make_header(dh)) + eq(header, 'Andr\xe9 Pirard <pirard@dom.ain>') + + def test_whitespace_eater_unicode_2(self): + eq = self.assertEqual + s = 'The =?iso-8859-1?b?cXVpY2sgYnJvd24gZm94?= jumped over the =?iso-8859-1?b?bGF6eSBkb2c=?=' + dh = decode_header(s) + eq(dh, [(b'The', None), (b'quick brown fox', 'iso-8859-1'), + (b'jumped over the', None), (b'lazy dog', 'iso-8859-1')]) + hu = str(make_header(dh)) + eq(hu, 'The quick brown fox jumped over the lazy dog') + + def test_rfc2047_missing_whitespace(self): + s = 'Sm=?ISO-8859-1?B?9g==?=rg=?ISO-8859-1?B?5Q==?=sbord' + dh = decode_header(s) + self.assertEqual(dh, [(s, None)]) + + def test_rfc2047_with_whitespace(self): + s = 'Sm =?ISO-8859-1?B?9g==?= rg =?ISO-8859-1?B?5Q==?= sbord' + dh = decode_header(s) + self.assertEqual(dh, [(b'Sm', None), (b'\xf6', 'iso-8859-1'), + (b'rg', None), (b'\xe5', 'iso-8859-1'), + (b'sbord', None)]) + + def test_rfc2047_B_bad_padding(self): + s = '=?iso-8859-1?B?%s?=' + data = [ # only test complete bytes + ('dm==', b'v'), ('dm=', b'v'), ('dm', b'v'), + ('dmk=', b'vi'), ('dmk', b'vi') + ] + for q, a in data: + dh = decode_header(s % q) + self.assertEqual(dh, [(a, 'iso-8859-1')]) + + def test_rfc2047_Q_invalid_digits(self): + # issue 10004. + s = '=?iso-8659-1?Q?andr=e9=zz?=' + self.assertEqual(decode_header(s), + [(b'andr\xe9=zz', 'iso-8659-1')]) + + +# Test the MIMEMessage class +class TestMIMEMessage(TestEmailBase): + def setUp(self): + with openfile('msg_11.txt') as fp: + self._text = fp.read() + + def test_type_error(self): + self.assertRaises(TypeError, MIMEMessage, 'a plain string') + + def test_valid_argument(self): + eq = self.assertEqual + unless = self.assertTrue + subject = 'A sub-message' + m = Message() + m['Subject'] = subject + r = MIMEMessage(m) + eq(r.get_content_type(), 'message/rfc822') + payload = r.get_payload() + unless(isinstance(payload, list)) + eq(len(payload), 1) + subpart = payload[0] + unless(subpart is m) + eq(subpart['subject'], subject) + + def test_bad_multipart(self): + eq = self.assertEqual + msg1 = Message() + msg1['Subject'] = 'subpart 1' + msg2 = Message() + msg2['Subject'] = 'subpart 2' + r = MIMEMessage(msg1) + self.assertRaises(errors.MultipartConversionError, r.attach, msg2) + + def test_generate(self): + # First craft the message to be encapsulated + m = Message() + m['Subject'] = 'An enclosed message' + m.set_payload('Here is the body of the message.\n') + r = MIMEMessage(m) + r['Subject'] = 'The enclosing message' + s = StringIO() + g = Generator(s) + g.flatten(r) + self.assertEqual(s.getvalue(), """\ +Content-Type: message/rfc822 +MIME-Version: 1.0 +Subject: The enclosing message + +Subject: An enclosed message + +Here is the body of the message. +""") + + def test_parse_message_rfc822(self): + eq = self.assertEqual + unless = self.assertTrue + msg = self._msgobj('msg_11.txt') + eq(msg.get_content_type(), 'message/rfc822') + payload = msg.get_payload() + unless(isinstance(payload, list)) + eq(len(payload), 1) + submsg = payload[0] + self.assertTrue(isinstance(submsg, Message)) + eq(submsg['subject'], 'An enclosed message') + eq(submsg.get_payload(), 'Here is the body of the message.\n') + + def test_dsn(self): + eq = self.assertEqual + unless = self.assertTrue + # msg 16 is a Delivery Status Notification, see RFC 1894 + msg = self._msgobj('msg_16.txt') + eq(msg.get_content_type(), 'multipart/report') + unless(msg.is_multipart()) + eq(len(msg.get_payload()), 3) + # Subpart 1 is a text/plain, human readable section + subpart = msg.get_payload(0) + eq(subpart.get_content_type(), 'text/plain') + eq(subpart.get_payload(), """\ +This report relates to a message you sent with the following header fields: + + Message-id: <002001c144a6$8752e060$56104586@oxy.edu> + Date: Sun, 23 Sep 2001 20:10:55 -0700 + From: "Ian T. Henry" <henryi@oxy.edu> + To: SoCal Raves <scr@socal-raves.org> + Subject: [scr] yeah for Ians!! + +Your message cannot be delivered to the following recipients: + + Recipient address: jangel1@cougar.noc.ucla.edu + Reason: recipient reached disk quota + +""") + # Subpart 2 contains the machine parsable DSN information. It + # consists of two blocks of headers, represented by two nested Message + # objects. + subpart = msg.get_payload(1) + eq(subpart.get_content_type(), 'message/delivery-status') + eq(len(subpart.get_payload()), 2) + # message/delivery-status should treat each block as a bunch of + # headers, i.e. a bunch of Message objects. + dsn1 = subpart.get_payload(0) + unless(isinstance(dsn1, Message)) + eq(dsn1['original-envelope-id'], '0GK500B4HD0888@cougar.noc.ucla.edu') + eq(dsn1.get_param('dns', header='reporting-mta'), '') + # Try a missing one <wink> + eq(dsn1.get_param('nsd', header='reporting-mta'), None) + dsn2 = subpart.get_payload(1) + unless(isinstance(dsn2, Message)) + eq(dsn2['action'], 'failed') + eq(dsn2.get_params(header='original-recipient'), + [('rfc822', ''), ('jangel1@cougar.noc.ucla.edu', '')]) + eq(dsn2.get_param('rfc822', header='final-recipient'), '') + # Subpart 3 is the original message + subpart = msg.get_payload(2) + eq(subpart.get_content_type(), 'message/rfc822') + payload = subpart.get_payload() + unless(isinstance(payload, list)) + eq(len(payload), 1) + subsubpart = payload[0] + unless(isinstance(subsubpart, Message)) + eq(subsubpart.get_content_type(), 'text/plain') + eq(subsubpart['message-id'], + '<002001c144a6$8752e060$56104586@oxy.edu>') + + def test_epilogue(self): + eq = self.ndiffAssertEqual + with openfile('msg_21.txt') as fp: + text = fp.read() + msg = Message() + msg['From'] = 'aperson@dom.ain' + msg['To'] = 'bperson@dom.ain' + msg['Subject'] = 'Test' + msg.preamble = 'MIME message' + msg.epilogue = 'End of MIME message\n' + msg1 = MIMEText('One') + msg2 = MIMEText('Two') + msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') + msg.attach(msg1) + msg.attach(msg2) + sfp = StringIO() + g = Generator(sfp) + g.flatten(msg) + eq(sfp.getvalue(), text) + + def test_no_nl_preamble(self): + eq = self.ndiffAssertEqual + msg = Message() + msg['From'] = 'aperson@dom.ain' + msg['To'] = 'bperson@dom.ain' + msg['Subject'] = 'Test' + msg.preamble = 'MIME message' + msg.epilogue = '' + msg1 = MIMEText('One') + msg2 = MIMEText('Two') + msg.add_header('Content-Type', 'multipart/mixed', boundary='BOUNDARY') + msg.attach(msg1) + msg.attach(msg2) + eq(msg.as_string(), """\ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: Test +Content-Type: multipart/mixed; boundary="BOUNDARY" + +MIME message +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +One +--BOUNDARY +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +Two +--BOUNDARY-- +""") + + def test_default_type(self): + eq = self.assertEqual + with openfile('msg_30.txt') as fp: + msg = email.message_from_file(fp) + container1 = msg.get_payload(0) + eq(container1.get_default_type(), 'message/rfc822') + eq(container1.get_content_type(), 'message/rfc822') + container2 = msg.get_payload(1) + eq(container2.get_default_type(), 'message/rfc822') + eq(container2.get_content_type(), 'message/rfc822') + container1a = container1.get_payload(0) + eq(container1a.get_default_type(), 'text/plain') + eq(container1a.get_content_type(), 'text/plain') + container2a = container2.get_payload(0) + eq(container2a.get_default_type(), 'text/plain') + eq(container2a.get_content_type(), 'text/plain') + + def test_default_type_with_explicit_container_type(self): + eq = self.assertEqual + with openfile('msg_28.txt') as fp: + msg = email.message_from_file(fp) + container1 = msg.get_payload(0) + eq(container1.get_default_type(), 'message/rfc822') + eq(container1.get_content_type(), 'message/rfc822') + container2 = msg.get_payload(1) + eq(container2.get_default_type(), 'message/rfc822') + eq(container2.get_content_type(), 'message/rfc822') + container1a = container1.get_payload(0) + eq(container1a.get_default_type(), 'text/plain') + eq(container1a.get_content_type(), 'text/plain') + container2a = container2.get_payload(0) + eq(container2a.get_default_type(), 'text/plain') + eq(container2a.get_content_type(), 'text/plain') + + def test_default_type_non_parsed(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + # Set up container + container = MIMEMultipart('digest', 'BOUNDARY') + container.epilogue = '' + # Set up subparts + subpart1a = MIMEText('message 1\n') + subpart2a = MIMEText('message 2\n') + subpart1 = MIMEMessage(subpart1a) + subpart2 = MIMEMessage(subpart2a) + container.attach(subpart1) + container.attach(subpart2) + eq(subpart1.get_content_type(), 'message/rfc822') + eq(subpart1.get_default_type(), 'message/rfc822') + eq(subpart2.get_content_type(), 'message/rfc822') + eq(subpart2.get_default_type(), 'message/rfc822') + neq(container.as_string(0), '''\ +Content-Type: multipart/digest; boundary="BOUNDARY" +MIME-Version: 1.0 + +--BOUNDARY +Content-Type: message/rfc822 +MIME-Version: 1.0 + +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +message 1 + +--BOUNDARY +Content-Type: message/rfc822 +MIME-Version: 1.0 + +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +message 2 + +--BOUNDARY-- +''') + del subpart1['content-type'] + del subpart1['mime-version'] + del subpart2['content-type'] + del subpart2['mime-version'] + eq(subpart1.get_content_type(), 'message/rfc822') + eq(subpart1.get_default_type(), 'message/rfc822') + eq(subpart2.get_content_type(), 'message/rfc822') + eq(subpart2.get_default_type(), 'message/rfc822') + neq(container.as_string(0), '''\ +Content-Type: multipart/digest; boundary="BOUNDARY" +MIME-Version: 1.0 + +--BOUNDARY + +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +message 1 + +--BOUNDARY + +Content-Type: text/plain; charset="us-ascii" +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit + +message 2 + +--BOUNDARY-- +''') + + def test_mime_attachments_in_constructor(self): + eq = self.assertEqual + text1 = MIMEText('') + text2 = MIMEText('') + msg = MIMEMultipart(_subparts=(text1, text2)) + eq(len(msg.get_payload()), 2) + eq(msg.get_payload(0), text1) + eq(msg.get_payload(1), text2) + + def test_default_multipart_constructor(self): + msg = MIMEMultipart() + self.assertTrue(msg.is_multipart()) + + +# A general test of parser->model->generator idempotency. IOW, read a message +# in, parse it into a message object tree, then without touching the tree, +# regenerate the plain text. The original text and the transformed text +# should be identical. Note: that we ignore the Unix-From since that may +# contain a changed date. +class TestIdempotent(TestEmailBase): + + linesep = '\n' + + def _msgobj(self, filename): + with openfile(filename) as fp: + data = fp.read() + msg = email.message_from_string(data) + return msg, data + + def _idempotent(self, msg, text, unixfrom=False): + eq = self.ndiffAssertEqual + s = StringIO() + g = Generator(s, maxheaderlen=0) + g.flatten(msg, unixfrom=unixfrom) + eq(text, s.getvalue()) + + def test_parse_text_message(self): + eq = self.assertEqual + msg, text = self._msgobj('msg_01.txt') + eq(msg.get_content_type(), 'text/plain') + eq(msg.get_content_maintype(), 'text') + eq(msg.get_content_subtype(), 'plain') + eq(msg.get_params()[1], ('charset', 'us-ascii')) + eq(msg.get_param('charset'), 'us-ascii') + eq(msg.preamble, None) + eq(msg.epilogue, None) + self._idempotent(msg, text) + + def test_parse_untyped_message(self): + eq = self.assertEqual + msg, text = self._msgobj('msg_03.txt') + eq(msg.get_content_type(), 'text/plain') + eq(msg.get_params(), None) + eq(msg.get_param('charset'), None) + self._idempotent(msg, text) + + def test_simple_multipart(self): + msg, text = self._msgobj('msg_04.txt') + self._idempotent(msg, text) + + def test_MIME_digest(self): + msg, text = self._msgobj('msg_02.txt') + self._idempotent(msg, text) + + def test_long_header(self): + msg, text = self._msgobj('msg_27.txt') + self._idempotent(msg, text) + + def test_MIME_digest_with_part_headers(self): + msg, text = self._msgobj('msg_28.txt') + self._idempotent(msg, text) + + def test_mixed_with_image(self): + msg, text = self._msgobj('msg_06.txt') + self._idempotent(msg, text) + + def test_multipart_report(self): + msg, text = self._msgobj('msg_05.txt') + self._idempotent(msg, text) + + def test_dsn(self): + msg, text = self._msgobj('msg_16.txt') + self._idempotent(msg, text) + + def test_preamble_epilogue(self): + msg, text = self._msgobj('msg_21.txt') + self._idempotent(msg, text) + + def test_multipart_one_part(self): + msg, text = self._msgobj('msg_23.txt') + self._idempotent(msg, text) + + def test_multipart_no_parts(self): + msg, text = self._msgobj('msg_24.txt') + self._idempotent(msg, text) + + def test_no_start_boundary(self): + msg, text = self._msgobj('msg_31.txt') + self._idempotent(msg, text) + + def test_rfc2231_charset(self): + msg, text = self._msgobj('msg_32.txt') + self._idempotent(msg, text) + + def test_more_rfc2231_parameters(self): + msg, text = self._msgobj('msg_33.txt') + self._idempotent(msg, text) + + def test_text_plain_in_a_multipart_digest(self): + msg, text = self._msgobj('msg_34.txt') + self._idempotent(msg, text) + + def test_nested_multipart_mixeds(self): + msg, text = self._msgobj('msg_12a.txt') + self._idempotent(msg, text) + + def test_message_external_body_idempotent(self): + msg, text = self._msgobj('msg_36.txt') + self._idempotent(msg, text) + + def test_message_delivery_status(self): + msg, text = self._msgobj('msg_43.txt') + self._idempotent(msg, text, unixfrom=True) + + def test_message_signed_idempotent(self): + msg, text = self._msgobj('msg_45.txt') + self._idempotent(msg, text) + + def test_content_type(self): + eq = self.assertEqual + unless = self.assertTrue + # Get a message object and reset the seek pointer for other tests + msg, text = self._msgobj('msg_05.txt') + eq(msg.get_content_type(), 'multipart/report') + # Test the Content-Type: parameters + params = {} + for pk, pv in msg.get_params(): + params[pk] = pv + eq(params['report-type'], 'delivery-status') + eq(params['boundary'], 'D1690A7AC1.996856090/mail.example.com') + eq(msg.preamble, 'This is a MIME-encapsulated message.' + self.linesep) + eq(msg.epilogue, self.linesep) + eq(len(msg.get_payload()), 3) + # Make sure the subparts are what we expect + msg1 = msg.get_payload(0) + eq(msg1.get_content_type(), 'text/plain') + eq(msg1.get_payload(), 'Yadda yadda yadda' + self.linesep) + msg2 = msg.get_payload(1) + eq(msg2.get_content_type(), 'text/plain') + eq(msg2.get_payload(), 'Yadda yadda yadda' + self.linesep) + msg3 = msg.get_payload(2) + eq(msg3.get_content_type(), 'message/rfc822') + self.assertTrue(isinstance(msg3, Message)) + payload = msg3.get_payload() + unless(isinstance(payload, list)) + eq(len(payload), 1) + msg4 = payload[0] + unless(isinstance(msg4, Message)) + eq(msg4.get_payload(), 'Yadda yadda yadda' + self.linesep) + + def test_parser(self): + eq = self.assertEqual + unless = self.assertTrue + msg, text = self._msgobj('msg_06.txt') + # Check some of the outer headers + eq(msg.get_content_type(), 'message/rfc822') + # Make sure the payload is a list of exactly one sub-Message, and that + # that submessage has a type of text/plain + payload = msg.get_payload() + unless(isinstance(payload, list)) + eq(len(payload), 1) + msg1 = payload[0] + self.assertTrue(isinstance(msg1, Message)) + eq(msg1.get_content_type(), 'text/plain') + self.assertTrue(isinstance(msg1.get_payload(), str)) + eq(msg1.get_payload(), self.linesep) + + + +# Test various other bits of the package's functionality +class TestMiscellaneous(TestEmailBase): + def test_message_from_string(self): + with openfile('msg_01.txt') as fp: + text = fp.read() + msg = email.message_from_string(text) + s = StringIO() + # Don't wrap/continue long headers since we're trying to test + # idempotency. + g = Generator(s, maxheaderlen=0) + g.flatten(msg) + self.assertEqual(text, s.getvalue()) + + def test_message_from_file(self): + with openfile('msg_01.txt') as fp: + text = fp.read() + fp.seek(0) + msg = email.message_from_file(fp) + s = StringIO() + # Don't wrap/continue long headers since we're trying to test + # idempotency. + g = Generator(s, maxheaderlen=0) + g.flatten(msg) + self.assertEqual(text, s.getvalue()) + + def test_message_from_string_with_class(self): + unless = self.assertTrue + with openfile('msg_01.txt') as fp: + text = fp.read() + + # Create a subclass + class MyMessage(Message): + pass + + msg = email.message_from_string(text, MyMessage) + unless(isinstance(msg, MyMessage)) + # Try something more complicated + with openfile('msg_02.txt') as fp: + text = fp.read() + msg = email.message_from_string(text, MyMessage) + for subpart in msg.walk(): + unless(isinstance(subpart, MyMessage)) + + def test_message_from_file_with_class(self): + unless = self.assertTrue + # Create a subclass + class MyMessage(Message): + pass + + with openfile('msg_01.txt') as fp: + msg = email.message_from_file(fp, MyMessage) + unless(isinstance(msg, MyMessage)) + # Try something more complicated + with openfile('msg_02.txt') as fp: + msg = email.message_from_file(fp, MyMessage) + for subpart in msg.walk(): + unless(isinstance(subpart, MyMessage)) + + def test__all__(self): + module = __import__('email') + # Can't use sorted() here due to Python 2.3 compatibility + all = module.__all__[:] + all.sort() + self.assertEqual(all, [ + 'base64mime', 'charset', 'encoders', 'errors', 'generator', + 'header', 'iterators', 'message', 'message_from_binary_file', + 'message_from_bytes', 'message_from_file', + 'message_from_string', 'mime', 'parser', + 'quoprimime', 'utils', + ]) + + def test_formatdate(self): + now = time.time() + self.assertEqual(utils.parsedate(utils.formatdate(now))[:6], + time.gmtime(now)[:6]) + + def test_formatdate_localtime(self): + now = time.time() + self.assertEqual( + utils.parsedate(utils.formatdate(now, localtime=True))[:6], + time.localtime(now)[:6]) + + def test_formatdate_usegmt(self): + now = time.time() + self.assertEqual( + utils.formatdate(now, localtime=False), + time.strftime('%a, %d %b %Y %H:%M:%S -0000', time.gmtime(now))) + self.assertEqual( + utils.formatdate(now, localtime=False, usegmt=True), + time.strftime('%a, %d %b %Y %H:%M:%S GMT', time.gmtime(now))) + + def test_parsedate_none(self): + self.assertEqual(utils.parsedate(''), None) + + def test_parsedate_compact(self): + # The FWS after the comma is optional + self.assertEqual(utils.parsedate('Wed,3 Apr 2002 14:58:26 +0800'), + utils.parsedate('Wed, 3 Apr 2002 14:58:26 +0800')) + + def test_parsedate_no_dayofweek(self): + eq = self.assertEqual + eq(utils.parsedate_tz('25 Feb 2003 13:47:26 -0800'), + (2003, 2, 25, 13, 47, 26, 0, 1, -1, -28800)) + + def test_parsedate_compact_no_dayofweek(self): + eq = self.assertEqual + eq(utils.parsedate_tz('5 Feb 2003 13:47:26 -0800'), + (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800)) + + def test_parsedate_no_space_before_positive_offset(self): + self.assertEqual(utils.parsedate_tz('Wed, 3 Apr 2002 14:58:26+0800'), + (2002, 4, 3, 14, 58, 26, 0, 1, -1, 28800)) + + def test_parsedate_no_space_before_negative_offset(self): + # Issue 1155362: we already handled '+' for this case. + self.assertEqual(utils.parsedate_tz('Wed, 3 Apr 2002 14:58:26-0800'), + (2002, 4, 3, 14, 58, 26, 0, 1, -1, -28800)) + + + def test_parsedate_accepts_time_with_dots(self): + eq = self.assertEqual + eq(utils.parsedate_tz('5 Feb 2003 13.47.26 -0800'), + (2003, 2, 5, 13, 47, 26, 0, 1, -1, -28800)) + eq(utils.parsedate_tz('5 Feb 2003 13.47 -0800'), + (2003, 2, 5, 13, 47, 0, 0, 1, -1, -28800)) + + def test_parsedate_acceptable_to_time_functions(self): + eq = self.assertEqual + timetup = utils.parsedate('5 Feb 2003 13:47:26 -0800') + t = int(time.mktime(timetup)) + eq(time.localtime(t)[:6], timetup[:6]) + eq(int(time.strftime('%Y', timetup)), 2003) + timetup = utils.parsedate_tz('5 Feb 2003 13:47:26 -0800') + t = int(time.mktime(timetup[:9])) + eq(time.localtime(t)[:6], timetup[:6]) + eq(int(time.strftime('%Y', timetup[:9])), 2003) + + def test_parsedate_y2k(self): + """Test for parsing a date with a two-digit year. + + Parsing a date with a two-digit year should return the correct + four-digit year. RFC822 allows two-digit years, but RFC2822 (which + obsoletes RFC822) requires four-digit years. + + """ + self.assertEqual(utils.parsedate_tz('25 Feb 03 13:47:26 -0800'), + utils.parsedate_tz('25 Feb 2003 13:47:26 -0800')) + self.assertEqual(utils.parsedate_tz('25 Feb 71 13:47:26 -0800'), + utils.parsedate_tz('25 Feb 1971 13:47:26 -0800')) + + def test_parseaddr_empty(self): + self.assertEqual(utils.parseaddr('<>'), ('', '')) + self.assertEqual(utils.formataddr(utils.parseaddr('<>')), '') + + def test_noquote_dump(self): + self.assertEqual( + utils.formataddr(('A Silly Person', 'person@dom.ain')), + 'A Silly Person <person@dom.ain>') + + def test_escape_dump(self): + self.assertEqual( + utils.formataddr(('A (Very) Silly Person', 'person@dom.ain')), + r'"A \(Very\) Silly Person" <person@dom.ain>') + a = r'A \(Special\) Person' + b = 'person@dom.ain' + self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b)) + + def test_escape_backslashes(self): + self.assertEqual( + utils.formataddr(('Arthur \Backslash\ Foobar', 'person@dom.ain')), + r'"Arthur \\Backslash\\ Foobar" <person@dom.ain>') + a = r'Arthur \Backslash\ Foobar' + b = 'person@dom.ain' + self.assertEqual(utils.parseaddr(utils.formataddr((a, b))), (a, b)) + + def test_quotes_unicode_names(self): + # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + name = "H\u00e4ns W\u00fcrst" + addr = 'person@dom.ain' + utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= <person@dom.ain>" + latin1_quopri = "=?iso-8859-1?q?H=E4ns_W=FCrst?= <person@dom.ain>" + self.assertEqual(utils.formataddr((name, addr)), utf8_base64) + self.assertEqual(utils.formataddr((name, addr), 'iso-8859-1'), + latin1_quopri) + + def test_accepts_any_charset_like_object(self): + # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + name = "H\u00e4ns W\u00fcrst" + addr = 'person@dom.ain' + utf8_base64 = "=?utf-8?b?SMOkbnMgV8O8cnN0?= <person@dom.ain>" + foobar = "FOOBAR" + class CharsetMock: + def header_encode(self, string): + return foobar + mock = CharsetMock() + mock_expected = "%s <%s>" % (foobar, addr) + self.assertEqual(utils.formataddr((name, addr), mock), mock_expected) + self.assertEqual(utils.formataddr((name, addr), Charset('utf-8')), + utf8_base64) + + def test_invalid_charset_like_object_raises_error(self): + # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + name = "H\u00e4ns W\u00fcrst" + addr = 'person@dom.ain' + # A object without a header_encode method: + bad_charset = object() + self.assertRaises(AttributeError, utils.formataddr, (name, addr), + bad_charset) + + def test_unicode_address_raises_error(self): + # issue 1690608. email.utils.formataddr() should be rfc2047 aware. + addr = 'pers\u00f6n@dom.in' + self.assertRaises(UnicodeError, utils.formataddr, (None, addr)) + self.assertRaises(UnicodeError, utils.formataddr, ("Name", addr)) + + def test_name_with_dot(self): + x = 'John X. Doe <jxd@example.com>' + y = '"John X. Doe" <jxd@example.com>' + a, b = ('John X. Doe', 'jxd@example.com') + self.assertEqual(utils.parseaddr(x), (a, b)) + self.assertEqual(utils.parseaddr(y), (a, b)) + # formataddr() quotes the name if there's a dot in it + self.assertEqual(utils.formataddr((a, b)), y) + + def test_parseaddr_preserves_quoted_pairs_in_addresses(self): + # issue 10005. Note that in the third test the second pair of + # backslashes is not actually a quoted pair because it is not inside a + # comment or quoted string: the address being parsed has a quoted + # string containing a quoted backslash, followed by 'example' and two + # backslashes, followed by another quoted string containing a space and + # the word 'example'. parseaddr copies those two backslashes + # literally. Per rfc5322 this is not technically correct since a \ may + # not appear in an address outside of a quoted string. It is probably + # a sensible Postel interpretation, though. + eq = self.assertEqual + eq(utils.parseaddr('""example" example"@example.com'), + ('', '""example" example"@example.com')) + eq(utils.parseaddr('"\\"example\\" example"@example.com'), + ('', '"\\"example\\" example"@example.com')) + eq(utils.parseaddr('"\\\\"example\\\\" example"@example.com'), + ('', '"\\\\"example\\\\" example"@example.com')) + + def test_parseaddr_preserves_spaces_in_local_part(self): + # issue 9286. A normal RFC5322 local part should not contain any + # folding white space, but legacy local parts can (they are a sequence + # of atoms, not dotatoms). On the other hand we strip whitespace from + # before the @ and around dots, on the assumption that the whitespace + # around the punctuation is a mistake in what would otherwise be + # an RFC5322 local part. Leading whitespace is, usual, stripped as well. + self.assertEqual(('', "merwok wok@xample.com"), + utils.parseaddr("merwok wok@xample.com")) + self.assertEqual(('', "merwok wok@xample.com"), + utils.parseaddr("merwok wok@xample.com")) + self.assertEqual(('', "merwok wok@xample.com"), + utils.parseaddr(" merwok wok @xample.com")) + self.assertEqual(('', 'merwok"wok" wok@xample.com'), + utils.parseaddr('merwok"wok" wok@xample.com')) + self.assertEqual(('', 'merwok.wok.wok@xample.com'), + utils.parseaddr('merwok. wok . wok@xample.com')) + + def test_multiline_from_comment(self): + x = """\ +Foo +\tBar <foo@example.com>""" + self.assertEqual(utils.parseaddr(x), ('Foo Bar', 'foo@example.com')) + + def test_quote_dump(self): + self.assertEqual( + utils.formataddr(('A Silly; Person', 'person@dom.ain')), + r'"A Silly; Person" <person@dom.ain>') + + def test_charset_richcomparisons(self): + eq = self.assertEqual + ne = self.assertNotEqual + cset1 = Charset() + cset2 = Charset() + eq(cset1, 'us-ascii') + eq(cset1, 'US-ASCII') + eq(cset1, 'Us-AsCiI') + eq('us-ascii', cset1) + eq('US-ASCII', cset1) + eq('Us-AsCiI', cset1) + ne(cset1, 'usascii') + ne(cset1, 'USASCII') + ne(cset1, 'UsAsCiI') + ne('usascii', cset1) + ne('USASCII', cset1) + ne('UsAsCiI', cset1) + eq(cset1, cset2) + eq(cset2, cset1) + + def test_getaddresses(self): + eq = self.assertEqual + eq(utils.getaddresses(['aperson@dom.ain (Al Person)', + 'Bud Person <bperson@dom.ain>']), + [('Al Person', 'aperson@dom.ain'), + ('Bud Person', 'bperson@dom.ain')]) + + def test_getaddresses_nasty(self): + eq = self.assertEqual + eq(utils.getaddresses(['foo: ;']), [('', '')]) + eq(utils.getaddresses( + ['[]*-- =~$']), + [('', ''), ('', ''), ('', '*--')]) + eq(utils.getaddresses( + ['foo: ;', '"Jason R. Mastaler" <jason@dom.ain>']), + [('', ''), ('Jason R. Mastaler', 'jason@dom.ain')]) + + def test_getaddresses_embedded_comment(self): + """Test proper handling of a nested comment""" + eq = self.assertEqual + addrs = utils.getaddresses(['User ((nested comment)) <foo@bar.com>']) + eq(addrs[0][1], 'foo@bar.com') + + def test_utils_quote_unquote(self): + eq = self.assertEqual + msg = Message() + msg.add_header('content-disposition', 'attachment', + filename='foo\\wacky"name') + eq(msg.get_filename(), 'foo\\wacky"name') + + def test_get_body_encoding_with_bogus_charset(self): + charset = Charset('not a charset') + self.assertEqual(charset.get_body_encoding(), 'base64') + + def test_get_body_encoding_with_uppercase_charset(self): + eq = self.assertEqual + msg = Message() + msg['Content-Type'] = 'text/plain; charset=UTF-8' + eq(msg['content-type'], 'text/plain; charset=UTF-8') + charsets = msg.get_charsets() + eq(len(charsets), 1) + eq(charsets[0], 'utf-8') + charset = Charset(charsets[0]) + eq(charset.get_body_encoding(), 'base64') + msg.set_payload(b'hello world', charset=charset) + eq(msg.get_payload(), 'aGVsbG8gd29ybGQ=\n') + eq(msg.get_payload(decode=True), b'hello world') + eq(msg['content-transfer-encoding'], 'base64') + # Try another one + msg = Message() + msg['Content-Type'] = 'text/plain; charset="US-ASCII"' + charsets = msg.get_charsets() + eq(len(charsets), 1) + eq(charsets[0], 'us-ascii') + charset = Charset(charsets[0]) + eq(charset.get_body_encoding(), encoders.encode_7or8bit) + msg.set_payload('hello world', charset=charset) + eq(msg.get_payload(), 'hello world') + eq(msg['content-transfer-encoding'], '7bit') + + def test_charsets_case_insensitive(self): + lc = Charset('us-ascii') + uc = Charset('US-ASCII') + self.assertEqual(lc.get_body_encoding(), uc.get_body_encoding()) + + def test_partial_falls_inside_message_delivery_status(self): + eq = self.ndiffAssertEqual + # The Parser interface provides chunks of data to FeedParser in 8192 + # byte gulps. SF bug #1076485 found one of those chunks inside + # message/delivery-status header block, which triggered an + # unreadline() of NeedMoreData. + msg = self._msgobj('msg_43.txt') + sfp = StringIO() + iterators._structure(msg, sfp) + eq(sfp.getvalue(), """\ +multipart/report + text/plain + message/delivery-status + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/plain + text/rfc822-headers +""") + + def test_make_msgid_domain(self): + self.assertEqual( + email.utils.make_msgid(domain='testdomain-string')[-19:], + '@testdomain-string>') + + +# Test the iterator/generators +class TestIterators(TestEmailBase): + def test_body_line_iterator(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + # First a simple non-multipart message + msg = self._msgobj('msg_01.txt') + it = iterators.body_line_iterator(msg) + lines = list(it) + eq(len(lines), 6) + neq(EMPTYSTRING.join(lines), msg.get_payload()) + # Now a more complicated multipart + msg = self._msgobj('msg_02.txt') + it = iterators.body_line_iterator(msg) + lines = list(it) + eq(len(lines), 43) + with openfile('msg_19.txt') as fp: + neq(EMPTYSTRING.join(lines), fp.read()) + + def test_typed_subpart_iterator(self): + eq = self.assertEqual + msg = self._msgobj('msg_04.txt') + it = iterators.typed_subpart_iterator(msg, 'text') + lines = [] + subparts = 0 + for subpart in it: + subparts += 1 + lines.append(subpart.get_payload()) + eq(subparts, 2) + eq(EMPTYSTRING.join(lines), """\ +a simple kind of mirror +to reflect upon our own +a simple kind of mirror +to reflect upon our own +""") + + def test_typed_subpart_iterator_default_type(self): + eq = self.assertEqual + msg = self._msgobj('msg_03.txt') + it = iterators.typed_subpart_iterator(msg, 'text', 'plain') + lines = [] + subparts = 0 + for subpart in it: + subparts += 1 + lines.append(subpart.get_payload()) + eq(subparts, 1) + eq(EMPTYSTRING.join(lines), """\ + +Hi, + +Do you like this message? + +-Me +""") + + def test_pushCR_LF(self): + '''FeedParser BufferedSubFile.push() assumed it received complete + line endings. A CR ending one push() followed by a LF starting + the next push() added an empty line. + ''' + imt = [ + ("a\r \n", 2), + ("b", 0), + ("c\n", 1), + ("", 0), + ("d\r\n", 1), + ("e\r", 0), + ("\nf", 1), + ("\r\n", 1), + ] + from email.feedparser import BufferedSubFile, NeedMoreData + bsf = BufferedSubFile() + om = [] + nt = 0 + for il, n in imt: + bsf.push(il) + nt += n + n1 = 0 + while True: + ol = bsf.readline() + if ol == NeedMoreData: + break + om.append(ol) + n1 += 1 + self.assertTrue(n == n1) + self.assertTrue(len(om) == nt) + self.assertTrue(''.join([il for il, n in imt]) == ''.join(om)) + + + +class TestParsers(TestEmailBase): + + def test_header_parser(self): + eq = self.assertEqual + # Parse only the headers of a complex multipart MIME document + with openfile('msg_02.txt') as fp: + msg = HeaderParser().parse(fp) + eq(msg['from'], 'ppp-request@zzz.org') + eq(msg['to'], 'ppp@zzz.org') + eq(msg.get_content_type(), 'multipart/mixed') + self.assertFalse(msg.is_multipart()) + self.assertTrue(isinstance(msg.get_payload(), str)) + + def test_bytes_header_parser(self): + eq = self.assertEqual + # Parse only the headers of a complex multipart MIME document + with openfile('msg_02.txt', 'rb') as fp: + msg = email.parser.BytesHeaderParser().parse(fp) + eq(msg['from'], 'ppp-request@zzz.org') + eq(msg['to'], 'ppp@zzz.org') + eq(msg.get_content_type(), 'multipart/mixed') + self.assertFalse(msg.is_multipart()) + self.assertTrue(isinstance(msg.get_payload(), str)) + self.assertTrue(isinstance(msg.get_payload(decode=True), bytes)) + + def test_whitespace_continuation(self): + eq = self.assertEqual + # This message contains a line after the Subject: header that has only + # whitespace, but it is not empty! + msg = email.message_from_string("""\ +From: aperson@dom.ain +To: bperson@dom.ain +Subject: the next line has a space on it +\x20 +Date: Mon, 8 Apr 2002 15:09:19 -0400 +Message-ID: spam + +Here's the message body +""") + eq(msg['subject'], 'the next line has a space on it\n ') + eq(msg['message-id'], 'spam') + eq(msg.get_payload(), "Here's the message body\n") + + def test_whitespace_continuation_last_header(self): + eq = self.assertEqual + # Like the previous test, but the subject line is the last + # header. + msg = email.message_from_string("""\ +From: aperson@dom.ain +To: bperson@dom.ain +Date: Mon, 8 Apr 2002 15:09:19 -0400 +Message-ID: spam +Subject: the next line has a space on it +\x20 + +Here's the message body +""") + eq(msg['subject'], 'the next line has a space on it\n ') + eq(msg['message-id'], 'spam') + eq(msg.get_payload(), "Here's the message body\n") + + def test_crlf_separation(self): + eq = self.assertEqual + with openfile('msg_26.txt', newline='\n') as fp: + msg = Parser().parse(fp) + eq(len(msg.get_payload()), 2) + part1 = msg.get_payload(0) + eq(part1.get_content_type(), 'text/plain') + eq(part1.get_payload(), 'Simple email with attachment.\r\n\r\n') + part2 = msg.get_payload(1) + eq(part2.get_content_type(), 'application/riscos') + + def test_crlf_flatten(self): + # Using newline='\n' preserves the crlfs in this input file. + with openfile('msg_26.txt', newline='\n') as fp: + text = fp.read() + msg = email.message_from_string(text) + s = StringIO() + g = Generator(s) + g.flatten(msg, linesep='\r\n') + self.assertEqual(s.getvalue(), text) + + def test_crlf_control_via_policy(self): + with openfile('msg_26.txt', newline='\n') as fp: + text = fp.read() + msg = email.message_from_string(text) + s = StringIO() + g = email.generator.Generator(s, policy=email.policy.SMTP) + g.flatten(msg) + self.assertEqual(s.getvalue(), text) + + def test_flatten_linesep_overrides_policy(self): + # msg_27 is lf separated + with openfile('msg_27.txt', newline='\n') as fp: + text = fp.read() + msg = email.message_from_string(text) + s = StringIO() + g = email.generator.Generator(s, policy=email.policy.SMTP) + g.flatten(msg, linesep='\n') + self.assertEqual(s.getvalue(), text) + + maxDiff = None + + def test_multipart_digest_with_extra_mime_headers(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + with openfile('msg_28.txt') as fp: + msg = email.message_from_file(fp) + # Structure is: + # multipart/digest + # message/rfc822 + # text/plain + # message/rfc822 + # text/plain + eq(msg.is_multipart(), 1) + eq(len(msg.get_payload()), 2) + part1 = msg.get_payload(0) + eq(part1.get_content_type(), 'message/rfc822') + eq(part1.is_multipart(), 1) + eq(len(part1.get_payload()), 1) + part1a = part1.get_payload(0) + eq(part1a.is_multipart(), 0) + eq(part1a.get_content_type(), 'text/plain') + neq(part1a.get_payload(), 'message 1\n') + # next message/rfc822 + part2 = msg.get_payload(1) + eq(part2.get_content_type(), 'message/rfc822') + eq(part2.is_multipart(), 1) + eq(len(part2.get_payload()), 1) + part2a = part2.get_payload(0) + eq(part2a.is_multipart(), 0) + eq(part2a.get_content_type(), 'text/plain') + neq(part2a.get_payload(), 'message 2\n') + + def test_three_lines(self): + # A bug report by Andrew McNamara + lines = ['From: Andrew Person <aperson@dom.ain', + 'Subject: Test', + 'Date: Tue, 20 Aug 2002 16:43:45 +1000'] + msg = email.message_from_string(NL.join(lines)) + self.assertEqual(msg['date'], 'Tue, 20 Aug 2002 16:43:45 +1000') + + def test_strip_line_feed_and_carriage_return_in_headers(self): + eq = self.assertEqual + # For [ 1002475 ] email message parser doesn't handle \r\n correctly + value1 = 'text' + value2 = 'more text' + m = 'Header: %s\r\nNext-Header: %s\r\n\r\nBody\r\n\r\n' % ( + value1, value2) + msg = email.message_from_string(m) + eq(msg.get('Header'), value1) + eq(msg.get('Next-Header'), value2) + + def test_rfc2822_header_syntax(self): + eq = self.assertEqual + m = '>From: foo\nFrom: bar\n!"#QUX;~: zoo\n\nbody' + msg = email.message_from_string(m) + eq(len(msg), 3) + eq(sorted(field for field in msg), ['!"#QUX;~', '>From', 'From']) + eq(msg.get_payload(), 'body') + + def test_rfc2822_space_not_allowed_in_header(self): + eq = self.assertEqual + m = '>From foo@example.com 11:25:53\nFrom: bar\n!"#QUX;~: zoo\n\nbody' + msg = email.message_from_string(m) + eq(len(msg.keys()), 0) + + def test_rfc2822_one_character_header(self): + eq = self.assertEqual + m = 'A: first header\nB: second header\nCC: third header\n\nbody' + msg = email.message_from_string(m) + headers = msg.keys() + headers.sort() + eq(headers, ['A', 'B', 'CC']) + eq(msg.get_payload(), 'body') + + def test_CRLFLF_at_end_of_part(self): + # issue 5610: feedparser should not eat two chars from body part ending + # with "\r\n\n". + m = ( + "From: foo@bar.com\n" + "To: baz\n" + "Mime-Version: 1.0\n" + "Content-Type: multipart/mixed; boundary=BOUNDARY\n" + "\n" + "--BOUNDARY\n" + "Content-Type: text/plain\n" + "\n" + "body ending with CRLF newline\r\n" + "\n" + "--BOUNDARY--\n" + ) + msg = email.message_from_string(m) + self.assertTrue(msg.get_payload(0).get_payload().endswith('\r\n')) + + +class Test8BitBytesHandling(unittest.TestCase): + # In Python3 all input is string, but that doesn't work if the actual input + # uses an 8bit transfer encoding. To hack around that, in email 5.1 we + # decode byte streams using the surrogateescape error handler, and + # reconvert to binary at appropriate places if we detect surrogates. This + # doesn't allow us to transform headers with 8bit bytes (they get munged), + # but it does allow us to parse and preserve them, and to decode body + # parts that use an 8bit CTE. + + bodytest_msg = textwrap.dedent("""\ + From: foo@bar.com + To: baz + Mime-Version: 1.0 + Content-Type: text/plain; charset={charset} + Content-Transfer-Encoding: {cte} + + {bodyline} + """) + + def test_known_8bit_CTE(self): + m = self.bodytest_msg.format(charset='utf-8', + cte='8bit', + bodyline='pöstal').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(), "pöstal\n") + self.assertEqual(msg.get_payload(decode=True), + "pöstal\n".encode('utf-8')) + + def test_unknown_8bit_CTE(self): + m = self.bodytest_msg.format(charset='notavalidcharset', + cte='8bit', + bodyline='pöstal').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(), "p\uFFFD\uFFFDstal\n") + self.assertEqual(msg.get_payload(decode=True), + "pöstal\n".encode('utf-8')) + + def test_8bit_in_quopri_body(self): + # This is non-RFC compliant data...without 'decode' the library code + # decodes the body using the charset from the headers, and because the + # source byte really is utf-8 this works. This is likely to fail + # against real dirty data (ie: produce mojibake), but the data is + # invalid anyway so it is as good a guess as any. But this means that + # this test just confirms the current behavior; that behavior is not + # necessarily the best possible behavior. With 'decode' it is + # returning the raw bytes, so that test should be of correct behavior, + # or at least produce the same result that email4 did. + m = self.bodytest_msg.format(charset='utf-8', + cte='quoted-printable', + bodyline='p=C3=B6stál').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(), 'p=C3=B6stál\n') + self.assertEqual(msg.get_payload(decode=True), + 'pöstál\n'.encode('utf-8')) + + def test_invalid_8bit_in_non_8bit_cte_uses_replace(self): + # This is similar to the previous test, but proves that if the 8bit + # byte is undecodeable in the specified charset, it gets replaced + # by the unicode 'unknown' character. Again, this may or may not + # be the ideal behavior. Note that if decode=False none of the + # decoders will get involved, so this is the only test we need + # for this behavior. + m = self.bodytest_msg.format(charset='ascii', + cte='quoted-printable', + bodyline='p=C3=B6stál').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(), 'p=C3=B6st\uFFFD\uFFFDl\n') + self.assertEqual(msg.get_payload(decode=True), + 'pöstál\n'.encode('utf-8')) + + def test_8bit_in_base64_body(self): + # Sticking an 8bit byte in a base64 block makes it undecodable by + # normal means, so the block is returned undecoded, but as bytes. + m = self.bodytest_msg.format(charset='utf-8', + cte='base64', + bodyline='cMO2c3RhbAá=').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(decode=True), + 'cMO2c3RhbAá=\n'.encode('utf-8')) + + def test_8bit_in_uuencode_body(self): + # Sticking an 8bit byte in a uuencode block makes it undecodable by + # normal means, so the block is returned undecoded, but as bytes. + m = self.bodytest_msg.format(charset='utf-8', + cte='uuencode', + bodyline='<,.V<W1A; á ').encode('utf-8') + msg = email.message_from_bytes(m) + self.assertEqual(msg.get_payload(decode=True), + '<,.V<W1A; á \n'.encode('utf-8')) + + + headertest_headers = ( + ('From: foo@bar.com', ('From', 'foo@bar.com')), + ('To: báz', ('To', '=?unknown-8bit?q?b=C3=A1z?=')), + ('Subject: Maintenant je vous présente mon collègue, le pouf célèbre\n' + '\tJean de Baddie', + ('Subject', '=?unknown-8bit?q?Maintenant_je_vous_pr=C3=A9sente_mon_' + 'coll=C3=A8gue=2C_le_pouf_c=C3=A9l=C3=A8bre?=\n' + ' =?unknown-8bit?q?_Jean_de_Baddie?=')), + ('From: göst', ('From', '=?unknown-8bit?b?Z8O2c3Q=?=')), + ) + headertest_msg = ('\n'.join([src for (src, _) in headertest_headers]) + + '\nYes, they are flying.\n').encode('utf-8') + + def test_get_8bit_header(self): + msg = email.message_from_bytes(self.headertest_msg) + self.assertEqual(str(msg.get('to')), 'b\uFFFD\uFFFDz') + self.assertEqual(str(msg['to']), 'b\uFFFD\uFFFDz') + + def test_print_8bit_headers(self): + msg = email.message_from_bytes(self.headertest_msg) + self.assertEqual(str(msg), + textwrap.dedent("""\ + From: {} + To: {} + Subject: {} + From: {} + + Yes, they are flying. + """).format(*[expected[1] for (_, expected) in + self.headertest_headers])) + + def test_values_with_8bit_headers(self): + msg = email.message_from_bytes(self.headertest_msg) + self.assertListEqual([str(x) for x in msg.values()], + ['foo@bar.com', + 'b\uFFFD\uFFFDz', + 'Maintenant je vous pr\uFFFD\uFFFDsente mon ' + 'coll\uFFFD\uFFFDgue, le pouf ' + 'c\uFFFD\uFFFDl\uFFFD\uFFFDbre\n' + '\tJean de Baddie', + "g\uFFFD\uFFFDst"]) + + def test_items_with_8bit_headers(self): + msg = email.message_from_bytes(self.headertest_msg) + self.assertListEqual([(str(x), str(y)) for (x, y) in msg.items()], + [('From', 'foo@bar.com'), + ('To', 'b\uFFFD\uFFFDz'), + ('Subject', 'Maintenant je vous ' + 'pr\uFFFD\uFFFDsente ' + 'mon coll\uFFFD\uFFFDgue, le pouf ' + 'c\uFFFD\uFFFDl\uFFFD\uFFFDbre\n' + '\tJean de Baddie'), + ('From', 'g\uFFFD\uFFFDst')]) + + def test_get_all_with_8bit_headers(self): + msg = email.message_from_bytes(self.headertest_msg) + self.assertListEqual([str(x) for x in msg.get_all('from')], + ['foo@bar.com', + 'g\uFFFD\uFFFDst']) + + def test_get_content_type_with_8bit(self): + msg = email.message_from_bytes(textwrap.dedent("""\ + Content-Type: text/pl\xA7in; charset=utf-8 + """).encode('latin-1')) + self.assertEqual(msg.get_content_type(), "text/pl\uFFFDin") + self.assertEqual(msg.get_content_maintype(), "text") + self.assertEqual(msg.get_content_subtype(), "pl\uFFFDin") + + def test_get_params_with_8bit(self): + msg = email.message_from_bytes( + 'X-Header: foo=\xa7ne; b\xa7r=two; baz=three\n'.encode('latin-1')) + self.assertEqual(msg.get_params(header='x-header'), + [('foo', '\uFFFDne'), ('b\uFFFDr', 'two'), ('baz', 'three')]) + self.assertEqual(msg.get_param('Foo', header='x-header'), '\uFFFdne') + # XXX: someday you might be able to get 'b\xa7r', for now you can't. + self.assertEqual(msg.get_param('b\xa7r', header='x-header'), None) + + def test_get_rfc2231_params_with_8bit(self): + msg = email.message_from_bytes(textwrap.dedent("""\ + Content-Type: text/plain; charset=us-ascii; + title*=us-ascii'en'This%20is%20not%20f\xa7n""" + ).encode('latin-1')) + self.assertEqual(msg.get_param('title'), + ('us-ascii', 'en', 'This is not f\uFFFDn')) + + def test_set_rfc2231_params_with_8bit(self): + msg = email.message_from_bytes(textwrap.dedent("""\ + Content-Type: text/plain; charset=us-ascii; + title*=us-ascii'en'This%20is%20not%20f\xa7n""" + ).encode('latin-1')) + msg.set_param('title', 'test') + self.assertEqual(msg.get_param('title'), 'test') + + def test_del_rfc2231_params_with_8bit(self): + msg = email.message_from_bytes(textwrap.dedent("""\ + Content-Type: text/plain; charset=us-ascii; + title*=us-ascii'en'This%20is%20not%20f\xa7n""" + ).encode('latin-1')) + msg.del_param('title') + self.assertEqual(msg.get_param('title'), None) + self.assertEqual(msg.get_content_maintype(), 'text') + + def test_get_payload_with_8bit_cte_header(self): + msg = email.message_from_bytes(textwrap.dedent("""\ + Content-Transfer-Encoding: b\xa7se64 + Content-Type: text/plain; charset=latin-1 + + payload + """).encode('latin-1')) + self.assertEqual(msg.get_payload(), 'payload\n') + self.assertEqual(msg.get_payload(decode=True), b'payload\n') + + non_latin_bin_msg = textwrap.dedent("""\ + From: foo@bar.com + To: báz + Subject: Maintenant je vous présente mon collègue, le pouf célèbre + \tJean de Baddie + Mime-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + + Да, они летят. + """).encode('utf-8') + + def test_bytes_generator(self): + msg = email.message_from_bytes(self.non_latin_bin_msg) + out = BytesIO() + email.generator.BytesGenerator(out).flatten(msg) + self.assertEqual(out.getvalue(), self.non_latin_bin_msg) + + def test_bytes_generator_handles_None_body(self): + #Issue 11019 + msg = email.message.Message() + out = BytesIO() + email.generator.BytesGenerator(out).flatten(msg) + self.assertEqual(out.getvalue(), b"\n") + + non_latin_bin_msg_as7bit_wrapped = textwrap.dedent("""\ + From: foo@bar.com + To: =?unknown-8bit?q?b=C3=A1z?= + Subject: =?unknown-8bit?q?Maintenant_je_vous_pr=C3=A9sente_mon_coll=C3=A8gue?= + =?unknown-8bit?q?=2C_le_pouf_c=C3=A9l=C3=A8bre?= + =?unknown-8bit?q?_Jean_de_Baddie?= + Mime-Version: 1.0 + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: base64 + + 0JTQsCwg0L7QvdC4INC70LXRgtGP0YIuCg== + """) + + def test_generator_handles_8bit(self): + msg = email.message_from_bytes(self.non_latin_bin_msg) + out = StringIO() + email.generator.Generator(out).flatten(msg) + self.assertEqual(out.getvalue(), self.non_latin_bin_msg_as7bit_wrapped) + + def test_bytes_generator_with_unix_from(self): + # The unixfrom contains a current date, so we can't check it + # literally. Just make sure the first word is 'From' and the + # rest of the message matches the input. + msg = email.message_from_bytes(self.non_latin_bin_msg) + out = BytesIO() + email.generator.BytesGenerator(out).flatten(msg, unixfrom=True) + lines = out.getvalue().split(b'\n') + self.assertEqual(lines[0].split()[0], b'From') + self.assertEqual(b'\n'.join(lines[1:]), self.non_latin_bin_msg) + + non_latin_bin_msg_as7bit = non_latin_bin_msg_as7bit_wrapped.split('\n') + non_latin_bin_msg_as7bit[2:4] = [ + 'Subject: =?unknown-8bit?q?Maintenant_je_vous_pr=C3=A9sente_mon_' + 'coll=C3=A8gue=2C_le_pouf_c=C3=A9l=C3=A8bre?='] + non_latin_bin_msg_as7bit = '\n'.join(non_latin_bin_msg_as7bit) + + def test_message_from_binary_file(self): + fn = 'test.msg' + self.addCleanup(unlink, fn) + with open(fn, 'wb') as testfile: + testfile.write(self.non_latin_bin_msg) + with open(fn, 'rb') as testfile: + m = email.parser.BytesParser().parse(testfile) + self.assertEqual(str(m), self.non_latin_bin_msg_as7bit) + + latin_bin_msg = textwrap.dedent("""\ + From: foo@bar.com + To: Dinsdale + Subject: Nudge nudge, wink, wink + Mime-Version: 1.0 + Content-Type: text/plain; charset="latin-1" + Content-Transfer-Encoding: 8bit + + oh là là, know what I mean, know what I mean? + """).encode('latin-1') + + latin_bin_msg_as7bit = textwrap.dedent("""\ + From: foo@bar.com + To: Dinsdale + Subject: Nudge nudge, wink, wink + Mime-Version: 1.0 + Content-Type: text/plain; charset="iso-8859-1" + Content-Transfer-Encoding: quoted-printable + + oh l=E0 l=E0, know what I mean, know what I mean? + """) + + def test_string_generator_reencodes_to_quopri_when_appropriate(self): + m = email.message_from_bytes(self.latin_bin_msg) + self.assertEqual(str(m), self.latin_bin_msg_as7bit) + + def test_decoded_generator_emits_unicode_body(self): + m = email.message_from_bytes(self.latin_bin_msg) + out = StringIO() + email.generator.DecodedGenerator(out).flatten(m) + #DecodedHeader output contains an extra blank line compared + #to the input message. RDM: not sure if this is a bug or not, + #but it is not specific to the 8bit->7bit conversion. + self.assertEqual(out.getvalue(), + self.latin_bin_msg.decode('latin-1')+'\n') + + def test_bytes_feedparser(self): + bfp = email.feedparser.BytesFeedParser() + for i in range(0, len(self.latin_bin_msg), 10): + bfp.feed(self.latin_bin_msg[i:i+10]) + m = bfp.close() + self.assertEqual(str(m), self.latin_bin_msg_as7bit) + + def test_crlf_flatten(self): + with openfile('msg_26.txt', 'rb') as fp: + text = fp.read() + msg = email.message_from_bytes(text) + s = BytesIO() + g = email.generator.BytesGenerator(s) + g.flatten(msg, linesep='\r\n') + self.assertEqual(s.getvalue(), text) + + def test_8bit_multipart(self): + # Issue 11605 + source = textwrap.dedent("""\ + Date: Fri, 18 Mar 2011 17:15:43 +0100 + To: foo@example.com + From: foodwatch-Newsletter <bar@example.com> + Subject: Aktuelles zu Japan, Klonfleisch und Smiley-System + Message-ID: <76a486bee62b0d200f33dc2ca08220ad@localhost.localdomain> + MIME-Version: 1.0 + Content-Type: multipart/alternative; + boundary="b1_76a486bee62b0d200f33dc2ca08220ad" + + --b1_76a486bee62b0d200f33dc2ca08220ad + Content-Type: text/plain; charset="utf-8" + Content-Transfer-Encoding: 8bit + + Guten Tag, , + + mit großer Betroffenheit verfolgen auch wir im foodwatch-Team die + Nachrichten aus Japan. + + + --b1_76a486bee62b0d200f33dc2ca08220ad + Content-Type: text/html; charset="utf-8" + Content-Transfer-Encoding: 8bit + + <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" + "http://www.w3.org/TR/html4/loose.dtd"> + <html lang="de"> + <head> + <title>foodwatch - Newsletter</title> + </head> + <body> + <p>mit großer Betroffenheit verfolgen auch wir im foodwatch-Team + die Nachrichten aus Japan.</p> + </body> + </html> + --b1_76a486bee62b0d200f33dc2ca08220ad-- + + """).encode('utf-8') + msg = email.message_from_bytes(source) + s = BytesIO() + g = email.generator.BytesGenerator(s) + g.flatten(msg) + self.assertEqual(s.getvalue(), source) + + def test_crlf_control_via_policy(self): + # msg_26 is crlf terminated + with openfile('msg_26.txt', 'rb') as fp: + text = fp.read() + msg = email.message_from_bytes(text) + s = BytesIO() + g = email.generator.BytesGenerator(s, policy=email.policy.SMTP) + g.flatten(msg) + self.assertEqual(s.getvalue(), text) + + def test_flatten_linesep_overrides_policy(self): + # msg_27 is lf separated + with openfile('msg_27.txt', 'rb') as fp: + text = fp.read() + msg = email.message_from_bytes(text) + s = BytesIO() + g = email.generator.BytesGenerator(s, policy=email.policy.SMTP) + g.flatten(msg, linesep='\n') + self.assertEqual(s.getvalue(), text) + + def test_must_be_7bit_handles_unknown_8bit(self): + msg = email.message_from_bytes(self.non_latin_bin_msg) + out = BytesIO() + g = email.generator.BytesGenerator(out, + policy=email.policy.default.clone(must_be_7bit=True)) + g.flatten(msg) + self.assertEqual(out.getvalue(), + self.non_latin_bin_msg_as7bit_wrapped.encode('ascii')) + + def test_must_be_7bit_transforms_8bit_cte(self): + msg = email.message_from_bytes(self.latin_bin_msg) + out = BytesIO() + g = email.generator.BytesGenerator(out, + policy=email.policy.default.clone(must_be_7bit=True)) + g.flatten(msg) + self.assertEqual(out.getvalue(), + self.latin_bin_msg_as7bit.encode('ascii')) + + maxDiff = None + + +class BaseTestBytesGeneratorIdempotent: + + maxDiff = None + + def _msgobj(self, filename): + with openfile(filename, 'rb') as fp: + data = fp.read() + data = self.normalize_linesep_regex.sub(self.blinesep, data) + msg = email.message_from_bytes(data) + return msg, data + + def _idempotent(self, msg, data, unixfrom=False): + b = BytesIO() + g = email.generator.BytesGenerator(b, maxheaderlen=0) + g.flatten(msg, unixfrom=unixfrom, linesep=self.linesep) + self.assertEqual(data, b.getvalue()) + + +class TestBytesGeneratorIdempotentNL(BaseTestBytesGeneratorIdempotent, + TestIdempotent): + linesep = '\n' + blinesep = b'\n' + normalize_linesep_regex = re.compile(br'\r\n') + + +class TestBytesGeneratorIdempotentCRLF(BaseTestBytesGeneratorIdempotent, + TestIdempotent): + linesep = '\r\n' + blinesep = b'\r\n' + normalize_linesep_regex = re.compile(br'(?<!\r)\n') + + +class TestBase64(unittest.TestCase): + def test_len(self): + eq = self.assertEqual + eq(base64mime.header_length('hello'), + len(base64mime.body_encode(b'hello', eol=''))) + for size in range(15): + if size == 0 : bsize = 0 + elif size <= 3 : bsize = 4 + elif size <= 6 : bsize = 8 + elif size <= 9 : bsize = 12 + elif size <= 12: bsize = 16 + else : bsize = 20 + eq(base64mime.header_length('x' * size), bsize) + + def test_decode(self): + eq = self.assertEqual + eq(base64mime.decode(''), b'') + eq(base64mime.decode('aGVsbG8='), b'hello') + + def test_encode(self): + eq = self.assertEqual + eq(base64mime.body_encode(b''), b'') + eq(base64mime.body_encode(b'hello'), 'aGVsbG8=\n') + # Test the binary flag + eq(base64mime.body_encode(b'hello\n'), 'aGVsbG8K\n') + # Test the maxlinelen arg + eq(base64mime.body_encode(b'xxxx ' * 20, maxlinelen=40), """\ +eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg +eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg +eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg +eHh4eCB4eHh4IA== +""") + # Test the eol argument + eq(base64mime.body_encode(b'xxxx ' * 20, maxlinelen=40, eol='\r\n'), + """\ +eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r +eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r +eHh4eCB4eHh4IHh4eHggeHh4eCB4eHh4IHh4eHgg\r +eHh4eCB4eHh4IA==\r +""") + + def test_header_encode(self): + eq = self.assertEqual + he = base64mime.header_encode + eq(he('hello'), '=?iso-8859-1?b?aGVsbG8=?=') + eq(he('hello\r\nworld'), '=?iso-8859-1?b?aGVsbG8NCndvcmxk?=') + eq(he('hello\nworld'), '=?iso-8859-1?b?aGVsbG8Kd29ybGQ=?=') + # Test the charset option + eq(he('hello', charset='iso-8859-2'), '=?iso-8859-2?b?aGVsbG8=?=') + eq(he('hello\nworld'), '=?iso-8859-1?b?aGVsbG8Kd29ybGQ=?=') + + + +class TestQuopri(unittest.TestCase): + def setUp(self): + # Set of characters (as byte integers) that don't need to be encoded + # in headers. + self.hlit = list(chain( + range(ord('a'), ord('z') + 1), + range(ord('A'), ord('Z') + 1), + range(ord('0'), ord('9') + 1), + (c for c in b'!*+-/'))) + # Set of characters (as byte integers) that do need to be encoded in + # headers. + self.hnon = [c for c in range(256) if c not in self.hlit] + assert len(self.hlit) + len(self.hnon) == 256 + # Set of characters (as byte integers) that don't need to be encoded + # in bodies. + self.blit = list(range(ord(' '), ord('~') + 1)) + self.blit.append(ord('\t')) + self.blit.remove(ord('=')) + # Set of characters (as byte integers) that do need to be encoded in + # bodies. + self.bnon = [c for c in range(256) if c not in self.blit] + assert len(self.blit) + len(self.bnon) == 256 + + def test_quopri_header_check(self): + for c in self.hlit: + self.assertFalse(quoprimime.header_check(c), + 'Should not be header quopri encoded: %s' % chr(c)) + for c in self.hnon: + self.assertTrue(quoprimime.header_check(c), + 'Should be header quopri encoded: %s' % chr(c)) + + def test_quopri_body_check(self): + for c in self.blit: + self.assertFalse(quoprimime.body_check(c), + 'Should not be body quopri encoded: %s' % chr(c)) + for c in self.bnon: + self.assertTrue(quoprimime.body_check(c), + 'Should be body quopri encoded: %s' % chr(c)) + + def test_header_quopri_len(self): + eq = self.assertEqual + eq(quoprimime.header_length(b'hello'), 5) + # RFC 2047 chrome is not included in header_length(). + eq(len(quoprimime.header_encode(b'hello', charset='xxx')), + quoprimime.header_length(b'hello') + + # =?xxx?q?...?= means 10 extra characters + 10) + eq(quoprimime.header_length(b'h@e@l@l@o@'), 20) + # RFC 2047 chrome is not included in header_length(). + eq(len(quoprimime.header_encode(b'h@e@l@l@o@', charset='xxx')), + quoprimime.header_length(b'h@e@l@l@o@') + + # =?xxx?q?...?= means 10 extra characters + 10) + for c in self.hlit: + eq(quoprimime.header_length(bytes([c])), 1, + 'expected length 1 for %r' % chr(c)) + for c in self.hnon: + # Space is special; it's encoded to _ + if c == ord(' '): + continue + eq(quoprimime.header_length(bytes([c])), 3, + 'expected length 3 for %r' % chr(c)) + eq(quoprimime.header_length(b' '), 1) + + def test_body_quopri_len(self): + eq = self.assertEqual + for c in self.blit: + eq(quoprimime.body_length(bytes([c])), 1) + for c in self.bnon: + eq(quoprimime.body_length(bytes([c])), 3) + + def test_quote_unquote_idempotent(self): + for x in range(256): + c = chr(x) + self.assertEqual(quoprimime.unquote(quoprimime.quote(c)), c) + + def _test_header_encode(self, header, expected_encoded_header, charset=None): + if charset is None: + encoded_header = quoprimime.header_encode(header) + else: + encoded_header = quoprimime.header_encode(header, charset) + self.assertEqual(encoded_header, expected_encoded_header) + + def test_header_encode_null(self): + self._test_header_encode(b'', '') + + def test_header_encode_one_word(self): + self._test_header_encode(b'hello', '=?iso-8859-1?q?hello?=') + + def test_header_encode_two_lines(self): + self._test_header_encode(b'hello\nworld', + '=?iso-8859-1?q?hello=0Aworld?=') + + def test_header_encode_non_ascii(self): + self._test_header_encode(b'hello\xc7there', + '=?iso-8859-1?q?hello=C7there?=') + + def test_header_encode_alt_charset(self): + self._test_header_encode(b'hello', '=?iso-8859-2?q?hello?=', + charset='iso-8859-2') + + def _test_header_decode(self, encoded_header, expected_decoded_header): + decoded_header = quoprimime.header_decode(encoded_header) + self.assertEqual(decoded_header, expected_decoded_header) + + def test_header_decode_null(self): + self._test_header_decode('', '') + + def test_header_decode_one_word(self): + self._test_header_decode('hello', 'hello') + + def test_header_decode_two_lines(self): + self._test_header_decode('hello=0Aworld', 'hello\nworld') + + def test_header_decode_non_ascii(self): + self._test_header_decode('hello=C7there', 'hello\xc7there') + + def _test_decode(self, encoded, expected_decoded, eol=None): + if eol is None: + decoded = quoprimime.decode(encoded) + else: + decoded = quoprimime.decode(encoded, eol=eol) + self.assertEqual(decoded, expected_decoded) + + def test_decode_null_word(self): + self._test_decode('', '') + + def test_decode_null_line_null_word(self): + self._test_decode('\r\n', '\n') + + def test_decode_one_word(self): + self._test_decode('hello', 'hello') + + def test_decode_one_word_eol(self): + self._test_decode('hello', 'hello', eol='X') + + def test_decode_one_line(self): + self._test_decode('hello\r\n', 'hello\n') + + def test_decode_one_line_lf(self): + self._test_decode('hello\n', 'hello\n') + + def test_decode_one_line_cr(self): + self._test_decode('hello\r', 'hello\n') + + def test_decode_one_line_nl(self): + self._test_decode('hello\n', 'helloX', eol='X') + + def test_decode_one_line_crnl(self): + self._test_decode('hello\r\n', 'helloX', eol='X') + + def test_decode_one_line_one_word(self): + self._test_decode('hello\r\nworld', 'hello\nworld') + + def test_decode_one_line_one_word_eol(self): + self._test_decode('hello\r\nworld', 'helloXworld', eol='X') + + def test_decode_two_lines(self): + self._test_decode('hello\r\nworld\r\n', 'hello\nworld\n') + + def test_decode_two_lines_eol(self): + self._test_decode('hello\r\nworld\r\n', 'helloXworldX', eol='X') + + def test_decode_one_long_line(self): + self._test_decode('Spam' * 250, 'Spam' * 250) + + def test_decode_one_space(self): + self._test_decode(' ', '') + + def test_decode_multiple_spaces(self): + self._test_decode(' ' * 5, '') + + def test_decode_one_line_trailing_spaces(self): + self._test_decode('hello \r\n', 'hello\n') + + def test_decode_two_lines_trailing_spaces(self): + self._test_decode('hello \r\nworld \r\n', 'hello\nworld\n') + + def test_decode_quoted_word(self): + self._test_decode('=22quoted=20words=22', '"quoted words"') + + def test_decode_uppercase_quoting(self): + self._test_decode('ab=CD=EF', 'ab\xcd\xef') + + def test_decode_lowercase_quoting(self): + self._test_decode('ab=cd=ef', 'ab\xcd\xef') + + def test_decode_soft_line_break(self): + self._test_decode('soft line=\r\nbreak', 'soft linebreak') + + def test_decode_false_quoting(self): + self._test_decode('A=1,B=A ==> A+B==2', 'A=1,B=A ==> A+B==2') + + def _test_encode(self, body, expected_encoded_body, maxlinelen=None, eol=None): + kwargs = {} + if maxlinelen is None: + # Use body_encode's default. + maxlinelen = 76 + else: + kwargs['maxlinelen'] = maxlinelen + if eol is None: + # Use body_encode's default. + eol = '\n' + else: + kwargs['eol'] = eol + encoded_body = quoprimime.body_encode(body, **kwargs) + self.assertEqual(encoded_body, expected_encoded_body) + if eol == '\n' or eol == '\r\n': + # We know how to split the result back into lines, so maxlinelen + # can be checked. + for line in encoded_body.splitlines(): + self.assertLessEqual(len(line), maxlinelen) + + def test_encode_null(self): + self._test_encode('', '') + + def test_encode_null_lines(self): + self._test_encode('\n\n', '\n\n') + + def test_encode_one_line(self): + self._test_encode('hello\n', 'hello\n') + + def test_encode_one_line_crlf(self): + self._test_encode('hello\r\n', 'hello\n') + + def test_encode_one_line_eol(self): + self._test_encode('hello\n', 'hello\r\n', eol='\r\n') + + def test_encode_one_space(self): + self._test_encode(' ', '=20') + + def test_encode_one_line_one_space(self): + self._test_encode(' \n', '=20\n') + +# XXX: body_encode() expect strings, but uses ord(char) from these strings +# to index into a 256-entry list. For code points above 255, this will fail. +# Should there be a check for 8-bit only ord() values in body, or at least +# a comment about the expected input? + + def test_encode_two_lines_one_space(self): + self._test_encode(' \n \n', '=20\n=20\n') + + def test_encode_one_word_trailing_spaces(self): + self._test_encode('hello ', 'hello =20') + + def test_encode_one_line_trailing_spaces(self): + self._test_encode('hello \n', 'hello =20\n') + + def test_encode_one_word_trailing_tab(self): + self._test_encode('hello \t', 'hello =09') + + def test_encode_one_line_trailing_tab(self): + self._test_encode('hello \t\n', 'hello =09\n') + + def test_encode_trailing_space_before_maxlinelen(self): + self._test_encode('abcd \n1234', 'abcd =\n\n1234', maxlinelen=6) + + def test_encode_trailing_space_at_maxlinelen(self): + self._test_encode('abcd \n1234', 'abcd=\n=20\n1234', maxlinelen=5) + + def test_encode_trailing_space_beyond_maxlinelen(self): + self._test_encode('abcd \n1234', 'abc=\nd=20\n1234', maxlinelen=4) + + def test_encode_whitespace_lines(self): + self._test_encode(' \n' * 5, '=20\n' * 5) + + def test_encode_quoted_equals(self): + self._test_encode('a = b', 'a =3D b') + + def test_encode_one_long_string(self): + self._test_encode('x' * 100, 'x' * 75 + '=\n' + 'x' * 25) + + def test_encode_one_long_line(self): + self._test_encode('x' * 100 + '\n', 'x' * 75 + '=\n' + 'x' * 25 + '\n') + + def test_encode_one_very_long_line(self): + self._test_encode('x' * 200 + '\n', + 2 * ('x' * 75 + '=\n') + 'x' * 50 + '\n') + + def test_encode_one_long_line(self): + self._test_encode('x' * 100 + '\n', 'x' * 75 + '=\n' + 'x' * 25 + '\n') + + def test_encode_shortest_maxlinelen(self): + self._test_encode('=' * 5, '=3D=\n' * 4 + '=3D', maxlinelen=4) + + def test_encode_maxlinelen_too_small(self): + self.assertRaises(ValueError, self._test_encode, '', '', maxlinelen=3) + + def test_encode(self): + eq = self.assertEqual + eq(quoprimime.body_encode(''), '') + eq(quoprimime.body_encode('hello'), 'hello') + # Test the binary flag + eq(quoprimime.body_encode('hello\r\nworld'), 'hello\nworld') + # Test the maxlinelen arg + eq(quoprimime.body_encode('xxxx ' * 20, maxlinelen=40), """\ +xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx= + xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx= +x xxxx xxxx xxxx xxxx=20""") + # Test the eol argument + eq(quoprimime.body_encode('xxxx ' * 20, maxlinelen=40, eol='\r\n'), + """\ +xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxxx=\r + xxxx xxxx xxxx xxxx xxxx xxxx xxxx xxx=\r +x xxxx xxxx xxxx xxxx=20""") + eq(quoprimime.body_encode("""\ +one line + +two line"""), """\ +one line + +two line""") + + + +# Test the Charset class +class TestCharset(unittest.TestCase): + def tearDown(self): + from email import charset as CharsetModule + try: + del CharsetModule.CHARSETS['fake'] + except KeyError: + pass + + def test_codec_encodeable(self): + eq = self.assertEqual + # Make sure us-ascii = no Unicode conversion + c = Charset('us-ascii') + eq(c.header_encode('Hello World!'), 'Hello World!') + # Test 8-bit idempotency with us-ascii + s = '\xa4\xa2\xa4\xa4\xa4\xa6\xa4\xa8\xa4\xaa' + self.assertRaises(UnicodeError, c.header_encode, s) + c = Charset('utf-8') + eq(c.header_encode(s), '=?utf-8?b?wqTCosKkwqTCpMKmwqTCqMKkwqo=?=') + + def test_body_encode(self): + eq = self.assertEqual + # Try a charset with QP body encoding + c = Charset('iso-8859-1') + eq('hello w=F6rld', c.body_encode('hello w\xf6rld')) + # Try a charset with Base64 body encoding + c = Charset('utf-8') + eq('aGVsbG8gd29ybGQ=\n', c.body_encode(b'hello world')) + # Try a charset with None body encoding + c = Charset('us-ascii') + eq('hello world', c.body_encode('hello world')) + # Try the convert argument, where input codec != output codec + c = Charset('euc-jp') + # With apologies to Tokio Kikuchi ;) + # XXX FIXME +## try: +## eq('\x1b$B5FCO;~IW\x1b(B', +## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7')) +## eq('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', +## c.body_encode('\xb5\xc6\xc3\xcf\xbb\xfe\xc9\xd7', False)) +## except LookupError: +## # We probably don't have the Japanese codecs installed +## pass + # Testing SF bug #625509, which we have to fake, since there are no + # built-in encodings where the header encoding is QP but the body + # encoding is not. + from email import charset as CharsetModule + CharsetModule.add_charset('fake', CharsetModule.QP, None, 'utf-8') + c = Charset('fake') + eq('hello world', c.body_encode('hello world')) + + def test_unicode_charset_name(self): + charset = Charset('us-ascii') + self.assertEqual(str(charset), 'us-ascii') + self.assertRaises(errors.CharsetError, Charset, 'asc\xffii') + + + +# Test multilingual MIME headers. +class TestHeader(TestEmailBase): + def test_simple(self): + eq = self.ndiffAssertEqual + h = Header('Hello World!') + eq(h.encode(), 'Hello World!') + h.append(' Goodbye World!') + eq(h.encode(), 'Hello World! Goodbye World!') + + def test_simple_surprise(self): + eq = self.ndiffAssertEqual + h = Header('Hello World!') + eq(h.encode(), 'Hello World!') + h.append('Goodbye World!') + eq(h.encode(), 'Hello World! Goodbye World!') + + def test_header_needs_no_decoding(self): + h = 'no decoding needed' + self.assertEqual(decode_header(h), [(h, None)]) + + def test_long(self): + h = Header("I am the very model of a modern Major-General; I've information vegetable, animal, and mineral; I know the kings of England, and I quote the fights historical from Marathon to Waterloo, in order categorical; I'm very well acquainted, too, with matters mathematical; I understand equations, both the simple and quadratical; about binomial theorem I'm teeming with a lot o' news, with many cheerful facts about the square of the hypotenuse.", + maxlinelen=76) + for l in h.encode(splitchars=' ').split('\n '): + self.assertTrue(len(l) <= 76) + + def test_multilingual(self): + eq = self.ndiffAssertEqual + g = Charset("iso-8859-1") + cz = Charset("iso-8859-2") + utf8 = Charset("utf-8") + g_head = (b'Die Mieter treten hier ein werden mit einem ' + b'Foerderband komfortabel den Korridor entlang, ' + b'an s\xfcdl\xfcndischen Wandgem\xe4lden vorbei, ' + b'gegen die rotierenden Klingen bef\xf6rdert. ') + cz_head = (b'Finan\xe8ni metropole se hroutily pod tlakem jejich ' + b'd\xf9vtipu.. ') + utf8_head = ('\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f' + '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00' + '\u90e8\u306f\u30c9\u30a4\u30c4\u8a9e\u3067\u3059\u304c' + '\u3001\u3042\u3068\u306f\u3067\u305f\u3089\u3081\u3067' + '\u3059\u3002\u5b9f\u969b\u306b\u306f\u300cWenn ist das ' + 'Nunstuck git und Slotermeyer? Ja! Beiherhund das Oder ' + 'die Flipperwaldt gersput.\u300d\u3068\u8a00\u3063\u3066' + '\u3044\u307e\u3059\u3002') + h = Header(g_head, g) + h.append(cz_head, cz) + h.append(utf8_head, utf8) + enc = h.encode(maxlinelen=76) + eq(enc, """\ +=?iso-8859-1?q?Die_Mieter_treten_hier_ein_werden_mit_einem_Foerderband_kom?= + =?iso-8859-1?q?fortabel_den_Korridor_entlang=2C_an_s=FCdl=FCndischen_Wand?= + =?iso-8859-1?q?gem=E4lden_vorbei=2C_gegen_die_rotierenden_Klingen_bef=F6r?= + =?iso-8859-1?q?dert=2E_?= =?iso-8859-2?q?Finan=E8ni_metropole_se_hroutily?= + =?iso-8859-2?q?_pod_tlakem_jejich_d=F9vtipu=2E=2E_?= =?utf-8?b?5q2j56K6?= + =?utf-8?b?44Gr6KiA44GG44Go57+76Kiz44Gv44GV44KM44Gm44GE44G+44Gb44KT44CC?= + =?utf-8?b?5LiA6YOo44Gv44OJ44Kk44OE6Kqe44Gn44GZ44GM44CB44GC44Go44Gv44Gn?= + =?utf-8?b?44Gf44KJ44KB44Gn44GZ44CC5a6f6Zqb44Gr44Gv44CMV2VubiBpc3QgZGFz?= + =?utf-8?b?IE51bnN0dWNrIGdpdCB1bmQgU2xvdGVybWV5ZXI/IEphISBCZWloZXJodW5k?= + =?utf-8?b?IGRhcyBPZGVyIGRpZSBGbGlwcGVyd2FsZHQgZ2Vyc3B1dC7jgI3jgajoqIA=?= + =?utf-8?b?44Gj44Gm44GE44G+44GZ44CC?=""") + decoded = decode_header(enc) + eq(len(decoded), 3) + eq(decoded[0], (g_head, 'iso-8859-1')) + eq(decoded[1], (cz_head, 'iso-8859-2')) + eq(decoded[2], (utf8_head.encode('utf-8'), 'utf-8')) + ustr = str(h) + eq(ustr, + (b'Die Mieter treten hier ein werden mit einem Foerderband ' + b'komfortabel den Korridor entlang, an s\xc3\xbcdl\xc3\xbcndischen ' + b'Wandgem\xc3\xa4lden vorbei, gegen die rotierenden Klingen ' + b'bef\xc3\xb6rdert. Finan\xc4\x8dni metropole se hroutily pod ' + b'tlakem jejich d\xc5\xafvtipu.. \xe6\xad\xa3\xe7\xa2\xba\xe3\x81' + b'\xab\xe8\xa8\x80\xe3\x81\x86\xe3\x81\xa8\xe7\xbf\xbb\xe8\xa8\xb3' + b'\xe3\x81\xaf\xe3\x81\x95\xe3\x82\x8c\xe3\x81\xa6\xe3\x81\x84\xe3' + b'\x81\xbe\xe3\x81\x9b\xe3\x82\x93\xe3\x80\x82\xe4\xb8\x80\xe9\x83' + b'\xa8\xe3\x81\xaf\xe3\x83\x89\xe3\x82\xa4\xe3\x83\x84\xe8\xaa\x9e' + b'\xe3\x81\xa7\xe3\x81\x99\xe3\x81\x8c\xe3\x80\x81\xe3\x81\x82\xe3' + b'\x81\xa8\xe3\x81\xaf\xe3\x81\xa7\xe3\x81\x9f\xe3\x82\x89\xe3\x82' + b'\x81\xe3\x81\xa7\xe3\x81\x99\xe3\x80\x82\xe5\xae\x9f\xe9\x9a\x9b' + b'\xe3\x81\xab\xe3\x81\xaf\xe3\x80\x8cWenn ist das Nunstuck git ' + b'und Slotermeyer? Ja! Beiherhund das Oder die Flipperwaldt ' + b'gersput.\xe3\x80\x8d\xe3\x81\xa8\xe8\xa8\x80\xe3\x81\xa3\xe3\x81' + b'\xa6\xe3\x81\x84\xe3\x81\xbe\xe3\x81\x99\xe3\x80\x82' + ).decode('utf-8')) + # Test make_header() + newh = make_header(decode_header(enc)) + eq(newh, h) + + def test_empty_header_encode(self): + h = Header() + self.assertEqual(h.encode(), '') + + def test_header_ctor_default_args(self): + eq = self.ndiffAssertEqual + h = Header() + eq(h, '') + h.append('foo', Charset('iso-8859-1')) + eq(h, 'foo') + + def test_explicit_maxlinelen(self): + eq = self.ndiffAssertEqual + hstr = ('A very long line that must get split to something other ' + 'than at the 76th character boundary to test the non-default ' + 'behavior') + h = Header(hstr) + eq(h.encode(), '''\ +A very long line that must get split to something other than at the 76th + character boundary to test the non-default behavior''') + eq(str(h), hstr) + h = Header(hstr, header_name='Subject') + eq(h.encode(), '''\ +A very long line that must get split to something other than at the + 76th character boundary to test the non-default behavior''') + eq(str(h), hstr) + h = Header(hstr, maxlinelen=1024, header_name='Subject') + eq(h.encode(), hstr) + eq(str(h), hstr) + + def test_quopri_splittable(self): + eq = self.ndiffAssertEqual + h = Header(charset='iso-8859-1', maxlinelen=20) + x = 'xxxx ' * 20 + h.append(x) + s = h.encode() + eq(s, """\ +=?iso-8859-1?q?xxx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_x?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?x_?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?xx?= + =?iso-8859-1?q?_?=""") + eq(x, str(make_header(decode_header(s)))) + h = Header(charset='iso-8859-1', maxlinelen=40) + h.append('xxxx ' * 20) + s = h.encode() + eq(s, """\ +=?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xxx?= + =?iso-8859-1?q?x_xxxx_xxxx_xxxx_xxxx_?= + =?iso-8859-1?q?xxxx_xxxx_xxxx_xxxx_xx?= + =?iso-8859-1?q?xx_xxxx_xxxx_xxxx_xxxx?= + =?iso-8859-1?q?_xxxx_xxxx_?=""") + eq(x, str(make_header(decode_header(s)))) + + def test_base64_splittable(self): + eq = self.ndiffAssertEqual + h = Header(charset='koi8-r', maxlinelen=20) + x = 'xxxx ' * 20 + h.append(x) + s = h.encode() + eq(s, """\ +=?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IHh4?= + =?koi8-r?b?eHgg?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?eCB4?= + =?koi8-r?b?eHh4?= + =?koi8-r?b?IA==?=""") + eq(x, str(make_header(decode_header(s)))) + h = Header(charset='koi8-r', maxlinelen=40) + h.append(x) + s = h.encode() + eq(s, """\ +=?koi8-r?b?eHh4eCB4eHh4IHh4eHggeHh4?= + =?koi8-r?b?eCB4eHh4IHh4eHggeHh4eCB4?= + =?koi8-r?b?eHh4IHh4eHggeHh4eCB4eHh4?= + =?koi8-r?b?IHh4eHggeHh4eCB4eHh4IHh4?= + =?koi8-r?b?eHggeHh4eCB4eHh4IHh4eHgg?= + =?koi8-r?b?eHh4eCB4eHh4IA==?=""") + eq(x, str(make_header(decode_header(s)))) + + def test_us_ascii_header(self): + eq = self.assertEqual + s = 'hello' + x = decode_header(s) + eq(x, [('hello', None)]) + h = make_header(x) + eq(s, h.encode()) + + def test_string_charset(self): + eq = self.assertEqual + h = Header() + h.append('hello', 'iso-8859-1') + eq(h, 'hello') + +## def test_unicode_error(self): +## raises = self.assertRaises +## raises(UnicodeError, Header, u'[P\xf6stal]', 'us-ascii') +## raises(UnicodeError, Header, '[P\xf6stal]', 'us-ascii') +## h = Header() +## raises(UnicodeError, h.append, u'[P\xf6stal]', 'us-ascii') +## raises(UnicodeError, h.append, '[P\xf6stal]', 'us-ascii') +## raises(UnicodeError, Header, u'\u83ca\u5730\u6642\u592b', 'iso-8859-1') + + def test_utf8_shortest(self): + eq = self.assertEqual + h = Header('p\xf6stal', 'utf-8') + eq(h.encode(), '=?utf-8?q?p=C3=B6stal?=') + h = Header('\u83ca\u5730\u6642\u592b', 'utf-8') + eq(h.encode(), '=?utf-8?b?6I+K5Zyw5pmC5aSr?=') + + def test_bad_8bit_header(self): + raises = self.assertRaises + eq = self.assertEqual + x = b'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' + raises(UnicodeError, Header, x) + h = Header() + raises(UnicodeError, h.append, x) + e = x.decode('utf-8', 'replace') + eq(str(Header(x, errors='replace')), e) + h.append(x, errors='replace') + eq(str(h), e) + + def test_escaped_8bit_header(self): + x = b'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' + e = x.decode('ascii', 'surrogateescape') + h = Header(e, charset=email.charset.UNKNOWN8BIT) + self.assertEqual(str(h), + 'Ynwp4dUEbay Auction Semiar- No Charge \uFFFD Earn Big') + self.assertEqual(email.header.decode_header(h), [(x, 'unknown-8bit')]) + + def test_header_handles_binary_unknown8bit(self): + x = b'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' + h = Header(x, charset=email.charset.UNKNOWN8BIT) + self.assertEqual(str(h), + 'Ynwp4dUEbay Auction Semiar- No Charge \uFFFD Earn Big') + self.assertEqual(email.header.decode_header(h), [(x, 'unknown-8bit')]) + + def test_make_header_handles_binary_unknown8bit(self): + x = b'Ynwp4dUEbay Auction Semiar- No Charge \x96 Earn Big' + h = Header(x, charset=email.charset.UNKNOWN8BIT) + h2 = email.header.make_header(email.header.decode_header(h)) + self.assertEqual(str(h2), + 'Ynwp4dUEbay Auction Semiar- No Charge \uFFFD Earn Big') + self.assertEqual(email.header.decode_header(h2), [(x, 'unknown-8bit')]) + + def test_modify_returned_list_does_not_change_header(self): + h = Header('test') + chunks = email.header.decode_header(h) + chunks.append(('ascii', 'test2')) + self.assertEqual(str(h), 'test') + + def test_encoded_adjacent_nonencoded(self): + eq = self.assertEqual + h = Header() + h.append('hello', 'iso-8859-1') + h.append('world') + s = h.encode() + eq(s, '=?iso-8859-1?q?hello?= world') + h = make_header(decode_header(s)) + eq(h.encode(), s) + + def test_whitespace_eater(self): + eq = self.assertEqual + s = 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztk=?= =?koi8-r?q?=CA?= zz.' + parts = decode_header(s) + eq(parts, [(b'Subject:', None), (b'\xf0\xd2\xcf\xd7\xc5\xd2\xcb\xc1 \xce\xc1 \xc6\xc9\xce\xc1\xcc\xd8\xce\xd9\xca', 'koi8-r'), (b'zz.', None)]) + hdr = make_header(parts) + eq(hdr.encode(), + 'Subject: =?koi8-r?b?8NLP18XSy8EgzsEgxsnOwczYztnK?= zz.') + + def test_broken_base64_header(self): + raises = self.assertRaises + s = 'Subject: =?EUC-KR?B?CSixpLDtKSC/7Liuvsax4iC6uLmwMcijIKHaILzSwd/H0SC8+LCjwLsgv7W/+Mj3I ?=' + raises(errors.HeaderParseError, decode_header, s) + + def test_shift_jis_charset(self): + h = Header('文', charset='shift_jis') + self.assertEqual(h.encode(), '=?iso-2022-jp?b?GyRCSjgbKEI=?=') + + def test_flatten_header_with_no_value(self): + # Issue 11401 (regression from email 4.x) Note that the space after + # the header doesn't reflect the input, but this is also the way + # email 4.x behaved. At some point it would be nice to fix that. + msg = email.message_from_string("EmptyHeader:") + self.assertEqual(str(msg), "EmptyHeader: \n\n") + + def test_encode_preserves_leading_ws_on_value(self): + msg = Message() + msg['SomeHeader'] = ' value with leading ws' + self.assertEqual(str(msg), "SomeHeader: value with leading ws\n\n") + + + +# Test RFC 2231 header parameters (en/de)coding +class TestRFC2231(TestEmailBase): + def test_get_param(self): + eq = self.assertEqual + msg = self._msgobj('msg_29.txt') + eq(msg.get_param('title'), + ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) + eq(msg.get_param('title', unquote=False), + ('us-ascii', 'en', '"This is even more ***fun*** isn\'t it!"')) + + def test_set_param(self): + eq = self.ndiffAssertEqual + msg = Message() + msg.set_param('title', 'This is even more ***fun*** isn\'t it!', + charset='us-ascii') + eq(msg.get_param('title'), + ('us-ascii', '', 'This is even more ***fun*** isn\'t it!')) + msg.set_param('title', 'This is even more ***fun*** isn\'t it!', + charset='us-ascii', language='en') + eq(msg.get_param('title'), + ('us-ascii', 'en', 'This is even more ***fun*** isn\'t it!')) + msg = self._msgobj('msg_01.txt') + msg.set_param('title', 'This is even more ***fun*** isn\'t it!', + charset='us-ascii', language='en') + eq(msg.as_string(maxheaderlen=78), """\ +Return-Path: <bbb@zzz.org> +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) +\tid 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 +Content-Type: text/plain; charset=us-ascii; + title*=us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21 + + +Hi, + +Do you like this message? + +-Me +""") + + def test_set_param_requote(self): + msg = Message() + msg.set_param('title', 'foo') + self.assertEqual(msg['content-type'], 'text/plain; title="foo"') + msg.set_param('title', 'bar', requote=False) + self.assertEqual(msg['content-type'], 'text/plain; title=bar') + # tspecial is still quoted. + msg.set_param('title', "(bar)bell", requote=False) + self.assertEqual(msg['content-type'], 'text/plain; title="(bar)bell"') + + def test_del_param(self): + eq = self.ndiffAssertEqual + msg = self._msgobj('msg_01.txt') + msg.set_param('foo', 'bar', charset='us-ascii', language='en') + msg.set_param('title', 'This is even more ***fun*** isn\'t it!', + charset='us-ascii', language='en') + msg.del_param('foo', header='Content-Type') + eq(msg.as_string(maxheaderlen=78), """\ +Return-Path: <bbb@zzz.org> +Delivered-To: bbb@zzz.org +Received: by mail.zzz.org (Postfix, from userid 889) +\tid 27CEAD38CC; Fri, 4 May 2001 14:05:44 -0400 (EDT) +MIME-Version: 1.0 +Content-Transfer-Encoding: 7bit +Message-ID: <15090.61304.110929.45684@aaa.zzz.org> +From: bbb@ddd.com (John X. Doe) +To: bbb@zzz.org +Subject: This is a test message +Date: Fri, 4 May 2001 14:05:44 -0400 +Content-Type: text/plain; charset="us-ascii"; + title*=us-ascii'en'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20isn%27t%20it%21 + + +Hi, + +Do you like this message? + +-Me +""") + + def test_rfc2231_get_content_charset(self): + eq = self.assertEqual + msg = self._msgobj('msg_32.txt') + eq(msg.get_content_charset(), 'us-ascii') + + def test_rfc2231_parse_rfc_quoting(self): + m = textwrap.dedent('''\ + Content-Disposition: inline; + \tfilename*0*=''This%20is%20even%20more%20; + \tfilename*1*=%2A%2A%2Afun%2A%2A%2A%20; + \tfilename*2="is it not.pdf" + + ''') + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + self.assertEqual(m, msg.as_string()) + + def test_rfc2231_parse_extra_quoting(self): + m = textwrap.dedent('''\ + Content-Disposition: inline; + \tfilename*0*="''This%20is%20even%20more%20"; + \tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; + \tfilename*2="is it not.pdf" + + ''') + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + self.assertEqual(m, msg.as_string()) + + def test_rfc2231_no_language_or_charset(self): + m = '''\ +Content-Transfer-Encoding: 8bit +Content-Disposition: inline; filename="file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm" +Content-Type: text/html; NAME*0=file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEM; NAME*1=P_nsmail.htm + +''' + msg = email.message_from_string(m) + param = msg.get_param('NAME') + self.assertFalse(isinstance(param, tuple)) + self.assertEqual( + param, + 'file____C__DOCUMENTS_20AND_20SETTINGS_FABIEN_LOCAL_20SETTINGS_TEMP_nsmail.htm') + + def test_rfc2231_no_language_or_charset_in_filename(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="''This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + + def test_rfc2231_no_language_or_charset_in_filename_encoded(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="''This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + + def test_rfc2231_partly_encoded(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0="''This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual( + msg.get_filename(), + 'This%20is%20even%20more%20***fun*** is it not.pdf') + + def test_rfc2231_partly_nonencoded(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0="This%20is%20even%20more%20"; +\tfilename*1="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual( + msg.get_filename(), + 'This%20is%20even%20more%20%2A%2A%2Afun%2A%2A%2A%20is it not.pdf') + + def test_rfc2231_no_language_or_charset_in_boundary(self): + m = '''\ +Content-Type: multipart/alternative; +\tboundary*0*="''This%20is%20even%20more%20"; +\tboundary*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tboundary*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_boundary(), + 'This is even more ***fun*** is it not.pdf') + + def test_rfc2231_no_language_or_charset_in_charset(self): + # This is a nonsensical charset value, but tests the code anyway + m = '''\ +Content-Type: text/plain; +\tcharset*0*="This%20is%20even%20more%20"; +\tcharset*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tcharset*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_content_charset(), + 'this is even more ***fun*** is it not.pdf') + + def test_rfc2231_bad_encoding_in_filename(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="bogus'xx'This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2="is it not.pdf" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf') + + def test_rfc2231_bad_encoding_in_charset(self): + m = """\ +Content-Type: text/plain; charset*=bogus''utf-8%E2%80%9D + +""" + msg = email.message_from_string(m) + # This should return None because non-ascii characters in the charset + # are not allowed. + self.assertEqual(msg.get_content_charset(), None) + + def test_rfc2231_bad_character_in_charset(self): + m = """\ +Content-Type: text/plain; charset*=ascii''utf-8%E2%80%9D + +""" + msg = email.message_from_string(m) + # This should return None because non-ascii characters in the charset + # are not allowed. + self.assertEqual(msg.get_content_charset(), None) + + def test_rfc2231_bad_character_in_filename(self): + m = '''\ +Content-Disposition: inline; +\tfilename*0*="ascii'xx'This%20is%20even%20more%20"; +\tfilename*1*="%2A%2A%2Afun%2A%2A%2A%20"; +\tfilename*2*="is it not.pdf%E2" + +''' + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), + 'This is even more ***fun*** is it not.pdf\ufffd') + + def test_rfc2231_unknown_encoding(self): + m = """\ +Content-Transfer-Encoding: 8bit +Content-Disposition: inline; filename*=X-UNKNOWN''myfile.txt + +""" + msg = email.message_from_string(m) + self.assertEqual(msg.get_filename(), 'myfile.txt') + + def test_rfc2231_single_tick_in_filename_extended(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; +\tname*0*=\"Frank's\"; name*1*=\" Document\" + +""" + msg = email.message_from_string(m) + charset, language, s = msg.get_param('name') + eq(charset, None) + eq(language, None) + eq(s, "Frank's Document") + + def test_rfc2231_single_tick_in_filename(self): + m = """\ +Content-Type: application/x-foo; name*0=\"Frank's\"; name*1=\" Document\" + +""" + msg = email.message_from_string(m) + param = msg.get_param('name') + self.assertFalse(isinstance(param, tuple)) + self.assertEqual(param, "Frank's Document") + + def test_rfc2231_tick_attack_extended(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; +\tname*0*=\"us-ascii'en-us'Frank's\"; name*1*=\" Document\" + +""" + msg = email.message_from_string(m) + charset, language, s = msg.get_param('name') + eq(charset, 'us-ascii') + eq(language, 'en-us') + eq(s, "Frank's Document") + + def test_rfc2231_tick_attack(self): + m = """\ +Content-Type: application/x-foo; +\tname*0=\"us-ascii'en-us'Frank's\"; name*1=\" Document\" + +""" + msg = email.message_from_string(m) + param = msg.get_param('name') + self.assertFalse(isinstance(param, tuple)) + self.assertEqual(param, "us-ascii'en-us'Frank's Document") + + def test_rfc2231_no_extended_values(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; name=\"Frank's Document\" + +""" + msg = email.message_from_string(m) + eq(msg.get_param('name'), "Frank's Document") + + def test_rfc2231_encoded_then_unencoded_segments(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; +\tname*0*=\"us-ascii'en-us'My\"; +\tname*1=\" Document\"; +\tname*2*=\" For You\" + +""" + msg = email.message_from_string(m) + charset, language, s = msg.get_param('name') + eq(charset, 'us-ascii') + eq(language, 'en-us') + eq(s, 'My Document For You') + + def test_rfc2231_unencoded_then_encoded_segments(self): + eq = self.assertEqual + m = """\ +Content-Type: application/x-foo; +\tname*0=\"us-ascii'en-us'My\"; +\tname*1*=\" Document\"; +\tname*2*=\" For You\" + +""" + msg = email.message_from_string(m) + charset, language, s = msg.get_param('name') + eq(charset, 'us-ascii') + eq(language, 'en-us') + eq(s, 'My Document For You') + + + +# Tests to ensure that signed parts of an email are completely preserved, as +# required by RFC1847 section 2.1. Note that these are incomplete, because the +# email package does not currently always preserve the body. See issue 1670765. +class TestSigned(TestEmailBase): + + def _msg_and_obj(self, filename): + with openfile(filename) as fp: + original = fp.read() + msg = email.message_from_string(original) + return original, msg + + def _signed_parts_eq(self, original, result): + # Extract the first mime part of each message + import re + repart = re.compile(r'^--([^\n]+)\n(.*?)\n--\1$', re.S | re.M) + inpart = repart.search(original).group(2) + outpart = repart.search(result).group(2) + self.assertEqual(outpart, inpart) + + def test_long_headers_as_string(self): + original, msg = self._msg_and_obj('msg_45.txt') + result = msg.as_string() + self._signed_parts_eq(original, result) + + def test_long_headers_as_string_maxheaderlen(self): + original, msg = self._msg_and_obj('msg_45.txt') + result = msg.as_string(maxheaderlen=60) + self._signed_parts_eq(original, result) + + def test_long_headers_flatten(self): + original, msg = self._msg_and_obj('msg_45.txt') + fp = StringIO() + Generator(fp).flatten(msg) + result = fp.getvalue() + self._signed_parts_eq(original, result) + + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_generator.py b/Lib/test/test_email/test_generator.py new file mode 100644 index 0000000..35ca6c5 --- /dev/null +++ b/Lib/test/test_email/test_generator.py @@ -0,0 +1,136 @@ +import io +import textwrap +import unittest +from email import message_from_string, message_from_bytes +from email.generator import Generator, BytesGenerator +from email import policy +from test.test_email import TestEmailBase + +# XXX: move generator tests from test_email into here at some point. + + +class TestGeneratorBase(): + + long_subject = { + 0: textwrap.dedent("""\ + To: whom_it_may_concern@example.com + From: nobody_you_want_to_know@example.com + Subject: We the willing led by the unknowing are doing the + impossible for the ungrateful. We have done so much for so long with so little + we are now qualified to do anything with nothing. + + None + """), + 40: textwrap.dedent("""\ + To: whom_it_may_concern@example.com + From:\x20 + nobody_you_want_to_know@example.com + Subject: We the willing led by the + unknowing are doing the + impossible for the ungrateful. We have + done so much for so long with so little + we are now qualified to do anything + with nothing. + + None + """), + 20: textwrap.dedent("""\ + To:\x20 + whom_it_may_concern@example.com + From:\x20 + nobody_you_want_to_know@example.com + Subject: We the + willing led by the + unknowing are doing + the + impossible for the + ungrateful. We have + done so much for so + long with so little + we are now + qualified to do + anything with + nothing. + + None + """), + } + long_subject[100] = long_subject[0] + + def maxheaderlen_parameter_test(self, n): + msg = self.msgmaker(self.long_subject[0]) + s = self.ioclass() + g = self.genclass(s, maxheaderlen=n) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.long_subject[n]) + + def test_maxheaderlen_parameter_0(self): + self.maxheaderlen_parameter_test(0) + + def test_maxheaderlen_parameter_100(self): + self.maxheaderlen_parameter_test(100) + + def test_maxheaderlen_parameter_40(self): + self.maxheaderlen_parameter_test(40) + + def test_maxheaderlen_parameter_20(self): + self.maxheaderlen_parameter_test(20) + + def maxheaderlen_policy_test(self, n): + msg = self.msgmaker(self.long_subject[0]) + s = self.ioclass() + g = self.genclass(s, policy=policy.default.clone(max_line_length=n)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.long_subject[n]) + + def test_maxheaderlen_policy_0(self): + self.maxheaderlen_policy_test(0) + + def test_maxheaderlen_policy_100(self): + self.maxheaderlen_policy_test(100) + + def test_maxheaderlen_policy_40(self): + self.maxheaderlen_policy_test(40) + + def test_maxheaderlen_policy_20(self): + self.maxheaderlen_policy_test(20) + + def maxheaderlen_parm_overrides_policy_test(self, n): + msg = self.msgmaker(self.long_subject[0]) + s = self.ioclass() + g = self.genclass(s, maxheaderlen=n, + policy=policy.default.clone(max_line_length=10)) + g.flatten(msg) + self.assertEqual(s.getvalue(), self.long_subject[n]) + + def test_maxheaderlen_parm_overrides_policy_0(self): + self.maxheaderlen_parm_overrides_policy_test(0) + + def test_maxheaderlen_parm_overrides_policy_100(self): + self.maxheaderlen_parm_overrides_policy_test(100) + + def test_maxheaderlen_parm_overrides_policy_40(self): + self.maxheaderlen_parm_overrides_policy_test(40) + + def test_maxheaderlen_parm_overrides_policy_20(self): + self.maxheaderlen_parm_overrides_policy_test(20) + + +class TestGenerator(TestGeneratorBase, TestEmailBase): + + msgmaker = staticmethod(message_from_string) + genclass = Generator + ioclass = io.StringIO + + +class TestBytesGenerator(TestGeneratorBase, TestEmailBase): + + msgmaker = staticmethod(message_from_bytes) + genclass = BytesGenerator + ioclass = io.BytesIO + long_subject = {key: x.encode('ascii') + for key, x in TestGeneratorBase.long_subject.items()} + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_policy.py b/Lib/test/test_email/test_policy.py new file mode 100644 index 0000000..086ce40 --- /dev/null +++ b/Lib/test/test_email/test_policy.py @@ -0,0 +1,148 @@ +import types +import unittest +import email.policy + +class PolicyAPITests(unittest.TestCase): + + longMessage = True + + # These default values are the ones set on email.policy.default. + # If any of these defaults change, the docs must be updated. + policy_defaults = { + 'max_line_length': 78, + 'linesep': '\n', + 'must_be_7bit': False, + 'raise_on_defect': False, + } + + # For each policy under test, we give here the values of the attributes + # that are different from the defaults for that policy. + policies = { + email.policy.Policy(): {}, + email.policy.default: {}, + email.policy.SMTP: {'linesep': '\r\n'}, + email.policy.HTTP: {'linesep': '\r\n', 'max_line_length': None}, + email.policy.strict: {'raise_on_defect': True}, + } + + def test_defaults(self): + for policy, changed_defaults in self.policies.items(): + expected = self.policy_defaults.copy() + expected.update(changed_defaults) + for attr, value in expected.items(): + self.assertEqual(getattr(policy, attr), value, + ("change {} docs/docstrings if defaults have " + "changed").format(policy)) + + def test_all_attributes_covered(self): + for attr in dir(email.policy.default): + if (attr.startswith('_') or + isinstance(getattr(email.policy.Policy, attr), + types.FunctionType)): + continue + else: + self.assertIn(attr, self.policy_defaults, + "{} is not fully tested".format(attr)) + + def test_policy_is_immutable(self): + for policy in self.policies: + for attr in self.policy_defaults: + with self.assertRaisesRegex(AttributeError, attr+".*read-only"): + setattr(policy, attr, None) + with self.assertRaisesRegex(AttributeError, 'no attribute.*foo'): + policy.foo = None + + def test_set_policy_attrs_when_calledl(self): + testattrdict = { attr: None for attr in self.policy_defaults } + for policyclass in self.policies: + policy = policyclass.clone(**testattrdict) + for attr in self.policy_defaults: + self.assertIsNone(getattr(policy, attr)) + + def test_reject_non_policy_keyword_when_called(self): + for policyclass in self.policies: + with self.assertRaises(TypeError): + policyclass(this_keyword_should_not_be_valid=None) + with self.assertRaises(TypeError): + policyclass(newtline=None) + + def test_policy_addition(self): + expected = self.policy_defaults.copy() + p1 = email.policy.default.clone(max_line_length=100) + p2 = email.policy.default.clone(max_line_length=50) + added = p1 + p2 + expected.update(max_line_length=50) + for attr, value in expected.items(): + self.assertEqual(getattr(added, attr), value) + added = p2 + p1 + expected.update(max_line_length=100) + for attr, value in expected.items(): + self.assertEqual(getattr(added, attr), value) + added = added + email.policy.default + for attr, value in expected.items(): + self.assertEqual(getattr(added, attr), value) + + def test_register_defect(self): + class Dummy: + def __init__(self): + self.defects = [] + obj = Dummy() + defect = object() + policy = email.policy.Policy() + policy.register_defect(obj, defect) + self.assertEqual(obj.defects, [defect]) + defect2 = object() + policy.register_defect(obj, defect2) + self.assertEqual(obj.defects, [defect, defect2]) + + class MyObj: + def __init__(self): + self.defects = [] + + class MyDefect(Exception): + pass + + def test_handle_defect_raises_on_strict(self): + foo = self.MyObj() + defect = self.MyDefect("the telly is broken") + with self.assertRaisesRegex(self.MyDefect, "the telly is broken"): + email.policy.strict.handle_defect(foo, defect) + + def test_handle_defect_registers_defect(self): + foo = self.MyObj() + defect1 = self.MyDefect("one") + email.policy.default.handle_defect(foo, defect1) + self.assertEqual(foo.defects, [defect1]) + defect2 = self.MyDefect("two") + email.policy.default.handle_defect(foo, defect2) + self.assertEqual(foo.defects, [defect1, defect2]) + + class MyPolicy(email.policy.Policy): + defects = [] + def register_defect(self, obj, defect): + self.defects.append(defect) + + def test_overridden_register_defect_still_raises(self): + foo = self.MyObj() + defect = self.MyDefect("the telly is broken") + with self.assertRaisesRegex(self.MyDefect, "the telly is broken"): + self.MyPolicy(raise_on_defect=True).handle_defect(foo, defect) + + def test_overriden_register_defect_works(self): + foo = self.MyObj() + defect1 = self.MyDefect("one") + my_policy = self.MyPolicy() + my_policy.handle_defect(foo, defect1) + self.assertEqual(my_policy.defects, [defect1]) + self.assertEqual(foo.defects, []) + defect2 = self.MyDefect("two") + my_policy.handle_defect(foo, defect2) + self.assertEqual(my_policy.defects, [defect1, defect2]) + self.assertEqual(foo.defects, []) + + # XXX: Need subclassing tests. + # For adding subclassed objects, make sure the usual rules apply (subclass + # wins), but that the order still works (right overrides left). + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_email/test_utils.py b/Lib/test/test_email/test_utils.py new file mode 100644 index 0000000..e003a64 --- /dev/null +++ b/Lib/test/test_email/test_utils.py @@ -0,0 +1,45 @@ +import datetime +from email import utils +import unittest + +class DateTimeTests(unittest.TestCase): + + datestring = 'Sun, 23 Sep 2001 20:10:55' + dateargs = (2001, 9, 23, 20, 10, 55) + offsetstring = ' -0700' + utcoffset = datetime.timedelta(hours=-7) + tz = datetime.timezone(utcoffset) + naive_dt = datetime.datetime(*dateargs) + aware_dt = datetime.datetime(*dateargs, tzinfo=tz) + + def test_naive_datetime(self): + self.assertEqual(utils.format_datetime(self.naive_dt), + self.datestring + ' -0000') + + def test_aware_datetime(self): + self.assertEqual(utils.format_datetime(self.aware_dt), + self.datestring + self.offsetstring) + + def test_usegmt(self): + utc_dt = datetime.datetime(*self.dateargs, + tzinfo=datetime.timezone.utc) + self.assertEqual(utils.format_datetime(utc_dt, usegmt=True), + self.datestring + ' GMT') + + def test_usegmt_with_naive_datetime_raises(self): + with self.assertRaises(ValueError): + utils.format_datetime(self.naive_dt, usegmt=True) + + def test_usegmt_with_non_utc_datetime_raises(self): + with self.assertRaises(ValueError): + utils.format_datetime(self.aware_dt, usegmt=True) + + def test_parsedate_to_datetime(self): + self.assertEqual( + utils.parsedate_to_datetime(self.datestring + self.offsetstring), + self.aware_dt) + + def test_parsedate_to_datetime_naive(self): + self.assertEqual( + utils.parsedate_to_datetime(self.datestring + ' -0000'), + self.naive_dt) diff --git a/Lib/test/test_email/torture_test.py b/Lib/test/test_email/torture_test.py new file mode 100644 index 0000000..544b1bb --- /dev/null +++ b/Lib/test/test_email/torture_test.py @@ -0,0 +1,136 @@ +# Copyright (C) 2002-2004 Python Software Foundation +# +# A torture test of the email package. This should not be run as part of the +# standard Python test suite since it requires several meg of email messages +# collected in the wild. These source messages are not checked into the +# Python distro, but are available as part of the standalone email package at +# http://sf.net/projects/mimelib + +import sys +import os +import unittest +from io import StringIO +from types import ListType + +from email.test.test_email import TestEmailBase +from test.support import TestSkipped, run_unittest + +import email +from email import __file__ as testfile +from email.iterators import _structure + +def openfile(filename): + from os.path import join, dirname, abspath + path = abspath(join(dirname(testfile), os.pardir, 'moredata', filename)) + return open(path, 'r') + +# Prevent this test from running in the Python distro +try: + openfile('crispin-torture.txt') +except IOError: + raise TestSkipped + + + +class TortureBase(TestEmailBase): + def _msgobj(self, filename): + fp = openfile(filename) + try: + msg = email.message_from_file(fp) + finally: + fp.close() + return msg + + + +class TestCrispinTorture(TortureBase): + # Mark Crispin's torture test from the SquirrelMail project + def test_mondo_message(self): + eq = self.assertEqual + neq = self.ndiffAssertEqual + msg = self._msgobj('crispin-torture.txt') + payload = msg.get_payload() + eq(type(payload), ListType) + eq(len(payload), 12) + eq(msg.preamble, None) + eq(msg.epilogue, '\n') + # Probably the best way to verify the message is parsed correctly is to + # dump its structure and compare it against the known structure. + fp = StringIO() + _structure(msg, fp=fp) + neq(fp.getvalue(), """\ +multipart/mixed + text/plain + message/rfc822 + multipart/alternative + text/plain + multipart/mixed + text/richtext + application/andrew-inset + message/rfc822 + audio/basic + audio/basic + image/pbm + message/rfc822 + multipart/mixed + multipart/mixed + text/plain + audio/x-sun + multipart/mixed + image/gif + image/gif + application/x-be2 + application/atomicmail + audio/x-sun + message/rfc822 + multipart/mixed + text/plain + image/pgm + text/plain + message/rfc822 + multipart/mixed + text/plain + image/pbm + message/rfc822 + application/postscript + image/gif + message/rfc822 + multipart/mixed + audio/basic + audio/basic + message/rfc822 + multipart/mixed + application/postscript + text/plain + message/rfc822 + multipart/mixed + text/plain + multipart/parallel + image/gif + audio/basic + application/atomicmail + message/rfc822 + audio/x-sun +""") + + +def _testclasses(): + mod = sys.modules[__name__] + return [getattr(mod, name) for name in dir(mod) if name.startswith('Test')] + + +def suite(): + suite = unittest.TestSuite() + for testclass in _testclasses(): + suite.addTest(unittest.makeSuite(testclass)) + return suite + + +def test_main(): + for testclass in _testclasses(): + run_unittest(testclass) + + + +if __name__ == '__main__': + unittest.main(defaultTest='suite') diff --git a/Lib/test/test_exceptions.py b/Lib/test/test_exceptions.py index 0a7ddd4..a7683ac 100644 --- a/Lib/test/test_exceptions.py +++ b/Lib/test/test_exceptions.py @@ -8,7 +8,7 @@ import weakref import errno from test.support import (TESTFN, unlink, run_unittest, captured_output, - gc_collect, cpython_only) + gc_collect, cpython_only, no_tracing) # XXX This is not really enough, each *operation* should be tested! @@ -46,8 +46,8 @@ class ExceptionTests(unittest.TestCase): fp.close() unlink(TESTFN) - self.raise_catch(IOError, "IOError") - self.assertRaises(IOError, open, 'this file does not exist', 'r') + self.raise_catch(OSError, "OSError") + self.assertRaises(OSError, open, 'this file does not exist', 'r') self.raise_catch(ImportError, "ImportError") self.assertRaises(ImportError, __import__, "undefined_module") @@ -192,11 +192,35 @@ class ExceptionTests(unittest.TestCase): except NameError: pass else: - self.assertEqual(str(WindowsError(1001)), "1001") - self.assertEqual(str(WindowsError(1001, "message")), - "[Error 1001] message") - self.assertEqual(WindowsError(1001, "message").errno, 22) - self.assertEqual(WindowsError(1001, "message").winerror, 1001) + self.assertIs(WindowsError, OSError) + self.assertEqual(str(OSError(1001)), "1001") + self.assertEqual(str(OSError(1001, "message")), + "[Errno 1001] message") + # POSIX errno (9 aka EBADF) is untranslated + w = OSError(9, 'foo', 'bar') + self.assertEqual(w.errno, 9) + self.assertEqual(w.winerror, None) + self.assertEqual(str(w), "[Errno 9] foo: 'bar'") + # ERROR_PATH_NOT_FOUND (win error 3) becomes ENOENT (2) + w = OSError(0, 'foo', 'bar', 3) + self.assertEqual(w.errno, 2) + self.assertEqual(w.winerror, 3) + self.assertEqual(w.strerror, 'foo') + self.assertEqual(w.filename, 'bar') + self.assertEqual(str(w), "[Error 3] foo: 'bar'") + # Unknown win error becomes EINVAL (22) + w = OSError(0, 'foo', None, 1001) + self.assertEqual(w.errno, 22) + self.assertEqual(w.winerror, 1001) + self.assertEqual(w.strerror, 'foo') + self.assertEqual(w.filename, None) + self.assertEqual(str(w), "[Error 1001] foo") + # Non-numeric "errno" + w = OSError('bar', 'foo') + self.assertEqual(w.errno, 'bar') + self.assertEqual(w.winerror, None) + self.assertEqual(w.strerror, 'foo') + self.assertEqual(w.filename, None) def testAttributes(self): # test that exception attributes are happy @@ -274,11 +298,12 @@ class ExceptionTests(unittest.TestCase): 'start' : 0, 'end' : 1}), ] try: + # More tests are in test_WindowsError exceptionList.append( (WindowsError, (1, 'strErrorStr', 'filenameStr'), {'args' : (1, 'strErrorStr'), - 'strerror' : 'strErrorStr', 'winerror' : 1, - 'errno' : 22, 'filename' : 'filenameStr'}) + 'strerror' : 'strErrorStr', 'winerror' : None, + 'errno' : 1, 'filename' : 'filenameStr'}) ) except NameError: pass @@ -389,6 +414,7 @@ class ExceptionTests(unittest.TestCase): x = DerivedException(fancy_arg=42) self.assertEqual(x.fancy_arg, 42) + @no_tracing def testInfiniteRecursion(self): def f(): return f() @@ -721,6 +747,7 @@ class ExceptionTests(unittest.TestCase): u.start = 1000 self.assertEqual(str(u), "can't translate characters in position 1000-4: 965230951443685724997") + @no_tracing def test_badisinstance(self): # Bug #2542: if issubclass(e, MyException) raises an exception, # it should be ignored @@ -831,6 +858,7 @@ class ExceptionTests(unittest.TestCase): self.fail("MemoryError not raised") self.assertEqual(wr(), None) + @no_tracing def test_recursion_error_cleanup(self): # Same test as above, but with "recursion exceeded" errors class C: diff --git a/Lib/test/test_extcall.py b/Lib/test/test_extcall.py index 1f7f630..6b6c12d 100644 --- a/Lib/test/test_extcall.py +++ b/Lib/test/test_extcall.py @@ -66,17 +66,17 @@ Verify clearing of SF bug #733667 >>> g() Traceback (most recent call last): ... - TypeError: g() takes at least 1 argument (0 given) + TypeError: g() missing 1 required positional argument: 'x' >>> g(*()) Traceback (most recent call last): ... - TypeError: g() takes at least 1 argument (0 given) + TypeError: g() missing 1 required positional argument: 'x' >>> g(*(), **{}) Traceback (most recent call last): ... - TypeError: g() takes at least 1 argument (0 given) + TypeError: g() missing 1 required positional argument: 'x' >>> g(1) 1 () {} @@ -151,7 +151,7 @@ What about willful misconduct? >>> g(1, 2, 3, **{'x': 4, 'y': 5}) Traceback (most recent call last): ... - TypeError: g() got multiple values for keyword argument 'x' + TypeError: g() got multiple values for argument 'x' >>> f(**{1:2}) Traceback (most recent call last): @@ -263,29 +263,80 @@ the function call setup. See <http://bugs.python.org/issue2016>. >>> f(**x) 1 2 -A obscure message: +Too many arguments: - >>> def f(a, b): - ... pass - >>> f(b=1) + >>> def f(): pass + >>> f(1) + Traceback (most recent call last): + ... + TypeError: f() takes 0 positional arguments but 1 was given + >>> def f(a): pass + >>> f(1, 2) Traceback (most recent call last): ... - TypeError: f() takes exactly 2 arguments (1 given) + TypeError: f() takes 1 positional argument but 2 were given + >>> def f(a, b=1): pass + >>> f(1, 2, 3) + Traceback (most recent call last): + ... + TypeError: f() takes from 1 to 2 positional arguments but 3 were given + >>> def f(*, kw): pass + >>> f(1, kw=3) + Traceback (most recent call last): + ... + TypeError: f() takes 0 positional arguments but 1 positional argument (and 1 keyword-only argument) were given + >>> def f(*, kw, b): pass + >>> f(1, 2, 3, b=3, kw=3) + Traceback (most recent call last): + ... + TypeError: f() takes 0 positional arguments but 3 positional arguments (and 2 keyword-only arguments) were given + >>> def f(a, b=2, *, kw): pass + >>> f(2, 3, 4, kw=4) + Traceback (most recent call last): + ... + TypeError: f() takes from 1 to 2 positional arguments but 3 positional arguments (and 1 keyword-only argument) were given -The number of arguments passed in includes keywords: +Too few and missing arguments: - >>> def f(a): - ... pass - >>> f(6, a=4, *(1, 2, 3)) + >>> def f(a): pass + >>> f() Traceback (most recent call last): ... - TypeError: f() takes exactly 1 positional argument (5 given) - >>> def f(a, *, kw): - ... pass - >>> f(6, 4, kw=4) + TypeError: f() missing 1 required positional argument: 'a' + >>> def f(a, b): pass + >>> f() Traceback (most recent call last): ... - TypeError: f() takes exactly 1 positional argument (3 given) + TypeError: f() missing 2 required positional arguments: 'a' and 'b' + >>> def f(a, b, c): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 3 required positional arguments: 'a', 'b', and 'c' + >>> def f(a, b, c, d, e): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 5 required positional arguments: 'a', 'b', 'c', 'd', and 'e' + >>> def f(a, b=4, c=5, d=5): pass + >>> f(c=12, b=9) + Traceback (most recent call last): + ... + TypeError: f() missing 1 required positional argument: 'a' + +Same with keyword only args: + + >>> def f(*, w): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 1 required keyword-only argument: 'w' + >>> def f(*, a, b, c, d, e): pass + >>> f() + Traceback (most recent call last): + ... + TypeError: f() missing 5 required keyword-only arguments: 'a', 'b', 'c', 'd', and 'e' + """ import sys diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py new file mode 100644 index 0000000..977cb39 --- /dev/null +++ b/Lib/test/test_faulthandler.py @@ -0,0 +1,554 @@ +from contextlib import contextmanager +import datetime +import faulthandler +import os +import re +import signal +import subprocess +import sys +from test import support, script_helper +import tempfile +import unittest + +try: + import threading + HAVE_THREADS = True +except ImportError: + HAVE_THREADS = False + +TIMEOUT = 0.5 + +try: + from resource import setrlimit, RLIMIT_CORE, error as resource_error +except ImportError: + prepare_subprocess = None +else: + def prepare_subprocess(): + # don't create core file + try: + setrlimit(RLIMIT_CORE, (0, 0)) + except (ValueError, resource_error): + pass + +def expected_traceback(lineno1, lineno2, header, min_count=1): + regex = header + regex += ' File "<string>", line %s in func\n' % lineno1 + regex += ' File "<string>", line %s in <module>' % lineno2 + if 1 < min_count: + return '^' + (regex + '\n') * (min_count - 1) + regex + else: + return '^' + regex + '$' + +@contextmanager +def temporary_filename(): + filename = tempfile.mktemp() + try: + yield filename + finally: + support.unlink(filename) + +class FaultHandlerTests(unittest.TestCase): + def get_output(self, code, filename=None): + """ + Run the specified code in Python (in a new child process) and read the + output from the standard error or from a file (if filename is set). + Return the output lines as a list. + + Strip the reference count from the standard error for Python debug + build, and replace "Current thread 0x00007f8d8fbd9700" by "Current + thread XXX". + """ + options = {} + if prepare_subprocess: + options['preexec_fn'] = prepare_subprocess + process = script_helper.spawn_python('-c', code, **options) + stdout, stderr = process.communicate() + exitcode = process.wait() + output = support.strip_python_stderr(stdout) + output = output.decode('ascii', 'backslashreplace') + if filename: + self.assertEqual(output, '') + with open(filename, "rb") as fp: + output = fp.read() + output = output.decode('ascii', 'backslashreplace') + output = re.sub('Current thread 0x[0-9a-f]+', + 'Current thread XXX', + output) + return output.splitlines(), exitcode + + def check_fatal_error(self, code, line_number, name_regex, + filename=None, all_threads=True, other_regex=None): + """ + Check that the fault handler for fatal errors is enabled and check the + traceback from the child process output. + + Raise an error if the output doesn't match the expected format. + """ + if all_threads: + header = 'Current thread XXX' + else: + header = 'Traceback (most recent call first)' + regex = """ +^Fatal Python error: {name} + +{header}: + File "<string>", line {lineno} in <module>$ +""".strip() + regex = regex.format( + lineno=line_number, + name=name_regex, + header=re.escape(header)) + if other_regex: + regex += '|' + other_regex + output, exitcode = self.get_output(code, filename) + output = '\n'.join(output) + self.assertRegex(output, regex) + self.assertNotEqual(exitcode, 0) + + def test_read_null(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._read_null() +""".strip(), + 3, + # Issue #12700: Read NULL raises SIGILL on Mac OS X Lion + '(?:Segmentation fault|Bus error|Illegal instruction)') + + def test_sigsegv(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigsegv() +""".strip(), + 3, + 'Segmentation fault') + + def test_sigabrt(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigabrt() +""".strip(), + 3, + 'Aborted') + + @unittest.skipIf(sys.platform == 'win32', + "SIGFPE cannot be caught on Windows") + def test_sigfpe(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigfpe() +""".strip(), + 3, + 'Floating point exception') + + @unittest.skipIf(not hasattr(faulthandler, '_sigbus'), + "need faulthandler._sigbus()") + def test_sigbus(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigbus() +""".strip(), + 3, + 'Bus error') + + @unittest.skipIf(not hasattr(faulthandler, '_sigill'), + "need faulthandler._sigill()") + def test_sigill(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._sigill() +""".strip(), + 3, + 'Illegal instruction') + + def test_fatal_error(self): + self.check_fatal_error(""" +import faulthandler +faulthandler._fatal_error(b'xyz') +""".strip(), + 2, + 'xyz') + + @unittest.skipIf(sys.platform.startswith('openbsd') and HAVE_THREADS, + "Issue #12868: sigaltstack() doesn't work on " + "OpenBSD if Python is compiled with pthread") + @unittest.skipIf(not hasattr(faulthandler, '_stack_overflow'), + 'need faulthandler._stack_overflow()') + def test_stack_overflow(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._stack_overflow() +""".strip(), + 3, + '(?:Segmentation fault|Bus error)', + other_regex='unable to raise a stack overflow') + + def test_gil_released(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable() +faulthandler._read_null(True) +""".strip(), + 3, + '(?:Segmentation fault|Bus error|Illegal instruction)') + + def test_enable_file(self): + with temporary_filename() as filename: + self.check_fatal_error(""" +import faulthandler +output = open({filename}, 'wb') +faulthandler.enable(output) +faulthandler._read_null() +""".strip().format(filename=repr(filename)), + 4, + '(?:Segmentation fault|Bus error|Illegal instruction)', + filename=filename) + + def test_enable_single_thread(self): + self.check_fatal_error(""" +import faulthandler +faulthandler.enable(all_threads=False) +faulthandler._read_null() +""".strip(), + 3, + '(?:Segmentation fault|Bus error|Illegal instruction)', + all_threads=False) + + def test_disable(self): + code = """ +import faulthandler +faulthandler.enable() +faulthandler.disable() +faulthandler._read_null() +""".strip() + not_expected = 'Fatal Python error' + stderr, exitcode = self.get_output(code) + stder = '\n'.join(stderr) + self.assertTrue(not_expected not in stderr, + "%r is present in %r" % (not_expected, stderr)) + self.assertNotEqual(exitcode, 0) + + def test_is_enabled(self): + orig_stderr = sys.stderr + try: + # regrtest may replace sys.stderr by io.StringIO object, but + # faulthandler.enable() requires that sys.stderr has a fileno() + # method + sys.stderr = sys.__stderr__ + + was_enabled = faulthandler.is_enabled() + try: + faulthandler.enable() + self.assertTrue(faulthandler.is_enabled()) + faulthandler.disable() + self.assertFalse(faulthandler.is_enabled()) + finally: + if was_enabled: + faulthandler.enable() + else: + faulthandler.disable() + finally: + sys.stderr = orig_stderr + + def check_dump_traceback(self, filename): + """ + Explicitly call dump_traceback() function and check its output. + Raise an error if the output doesn't match the expected format. + """ + code = """ +import faulthandler + +def funcB(): + if {has_filename}: + with open({filename}, "wb") as fp: + faulthandler.dump_traceback(fp, all_threads=False) + else: + faulthandler.dump_traceback(all_threads=False) + +def funcA(): + funcB() + +funcA() +""".strip() + code = code.format( + filename=repr(filename), + has_filename=bool(filename), + ) + if filename: + lineno = 6 + else: + lineno = 8 + expected = [ + 'Traceback (most recent call first):', + ' File "<string>", line %s in funcB' % lineno, + ' File "<string>", line 11 in funcA', + ' File "<string>", line 13 in <module>' + ] + trace, exitcode = self.get_output(code, filename) + self.assertEqual(trace, expected) + self.assertEqual(exitcode, 0) + + def test_dump_traceback(self): + self.check_dump_traceback(None) + + def test_dump_traceback_file(self): + with temporary_filename() as filename: + self.check_dump_traceback(filename) + + @unittest.skipIf(not HAVE_THREADS, 'need threads') + def check_dump_traceback_threads(self, filename): + """ + Call explicitly dump_traceback(all_threads=True) and check the output. + Raise an error if the output doesn't match the expected format. + """ + code = """ +import faulthandler +from threading import Thread, Event +import time + +def dump(): + if {filename}: + with open({filename}, "wb") as fp: + faulthandler.dump_traceback(fp, all_threads=True) + else: + faulthandler.dump_traceback(all_threads=True) + +class Waiter(Thread): + # avoid blocking if the main thread raises an exception. + daemon = True + + def __init__(self): + Thread.__init__(self) + self.running = Event() + self.stop = Event() + + def run(self): + self.running.set() + self.stop.wait() + +waiter = Waiter() +waiter.start() +waiter.running.wait() +dump() +waiter.stop.set() +waiter.join() +""".strip() + code = code.format(filename=repr(filename)) + output, exitcode = self.get_output(code, filename) + output = '\n'.join(output) + if filename: + lineno = 8 + else: + lineno = 10 + regex = """ +^Thread 0x[0-9a-f]+: +(?: File ".*threading.py", line [0-9]+ in [_a-z]+ +){{1,3}} File "<string>", line 23 in run + File ".*threading.py", line [0-9]+ in _bootstrap_inner + File ".*threading.py", line [0-9]+ in _bootstrap + +Current thread XXX: + File "<string>", line {lineno} in dump + File "<string>", line 28 in <module>$ +""".strip() + regex = regex.format(lineno=lineno) + self.assertRegex(output, regex) + self.assertEqual(exitcode, 0) + + def test_dump_traceback_threads(self): + self.check_dump_traceback_threads(None) + + def test_dump_traceback_threads_file(self): + with temporary_filename() as filename: + self.check_dump_traceback_threads(filename) + + def _check_dump_tracebacks_later(self, repeat, cancel, filename, loops): + """ + Check how many times the traceback is written in timeout x 2.5 seconds, + or timeout x 3.5 seconds if cancel is True: 1, 2 or 3 times depending + on repeat and cancel options. + + Raise an error if the output doesn't match the expect format. + """ + timeout_str = str(datetime.timedelta(seconds=TIMEOUT)) + code = """ +import faulthandler +import time + +def func(timeout, repeat, cancel, file, loops): + for loop in range(loops): + faulthandler.dump_tracebacks_later(timeout, repeat=repeat, file=file) + if cancel: + faulthandler.cancel_dump_tracebacks_later() + time.sleep(timeout * 5) + faulthandler.cancel_dump_tracebacks_later() + +timeout = {timeout} +repeat = {repeat} +cancel = {cancel} +loops = {loops} +if {has_filename}: + file = open({filename}, "wb") +else: + file = None +func(timeout, repeat, cancel, file, loops) +if file is not None: + file.close() +""".strip() + code = code.format( + timeout=TIMEOUT, + repeat=repeat, + cancel=cancel, + loops=loops, + has_filename=bool(filename), + filename=repr(filename), + ) + trace, exitcode = self.get_output(code, filename) + trace = '\n'.join(trace) + + if not cancel: + count = loops + if repeat: + count *= 2 + header = r'Timeout \(%s\)!\nThread 0x[0-9a-f]+:\n' % timeout_str + regex = expected_traceback(9, 20, header, min_count=count) + self.assertRegex(trace, regex) + else: + self.assertEqual(trace, '') + self.assertEqual(exitcode, 0) + + @unittest.skipIf(not hasattr(faulthandler, 'dump_tracebacks_later'), + 'need faulthandler.dump_tracebacks_later()') + def check_dump_tracebacks_later(self, repeat=False, cancel=False, + file=False, twice=False): + if twice: + loops = 2 + else: + loops = 1 + if file: + with temporary_filename() as filename: + self._check_dump_tracebacks_later(repeat, cancel, + filename, loops) + else: + self._check_dump_tracebacks_later(repeat, cancel, None, loops) + + def test_dump_tracebacks_later(self): + self.check_dump_tracebacks_later() + + def test_dump_tracebacks_later_repeat(self): + self.check_dump_tracebacks_later(repeat=True) + + def test_dump_tracebacks_later_cancel(self): + self.check_dump_tracebacks_later(cancel=True) + + def test_dump_tracebacks_later_file(self): + self.check_dump_tracebacks_later(file=True) + + def test_dump_tracebacks_later_twice(self): + self.check_dump_tracebacks_later(twice=True) + + @unittest.skipIf(not hasattr(faulthandler, "register"), + "need faulthandler.register") + def check_register(self, filename=False, all_threads=False, + unregister=False, chain=False): + """ + Register a handler displaying the traceback on a user signal. Raise the + signal and check the written traceback. + + If chain is True, check that the previous signal handler is called. + + Raise an error if the output doesn't match the expected format. + """ + signum = signal.SIGUSR1 + code = """ +import faulthandler +import os +import signal +import sys + +def func(signum): + os.kill(os.getpid(), signum) + +def handler(signum, frame): + handler.called = True +handler.called = False + +exitcode = 0 +signum = {signum} +unregister = {unregister} +chain = {chain} + +if {has_filename}: + file = open({filename}, "wb") +else: + file = None +if chain: + signal.signal(signum, handler) +faulthandler.register(signum, file=file, + all_threads={all_threads}, chain={chain}) +if unregister: + faulthandler.unregister(signum) +func(signum) +if chain and not handler.called: + if file is not None: + output = file + else: + output = sys.stderr + print("Error: signal handler not called!", file=output) + exitcode = 1 +if file is not None: + file.close() +sys.exit(exitcode) +""".strip() + code = code.format( + filename=repr(filename), + has_filename=bool(filename), + all_threads=all_threads, + signum=signum, + unregister=unregister, + chain=chain, + ) + trace, exitcode = self.get_output(code, filename) + trace = '\n'.join(trace) + if not unregister: + if all_threads: + regex = 'Current thread XXX:\n' + else: + regex = 'Traceback \(most recent call first\):\n' + regex = expected_traceback(7, 28, regex) + self.assertRegex(trace, regex) + else: + self.assertEqual(trace, '') + if unregister: + self.assertNotEqual(exitcode, 0) + else: + self.assertEqual(exitcode, 0) + + def test_register(self): + self.check_register() + + def test_unregister(self): + self.check_register(unregister=True) + + def test_register_file(self): + with temporary_filename() as filename: + self.check_register(filename=filename) + + def test_register_threads(self): + self.check_register(all_threads=True) + + def test_register_chain(self): + self.check_register(chain=True) + + +def test_main(): + support.run_unittest(FaultHandlerTests) + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_fileinput.py b/Lib/test/test_fileinput.py index f312882..1e70641 100644 --- a/Lib/test/test_fileinput.py +++ b/Lib/test/test_fileinput.py @@ -2,18 +2,33 @@ Tests for fileinput module. Nick Mathewson ''' - +import os +import sys +import re +import fileinput +import collections +import builtins import unittest -from test.support import verbose, TESTFN, run_unittest -from test.support import unlink as safe_unlink -import sys, re + +try: + import bz2 +except ImportError: + bz2 = None +try: + import gzip +except ImportError: + gzip = None + from io import StringIO from fileinput import FileInput, hook_encoded +from test.support import verbose, TESTFN, run_unittest +from test.support import unlink as safe_unlink + + # The fileinput module has 2 interfaces: the FileInput class which does # all the work, and a few functions (input, etc.) that use a global _state -# variable. We only test the FileInput class, since the other functions -# only provide a thin facade over FileInput. +# variable. # Write lines (a list of lines) to temp file number i, and return the # temp file's name. @@ -121,7 +136,16 @@ class BufferSizesTests(unittest.TestCase): self.assertEqual(int(m.group(1)), fi.filelineno()) fi.close() +class UnconditionallyRaise: + def __init__(self, exception_type): + self.exception_type = exception_type + self.invoked = False + def __call__(self, *args, **kwargs): + self.invoked = True + raise self.exception_type() + class FileInputTests(unittest.TestCase): + def test_zero_byte_files(self): t1 = t2 = t3 = t4 = None try: @@ -219,17 +243,20 @@ class FileInputTests(unittest.TestCase): self.fail("FileInput should check openhook for being callable") except ValueError: pass - # XXX The rot13 codec was removed. - # So this test needs to be changed to use something else. - # (Or perhaps the API needs to change so we can just pass - # an encoding rather than using a hook?) -## try: -## t1 = writeTmp(1, ["A\nB"], mode="wb") -## fi = FileInput(files=t1, openhook=hook_encoded("rot13")) -## lines = list(fi) -## self.assertEqual(lines, ["N\n", "O"]) -## finally: -## remove_tempfiles(t1) + + class CustomOpenHook: + def __init__(self): + self.invoked = False + def __call__(self, *args): + self.invoked = True + return open(*args) + + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + custom_open_hook = CustomOpenHook() + with FileInput([t], openhook=custom_open_hook) as fi: + fi.readline() + self.assertTrue(custom_open_hook.invoked, "openhook not invoked") def test_context_manager(self): try: @@ -254,9 +281,576 @@ class FileInputTests(unittest.TestCase): finally: remove_tempfiles(t1) + def test_empty_files_list_specified_to_constructor(self): + with FileInput(files=[]) as fi: + self.assertEqual(fi._files, ('-',)) + + def test__getitem__(self): + """Tests invoking FileInput.__getitem__() with the current + line number""" + t = writeTmp(1, ["line1\n", "line2\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t]) as fi: + retval1 = fi[0] + self.assertEqual(retval1, "line1\n") + retval2 = fi[1] + self.assertEqual(retval2, "line2\n") + + def test__getitem__invalid_key(self): + """Tests invoking FileInput.__getitem__() with an index unequal to + the line number""" + t = writeTmp(1, ["line1\n", "line2\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t]) as fi: + with self.assertRaises(RuntimeError) as cm: + fi[1] + self.assertEqual(cm.exception.args, ("accessing lines out of order",)) + + def test__getitem__eof(self): + """Tests invoking FileInput.__getitem__() with the line number but at + end-of-input""" + t = writeTmp(1, []) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t]) as fi: + with self.assertRaises(IndexError) as cm: + fi[0] + self.assertEqual(cm.exception.args, ("end of input reached",)) + + def test_nextfile_oserror_deleting_backup(self): + """Tests invoking FileInput.nextfile() when the attempt to delete + the backup file would raise OSError. This error is expected to be + silently ignored""" + + os_unlink_orig = os.unlink + os_unlink_replacement = UnconditionallyRaise(OSError) + try: + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t], inplace=True) as fi: + next(fi) # make sure the file is opened + os.unlink = os_unlink_replacement + fi.nextfile() + finally: + os.unlink = os_unlink_orig + + # sanity check to make sure that our test scenario was actually hit + self.assertTrue(os_unlink_replacement.invoked, + "os.unlink() was not invoked") + + def test_readline_os_fstat_raises_OSError(self): + """Tests invoking FileInput.readline() when os.fstat() raises OSError. + This exception should be silently discarded.""" + + os_fstat_orig = os.fstat + os_fstat_replacement = UnconditionallyRaise(OSError) + try: + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t], inplace=True) as fi: + os.fstat = os_fstat_replacement + fi.readline() + finally: + os.fstat = os_fstat_orig + + # sanity check to make sure that our test scenario was actually hit + self.assertTrue(os_fstat_replacement.invoked, + "os.fstat() was not invoked") + + @unittest.skipIf(not hasattr(os, "chmod"), "os.chmod does not exist") + def test_readline_os_chmod_raises_OSError(self): + """Tests invoking FileInput.readline() when os.chmod() raises OSError. + This exception should be silently discarded.""" + + os_chmod_orig = os.chmod + os_chmod_replacement = UnconditionallyRaise(OSError) + try: + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t], inplace=True) as fi: + os.chmod = os_chmod_replacement + fi.readline() + finally: + os.chmod = os_chmod_orig + + # sanity check to make sure that our test scenario was actually hit + self.assertTrue(os_chmod_replacement.invoked, + "os.fstat() was not invoked") + + def test_fileno_when_ValueError_raised(self): + class FilenoRaisesValueError(UnconditionallyRaise): + def __init__(self): + UnconditionallyRaise.__init__(self, ValueError) + def fileno(self): + self.__call__() + + unconditionally_raise_ValueError = FilenoRaisesValueError() + t = writeTmp(1, ["\n"]) + self.addCleanup(remove_tempfiles, t) + with FileInput(files=[t]) as fi: + file_backup = fi._file + try: + fi._file = unconditionally_raise_ValueError + result = fi.fileno() + finally: + fi._file = file_backup # make sure the file gets cleaned up + + # sanity check to make sure that our test scenario was actually hit + self.assertTrue(unconditionally_raise_ValueError.invoked, + "_file.fileno() was not invoked") + + self.assertEqual(result, -1, "fileno() should return -1") + +class MockFileInput: + """A class that mocks out fileinput.FileInput for use during unit tests""" + + def __init__(self, files=None, inplace=False, backup="", bufsize=0, + mode="r", openhook=None): + self.files = files + self.inplace = inplace + self.backup = backup + self.bufsize = bufsize + self.mode = mode + self.openhook = openhook + self._file = None + self.invocation_counts = collections.defaultdict(lambda: 0) + self.return_values = {} + + def close(self): + self.invocation_counts["close"] += 1 + + def nextfile(self): + self.invocation_counts["nextfile"] += 1 + return self.return_values["nextfile"] + + def filename(self): + self.invocation_counts["filename"] += 1 + return self.return_values["filename"] + + def lineno(self): + self.invocation_counts["lineno"] += 1 + return self.return_values["lineno"] + + def filelineno(self): + self.invocation_counts["filelineno"] += 1 + return self.return_values["filelineno"] + + def fileno(self): + self.invocation_counts["fileno"] += 1 + return self.return_values["fileno"] + + def isfirstline(self): + self.invocation_counts["isfirstline"] += 1 + return self.return_values["isfirstline"] + + def isstdin(self): + self.invocation_counts["isstdin"] += 1 + return self.return_values["isstdin"] + +class BaseFileInputGlobalMethodsTest(unittest.TestCase): + """Base class for unit tests for the global function of + the fileinput module.""" + + def setUp(self): + self._orig_state = fileinput._state + self._orig_FileInput = fileinput.FileInput + fileinput.FileInput = MockFileInput + + def tearDown(self): + fileinput.FileInput = self._orig_FileInput + fileinput._state = self._orig_state + + def assertExactlyOneInvocation(self, mock_file_input, method_name): + # assert that the method with the given name was invoked once + actual_count = mock_file_input.invocation_counts[method_name] + self.assertEqual(actual_count, 1, method_name) + # assert that no other unexpected methods were invoked + actual_total_count = len(mock_file_input.invocation_counts) + self.assertEqual(actual_total_count, 1) + +class Test_fileinput_input(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.input()""" + + def test_state_is_not_None_and_state_file_is_not_None(self): + """Tests invoking fileinput.input() when fileinput._state is not None + and its _file attribute is also not None. Expect RuntimeError to + be raised with a meaningful error message and for fileinput._state + to *not* be modified.""" + instance = MockFileInput() + instance._file = object() + fileinput._state = instance + with self.assertRaises(RuntimeError) as cm: + fileinput.input() + self.assertEqual(("input() already active",), cm.exception.args) + self.assertIs(instance, fileinput._state, "fileinput._state") + + def test_state_is_not_None_and_state_file_is_None(self): + """Tests invoking fileinput.input() when fileinput._state is not None + but its _file attribute *is* None. Expect it to create and return + a new fileinput.FileInput object with all method parameters passed + explicitly to the __init__() method; also ensure that + fileinput._state is set to the returned instance.""" + instance = MockFileInput() + instance._file = None + fileinput._state = instance + self.do_test_call_input() + + def test_state_is_None(self): + """Tests invoking fileinput.input() when fileinput._state is None + Expect it to create and return a new fileinput.FileInput object + with all method parameters passed explicitly to the __init__() + method; also ensure that fileinput._state is set to the returned + instance.""" + fileinput._state = None + self.do_test_call_input() + + def do_test_call_input(self): + """Tests that fileinput.input() creates a new fileinput.FileInput + object, passing the given parameters unmodified to + fileinput.FileInput.__init__(). Note that this test depends on the + monkey patching of fileinput.FileInput done by setUp().""" + files = object() + inplace = object() + backup = object() + bufsize = object() + mode = object() + openhook = object() + + # call fileinput.input() with different values for each argument + result = fileinput.input(files=files, inplace=inplace, backup=backup, + bufsize=bufsize, + mode=mode, openhook=openhook) + + # ensure fileinput._state was set to the returned object + self.assertIs(result, fileinput._state, "fileinput._state") + + # ensure the parameters to fileinput.input() were passed directly + # to FileInput.__init__() + self.assertIs(files, result.files, "files") + self.assertIs(inplace, result.inplace, "inplace") + self.assertIs(backup, result.backup, "backup") + self.assertIs(bufsize, result.bufsize, "bufsize") + self.assertIs(mode, result.mode, "mode") + self.assertIs(openhook, result.openhook, "openhook") + +class Test_fileinput_close(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.close()""" + + def test_state_is_None(self): + """Tests that fileinput.close() does nothing if fileinput._state + is None""" + fileinput._state = None + fileinput.close() + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests that fileinput.close() invokes close() on fileinput._state + and sets _state=None""" + instance = MockFileInput() + fileinput._state = instance + fileinput.close() + self.assertExactlyOneInvocation(instance, "close") + self.assertIsNone(fileinput._state) + +class Test_fileinput_nextfile(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.nextfile()""" + + def test_state_is_None(self): + """Tests fileinput.nextfile() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.nextfile() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.nextfile() when fileinput._state is not None. + Ensure that it invokes fileinput._state.nextfile() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + nextfile_retval = object() + instance = MockFileInput() + instance.return_values["nextfile"] = nextfile_retval + fileinput._state = instance + retval = fileinput.nextfile() + self.assertExactlyOneInvocation(instance, "nextfile") + self.assertIs(retval, nextfile_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_filename(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.filename()""" + + def test_state_is_None(self): + """Tests fileinput.filename() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.filename() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.filename() when fileinput._state is not None. + Ensure that it invokes fileinput._state.filename() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + filename_retval = object() + instance = MockFileInput() + instance.return_values["filename"] = filename_retval + fileinput._state = instance + retval = fileinput.filename() + self.assertExactlyOneInvocation(instance, "filename") + self.assertIs(retval, filename_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_lineno(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.lineno()""" + + def test_state_is_None(self): + """Tests fileinput.lineno() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.lineno() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.lineno() when fileinput._state is not None. + Ensure that it invokes fileinput._state.lineno() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + lineno_retval = object() + instance = MockFileInput() + instance.return_values["lineno"] = lineno_retval + fileinput._state = instance + retval = fileinput.lineno() + self.assertExactlyOneInvocation(instance, "lineno") + self.assertIs(retval, lineno_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_filelineno(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.filelineno()""" + + def test_state_is_None(self): + """Tests fileinput.filelineno() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.filelineno() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.filelineno() when fileinput._state is not None. + Ensure that it invokes fileinput._state.filelineno() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + filelineno_retval = object() + instance = MockFileInput() + instance.return_values["filelineno"] = filelineno_retval + fileinput._state = instance + retval = fileinput.filelineno() + self.assertExactlyOneInvocation(instance, "filelineno") + self.assertIs(retval, filelineno_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_fileno(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.fileno()""" + + def test_state_is_None(self): + """Tests fileinput.fileno() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.fileno() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.fileno() when fileinput._state is not None. + Ensure that it invokes fileinput._state.fileno() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + fileno_retval = object() + instance = MockFileInput() + instance.return_values["fileno"] = fileno_retval + instance.fileno_retval = fileno_retval + fileinput._state = instance + retval = fileinput.fileno() + self.assertExactlyOneInvocation(instance, "fileno") + self.assertIs(retval, fileno_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_isfirstline(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.isfirstline()""" + + def test_state_is_None(self): + """Tests fileinput.isfirstline() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.isfirstline() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.isfirstline() when fileinput._state is not None. + Ensure that it invokes fileinput._state.isfirstline() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + isfirstline_retval = object() + instance = MockFileInput() + instance.return_values["isfirstline"] = isfirstline_retval + fileinput._state = instance + retval = fileinput.isfirstline() + self.assertExactlyOneInvocation(instance, "isfirstline") + self.assertIs(retval, isfirstline_retval) + self.assertIs(fileinput._state, instance) + +class Test_fileinput_isstdin(BaseFileInputGlobalMethodsTest): + """Unit tests for fileinput.isstdin()""" + + def test_state_is_None(self): + """Tests fileinput.isstdin() when fileinput._state is None. + Ensure that it raises RuntimeError with a meaningful error message + and does not modify fileinput._state""" + fileinput._state = None + with self.assertRaises(RuntimeError) as cm: + fileinput.isstdin() + self.assertEqual(("no active input()",), cm.exception.args) + self.assertIsNone(fileinput._state) + + def test_state_is_not_None(self): + """Tests fileinput.isstdin() when fileinput._state is not None. + Ensure that it invokes fileinput._state.isstdin() exactly once, + returns whatever it returns, and does not modify fileinput._state + to point to a different object.""" + isstdin_retval = object() + instance = MockFileInput() + instance.return_values["isstdin"] = isstdin_retval + fileinput._state = instance + retval = fileinput.isstdin() + self.assertExactlyOneInvocation(instance, "isstdin") + self.assertIs(retval, isstdin_retval) + self.assertIs(fileinput._state, instance) + +class InvocationRecorder: + def __init__(self): + self.invocation_count = 0 + def __call__(self, *args, **kwargs): + self.invocation_count += 1 + self.last_invocation = (args, kwargs) + +class Test_hook_compressed(unittest.TestCase): + """Unit tests for fileinput.hook_compressed()""" + + def setUp(self): + self.fake_open = InvocationRecorder() + + def test_empty_string(self): + self.do_test_use_builtin_open("", 1) + + def test_no_ext(self): + self.do_test_use_builtin_open("abcd", 2) + + @unittest.skipUnless(gzip, "Requires gzip and zlib") + def test_gz_ext_fake(self): + original_open = gzip.open + gzip.open = self.fake_open + try: + result = fileinput.hook_compressed("test.gz", 3) + finally: + gzip.open = original_open + + self.assertEqual(self.fake_open.invocation_count, 1) + self.assertEqual(self.fake_open.last_invocation, (("test.gz", 3), {})) + + @unittest.skipUnless(bz2, "Requires bz2") + def test_bz2_ext_fake(self): + original_open = bz2.BZ2File + bz2.BZ2File = self.fake_open + try: + result = fileinput.hook_compressed("test.bz2", 4) + finally: + bz2.BZ2File = original_open + + self.assertEqual(self.fake_open.invocation_count, 1) + self.assertEqual(self.fake_open.last_invocation, (("test.bz2", 4), {})) + + def test_blah_ext(self): + self.do_test_use_builtin_open("abcd.blah", 5) + + def test_gz_ext_builtin(self): + self.do_test_use_builtin_open("abcd.Gz", 6) + + def test_bz2_ext_builtin(self): + self.do_test_use_builtin_open("abcd.Bz2", 7) + + def do_test_use_builtin_open(self, filename, mode): + original_open = self.replace_builtin_open(self.fake_open) + try: + result = fileinput.hook_compressed(filename, mode) + finally: + self.replace_builtin_open(original_open) + + self.assertEqual(self.fake_open.invocation_count, 1) + self.assertEqual(self.fake_open.last_invocation, + ((filename, mode), {})) + + @staticmethod + def replace_builtin_open(new_open_func): + original_open = builtins.open + builtins.open = new_open_func + return original_open + +class Test_hook_encoded(unittest.TestCase): + """Unit tests for fileinput.hook_encoded()""" + + def test(self): + encoding = object() + result = fileinput.hook_encoded(encoding) + + fake_open = InvocationRecorder() + original_open = builtins.open + builtins.open = fake_open + try: + filename = object() + mode = object() + open_result = result(filename, mode) + finally: + builtins.open = original_open + + self.assertEqual(fake_open.invocation_count, 1) + + args, kwargs = fake_open.last_invocation + self.assertIs(args[0], filename) + self.assertIs(args[1], mode) + self.assertIs(kwargs.pop('encoding'), encoding) + self.assertFalse(kwargs) def test_main(): - run_unittest(BufferSizesTests, FileInputTests) + run_unittest( + BufferSizesTests, + FileInputTests, + Test_fileinput_input, + Test_fileinput_close, + Test_fileinput_nextfile, + Test_fileinput_filename, + Test_fileinput_lineno, + Test_fileinput_filelineno, + Test_fileinput_fileno, + Test_fileinput_isfirstline, + Test_fileinput_isstdin, + Test_hook_compressed, + Test_hook_encoded, + ) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_float.py b/Lib/test/test_float.py index 4d7bbba..dc0c291 100644 --- a/Lib/test/test_float.py +++ b/Lib/test/test_float.py @@ -88,7 +88,7 @@ class GeneralFloatCases(unittest.TestCase): self.assertRaises(ValueError, float, " -0x3.p-1 ") self.assertRaises(ValueError, float, " +0x3.p-1 ") self.assertEqual(float(" 25.e-1 "), 2.5) - self.assertEqual(support.fcmp(float(" .25e-1 "), .025), 0) + self.assertAlmostEqual(float(" .25e-1 "), .025) def test_floatconversion(self): # Make sure that calls to __float__() work properly diff --git a/Lib/test/test_ftplib.py b/Lib/test/test_ftplib.py index 5b69b0a..f73b8af 100644 --- a/Lib/test/test_ftplib.py +++ b/Lib/test/test_ftplib.py @@ -22,10 +22,25 @@ from test.support import HOST threading = support.import_module('threading') # the dummy data returned by server over the data channel when -# RETR, LIST and NLST commands are issued +# RETR, LIST, NLST, MLSD commands are issued RETR_DATA = 'abcde12345\r\n' * 1000 LIST_DATA = 'foo\r\nbar\r\n' NLST_DATA = 'foo\r\nbar\r\n' +MLSD_DATA = ("type=cdir;perm=el;unique==keVO1+ZF4; test\r\n" + "type=pdir;perm=e;unique==keVO1+d?3; ..\r\n" + "type=OS.unix=slink:/foobar;perm=;unique==keVO1+4G4; foobar\r\n" + "type=OS.unix=chr-13/29;perm=;unique==keVO1+5G4; device\r\n" + "type=OS.unix=blk-11/108;perm=;unique==keVO1+6G4; block\r\n" + "type=file;perm=awr;unique==keVO1+8G4; writable\r\n" + "type=dir;perm=cpmel;unique==keVO1+7G4; promiscuous\r\n" + "type=dir;perm=;unique==keVO1+1t2; no-exec\r\n" + "type=file;perm=r;unique==keVO1+EG4; two words\r\n" + "type=file;perm=r;unique==keVO1+IH4; leading space\r\n" + "type=file;perm=r;unique==keVO1+1G4; file1\r\n" + "type=dir;perm=cpmel;unique==keVO1+7G4; incoming\r\n" + "type=file;perm=r;unique==keVO1+1G4; file2\r\n" + "type=file;perm=r;unique==keVO1+1G4; file3\r\n" + "type=file;perm=r;unique==keVO1+1G4; file4\r\n") class DummyDTPHandler(asynchat.async_chat): @@ -49,6 +64,11 @@ class DummyDTPHandler(asynchat.async_chat): self.dtp_conn_closed = True def push(self, what): + if self.baseclass.next_data is not None: + what = self.baseclass.next_data + self.baseclass.next_data = None + if not what: + return self.close_when_done() super(DummyDTPHandler, self).push(what.encode('ascii')) def handle_error(self): @@ -69,6 +89,7 @@ class DummyFTPHandler(asynchat.async_chat): self.last_received_cmd = None self.last_received_data = '' self.next_response = '' + self.next_data = None self.rest = None self.push('220 welcome') @@ -104,7 +125,7 @@ class DummyFTPHandler(asynchat.async_chat): addr = list(map(int, arg.split(','))) ip = '%d.%d.%d.%d' %tuple(addr[:4]) port = (addr[4] * 256) + addr[5] - s = socket.create_connection((ip, port), timeout=10) + s = socket.create_connection((ip, port), timeout=2) self.dtp = self.dtp_handler(s, baseclass=self) self.push('200 active data connection established') @@ -122,7 +143,7 @@ class DummyFTPHandler(asynchat.async_chat): def cmd_eprt(self, arg): af, ip, port = arg.split(arg[0])[1:-1] port = int(port) - s = socket.create_connection((ip, port), timeout=10) + s = socket.create_connection((ip, port), timeout=2) self.dtp = self.dtp_handler(s, baseclass=self) self.push('200 active data connection established') @@ -213,6 +234,14 @@ class DummyFTPHandler(asynchat.async_chat): self.dtp.push(NLST_DATA) self.dtp.close_when_done() + def cmd_opts(self, arg): + self.push('200 opts ok') + + def cmd_mlsd(self, arg): + self.push('125 mlsd ok') + self.dtp.push(MLSD_DATA) + self.dtp.close_when_done() + class DummyFTPServer(asyncore.dispatcher, threading.Thread): @@ -274,11 +303,11 @@ if ssl is not None: _ssl_closing = False def secure_connection(self): - self.del_channel() socket = ssl.wrap_socket(self.socket, suppress_ragged_eofs=False, certfile=CERTFILE, server_side=True, do_handshake_on_connect=False, ssl_version=ssl.PROTOCOL_SSLv23) + self.del_channel() self.set_socket(socket) self._ssl_accepting = True @@ -313,7 +342,10 @@ if ssl is not None: # http://www.mail-archive.com/openssl-users@openssl.org/msg60710.html pass self._ssl_closing = False - super(SSLConnection, self).close() + if getattr(self, '_ccc', False) == False: + super(SSLConnection, self).close() + else: + pass def handle_read_event(self): if self._ssl_accepting: @@ -381,12 +413,18 @@ if ssl is not None: def __init__(self, conn): DummyFTPHandler.__init__(self, conn) self.secure_data_channel = False + self._ccc = False def cmd_auth(self, line): """Set up secure control channel.""" self.push('234 AUTH TLS successful') self.secure_connection() + def cmd_ccc(self, line): + self.push('220 Reverting back to clear-text') + self._ccc = True + self._do_ssl_shutdown() + def cmd_pbsz(self, line): """Negotiate size of buffer for secure data transfer. For TLS/SSL the only valid value for the parameter is '0'. @@ -416,7 +454,7 @@ class TestFTPClass(TestCase): def setUp(self): self.server = DummyFTPServer((HOST, 0)) self.server.start() - self.client = ftplib.FTP(timeout=10) + self.client = ftplib.FTP(timeout=2) self.client.connect(self.server.host, self.server.port) def tearDown(self): @@ -558,6 +596,64 @@ class TestFTPClass(TestCase): self.client.dir(lambda x: l.append(x)) self.assertEqual(''.join(l), LIST_DATA.replace('\r\n', '')) + def test_mlsd(self): + list(self.client.mlsd()) + list(self.client.mlsd(path='/')) + list(self.client.mlsd(path='/', facts=['size', 'type'])) + + ls = list(self.client.mlsd()) + for name, facts in ls: + self.assertIsInstance(name, str) + self.assertIsInstance(facts, dict) + self.assertTrue(name) + self.assertIn('type', facts) + self.assertIn('perm', facts) + self.assertIn('unique', facts) + + def set_data(data): + self.server.handler_instance.next_data = data + + def test_entry(line, type=None, perm=None, unique=None, name=None): + type = 'type' if type is None else type + perm = 'perm' if perm is None else perm + unique = 'unique' if unique is None else unique + name = 'name' if name is None else name + set_data(line) + _name, facts = next(self.client.mlsd()) + self.assertEqual(_name, name) + self.assertEqual(facts['type'], type) + self.assertEqual(facts['perm'], perm) + self.assertEqual(facts['unique'], unique) + + # plain + test_entry('type=type;perm=perm;unique=unique; name\r\n') + # "=" in fact value + test_entry('type=ty=pe;perm=perm;unique=unique; name\r\n', type="ty=pe") + test_entry('type==type;perm=perm;unique=unique; name\r\n', type="=type") + test_entry('type=t=y=pe;perm=perm;unique=unique; name\r\n', type="t=y=pe") + test_entry('type=====;perm=perm;unique=unique; name\r\n', type="====") + # spaces in name + test_entry('type=type;perm=perm;unique=unique; na me\r\n', name="na me") + test_entry('type=type;perm=perm;unique=unique; name \r\n', name="name ") + test_entry('type=type;perm=perm;unique=unique; name\r\n', name=" name") + test_entry('type=type;perm=perm;unique=unique; n am e\r\n', name="n am e") + # ";" in name + test_entry('type=type;perm=perm;unique=unique; na;me\r\n', name="na;me") + test_entry('type=type;perm=perm;unique=unique; ;name\r\n', name=";name") + test_entry('type=type;perm=perm;unique=unique; ;name;\r\n', name=";name;") + test_entry('type=type;perm=perm;unique=unique; ;;;;\r\n', name=";;;;") + # case sensitiveness + set_data('Type=type;TyPe=perm;UNIQUE=unique; name\r\n') + _name, facts = next(self.client.mlsd()) + for x in facts: + self.assertTrue(x.islower()) + # no data (directory empty) + set_data('') + self.assertRaises(StopIteration, next, self.client.mlsd()) + set_data('') + for x in self.client.mlsd(): + self.fail("unexpected data %s" % data) + def test_makeport(self): with self.client.makeport(): # IPv4 is in use, just make sure send_eprt has not been used @@ -584,7 +680,7 @@ class TestFTPClass(TestCase): return True # base test - with ftplib.FTP(timeout=10) as self.client: + with ftplib.FTP(timeout=2) as self.client: self.client.connect(self.server.host, self.server.port) self.client.sendcmd('noop') self.assertTrue(is_client_connected()) @@ -592,7 +688,7 @@ class TestFTPClass(TestCase): self.assertFalse(is_client_connected()) # QUIT sent inside the with block - with ftplib.FTP(timeout=10) as self.client: + with ftplib.FTP(timeout=2) as self.client: self.client.connect(self.server.host, self.server.port) self.client.sendcmd('noop') self.client.quit() @@ -602,7 +698,7 @@ class TestFTPClass(TestCase): # force a wrong response code to be sent on QUIT: error_perm # is expected and the connection is supposed to be closed try: - with ftplib.FTP(timeout=10) as self.client: + with ftplib.FTP(timeout=2) as self.client: self.client.connect(self.server.host, self.server.port) self.client.sendcmd('noop') self.server.handler_instance.next_response = '550 error on quit' @@ -616,6 +712,30 @@ class TestFTPClass(TestCase): self.assertEqual(self.server.handler_instance.last_received_cmd, 'quit') self.assertFalse(is_client_connected()) + def test_source_address(self): + self.client.quit() + port = support.find_unused_port() + try: + self.client.connect(self.server.host, self.server.port, + source_address=(HOST, port)) + self.assertEqual(self.client.sock.getsockname()[1], port) + self.client.quit() + except IOError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + + def test_source_address_passive_connection(self): + port = support.find_unused_port() + self.client.source_address = (HOST, port) + try: + with self.client.transfercmd('list') as sock: + self.assertEqual(sock.getsockname()[1], port) + except IOError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + def test_parse257(self): self.assertEqual(ftplib.parse257('257 "/foo/bar"'), '/foo/bar') self.assertEqual(ftplib.parse257('257 "/foo/bar" created'), '/foo/bar') @@ -632,7 +752,7 @@ class TestFTPClass(TestCase): class TestIPv6Environment(TestCase): def setUp(self): - self.server = DummyFTPServer((HOST, 0), af=socket.AF_INET6) + self.server = DummyFTPServer(('::1', 0), af=socket.AF_INET6) self.server.start() self.client = ftplib.FTP() self.client.connect(self.server.host, self.server.port) @@ -676,7 +796,7 @@ class TestTLS_FTPClassMixin(TestFTPClass): def setUp(self): self.server = DummyTLS_FTPServer((HOST, 0)) self.server.start() - self.client = ftplib.FTP_TLS(timeout=10) + self.client = ftplib.FTP_TLS(timeout=2) self.client.connect(self.server.host, self.server.port) # enable TLS self.client.auth() @@ -689,7 +809,7 @@ class TestTLS_FTPClass(TestCase): def setUp(self): self.server = DummyTLS_FTPServer((HOST, 0)) self.server.start() - self.client = ftplib.FTP_TLS(timeout=10) + self.client = ftplib.FTP_TLS(timeout=2) self.client.connect(self.server.host, self.server.port) def tearDown(self): @@ -749,7 +869,7 @@ class TestTLS_FTPClass(TestCase): self.assertRaises(ValueError, ftplib.FTP_TLS, certfile=CERTFILE, keyfile=CERTFILE, context=ctx) - self.client = ftplib.FTP_TLS(context=ctx, timeout=10) + self.client = ftplib.FTP_TLS(context=ctx, timeout=2) self.client.connect(self.server.host, self.server.port) self.assertNotIsInstance(self.client.sock, ssl.SSLSocket) self.client.auth() @@ -761,6 +881,13 @@ class TestTLS_FTPClass(TestCase): self.assertIs(sock.context, ctx) self.assertIsInstance(sock, ssl.SSLSocket) + def test_ccc(self): + self.assertRaises(ValueError, self.client.ccc) + self.client.login(secure=True) + self.assertIsInstance(self.client.sock, ssl.SSLSocket) + self.client.ccc() + self.assertRaises(ValueError, self.client.sock.unwrap) + class TestTimeouts(TestCase): @@ -857,13 +984,8 @@ class TestTimeouts(TestCase): def test_main(): tests = [TestFTPClass, TestTimeouts] - if socket.has_ipv6: - try: - DummyFTPServer((HOST, 0), af=socket.AF_INET6) - except socket.error: - pass - else: - tests.append(TestIPv6Environment) + if support.IPV6_ENABLED: + tests.append(TestIPv6Environment) if ssl is not None: tests.extend([TestTLS_FTPClassMixin, TestTLS_FTPClass]) diff --git a/Lib/test/test_functools.py b/Lib/test/test_functools.py index 270cab0..c4d9fe6 100644 --- a/Lib/test/test_functools.py +++ b/Lib/test/test_functools.py @@ -436,19 +436,82 @@ class TestReduce(unittest.TestCase): self.assertEqual(self.func(add, d), "".join(d.keys())) class TestCmpToKey(unittest.TestCase): + def test_cmp_to_key(self): + def cmp1(x, y): + return (x > y) - (x < y) + key = functools.cmp_to_key(cmp1) + self.assertEqual(key(3), key(3)) + self.assertGreater(key(3), key(1)) + def cmp2(x, y): + return int(x) - int(y) + key = functools.cmp_to_key(cmp2) + self.assertEqual(key(4.0), key('4')) + self.assertLess(key(2), key('35')) + + def test_cmp_to_key_arguments(self): + def cmp1(x, y): + return (x > y) - (x < y) + key = functools.cmp_to_key(mycmp=cmp1) + self.assertEqual(key(obj=3), key(obj=3)) + self.assertGreater(key(obj=3), key(obj=1)) + with self.assertRaises((TypeError, AttributeError)): + key(3) > 1 # rhs is not a K object + with self.assertRaises((TypeError, AttributeError)): + 1 < key(3) # lhs is not a K object + with self.assertRaises(TypeError): + key = functools.cmp_to_key() # too few args + with self.assertRaises(TypeError): + key = functools.cmp_to_key(cmp1, None) # too many args + key = functools.cmp_to_key(cmp1) + with self.assertRaises(TypeError): + key() # too few args + with self.assertRaises(TypeError): + key(None, None) # too many args + + def test_bad_cmp(self): + def cmp1(x, y): + raise ZeroDivisionError + key = functools.cmp_to_key(cmp1) + with self.assertRaises(ZeroDivisionError): + key(3) > key(1) + + class BadCmp: + def __lt__(self, other): + raise ZeroDivisionError + def cmp1(x, y): + return BadCmp() + with self.assertRaises(ZeroDivisionError): + key(3) > key(1) + + def test_obj_field(self): + def cmp1(x, y): + return (x > y) - (x < y) + key = functools.cmp_to_key(mycmp=cmp1) + self.assertEqual(key(50).obj, 50) + + def test_sort_int(self): def mycmp(x, y): return y - x self.assertEqual(sorted(range(5), key=functools.cmp_to_key(mycmp)), [4, 3, 2, 1, 0]) + def test_sort_int_str(self): + def mycmp(x, y): + x, y = int(x), int(y) + return (x > y) - (x < y) + values = [5, '3', 7, 2, '0', '1', 4, '10', 1] + values = sorted(values, key=functools.cmp_to_key(mycmp)) + self.assertEqual([int(value) for value in values], + [0, 1, 1, 2, 3, 4, 5, 7, 10]) + def test_hash(self): def mycmp(x, y): return y - x key = functools.cmp_to_key(mycmp) k = key(10) self.assertRaises(TypeError, hash, k) - self.assertFalse(isinstance(k, collections.Hashable)) + self.assertNotIsInstance(k, collections.Hashable) class TestTotalOrdering(unittest.TestCase): @@ -671,6 +734,22 @@ class TestLRU(unittest.TestCase): with self.assertRaises(IndexError): func(15) + def test_lru_with_types(self): + for maxsize in (None, 100): + @functools.lru_cache(maxsize=maxsize, typed=True) + def square(x): + return x * x + self.assertEqual(square(3), 9) + self.assertEqual(type(square(3)), type(9)) + self.assertEqual(square(3.0), 9.0) + self.assertEqual(type(square(3.0)), type(9.0)) + self.assertEqual(square(x=3), 9) + self.assertEqual(type(square(x=3)), type(9)) + self.assertEqual(square(x=3.0), 9.0) + self.assertEqual(type(square(x=3.0)), type(9.0)) + self.assertEqual(square.cache_info().hits, 4) + self.assertEqual(square.cache_info().misses, 4) + def test_main(verbose=None): test_classes = ( TestPartial, diff --git a/Lib/test/test_future.py b/Lib/test/test_future.py index c6689a1..3a25eb1 100644 --- a/Lib/test/test_future.py +++ b/Lib/test/test_future.py @@ -13,14 +13,14 @@ def get_error_location(msg): class FutureTest(unittest.TestCase): def test_future1(self): - support.unload('test_future1') - from test import test_future1 - self.assertEqual(test_future1.result, 6) + support.unload('future_test1') + from test import future_test1 + self.assertEqual(future_test1.result, 6) def test_future2(self): - support.unload('test_future2') - from test import test_future2 - self.assertEqual(test_future2.result, 6) + support.unload('future_test2') + from test import future_test2 + self.assertEqual(future_test2.result, 6) def test_future3(self): support.unload('test_future3') diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 100c767..19313db 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1,5 +1,6 @@ import unittest -from test.support import verbose, run_unittest, strip_python_stderr +from test.support import (verbose, refcount_test, run_unittest, + strip_python_stderr) import sys import gc import weakref @@ -175,6 +176,7 @@ class GCTests(unittest.TestCase): del d self.assertEqual(gc.collect(), 2) + @refcount_test def test_frame(self): def f(): frame = sys._getframe() @@ -242,6 +244,7 @@ class GCTests(unittest.TestCase): # For example, disposed tuples are not freed, but reused. # To minimize variations, though, we first store the get_count() results # and check them at the end. + @refcount_test def test_get_count(self): gc.collect() a, b, c = gc.get_count() @@ -255,6 +258,7 @@ class GCTests(unittest.TestCase): # created (the list). self.assertGreater(d, a) + @refcount_test def test_collect_generations(self): gc.collect() # This object will "trickle" into generation N + 1 after diff --git a/Lib/test/test_genericpath.py b/Lib/test/test_genericpath.py index 50638a1..ebb8396 100644 --- a/Lib/test/test_genericpath.py +++ b/Lib/test/test_genericpath.py @@ -2,11 +2,12 @@ Tests common to genericpath, macpath, ntpath and posixpath """ -import unittest -from test import support -import os import genericpath +import os import sys +import unittest +import warnings +from test import support def safe_rmdir(dirname): @@ -258,15 +259,21 @@ class CommonTest(GenericTest): def test_abspath(self): self.assertIn("foo", self.pathmodule.abspath("foo")) - self.assertIn(b"foo", self.pathmodule.abspath(b"foo")) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.assertIn(b"foo", self.pathmodule.abspath(b"foo")) # Abspath returns bytes when the arg is bytes - for path in (b'', b'foo', b'f\xf2\xf2', b'/foo', b'C:\\'): - self.assertIsInstance(self.pathmodule.abspath(path), bytes) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + for path in (b'', b'foo', b'f\xf2\xf2', b'/foo', b'C:\\'): + self.assertIsInstance(self.pathmodule.abspath(path), bytes) def test_realpath(self): self.assertIn("foo", self.pathmodule.realpath("foo")) - self.assertIn(b"foo", self.pathmodule.realpath(b"foo")) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.assertIn(b"foo", self.pathmodule.realpath(b"foo")) def test_normpath_issue5827(self): # Make sure normpath preserves unicode @@ -296,8 +303,10 @@ class CommonTest(GenericTest): "Mac OS X denies the creation of a directory with an invalid utf8 name") def test_nonascii_abspath(self): # Test non-ASCII, non-UTF8 bytes in the path. - with support.temp_cwd(b'\xe7w\xf0'): - self.test_abspath() + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with support.temp_cwd(b'\xe7w\xf0'): + self.test_abspath() def test_main(): diff --git a/Lib/test/test_genexps.py b/Lib/test/test_genexps.py index 1f46af1..413043c 100644 --- a/Lib/test/test_genexps.py +++ b/Lib/test/test_genexps.py @@ -257,11 +257,15 @@ Verify that genexps are weakly referencable """ +import sys -__test__ = {'doctests' : doctests} +# Trace function can throw off the tuple reuse test. +if hasattr(sys, 'gettrace') and sys.gettrace(): + __test__ = {} +else: + __test__ = {'doctests' : doctests} def test_main(verbose=None): - import sys from test import support from test import test_genexps support.run_doctest(test_genexps, verbose) diff --git a/Lib/test/test_getargs2.py b/Lib/test/test_getargs2.py index 3d9c06a..768ea8d 100644 --- a/Lib/test/test_getargs2.py +++ b/Lib/test/test_getargs2.py @@ -294,6 +294,15 @@ class Keywords_TestCase(unittest.TestCase): self.fail('TypeError should have been raised') class Bytes_TestCase(unittest.TestCase): + def test_c(self): + from _testcapi import getargs_c + self.assertRaises(TypeError, getargs_c, b'abc') # len > 1 + self.assertEqual(getargs_c(b'a'), b'a') + self.assertEqual(getargs_c(bytearray(b'a')), b'a') + self.assertRaises(TypeError, getargs_c, memoryview(b'a')) + self.assertRaises(TypeError, getargs_c, 's') + self.assertRaises(TypeError, getargs_c, None) + def test_s(self): from _testcapi import getargs_s self.assertEqual(getargs_s('abc\xe9'), b'abc\xc3\xa9') diff --git a/Lib/test/test_glob.py b/Lib/test/test_glob.py index 1560a6b..6ee08db 100644 --- a/Lib/test/test_glob.py +++ b/Lib/test/test_glob.py @@ -1,5 +1,6 @@ import unittest -from test.support import run_unittest, TESTFN, skip_unless_symlink, can_symlink +from test.support import (run_unittest, TESTFN, skip_unless_symlink, + can_symlink, create_empty_file) import glob import os import shutil @@ -14,8 +15,7 @@ class GlobTests(unittest.TestCase): base, file = os.path.split(filename) if not os.path.exists(base): os.makedirs(base) - f = open(filename, 'w') - f.close() + create_empty_file(filename) def setUp(self): self.tempdir = TESTFN+"_dir" diff --git a/Lib/test/test_grammar.py b/Lib/test/test_grammar.py index 329b258..32dc15e 100644 --- a/Lib/test/test_grammar.py +++ b/Lib/test/test_grammar.py @@ -10,7 +10,7 @@ from sys import * class TokenTests(unittest.TestCase): - def testBackslash(self): + def test_backslash(self): # Backslash means line continuation: x = 1 \ + 1 @@ -20,7 +20,7 @@ class TokenTests(unittest.TestCase): x = 0 self.assertEqual(x, 0, 'backslash ending comment') - def testPlainIntegers(self): + def test_plain_integers(self): self.assertEqual(type(000), type(0)) self.assertEqual(0xff, 255) self.assertEqual(0o377, 255) @@ -56,7 +56,7 @@ class TokenTests(unittest.TestCase): else: self.fail('Weird maxsize value %r' % maxsize) - def testLongIntegers(self): + def test_long_integers(self): x = 0 x = 0xffffffffffffffff x = 0Xffffffffffffffff @@ -66,7 +66,7 @@ class TokenTests(unittest.TestCase): x = 0b100000000000000000000000000000000000000000000000000000000000000000000 x = 0B111111111111111111111111111111111111111111111111111111111111111111111 - def testFloats(self): + def test_floats(self): x = 3.14 x = 314. x = 0.314 @@ -80,7 +80,7 @@ class TokenTests(unittest.TestCase): x = .3e14 x = 3.1e4 - def testStringLiterals(self): + def test_string_literals(self): x = ''; y = ""; self.assertTrue(len(x) == 0 and x == y) x = '\''; y = "'"; self.assertTrue(len(x) == 1 and x == y and ord(x) == 39) x = '"'; y = "\""; self.assertTrue(len(x) == 1 and x == y and ord(x) == 34) @@ -120,11 +120,18 @@ the \'lazy\' dog.\n\ ' self.assertEqual(x, y) - def testEllipsis(self): + def test_ellipsis(self): x = ... self.assertTrue(x is Ellipsis) self.assertRaises(SyntaxError, eval, ".. .") + def test_eof_error(self): + samples = ("def foo(", "\ndef foo(", "def foo(\n") + for s in samples: + with self.assertRaises(SyntaxError) as cm: + compile(s, "<test>", "exec") + self.assertIn("unexpected EOF", str(cm.exception)) + class GrammarTests(unittest.TestCase): # single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE @@ -136,11 +143,11 @@ class GrammarTests(unittest.TestCase): # expr_input: testlist NEWLINE # XXX Hard to test -- used only in calls to input() - def testEvalInput(self): + def test_eval_input(self): # testlist ENDMARKER x = eval('1, 0 or 1') - def testFuncdef(self): + def test_funcdef(self): ### [decorators] 'def' NAME parameters ['->' test] ':' suite ### decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE ### decorators: decorator+ @@ -324,7 +331,7 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "f(*g(1=2))") check_syntax_error(self, "f(**g(1=2))") - def testLambdef(self): + def test_lambdef(self): ### lambdef: 'lambda' [varargslist] ':' test l1 = lambda : 0 self.assertEqual(l1(), 0) @@ -346,7 +353,7 @@ class GrammarTests(unittest.TestCase): ### stmt: simple_stmt | compound_stmt # Tested below - def testSimpleStmt(self): + def test_simple_stmt(self): ### simple_stmt: small_stmt (';' small_stmt)* [';'] x = 1; pass; del x def foo(): @@ -357,7 +364,7 @@ class GrammarTests(unittest.TestCase): ### small_stmt: expr_stmt | pass_stmt | del_stmt | flow_stmt | import_stmt | global_stmt | access_stmt # Tested below - def testExprStmt(self): + def test_expr_stmt(self): # (exprlist '=')* exprlist 1 1, 2, 3 @@ -370,7 +377,7 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "x + 1 = 1") check_syntax_error(self, "a + 1 = b + 2") - def testDelStmt(self): + def test_del_stmt(self): # 'del' exprlist abc = [1,2,3] x, y, z = abc @@ -379,18 +386,18 @@ class GrammarTests(unittest.TestCase): del abc del x, y, (z, xyz) - def testPassStmt(self): + def test_pass_stmt(self): # 'pass' pass # flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt # Tested below - def testBreakStmt(self): + def test_break_stmt(self): # 'break' while 1: break - def testContinueStmt(self): + def test_continue_stmt(self): # 'continue' i = 1 while i: i = 0; continue @@ -442,7 +449,7 @@ class GrammarTests(unittest.TestCase): self.fail("continue then break in try/except in loop broken!") test_inner() - def testReturn(self): + def test_return(self): # 'return' [testlist] def g1(): return def g2(): return 1 @@ -450,17 +457,17 @@ class GrammarTests(unittest.TestCase): x = g2() check_syntax_error(self, "class foo:return 1") - def testYield(self): + def test_yield(self): check_syntax_error(self, "class foo:yield 1") - def testRaise(self): + def test_raise(self): # 'raise' test [',' test] try: raise RuntimeError('just testing') except RuntimeError: pass try: raise KeyboardInterrupt except KeyboardInterrupt: pass - def testImport(self): + def test_import(self): # 'import' dotted_as_names import sys import time, sys @@ -473,13 +480,13 @@ class GrammarTests(unittest.TestCase): from sys import (path, argv) from sys import (path, argv,) - def testGlobal(self): + def test_global(self): # 'global' NAME (',' NAME)* global a global a, b global one, two, three, four, five, six, seven, eight, nine, ten - def testNonlocal(self): + def test_nonlocal(self): # 'nonlocal' NAME (',' NAME)* x = 0 y = 0 @@ -487,7 +494,7 @@ class GrammarTests(unittest.TestCase): nonlocal x nonlocal x, y - def testAssert(self): + def test_assert(self): # assertTruestmt: 'assert' test [',' test] assert 1 assert 1, 1 @@ -504,7 +511,7 @@ class GrammarTests(unittest.TestCase): ### compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | funcdef | classdef # Tested below - def testIf(self): + def test_if(self): # 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] if 1: pass if 1: pass @@ -517,7 +524,7 @@ class GrammarTests(unittest.TestCase): elif 0: pass else: pass - def testWhile(self): + def test_while(self): # 'while' test ':' suite ['else' ':' suite] while 0: pass while 0: pass @@ -532,7 +539,7 @@ class GrammarTests(unittest.TestCase): x = 2 self.assertEqual(x, 2) - def testFor(self): + def test_for(self): # 'for' exprlist 'in' exprlist ':' suite ['else' ':' suite] for i in 1, 2, 3: pass for i, j, k in (): pass @@ -559,7 +566,7 @@ class GrammarTests(unittest.TestCase): result.append(x) self.assertEqual(result, [1, 2, 3]) - def testTry(self): + def test_try(self): ### try_stmt: 'try' ':' suite (except_clause ':' suite)+ ['else' ':' suite] ### | 'try' ':' suite 'finally' ':' suite ### except_clause: 'except' [expr ['as' expr]] @@ -582,7 +589,7 @@ class GrammarTests(unittest.TestCase): try: pass finally: pass - def testSuite(self): + def test_suite(self): # simple_stmt | NEWLINE INDENT NEWLINE* (stmt NEWLINE*)+ DEDENT if 1: pass if 1: @@ -597,7 +604,7 @@ class GrammarTests(unittest.TestCase): pass # - def testTest(self): + def test_test(self): ### and_test ('or' and_test)* ### and_test: not_test ('and' not_test)* ### not_test: 'not' not_test | comparison @@ -608,7 +615,7 @@ class GrammarTests(unittest.TestCase): if not 1 and 1 and 1: pass if 1 and 1 or 1 and 1 and 1 or not 1 and 1: pass - def testComparison(self): + def test_comparison(self): ### comparison: expr (comp_op expr)* ### comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not' if 1: pass @@ -625,36 +632,36 @@ class GrammarTests(unittest.TestCase): if 1 not in (): pass if 1 < 1 > 1 == 1 >= 1 <= 1 != 1 in 1 not in 1 is 1 is not 1: pass - def testBinaryMaskOps(self): + def test_binary_mask_ops(self): x = 1 & 1 x = 1 ^ 1 x = 1 | 1 - def testShiftOps(self): + def test_shift_ops(self): x = 1 << 1 x = 1 >> 1 x = 1 << 1 >> 1 - def testAdditiveOps(self): + def test_additive_ops(self): x = 1 x = 1 + 1 x = 1 - 1 - 1 x = 1 - 1 + 1 - 1 + 1 - def testMultiplicativeOps(self): + def test_multiplicative_ops(self): x = 1 * 1 x = 1 / 1 x = 1 % 1 x = 1 / 1 * 1 % 1 - def testUnaryOps(self): + def test_unary_ops(self): x = +1 x = -1 x = ~1 x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 x = -1*1/1 + 1*1 - ---1*1 - def testSelectors(self): + def test_selectors(self): ### trailer: '(' [testlist] ')' | '[' subscript ']' | '.' NAME ### subscript: expr | [expr] ':' [expr] @@ -684,7 +691,7 @@ class GrammarTests(unittest.TestCase): L.sort(key=lambda x: x if isinstance(x, tuple) else ()) self.assertEqual(str(L), '[1, (1,), (1, 2), (1, 2, 3)]') - def testAtoms(self): + def test_atoms(self): ### atom: '(' [testlist] ')' | '[' [testlist] ']' | '{' [dictsetmaker] '}' | NAME | NUMBER | STRING ### dictsetmaker: (test ':' test (',' test ':' test)* [',']) | (test (',' test)* [',']) @@ -719,7 +726,7 @@ class GrammarTests(unittest.TestCase): ### testlist: test (',' test)* [','] # These have been exercised enough above - def testClassdef(self): + def test_classdef(self): # 'class' NAME ['(' [testlist] ')'] ':' suite class B: pass class B2(): pass @@ -738,14 +745,14 @@ class GrammarTests(unittest.TestCase): @class_decorator class G: pass - def testDictcomps(self): + def test_dictcomps(self): # dictorsetmaker: ( (test ':' test (comp_for | # (',' test ':' test)* [','])) | # (test (comp_for | (',' test)* [','])) ) nums = [1, 2, 3] self.assertEqual({i:i+1 for i in nums}, {1: 2, 2: 3, 3: 4}) - def testListcomps(self): + def test_listcomps(self): # list comprehension tests nums = [1, 2, 3, 4, 5] strs = ["Apple", "Banana", "Coconut"] @@ -808,7 +815,7 @@ class GrammarTests(unittest.TestCase): self.assertEqual(x, [('Boeing', 'Airliner'), ('Boeing', 'Engine'), ('Ford', 'Engine'), ('Macdonalds', 'Cheeseburger')]) - def testGenexps(self): + def test_genexps(self): # generator expression tests g = ([x for x in range(10)] for x in range(1)) self.assertEqual(next(g), [x for x in range(10)]) @@ -843,7 +850,7 @@ class GrammarTests(unittest.TestCase): check_syntax_error(self, "foo(x for x in range(10), 100)") check_syntax_error(self, "foo(100, x for x in range(10))") - def testComprehensionSpecials(self): + def test_comprehension_specials(self): # test for outmost iterable precomputation x = 10; g = (i for i in range(x)); x = 5 self.assertEqual(len(list(g)), 10) @@ -882,7 +889,7 @@ class GrammarTests(unittest.TestCase): with manager() as x, manager(): pass - def testIfElseExpr(self): + def test_if_else_expr(self): # Test ifelse expressions in various cases def _checkeval(msg, ret): "helper to check that evaluation of expressions is done correctly" diff --git a/Lib/test/test_gzip.py b/Lib/test/test_gzip.py index 2b0ac36..9c7a96e 100644 --- a/Lib/test/test_gzip.py +++ b/Lib/test/test_gzip.py @@ -64,6 +64,21 @@ class TestGzip(unittest.TestCase): d = f.read() self.assertEqual(d, data1*50) + def test_read1(self): + self.test_write() + blocks = [] + nread = 0 + with gzip.GzipFile(self.filename, 'r') as f: + while True: + d = f.read1() + if not d: + break + blocks.append(d) + nread += len(d) + # Check that position was updated correctly (see issue10791). + self.assertEqual(f.tell(), nread) + self.assertEqual(b''.join(blocks), data1 * 50) + def test_io_on_closed_object(self): # Test that I/O operations on closed GzipFile objects raise a # ValueError, just like the corresponding functions on file objects. @@ -124,7 +139,7 @@ class TestGzip(unittest.TestCase): with io.BufferedReader(f) as r: lines = [line for line in r] - self.assertEqual(lines, 50 * data1.splitlines(True)) + self.assertEqual(lines, 50 * data1.splitlines(keepends=True)) def test_readline(self): self.test_write() @@ -323,6 +338,14 @@ class TestGzip(unittest.TestCase): self.assertEqual(f.read(100), b'') self.assertEqual(nread, len(uncompressed)) + def test_textio_readlines(self): + # Issue #10791: TextIOWrapper.readlines() fails when wrapping GzipFile. + lines = (data1 * 50).decode("ascii").splitlines(keepends=True) + self.test_write() + with gzip.GzipFile(self.filename, 'r') as f: + with io.TextIOWrapper(f, encoding="ascii") as t: + self.assertEqual(t.readlines(), lines) + # Testing compress/decompress shortcut functions def test_compress(self): diff --git a/Lib/test/test_hash.py b/Lib/test/test_hash.py index fea1025..779e485 100644 --- a/Lib/test/test_hash.py +++ b/Lib/test/test_hash.py @@ -107,8 +107,7 @@ class DefaultIterSeq(object): return self.seq[index] class HashBuiltinsTestCase(unittest.TestCase): - hashes_to_check = [range(10), - enumerate(range(10)), + hashes_to_check = [enumerate(range(10)), iter(DefaultIterSeq()), iter(lambda: 0, 0), ] diff --git a/Lib/test/test_http_cookiejar.py b/Lib/test/test_http_cookiejar.py index 41e0dfd..a35ec95 100644 --- a/Lib/test/test_http_cookiejar.py +++ b/Lib/test/test_http_cookiejar.py @@ -248,18 +248,19 @@ class FileCookieJarTests(unittest.TestCase): self.assertEqual(c._cookies["www.acme.com"]["/"]["boo"].value, None) def test_bad_magic(self): - # IOErrors (eg. file doesn't exist) are allowed to propagate + # OSErrors (eg. file doesn't exist) are allowed to propagate filename = test.support.TESTFN for cookiejar_class in LWPCookieJar, MozillaCookieJar: c = cookiejar_class() try: c.load(filename="for this test to work, a file with this " "filename should not exist") - except IOError as exc: - # exactly IOError, not LoadError - self.assertIs(exc.__class__, IOError) + except OSError as exc: + # an OSError subclass (likely FileNotFoundError), but not + # LoadError + self.assertIsNot(exc.__class__, LoadError) else: - self.fail("expected IOError for invalid filename") + self.fail("expected OSError for invalid filename") # Invalid contents of cookies file (eg. bad magic string) # causes a LoadError. try: diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index ff03321..a10c09d 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -508,8 +508,7 @@ class HTTPSTest(TestCase): def test_local_good_hostname(self): # The (valid) cert validates the HTTP hostname import ssl - from test.ssl_servers import make_https_server - server = make_https_server(self, CERT_localhost) + server = self.make_server(CERT_localhost) context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.verify_mode = ssl.CERT_REQUIRED context.load_verify_locations(CERT_localhost) @@ -517,12 +516,12 @@ class HTTPSTest(TestCase): h.request('GET', '/nonexistent') resp = h.getresponse() self.assertEqual(resp.status, 404) + del server def test_local_bad_hostname(self): # The (valid) cert doesn't validate the HTTP hostname import ssl - from test.ssl_servers import make_https_server - server = make_https_server(self, CERT_fakehostname) + server = self.make_server(CERT_fakehostname) context = ssl.SSLContext(ssl.PROTOCOL_TLSv1) context.verify_mode = ssl.CERT_REQUIRED context.load_verify_locations(CERT_fakehostname) @@ -540,6 +539,7 @@ class HTTPSTest(TestCase): h.request('GET', '/nonexistent') resp = h.getresponse() self.assertEqual(resp.status, 404) + del server @unittest.skipIf(not hasattr(client, 'HTTPSConnection'), 'http.client.HTTPSConnection not available') diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py index e83c048..cc15dd6 100644 --- a/Lib/test/test_httpservers.py +++ b/Lib/test/test_httpservers.py @@ -465,6 +465,23 @@ class RejectingSocketlessRequestHandler(SocketlessRequestHandler): self.send_error(417) return False + +class AuditableBytesIO: + + def __init__(self): + self.datas = [] + + def write(self, data): + self.datas.append(data) + + def getData(self): + return b''.join(self.datas) + + @property + def numWrites(self): + return len(self.datas) + + class BaseHTTPRequestHandlerTestCase(unittest.TestCase): """Test the functionality of the BaseHTTPServer. @@ -531,27 +548,49 @@ class BaseHTTPRequestHandlerTestCase(unittest.TestCase): self.verify_get_called() self.assertEqual(result[-1], b'<html><body>Data</body></html>\r\n') - def test_header_buffering(self): + def test_header_buffering_of_send_error(self): - def _readAndReseek(f): - pos = f.tell() - f.seek(0) - data = f.read() - f.seek(pos) - return data + input = BytesIO(b'GET / HTTP/1.1\r\n\r\n') + output = AuditableBytesIO() + handler = SocketlessRequestHandler() + handler.rfile = input + handler.wfile = output + handler.request_version = 'HTTP/1.1' + handler.requestline = '' + handler.command = None + + handler.send_error(418) + self.assertEqual(output.numWrites, 2) + + def test_header_buffering_of_send_response_only(self): input = BytesIO(b'GET / HTTP/1.1\r\n\r\n') - output = BytesIO() - self.handler.rfile = input - self.handler.wfile = output - self.handler.request_version = 'HTTP/1.1' + output = AuditableBytesIO() + handler = SocketlessRequestHandler() + handler.rfile = input + handler.wfile = output + handler.request_version = 'HTTP/1.1' - self.handler.send_header('Foo', 'foo') - self.handler.send_header('bar', 'bar') - self.assertEqual(_readAndReseek(output), b'') - self.handler.end_headers() - self.assertEqual(_readAndReseek(output), - b'Foo: foo\r\nbar: bar\r\n\r\n') + handler.send_response_only(418) + self.assertEqual(output.numWrites, 0) + handler.end_headers() + self.assertEqual(output.numWrites, 1) + + def test_header_buffering_of_send_header(self): + + input = BytesIO(b'GET / HTTP/1.1\r\n\r\n') + output = AuditableBytesIO() + handler = SocketlessRequestHandler() + handler.rfile = input + handler.wfile = output + handler.request_version = 'HTTP/1.1' + + handler.send_header('Foo', 'foo') + handler.send_header('bar', 'bar') + self.assertEqual(output.numWrites, 0) + handler.end_headers() + self.assertEqual(output.getData(), b'Foo: foo\r\nbar: bar\r\n\r\n') + self.assertEqual(output.numWrites, 1) def test_header_unbuffered_when_continue(self): diff --git a/Lib/test/test_imaplib.py b/Lib/test/test_imaplib.py index 8034000..c4c7ecc 100644 --- a/Lib/test/test_imaplib.py +++ b/Lib/test/test_imaplib.py @@ -258,11 +258,58 @@ class RemoteIMAP_SSLTest(RemoteIMAPTest): port = 993 imap_class = IMAP4_SSL + def setUp(self): + pass + + def tearDown(self): + pass + + def create_ssl_context(self): + ssl_context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + ssl_context.load_cert_chain(CERTFILE) + return ssl_context + + def check_logincapa(self, server): + try: + for cap in server.capabilities: + self.assertIsInstance(cap, str) + self.assertFalse('LOGINDISABLED' in server.capabilities) + self.assertTrue('AUTH=PLAIN' in server.capabilities) + rs = server.login(self.username, self.password) + self.assertEqual(rs[0], 'OK') + finally: + server.logout() + def test_logincapa(self): - for cap in self.server.capabilities: - self.assertIsInstance(cap, str) - self.assertFalse('LOGINDISABLED' in self.server.capabilities) - self.assertTrue('AUTH=PLAIN' in self.server.capabilities) + with transient_internet(self.host): + _server = self.imap_class(self.host, self.port) + self.check_logincapa(_server) + + def test_logincapa_with_client_certfile(self): + with transient_internet(self.host): + _server = self.imap_class(self.host, self.port, certfile=CERTFILE) + self.check_logincapa(_server) + + def test_logincapa_with_client_ssl_context(self): + with transient_internet(self.host): + _server = self.imap_class(self.host, self.port, ssl_context=self.create_ssl_context()) + self.check_logincapa(_server) + + def test_logout(self): + with transient_internet(self.host): + _server = self.imap_class(self.host, self.port) + rs = _server.logout() + self.assertEqual(rs[0], 'BYE') + + def test_ssl_context_certfile_exclusive(self): + with transient_internet(self.host): + self.assertRaises(ValueError, self.imap_class, self.host, self.port, + certfile=CERTFILE, ssl_context=self.create_ssl_context()) + + def test_ssl_context_keyfile_exclusive(self): + with transient_internet(self.host): + self.assertRaises(ValueError, self.imap_class, self.host, self.port, + keyfile=CERTFILE, ssl_context=self.create_ssl_context()) def test_main(): diff --git a/Lib/test/test_imp.py b/Lib/test/test_imp.py index 551ad1b..3041218 100644 --- a/Lib/test/test_imp.py +++ b/Lib/test/test_imp.py @@ -58,6 +58,12 @@ class ImportTests(unittest.TestCase): with imp.find_module('module_' + mod, self.test_path)[0] as fd: self.assertEqual(fd.encoding, encoding) + path = [os.path.dirname(__file__)] + self.assertRaisesRegex(SyntaxError, + r"Non-UTF-8 code starting with '\\xf6'" + r" in file .*badsyntax_pep3120.py", + imp.find_module, 'badsyntax_pep3120', path) + def test_issue1267(self): for mod, encoding, _ in self.test_strings: fp, filename, info = imp.find_module('module_' + mod, @@ -215,6 +221,10 @@ class PEP3147Tests(unittest.TestCase): self.assertEqual( imp.cache_from_source('/foo/bar/baz/qux.py', True), '/foo/bar/baz/__pycache__/qux.{}.pyc'.format(self.tag)) + # Directory with a dot, filename without dot + self.assertEqual( + imp.cache_from_source('/foo.bar/file', True), + '/foo.bar/__pycache__/file{}.pyc'.format(self.tag)) def test_cache_from_source_optimized(self): # Given the path to a .py file, return the path to its PEP 3147 @@ -314,8 +324,7 @@ class PEP3147Tests(unittest.TestCase): shutil.rmtree('pep3147') self.addCleanup(cleanup) # Touch the __init__.py file. - with open('pep3147/__init__.py', 'w'): - pass + support.create_empty_file('pep3147/__init__.py') m = __import__('pep3147') # Ensure we load the pyc file. support.forget('pep3147') diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py index 63b8776..86ef40e 100644 --- a/Lib/test/test_import.py +++ b/Lib/test/test_import.py @@ -15,7 +15,7 @@ import textwrap from test.support import ( EnvironmentVarGuard, TESTFN, check_warnings, forget, is_jython, make_legacy_pyc, rmtree, run_unittest, swap_attr, swap_item, temp_umask, - unlink, unload) + unlink, unload, create_empty_file) from test import script_helper @@ -97,24 +97,22 @@ class ImportTests(unittest.TestCase): @unittest.skipUnless(os.name == 'posix', "test meaningful only on posix systems") - def test_execute_bit_not_copied(self): - # Issue 6070: under posix .pyc files got their execute bit set if - # the .py file had the execute bit set, but they aren't executable. - with temp_umask(0o022): + def test_creation_mode(self): + mask = 0o022 + with temp_umask(mask): sys.path.insert(0, os.curdir) try: fname = TESTFN + os.extsep + "py" - open(fname, 'w').close() - os.chmod(fname, (stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH | - stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH)) + create_empty_file(fname) __import__(TESTFN) fn = imp.cache_from_source(fname) if not os.path.exists(fn): self.fail("__import__ did not result in creation of " "either a .pyc or .pyo file") s = os.stat(fn) - self.assertEqual(stat.S_IMODE(s.st_mode), - stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH) + # Check that the umask is respected, and the executable bits + # aren't set. + self.assertEqual(stat.S_IMODE(s.st_mode), 0o666 & ~mask) finally: del sys.path[0] remove_files(TESTFN) @@ -295,8 +293,6 @@ class ImportTests(unittest.TestCase): self.skipTest('path is not encodable to {}'.format(encoding)) with self.assertRaises(ImportError) as c: __import__(path) - self.assertEqual("Import by filename is not supported.", - c.exception.args[0]) def test_import_in_del_does_not_crash(self): # Issue 4236 diff --git a/Lib/test/test_importhooks.py b/Lib/test/test_importhooks.py index ec6730e..7a25657 100644 --- a/Lib/test/test_importhooks.py +++ b/Lib/test/test_importhooks.py @@ -51,7 +51,7 @@ class TestImporter: def __init__(self, path=test_path): if path != test_path: - # if out class is on sys.path_hooks, we must raise + # if our class is on sys.path_hooks, we must raise # ImportError for any path item that we can't handle. raise ImportError self.path = path @@ -229,7 +229,9 @@ class ImportHooksTestCase(ImportHooksBaseTestCase): i = ImpWrapper() sys.meta_path.append(i) sys.path_hooks.append(ImpWrapper) - mnames = ("colorsys", "urllib.parse", "distutils.core") + mnames = ( + "colorsys", "urllib.parse", "distutils.core", "sys", + ) for mname in mnames: parent = mname.split(".")[0] for n in list(sys.modules): @@ -237,7 +239,8 @@ class ImportHooksTestCase(ImportHooksBaseTestCase): del sys.modules[n] for mname in mnames: m = __import__(mname, globals(), locals(), ["__dummy__"]) - m.__loader__ # to make sure we actually handled the import + # to make sure we actually handled the import + self.assertTrue(hasattr(m, "__loader__")) def test_main(): diff --git a/Lib/test/test_inspect.py b/Lib/test/test_inspect.py index 7666fe4..06132f2 100644 --- a/Lib/test/test_inspect.py +++ b/Lib/test/test_inspect.py @@ -304,7 +304,7 @@ class TestRetrievingSourceCode(GetSourceBase): getlines = linecache.getlines def monkey(filename, module_globals=None): if filename == fn: - return source.splitlines(True) + return source.splitlines(keepends=True) else: return getlines(filename, module_globals) linecache.getlines = monkey diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index df7ca15..5954999 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -49,7 +49,7 @@ except ImportError: def _default_chunk_size(): """Get the default TextIOWrapper chunk size""" - with open(__file__, "r", encoding="latin1") as f: + with open(__file__, "r", encoding="latin-1") as f: return f._CHUNK_SIZE @@ -629,6 +629,15 @@ class IOTest(unittest.TestCase): for obj in test: self.assertTrue(hasattr(obj, "__dict__")) + def test_opener(self): + with self.open(support.TESTFN, "w") as f: + f.write("egg\n") + fd = os.open(support.TESTFN, os.O_RDONLY) + def opener(path, flags): + return fd + with self.open("non-existent", "r", opener=opener) as f: + self.assertEqual(f.read(), "egg\n") + class CIOTest(IOTest): def test_IOBase_finalize(self): @@ -830,6 +839,12 @@ class BufferedReaderTest(unittest.TestCase, CommonBufferedTests): self.assertEqual(b, b"gf") self.assertEqual(bufio.readinto(b), 0) self.assertEqual(b, b"gf") + rawio = self.MockRawIO((b"abc", None)) + bufio = self.tp(rawio) + self.assertEqual(bufio.readinto(b), 2) + self.assertEqual(b, b"ab") + self.assertEqual(bufio.readinto(b), 1) + self.assertEqual(b, b"cb") def test_readlines(self): def bufio(): @@ -1797,11 +1812,11 @@ class TextIOWrapperTest(unittest.TestCase): r = self.BytesIO(b"\xc3\xa9\n\n") b = self.BufferedReader(r, 1000) t = self.TextIOWrapper(b) - t.__init__(b, encoding="latin1", newline="\r\n") - self.assertEqual(t.encoding, "latin1") + t.__init__(b, encoding="latin-1", newline="\r\n") + self.assertEqual(t.encoding, "latin-1") self.assertEqual(t.line_buffering, False) - t.__init__(b, encoding="utf8", line_buffering=True) - self.assertEqual(t.encoding, "utf8") + t.__init__(b, encoding="utf-8", line_buffering=True) + self.assertEqual(t.encoding, "utf-8") self.assertEqual(t.line_buffering, True) self.assertEqual("\xe9\n", t.readline()) self.assertRaises(TypeError, t.__init__, b, newline=42) @@ -1851,8 +1866,8 @@ class TextIOWrapperTest(unittest.TestCase): def test_encoding(self): # Check the encoding attribute is always set, and valid b = self.BytesIO() - t = self.TextIOWrapper(b, encoding="utf8") - self.assertEqual(t.encoding, "utf8") + t = self.TextIOWrapper(b, encoding="utf-8") + self.assertEqual(t.encoding, "utf-8") t = self.TextIOWrapper(b) self.assertTrue(t.encoding is not None) codecs.lookup(t.encoding) @@ -1945,8 +1960,8 @@ class TextIOWrapperTest(unittest.TestCase): testdata = b"AAA\nBB\x00B\nCCC\rDDD\rEEE\r\nFFF\r\nGGG" normalized = testdata.replace(b"\r\n", b"\n").replace(b"\r", b"\n") for newline, expected in [ - (None, normalized.decode("ascii").splitlines(True)), - ("", testdata.decode("ascii").splitlines(True)), + (None, normalized.decode("ascii").splitlines(keepends=True)), + ("", testdata.decode("ascii").splitlines(keepends=True)), ("\n", ["AAA\n", "BB\x00B\n", "CCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), ("\r\n", ["AAA\nBB\x00B\nCCC\rDDD\rEEE\r\n", "FFF\r\n", "GGG"]), ("\r", ["AAA\nBB\x00B\nCCC\r", "DDD\r", "EEE\r", "\nFFF\r", "\nGGG"]), @@ -2031,7 +2046,7 @@ class TextIOWrapperTest(unittest.TestCase): def test_basic_io(self): for chunksize in (1, 2, 3, 4, 5, 15, 16, 17, 31, 32, 33, 63, 64, 65): - for enc in "ascii", "latin1", "utf8" :# , "utf-16-be", "utf-16-le": + for enc in "ascii", "latin-1", "utf-8" :# , "utf-16-be", "utf-16-le": f = self.open(support.TESTFN, "w+", encoding=enc) f._CHUNK_SIZE = chunksize self.assertEqual(f.write("abc"), 3) @@ -2081,7 +2096,7 @@ class TextIOWrapperTest(unittest.TestCase): self.assertEqual(rlines, wlines) def test_telling(self): - f = self.open(support.TESTFN, "w+", encoding="utf8") + f = self.open(support.TESTFN, "w+", encoding="utf-8") p0 = f.tell() f.write("\xff\n") p1 = f.tell() @@ -2327,6 +2342,7 @@ class TextIOWrapperTest(unittest.TestCase): with self.open(support.TESTFN, "w", errors="replace") as f: self.assertEqual(f.errors, "replace") + @support.no_tracing @unittest.skipUnless(threading, 'Threading required for this test.') def test_threads_write(self): # Issue6750: concurrent writes could duplicate data @@ -2644,12 +2660,6 @@ class MiscIOTest(unittest.TestCase): def test_blockingioerror(self): # Various BlockingIOError issues - self.assertRaises(TypeError, self.BlockingIOError) - self.assertRaises(TypeError, self.BlockingIOError, 1) - self.assertRaises(TypeError, self.BlockingIOError, 1, 2, 3, 4) - self.assertRaises(TypeError, self.BlockingIOError, 1, "", None) - b = self.BlockingIOError(1, "") - self.assertEqual(b.characters_written, 0) class C(str): pass c = C("") @@ -2791,6 +2801,7 @@ class MiscIOTest(unittest.TestCase): except self.BlockingIOError as e: self.assertEqual(e.args[0], errno.EAGAIN) + self.assertEqual(e.args[2], e.characters_written) sent[-1] = sent[-1][:e.characters_written] received.append(rf.read()) msg = b'BLOCKED' @@ -2803,6 +2814,7 @@ class MiscIOTest(unittest.TestCase): break except self.BlockingIOError as e: self.assertEqual(e.args[0], errno.EAGAIN) + self.assertEqual(e.args[2], e.characters_written) self.assertEqual(e.characters_written, 0) received.append(rf.read()) @@ -2833,14 +2845,14 @@ class SignalsTest(unittest.TestCase): 1/0 @unittest.skipUnless(threading, 'Threading required for this test.') - @unittest.skipIf(sys.platform in ('freebsd5', 'freebsd6', 'freebsd7'), - 'issue #12429: skip test on FreeBSD <= 7') def check_interrupted_write(self, item, bytes, **fdopen_kwargs): """Check that a partial write, when it gets interrupted, properly invokes the signal handler, and bubbles up the exception raised in the latter.""" read_results = [] def _read(): + if hasattr(signal, 'pthread_sigmask'): + signal.pthread_sigmask(signal.SIG_BLOCK, [signal.SIGALRM]) s = os.read(r, 1) read_results.append(s) t = threading.Thread(target=_read) @@ -2858,7 +2870,7 @@ class SignalsTest(unittest.TestCase): # The buffered IO layer must check for pending signal # handlers, which in this case will invoke alarm_interrupt(). self.assertRaises(ZeroDivisionError, - wio.write, item * (1024 * 1024)) + wio.write, item * (support.PIPE_MAX_SIZE // len(item))) t.join() # We got one byte, get another one and check that it isn't a # repeat of the first one. @@ -2885,6 +2897,7 @@ class SignalsTest(unittest.TestCase): def test_interrupted_write_text(self): self.check_interrupted_write("xy", b"xy", mode="w", encoding="ascii") + @support.no_tracing def check_reentrant_write(self, data, **fdopen_kwargs): def on_alarm(*args): # Will be called reentrantly from the same thread diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 8cdc597..957991c 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -69,11 +69,21 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(list(accumulate('abc')), ['a', 'ab', 'abc']) # works with non-numeric self.assertEqual(list(accumulate([])), []) # empty iterable self.assertEqual(list(accumulate([7])), [7]) # iterable of length one - self.assertRaises(TypeError, accumulate, range(10), 5) # too many args + self.assertRaises(TypeError, accumulate, range(10), 5, 6) # too many args self.assertRaises(TypeError, accumulate) # too few args self.assertRaises(TypeError, accumulate, x=range(10)) # unexpected kwd arg self.assertRaises(TypeError, list, accumulate([1, []])) # args that don't add + s = [2, 8, 9, 5, 7, 0, 3, 4, 1, 6] + self.assertEqual(list(accumulate(s, min)), + [2, 2, 2, 2, 2, 0, 0, 0, 0, 0]) + self.assertEqual(list(accumulate(s, max)), + [2, 8, 9, 9, 9, 9, 9, 9, 9, 9]) + self.assertEqual(list(accumulate(s, operator.mul)), + [2, 16, 144, 720, 5040, 0, 0, 0, 0, 0]) + with self.assertRaises(TypeError): + list(accumulate(s, chr)) # unary-operation + def test_chain(self): def chain2(*iterables): @@ -158,7 +168,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(result, list(combinations2(values, r))) # matches second pure python version self.assertEqual(result, list(combinations3(values, r))) # matches second pure python version - # Test implementation detail: tuple re-use + @support.impl_detail("tuple reuse is specific to CPython") + def test_combinations_tuple_reuse(self): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) @@ -228,7 +239,9 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(result, list(cwr1(values, r))) # matches first pure python version self.assertEqual(result, list(cwr2(values, r))) # matches second pure python version - # Test implementation detail: tuple re-use + @support.impl_detail("tuple reuse is specific to CPython") + def test_combinations_with_replacement_tuple_reuse(self): + cwr = combinations_with_replacement self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) @@ -292,7 +305,8 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(result, list(permutations(values, None))) # test r as None self.assertEqual(result, list(permutations(values))) # test default r - # Test implementation detail: tuple re-use + @support.impl_detail("tuple resuse is CPython specific") + def test_permutations_tuple_reuse(self): self.assertEqual(len(set(map(id, permutations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(permutations('abcde', 3))))), 1) @@ -556,11 +570,13 @@ class TestBasicOps(unittest.TestCase): self.assertEqual(list(zip()), lzip()) self.assertRaises(TypeError, zip, 3) self.assertRaises(TypeError, zip, range(3), 3) - # Check tuple re-use (implementation detail) self.assertEqual([tuple(list(pair)) for pair in zip('abc', 'def')], lzip('abc', 'def')) self.assertEqual([pair for pair in zip('abc', 'def')], lzip('abc', 'def')) + + @support.impl_detail("tuple reuse is specific to CPython") + def test_zip_tuple_reuse(self): ids = list(map(id, zip('abc', 'def'))) self.assertEqual(min(ids), max(ids)) ids = list(map(id, list(zip('abc', 'def')))) @@ -603,11 +619,13 @@ class TestBasicOps(unittest.TestCase): else: self.fail('Did not raise Type in: ' + stmt) - # Check tuple re-use (implementation detail) self.assertEqual([tuple(list(pair)) for pair in zip_longest('abc', 'def')], list(zip('abc', 'def'))) self.assertEqual([pair for pair in zip_longest('abc', 'def')], list(zip('abc', 'def'))) + + @support.impl_detail("tuple reuse is specific to CPython") + def test_zip_longest_tuple_reuse(self): ids = list(map(id, zip_longest('abc', 'def'))) self.assertEqual(min(ids), max(ids)) ids = list(map(id, list(zip_longest('abc', 'def')))) @@ -711,7 +729,8 @@ class TestBasicOps(unittest.TestCase): args = map(iter, args) self.assertEqual(len(list(product(*args))), expected_len) - # Test implementation detail: tuple re-use + @support.impl_detail("tuple reuse is specific to CPython") + def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py index 3aebd68..61d40f9 100644 --- a/Lib/test/test_keywordonlyarg.py +++ b/Lib/test/test_keywordonlyarg.py @@ -78,7 +78,7 @@ class KeywordOnlyArgTestCase(unittest.TestCase): pass with self.assertRaises(TypeError) as exc: f(1, 2, 3) - expected = "f() takes at most 2 positional arguments (3 given)" + expected = "f() takes from 1 to 2 positional arguments but 3 were given" self.assertEqual(str(exc.exception), expected) def testSyntaxErrorForFunctionCall(self): diff --git a/Lib/test/test_lib2to3.py b/Lib/test/test_lib2to3.py index 1afaf70..df4c37b 100644 --- a/Lib/test/test_lib2to3.py +++ b/Lib/test/test_lib2to3.py @@ -9,8 +9,8 @@ from test.support import run_unittest def suite(): tests = unittest.TestSuite() loader = unittest.TestLoader() - for m in (test_fixers, test_pytree,test_util, test_refactor, - test_parser, test_main_): + for m in (test_fixers, test_pytree, test_util, test_refactor, test_parser, + test_main_): tests.addTests(loader.loadTestsFromModule(m)) return tests diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index 30480c1..44081b9 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -404,6 +404,8 @@ class TestMiscellaneous(unittest.TestCase): # Unsupported locale on this system self.skipTest('test needs Turkish locale') loc = locale.getlocale(locale.LC_CTYPE) + if verbose: + print('got locale %a' % (loc,)) locale.setlocale(locale.LC_CTYPE, loc) self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index ab66596..5e192d5 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -36,20 +36,39 @@ import queue import re import select import socket -from socketserver import ThreadingTCPServer, StreamRequestHandler import struct import sys import tempfile -from test.support import captured_stdout, run_with_locale, run_unittest +from test.support import captured_stdout, run_with_locale, run_unittest, patch from test.support import TestHandler, Matcher import textwrap +import time import unittest import warnings import weakref try: import threading + # The following imports are needed only for tests which + # require threading + import asynchat + import asyncore + import errno + from http.server import HTTPServer, BaseHTTPRequestHandler + import smtpd + from urllib.parse import urlparse, parse_qs + from socketserver import (ThreadingUDPServer, DatagramRequestHandler, + ThreadingTCPServer, StreamRequestHandler) except ImportError: threading = None +try: + import win32evtlog +except ImportError: + win32evtlog = None +try: + import win32evtlogutil +except ImportError: + win32evtlogutil = None + win32evtlog = None class BaseTest(unittest.TestCase): @@ -77,9 +96,7 @@ class BaseTest(unittest.TestCase): finally: logging._releaseLock() - # Set two unused loggers: one non-ASCII and one Unicode. - # This is to test correct operation when sorting existing - # loggers in the configuration code. See issue 8201. + # Set two unused loggers self.logger1 = logging.getLogger("\xab\xd7\xbb") self.logger2 = logging.getLogger("\u013f\u00d6\u0047") @@ -140,8 +157,7 @@ class BaseTest(unittest.TestCase): except AttributeError: # StringIO.StringIO lacks a reset() method. actual_lines = stream.getvalue().splitlines() - self.assertEqual(len(actual_lines), len(expected_values), - '%s vs. %s' % (actual_lines, expected_values)) + self.assertEqual(len(actual_lines), len(expected_values)) for actual, expected in zip(actual_lines, expected_values): match = pat.search(actual) if not match: @@ -179,17 +195,17 @@ class BuiltinLevelsTest(BaseTest): INF.log(logging.CRITICAL, m()) INF.error(m()) - INF.warn(m()) + INF.warning(m()) INF.info(m()) DEB.log(logging.CRITICAL, m()) DEB.error(m()) - DEB.warn (m()) - DEB.info (m()) + DEB.warning(m()) + DEB.info(m()) DEB.debug(m()) # These should not log. - ERR.warn(m()) + ERR.warning(m()) ERR.info(m()) ERR.debug(m()) @@ -223,7 +239,7 @@ class BuiltinLevelsTest(BaseTest): INF_ERR.error(m()) # These should not log. - INF_ERR.warn(m()) + INF_ERR.warning(m()) INF_ERR.info(m()) INF_ERR.debug(m()) @@ -247,14 +263,14 @@ class BuiltinLevelsTest(BaseTest): # These should log. INF_UNDEF.log(logging.CRITICAL, m()) INF_UNDEF.error(m()) - INF_UNDEF.warn(m()) + INF_UNDEF.warning(m()) INF_UNDEF.info(m()) INF_ERR_UNDEF.log(logging.CRITICAL, m()) INF_ERR_UNDEF.error(m()) # These should not log. INF_UNDEF.debug(m()) - INF_ERR_UNDEF.warn(m()) + INF_ERR_UNDEF.warning(m()) INF_ERR_UNDEF.info(m()) INF_ERR_UNDEF.debug(m()) @@ -293,8 +309,6 @@ class BuiltinLevelsTest(BaseTest): ('INF.BADPARENT', 'INFO', '4'), ]) - def test_invalid_name(self): - self.assertRaises(TypeError, logging.getLogger, any) class BasicFilterTest(BaseTest): @@ -353,6 +367,10 @@ class BasicFilterTest(BaseTest): finally: handler.removeFilter(filterfunc) + def test_empty_filter(self): + f = logging.Filter() + r = logging.makeLogRecord({'name': 'spam.eggs'}) + self.assertTrue(f.filter(r)) # # First, we define our levels. There can be as many as you want - the only @@ -496,6 +514,438 @@ class CustomLevelsAndFiltersTest(BaseTest): handler.removeFilter(garr) +class HandlerTest(BaseTest): + def test_name(self): + h = logging.Handler() + h.name = 'generic' + self.assertEqual(h.name, 'generic') + h.name = 'anothergeneric' + self.assertEqual(h.name, 'anothergeneric') + self.assertRaises(NotImplementedError, h.emit, None) + + def test_builtin_handlers(self): + # We can't actually *use* too many handlers in the tests, + # but we can try instantiating them with various options + if sys.platform in ('linux', 'darwin'): + for existing in (True, False): + fd, fn = tempfile.mkstemp() + os.close(fd) + if not existing: + os.unlink(fn) + h = logging.handlers.WatchedFileHandler(fn, delay=True) + if existing: + dev, ino = h.dev, h.ino + self.assertNotEqual(dev, -1) + self.assertNotEqual(ino, -1) + r = logging.makeLogRecord({'msg': 'Test'}) + h.handle(r) + # Now remove the file. + os.unlink(fn) + self.assertFalse(os.path.exists(fn)) + # The next call should recreate the file. + h.handle(r) + self.assertTrue(os.path.exists(fn)) + else: + self.assertEqual(h.dev, -1) + self.assertEqual(h.ino, -1) + h.close() + if existing: + os.unlink(fn) + if sys.platform == 'darwin': + sockname = '/var/run/syslog' + else: + sockname = '/dev/log' + try: + h = logging.handlers.SysLogHandler(sockname) + self.assertEqual(h.facility, h.LOG_USER) + self.assertTrue(h.unixsocket) + h.close() + except socket.error: # syslogd might not be available + pass + for method in ('GET', 'POST', 'PUT'): + if method == 'PUT': + self.assertRaises(ValueError, logging.handlers.HTTPHandler, + 'localhost', '/log', method) + else: + h = logging.handlers.HTTPHandler('localhost', '/log', method) + h.close() + h = logging.handlers.BufferingHandler(0) + r = logging.makeLogRecord({}) + self.assertTrue(h.shouldFlush(r)) + h.close() + h = logging.handlers.BufferingHandler(1) + self.assertFalse(h.shouldFlush(r)) + h.close() + +class BadStream(object): + def write(self, data): + raise RuntimeError('deliberate mistake') + +class TestStreamHandler(logging.StreamHandler): + def handleError(self, record): + self.error_record = record + +class StreamHandlerTest(BaseTest): + def test_error_handling(self): + h = TestStreamHandler(BadStream()) + r = logging.makeLogRecord({}) + old_raise = logging.raiseExceptions + old_stderr = sys.stderr + try: + h.handle(r) + self.assertIs(h.error_record, r) + h = logging.StreamHandler(BadStream()) + sys.stderr = sio = io.StringIO() + h.handle(r) + self.assertIn('\nRuntimeError: deliberate mistake\n', + sio.getvalue()) + logging.raiseExceptions = False + sys.stderr = sio = io.StringIO() + h.handle(r) + self.assertEqual('', sio.getvalue()) + finally: + logging.raiseExceptions = old_raise + sys.stderr = old_stderr + +# -- The following section could be moved into a server_helper.py module +# -- if it proves to be of wider utility than just test_logging + +if threading: + class TestSMTPChannel(smtpd.SMTPChannel): + """ + This derived class has had to be created because smtpd does not + support use of custom channel maps, although they are allowed by + asyncore's design. Issue #11959 has been raised to address this, + and if resolved satisfactorily, some of this code can be removed. + """ + def __init__(self, server, conn, addr, sockmap): + asynchat.async_chat.__init__(self, conn, sockmap) + self.smtp_server = server + self.conn = conn + self.addr = addr + self.received_lines = [] + self.smtp_state = self.COMMAND + self.seen_greeting = '' + self.mailfrom = None + self.rcpttos = [] + self.received_data = '' + self.fqdn = socket.getfqdn() + self.num_bytes = 0 + try: + self.peer = conn.getpeername() + except socket.error as err: + # a race condition may occur if the other end is closing + # before we can get the peername + self.close() + if err.args[0] != errno.ENOTCONN: + raise + return + self.push('220 %s %s' % (self.fqdn, smtpd.__version__)) + self.set_terminator(b'\r\n') + + + class TestSMTPServer(smtpd.SMTPServer): + """ + This class implements a test SMTP server. + + :param addr: A (host, port) tuple which the server listens on. + You can specify a port value of zero: the server's + *port* attribute will hold the actual port number + used, which can be used in client connections. + :param handler: A callable which will be called to process + incoming messages. The handler will be passed + the client address tuple, who the message is from, + a list of recipients and the message data. + :param poll_interval: The interval, in seconds, used in the underlying + :func:`select` or :func:`poll` call by + :func:`asyncore.loop`. + :param sockmap: A dictionary which will be used to hold + :class:`asyncore.dispatcher` instances used by + :func:`asyncore.loop`. This avoids changing the + :mod:`asyncore` module's global state. + """ + channel_class = TestSMTPChannel + + def __init__(self, addr, handler, poll_interval, sockmap): + self._localaddr = addr + self._remoteaddr = None + self.sockmap = sockmap + asyncore.dispatcher.__init__(self, map=sockmap) + try: + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + sock.setblocking(0) + self.set_socket(sock, map=sockmap) + # try to re-use a server port if possible + self.set_reuse_addr() + self.bind(addr) + self.port = sock.getsockname()[1] + self.listen(5) + except: + self.close() + raise + self._handler = handler + self._thread = None + self.poll_interval = poll_interval + + def handle_accepted(self, conn, addr): + """ + Redefined only because the base class does not pass in a + map, forcing use of a global in :mod:`asyncore`. + """ + channel = self.channel_class(self, conn, addr, self.sockmap) + + def process_message(self, peer, mailfrom, rcpttos, data): + """ + Delegates to the handler passed in to the server's constructor. + + Typically, this will be a test case method. + :param peer: The client (host, port) tuple. + :param mailfrom: The address of the sender. + :param rcpttos: The addresses of the recipients. + :param data: The message. + """ + self._handler(peer, mailfrom, rcpttos, data) + + def start(self): + """ + Start the server running on a separate daemon thread. + """ + self._thread = t = threading.Thread(target=self.serve_forever, + args=(self.poll_interval,)) + t.setDaemon(True) + t.start() + + def serve_forever(self, poll_interval): + """ + Run the :mod:`asyncore` loop until normal termination + conditions arise. + :param poll_interval: The interval, in seconds, used in the underlying + :func:`select` or :func:`poll` call by + :func:`asyncore.loop`. + """ + try: + asyncore.loop(poll_interval, map=self.sockmap) + except select.error: + # On FreeBSD 8, closing the server repeatably + # raises this error. We swallow it if the + # server has been closed. + if self.connected or self.accepting: + raise + + def stop(self, timeout=None): + """ + Stop the thread by closing the server instance. + Wait for the server thread to terminate. + + :param timeout: How long to wait for the server thread + to terminate. + """ + self.close() + self._thread.join(timeout) + self._thread = None + + class ControlMixin(object): + """ + This mixin is used to start a server on a separate thread, and + shut it down programmatically. Request handling is simplified - instead + of needing to derive a suitable RequestHandler subclass, you just + provide a callable which will be passed each received request to be + processed. + + :param handler: A handler callable which will be called with a + single parameter - the request - in order to + process the request. This handler is called on the + server thread, effectively meaning that requests are + processed serially. While not quite Web scale ;-), + this should be fine for testing applications. + :param poll_interval: The polling interval in seconds. + """ + def __init__(self, handler, poll_interval): + self._thread = None + self.poll_interval = poll_interval + self._handler = handler + self.ready = threading.Event() + + def start(self): + """ + Create a daemon thread to run the server, and start it. + """ + self._thread = t = threading.Thread(target=self.serve_forever, + args=(self.poll_interval,)) + t.setDaemon(True) + t.start() + + def serve_forever(self, poll_interval): + """ + Run the server. Set the ready flag before entering the + service loop. + """ + self.ready.set() + super(ControlMixin, self).serve_forever(poll_interval) + + def stop(self, timeout=None): + """ + Tell the server thread to stop, and wait for it to do so. + + :param timeout: How long to wait for the server thread + to terminate. + """ + self.shutdown() + if self._thread is not None: + self._thread.join(timeout) + self._thread = None + self.server_close() + self.ready.clear() + + class TestHTTPServer(ControlMixin, HTTPServer): + """ + An HTTP server which is controllable using :class:`ControlMixin`. + + :param addr: A tuple with the IP address and port to listen on. + :param handler: A handler callable which will be called with a + single parameter - the request - in order to + process the request. + :param poll_interval: The polling interval in seconds. + :param log: Pass ``True`` to enable log messages. + """ + def __init__(self, addr, handler, poll_interval=0.5, + log=False, sslctx=None): + class DelegatingHTTPRequestHandler(BaseHTTPRequestHandler): + def __getattr__(self, name, default=None): + if name.startswith('do_'): + return self.process_request + raise AttributeError(name) + + def process_request(self): + self.server._handler(self) + + def log_message(self, format, *args): + if log: + super(DelegatingHTTPRequestHandler, + self).log_message(format, *args) + HTTPServer.__init__(self, addr, DelegatingHTTPRequestHandler) + ControlMixin.__init__(self, handler, poll_interval) + self.sslctx = sslctx + + def get_request(self): + try: + sock, addr = self.socket.accept() + if self.sslctx: + sock = self.sslctx.wrap_socket(sock, server_side=True) + except socket.error as e: + # socket errors are silenced by the caller, print them here + sys.stderr.write("Got an error:\n%s\n" % e) + raise + return sock, addr + + class TestTCPServer(ControlMixin, ThreadingTCPServer): + """ + A TCP server which is controllable using :class:`ControlMixin`. + + :param addr: A tuple with the IP address and port to listen on. + :param handler: A handler callable which will be called with a single + parameter - the request - in order to process the request. + :param poll_interval: The polling interval in seconds. + :bind_and_activate: If True (the default), binds the server and starts it + listening. If False, you need to call + :meth:`server_bind` and :meth:`server_activate` at + some later time before calling :meth:`start`, so that + the server will set up the socket and listen on it. + """ + + allow_reuse_address = True + + def __init__(self, addr, handler, poll_interval=0.5, + bind_and_activate=True): + class DelegatingTCPRequestHandler(StreamRequestHandler): + + def handle(self): + self.server._handler(self) + ThreadingTCPServer.__init__(self, addr, DelegatingTCPRequestHandler, + bind_and_activate) + ControlMixin.__init__(self, handler, poll_interval) + + def server_bind(self): + super(TestTCPServer, self).server_bind() + self.port = self.socket.getsockname()[1] + + class TestUDPServer(ControlMixin, ThreadingUDPServer): + """ + A UDP server which is controllable using :class:`ControlMixin`. + + :param addr: A tuple with the IP address and port to listen on. + :param handler: A handler callable which will be called with a + single parameter - the request - in order to + process the request. + :param poll_interval: The polling interval for shutdown requests, + in seconds. + :bind_and_activate: If True (the default), binds the server and + starts it listening. If False, you need to + call :meth:`server_bind` and + :meth:`server_activate` at some later time + before calling :meth:`start`, so that the server will + set up the socket and listen on it. + """ + def __init__(self, addr, handler, poll_interval=0.5, + bind_and_activate=True): + class DelegatingUDPRequestHandler(DatagramRequestHandler): + + def handle(self): + self.server._handler(self) + + def finish(self): + data = self.wfile.getvalue() + if data: + try: + super(DelegatingUDPRequestHandler, self).finish() + except socket.error: + if not self.server._closed: + raise + + ThreadingUDPServer.__init__(self, addr, + DelegatingUDPRequestHandler, + bind_and_activate) + ControlMixin.__init__(self, handler, poll_interval) + self._closed = False + + def server_bind(self): + super(TestUDPServer, self).server_bind() + self.port = self.socket.getsockname()[1] + + def server_close(self): + super(TestUDPServer, self).server_close() + self._closed = True + +# - end of server_helper section + +@unittest.skipUnless(threading, 'Threading required for this test.') +class SMTPHandlerTest(BaseTest): + def test_basic(self): + sockmap = {} + server = TestSMTPServer(('localhost', 0), self.process_message, 0.001, + sockmap) + server.start() + addr = ('localhost', server.port) + h = logging.handlers.SMTPHandler(addr, 'me', 'you', 'Log') + self.assertEqual(h.toaddrs, ['you']) + self.messages = [] + r = logging.makeLogRecord({'msg': 'Hello'}) + self.handled = threading.Event() + h.handle(r) + self.handled.wait() + server.stop() + self.assertEqual(len(self.messages), 1) + peer, mailfrom, rcpttos, data = self.messages[0] + self.assertEqual(mailfrom, 'me') + self.assertEqual(rcpttos, ['you']) + self.assertIn('\nSubject: Log\n', data) + self.assertTrue(data.endswith('\n\nHello')) + h.close() + + def process_message(self, *args): + self.messages.append(args) + self.handled.set() + class MemoryHandlerTest(BaseTest): """Tests for the MemoryHandler.""" @@ -523,7 +973,7 @@ class MemoryHandlerTest(BaseTest): self.mem_logger.info(self.next_message()) self.assert_log_lines([]) # This will flush because the level is >= logging.WARNING - self.mem_logger.warn(self.next_message()) + self.mem_logger.warning(self.next_message()) lines = [ ('DEBUG', '1'), ('INFO', '2'), @@ -868,116 +1318,280 @@ class ConfigFileTest(BaseTest): # Original logger output is empty. self.assert_log_lines([]) -class LogRecordStreamHandler(StreamRequestHandler): - """Handler for a streaming logging request. It saves the log message in the - TCP server's 'log_output' attribute.""" +@unittest.skipUnless(threading, 'Threading required for this test.') +class SocketHandlerTest(BaseTest): - TCP_LOG_END = "!!!END!!!" + """Test for SocketHandler objects.""" - def handle(self): - """Handle multiple requests - each expected to be of 4-byte length, - followed by the LogRecord in pickle format. Logs the record - according to whatever policy is configured locally.""" + def setUp(self): + """Set up a TCP server to receive log messages, and a SocketHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + addr = ('localhost', 0) + self.server = server = TestTCPServer(addr, self.handle_socket, + 0.01) + server.start() + server.ready.wait() + self.sock_hdlr = logging.handlers.SocketHandler('localhost', + server.port) + self.log_output = '' + self.root_logger.removeHandler(self.root_logger.handlers[0]) + self.root_logger.addHandler(self.sock_hdlr) + self.handled = threading.Semaphore(0) + + def tearDown(self): + """Shutdown the TCP server.""" + try: + self.server.stop(2.0) + self.root_logger.removeHandler(self.sock_hdlr) + self.sock_hdlr.close() + finally: + BaseTest.tearDown(self) + + def handle_socket(self, request): + conn = request.connection while True: - chunk = self.connection.recv(4) + chunk = conn.recv(4) if len(chunk) < 4: break slen = struct.unpack(">L", chunk)[0] - chunk = self.connection.recv(slen) + chunk = conn.recv(slen) while len(chunk) < slen: - chunk = chunk + self.connection.recv(slen - len(chunk)) - obj = self.unpickle(chunk) + chunk = chunk + conn.recv(slen - len(chunk)) + obj = pickle.loads(chunk) record = logging.makeLogRecord(obj) - self.handle_log_record(record) + self.log_output += record.msg + '\n' + self.handled.release() - def unpickle(self, data): - return pickle.loads(data) + def test_output(self): + # The log message sent to the SocketHandler is properly received. + logger = logging.getLogger("tcp") + logger.error("spam") + self.handled.acquire() + logger.debug("eggs") + self.handled.acquire() + self.assertEqual(self.log_output, "spam\neggs\n") - def handle_log_record(self, record): - # If the end-of-messages sentinel is seen, tell the server to - # terminate. - if self.TCP_LOG_END in record.msg: - self.server.abort = 1 - return - self.server.log_output += record.msg + "\n" + def test_noserver(self): + # Kill the server + self.server.stop(2.0) + #The logging call should try to connect, which should fail + try: + raise RuntimeError('Deliberate mistake') + except RuntimeError: + self.root_logger.exception('Never sent') + self.root_logger.error('Never sent, either') + now = time.time() + self.assertTrue(self.sock_hdlr.retryTime > now) + time.sleep(self.sock_hdlr.retryTime - now + 0.001) + self.root_logger.error('Nor this') -class LogRecordSocketReceiver(ThreadingTCPServer): +@unittest.skipUnless(threading, 'Threading required for this test.') +class DatagramHandlerTest(BaseTest): - """A simple-minded TCP socket-based logging receiver suitable for test - purposes.""" + """Test for DatagramHandler.""" - allow_reuse_address = 1 - log_output = "" + def setUp(self): + """Set up a UDP server to receive log messages, and a DatagramHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + addr = ('localhost', 0) + self.server = server = TestUDPServer(addr, self.handle_datagram, 0.01) + server.start() + server.ready.wait() + self.sock_hdlr = logging.handlers.DatagramHandler('localhost', + server.port) + self.log_output = '' + self.root_logger.removeHandler(self.root_logger.handlers[0]) + self.root_logger.addHandler(self.sock_hdlr) + self.handled = threading.Event() - def __init__(self, host='localhost', - port=logging.handlers.DEFAULT_TCP_LOGGING_PORT, - handler=LogRecordStreamHandler): - ThreadingTCPServer.__init__(self, (host, port), handler) - self.abort = False - self.timeout = 0.1 - self.finished = threading.Event() + def tearDown(self): + """Shutdown the UDP server.""" + try: + self.server.stop(2.0) + self.root_logger.removeHandler(self.sock_hdlr) + self.sock_hdlr.close() + finally: + BaseTest.tearDown(self) + + def handle_datagram(self, request): + slen = struct.pack('>L', 0) # length of prefix + packet = request.packet[len(slen):] + obj = pickle.loads(packet) + record = logging.makeLogRecord(obj) + self.log_output += record.msg + '\n' + self.handled.set() - def serve_until_stopped(self): - while not self.abort: - rd, wr, ex = select.select([self.socket.fileno()], [], [], - self.timeout) - if rd: - self.handle_request() - # Notify the main thread that we're about to exit - self.finished.set() - # close the listen socket - self.server_close() + def test_output(self): + # The log message sent to the DatagramHandler is properly received. + logger = logging.getLogger("udp") + logger.error("spam") + self.handled.wait() + self.handled.clear() + logger.error("eggs") + self.handled.wait() + self.assertEqual(self.log_output, "spam\neggs\n") @unittest.skipUnless(threading, 'Threading required for this test.') -class SocketHandlerTest(BaseTest): +class SysLogHandlerTest(BaseTest): - """Test for SocketHandler objects.""" + """Test for SysLogHandler using UDP.""" def setUp(self): - """Set up a TCP server to receive log messages, and a SocketHandler + """Set up a UDP server to receive log messages, and a SysLogHandler pointing to that server's address and port.""" BaseTest.setUp(self) - self.tcpserver = LogRecordSocketReceiver(port=0) - self.port = self.tcpserver.socket.getsockname()[1] - self.threads = [ - threading.Thread(target=self.tcpserver.serve_until_stopped)] - for thread in self.threads: - thread.start() - - self.sock_hdlr = logging.handlers.SocketHandler('localhost', self.port) - self.sock_hdlr.setFormatter(self.root_formatter) + addr = ('localhost', 0) + self.server = server = TestUDPServer(addr, self.handle_datagram, + 0.01) + server.start() + server.ready.wait() + self.sl_hdlr = logging.handlers.SysLogHandler(('localhost', + server.port)) + self.log_output = '' self.root_logger.removeHandler(self.root_logger.handlers[0]) - self.root_logger.addHandler(self.sock_hdlr) + self.root_logger.addHandler(self.sl_hdlr) + self.handled = threading.Event() def tearDown(self): - """Shutdown the TCP server.""" + """Shutdown the UDP server.""" try: - self.tcpserver.abort = True - del self.tcpserver - self.root_logger.removeHandler(self.sock_hdlr) - self.sock_hdlr.close() - for thread in self.threads: - thread.join(2.0) + self.server.stop(2.0) + self.root_logger.removeHandler(self.sl_hdlr) + self.sl_hdlr.close() finally: BaseTest.tearDown(self) - def get_output(self): - """Get the log output as received by the TCP server.""" - # Signal the TCP receiver and wait for it to terminate. - self.root_logger.critical(LogRecordStreamHandler.TCP_LOG_END) - self.tcpserver.finished.wait(2.0) - return self.tcpserver.log_output + def handle_datagram(self, request): + self.log_output = request.packet + self.handled.set() def test_output(self): - # The log message sent to the SocketHandler is properly received. - logger = logging.getLogger("tcp") - logger.error("spam") - logger.debug("eggs") - self.assertEqual(self.get_output(), "spam\neggs\n") + # The log message sent to the SysLogHandler is properly received. + logger = logging.getLogger("slh") + logger.error("sp\xe4m") + self.handled.wait() + self.assertEqual(self.log_output, b'<11>\xef\xbb\xbfsp\xc3\xa4m\x00') + self.handled.clear() + self.sl_hdlr.append_nul = False + logger.error("sp\xe4m") + self.handled.wait() + self.assertEqual(self.log_output, b'<11>\xef\xbb\xbfsp\xc3\xa4m') + self.handled.clear() + self.sl_hdlr.ident = "h\xe4m-" + logger.error("sp\xe4m") + self.handled.wait() + self.assertEqual(self.log_output, b'<11>\xef\xbb\xbfh\xc3\xa4m-sp\xc3\xa4m') + + +@unittest.skipUnless(threading, 'Threading required for this test.') +class HTTPHandlerTest(BaseTest): + """Test for HTTPHandler.""" + + PEMFILE = """-----BEGIN RSA PRIVATE KEY----- +MIICXQIBAAKBgQDGT4xS5r91rbLJQK2nUDenBhBG6qFk+bVOjuAGC/LSHlAoBnvG +zQG3agOG+e7c5z2XT8m2ktORLqG3E4mYmbxgyhDrzP6ei2Anc+pszmnxPoK3Puh5 +aXV+XKt0bU0C1m2+ACmGGJ0t3P408art82nOxBw8ZHgIg9Dtp6xIUCyOqwIDAQAB +AoGBAJFTnFboaKh5eUrIzjmNrKsG44jEyy+vWvHN/FgSC4l103HxhmWiuL5Lv3f7 +0tMp1tX7D6xvHwIG9VWvyKb/Cq9rJsDibmDVIOslnOWeQhG+XwJyitR0pq/KlJIB +5LjORcBw795oKWOAi6RcOb1ON59tysEFYhAGQO9k6VL621gRAkEA/Gb+YXULLpbs +piXN3q4zcHzeaVANo69tUZ6TjaQqMeTxE4tOYM0G0ZoSeHEdaP59AOZGKXXNGSQy +2z/MddcYGQJBAMkjLSYIpOLJY11ja8OwwswFG2hEzHe0cS9bzo++R/jc1bHA5R0Y +i6vA5iPi+wopPFvpytdBol7UuEBe5xZrxWMCQQCWxELRHiP2yWpEeLJ3gGDzoXMN +PydWjhRju7Bx3AzkTtf+D6lawz1+eGTuEss5i0JKBkMEwvwnN2s1ce+EuF4JAkBb +E96h1lAzkVW5OAfYOPY8RCPA90ZO/hoyg7PpSxR0ECuDrgERR8gXIeYUYfejBkEa +rab4CfRoVJKKM28Yq/xZAkBvuq670JRCwOgfUTdww7WpdOQBYPkzQccsKNCslQW8 +/DyW6y06oQusSENUvynT6dr3LJxt/NgZPhZX2+k1eYDV +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICGzCCAYSgAwIBAgIJAIq84a2Q/OvlMA0GCSqGSIb3DQEBBQUAMBQxEjAQBgNV +BAMTCWxvY2FsaG9zdDAeFw0xMTA1MjExMDIzMzNaFw03NTAzMjEwMzU1MTdaMBQx +EjAQBgNVBAMTCWxvY2FsaG9zdDCBnzANBgkqhkiG9w0BAQEFAAOBjQAwgYkCgYEA +xk+MUua/da2yyUCtp1A3pwYQRuqhZPm1To7gBgvy0h5QKAZ7xs0Bt2oDhvnu3Oc9 +l0/JtpLTkS6htxOJmJm8YMoQ68z+notgJ3PqbM5p8T6Ctz7oeWl1flyrdG1NAtZt +vgAphhidLdz+NPGq7fNpzsQcPGR4CIPQ7aesSFAsjqsCAwEAAaN1MHMwHQYDVR0O +BBYEFLWaUPO6N7efGiuoS9i3DVYcUwn0MEQGA1UdIwQ9MDuAFLWaUPO6N7efGiuo +S9i3DVYcUwn0oRikFjAUMRIwEAYDVQQDEwlsb2NhbGhvc3SCCQCKvOGtkPzr5TAM +BgNVHRMEBTADAQH/MA0GCSqGSIb3DQEBBQUAA4GBAMK5whPjLNQK1Ivvk88oqJqq +4f889OwikGP0eUhOBhbFlsZs+jq5YZC2UzHz+evzKBlgAP1u4lP/cB85CnjvWqM+ +1c/lywFHQ6HOdDeQ1L72tSYMrNOG4XNmLn0h7rx6GoTU7dcFRfseahBCq8mv0IDt +IRbTpvlHWPjsSvHz0ZOH +-----END CERTIFICATE-----""" + + def setUp(self): + """Set up an HTTP server to receive log messages, and a HTTPHandler + pointing to that server's address and port.""" + BaseTest.setUp(self) + self.handled = threading.Event() + + def handle_request(self, request): + self.command = request.command + self.log_data = urlparse(request.path) + if self.command == 'POST': + try: + rlen = int(request.headers['Content-Length']) + self.post_data = request.rfile.read(rlen) + except: + self.post_data = None + request.send_response(200) + request.end_headers() + self.handled.set() + def test_output(self): + # The log message sent to the HTTPHandler is properly received. + logger = logging.getLogger("http") + root_logger = self.root_logger + root_logger.removeHandler(self.root_logger.handlers[0]) + for secure in (False, True): + addr = ('localhost', 0) + if secure: + try: + import ssl + fd, fn = tempfile.mkstemp() + os.close(fd) + with open(fn, 'w') as f: + f.write(self.PEMFILE) + sslctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + sslctx.load_cert_chain(fn) + os.unlink(fn) + except ImportError: + sslctx = None + else: + sslctx = None + self.server = server = TestHTTPServer(addr, self.handle_request, + 0.01, sslctx=sslctx) + server.start() + server.ready.wait() + host = 'localhost:%d' % server.server_port + secure_client = secure and sslctx + self.h_hdlr = logging.handlers.HTTPHandler(host, '/frob', + secure=secure_client) + self.log_data = None + root_logger.addHandler(self.h_hdlr) + + for method in ('GET', 'POST'): + self.h_hdlr.method = method + self.handled.clear() + msg = "sp\xe4m" + logger.error(msg) + self.handled.wait() + self.assertEqual(self.log_data.path, '/frob') + self.assertEqual(self.command, method) + if method == 'GET': + d = parse_qs(self.log_data.query) + else: + d = parse_qs(self.post_data.decode('utf-8')) + self.assertEqual(d['name'], ['http']) + self.assertEqual(d['funcName'], ['test_output']) + self.assertEqual(d['msg'], [msg]) + + self.server.stop(2.0) + self.root_logger.removeHandler(self.h_hdlr) + self.h_hdlr.close() class MemoryTest(BaseTest): @@ -1085,28 +1699,39 @@ class WarningsTest(BaseTest): def test_warnings(self): with warnings.catch_warnings(): logging.captureWarnings(True) - try: - warnings.filterwarnings("always", category=UserWarning) - file = io.StringIO() - h = logging.StreamHandler(file) - logger = logging.getLogger("py.warnings") - logger.addHandler(h) - warnings.warn("I'm warning you...") - logger.removeHandler(h) - s = file.getvalue() - h.close() - self.assertTrue(s.find("UserWarning: I'm warning you...\n") > 0) - - #See if an explicit file uses the original implementation - file = io.StringIO() - warnings.showwarning("Explicit", UserWarning, "dummy.py", 42, - file, "Dummy line") - s = file.getvalue() - file.close() - self.assertEqual(s, - "dummy.py:42: UserWarning: Explicit\n Dummy line\n") - finally: - logging.captureWarnings(False) + self.addCleanup(logging.captureWarnings, False) + warnings.filterwarnings("always", category=UserWarning) + stream = io.StringIO() + h = logging.StreamHandler(stream) + logger = logging.getLogger("py.warnings") + logger.addHandler(h) + warnings.warn("I'm warning you...") + logger.removeHandler(h) + s = stream.getvalue() + h.close() + self.assertTrue(s.find("UserWarning: I'm warning you...\n") > 0) + + #See if an explicit file uses the original implementation + a_file = io.StringIO() + warnings.showwarning("Explicit", UserWarning, "dummy.py", 42, + a_file, "Dummy line") + s = a_file.getvalue() + a_file.close() + self.assertEqual(s, + "dummy.py:42: UserWarning: Explicit\n Dummy line\n") + + def test_warnings_no_handlers(self): + with warnings.catch_warnings(): + logging.captureWarnings(True) + self.addCleanup(logging.captureWarnings, False) + + # confirm our assumption: no loggers are set + logger = logging.getLogger("py.warnings") + self.assertEqual(logger.handlers, []) + + warnings.showwarning("Explicit", UserWarning, "dummy.py", 42) + self.assertEqual(len(logger.handlers), 1) + self.assertIsInstance(logger.handlers[0], logging.NullHandler) def formatFunc(format, datefmt=None): @@ -1959,6 +2584,7 @@ class ConfigDictTest(BaseTest): logging.config.stopListening() t.join(2.0) + @unittest.skipUnless(threading, 'Threading required for this test.') def test_listen_config_10_ok(self): with captured_stdout() as output: self.setup_via_listener(json.dumps(self.config10)) @@ -1978,6 +2604,7 @@ class ConfigDictTest(BaseTest): ('ERROR', '4'), ], stream=output) + @unittest.skipUnless(threading, 'Threading required for this test.') def test_listen_config_1_ok(self): with captured_stdout() as output: self.setup_via_listener(textwrap.dedent(ConfigFileTest.config1)) @@ -1992,6 +2619,27 @@ class ConfigDictTest(BaseTest): # Original logger output is empty. self.assert_log_lines([]) + def test_baseconfig(self): + d = { + 'atuple': (1, 2, 3), + 'alist': ['a', 'b', 'c'], + 'adict': {'d': 'e', 'f': 3 }, + 'nest1': ('g', ('h', 'i'), 'j'), + 'nest2': ['k', ['l', 'm'], 'n'], + 'nest3': ['o', 'cfg://alist', 'p'], + } + bc = logging.config.BaseConfigurator(d) + self.assertEqual(bc.convert('cfg://atuple[1]'), 2) + self.assertEqual(bc.convert('cfg://alist[1]'), 'b') + self.assertEqual(bc.convert('cfg://nest1[1][0]'), 'h') + self.assertEqual(bc.convert('cfg://nest2[1][1]'), 'm') + self.assertEqual(bc.convert('cfg://adict.d'), 'e') + self.assertEqual(bc.convert('cfg://adict[f]'), 3) + v = bc.convert('cfg://nest3') + self.assertEqual(v.pop(1), ['a', 'b', 'c']) + self.assertRaises(KeyError, bc.convert, 'cfg://nosuch') + self.assertRaises(ValueError, bc.convert, 'cfg://!') + self.assertRaises(KeyError, bc.convert, 'cfg://adict[2]') class ManagerTest(BaseTest): def test_manager_loggerclass(self): @@ -2010,6 +2658,11 @@ class ManagerTest(BaseTest): self.assertEqual(logged, ['should appear in logged']) + def test_set_log_record_factory(self): + man = logging.Manager(None) + expected = object() + man.setLogRecordFactory(expected) + self.assertEqual(man.logRecordFactory, expected) class ChildLoggerTest(BaseTest): def test_child_loggers(self): @@ -2111,6 +2764,18 @@ class QueueHandlerTest(BaseTest): self.assertTrue(handler.matches(levelno=logging.ERROR, message='2')) self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3')) +ZERO = datetime.timedelta(0) + +class UTC(datetime.tzinfo): + def utcoffset(self, dt): + return ZERO + + dst = utcoffset + + def tzname(self, dt): + return 'UTC' + +utc = UTC() class FormatterTest(unittest.TestCase): def setUp(self): @@ -2184,6 +2849,69 @@ class FormatterTest(unittest.TestCase): f = logging.Formatter('asctime', style='$') self.assertFalse(f.usesTime()) + def test_invalid_style(self): + self.assertRaises(ValueError, logging.Formatter, None, None, 'x') + + def test_time(self): + r = self.get_record() + dt = datetime.datetime(1993,4,21,8,3,0,0,utc) + r.created = time.mktime(dt.timetuple()) - time.timezone + r.msecs = 123 + f = logging.Formatter('%(asctime)s %(message)s') + f.converter = time.gmtime + self.assertEqual(f.formatTime(r), '1993-04-21 08:03:00,123') + self.assertEqual(f.formatTime(r, '%Y:%d'), '1993:21') + f.format(r) + self.assertEqual(r.asctime, '1993-04-21 08:03:00,123') + +class TestBufferingFormatter(logging.BufferingFormatter): + def formatHeader(self, records): + return '[(%d)' % len(records) + + def formatFooter(self, records): + return '(%d)]' % len(records) + +class BufferingFormatterTest(unittest.TestCase): + def setUp(self): + self.records = [ + logging.makeLogRecord({'msg': 'one'}), + logging.makeLogRecord({'msg': 'two'}), + ] + + def test_default(self): + f = logging.BufferingFormatter() + self.assertEqual('', f.format([])) + self.assertEqual('onetwo', f.format(self.records)) + + def test_custom(self): + f = TestBufferingFormatter() + self.assertEqual('[(2)onetwo(2)]', f.format(self.records)) + lf = logging.Formatter('<%(message)s>') + f = TestBufferingFormatter(lf) + self.assertEqual('[(2)<one><two>(2)]', f.format(self.records)) + +class ExceptionTest(BaseTest): + def test_formatting(self): + r = self.root_logger + h = RecordingHandler() + r.addHandler(h) + try: + raise RuntimeError('deliberate mistake') + except: + logging.exception('failed', stack_info=True) + r.removeHandler(h) + h.close() + r = h.records[0] + self.assertTrue(r.exc_text.startswith('Traceback (most recent ' + 'call last):\n')) + self.assertTrue(r.exc_text.endswith('\nRuntimeError: ' + 'deliberate mistake')) + self.assertTrue(r.stack_info.startswith('Stack (most recent ' + 'call last):\n')) + self.assertTrue(r.stack_info.endswith('logging.exception(\'failed\', ' + 'stack_info=True)')) + + class LastResortTest(BaseTest): def test_last_resort(self): # Test the last resort handler @@ -2194,6 +2922,8 @@ class LastResortTest(BaseTest): old_raise_exceptions = logging.raiseExceptions try: sys.stderr = sio = io.StringIO() + root.debug('This should not appear') + self.assertEqual(sio.getvalue(), '') root.warning('This is your final chance!') self.assertEqual(sio.getvalue(), 'This is your final chance!\n') #No handlers and no last resort, so 'No handlers' message @@ -2218,6 +2948,586 @@ class LastResortTest(BaseTest): logging.raiseExceptions = old_raise_exceptions +class FakeHandler: + + def __init__(self, identifier, called): + for method in ('acquire', 'flush', 'close', 'release'): + setattr(self, method, self.record_call(identifier, method, called)) + + def record_call(self, identifier, method_name, called): + def inner(): + called.append('{} - {}'.format(identifier, method_name)) + return inner + + +class RecordingHandler(logging.NullHandler): + + def __init__(self, *args, **kwargs): + super(RecordingHandler, self).__init__(*args, **kwargs) + self.records = [] + + def handle(self, record): + """Keep track of all the emitted records.""" + self.records.append(record) + + +class ShutdownTest(BaseTest): + + """Test suite for the shutdown method.""" + + def setUp(self): + super(ShutdownTest, self).setUp() + self.called = [] + + raise_exceptions = logging.raiseExceptions + self.addCleanup(setattr, logging, 'raiseExceptions', raise_exceptions) + + def raise_error(self, error): + def inner(): + raise error() + return inner + + def test_no_failure(self): + # create some fake handlers + handler0 = FakeHandler(0, self.called) + handler1 = FakeHandler(1, self.called) + handler2 = FakeHandler(2, self.called) + + # create live weakref to those handlers + handlers = map(logging.weakref.ref, [handler0, handler1, handler2]) + + logging.shutdown(handlerList=list(handlers)) + + expected = ['2 - acquire', '2 - flush', '2 - close', '2 - release', + '1 - acquire', '1 - flush', '1 - close', '1 - release', + '0 - acquire', '0 - flush', '0 - close', '0 - release'] + self.assertEqual(expected, self.called) + + def _test_with_failure_in_method(self, method, error): + handler = FakeHandler(0, self.called) + setattr(handler, method, self.raise_error(error)) + handlers = [logging.weakref.ref(handler)] + + logging.shutdown(handlerList=list(handlers)) + + self.assertEqual('0 - release', self.called[-1]) + + def test_with_ioerror_in_acquire(self): + self._test_with_failure_in_method('acquire', IOError) + + def test_with_ioerror_in_flush(self): + self._test_with_failure_in_method('flush', IOError) + + def test_with_ioerror_in_close(self): + self._test_with_failure_in_method('close', IOError) + + def test_with_valueerror_in_acquire(self): + self._test_with_failure_in_method('acquire', ValueError) + + def test_with_valueerror_in_flush(self): + self._test_with_failure_in_method('flush', ValueError) + + def test_with_valueerror_in_close(self): + self._test_with_failure_in_method('close', ValueError) + + def test_with_other_error_in_acquire_without_raise(self): + logging.raiseExceptions = False + self._test_with_failure_in_method('acquire', IndexError) + + def test_with_other_error_in_flush_without_raise(self): + logging.raiseExceptions = False + self._test_with_failure_in_method('flush', IndexError) + + def test_with_other_error_in_close_without_raise(self): + logging.raiseExceptions = False + self._test_with_failure_in_method('close', IndexError) + + def test_with_other_error_in_acquire_with_raise(self): + logging.raiseExceptions = True + self.assertRaises(IndexError, self._test_with_failure_in_method, + 'acquire', IndexError) + + def test_with_other_error_in_flush_with_raise(self): + logging.raiseExceptions = True + self.assertRaises(IndexError, self._test_with_failure_in_method, + 'flush', IndexError) + + def test_with_other_error_in_close_with_raise(self): + logging.raiseExceptions = True + self.assertRaises(IndexError, self._test_with_failure_in_method, + 'close', IndexError) + + +class ModuleLevelMiscTest(BaseTest): + + """Test suite for some module level methods.""" + + def test_disable(self): + old_disable = logging.root.manager.disable + # confirm our assumptions are correct + self.assertEqual(old_disable, 0) + self.addCleanup(logging.disable, old_disable) + + logging.disable(83) + self.assertEqual(logging.root.manager.disable, 83) + + def _test_log(self, method, level=None): + called = [] + patch(self, logging, 'basicConfig', + lambda *a, **kw: called.append((a, kw))) + + recording = RecordingHandler() + logging.root.addHandler(recording) + + log_method = getattr(logging, method) + if level is not None: + log_method(level, "test me: %r", recording) + else: + log_method("test me: %r", recording) + + self.assertEqual(len(recording.records), 1) + record = recording.records[0] + self.assertEqual(record.getMessage(), "test me: %r" % recording) + + expected_level = level if level is not None else getattr(logging, method.upper()) + self.assertEqual(record.levelno, expected_level) + + # basicConfig was not called! + self.assertEqual(called, []) + + def test_log(self): + self._test_log('log', logging.ERROR) + + def test_debug(self): + self._test_log('debug') + + def test_info(self): + self._test_log('info') + + def test_warning(self): + self._test_log('warning') + + def test_error(self): + self._test_log('error') + + def test_critical(self): + self._test_log('critical') + + def test_set_logger_class(self): + self.assertRaises(TypeError, logging.setLoggerClass, object) + + class MyLogger(logging.Logger): + pass + + logging.setLoggerClass(MyLogger) + self.assertEqual(logging.getLoggerClass(), MyLogger) + + logging.setLoggerClass(logging.Logger) + self.assertEqual(logging.getLoggerClass(), logging.Logger) + +class LogRecordTest(BaseTest): + def test_str_rep(self): + r = logging.makeLogRecord({}) + s = str(r) + self.assertTrue(s.startswith('<LogRecord: ')) + self.assertTrue(s.endswith('>')) + + def test_dict_arg(self): + h = RecordingHandler() + r = logging.getLogger() + r.addHandler(h) + d = {'less' : 'more' } + logging.warning('less is %(less)s', d) + self.assertIs(h.records[0].args, d) + self.assertEqual(h.records[0].message, 'less is more') + r.removeHandler(h) + h.close() + + def test_multiprocessing(self): + r = logging.makeLogRecord({}) + self.assertEqual(r.processName, 'MainProcess') + try: + import multiprocessing as mp + r = logging.makeLogRecord({}) + self.assertEqual(r.processName, mp.current_process().name) + except ImportError: + pass + + def test_optional(self): + r = logging.makeLogRecord({}) + NOT_NONE = self.assertIsNotNone + if threading: + NOT_NONE(r.thread) + NOT_NONE(r.threadName) + NOT_NONE(r.process) + NOT_NONE(r.processName) + log_threads = logging.logThreads + log_processes = logging.logProcesses + log_multiprocessing = logging.logMultiprocessing + try: + logging.logThreads = False + logging.logProcesses = False + logging.logMultiprocessing = False + r = logging.makeLogRecord({}) + NONE = self.assertIsNone + NONE(r.thread) + NONE(r.threadName) + NONE(r.process) + NONE(r.processName) + finally: + logging.logThreads = log_threads + logging.logProcesses = log_processes + logging.logMultiprocessing = log_multiprocessing + +class BasicConfigTest(unittest.TestCase): + + """Test suite for logging.basicConfig.""" + + def setUp(self): + super(BasicConfigTest, self).setUp() + self.handlers = logging.root.handlers + self.saved_handlers = logging._handlers.copy() + self.saved_handler_list = logging._handlerList[:] + self.original_logging_level = logging.root.level + self.addCleanup(self.cleanup) + logging.root.handlers = [] + + def tearDown(self): + for h in logging.root.handlers[:]: + logging.root.removeHandler(h) + h.close() + super(BasicConfigTest, self).tearDown() + + def cleanup(self): + setattr(logging.root, 'handlers', self.handlers) + logging._handlers.clear() + logging._handlers.update(self.saved_handlers) + logging._handlerList[:] = self.saved_handler_list + logging.root.level = self.original_logging_level + + def test_no_kwargs(self): + logging.basicConfig() + + # handler defaults to a StreamHandler to sys.stderr + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.StreamHandler) + self.assertEqual(handler.stream, sys.stderr) + + formatter = handler.formatter + # format defaults to logging.BASIC_FORMAT + self.assertEqual(formatter._style._fmt, logging.BASIC_FORMAT) + # datefmt defaults to None + self.assertIsNone(formatter.datefmt) + # style defaults to % + self.assertIsInstance(formatter._style, logging.PercentStyle) + + # level is not explicitly set + self.assertEqual(logging.root.level, self.original_logging_level) + + def test_filename(self): + logging.basicConfig(filename='test.log') + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.FileHandler) + + expected = logging.FileHandler('test.log', 'a') + self.addCleanup(expected.close) + self.assertEqual(handler.stream.mode, expected.stream.mode) + self.assertEqual(handler.stream.name, expected.stream.name) + + def test_filemode(self): + logging.basicConfig(filename='test.log', filemode='wb') + + handler = logging.root.handlers[0] + expected = logging.FileHandler('test.log', 'wb') + self.addCleanup(expected.close) + self.assertEqual(handler.stream.mode, expected.stream.mode) + + def test_stream(self): + stream = io.StringIO() + self.addCleanup(stream.close) + logging.basicConfig(stream=stream) + + self.assertEqual(len(logging.root.handlers), 1) + handler = logging.root.handlers[0] + self.assertIsInstance(handler, logging.StreamHandler) + self.assertEqual(handler.stream, stream) + + def test_format(self): + logging.basicConfig(format='foo') + + formatter = logging.root.handlers[0].formatter + self.assertEqual(formatter._style._fmt, 'foo') + + def test_datefmt(self): + logging.basicConfig(datefmt='bar') + + formatter = logging.root.handlers[0].formatter + self.assertEqual(formatter.datefmt, 'bar') + + def test_style(self): + logging.basicConfig(style='$') + + formatter = logging.root.handlers[0].formatter + self.assertIsInstance(formatter._style, logging.StringTemplateStyle) + + def test_level(self): + old_level = logging.root.level + self.addCleanup(logging.root.setLevel, old_level) + + logging.basicConfig(level=57) + self.assertEqual(logging.root.level, 57) + # Test that second call has no effect + logging.basicConfig(level=58) + self.assertEqual(logging.root.level, 57) + + def test_incompatible(self): + assertRaises = self.assertRaises + handlers = [logging.StreamHandler()] + stream = sys.stderr + assertRaises(ValueError, logging.basicConfig, filename='test.log', + stream=stream) + assertRaises(ValueError, logging.basicConfig, filename='test.log', + handlers=handlers) + assertRaises(ValueError, logging.basicConfig, stream=stream, + handlers=handlers) + + def test_handlers(self): + handlers = [ + logging.StreamHandler(), + logging.StreamHandler(sys.stdout), + logging.StreamHandler(), + ] + f = logging.Formatter() + handlers[2].setFormatter(f) + logging.basicConfig(handlers=handlers) + self.assertIs(handlers[0], logging.root.handlers[0]) + self.assertIs(handlers[1], logging.root.handlers[1]) + self.assertIs(handlers[2], logging.root.handlers[2]) + self.assertIsNotNone(handlers[0].formatter) + self.assertIsNotNone(handlers[1].formatter) + self.assertIs(handlers[2].formatter, f) + self.assertIs(handlers[0].formatter, handlers[1].formatter) + + def _test_log(self, method, level=None): + # logging.root has no handlers so basicConfig should be called + called = [] + + old_basic_config = logging.basicConfig + def my_basic_config(*a, **kw): + old_basic_config() + old_level = logging.root.level + logging.root.setLevel(100) # avoid having messages in stderr + self.addCleanup(logging.root.setLevel, old_level) + called.append((a, kw)) + + patch(self, logging, 'basicConfig', my_basic_config) + + log_method = getattr(logging, method) + if level is not None: + log_method(level, "test me") + else: + log_method("test me") + + # basicConfig was called with no arguments + self.assertEqual(called, [((), {})]) + + def test_log(self): + self._test_log('log', logging.WARNING) + + def test_debug(self): + self._test_log('debug') + + def test_info(self): + self._test_log('info') + + def test_warning(self): + self._test_log('warning') + + def test_error(self): + self._test_log('error') + + def test_critical(self): + self._test_log('critical') + + +class LoggerAdapterTest(unittest.TestCase): + + def setUp(self): + super(LoggerAdapterTest, self).setUp() + old_handler_list = logging._handlerList[:] + + self.recording = RecordingHandler() + self.logger = logging.root + self.logger.addHandler(self.recording) + self.addCleanup(self.logger.removeHandler, self.recording) + self.addCleanup(self.recording.close) + + def cleanup(): + logging._handlerList[:] = old_handler_list + + self.addCleanup(cleanup) + self.addCleanup(logging.shutdown) + self.adapter = logging.LoggerAdapter(logger=self.logger, extra=None) + + def test_exception(self): + msg = 'testing exception: %r' + exc = None + try: + 1 / 0 + except ZeroDivisionError as e: + exc = e + self.adapter.exception(msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.ERROR) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.exc_info, + (exc.__class__, exc, exc.__traceback__)) + + def test_critical(self): + msg = 'critical test! %r' + self.adapter.critical(msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.CRITICAL) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + + def test_is_enabled_for(self): + old_disable = self.adapter.logger.manager.disable + self.adapter.logger.manager.disable = 33 + self.addCleanup(setattr, self.adapter.logger.manager, 'disable', + old_disable) + self.assertFalse(self.adapter.isEnabledFor(32)) + + def test_has_handlers(self): + self.assertTrue(self.adapter.hasHandlers()) + + for handler in self.logger.handlers: + self.logger.removeHandler(handler) + + self.assertFalse(self.logger.hasHandlers()) + self.assertFalse(self.adapter.hasHandlers()) + + +class LoggerTest(BaseTest): + + def setUp(self): + super(LoggerTest, self).setUp() + self.recording = RecordingHandler() + self.logger = logging.Logger(name='blah') + self.logger.addHandler(self.recording) + self.addCleanup(self.logger.removeHandler, self.recording) + self.addCleanup(self.recording.close) + self.addCleanup(logging.shutdown) + + def test_set_invalid_level(self): + self.assertRaises(TypeError, self.logger.setLevel, object()) + + def test_exception(self): + msg = 'testing exception: %r' + exc = None + try: + 1 / 0 + except ZeroDivisionError as e: + exc = e + self.logger.exception(msg, self.recording) + + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertEqual(record.levelno, logging.ERROR) + self.assertEqual(record.msg, msg) + self.assertEqual(record.args, (self.recording,)) + self.assertEqual(record.exc_info, + (exc.__class__, exc, exc.__traceback__)) + + def test_log_invalid_level_with_raise(self): + old_raise = logging.raiseExceptions + self.addCleanup(setattr, logging, 'raiseExecptions', old_raise) + + logging.raiseExceptions = True + self.assertRaises(TypeError, self.logger.log, '10', 'test message') + + def test_log_invalid_level_no_raise(self): + old_raise = logging.raiseExceptions + self.addCleanup(setattr, logging, 'raiseExecptions', old_raise) + + logging.raiseExceptions = False + self.logger.log('10', 'test message') # no exception happens + + def test_find_caller_with_stack_info(self): + called = [] + patch(self, logging.traceback, 'print_stack', + lambda f, file: called.append(file.getvalue())) + + self.logger.findCaller(stack_info=True) + + self.assertEqual(len(called), 1) + self.assertEqual('Stack (most recent call last):\n', called[0]) + + def test_make_record_with_extra_overwrite(self): + name = 'my record' + level = 13 + fn = lno = msg = args = exc_info = func = sinfo = None + rv = logging._logRecordFactory(name, level, fn, lno, msg, args, + exc_info, func, sinfo) + + for key in ('message', 'asctime') + tuple(rv.__dict__.keys()): + extra = {key: 'some value'} + self.assertRaises(KeyError, self.logger.makeRecord, name, level, + fn, lno, msg, args, exc_info, + extra=extra, sinfo=sinfo) + + def test_make_record_with_extra_no_overwrite(self): + name = 'my record' + level = 13 + fn = lno = msg = args = exc_info = func = sinfo = None + extra = {'valid_key': 'some value'} + result = self.logger.makeRecord(name, level, fn, lno, msg, args, + exc_info, extra=extra, sinfo=sinfo) + self.assertIn('valid_key', result.__dict__) + + def test_has_handlers(self): + self.assertTrue(self.logger.hasHandlers()) + + for handler in self.logger.handlers: + self.logger.removeHandler(handler) + self.assertFalse(self.logger.hasHandlers()) + + def test_has_handlers_no_propagate(self): + child_logger = logging.getLogger('blah.child') + child_logger.propagate = False + self.assertFalse(child_logger.hasHandlers()) + + def test_is_enabled_for(self): + old_disable = self.logger.manager.disable + self.logger.manager.disable = 23 + self.addCleanup(setattr, self.logger.manager, 'disable', old_disable) + self.assertFalse(self.logger.isEnabledFor(22)) + + def test_root_logger_aliases(self): + root = logging.getLogger() + self.assertIs(root, logging.root) + self.assertIs(root, logging.getLogger(None)) + self.assertIs(root, logging.getLogger('')) + self.assertIs(root, logging.getLogger('foo').root) + self.assertIs(root, logging.getLogger('foo.bar').root) + self.assertIs(root, logging.getLogger('foo').parent) + + self.assertIsNot(root, logging.getLogger('\0')) + self.assertIsNot(root, logging.getLogger('foo.bar').parent) + + def test_invalid_names(self): + self.assertRaises(TypeError, logging.getLogger, any) + self.assertRaises(TypeError, logging.getLogger, b'foo') + + class BaseFileTest(BaseTest): "Base class for handler tests that write log files" @@ -2237,10 +3547,21 @@ class BaseFileTest(BaseTest): def assertLogFile(self, filename): "Assert a log file is there and register it for deletion" self.assertTrue(os.path.exists(filename), - msg="Log file %r does not exist") + msg="Log file %r does not exist" % filename) self.rmfiles.append(filename) +class FileHandlerTest(BaseFileTest): + def test_delay(self): + os.unlink(self.fn) + fh = logging.FileHandler(self.fn, delay=True) + self.assertIsNone(fh.stream) + self.assertFalse(os.path.exists(self.fn)) + fh.handle(logging.makeLogRecord({})) + self.assertIsNotNone(fh.stream) + self.assertTrue(os.path.exists(self.fn)) + fh.close() + class RotatingFileHandlerTest(BaseFileTest): def next_rec(self): return logging.LogRecord('n', logging.DEBUG, 'p', 1, @@ -2278,8 +3599,49 @@ class RotatingFileHandlerTest(BaseFileTest): rh.close() class TimedRotatingFileHandlerTest(BaseFileTest): - # test methods added below - pass + # other test methods added below + def test_rollover(self): + fh = logging.handlers.TimedRotatingFileHandler(self.fn, 'S', + backupCount=1) + r = logging.makeLogRecord({'msg': 'testing'}) + fh.emit(r) + self.assertLogFile(self.fn) + time.sleep(1.01) # just a little over a second ... + fh.emit(r) + fh.close() + # At this point, we should have a recent rotated file which we + # can test for the existence of. However, in practice, on some + # machines which run really slowly, we don't know how far back + # in time to go to look for the log file. So, we go back a fair + # bit, and stop as soon as we see a rotated file. In theory this + # could of course still fail, but the chances are lower. + found = False + now = datetime.datetime.now() + GO_BACK = 5 * 60 # seconds + for secs in range(GO_BACK): + prev = now - datetime.timedelta(seconds=secs) + fn = self.fn + prev.strftime(".%Y-%m-%d_%H-%M-%S") + found = os.path.exists(fn) + if found: + self.rmfiles.append(fn) + break + msg = 'No rotated files found, went back %d seconds' % GO_BACK + if not found: + #print additional diagnostics + dn, fn = os.path.split(self.fn) + files = [f for f in os.listdir(dn) if f.startswith(fn)] + print('Test time: %s' % now.strftime("%Y-%m-%d %H-%M-%S"), file=sys.stderr) + print('The only matching files are: %s' % files, file=sys.stderr) + self.assertTrue(found, msg=msg) + + def test_invalid(self): + assertRaises = self.assertRaises + assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, + self.fn, 'X', delay=True) + assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, + self.fn, 'W', delay=True) + assertRaises(ValueError, logging.handlers.TimedRotatingFileHandler, + self.fn, 'W7', delay=True) def secs(**kw): return datetime.timedelta(**kw) // datetime.timedelta(seconds=1) @@ -2328,19 +3690,51 @@ for when, exp in (('S', 1), rh.close() setattr(TimedRotatingFileHandlerTest, "test_compute_rollover_%s" % when, test_compute_rollover) + +@unittest.skipUnless(win32evtlog, 'win32evtlog/win32evtlogutil required for this test.') +class NTEventLogHandlerTest(BaseTest): + def test_basic(self): + logtype = 'Application' + elh = win32evtlog.OpenEventLog(None, logtype) + num_recs = win32evtlog.GetNumberOfEventLogRecords(elh) + h = logging.handlers.NTEventLogHandler('test_logging') + r = logging.makeLogRecord({'msg': 'Test Log Message'}) + h.handle(r) + h.close() + # Now see if the event is recorded + self.assertTrue(num_recs < win32evtlog.GetNumberOfEventLogRecords(elh)) + flags = win32evtlog.EVENTLOG_BACKWARDS_READ | \ + win32evtlog.EVENTLOG_SEQUENTIAL_READ + found = False + GO_BACK = 100 + events = win32evtlog.ReadEventLog(elh, flags, GO_BACK) + for e in events: + if e.SourceName != 'test_logging': + continue + msg = win32evtlogutil.SafeFormatMessage(e, logtype) + if msg != 'Test Log Message\r\n': + continue + found = True + break + msg = 'Record not found in event log, went back %d records' % GO_BACK + self.assertTrue(found, msg=msg) + # Set the locale to the platform-dependent default. I have no idea # why the test does this, but in any case we save the current locale # first and restore it at the end. @run_with_locale('LC_ALL', '') def test_main(): run_unittest(BuiltinLevelsTest, BasicFilterTest, - CustomLevelsAndFiltersTest, MemoryHandlerTest, - ConfigFileTest, SocketHandlerTest, MemoryTest, - EncodingTest, WarningsTest, ConfigDictTest, ManagerTest, - FormatterTest, - LogRecordFactoryTest, ChildLoggerTest, QueueHandlerTest, - RotatingFileHandlerTest, - LastResortTest, + CustomLevelsAndFiltersTest, HandlerTest, MemoryHandlerTest, + ConfigFileTest, SocketHandlerTest, DatagramHandlerTest, + MemoryTest, EncodingTest, WarningsTest, ConfigDictTest, + ManagerTest, FormatterTest, BufferingFormatterTest, + StreamHandlerTest, LogRecordFactoryTest, ChildLoggerTest, + QueueHandlerTest, ShutdownTest, ModuleLevelMiscTest, + BasicConfigTest, LoggerAdapterTest, LoggerTest, + SMTPHandlerTest, FileHandlerTest, RotatingFileHandlerTest, + LastResortTest, LogRecordTest, ExceptionTest, + SysLogHandlerTest, HTTPHandlerTest, NTEventLogHandlerTest, TimedRotatingFileHandlerTest ) diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index 04066ec..05b3e3e 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -43,6 +43,53 @@ DBL_MIN_EXP = sys.float_info.min_exp DBL_MANT_DIG = sys.float_info.mant_dig DBL_MIN_OVERFLOW = 2**DBL_MAX_EXP - 2**(DBL_MAX_EXP - DBL_MANT_DIG - 1) + +# Pure Python version of correctly-rounded integer-to-float conversion. +def int_to_float(n): + """ + Correctly-rounded integer-to-float conversion. + + """ + # Constants, depending only on the floating-point format in use. + # We use an extra 2 bits of precision for rounding purposes. + PRECISION = sys.float_info.mant_dig + 2 + SHIFT_MAX = sys.float_info.max_exp - PRECISION + Q_MAX = 1 << PRECISION + ROUND_HALF_TO_EVEN_CORRECTION = [0, -1, -2, 1, 0, -1, 2, 1] + + # Reduce to the case where n is positive. + if n == 0: + return 0.0 + elif n < 0: + return -int_to_float(-n) + + # Convert n to a 'floating-point' number q * 2**shift, where q is an + # integer with 'PRECISION' significant bits. When shifting n to create q, + # the least significant bit of q is treated as 'sticky'. That is, the + # least significant bit of q is set if either the corresponding bit of n + # was already set, or any one of the bits of n lost in the shift was set. + shift = n.bit_length() - PRECISION + q = n << -shift if shift < 0 else (n >> shift) | bool(n & ~(-1 << shift)) + + # Round half to even (actually rounds to the nearest multiple of 4, + # rounding ties to a multiple of 8). + q += ROUND_HALF_TO_EVEN_CORRECTION[q & 7] + + # Detect overflow. + if shift + (q == Q_MAX) > SHIFT_MAX: + raise OverflowError("integer too large to convert to float") + + # Checks: q is exactly representable, and q**2**shift doesn't overflow. + assert q % 4 == 0 and q // 4 <= 2**(sys.float_info.mant_dig) + assert q * 2**shift <= sys.float_info.max + + # Some circularity here, since float(q) is doing an int-to-float + # conversion. But here q is of bounded size, and is exactly representable + # as a float. In a low-level C-like language, this operation would be a + # simple cast (e.g., from unsigned long long to double). + return math.ldexp(float(q), shift) + + # pure Python version of correctly-rounded true division def truediv(a, b): """Correctly-rounded true division for integers.""" @@ -367,6 +414,23 @@ class LongTest(unittest.TestCase): return 1729 self.assertEqual(int(LongTrunc()), 1729) + def check_float_conversion(self, n): + # Check that int -> float conversion behaviour matches + # that of the pure Python version above. + try: + actual = float(n) + except OverflowError: + actual = 'overflow' + + try: + expected = int_to_float(n) + except OverflowError: + expected = 'overflow' + + msg = ("Error in conversion of integer {} to float. " + "Got {}, expected {}.".format(n, actual, expected)) + self.assertEqual(actual, expected, msg) + @support.requires_IEEE_754 def test_float_conversion(self): @@ -421,6 +485,22 @@ class LongTest(unittest.TestCase): y = 2**p * 2**53 self.assertEqual(int(float(x)), y) + # Compare builtin float conversion with pure Python int_to_float + # function above. + test_values = [ + int_dbl_max-1, int_dbl_max, int_dbl_max+1, + halfway-1, halfway, halfway + 1, + top_power-1, top_power, top_power+1, + 2*top_power-1, 2*top_power, top_power*top_power, + ] + test_values.extend(exact_values) + for p in range(-4, 8): + for x in range(-128, 128): + test_values.append(2**(p+53) + x) + for value in test_values: + self.check_float_conversion(value) + self.check_float_conversion(-value) + def test_float_overflow(self): for x in -2.0, -1.0, 0.0, 1.0, 2.0: self.assertEqual(float(int(x)), x) diff --git a/Lib/test/test_mailbox.py b/Lib/test/test_mailbox.py index 007c419..a1c684e 100644 --- a/Lib/test/test_mailbox.py +++ b/Lib/test/test_mailbox.py @@ -95,14 +95,14 @@ class TestMailbox(TestBase): """) def test_add_invalid_8bit_bytes_header(self): - key = self._box.add(self._nonascii_msg.encode('latin1')) + key = self._box.add(self._nonascii_msg.encode('latin-1')) self.assertEqual(len(self._box), 1) self.assertEqual(self._box.get_bytes(key), - self._nonascii_msg.encode('latin1')) + self._nonascii_msg.encode('latin-1')) def test_invalid_nonascii_header_as_string(self): subj = self._nonascii_msg.splitlines()[1] - key = self._box.add(subj.encode('latin1')) + key = self._box.add(subj.encode('latin-1')) self.assertEqual(self._box.get_string(key), 'Subject: =?unknown-8bit?b?RmFsaW5hcHThciBo4Xpob3pzeuFsbO104XNz' 'YWwuIE3hciByZW5kZWx06Ww/?=\n\n') @@ -929,8 +929,7 @@ class TestMaildir(TestMailbox): # the mtime and should cause a re-read. Note that "sleep # emulation" is still in effect, as skewfactor is -3. filename = os.path.join(self._path, 'cur', 'stray-file') - f = open(filename, 'w') - f.close() + support.create_empty_file(filename) os.unlink(filename) self._box._refresh() self.assertTrue(refreshed()) diff --git a/Lib/test/test_mailcap.py b/Lib/test/test_mailcap.py new file mode 100644 index 0000000..a4cd09c --- /dev/null +++ b/Lib/test/test_mailcap.py @@ -0,0 +1,221 @@ +import mailcap +import os +import shutil +import test.support +import unittest + +# Location of mailcap file +MAILCAPFILE = test.support.findfile("mailcap.txt") + +# Dict to act as mock mailcap entry for this test +# The keys and values should match the contents of MAILCAPFILE +MAILCAPDICT = { + 'application/x-movie': + [{'compose': 'moviemaker %s', + 'x11-bitmap': '"/usr/lib/Zmail/bitmaps/movie.xbm"', + 'description': '"Movie"', + 'view': 'movieplayer %s'}], + 'application/*': + [{'copiousoutput': '', + 'view': 'echo "This is \\"%t\\" but is 50 \\% Greek to me" \\; cat %s'}], + 'audio/basic': + [{'edit': 'audiocompose %s', + 'compose': 'audiocompose %s', + 'description': '"An audio fragment"', + 'view': 'showaudio %s'}], + 'video/mpeg': + [{'view': 'mpeg_play %s'}], + 'application/postscript': + [{'needsterminal': '', 'view': 'ps-to-terminal %s'}, + {'compose': 'idraw %s', 'view': 'ps-to-terminal %s'}], + 'application/x-dvi': + [{'view': 'xdvi %s'}], + 'message/external-body': + [{'composetyped': 'extcompose %s', + 'description': '"A reference to data stored in an external location"', + 'needsterminal': '', + 'view': 'showexternal %s %{access-type} %{name} %{site} %{directory} %{mode} %{server}'}], + 'text/richtext': + [{'test': 'test "`echo %{charset} | tr \'[A-Z]\' \'[a-z]\'`" = iso-8859-8', + 'copiousoutput': '', + 'view': 'shownonascii iso-8859-8 -e richtext -p %s'}], + 'image/x-xwindowdump': + [{'view': 'display %s'}], + 'audio/*': + [{'view': '/usr/local/bin/showaudio %t'}], + 'video/*': + [{'view': 'animate %s'}], + 'application/frame': + [{'print': '"cat %s | lp"', 'view': 'showframe %s'}], + 'image/rgb': + [{'view': 'display %s'}] +} + + +class HelperFunctionTest(unittest.TestCase): + + def test_listmailcapfiles(self): + # The return value for listmailcapfiles() will vary by system. + # So verify that listmailcapfiles() returns a list of strings that is of + # non-zero length. + mcfiles = mailcap.listmailcapfiles() + self.assertIsInstance(mcfiles, list) + for m in mcfiles: + self.assertIsInstance(m, str) + with test.support.EnvironmentVarGuard() as env: + # According to RFC 1524, if MAILCAPS env variable exists, use that + # and only that. + if "MAILCAPS" in env: + env_mailcaps = env["MAILCAPS"].split(os.pathsep) + else: + env_mailcaps = ["/testdir1/.mailcap", "/testdir2/mailcap"] + env["MAILCAPS"] = os.pathsep.join(env_mailcaps) + mcfiles = mailcap.listmailcapfiles() + self.assertEqual(env_mailcaps, mcfiles) + + def test_readmailcapfile(self): + # Test readmailcapfile() using test file. It should match MAILCAPDICT. + with open(MAILCAPFILE, 'r') as mcf: + d = mailcap.readmailcapfile(mcf) + self.assertDictEqual(d, MAILCAPDICT) + + def test_lookup(self): + # Test without key + expected = [{'view': 'mpeg_play %s'}, {'view': 'animate %s'}] + actual = mailcap.lookup(MAILCAPDICT, 'video/mpeg') + self.assertListEqual(expected, actual) + + # Test with key + key = 'compose' + expected = [{'edit': 'audiocompose %s', + 'compose': 'audiocompose %s', + 'description': '"An audio fragment"', + 'view': 'showaudio %s'}] + actual = mailcap.lookup(MAILCAPDICT, 'audio/basic', key) + self.assertListEqual(expected, actual) + + def test_subst(self): + plist = ['id=1', 'number=2', 'total=3'] + # test case: ([field, MIMEtype, filename, plist=[]], <expected string>) + test_cases = [ + (["", "audio/*", "foo.txt"], ""), + (["echo foo", "audio/*", "foo.txt"], "echo foo"), + (["echo %s", "audio/*", "foo.txt"], "echo foo.txt"), + (["echo %t", "audio/*", "foo.txt"], "echo audio/*"), + (["echo \%t", "audio/*", "foo.txt"], "echo %t"), + (["echo foo", "audio/*", "foo.txt", plist], "echo foo"), + (["echo %{total}", "audio/*", "foo.txt", plist], "echo 3") + ] + for tc in test_cases: + self.assertEqual(mailcap.subst(*tc[0]), tc[1]) + + +class GetcapsTest(unittest.TestCase): + + def test_mock_getcaps(self): + # Test mailcap.getcaps() using mock mailcap file in this dir. + # Temporarily override any existing system mailcap file by pointing the + # MAILCAPS environment variable to our mock file. + with test.support.EnvironmentVarGuard() as env: + env["MAILCAPS"] = MAILCAPFILE + caps = mailcap.getcaps() + self.assertDictEqual(caps, MAILCAPDICT) + + def test_system_mailcap(self): + # Test mailcap.getcaps() with mailcap file(s) on system, if any. + caps = mailcap.getcaps() + self.assertIsInstance(caps, dict) + mailcapfiles = mailcap.listmailcapfiles() + existingmcfiles = [mcf for mcf in mailcapfiles if os.path.exists(mcf)] + if existingmcfiles: + # At least 1 mailcap file exists, so test that. + for (k, v) in caps.items(): + self.assertIsInstance(k, str) + self.assertIsInstance(v, list) + for e in v: + self.assertIsInstance(e, dict) + else: + # No mailcap files on system. getcaps() should return empty dict. + self.assertEqual({}, caps) + + +class FindmatchTest(unittest.TestCase): + + def test_findmatch(self): + + # default findmatch arguments + c = MAILCAPDICT + fname = "foo.txt" + plist = ["access-type=default", "name=john", "site=python.org", + "directory=/tmp", "mode=foo", "server=bar"] + audio_basic_entry = { + 'edit': 'audiocompose %s', + 'compose': 'audiocompose %s', + 'description': '"An audio fragment"', + 'view': 'showaudio %s' + } + audio_entry = {"view": "/usr/local/bin/showaudio %t"} + video_entry = {'view': 'animate %s'} + message_entry = { + 'composetyped': 'extcompose %s', + 'description': '"A reference to data stored in an external location"', 'needsterminal': '', + 'view': 'showexternal %s %{access-type} %{name} %{site} %{directory} %{mode} %{server}' + } + + # test case: (findmatch args, findmatch keyword args, expected output) + # positional args: caps, MIMEtype + # keyword args: key="view", filename="/dev/null", plist=[] + # output: (command line, mailcap entry) + cases = [ + ([{}, "video/mpeg"], {}, (None, None)), + ([c, "foo/bar"], {}, (None, None)), + ([c, "video/mpeg"], {}, ('mpeg_play /dev/null', {'view': 'mpeg_play %s'})), + ([c, "audio/basic", "edit"], {}, ("audiocompose /dev/null", audio_basic_entry)), + ([c, "audio/basic", "compose"], {}, ("audiocompose /dev/null", audio_basic_entry)), + ([c, "audio/basic", "description"], {}, ('"An audio fragment"', audio_basic_entry)), + ([c, "audio/basic", "foobar"], {}, (None, None)), + ([c, "video/*"], {"filename": fname}, ("animate %s" % fname, video_entry)), + ([c, "audio/basic", "compose"], + {"filename": fname}, + ("audiocompose %s" % fname, audio_basic_entry)), + ([c, "audio/basic"], + {"key": "description", "filename": fname}, + ('"An audio fragment"', audio_basic_entry)), + ([c, "audio/*"], + {"filename": fname}, + ("/usr/local/bin/showaudio audio/*", audio_entry)), + ([c, "message/external-body"], + {"plist": plist}, + ("showexternal /dev/null default john python.org /tmp foo bar", message_entry)) + ] + self._run_cases(cases) + + @unittest.skipUnless(os.name == "posix", "Requires 'test' command on system") + def test_test(self): + # findmatch() will automatically check any "test" conditions and skip + # the entry if the check fails. + caps = {"test/pass": [{"test": "test 1 -eq 1"}], + "test/fail": [{"test": "test 1 -eq 0"}]} + # test case: (findmatch args, findmatch keyword args, expected output) + # positional args: caps, MIMEtype, key ("test") + # keyword args: N/A + # output: (command line, mailcap entry) + cases = [ + # findmatch will return the mailcap entry for test/pass because it evaluates to true + ([caps, "test/pass", "test"], {}, ("test 1 -eq 1", {"test": "test 1 -eq 1"})), + # findmatch will return None because test/fail evaluates to false + ([caps, "test/fail", "test"], {}, (None, None)) + ] + self._run_cases(cases) + + def _run_cases(self, cases): + for c in cases: + self.assertEqual(mailcap.findmatch(*c[0], **c[1]), c[2]) + + +def test_main(): + test.support.run_unittest(HelperFunctionTest, GetcapsTest, FindmatchTest) + + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_marshal.py b/Lib/test/test_marshal.py index cd100f9..dec8129 100644 --- a/Lib/test/test_marshal.py +++ b/Lib/test/test_marshal.py @@ -5,6 +5,7 @@ import marshal import sys import unittest import os +import types class HelperMixin: def helper(self, sample, *extra): @@ -113,6 +114,22 @@ class CodeTestCase(unittest.TestCase): codes = (ExceptionTestCase.test_exceptions.__code__,) * count marshal.loads(marshal.dumps(codes)) + def test_different_filenames(self): + co1 = compile("x", "f1", "exec") + co2 = compile("y", "f2", "exec") + co1, co2 = marshal.loads(marshal.dumps((co1, co2))) + self.assertEqual(co1.co_filename, "f1") + self.assertEqual(co2.co_filename, "f2") + + @support.cpython_only + def test_same_filename_used(self): + s = """def f(): pass\ndef g(): pass""" + co = compile(s, "myfile", "exec") + co = marshal.loads(marshal.dumps(co)) + for obj in co.co_consts: + if isinstance(obj, types.CodeType): + self.assertIs(co.co_filename, obj.co_filename) + class ContainerTestCase(unittest.TestCase, HelperMixin): d = {'astring': 'foo@bar.baz.spam', 'afloat': 7283.43, diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index dddc889..715003a 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -2,11 +2,12 @@ # XXXX Should not do tests around zero only from test.support import run_unittest, verbose, requires_IEEE_754 +from test import support import unittest import math import os +import platform import sys -import random import struct import sysconfig @@ -457,12 +458,12 @@ class MathTests(unittest.TestCase): def testFmod(self): self.assertRaises(TypeError, math.fmod) - self.ftest('fmod(10,1)', math.fmod(10,1), 0) - self.ftest('fmod(10,0.5)', math.fmod(10,0.5), 0) - self.ftest('fmod(10,1.5)', math.fmod(10,1.5), 1) - self.ftest('fmod(-10,1)', math.fmod(-10,1), 0) - self.ftest('fmod(-10,0.5)', math.fmod(-10,0.5), 0) - self.ftest('fmod(-10,1.5)', math.fmod(-10,1.5), -1) + self.ftest('fmod(10, 1)', math.fmod(10, 1), 0.0) + self.ftest('fmod(10, 0.5)', math.fmod(10, 0.5), 0.0) + self.ftest('fmod(10, 1.5)', math.fmod(10, 1.5), 1.0) + self.ftest('fmod(-10, 1)', math.fmod(-10, 1), -0.0) + self.ftest('fmod(-10, 0.5)', math.fmod(-10, 0.5), -0.0) + self.ftest('fmod(-10, 1.5)', math.fmod(-10, 1.5), -1.0) self.assertTrue(math.isnan(math.fmod(NAN, 1.))) self.assertTrue(math.isnan(math.fmod(1., NAN))) self.assertTrue(math.isnan(math.fmod(NAN, NAN))) @@ -650,6 +651,33 @@ class MathTests(unittest.TestCase): n= 2**90 self.assertAlmostEqual(math.log1p(n), math.log1p(float(n))) + @requires_IEEE_754 + def testLog2(self): + self.assertRaises(TypeError, math.log2) + + # Check some integer values + self.assertEqual(math.log2(1), 0.0) + self.assertEqual(math.log2(2), 1.0) + self.assertEqual(math.log2(4), 2.0) + + # Large integer values + self.assertEqual(math.log2(2**1023), 1023.0) + self.assertEqual(math.log2(2**1024), 1024.0) + self.assertEqual(math.log2(2**2000), 2000.0) + + self.assertRaises(ValueError, math.log2, -1.5) + self.assertRaises(ValueError, math.log2, NINF) + self.assertTrue(math.isnan(math.log2(NAN))) + + @requires_IEEE_754 + # log2() is not accurate enough on Mac OS X Tiger (10.4) + @support.requires_mac_ver(10, 5) + def testLog2Exact(self): + # Check that we get exact equality for log2 of powers of 2. + actual = [math.log2(math.ldexp(1.0, n)) for n in range(-1074, 1024)] + expected = [float(n) for n in range(-1074, 1024)] + self.assertEqual(actual, expected) + def testLog10(self): self.assertRaises(TypeError, math.log10) self.ftest('log10(0.1)', math.log10(0.1), -1) @@ -1010,7 +1038,6 @@ class MathTests(unittest.TestCase): @requires_IEEE_754 def test_mtestfile(self): - ALLOWED_ERROR = 20 # permitted error, in ulps fail_fmt = "{}:{}({!r}): expected {!r}, got {!r}" failures = [] diff --git a/Lib/test/test_metaclass.py b/Lib/test/test_metaclass.py index 219ab99..6862900 100644 --- a/Lib/test/test_metaclass.py +++ b/Lib/test/test_metaclass.py @@ -246,7 +246,13 @@ Test failures in looking up the __prepare__ method work. """ -__test__ = {'doctests' : doctests} +import sys + +# Trace function introduces __locals__ which causes various tests to fail. +if hasattr(sys, 'gettrace') and sys.gettrace(): + __test__ = {} +else: + __test__ = {'doctests' : doctests} def test_main(verbose=False): from test import support diff --git a/Lib/test/test_minidom.py b/Lib/test/test_minidom.py index f3fa1b8..392e40e 100644 --- a/Lib/test/test_minidom.py +++ b/Lib/test/test_minidom.py @@ -4,9 +4,7 @@ import pickle from test.support import verbose, run_unittest, findfile import unittest -import xml.dom import xml.dom.minidom -import xml.parsers.expat from xml.dom.minidom import parse, Node, Document, parseString from xml.dom.minidom import getDOMImplementation @@ -14,7 +12,6 @@ from xml.dom.minidom import getDOMImplementation tstfile = findfile("test.xml", subdir="xmltestdata") - # The tests of DocumentType importing use these helpers to construct # the documents to work with, since not all DOM builders actually # create the DocumentType nodes. @@ -282,6 +279,7 @@ class MinidomTest(unittest.TestCase): child.setAttribute("def", "ghi") self.confirm(len(child.attributes) == 1) + self.assertRaises(xml.dom.NotFoundErr, child.removeAttribute, "foo") child.removeAttribute("def") self.confirm(len(child.attributes) == 0) dom.unlink() @@ -293,6 +291,8 @@ class MinidomTest(unittest.TestCase): child.setAttributeNS("http://www.w3.org", "xmlns:python", "http://www.python.org") child.setAttributeNS("http://www.python.org", "python:abcattr", "foo") + self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNS, + "foo", "http://www.python.org") self.confirm(len(child.attributes) == 2) child.removeAttributeNS("http://www.python.org", "abcattr") self.confirm(len(child.attributes) == 1) @@ -304,11 +304,23 @@ class MinidomTest(unittest.TestCase): child.setAttribute("spam", "jam") self.confirm(len(child.attributes) == 1) node = child.getAttributeNode("spam") + self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNode, + None) child.removeAttributeNode(node) self.confirm(len(child.attributes) == 0 and child.getAttributeNode("spam") is None) + dom2 = Document() + child2 = dom2.appendChild(dom.createElement("foo")) + self.assertRaises(xml.dom.NotFoundErr, child.removeAttributeNode, + node) dom.unlink() + def testHasAttribute(self): + dom = Document() + child = dom.appendChild(dom.createElement("foo")) + child.setAttribute("spam", "jam") + self.confirm(child.hasAttribute("spam")) + def testChangeAttr(self): dom = parseString("<abc/>") el = dom.documentElement @@ -356,7 +368,16 @@ class MinidomTest(unittest.TestCase): def testGetAttribute(self): pass - def testGetAttributeNS(self): pass + def testGetAttributeNS(self): + dom = Document() + child = dom.appendChild( + dom.createElementNS("http://www.python.org", "python:abc")) + child.setAttributeNS("http://www.w3.org", "xmlns:python", + "http://www.python.org") + self.assertEqual(child.getAttributeNS("http://www.w3.org", "python"), + 'http://www.python.org') + self.assertEqual(child.getAttributeNS("http://www.w3.org", "other"), + '') def testGetAttributeNode(self): pass @@ -574,7 +595,13 @@ class MinidomTest(unittest.TestCase): def testFirstChild(self): pass - def testHasChildNodes(self): pass + def testHasChildNodes(self): + dom = parseString("<doc><foo/></doc>") + doc = dom.documentElement + self.assertTrue(dom.hasChildNodes()) + dom2 = parseString("<doc/>") + doc2 = dom2.documentElement + self.assertFalse(doc2.hasChildNodes()) def _testCloneElementCopiesAttributes(self, e1, e2, test): attrs1 = e1.attributes @@ -1023,41 +1050,6 @@ class MinidomTest(unittest.TestCase): "test NodeList.item()") doc.unlink() - def testSAX2DOM(self): - from xml.dom import pulldom - - sax2dom = pulldom.SAX2DOM() - sax2dom.startDocument() - sax2dom.startElement("doc", {}) - sax2dom.characters("text") - sax2dom.startElement("subelm", {}) - sax2dom.characters("text") - sax2dom.endElement("subelm") - sax2dom.characters("text") - sax2dom.endElement("doc") - sax2dom.endDocument() - - doc = sax2dom.document - root = doc.documentElement - (text1, elm1, text2) = root.childNodes - text3 = elm1.childNodes[0] - - self.confirm(text1.previousSibling is None and - text1.nextSibling is elm1 and - elm1.previousSibling is text1 and - elm1.nextSibling is text2 and - text2.previousSibling is elm1 and - text2.nextSibling is None and - text3.previousSibling is None and - text3.nextSibling is None, "testSAX2DOM - siblings") - - self.confirm(root.parentNode is doc and - text1.parentNode is root and - elm1.parentNode is root and - text2.parentNode is root and - text3.parentNode is elm1, "testSAX2DOM - parents") - doc.unlink() - def testEncodings(self): doc = parseString('<foo>€</foo>') self.assertEqual(doc.toxml(), @@ -1504,12 +1496,21 @@ class MinidomTest(unittest.TestCase): doc.appendChild(doc.createComment("foo--bar")) self.assertRaises(ValueError, doc.toxml) + def testEmptyXMLNSValue(self): doc = parseString("<element xmlns=''>\n" "<foo/>\n</element>") doc2 = parseString(doc.toxml()) self.confirm(doc2.namespaceURI == xml.dom.EMPTY_NAMESPACE) + def testDocRemoveChild(self): + doc = parse(tstfile) + title_tag = doc.documentElement.getElementsByTagName("TITLE")[0] + self.assertRaises( xml.dom.NotFoundErr, doc.removeChild, title_tag) + num_children_before = len(doc.childNodes) + doc.removeChild(doc.childNodes[0]) + num_children_after = len(doc.childNodes) + self.assertTrue(num_children_after == num_children_before - 1) def test_main(): run_unittest(MinidomTest) diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index 0e18aab..2230028 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -417,6 +417,35 @@ class MmapTests(unittest.TestCase): m[x] = b self.assertEqual(m[x], b) + def test_read_all(self): + m = mmap.mmap(-1, 16) + self.addCleanup(m.close) + + # With no parameters, or None or a negative argument, reads all + m.write(bytes(range(16))) + m.seek(0) + self.assertEqual(m.read(), bytes(range(16))) + m.seek(8) + self.assertEqual(m.read(), bytes(range(8, 16))) + m.seek(16) + self.assertEqual(m.read(), b'') + m.seek(3) + self.assertEqual(m.read(None), bytes(range(3, 16))) + m.seek(4) + self.assertEqual(m.read(-1), bytes(range(4, 16))) + m.seek(5) + self.assertEqual(m.read(-2), bytes(range(5, 16))) + m.seek(9) + self.assertEqual(m.read(-42), bytes(range(9, 16))) + + def test_read_invalid_arg(self): + m = mmap.mmap(-1, 16) + self.addCleanup(m.close) + + self.assertRaises(TypeError, m.read, 'foo') + self.assertRaises(TypeError, m.read, 5.5) + self.assertRaises(TypeError, m.read, [1, 2, 3]) + def test_extended_getslice(self): # Test extended slicing by comparing with list slicing. s = bytes(reversed(range(256))) @@ -534,8 +563,7 @@ class MmapTests(unittest.TestCase): f.close() def test_error(self): - self.assertTrue(issubclass(mmap.error, EnvironmentError)) - self.assertIn("mmap.error", str(mmap.error)) + self.assertIs(mmap.error, OSError) def test_io_methods(self): data = b"0123456789" diff --git a/Lib/test/test_modulefinder.py b/Lib/test/test_modulefinder.py index a184217..c5fc320 100644 --- a/Lib/test/test_modulefinder.py +++ b/Lib/test/test_modulefinder.py @@ -1,7 +1,7 @@ -import __future__ import os +import errno +import shutil import unittest -import distutils.dir_util import tempfile from test import support @@ -9,7 +9,7 @@ from test import support import modulefinder TEST_DIR = tempfile.mkdtemp() -TEST_PATH = [TEST_DIR, os.path.dirname(__future__.__file__)] +TEST_PATH = [TEST_DIR, os.path.dirname(tempfile.__file__)] # Each test description is a list of 5 items: # @@ -196,12 +196,17 @@ a/module.py from . import bar """] + def open_file(path): - ##print "#", os.path.abspath(path) dirname = os.path.dirname(path) - distutils.dir_util.mkpath(dirname) + try: + os.makedirs(dirname) + except OSError as e: + if e.errno != errno.EEXIST: + raise return open(path, "w") + def create_package(source): ofi = None try: @@ -216,6 +221,7 @@ def create_package(source): if ofi: ofi.close() + class ModuleFinderTest(unittest.TestCase): def _do_test(self, info, report=False): import_this, modules, missing, maybe_missing, source = info @@ -234,19 +240,17 @@ class ModuleFinderTest(unittest.TestCase): ## import traceback; traceback.print_exc() ## sys.path = opath ## return - modules = set(modules) - found = set(mf.modules.keys()) - more = list(found - modules) - less = list(modules - found) + modules = sorted(set(modules)) + found = sorted(mf.modules) # check if we found what we expected, not more, not less - self.assertEqual((more, less), ([], [])) + self.assertEqual(found, modules) # check for missing and maybe missing modules bad, maybe = mf.any_missing_maybe() self.assertEqual(bad, missing) self.assertEqual(maybe, maybe_missing) finally: - distutils.dir_util.remove_tree(TEST_DIR) + shutil.rmtree(TEST_DIR) def test_package(self): self._do_test(package_test) @@ -254,25 +258,23 @@ class ModuleFinderTest(unittest.TestCase): def test_maybe(self): self._do_test(maybe_test) - if getattr(__future__, "absolute_import", None): + def test_maybe_new(self): + self._do_test(maybe_test_new) - def test_maybe_new(self): - self._do_test(maybe_test_new) + def test_absolute_imports(self): + self._do_test(absolute_import_test) - def test_absolute_imports(self): - self._do_test(absolute_import_test) + def test_relative_imports(self): + self._do_test(relative_import_test) - def test_relative_imports(self): - self._do_test(relative_import_test) + def test_relative_imports_2(self): + self._do_test(relative_import_test_2) - def test_relative_imports_2(self): - self._do_test(relative_import_test_2) + def test_relative_imports_3(self): + self._do_test(relative_import_test_3) - def test_relative_imports_3(self): - self._do_test(relative_import_test_3) def test_main(): - distutils.log.set_threshold(distutils.log.WARN) support.run_unittest(ModuleFinderTest) if __name__ == "__main__": diff --git a/Lib/test/test_multibytecodec.py b/Lib/test/test_multibytecodec.py index 86c68dc..feb7bd5 100644 --- a/Lib/test/test_multibytecodec.py +++ b/Lib/test/test_multibytecodec.py @@ -247,20 +247,16 @@ class Test_ISO2022(unittest.TestCase): self.assertFalse(any(x > 0x80 for x in e)) def test_bug1572832(self): - if sys.maxunicode >= 0x10000: - myunichr = chr - else: - myunichr = lambda x: chr(0xD7C0+(x>>10)) + chr(0xDC00+(x&0x3FF)) - for x in range(0x10000, 0x110000): # Any ISO 2022 codec will cause the segfault - myunichr(x).encode('iso_2022_jp', 'ignore') + chr(x).encode('iso_2022_jp', 'ignore') class TestStateful(unittest.TestCase): text = '\u4E16\u4E16' encoding = 'iso-2022-jp' expected = b'\x1b$B@$@$' - expected_reset = b'\x1b$B@$@$\x1b(B' + reset = b'\x1b(B' + expected_reset = expected + reset def test_encode(self): self.assertEqual(self.text.encode(self.encoding), self.expected_reset) @@ -271,6 +267,8 @@ class TestStateful(unittest.TestCase): encoder.encode(char) for char in self.text) self.assertEqual(output, self.expected) + self.assertEqual(encoder.encode('', final=True), self.reset) + self.assertEqual(encoder.encode('', final=True), b'') def test_incrementalencoder_final(self): encoder = codecs.getincrementalencoder(self.encoding)() @@ -279,12 +277,14 @@ class TestStateful(unittest.TestCase): encoder.encode(char, index == last_index) for index, char in enumerate(self.text)) self.assertEqual(output, self.expected_reset) + self.assertEqual(encoder.encode('', final=True), b'') class TestHZStateful(TestStateful): text = '\u804a\u804a' encoding = 'hz' expected = b'~{ADAD' - expected_reset = b'~{ADAD~}' + reset = b'~}' + expected_reset = expected + reset def test_main(): support.run_unittest(__name__) diff --git a/Lib/test/test_multibytecodec_support.py b/Lib/test/test_multibytecodec_support.py index ef63b69..ac3b92b 100644 --- a/Lib/test/test_multibytecodec_support.py +++ b/Lib/test/test_multibytecodec_support.py @@ -264,21 +264,6 @@ class TestBase: self.assertEqual(ostream.getvalue(), self.tstring[0]) -if len('\U00012345') == 2: # ucs2 build - _unichr = chr - def chr(v): - if v >= 0x10000: - return _unichr(0xd800 + ((v - 0x10000) >> 10)) + \ - _unichr(0xdc00 + ((v - 0x10000) & 0x3ff)) - else: - return _unichr(v) - _ord = ord - def ord(c): - if len(c) == 2: - return 0x10000 + ((_ord(c[0]) - 0xd800) << 10) + \ - (ord(c[1]) - 0xdc00) - else: - return _ord(c) class TestBase_Mapping(unittest.TestCase): pass_enctest = [] diff --git a/Lib/test/test_multiprocessing.py b/Lib/test/test_multiprocessing.py index 45bf454..0bc056f 100644 --- a/Lib/test/test_multiprocessing.py +++ b/Lib/test/test_multiprocessing.py @@ -82,6 +82,23 @@ HAVE_GETVALUE = not getattr(_multiprocessing, 'HAVE_BROKEN_SEM_GETVALUE', False) WIN32 = (sys.platform == "win32") +if WIN32: + from _subprocess import WaitForSingleObject, INFINITE, WAIT_OBJECT_0 + + def wait_for_handle(handle, timeout): + if timeout is None or timeout < 0.0: + timeout = INFINITE + else: + timeout = int(1000 * timeout) + return WaitForSingleObject(handle, timeout) == WAIT_OBJECT_0 +else: + from select import select + _select = util._eintr_retry(select) + + def wait_for_handle(handle, timeout): + if timeout is not None and timeout < 0.0: + timeout = None + return handle in _select([handle], [], [], timeout)[0] try: MAXFD = os.sysconf("SC_OPEN_MAX") @@ -180,6 +197,18 @@ class _TestProcess(BaseTestCase): self.assertEqual(current.ident, os.getpid()) self.assertEqual(current.exitcode, None) + def test_daemon_argument(self): + if self.TYPE == "threads": + return + + # By default uses the current process's daemon flag. + proc0 = self.Process(target=self._test) + self.assertEqual(proc0.daemon, self.current_process().daemon) + proc1 = self.Process(target=self._test, daemon=True) + self.assertTrue(proc1.daemon) + proc2 = self.Process(target=self._test, daemon=False) + self.assertFalse(proc2.daemon) + @classmethod def _test(cls, q, *args, **kwds): current = cls.current_process() @@ -312,6 +341,26 @@ class _TestProcess(BaseTestCase): ] self.assertEqual(result, expected) + @classmethod + def _test_sentinel(cls, event): + event.wait(10.0) + + def test_sentinel(self): + if self.TYPE == "threads": + return + event = self.Event() + p = self.Process(target=self._test_sentinel, args=(event,)) + with self.assertRaises(ValueError): + p.sentinel + p.start() + self.addCleanup(p.join) + sentinel = p.sentinel + self.assertIsInstance(sentinel, int) + self.assertFalse(wait_for_handle(sentinel, timeout=0.0)) + event.set() + p.join() + self.assertTrue(wait_for_handle(sentinel, timeout=DELTA)) + # # # @@ -2075,9 +2124,15 @@ class TestInvalidHandle(unittest.TestCase): @unittest.skipIf(WIN32, "skipped on Windows") def test_invalid_handles(self): - conn = _multiprocessing.Connection(44977608) - self.assertRaises(IOError, conn.poll) - self.assertRaises(IOError, _multiprocessing.Connection, -1) + conn = multiprocessing.connection.Connection(44977608) + try: + self.assertRaises((ValueError, IOError), conn.poll) + finally: + # Hack private attribute _handle to avoid printing an error + # in conn.__del__ + conn._handle = None + self.assertRaises((ValueError, IOError), + multiprocessing.connection.Connection, -1) # # Functions used to create test cases from the base ones in this module diff --git a/Lib/test/test_nntplib.py b/Lib/test/test_nntplib.py index 786b5f0..3141389 100644 --- a/Lib/test/test_nntplib.py +++ b/Lib/test/test_nntplib.py @@ -1,4 +1,5 @@ import io +import socket import datetime import textwrap import unittest @@ -251,6 +252,26 @@ class NetworkedNNTPTestsMixin: # value setattr(cls, name, wrap_meth(meth)) + def test_with_statement(self): + def is_connected(): + if not hasattr(server, 'file'): + return False + try: + server.help() + except (socket.error, EOFError): + return False + return True + + with self.NNTP_CLASS(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) as server: + self.assertTrue(is_connected()) + self.assertTrue(server.help()) + self.assertFalse(is_connected()) + + with self.NNTP_CLASS(self.NNTP_HOST, timeout=TIMEOUT, usenetrc=False) as server: + server.quit() + self.assertFalse(is_connected()) + + NetworkedNNTPTestsMixin.wrap_methods() @@ -812,7 +833,7 @@ class NNTPv1v2TestsMixin: def _check_article_body(self, lines): self.assertEqual(len(lines), 4) - self.assertEqual(lines[-1].decode('utf8'), "-- Signed by André.") + self.assertEqual(lines[-1].decode('utf-8'), "-- Signed by André.") self.assertEqual(lines[-2], b"") self.assertEqual(lines[-3], b".Here is a dot-starting line.") self.assertEqual(lines[-4], b"This is just a test article.") @@ -1011,12 +1032,12 @@ class NNTPv1v2TestsMixin: self.assertEqual(resp, success_resp) # With an iterable of terminated lines def iterlines(b): - return iter(b.splitlines(True)) + return iter(b.splitlines(keepends=True)) resp = self._check_post_ihave_sub(func, *args, file_factory=iterlines) self.assertEqual(resp, success_resp) # With an iterable of non-terminated lines def iterlines(b): - return iter(b.splitlines(False)) + return iter(b.splitlines(keepends=False)) resp = self._check_post_ihave_sub(func, *args, file_factory=iterlines) self.assertEqual(resp, success_resp) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 6464950..2c9dab9 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1,10 +1,11 @@ import ntpath import os import sys +import unittest +import warnings from test.support import TestFailed from test import support, test_genericpath from tempfile import TemporaryFile -import unittest def tester(fn, wantResult): @@ -21,7 +22,9 @@ def tester(fn, wantResult): fn = fn.replace('["', '[b"') fn = fn.replace(", '", ", b'") fn = fn.replace(', "', ', b"') - gotResult = eval(fn) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + gotResult = eval(fn) if isinstance(wantResult, str): wantResult = wantResult.encode('ascii') elif isinstance(wantResult, tuple): diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 7b95612..d1ae757 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -631,7 +631,7 @@ class TestStandard(BaseTest): option_list=options) def test_required_value(self): - self.assertParseFail(["-a"], "-a option requires an argument") + self.assertParseFail(["-a"], "-a option requires 1 argument") def test_invalid_integer(self): self.assertParseFail(["-b", "5x"], @@ -1023,7 +1023,7 @@ class TestExtendAddTypes(BaseTest): TYPE_CHECKER["file"] = check_file def test_filetype_ok(self): - open(support.TESTFN, "w").close() + support.create_empty_file(support.TESTFN) self.assertParseOK(["--file", support.TESTFN, "-afoo"], {'file': support.TESTFN, 'a': 'foo'}, []) diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index efa28ea..b9e2f32 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -14,18 +14,25 @@ import shutil from test import support import contextlib import mmap +import platform +import re import uuid +import asyncore +import asynchat +import socket +try: + import threading +except ImportError: + threading = None # Detect whether we're on a Linux system that uses the (now outdated # and unmaintained) linuxthreads threading library. There's an issue # when combining linuxthreads with a failed execv call: see # http://bugs.python.org/issue4970. -if (hasattr(os, "confstr_names") and - "CS_GNU_LIBPTHREAD_VERSION" in os.confstr_names): - libpthread = os.confstr("CS_GNU_LIBPTHREAD_VERSION") - USING_LINUXTHREADS= libpthread.startswith("linuxthreads") +if hasattr(sys, 'thread_info') and sys.thread_info.version: + USING_LINUXTHREADS = sys.thread_info.version.startswith("linuxthreads") else: - USING_LINUXTHREADS= False + USING_LINUXTHREADS = False # Tests creating TESTFN class FileTests(unittest.TestCase): @@ -206,7 +213,9 @@ class StatAttributeTests(unittest.TestCase): fname = self.fname.encode(sys.getfilesystemencoding()) except UnicodeEncodeError: self.skipTest("cannot encode %a for the filesystem" % self.fname) - self.check_stat_attributes(fname) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.check_stat_attributes(fname) def test_statvfs_attributes(self): if not hasattr(os, "statvfs"): @@ -263,6 +272,21 @@ class StatAttributeTests(unittest.TestCase): st2 = os.stat(support.TESTFN) self.assertEqual(st2.st_mtime, int(st.st_mtime-delta)) + def test_utime_noargs(self): + # Issue #13327 removed the requirement to pass None as the + # second argument. Check that the previous methods of passing + # a time tuple or None work in addition to no argument. + st = os.stat(support.TESTFN) + # Doesn't set anything new, but sets the time tuple way + os.utime(support.TESTFN, (st.st_atime, st.st_mtime)) + # Set to the current time in the old explicit way. + os.utime(support.TESTFN, None) + st1 = os.stat(support.TESTFN) + # Set to the current time in the new way + os.utime(support.TESTFN) + st2 = os.stat(support.TESTFN) + self.assertAlmostEqual(st1.st_mtime, st2.st_mtime, delta=10) + # Restrict test to Win32, since there is no guarantee other # systems support centiseconds if sys.platform == 'win32': @@ -816,7 +840,9 @@ class LinkTests(unittest.TestCase): with open(file1, "w") as f1: f1.write("test") - os.link(file1, file2) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + os.link(file1, file2) with open(file1, "r") as f1, open(file2, "r") as f2: self.assertTrue(os.path.sameopenfile(f1.fileno(), f2.fileno())) @@ -919,8 +945,7 @@ if sys.platform != 'win32': os.mkdir(self.dir) try: for fn in bytesfn: - f = open(os.path.join(self.bdir, fn), "w") - f.close() + support.create_empty_file(os.path.join(self.bdir, fn)) fn = os.fsdecode(fn) if fn in self.unicodefn: raise ValueError("duplicate filename") @@ -1139,8 +1164,10 @@ class Win32SymlinkTests(unittest.TestCase): self.assertNotEqual(os.lstat(link), os.stat(link)) bytes_link = os.fsencode(link) - self.assertEqual(os.stat(bytes_link), os.stat(target)) - self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link)) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.assertEqual(os.stat(bytes_link), os.stat(target)) + self.assertNotEqual(os.lstat(bytes_link), os.stat(bytes_link)) def test_12084(self): level1 = os.path.abspath(support.TESTFN) @@ -1220,6 +1247,422 @@ class LoginTests(unittest.TestCase): self.assertNotEqual(len(user_name), 0) +@unittest.skipUnless(hasattr(os, 'getpriority') and hasattr(os, 'setpriority'), + "needs os.getpriority and os.setpriority") +class ProgramPriorityTests(unittest.TestCase): + """Tests for os.getpriority() and os.setpriority().""" + + def test_set_get_priority(self): + + base = os.getpriority(os.PRIO_PROCESS, os.getpid()) + os.setpriority(os.PRIO_PROCESS, os.getpid(), base + 1) + try: + new_prio = os.getpriority(os.PRIO_PROCESS, os.getpid()) + if base >= 19 and new_prio <= 19: + raise unittest.SkipTest( + "unable to reliably test setpriority at current nice level of %s" % base) + else: + self.assertEqual(new_prio, base + 1) + finally: + try: + os.setpriority(os.PRIO_PROCESS, os.getpid(), base) + except OSError as err: + if err.errno != errno.EACCES: + raise + + +if threading is not None: + class SendfileTestServer(asyncore.dispatcher, threading.Thread): + + class Handler(asynchat.async_chat): + + def __init__(self, conn): + asynchat.async_chat.__init__(self, conn) + self.in_buffer = [] + self.closed = False + self.push(b"220 ready\r\n") + + def handle_read(self): + data = self.recv(4096) + self.in_buffer.append(data) + + def get_data(self): + return b''.join(self.in_buffer) + + def handle_close(self): + self.close() + self.closed = True + + def handle_error(self): + raise + + def __init__(self, address): + threading.Thread.__init__(self) + asyncore.dispatcher.__init__(self) + self.create_socket(socket.AF_INET, socket.SOCK_STREAM) + self.bind(address) + self.listen(5) + self.host, self.port = self.socket.getsockname()[:2] + self.handler_instance = None + self._active = False + self._active_lock = threading.Lock() + + # --- public API + + @property + def running(self): + return self._active + + def start(self): + assert not self.running + self.__flag = threading.Event() + threading.Thread.start(self) + self.__flag.wait() + + def stop(self): + assert self.running + self._active = False + self.join() + + def wait(self): + # wait for handler connection to be closed, then stop the server + while not getattr(self.handler_instance, "closed", False): + time.sleep(0.001) + self.stop() + + # --- internals + + def run(self): + self._active = True + self.__flag.set() + while self._active and asyncore.socket_map: + self._active_lock.acquire() + asyncore.loop(timeout=0.001, count=1) + self._active_lock.release() + asyncore.close_all() + + def handle_accept(self): + conn, addr = self.accept() + self.handler_instance = self.Handler(conn) + + def handle_connect(self): + self.close() + handle_read = handle_connect + + def writable(self): + return 0 + + def handle_error(self): + raise + + +@unittest.skipUnless(threading is not None, "test needs threading module") +@unittest.skipUnless(hasattr(os, 'sendfile'), "test needs os.sendfile()") +class TestSendfile(unittest.TestCase): + + DATA = b"12345abcde" * 16 * 1024 # 160 KB + SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ + not sys.platform.startswith("solaris") and \ + not sys.platform.startswith("sunos") + + @classmethod + def setUpClass(cls): + with open(support.TESTFN, "wb") as f: + f.write(cls.DATA) + + @classmethod + def tearDownClass(cls): + support.unlink(support.TESTFN) + + def setUp(self): + self.server = SendfileTestServer((support.HOST, 0)) + self.server.start() + self.client = socket.socket() + self.client.connect((self.server.host, self.server.port)) + self.client.settimeout(1) + # synchronize by waiting for "220 ready" response + self.client.recv(1024) + self.sockno = self.client.fileno() + self.file = open(support.TESTFN, 'rb') + self.fileno = self.file.fileno() + + def tearDown(self): + self.file.close() + self.client.close() + if self.server.running: + self.server.stop() + + def sendfile_wrapper(self, sock, file, offset, nbytes, headers=[], trailers=[]): + """A higher level wrapper representing how an application is + supposed to use sendfile(). + """ + while 1: + try: + if self.SUPPORT_HEADERS_TRAILERS: + return os.sendfile(sock, file, offset, nbytes, headers, + trailers) + else: + return os.sendfile(sock, file, offset, nbytes) + except OSError as err: + if err.errno == errno.ECONNRESET: + # disconnected + raise + elif err.errno in (errno.EAGAIN, errno.EBUSY): + # we have to retry send data + continue + else: + raise + + def test_send_whole_file(self): + # normal send + total_sent = 0 + offset = 0 + nbytes = 4096 + while total_sent < len(self.DATA): + sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) + if sent == 0: + break + offset += sent + total_sent += sent + self.assertTrue(sent <= nbytes) + self.assertEqual(offset, total_sent) + + self.assertEqual(total_sent, len(self.DATA)) + self.client.shutdown(socket.SHUT_RDWR) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(len(data), len(self.DATA)) + self.assertEqual(data, self.DATA) + + def test_send_at_certain_offset(self): + # start sending a file at a certain offset + total_sent = 0 + offset = len(self.DATA) // 2 + must_send = len(self.DATA) - offset + nbytes = 4096 + while total_sent < must_send: + sent = self.sendfile_wrapper(self.sockno, self.fileno, offset, nbytes) + if sent == 0: + break + offset += sent + total_sent += sent + self.assertTrue(sent <= nbytes) + + self.client.shutdown(socket.SHUT_RDWR) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + expected = self.DATA[len(self.DATA) // 2:] + self.assertEqual(total_sent, len(expected)) + self.assertEqual(len(data), len(expected)) + self.assertEqual(data, expected) + + def test_offset_overflow(self): + # specify an offset > file size + offset = len(self.DATA) + 4096 + try: + sent = os.sendfile(self.sockno, self.fileno, offset, 4096) + except OSError as e: + # Solaris can raise EINVAL if offset >= file length, ignore. + if e.errno != errno.EINVAL: + raise + else: + self.assertEqual(sent, 0) + self.client.shutdown(socket.SHUT_RDWR) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(data, b'') + + def test_invalid_offset(self): + with self.assertRaises(OSError) as cm: + os.sendfile(self.sockno, self.fileno, -1, 4096) + self.assertEqual(cm.exception.errno, errno.EINVAL) + + # --- headers / trailers tests + + if SUPPORT_HEADERS_TRAILERS: + + def test_headers(self): + total_sent = 0 + sent = os.sendfile(self.sockno, self.fileno, 0, 4096, + headers=[b"x" * 512]) + total_sent += sent + offset = 4096 + nbytes = 4096 + while 1: + sent = self.sendfile_wrapper(self.sockno, self.fileno, + offset, nbytes) + if sent == 0: + break + total_sent += sent + offset += sent + + expected_data = b"x" * 512 + self.DATA + self.assertEqual(total_sent, len(expected_data)) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(hash(data), hash(expected_data)) + + def test_trailers(self): + TESTFN2 = support.TESTFN + "2" + with open(TESTFN2, 'wb') as f: + f.write(b"abcde") + with open(TESTFN2, 'rb')as f: + self.addCleanup(os.remove, TESTFN2) + os.sendfile(self.sockno, f.fileno(), 0, 4096, + trailers=[b"12345"]) + self.client.close() + self.server.wait() + data = self.server.handler_instance.get_data() + self.assertEqual(data, b"abcde12345") + + if hasattr(os, "SF_NODISKIO"): + def test_flags(self): + try: + os.sendfile(self.sockno, self.fileno, 0, 4096, + flags=os.SF_NODISKIO) + except OSError as err: + if err.errno not in (errno.EBUSY, errno.EAGAIN): + raise + + +def supports_extended_attributes(): + if not hasattr(os, "setxattr"): + return False + try: + with open(support.TESTFN, "wb") as fp: + try: + os.fsetxattr(fp.fileno(), b"user.test", b"") + except OSError as e: + if e.errno != errno.ENOTSUP: + raise + return False + finally: + support.unlink(support.TESTFN) + # Kernels < 2.6.39 don't respect setxattr flags. + kernel_version = platform.release() + m = re.match("2.6.(\d{1,2})", kernel_version) + return m is None or int(m.group(1)) >= 39 + + +@unittest.skipUnless(supports_extended_attributes(), + "no non-broken extended attribute support") +class ExtendedAttributeTests(unittest.TestCase): + + def tearDown(self): + support.unlink(support.TESTFN) + + def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr): + fn = support.TESTFN + open(fn, "wb").close() + with self.assertRaises(OSError) as cm: + getxattr(fn, s("user.test")) + self.assertEqual(cm.exception.errno, errno.ENODATA) + init_xattr = listxattr(fn) + self.assertIsInstance(init_xattr, list) + setxattr(fn, s("user.test"), b"") + xattr = set(init_xattr) + xattr.add("user.test") + self.assertEqual(set(listxattr(fn)), xattr) + self.assertEqual(getxattr(fn, b"user.test"), b"") + setxattr(fn, s("user.test"), b"hello", os.XATTR_REPLACE) + self.assertEqual(getxattr(fn, b"user.test"), b"hello") + with self.assertRaises(OSError) as cm: + setxattr(fn, s("user.test"), b"bye", os.XATTR_CREATE) + self.assertEqual(cm.exception.errno, errno.EEXIST) + with self.assertRaises(OSError) as cm: + setxattr(fn, s("user.test2"), b"bye", os.XATTR_REPLACE) + self.assertEqual(cm.exception.errno, errno.ENODATA) + setxattr(fn, s("user.test2"), b"foo", os.XATTR_CREATE) + xattr.add("user.test2") + self.assertEqual(set(listxattr(fn)), xattr) + removexattr(fn, s("user.test")) + with self.assertRaises(OSError) as cm: + getxattr(fn, s("user.test")) + self.assertEqual(cm.exception.errno, errno.ENODATA) + xattr.remove("user.test") + self.assertEqual(set(listxattr(fn)), xattr) + self.assertEqual(getxattr(fn, s("user.test2")), b"foo") + setxattr(fn, s("user.test"), b"a"*1024) + self.assertEqual(getxattr(fn, s("user.test")), b"a"*1024) + removexattr(fn, s("user.test")) + many = sorted("user.test{}".format(i) for i in range(100)) + for thing in many: + setxattr(fn, thing, b"x") + self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many)) + + def _check_xattrs(self, *args): + def make_bytes(s): + return bytes(s, "ascii") + self._check_xattrs_str(str, *args) + support.unlink(support.TESTFN) + self._check_xattrs_str(make_bytes, *args) + + def test_simple(self): + self._check_xattrs(os.getxattr, os.setxattr, os.removexattr, + os.listxattr) + + def test_lpath(self): + self._check_xattrs(os.lgetxattr, os.lsetxattr, os.lremovexattr, + os.llistxattr) + + def test_fds(self): + def getxattr(path, *args): + with open(path, "rb") as fp: + return os.fgetxattr(fp.fileno(), *args) + def setxattr(path, *args): + with open(path, "wb") as fp: + os.fsetxattr(fp.fileno(), *args) + def removexattr(path, *args): + with open(path, "wb") as fp: + os.fremovexattr(fp.fileno(), *args) + def listxattr(path, *args): + with open(path, "rb") as fp: + return os.flistxattr(fp.fileno(), *args) + self._check_xattrs(getxattr, setxattr, removexattr, listxattr) + + +@unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") +class Win32DeprecatedBytesAPI(unittest.TestCase): + def test_deprecated(self): + import nt + filename = os.fsencode(support.TESTFN) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + for func, *args in ( + (nt._getfullpathname, filename), + (nt._isdir, filename), + (os.access, filename, os.R_OK), + (os.chdir, filename), + (os.chmod, filename, 0o777), + (os.getcwdb,), + (os.link, filename, filename), + (os.listdir, filename), + (os.lstat, filename), + (os.mkdir, filename), + (os.open, filename, os.O_RDONLY), + (os.rename, filename, filename), + (os.rmdir, filename), + (os.startfile, filename), + (os.stat, filename), + (os.unlink, filename), + (os.utime, filename), + ): + self.assertRaises(DeprecationWarning, func, *args) + + @support.skip_unless_symlink + def test_symlink(self): + filename = os.fsencode(support.TESTFN) + with warnings.catch_warnings(): + warnings.simplefilter("error", DeprecationWarning) + self.assertRaises(DeprecationWarning, + os.symlink, filename, filename) + + +@support.reap_threads def test_main(): support.run_unittest( FileTests, @@ -1240,6 +1683,10 @@ def test_main(): PidTests, LoginTests, LinkTests, + TestSendfile, + ProgramPriorityTests, + ExtendedAttributeTests, + Win32DeprecatedBytesAPI, ) if __name__ == "__main__": diff --git a/Lib/test/test_ossaudiodev.py b/Lib/test/test_ossaudiodev.py index 9cb89d6..3908a05 100644 --- a/Lib/test/test_ossaudiodev.py +++ b/Lib/test/test_ossaudiodev.py @@ -170,6 +170,22 @@ class OSSAudioDevTests(unittest.TestCase): pass self.assertTrue(dsp.closed) + def test_on_closed(self): + dsp = ossaudiodev.open('w') + dsp.close() + self.assertRaises(ValueError, dsp.fileno) + self.assertRaises(ValueError, dsp.read, 1) + self.assertRaises(ValueError, dsp.write, b'x') + self.assertRaises(ValueError, dsp.writeall, b'x') + self.assertRaises(ValueError, dsp.bufsize) + self.assertRaises(ValueError, dsp.obufcount) + self.assertRaises(ValueError, dsp.obufcount) + self.assertRaises(ValueError, dsp.obuffree) + self.assertRaises(ValueError, dsp.getptr) + + mixer = ossaudiodev.openmixer() + mixer.close() + self.assertRaises(ValueError, mixer.fileno) def test_main(): try: diff --git a/Lib/test/test_osx_env.py b/Lib/test/test_osx_env.py index 8b3df37..24ec2b4 100644 --- a/Lib/test/test_osx_env.py +++ b/Lib/test/test_osx_env.py @@ -5,6 +5,7 @@ Test suite for OS X interpreter environment variables. from test.support import EnvironmentVarGuard, run_unittest import subprocess import sys +import sysconfig import unittest class OSXEnvironmentVariableTestCase(unittest.TestCase): @@ -27,8 +28,6 @@ class OSXEnvironmentVariableTestCase(unittest.TestCase): self._check_sys('PYTHONEXECUTABLE', '==', 'sys.executable') def test_main(): - from distutils import sysconfig - if sys.platform == 'darwin' and sysconfig.get_config_var('WITH_NEXT_FRAMEWORK'): run_unittest(OSXEnvironmentVariableTestCase) diff --git a/Lib/test/test_packaging.py b/Lib/test/test_packaging.py new file mode 100644 index 0000000..250d661 --- /dev/null +++ b/Lib/test/test_packaging.py @@ -0,0 +1,5 @@ +import sys +from packaging.tests.__main__ import test_main + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 020acd5..2b50fca 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -614,6 +614,13 @@ class STObjectTestCase(unittest.TestCase): # XXX tests for pickling and unpickling of ST objects should go here +class OtherParserCase(unittest.TestCase): + + def test_two_args_to_expr(self): + # See bug #12264 + with self.assertRaises(TypeError): + parser.expr("a", "b") + def test_main(): support.run_unittest( @@ -622,6 +629,7 @@ def test_main(): CompileTestCase, ParserStackLimitTestCase, STObjectTestCase, + OtherParserCase, ) diff --git a/Lib/test/test_pdb.py b/Lib/test/test_pdb.py index d861df5..c197aff 100644 --- a/Lib/test/test_pdb.py +++ b/Lib/test/test_pdb.py @@ -20,9 +20,12 @@ class PdbTestInput(object): def __enter__(self): self.real_stdin = sys.stdin sys.stdin = _FakeInput(self.input) + self.orig_trace = sys.gettrace() if hasattr(sys, 'gettrace') else None def __exit__(self, *exc): sys.stdin = self.real_stdin + if self.orig_trace: + sys.settrace(self.orig_trace) def test_pdb_displayhook(): diff --git a/Lib/test/test_peepholer.py b/Lib/test/test_peepholer.py index 53719d3..1e782cf 100644 --- a/Lib/test/test_peepholer.py +++ b/Lib/test/test_peepholer.py @@ -3,13 +3,16 @@ import re import sys from io import StringIO import unittest +from math import copysign def disassemble(func): f = StringIO() tmp = sys.stdout sys.stdout = f - dis.dis(func) - sys.stdout = tmp + try: + dis.dis(func) + finally: + sys.stdout = tmp result = f.getvalue() f.close() return result @@ -17,6 +20,7 @@ def disassemble(func): def dis_single(line): return disassemble(compile(line, '', 'single')) + class TestTranforms(unittest.TestCase): def test_unot(self): @@ -99,6 +103,12 @@ class TestTranforms(unittest.TestCase): self.assertIn(elem, asm) self.assertNotIn('BUILD_TUPLE', asm) + # Long tuples should be folded too. + asm = dis_single(repr(tuple(range(10000)))) + # One LOAD_CONST for the tuple, one for the None return value + self.assertEqual(asm.count('LOAD_CONST'), 2) + self.assertNotIn('BUILD_TUPLE', asm) + # Bug 1053819: Tuple of constants misidentified when presented with: # . . . opcode_with_arg 100 unary_opcode BUILD_TUPLE 1 . . . # The following would segfault upon compilation @@ -208,14 +218,13 @@ class TestTranforms(unittest.TestCase): # out of range asm = dis_single('"fuu"[10]') self.assertIn('BINARY_SUBSCR', asm) - # non-BMP char (see #5057) - asm = dis_single('"\U00012345"[0]') - self.assertIn('BINARY_SUBSCR', asm) - def test_folding_of_unaryops_on_constants(self): for line, elem in ( ('-0.5', '(-0.5)'), # unary negative + ('-0.0', '(-0.0)'), # -0.0 + ('-(1.0-1.0)','(-0.0)'), # -0.0 after folding + ('-0', '(0)'), # -0 ('~-2', '(1)'), # unary invert ('+1', '(1)'), # unary positive ): @@ -223,6 +232,13 @@ class TestTranforms(unittest.TestCase): self.assertIn(elem, asm, asm) self.assertNotIn('UNARY_', asm) + # Check that -0.0 works after marshaling + def negzero(): + return -(1.0-1.0) + + self.assertNotIn('UNARY_', disassemble(negzero)) + self.assertTrue(copysign(1.0, negzero()) < 0) + # Verify that unfoldables are skipped for line, elem in ( ('-"abc"', "('abc')"), # unary negative @@ -285,6 +301,25 @@ class TestTranforms(unittest.TestCase): asm = disassemble(f) self.assertNotIn('BINARY_ADD', asm) + def test_constant_folding(self): + # Issue #11244: aggressive constant folding. + exprs = [ + "3 * -5", + "-3 * 5", + "2 * (3 * 4)", + "(2 * 3) * 4", + "(-1, 2, 3)", + "(1, -2, 3)", + "(1, 2, -3)", + "(1, 2, -3) * 6", + "lambda x: x in {(3 * -5) + (-1 - 6), (1, -2, 3) * 2, None}", + ] + for e in exprs: + asm = dis_single(e) + self.assertNotIn('UNARY_', asm, e) + self.assertNotIn('BINARY_', asm, e) + self.assertNotIn('BUILD_', asm, e) + class TestBuglets(unittest.TestCase): def test_bug_11510(self): diff --git a/Lib/test/test_pep277.py b/Lib/test/test_pep277.py index 6d891e5..4b16cbb 100644 --- a/Lib/test/test_pep277.py +++ b/Lib/test/test_pep277.py @@ -1,6 +1,9 @@ # Test the Unicode versions of normal file functions # open, os.open, os.stat. os.listdir, os.rename, os.remove, os.mkdir, os.chdir, os.rmdir -import sys, os, unittest +import os +import sys +import unittest +import warnings from unicodedata import normalize from test import support @@ -38,8 +41,8 @@ if sys.platform != 'darwin': '17_\u2001\u2001\u2001A', '18_\u2003\u2003\u2003A', # == NFC('\u2001\u2001\u2001A') '19_\u0020\u0020\u0020A', # '\u0020' == ' ' == NFKC('\u2000') == - # NFKC('\u2001') == NFKC('\u2003') -]) + # NFKC('\u2001') == NFKC('\u2003') + ]) # Is it Unicode-friendly? @@ -71,7 +74,7 @@ class UnicodeFileTests(unittest.TestCase): def setUp(self): try: os.mkdir(support.TESTFN) - except OSError: + except FileExistsError: pass files = set() for name in self.files: @@ -90,15 +93,17 @@ class UnicodeFileTests(unittest.TestCase): return normalize(self.normal_form, s) return s - def _apply_failure(self, fn, filename, expected_exception, - check_fn_in_exception = True): + def _apply_failure(self, fn, filename, + expected_exception=FileNotFoundError, + check_filename=True): with self.assertRaises(expected_exception) as c: fn(filename) exc_filename = c.exception.filename - # the "filename" exception attribute may be encoded - if isinstance(exc_filename, bytes): - filename = filename.encode(sys.getfilesystemencoding()) - if check_fn_in_exception: + # listdir may append a wildcard to the filename + if fn is os.listdir and sys.platform == 'win32': + exc_filename, _, wildcard = exc_filename.rpartition(os.sep) + self.assertEqual(wildcard, '*.*') + if check_filename: self.assertEqual(exc_filename, filename, "Function '%s(%a) failed " "with bad filename in the exception: %a" % (fn.__name__, filename, exc_filename)) @@ -107,13 +112,18 @@ class UnicodeFileTests(unittest.TestCase): # Pass non-existing Unicode filenames all over the place. for name in self.files: name = "not_" + name - self._apply_failure(open, name, IOError) - self._apply_failure(os.stat, name, OSError) - self._apply_failure(os.chdir, name, OSError) - self._apply_failure(os.rmdir, name, OSError) - self._apply_failure(os.remove, name, OSError) - # listdir may append a wildcard to the filename, so dont check - self._apply_failure(os.listdir, name, OSError, False) + self._apply_failure(open, name) + self._apply_failure(os.stat, name) + self._apply_failure(os.chdir, name) + self._apply_failure(os.rmdir, name) + self._apply_failure(os.remove, name) + self._apply_failure(os.listdir, name) + + if sys.platform == 'win32': + # Windows is lunatic. Issue #13366. + _listdir_failure = NotADirectoryError, FileNotFoundError + else: + _listdir_failure = NotADirectoryError def test_open(self): for name in self.files: @@ -121,12 +131,13 @@ class UnicodeFileTests(unittest.TestCase): f.write((name+'\n').encode("utf-8")) f.close() os.stat(name) + self._apply_failure(os.listdir, name, self._listdir_failure) # Skip the test on darwin, because darwin does normalize the filename to # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, # NFKD in Python is useless, because darwin will normalize it later and so # open(), os.stat(), etc. don't raise any exception. - @unittest.skipIf(sys.platform == 'darwin', 'irrevelant test on Mac OS X') + @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') def test_normalize(self): files = set(self.files) others = set() @@ -134,21 +145,22 @@ class UnicodeFileTests(unittest.TestCase): others |= set(normalize(nf, file) for file in files) others -= files for name in others: - self._apply_failure(open, name, IOError) - self._apply_failure(os.stat, name, OSError) - self._apply_failure(os.chdir, name, OSError) - self._apply_failure(os.rmdir, name, OSError) - self._apply_failure(os.remove, name, OSError) - # listdir may append a wildcard to the filename, so dont check - self._apply_failure(os.listdir, name, OSError, False) + self._apply_failure(open, name) + self._apply_failure(os.stat, name) + self._apply_failure(os.chdir, name) + self._apply_failure(os.rmdir, name) + self._apply_failure(os.remove, name) + self._apply_failure(os.listdir, name) # Skip the test on darwin, because darwin uses a normalization different # than Python NFD normalization: filenames are different even if we use # Python NFD normalization. - @unittest.skipIf(sys.platform == 'darwin', 'irrevelant test on Mac OS X') + @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') def test_listdir(self): sf0 = set(self.files) - f1 = os.listdir(support.TESTFN.encode(sys.getfilesystemencoding())) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + f1 = os.listdir(support.TESTFN.encode(sys.getfilesystemencoding())) f2 = os.listdir(support.TESTFN) sf2 = set(os.path.join(support.TESTFN, f) for f in f2) self.assertEqual(sf0, sf2, "%a != %a" % (sf0, sf2)) diff --git a/Lib/test/test_pep292.py b/Lib/test/test_pep292.py index 119c7ea..6da8d2e 100644 --- a/Lib/test/test_pep292.py +++ b/Lib/test/test_pep292.py @@ -42,19 +42,6 @@ class TestTemplate(unittest.TestCase): s = Template('$who likes $$') eq(s.substitute(dict(who='tim', what='ham')), 'tim likes $') - def test_invalid(self): - class MyPattern(Template): - pattern = r""" - (?: - (?P<invalid>) | - (?P<escaped>%(delim)s) | - @(?P<named>%(id)s) | - @{(?P<braced>%(id)s)} - ) - """ - s = MyPattern('$') - self.assertRaises(ValueError, s.substitute, dict()) - def test_percents(self): eq = self.assertEqual s = Template('%(foo)s $foo ${foo}') @@ -172,6 +159,26 @@ class TestTemplate(unittest.TestCase): val = t.safe_substitute({'location': 'Cleveland'}) self.assertEqual(val, 'PyCon in Cleveland') + def test_invalid_with_no_lines(self): + # The error formatting for invalid templates + # has a special case for no data that the default + # pattern can't trigger (always has at least '$') + # So we craft a pattern that is always invalid + # with no leading data. + class MyTemplate(Template): + pattern = r""" + (?P<invalid>) | + unreachable( + (?P<named>) | + (?P<braced>) | + (?P<escaped>) + ) + """ + s = MyTemplate('') + with self.assertRaises(ValueError) as err: + s.substitute({}) + self.assertIn('line 1, col 1', str(err.exception)) + def test_unicode_values(self): s = Template('$who likes $what') d = dict(who='t\xffm', what='f\xfe\fed') diff --git a/Lib/test/test_pep3120.py b/Lib/test/test_pep3120.py index 09fedf0..496f8da 100644 --- a/Lib/test/test_pep3120.py +++ b/Lib/test/test_pep3120.py @@ -19,8 +19,8 @@ class PEP3120Test(unittest.TestCase): try: import test.badsyntax_pep3120 except SyntaxError as msg: - msg = str(msg) - self.assertTrue('UTF-8' in msg or 'utf8' in msg) + msg = str(msg).lower() + self.assertTrue('utf-8' in msg or 'utf8' in msg) else: self.fail("expected exception didn't occur") diff --git a/Lib/test/test_pep3131.py b/Lib/test/test_pep3131.py index df0f64d..2e6b90a 100644 --- a/Lib/test/test_pep3131.py +++ b/Lib/test/test_pep3131.py @@ -17,12 +17,7 @@ class PEP3131Test(unittest.TestCase): def test_non_bmp_normalized(self): 𝔘𝔫𝔦𝔠𝔬𝔡𝔢 = 1 - # On wide builds, this is normalized, but on narrow ones it is not. See - # #12746. - try: - self.assertIn("𝔘𝔫𝔦𝔠𝔬𝔡𝔢", dir()) - except AssertionError: - raise unittest.case._ExpectedFailure(sys.exc_info()) + self.assertIn("Unicode", dir()) def test_invalid(self): try: diff --git a/Lib/test/test_pep3151.py b/Lib/test/test_pep3151.py new file mode 100644 index 0000000..e327f42 --- /dev/null +++ b/Lib/test/test_pep3151.py @@ -0,0 +1,151 @@ +import builtins +import os +import select +import socket +import sys +import unittest +import errno +from errno import EEXIST + +from test import support + +class SubOSError(OSError): + pass + + +class HierarchyTest(unittest.TestCase): + + def test_builtin_errors(self): + self.assertEqual(OSError.__name__, 'OSError') + self.assertIs(IOError, OSError) + self.assertIs(EnvironmentError, OSError) + + def test_socket_errors(self): + self.assertIs(socket.error, IOError) + self.assertIs(socket.gaierror.__base__, OSError) + self.assertIs(socket.herror.__base__, OSError) + self.assertIs(socket.timeout.__base__, OSError) + + def test_select_error(self): + self.assertIs(select.error, OSError) + + # mmap.error is tested in test_mmap + + _pep_map = """ + +-- BlockingIOError EAGAIN, EALREADY, EWOULDBLOCK, EINPROGRESS + +-- ChildProcessError ECHILD + +-- ConnectionError + +-- BrokenPipeError EPIPE, ESHUTDOWN + +-- ConnectionAbortedError ECONNABORTED + +-- ConnectionRefusedError ECONNREFUSED + +-- ConnectionResetError ECONNRESET + +-- FileExistsError EEXIST + +-- FileNotFoundError ENOENT + +-- InterruptedError EINTR + +-- IsADirectoryError EISDIR + +-- NotADirectoryError ENOTDIR + +-- PermissionError EACCES, EPERM + +-- ProcessLookupError ESRCH + +-- TimeoutError ETIMEDOUT + """ + def _make_map(s): + _map = {} + for line in s.splitlines(): + line = line.strip('+- ') + if not line: + continue + excname, _, errnames = line.partition(' ') + for errname in filter(None, errnames.strip().split(', ')): + _map[getattr(errno, errname)] = getattr(builtins, excname) + return _map + _map = _make_map(_pep_map) + + def test_errno_mapping(self): + # The OSError constructor maps errnos to subclasses + # A sample test for the basic functionality + e = OSError(EEXIST, "Bad file descriptor") + self.assertIs(type(e), FileExistsError) + # Exhaustive testing + for errcode, exc in self._map.items(): + e = OSError(errcode, "Some message") + self.assertIs(type(e), exc) + othercodes = set(errno.errorcode) - set(self._map) + for errcode in othercodes: + e = OSError(errcode, "Some message") + self.assertIs(type(e), OSError) + + def test_OSError_subclass_mapping(self): + # When constructing an OSError subclass, errno mapping isn't done + e = SubOSError(EEXIST, "Bad file descriptor") + self.assertIs(type(e), SubOSError) + + def test_try_except(self): + filename = "some_hopefully_non_existing_file" + + # This checks that try .. except checks the concrete exception + # (FileNotFoundError) and not the base type specified when + # PyErr_SetFromErrnoWithFilenameObject was called. + # (it is therefore deliberate that it doesn't use assertRaises) + try: + open(filename) + except FileNotFoundError: + pass + else: + self.fail("should have raised a FileNotFoundError") + + # Another test for PyErr_SetExcFromWindowsErrWithFilenameObject() + self.assertFalse(os.path.exists(filename)) + try: + os.unlink(filename) + except FileNotFoundError: + pass + else: + self.fail("should have raised a FileNotFoundError") + + +class AttributesTest(unittest.TestCase): + + def test_windows_error(self): + if os.name == "nt": + self.assertIn('winerror', dir(OSError)) + else: + self.assertNotIn('winerror', dir(OSError)) + + def test_posix_error(self): + e = OSError(EEXIST, "File already exists", "foo.txt") + self.assertEqual(e.errno, EEXIST) + self.assertEqual(e.args[0], EEXIST) + self.assertEqual(e.strerror, "File already exists") + self.assertEqual(e.filename, "foo.txt") + if os.name == "nt": + self.assertEqual(e.winerror, None) + + @unittest.skipUnless(os.name == "nt", "Windows-specific test") + def test_errno_translation(self): + # ERROR_ALREADY_EXISTS (183) -> EEXIST + e = OSError(0, "File already exists", "foo.txt", 183) + self.assertEqual(e.winerror, 183) + self.assertEqual(e.errno, EEXIST) + self.assertEqual(e.args[0], EEXIST) + self.assertEqual(e.strerror, "File already exists") + self.assertEqual(e.filename, "foo.txt") + + def test_blockingioerror(self): + args = ("a", "b", "c", "d", "e") + for n in range(6): + e = BlockingIOError(*args[:n]) + with self.assertRaises(AttributeError): + e.characters_written + e = BlockingIOError("a", "b", 3) + self.assertEqual(e.characters_written, 3) + e.characters_written = 5 + self.assertEqual(e.characters_written, 5) + + # XXX VMSError not tested + + +def test_main(): + support.run_unittest(__name__) + +if __name__=="__main__": + test_main() diff --git a/Lib/test/test_pipes.py b/Lib/test/test_pipes.py index d5b886f..6a7b45f 100644 --- a/Lib/test/test_pipes.py +++ b/Lib/test/test_pipes.py @@ -79,21 +79,6 @@ class SimplePipeTests(unittest.TestCase): with open(TESTFN) as f: self.assertEqual(f.read(), d) - def testQuoting(self): - safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./' - unicode_sample = '\xe9\xe0\xdf' # e + acute accent, a + grave, sharp s - unsafe = '"`$\\!' + unicode_sample - - self.assertEqual(pipes.quote(''), "''") - self.assertEqual(pipes.quote(safeunquoted), safeunquoted) - self.assertEqual(pipes.quote('test file name'), "'test file name'") - for u in unsafe: - self.assertEqual(pipes.quote('test%sname' % u), - "'test%sname'" % u) - for u in unsafe: - self.assertEqual(pipes.quote("test%s'name'" % u), - "'test%s'\"'\"'name'\"'\"''" % u) - def testRepr(self): t = pipes.Template() self.assertEqual(repr(t), "<Template instance, steps=[]>") diff --git a/Lib/test/test_pkgimport.py b/Lib/test/test_pkgimport.py index c37e936..a8426b5 100644 --- a/Lib/test/test_pkgimport.py +++ b/Lib/test/test_pkgimport.py @@ -7,7 +7,7 @@ import tempfile import unittest from imp import cache_from_source -from test.support import run_unittest +from test.support import run_unittest, create_empty_file class TestImport(unittest.TestCase): @@ -29,7 +29,7 @@ class TestImport(unittest.TestCase): self.package_dir = os.path.join(self.test_dir, self.package_name) os.mkdir(self.package_dir) - open(os.path.join(self.package_dir, '__init__.py'), 'w').close() + create_empty_file(os.path.join(self.package_dir, '__init__.py')) self.module_path = os.path.join(self.package_dir, 'foo.py') def tearDown(self): diff --git a/Lib/test/test_platform.py b/Lib/test/test_platform.py index 8751aa8..cfe623a 100644 --- a/Lib/test/test_platform.py +++ b/Lib/test/test_platform.py @@ -1,8 +1,9 @@ -import sys import os -import unittest import platform import subprocess +import sys +import unittest +import warnings from test import support @@ -56,13 +57,11 @@ class PlatformTest(unittest.TestCase): def setUp(self): self.save_version = sys.version - self.save_subversion = sys.subversion self.save_mercurial = sys._mercurial self.save_platform = sys.platform def tearDown(self): sys.version = self.save_version - sys.subversion = self.save_subversion sys._mercurial = self.save_mercurial sys.platform = self.save_platform @@ -77,7 +76,7 @@ class PlatformTest(unittest.TestCase): ('IronPython', '1.0.0', '', '', '', '', '.NET 2.0.50727.42')), ): # branch and revision are not "parsed", but fetched - # from sys.subversion. Ignore them + # from sys._mercurial. Ignore them (name, version, branch, revision, buildno, builddate, compiler) \ = platform._sys_version(input) self.assertEqual( @@ -113,8 +112,6 @@ class PlatformTest(unittest.TestCase): if subversion is None: if hasattr(sys, "_mercurial"): del sys._mercurial - if hasattr(sys, "subversion"): - del sys.subversion else: sys._mercurial = subversion if sys_platform is not None: @@ -247,6 +244,38 @@ class PlatformTest(unittest.TestCase): ): self.assertEqual(platform._parse_release_file(input), output) + def test_popen(self): + mswindows = (sys.platform == "win32") + + if mswindows: + command = '"{}" -c "print(\'Hello\')"'.format(sys.executable) + else: + command = "'{}' -c 'print(\"Hello\")'".format(sys.executable) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with platform.popen(command) as stdout: + hello = stdout.read().strip() + stdout.close() + self.assertEqual(hello, "Hello") + + data = 'plop' + if mswindows: + command = '"{}" -c "import sys; data=sys.stdin.read(); exit(len(data))"' + else: + command = "'{}' -c 'import sys; data=sys.stdin.read(); exit(len(data))'" + command = command.format(sys.executable) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + with platform.popen(command, 'w') as stdin: + stdout = stdin.write(data) + ret = stdin.close() + self.assertIsNotNone(ret) + if os.name == 'nt': + returncode = ret + else: + returncode = ret >> 8 + self.assertEqual(returncode, len(data)) + def test_main(): support.run_unittest( diff --git a/Lib/test/test_poplib.py b/Lib/test/test_poplib.py index 81af569..0a3adcc 100644 --- a/Lib/test/test_poplib.py +++ b/Lib/test/test_poplib.py @@ -108,6 +108,10 @@ class DummyPOP3Handler(asynchat.async_chat): def cmd_apop(self, arg): self.push('+OK done nothing.') + def cmd_quit(self, arg): + self.push('+OK closing.') + self.close_when_done() + class DummyPOP3Server(asyncore.dispatcher, threading.Thread): @@ -165,10 +169,10 @@ class TestPOP3Class(TestCase): def setUp(self): self.server = DummyPOP3Server((HOST, PORT)) self.server.start() - self.client = poplib.POP3(self.server.host, self.server.port) + self.client = poplib.POP3(self.server.host, self.server.port, timeout=3) def tearDown(self): - self.client.quit() + self.client.close() self.server.stop() def test_getwelcome(self): @@ -228,6 +232,12 @@ class TestPOP3Class(TestCase): self.client.uidl() self.client.uidl('foo') + def test_quit(self): + resp = self.client.quit() + self.assertTrue(resp) + self.assertIsNone(self.client.sock) + self.assertIsNone(self.client.file) + SUPPORTS_SSL = False if hasattr(poplib, 'POP3_SSL'): @@ -274,6 +284,7 @@ if hasattr(poplib, 'POP3_SSL'): else: DummyPOP3Handler.handle_read(self) + class TestPOP3_SSLClass(TestPOP3Class): # repeat previous tests by using poplib.POP3_SSL diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 09f04ec..3c863a7 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -9,6 +9,7 @@ import errno import sys import time import os +import fcntl import pwd import shutil import stat @@ -42,7 +43,7 @@ class PosixTester(unittest.TestCase): NO_ARG_FUNCTIONS = [ "ctermid", "getcwd", "getcwdb", "uname", "times", "getloadavg", "getegid", "geteuid", "getgid", "getgroups", - "getpid", "getpgrp", "getppid", "getuid", + "getpid", "getpgrp", "getppid", "getuid", "sync", ] for name in NO_ARG_FUNCTIONS: @@ -137,6 +138,159 @@ class PosixTester(unittest.TestCase): finally: fp.close() + @unittest.skipUnless(hasattr(posix, 'truncate'), "test needs posix.truncate()") + def test_truncate(self): + with open(support.TESTFN, 'w') as fp: + fp.write('test') + fp.flush() + posix.truncate(support.TESTFN, 0) + + @unittest.skipUnless(hasattr(posix, 'fexecve'), "test needs posix.fexecve()") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + @unittest.skipUnless(hasattr(os, 'waitpid'), "test needs os.waitpid()") + def test_fexecve(self): + fp = os.open(sys.executable, os.O_RDONLY) + try: + pid = os.fork() + if pid == 0: + os.chdir(os.path.split(sys.executable)[0]) + posix.fexecve(fp, [sys.executable, '-c', 'pass'], os.environ) + else: + self.assertEqual(os.waitpid(pid, 0), (pid, 0)) + finally: + os.close(fp) + + @unittest.skipUnless(hasattr(posix, 'waitid'), "test needs posix.waitid()") + @unittest.skipUnless(hasattr(os, 'fork'), "test needs os.fork()") + def test_waitid(self): + pid = os.fork() + if pid == 0: + os.chdir(os.path.split(sys.executable)[0]) + posix.execve(sys.executable, [sys.executable, '-c', 'pass'], os.environ) + else: + res = posix.waitid(posix.P_PID, pid, posix.WEXITED) + self.assertEqual(pid, res.si_pid) + + @unittest.skipUnless(hasattr(posix, 'lockf'), "test needs posix.lockf()") + def test_lockf(self): + fd = os.open(support.TESTFN, os.O_WRONLY | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, os.SEEK_SET) + posix.lockf(fd, posix.F_LOCK, 4) + # section is locked + posix.lockf(fd, posix.F_ULOCK, 4) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'pread'), "test needs posix.pread()") + def test_pread(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(b'es', posix.pread(fd, 2, 1)) + # the first pread() shouldn't disturb the file offset + self.assertEqual(b'te', posix.read(fd, 2)) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'pwrite'), "test needs posix.pwrite()") + def test_pwrite(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test') + os.lseek(fd, 0, os.SEEK_SET) + posix.pwrite(fd, b'xx', 1) + self.assertEqual(b'txxt', posix.read(fd, 4)) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'posix_fallocate'), + "test needs posix.posix_fallocate()") + def test_posix_fallocate(self): + fd = os.open(support.TESTFN, os.O_WRONLY | os.O_CREAT) + try: + posix.posix_fallocate(fd, 0, 10) + except OSError as inst: + # issue10812, ZFS doesn't appear to support posix_fallocate, + # so skip Solaris-based since they are likely to have ZFS. + if inst.errno != errno.EINVAL or not sys.platform.startswith("sunos"): + raise + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'posix_fadvise'), + "test needs posix.posix_fadvise()") + def test_posix_fadvise(self): + fd = os.open(support.TESTFN, os.O_RDONLY) + try: + posix.posix_fadvise(fd, 0, 0, posix.POSIX_FADV_WILLNEED) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'futimes'), "test needs posix.futimes()") + def test_futimes(self): + now = time.time() + fd = os.open(support.TESTFN, os.O_RDONLY) + try: + posix.futimes(fd, None) + posix.futimes(fd) + self.assertRaises(TypeError, posix.futimes, fd, (None, None)) + self.assertRaises(TypeError, posix.futimes, fd, (now, None)) + self.assertRaises(TypeError, posix.futimes, fd, (None, now)) + posix.futimes(fd, (int(now), int(now))) + posix.futimes(fd, (now, now)) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'lutimes'), "test needs posix.lutimes()") + def test_lutimes(self): + now = time.time() + posix.lutimes(support.TESTFN, None) + self.assertRaises(TypeError, posix.lutimes, support.TESTFN, (None, None)) + self.assertRaises(TypeError, posix.lutimes, support.TESTFN, (now, None)) + self.assertRaises(TypeError, posix.lutimes, support.TESTFN, (None, now)) + posix.lutimes(support.TESTFN, (int(now), int(now))) + posix.lutimes(support.TESTFN, (now, now)) + posix.lutimes(support.TESTFN) + + @unittest.skipUnless(hasattr(posix, 'futimens'), "test needs posix.futimens()") + def test_futimens(self): + now = time.time() + fd = os.open(support.TESTFN, os.O_RDONLY) + try: + self.assertRaises(TypeError, posix.futimens, fd, (None, None), (None, None)) + self.assertRaises(TypeError, posix.futimens, fd, (now, 0), None) + self.assertRaises(TypeError, posix.futimens, fd, None, (now, 0)) + posix.futimens(fd, (int(now), int((now - int(now)) * 1e9)), + (int(now), int((now - int(now)) * 1e9))) + posix.futimens(fd) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'writev'), "test needs posix.writev()") + def test_writev(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.writev(fd, (b'test1', b'tt2', b't3')) + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(b'test1tt2t3', posix.read(fd, 10)) + finally: + os.close(fd) + + @unittest.skipUnless(hasattr(posix, 'readv'), "test needs posix.readv()") + def test_readv(self): + fd = os.open(support.TESTFN, os.O_RDWR | os.O_CREAT) + try: + os.write(fd, b'test1tt2t3') + os.lseek(fd, 0, os.SEEK_SET) + buf = [bytearray(i) for i in [5, 3, 2]] + self.assertEqual(posix.readv(fd, buf), 10) + self.assertEqual([b'test1', b'tt2', b't3'], [bytes(i) for i in buf]) + finally: + os.close(fd) + def test_dup(self): if hasattr(posix, 'dup'): fp = open(support.TESTFN) @@ -162,6 +316,13 @@ class PosixTester(unittest.TestCase): fp1.close() fp2.close() + @unittest.skipUnless(hasattr(os, 'O_CLOEXEC'), "needs os.O_CLOEXEC") + @support.requires_linux_version(2, 6, 23) + def test_oscloexec(self): + fd = os.open(support.TESTFN, os.O_RDONLY|os.O_CLOEXEC) + self.addCleanup(os.close, fd) + self.assertTrue(fcntl.fcntl(fd, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + def test_osexlock(self): if hasattr(posix, "O_EXLOCK"): fd = os.open(support.TESTFN, @@ -254,7 +415,7 @@ class PosixTester(unittest.TestCase): self.assertRaises(OSError, posix.chown, support.TESTFN, -1, -1) # re-create the file - open(support.TESTFN, 'w').close() + support.create_empty_file(support.TESTFN) self._test_all_chown_common(posix.chown, support.TESTFN) @unittest.skipUnless(hasattr(posix, 'fchown'), "test needs os.fchown()") @@ -290,6 +451,18 @@ class PosixTester(unittest.TestCase): if hasattr(posix, 'listdir'): self.assertTrue(support.TESTFN in posix.listdir()) + @unittest.skipUnless(hasattr(posix, 'fdlistdir'), "test needs posix.fdlistdir()") + def test_fdlistdir(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + self.assertEqual( + sorted(posix.listdir('.')), + sorted(posix.fdlistdir(f)) + ) + # Check the fd was closed by fdlistdir + with self.assertRaises(OSError) as ctx: + posix.close(f) + self.assertEqual(ctx.exception.errno, errno.EBADF) + def test_access(self): if hasattr(posix, 'access'): self.assertTrue(posix.access(support.TESTFN, os.R_OK)) @@ -310,6 +483,32 @@ class PosixTester(unittest.TestCase): os.close(reader) os.close(writer) + @unittest.skipUnless(hasattr(os, 'pipe2'), "test needs os.pipe2()") + @support.requires_linux_version(2, 6, 27) + def test_pipe2(self): + self.assertRaises(TypeError, os.pipe2, 'DEADBEEF') + self.assertRaises(TypeError, os.pipe2, 0, 0) + + # try calling with flags = 0, like os.pipe() + r, w = os.pipe2(0) + os.close(r) + os.close(w) + + # test flags + r, w = os.pipe2(os.O_CLOEXEC|os.O_NONBLOCK) + self.addCleanup(os.close, r) + self.addCleanup(os.close, w) + self.assertTrue(fcntl.fcntl(r, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + self.assertTrue(fcntl.fcntl(w, fcntl.F_GETFD) & fcntl.FD_CLOEXEC) + # try reading from an empty pipe: this should fail, not block + self.assertRaises(OSError, os.read, r, 1) + # try a write big enough to fill-up the pipe: this should either + # fail or perform a partial write, not block + try: + os.write(w, b'x' * support.PIPE_MAX_SIZE) + except OSError: + pass + def test_utime(self): if hasattr(posix, 'utime'): now = time.time() @@ -410,6 +609,21 @@ class PosixTester(unittest.TestCase): os.chdir(curdir) support.rmtree(base_path) + @unittest.skipUnless(hasattr(posix, 'getgrouplist'), "test needs posix.getgrouplist()") + @unittest.skipUnless(hasattr(pwd, 'getpwuid'), "test needs pwd.getpwuid()") + @unittest.skipUnless(hasattr(os, 'getuid'), "test needs os.getuid()") + def test_getgrouplist(self): + with os.popen('id -G') as idg: + groups = idg.read().strip() + + if not groups: + raise unittest.SkipTest("need working 'id -G'") + + self.assertEqual( + set([int(x) for x in groups.split()]), + set(posix.getgrouplist(pwd.getpwuid(os.getuid())[0], + pwd.getpwuid(os.getuid())[3]))) + @unittest.skipUnless(hasattr(os, 'getegid'), "test needs os.getegid()") def test_getgroups(self): with os.popen('id -G') as idg: @@ -426,6 +640,366 @@ class PosixTester(unittest.TestCase): set([int(x) for x in groups.split()]), set(posix.getgroups() + [posix.getegid()])) + # tests for the posix *at functions follow + + @unittest.skipUnless(hasattr(posix, 'faccessat'), "test needs posix.faccessat()") + def test_faccessat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + self.assertTrue(posix.faccessat(f, support.TESTFN, os.R_OK)) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'fchmodat'), "test needs posix.fchmodat()") + def test_fchmodat(self): + os.chmod(support.TESTFN, stat.S_IRUSR) + + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.fchmodat(f, support.TESTFN, stat.S_IRUSR | stat.S_IWUSR) + + s = posix.stat(support.TESTFN) + self.assertEqual(s[0] & stat.S_IRWXU, stat.S_IRUSR | stat.S_IWUSR) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'fchownat'), "test needs posix.fchownat()") + def test_fchownat(self): + support.unlink(support.TESTFN) + support.create_empty_file(support.TESTFN) + + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.fchownat(f, support.TESTFN, os.getuid(), os.getgid()) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'fstatat'), "test needs posix.fstatat()") + def test_fstatat(self): + support.unlink(support.TESTFN) + with open(support.TESTFN, 'w') as outfile: + outfile.write("testline\n") + + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + s1 = posix.stat(support.TESTFN) + s2 = posix.fstatat(f, support.TESTFN) + self.assertEqual(s1, s2) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'futimesat'), "test needs posix.futimesat()") + def test_futimesat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + now = time.time() + posix.futimesat(f, support.TESTFN, None) + posix.futimesat(f, support.TESTFN) + self.assertRaises(TypeError, posix.futimesat, f, support.TESTFN, (None, None)) + self.assertRaises(TypeError, posix.futimesat, f, support.TESTFN, (now, None)) + self.assertRaises(TypeError, posix.futimesat, f, support.TESTFN, (None, now)) + posix.futimesat(f, support.TESTFN, (int(now), int(now))) + posix.futimesat(f, support.TESTFN, (now, now)) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'linkat'), "test needs posix.linkat()") + def test_linkat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.linkat(f, support.TESTFN, f, support.TESTFN + 'link') + # should have same inodes + self.assertEqual(posix.stat(support.TESTFN)[1], + posix.stat(support.TESTFN + 'link')[1]) + finally: + posix.close(f) + support.unlink(support.TESTFN + 'link') + + @unittest.skipUnless(hasattr(posix, 'mkdirat'), "test needs posix.mkdirat()") + def test_mkdirat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.mkdirat(f, support.TESTFN + 'dir') + posix.stat(support.TESTFN + 'dir') # should not raise exception + finally: + posix.close(f) + support.rmtree(support.TESTFN + 'dir') + + @unittest.skipUnless(hasattr(posix, 'mknodat') and hasattr(stat, 'S_IFIFO'), + "don't have mknodat()/S_IFIFO") + def test_mknodat(self): + # Test using mknodat() to create a FIFO (the only use specified + # by POSIX). + support.unlink(support.TESTFN) + mode = stat.S_IFIFO | stat.S_IRUSR | stat.S_IWUSR + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.mknodat(f, support.TESTFN, mode, 0) + except OSError as e: + # Some old systems don't allow unprivileged users to use + # mknod(), or only support creating device nodes. + self.assertIn(e.errno, (errno.EPERM, errno.EINVAL)) + else: + self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode)) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'openat'), "test needs posix.openat()") + def test_openat(self): + support.unlink(support.TESTFN) + with open(support.TESTFN, 'w') as outfile: + outfile.write("testline\n") + a = posix.open(posix.getcwd(), posix.O_RDONLY) + b = posix.openat(a, support.TESTFN, posix.O_RDONLY) + try: + res = posix.read(b, 9).decode(encoding="utf-8") + self.assertEqual("testline\n", res) + finally: + posix.close(a) + posix.close(b) + + @unittest.skipUnless(hasattr(posix, 'readlinkat'), "test needs posix.readlinkat()") + def test_readlinkat(self): + os.symlink(support.TESTFN, support.TESTFN + 'link') + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + self.assertEqual(posix.readlink(support.TESTFN + 'link'), + posix.readlinkat(f, support.TESTFN + 'link')) + finally: + support.unlink(support.TESTFN + 'link') + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'renameat'), "test needs posix.renameat()") + def test_renameat(self): + support.unlink(support.TESTFN) + support.create_empty_file(support.TESTFN + 'ren') + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.renameat(f, support.TESTFN + 'ren', f, support.TESTFN) + except: + posix.rename(support.TESTFN + 'ren', support.TESTFN) + raise + else: + posix.stat(support.TESTFN) # should not throw exception + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'symlinkat'), "test needs posix.symlinkat()") + def test_symlinkat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.symlinkat(support.TESTFN, f, support.TESTFN + 'link') + self.assertEqual(posix.readlink(support.TESTFN + 'link'), support.TESTFN) + finally: + posix.close(f) + support.unlink(support.TESTFN + 'link') + + @unittest.skipUnless(hasattr(posix, 'unlinkat'), "test needs posix.unlinkat()") + def test_unlinkat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + support.create_empty_file(support.TESTFN + 'del') + posix.stat(support.TESTFN + 'del') # should not throw exception + try: + posix.unlinkat(f, support.TESTFN + 'del') + except: + support.unlink(support.TESTFN + 'del') + raise + else: + self.assertRaises(OSError, posix.stat, support.TESTFN + 'link') + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'utimensat'), "test needs posix.utimensat()") + def test_utimensat(self): + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + now = time.time() + posix.utimensat(f, support.TESTFN, None, None) + posix.utimensat(f, support.TESTFN) + posix.utimensat(f, support.TESTFN, flags=os.AT_SYMLINK_NOFOLLOW) + self.assertRaises(TypeError, posix.utimensat, f, support.TESTFN, (None, None), (None, None)) + self.assertRaises(TypeError, posix.utimensat, f, support.TESTFN, (now, 0), None) + self.assertRaises(TypeError, posix.utimensat, f, support.TESTFN, None, (now, 0)) + posix.utimensat(f, support.TESTFN, (int(now), int((now - int(now)) * 1e9)), + (int(now), int((now - int(now)) * 1e9))) + posix.utimensat(dirfd=f, path=support.TESTFN, + atime=(int(now), int((now - int(now)) * 1e9)), + mtime=(int(now), int((now - int(now)) * 1e9))) + finally: + posix.close(f) + + @unittest.skipUnless(hasattr(posix, 'mkfifoat'), "don't have mkfifoat()") + def test_mkfifoat(self): + support.unlink(support.TESTFN) + f = posix.open(posix.getcwd(), posix.O_RDONLY) + try: + posix.mkfifoat(f, support.TESTFN, stat.S_IRUSR | stat.S_IWUSR) + self.assertTrue(stat.S_ISFIFO(posix.stat(support.TESTFN).st_mode)) + finally: + posix.close(f) + + requires_sched_h = unittest.skipUnless(hasattr(posix, 'sched_yield'), + "don't have scheduling support") + requires_sched_affinity = unittest.skipUnless(hasattr(posix, 'cpu_set'), + "don't have sched affinity support") + + @requires_sched_h + def test_sched_yield(self): + # This has no error conditions (at least on Linux). + posix.sched_yield() + + @requires_sched_h + @unittest.skipUnless(hasattr(posix, 'sched_get_priority_max'), + "requires sched_get_priority_max()") + def test_sched_priority(self): + # Round-robin usually has interesting priorities. + pol = posix.SCHED_RR + lo = posix.sched_get_priority_min(pol) + hi = posix.sched_get_priority_max(pol) + self.assertIsInstance(lo, int) + self.assertIsInstance(hi, int) + self.assertGreaterEqual(hi, lo) + # OSX evidently just returns 15 without checking the argument. + if sys.platform != "darwin": + self.assertRaises(OSError, posix.sched_get_priority_min, -23) + self.assertRaises(OSError, posix.sched_get_priority_max, -23) + + @unittest.skipUnless(hasattr(posix, 'sched_setscheduler'), "can't change scheduler") + def test_get_and_set_scheduler_and_param(self): + possible_schedulers = [sched for name, sched in posix.__dict__.items() + if name.startswith("SCHED_")] + mine = posix.sched_getscheduler(0) + self.assertIn(mine, possible_schedulers) + try: + parent = posix.sched_getscheduler(os.getppid()) + except OSError as e: + if e.errno != errno.EPERM: + raise + else: + self.assertIn(parent, possible_schedulers) + self.assertRaises(OSError, posix.sched_getscheduler, -1) + self.assertRaises(OSError, posix.sched_getparam, -1) + param = posix.sched_getparam(0) + self.assertIsInstance(param.sched_priority, int) + try: + posix.sched_setscheduler(0, mine, param) + except OSError as e: + if e.errno != errno.EPERM: + raise + + # POSIX states that calling sched_setparam() on a process with a + # scheduling policy other than SCHED_FIFO or SCHED_RR is + # implementation-defined: FreeBSD returns EINVAL. + if not sys.platform.startswith('freebsd'): + posix.sched_setparam(0, param) + self.assertRaises(OSError, posix.sched_setparam, -1, param) + + self.assertRaises(OSError, posix.sched_setscheduler, -1, mine, param) + self.assertRaises(TypeError, posix.sched_setscheduler, 0, mine, None) + self.assertRaises(TypeError, posix.sched_setparam, 0, 43) + param = posix.sched_param(None) + self.assertRaises(TypeError, posix.sched_setparam, 0, param) + large = 214748364700 + param = posix.sched_param(large) + self.assertRaises(OverflowError, posix.sched_setparam, 0, param) + param = posix.sched_param(sched_priority=-large) + self.assertRaises(OverflowError, posix.sched_setparam, 0, param) + + @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") + def test_sched_rr_get_interval(self): + try: + interval = posix.sched_rr_get_interval(0) + except OSError as e: + # This likely means that sched_rr_get_interval is only valid for + # processes with the SCHED_RR scheduler in effect. + if e.errno != errno.EINVAL: + raise + self.skipTest("only works on SCHED_RR processes") + self.assertIsInstance(interval, float) + # Reasonable constraints, I think. + self.assertGreaterEqual(interval, 0.) + self.assertLess(interval, 1.) + + @requires_sched_affinity + def test_sched_affinity(self): + mask = posix.sched_getaffinity(0, 1024) + self.assertGreaterEqual(mask.count(), 1) + self.assertIsInstance(mask, posix.cpu_set) + self.assertRaises(OSError, posix.sched_getaffinity, -1, 1024) + empty = posix.cpu_set(10) + posix.sched_setaffinity(0, mask) + self.assertRaises(OSError, posix.sched_setaffinity, 0, empty) + self.assertRaises(OSError, posix.sched_setaffinity, -1, mask) + + @requires_sched_affinity + def test_cpu_set_basic(self): + s = posix.cpu_set(10) + self.assertEqual(len(s), 10) + self.assertEqual(s.count(), 0) + s.set(0) + s.set(9) + self.assertTrue(s.isset(0)) + self.assertTrue(s.isset(9)) + self.assertFalse(s.isset(5)) + self.assertEqual(s.count(), 2) + s.clear(0) + self.assertFalse(s.isset(0)) + self.assertEqual(s.count(), 1) + s.zero() + self.assertFalse(s.isset(0)) + self.assertFalse(s.isset(9)) + self.assertEqual(s.count(), 0) + self.assertRaises(ValueError, s.set, -1) + self.assertRaises(ValueError, s.set, 10) + self.assertRaises(ValueError, s.clear, -1) + self.assertRaises(ValueError, s.clear, 10) + self.assertRaises(ValueError, s.isset, -1) + self.assertRaises(ValueError, s.isset, 10) + + @requires_sched_affinity + def test_cpu_set_cmp(self): + self.assertNotEqual(posix.cpu_set(11), posix.cpu_set(12)) + l = posix.cpu_set(10) + r = posix.cpu_set(10) + self.assertEqual(l, r) + l.set(1) + self.assertNotEqual(l, r) + r.set(1) + self.assertEqual(l, r) + + @requires_sched_affinity + def test_cpu_set_bitwise(self): + l = posix.cpu_set(5) + l.set(0) + l.set(1) + r = posix.cpu_set(5) + r.set(1) + r.set(2) + b = l & r + self.assertEqual(b.count(), 1) + self.assertTrue(b.isset(1)) + b = l | r + self.assertEqual(b.count(), 3) + self.assertTrue(b.isset(0)) + self.assertTrue(b.isset(1)) + self.assertTrue(b.isset(2)) + b = l ^ r + self.assertEqual(b.count(), 2) + self.assertTrue(b.isset(0)) + self.assertFalse(b.isset(1)) + self.assertTrue(b.isset(2)) + b = l + b |= r + self.assertIs(b, l) + self.assertEqual(l.count(), 3) + + def test_rtld_constants(self): + # check presence of major RTLD_* constants + posix.RTLD_LAZY + posix.RTLD_NOW + posix.RTLD_GLOBAL + posix.RTLD_LOCAL + class PosixGroupsTester(unittest.TestCase): def setUp(self): @@ -461,9 +1035,11 @@ class PosixGroupsTester(unittest.TestCase): posix.setgroups(groups) self.assertListEqual(groups, posix.getgroups()) - def test_main(): - support.run_unittest(PosixTester, PosixGroupsTester) + try: + support.run_unittest(PosixTester, PosixGroupsTester) + finally: + support.reap_children() if __name__ == '__main__': test_main() diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index bb4559c..709ef04 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,10 +1,10 @@ -import unittest -from test import support, test_genericpath - -import posixpath import os +import posixpath import sys +import unittest +import warnings from posixpath import realpath, abspath, dirname, basename +from test import support, test_genericpath try: import posix @@ -231,7 +231,9 @@ class PosixPathTest(unittest.TestCase): def test_ismount(self): self.assertIs(posixpath.ismount("/"), True) - self.assertIs(posixpath.ismount(b"/"), True) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", DeprecationWarning) + self.assertIs(posixpath.ismount(b"/"), True) def test_ismount_non_existent(self): # Non-existent mountpoint. diff --git a/Lib/test/test_pulldom.py b/Lib/test/test_pulldom.py new file mode 100644 index 0000000..b81a595 --- /dev/null +++ b/Lib/test/test_pulldom.py @@ -0,0 +1,347 @@ +import io +import unittest +import sys +import xml.sax + +from xml.sax.xmlreader import AttributesImpl +from xml.dom import pulldom + +from test.support import run_unittest, findfile + + +tstfile = findfile("test.xml", subdir="xmltestdata") + +# A handy XML snippet, containing attributes, a namespace prefix, and a +# self-closing tag: +SMALL_SAMPLE = """<?xml version="1.0"?> +<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xdc="http://www.xml.com/books"> +<!-- A comment --> +<title>Introduction to XSL</title> +<hr/> +<p><xdc:author xdc:attrib="prefixed attribute" attrib="other attrib">A. Namespace</xdc:author></p> +</html>""" + + +class PullDOMTestCase(unittest.TestCase): + + def test_parse(self): + """Minimal test of DOMEventStream.parse()""" + + # This just tests that parsing from a stream works. Actual parser + # semantics are tested using parseString with a more focused XML + # fragment. + + # Test with a filename: + handler = pulldom.parse(tstfile) + self.addCleanup(handler.stream.close) + list(handler) + + # Test with a file object: + with open(tstfile, "rb") as fin: + list(pulldom.parse(fin)) + + def test_parse_semantics(self): + """Test DOMEventStream parsing semantics.""" + + items = pulldom.parseString(SMALL_SAMPLE) + evt, node = next(items) + # Just check the node is a Document: + self.assertTrue(hasattr(node, "createElement")) + self.assertEqual(pulldom.START_DOCUMENT, evt) + evt, node = next(items) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("html", node.tagName) + self.assertEqual(2, len(node.attributes)) + self.assertEqual(node.attributes.getNamedItem("xmlns:xdc").value, + "http://www.xml.com/books") + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) # Line break + evt, node = next(items) + # XXX - A comment should be reported here! + # self.assertEqual(pulldom.COMMENT, evt) + # Line break after swallowed comment: + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual("title", node.tagName) + title_node = node + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + self.assertEqual("Introduction to XSL", node.data) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("title", node.tagName) + self.assertTrue(title_node is node) + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("hr", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("hr", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("p", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("xdc:author", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("xdc:author", node.tagName) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + evt, node = next(items) + self.assertEqual(pulldom.CHARACTERS, evt) + evt, node = next(items) + self.assertEqual(pulldom.END_ELEMENT, evt) + # XXX No END_DOCUMENT item is ever obtained: + #evt, node = next(items) + #self.assertEqual(pulldom.END_DOCUMENT, evt) + + def test_expandItem(self): + """Ensure expandItem works as expected.""" + items = pulldom.parseString(SMALL_SAMPLE) + # Loop through the nodes until we get to a "title" start tag: + for evt, item in items: + if evt == pulldom.START_ELEMENT and item.tagName == "title": + items.expandNode(item) + self.assertEqual(1, len(item.childNodes)) + break + else: + self.fail("No \"title\" element detected in SMALL_SAMPLE!") + # Loop until we get to the next start-element: + for evt, node in items: + if evt == pulldom.START_ELEMENT: + break + self.assertEqual("hr", node.tagName, + "expandNode did not leave DOMEventStream in the correct state.") + # Attempt to expand a standalone element: + items.expandNode(node) + self.assertEqual(next(items)[0], pulldom.CHARACTERS) + evt, node = next(items) + self.assertEqual(node.tagName, "p") + items.expandNode(node) + next(items) # Skip character data + evt, node = next(items) + self.assertEqual(node.tagName, "html") + with self.assertRaises(StopIteration): + next(items) + items.clear() + self.assertIsNone(items.parser) + self.assertIsNone(items.stream) + + @unittest.expectedFailure + def test_comment(self): + """PullDOM does not receive "comment" events.""" + items = pulldom.parseString(SMALL_SAMPLE) + for evt, _ in items: + if evt == pulldom.COMMENT: + break + else: + self.fail("No comment was encountered") + + @unittest.expectedFailure + def test_end_document(self): + """PullDOM does not receive "end-document" events.""" + items = pulldom.parseString(SMALL_SAMPLE) + # Read all of the nodes up to and including </html>: + for evt, node in items: + if evt == pulldom.END_ELEMENT and node.tagName == "html": + break + try: + # Assert that the next node is END_DOCUMENT: + evt, node = next(items) + self.assertEqual(pulldom.END_DOCUMENT, evt) + except StopIteration: + self.fail( + "Ran out of events, but should have received END_DOCUMENT") + + +class ThoroughTestCase(unittest.TestCase): + """Test the hard-to-reach parts of pulldom.""" + + def test_thorough_parse(self): + """Test some of the hard-to-reach parts of PullDOM.""" + self._test_thorough(pulldom.parse(None, parser=SAXExerciser())) + + @unittest.expectedFailure + def test_sax2dom_fail(self): + """SAX2DOM can"t handle a PI before the root element.""" + pd = SAX2DOMTestHelper(None, SAXExerciser(), 12) + self._test_thorough(pd) + + def test_thorough_sax2dom(self): + """Test some of the hard-to-reach parts of SAX2DOM.""" + pd = SAX2DOMTestHelper(None, SAX2DOMExerciser(), 12) + self._test_thorough(pd, False) + + def _test_thorough(self, pd, before_root=True): + """Test some of the hard-to-reach parts of the parser, using a mock + parser.""" + + evt, node = next(pd) + self.assertEqual(pulldom.START_DOCUMENT, evt) + # Just check the node is a Document: + self.assertTrue(hasattr(node, "createElement")) + + if before_root: + evt, node = next(pd) + self.assertEqual(pulldom.COMMENT, evt) + self.assertEqual("a comment", node.data) + evt, node = next(pd) + self.assertEqual(pulldom.PROCESSING_INSTRUCTION, evt) + self.assertEqual("target", node.target) + self.assertEqual("data", node.data) + + evt, node = next(pd) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("html", node.tagName) + + evt, node = next(pd) + self.assertEqual(pulldom.COMMENT, evt) + self.assertEqual("a comment", node.data) + evt, node = next(pd) + self.assertEqual(pulldom.PROCESSING_INSTRUCTION, evt) + self.assertEqual("target", node.target) + self.assertEqual("data", node.data) + + evt, node = next(pd) + self.assertEqual(pulldom.START_ELEMENT, evt) + self.assertEqual("p", node.tagName) + + evt, node = next(pd) + self.assertEqual(pulldom.CHARACTERS, evt) + self.assertEqual("text", node.data) + evt, node = next(pd) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("p", node.tagName) + evt, node = next(pd) + self.assertEqual(pulldom.END_ELEMENT, evt) + self.assertEqual("html", node.tagName) + evt, node = next(pd) + self.assertEqual(pulldom.END_DOCUMENT, evt) + + +class SAXExerciser(object): + """A fake sax parser that calls some of the harder-to-reach sax methods to + ensure it emits the correct events""" + + def setContentHandler(self, handler): + self._handler = handler + + def parse(self, _): + h = self._handler + h.startDocument() + + # The next two items ensure that items preceding the first + # start_element are properly stored and emitted: + h.comment("a comment") + h.processingInstruction("target", "data") + + h.startElement("html", AttributesImpl({})) + + h.comment("a comment") + h.processingInstruction("target", "data") + + h.startElement("p", AttributesImpl({"class": "paraclass"})) + h.characters("text") + h.endElement("p") + h.endElement("html") + h.endDocument() + + def stub(self, *args, **kwargs): + """Stub method. Does nothing.""" + pass + setProperty = stub + setFeature = stub + + +class SAX2DOMExerciser(SAXExerciser): + """The same as SAXExerciser, but without the processing instruction and + comment before the root element, because S2D can"t handle it""" + + def parse(self, _): + h = self._handler + h.startDocument() + h.startElement("html", AttributesImpl({})) + h.comment("a comment") + h.processingInstruction("target", "data") + h.startElement("p", AttributesImpl({"class": "paraclass"})) + h.characters("text") + h.endElement("p") + h.endElement("html") + h.endDocument() + + +class SAX2DOMTestHelper(pulldom.DOMEventStream): + """Allows us to drive SAX2DOM from a DOMEventStream.""" + + def reset(self): + self.pulldom = pulldom.SAX2DOM() + # This content handler relies on namespace support + self.parser.setFeature(xml.sax.handler.feature_namespaces, 1) + self.parser.setContentHandler(self.pulldom) + + +class SAX2DOMTestCase(unittest.TestCase): + + def confirm(self, test, testname="Test"): + self.assertTrue(test, testname) + + def test_basic(self): + """Ensure SAX2DOM can parse from a stream.""" + with io.StringIO(SMALL_SAMPLE) as fin: + sd = SAX2DOMTestHelper(fin, xml.sax.make_parser(), + len(SMALL_SAMPLE)) + for evt, node in sd: + if evt == pulldom.START_ELEMENT and node.tagName == "html": + break + # Because the buffer is the same length as the XML, all the + # nodes should have been parsed and added: + self.assertGreater(len(node.childNodes), 0) + + def testSAX2DOM(self): + """Ensure SAX2DOM expands nodes as expected.""" + sax2dom = pulldom.SAX2DOM() + sax2dom.startDocument() + sax2dom.startElement("doc", {}) + sax2dom.characters("text") + sax2dom.startElement("subelm", {}) + sax2dom.characters("text") + sax2dom.endElement("subelm") + sax2dom.characters("text") + sax2dom.endElement("doc") + sax2dom.endDocument() + + doc = sax2dom.document + root = doc.documentElement + (text1, elm1, text2) = root.childNodes + text3 = elm1.childNodes[0] + + self.assertIsNone(text1.previousSibling) + self.assertIs(text1.nextSibling, elm1) + self.assertIs(elm1.previousSibling, text1) + self.assertIs(elm1.nextSibling, text2) + self.assertIs(text2.previousSibling, elm1) + self.assertIsNone(text2.nextSibling) + self.assertIsNone(text3.previousSibling) + self.assertIsNone(text3.nextSibling) + + self.assertIs(root.parentNode, doc) + self.assertIs(text1.parentNode, root) + self.assertIs(elm1.parentNode, root) + self.assertIs(text2.parentNode, root) + self.assertIs(text3.parentNode, elm1) + doc.unlink() + + +def test_main(): + run_unittest(PullDOMTestCase, ThoroughTestCase, SAX2DOMTestCase) + + +if __name__ == "__main__": + test_main() diff --git a/Lib/test/test_pydoc.py b/Lib/test/test_pydoc.py index c139fa4..3e5c450 100644 --- a/Lib/test/test_pydoc.py +++ b/Lib/test/test_pydoc.py @@ -198,7 +198,7 @@ war</tt></dd></dl> missing_pattern = "no Python documentation found for '%s'" # output pattern for module with bad imports -badimport_pattern = "problem in %s - ImportError: No module named %s" +badimport_pattern = "problem in %s - ImportError: No module named %r" def run_pydoc(module_name, *args, **env): """ @@ -236,8 +236,8 @@ def get_pydoc_text(module): def print_diffs(text1, text2): "Prints unified diffs for two texts" # XXX now obsolete, use unittest built-in support - lines1 = text1.splitlines(True) - lines2 = text2.splitlines(True) + lines1 = text1.splitlines(keepends=True) + lines2 = text2.splitlines(keepends=True) diffs = difflib.unified_diff(lines1, lines2, n=0, fromfile='expected', tofile='got') print('\n' + ''.join(diffs)) @@ -254,6 +254,8 @@ class PydocDocTest(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') def test_html_doc(self): result, doc_loc = get_pydoc_html(pydoc_mod) mod_file = inspect.getabsfile(pydoc_mod) @@ -269,6 +271,8 @@ class PydocDocTest(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') def test_text_doc(self): result, doc_loc = get_pydoc_text(pydoc_mod) expected_text = expected_text_pattern % \ @@ -310,6 +314,8 @@ class PydocDocTest(unittest.TestCase): @unittest.skipIf(sys.flags.optimize >= 2, 'Docstrings are omitted with -O2 and above') + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'trace function introduces __locals__ unexpectedly') def test_help_output_redirect(self): # issue 940286, if output is set in Helper, then all output from # Helper.help should be redirected diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index e02c1af..92c50c7 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -130,6 +130,13 @@ class TestRaise(unittest.TestCase): with self.assertRaises(TypeError): raise MyException + def test_assert_with_tuple_arg(self): + try: + assert False, (3,) + except AssertionError as e: + self.assertEqual(str(e), "(3,)") + + class TestCause(unittest.TestCase): def test_invalid_cause(self): diff --git a/Lib/test/test_range.py b/Lib/test/test_range.py index ede0791..2e335cc 100644 --- a/Lib/test/test_range.py +++ b/Lib/test/test_range.py @@ -507,6 +507,87 @@ class RangeTest(unittest.TestCase): for k in values - {0}: r[i:j:k] + def test_comparison(self): + test_ranges = [range(0), range(0, -1), range(1, 1, 3), + range(1), range(5, 6), range(5, 6, 2), + range(5, 7, 2), range(2), range(0, 4, 2), + range(0, 5, 2), range(0, 6, 2)] + test_tuples = list(map(tuple, test_ranges)) + + # Check that equality of ranges matches equality of the corresponding + # tuples for each pair from the test lists above. + ranges_eq = [a == b for a in test_ranges for b in test_ranges] + tuples_eq = [a == b for a in test_tuples for b in test_tuples] + self.assertEqual(ranges_eq, tuples_eq) + + # Check that != correctly gives the logical negation of == + ranges_ne = [a != b for a in test_ranges for b in test_ranges] + self.assertEqual(ranges_ne, [not x for x in ranges_eq]) + + # Equal ranges should have equal hashes. + for a in test_ranges: + for b in test_ranges: + if a == b: + self.assertEqual(hash(a), hash(b)) + + # Ranges are unequal to other types (even sequence types) + self.assertIs(range(0) == (), False) + self.assertIs(() == range(0), False) + self.assertIs(range(2) == [0, 1], False) + + # Huge integers aren't a problem. + self.assertEqual(range(0, 2**100 - 1, 2), + range(0, 2**100, 2)) + self.assertEqual(hash(range(0, 2**100 - 1, 2)), + hash(range(0, 2**100, 2))) + self.assertNotEqual(range(0, 2**100, 2), + range(0, 2**100 + 1, 2)) + self.assertEqual(range(2**200, 2**201 - 2**99, 2**100), + range(2**200, 2**201, 2**100)) + self.assertEqual(hash(range(2**200, 2**201 - 2**99, 2**100)), + hash(range(2**200, 2**201, 2**100))) + self.assertNotEqual(range(2**200, 2**201, 2**100), + range(2**200, 2**201 + 1, 2**100)) + + # Order comparisons are not implemented for ranges. + with self.assertRaises(TypeError): + range(0) < range(0) + with self.assertRaises(TypeError): + range(0) > range(0) + with self.assertRaises(TypeError): + range(0) <= range(0) + with self.assertRaises(TypeError): + range(0) >= range(0) + + + def test_attributes(self): + # test the start, stop and step attributes of range objects + self.assert_attrs(range(0), 0, 0, 1) + self.assert_attrs(range(10), 0, 10, 1) + self.assert_attrs(range(-10), 0, -10, 1) + self.assert_attrs(range(0, 10, 1), 0, 10, 1) + self.assert_attrs(range(0, 10, 3), 0, 10, 3) + self.assert_attrs(range(10, 0, -1), 10, 0, -1) + self.assert_attrs(range(10, 0, -3), 10, 0, -3) + + def assert_attrs(self, rangeobj, start, stop, step): + self.assertEqual(rangeobj.start, start) + self.assertEqual(rangeobj.stop, stop) + self.assertEqual(rangeobj.step, step) + + with self.assertRaises(AttributeError): + rangeobj.start = 0 + with self.assertRaises(AttributeError): + rangeobj.stop = 10 + with self.assertRaises(AttributeError): + rangeobj.step = 1 + + with self.assertRaises(AttributeError): + del rangeobj.start + with self.assertRaises(AttributeError): + del rangeobj.stop + with self.assertRaises(AttributeError): + del rangeobj.step def test_main(): test.support.run_unittest(RangeTest) diff --git a/Lib/test/test_re.py b/Lib/test/test_re.py index fe8bc34..d23c49b 100644 --- a/Lib/test/test_re.py +++ b/Lib/test/test_re.py @@ -428,7 +428,7 @@ class ReTests(unittest.TestCase): self.assertEqual(m.span(), span) def test_re_escape(self): - alnum_chars = string.ascii_letters + string.digits + alnum_chars = string.ascii_letters + string.digits + '_' p = ''.join(chr(i) for i in range(256)) for c in p: if c in alnum_chars: @@ -441,7 +441,7 @@ class ReTests(unittest.TestCase): self.assertMatch(re.escape(p), p) def test_re_escape_byte(self): - alnum_chars = (string.ascii_letters + string.digits).encode('ascii') + alnum_chars = (string.ascii_letters + string.digits + '_').encode('ascii') p = bytes(range(256)) for i in p: b = bytes([i]) @@ -780,6 +780,13 @@ class ReTests(unittest.TestCase): self.assertRaises(OverflowError, _sre.compile, "abc", 0, [long_overflow]) self.assertRaises(TypeError, _sre.compile, {}, 0, []) + def test_search_dot_unicode(self): + self.assertIsNotNone(re.search("123.*-", '123abc-')) + self.assertIsNotNone(re.search("123.*-", '123\xe9-')) + self.assertIsNotNone(re.search("123.*-", '123\u20ac-')) + self.assertIsNotNone(re.search("123.*-", '123\U0010ffff-')) + self.assertIsNotNone(re.search("123.*-", '123\xe9\u20ac\U0010ffff-')) + def run_re_tests(): from test.re_tests import tests, SUCCEED, FAIL, SYNTAX_ERROR if verbose: diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index b0dc4d7..439fa33 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -8,7 +8,7 @@ import os import shutil import unittest -from test.support import run_unittest +from test.support import run_unittest, create_empty_file from reprlib import repr as r # Don't shadow builtin repr from reprlib import Repr from reprlib import recursive_repr @@ -193,10 +193,9 @@ class ReprTests(unittest.TestCase): r(y) r(z) -def touch(path, text=''): - fp = open(path, 'w') - fp.write(text) - fp.close() +def write_file(path, text): + with open(path, 'w', encoding='ASCII') as fp: + fp.write(text) class LongReprTest(unittest.TestCase): def setUp(self): @@ -206,10 +205,10 @@ class LongReprTest(unittest.TestCase): # Make the package and subpackage shutil.rmtree(self.pkgname, ignore_errors=True) os.mkdir(self.pkgname) - touch(os.path.join(self.pkgname, '__init__.py')) + create_empty_file(os.path.join(self.pkgname, '__init__.py')) shutil.rmtree(self.subpkgname, ignore_errors=True) os.mkdir(self.subpkgname) - touch(os.path.join(self.subpkgname, '__init__.py')) + create_empty_file(os.path.join(self.subpkgname, '__init__.py')) # Remember where we are self.here = os.getcwd() sys.path.insert(0, self.here) @@ -231,15 +230,15 @@ class LongReprTest(unittest.TestCase): def test_module(self): eq = self.assertEqual - touch(os.path.join(self.subpkgname, self.pkgname + '.py')) + create_empty_file(os.path.join(self.subpkgname, self.pkgname + '.py')) from areallylongpackageandmodulenametotestreprtruncation.areallylongpackageandmodulenametotestreprtruncation import areallylongpackageandmodulenametotestreprtruncation eq(repr(areallylongpackageandmodulenametotestreprtruncation), - "<module '%s' from '%s'>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__)) + "<module %r from %r>" % (areallylongpackageandmodulenametotestreprtruncation.__name__, areallylongpackageandmodulenametotestreprtruncation.__file__)) eq(repr(sys), "<module 'sys' (built-in)>") def test_type(self): eq = self.assertEqual - touch(os.path.join(self.subpkgname, 'foo.py'), '''\ + write_file(os.path.join(self.subpkgname, 'foo.py'), '''\ class foo(object): pass ''') @@ -253,7 +252,7 @@ class foo(object): pass def test_class(self): - touch(os.path.join(self.subpkgname, 'bar.py'), '''\ + write_file(os.path.join(self.subpkgname, 'bar.py'), '''\ class bar: pass ''') @@ -262,7 +261,7 @@ class bar: self.assertEqual(repr(bar.bar), "<class '%s.bar'>" % bar.__name__) def test_instance(self): - touch(os.path.join(self.subpkgname, 'baz.py'), '''\ + write_file(os.path.join(self.subpkgname, 'baz.py'), '''\ class baz: pass ''') @@ -273,7 +272,7 @@ class baz: def test_method(self): eq = self.assertEqual - touch(os.path.join(self.subpkgname, 'qux.py'), '''\ + write_file(os.path.join(self.subpkgname, 'qux.py'), '''\ class aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa: def amethod(self): pass ''') diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index f8f3717..0b629dc 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -220,6 +220,7 @@ class MiscTest(unittest.TestCase): for func in (do, operator.not_): self.assertRaises(Exc, func, Bad()) + @support.no_tracing def test_recursion(self): # Check that comparison for recursive objects fails gracefully from collections import UserList diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index 7ffb6af..2cede19 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -6,7 +6,9 @@ import sys import re import tempfile import py_compile -from test.support import forget, make_legacy_pyc, run_unittest, unload, verbose +from test.support import ( + forget, make_legacy_pyc, run_unittest, unload, verbose, no_tracing, + create_empty_file) from test.script_helper import ( make_pkg, make_script, make_zip_pkg, make_zip_script, temp_dir) @@ -112,8 +114,7 @@ class RunModuleTest(unittest.TestCase): def _add_pkg_dir(self, pkg_dir): os.mkdir(pkg_dir) pkg_fname = os.path.join(pkg_dir, "__init__.py") - pkg_file = open(pkg_fname, "w") - pkg_file.close() + create_empty_file(pkg_fname) return pkg_fname def _make_pkg(self, source, depth, mod_base="runpy_test"): @@ -218,8 +219,7 @@ class RunModuleTest(unittest.TestCase): module_dir = os.path.join(module_dir, pkg_name) # Add sibling module sibling_fname = os.path.join(module_dir, "sibling.py") - sibling_file = open(sibling_fname, "w") - sibling_file.close() + create_empty_file(sibling_fname) if verbose: print(" Added sibling module:", sibling_fname) # Add nephew module uncle_dir = os.path.join(parent_dir, "uncle") @@ -229,8 +229,7 @@ class RunModuleTest(unittest.TestCase): self._add_pkg_dir(cousin_dir) if verbose: print(" Added cousin package:", cousin_dir) nephew_fname = os.path.join(cousin_dir, "nephew.py") - nephew_file = open(nephew_fname, "w") - nephew_file.close() + create_empty_file(nephew_fname) if verbose: print(" Added nephew module:", nephew_fname) def _check_relative_imports(self, depth, run_name=None): @@ -395,6 +394,7 @@ argv0 = sys.argv[0] msg = "can't find '__main__' module in %r" % zip_name self._check_import_error(zip_name, msg) + @no_tracing def test_main_recursion_error(self): with temp_dir() as script_dir, temp_dir() as dummy_dir: mod_name = '__main__' diff --git a/Lib/test/test_sax.py b/Lib/test/test_sax.py index 1225d6e..8e00889 100644 --- a/Lib/test/test_sax.py +++ b/Lib/test/test_sax.py @@ -20,8 +20,8 @@ import unittest TEST_XMLFILE = findfile("test.xml", subdir="xmltestdata") TEST_XMLFILE_OUT = findfile("test.xml.out", subdir="xmltestdata") try: - TEST_XMLFILE.encode("utf8") - TEST_XMLFILE_OUT.encode("utf8") + TEST_XMLFILE.encode("utf-8") + TEST_XMLFILE_OUT.encode("utf-8") except UnicodeEncodeError: raise unittest.SkipTest("filename is not encodable to utf8") diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index fbc87aa..129a18a 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -1,5 +1,5 @@ import unittest -from test.support import check_syntax_error, run_unittest +from test.support import check_syntax_error, cpython_only, run_unittest class ScopeTests(unittest.TestCase): @@ -496,23 +496,22 @@ class ScopeTests(unittest.TestCase): self.assertNotIn("x", varnames) self.assertIn("y", varnames) + @cpython_only def testLocalsClass_WithTrace(self): # Issue23728: after the trace function returns, the locals() # dictionary is used to update all variables, this used to # include free variables. But in class statements, free # variables are not inserted... import sys + self.addCleanup(sys.settrace, sys.gettrace()) sys.settrace(lambda a,b,c:None) - try: - x = 12 + x = 12 - class C: - def f(self): - return x + class C: + def f(self): + return x - self.assertEqual(x, 12) # Used to raise UnboundLocalError - finally: - sys.settrace(None) + self.assertEqual(x, 12) # Used to raise UnboundLocalError def testBoundAndFree(self): # var is bound and free in class @@ -527,6 +526,7 @@ class ScopeTests(unittest.TestCase): inst = f(3)() self.assertEqual(inst.a, inst.m()) + @cpython_only def testInteractionWithTraceFunc(self): import sys @@ -543,6 +543,7 @@ class ScopeTests(unittest.TestCase): class TestClass: pass + self.addCleanup(sys.settrace, sys.gettrace()) sys.settrace(tracer) adaptgetter("foo", TestClass, (1, "")) sys.settrace(None) diff --git a/Lib/test/test_select.py b/Lib/test/test_select.py index fe92f45..3144c54 100644 --- a/Lib/test/test_select.py +++ b/Lib/test/test_select.py @@ -1,8 +1,9 @@ -from test import support -import unittest -import select +import errno import os +import select import sys +import unittest +from test import support @unittest.skipIf(sys.platform[:3] in ('win', 'os2', 'riscos'), "can't easily test on this system") @@ -20,6 +21,21 @@ class SelectTestCase(unittest.TestCase): self.assertRaises(TypeError, select.select, [self.Nope()], [], []) self.assertRaises(TypeError, select.select, [self.Almost()], [], []) self.assertRaises(TypeError, select.select, [], [], [], "not a number") + self.assertRaises(ValueError, select.select, [], [], [], -1) + + # Issue #12367: http://www.freebsd.org/cgi/query-pr.cgi?pr=kern/155606 + @unittest.skipIf(sys.platform.startswith('freebsd'), + 'skip because of a FreeBSD bug: kern/155606') + def test_errno(self): + with open(__file__, 'rb') as fp: + fd = fp.fileno() + fp.close() + try: + select.select([fd], [], [], 0) + except select.error as err: + self.assertEqual(err.errno, errno.EBADF) + else: + self.fail("exception not raised") def test_returned_list_identity(self): # See issue #8329 diff --git a/Lib/test/test_shelve.py b/Lib/test/test_shelve.py index 3e73f52..13c1265 100644 --- a/Lib/test/test_shelve.py +++ b/Lib/test/test_shelve.py @@ -2,7 +2,7 @@ import unittest import shelve import glob from test import support -from collections import MutableMapping +from collections.abc import MutableMapping from test.test_dbm import dbm_iterator def L1(s): @@ -129,8 +129,8 @@ class TestCase(unittest.TestCase): shelve.Shelf(d)[key] = [1] self.assertIn(key.encode('utf-8'), d) # but a different one can be given - shelve.Shelf(d, keyencoding='latin1')[key] = [1] - self.assertIn(key.encode('latin1'), d) + shelve.Shelf(d, keyencoding='latin-1')[key] = [1] + self.assertIn(key.encode('latin-1'), d) # with all consequences s = shelve.Shelf(d, keyencoding='ascii') self.assertRaises(UnicodeEncodeError, s.__setitem__, key, [1]) diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index 25e4b6d..d4463f30 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -1,6 +1,7 @@ -import unittest -import os, sys, io +import io import shlex +import string +import unittest from test import support @@ -173,6 +174,22 @@ class ShlexTest(unittest.TestCase): "%s: %s != %s" % (self.data[i][0], l, self.data[i][1:])) + def testQuote(self): + safeunquoted = string.ascii_letters + string.digits + '@%_-+=:,./' + unicode_sample = '\xe9\xe0\xdf' # e + acute accent, a + grave, sharp s + unsafe = '"`$\\!' + unicode_sample + + self.assertEqual(shlex.quote(''), "''") + self.assertEqual(shlex.quote(safeunquoted), safeunquoted) + self.assertEqual(shlex.quote('test file name'), "'test file name'") + for u in unsafe: + self.assertEqual(shlex.quote('test%sname' % u), + "'test%sname'" % u) + for u in unsafe: + self.assertEqual(shlex.quote("test%s'name'" % u), + "'test%s'\"'\"'name'\"'\"''" % u) + + # Allow this test to be used with old shlex.py if not getattr(shlex, "split", None): for methname in dir(ShlexTest): diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index e2310e2..b169803 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -21,7 +21,7 @@ import tarfile import warnings from test import support -from test.support import TESTFN, check_warnings, captured_stdout +from test.support import TESTFN, check_warnings, captured_stdout, requires_zlib try: import bz2 @@ -39,11 +39,6 @@ except ImportError: UID_GID_SUPPORT = False try: - import zlib -except ImportError: - zlib = None - -try: import zipfile ZIP_SUPPORT = True except ImportError: @@ -64,6 +59,31 @@ def mock_rename(func): os.rename = builtin_rename return wrap +def write_file(path, content, binary=False): + """Write *content* to a file located at *path*. + + If *path* is a tuple instead of a string, os.path.join will be used to + make a path. If *binary* is true, the file will be opened in binary + mode. + """ + if isinstance(path, tuple): + path = os.path.join(*path) + with open(path, 'wb' if binary else 'w') as fp: + fp.write(content) + +def read_file(path, binary=False): + """Return contents from a file located at *path*. + + If *path* is a tuple instead of a string, os.path.join will be used to + make a path. If *binary* is true, the file will be opened in binary + mode. + """ + if isinstance(path, tuple): + path = os.path.join(*path) + with open(path, 'rb' if binary else 'r') as fp: + return fp.read() + + class TestShutil(unittest.TestCase): def setUp(self): @@ -76,19 +96,6 @@ class TestShutil(unittest.TestCase): d = self.tempdirs.pop() shutil.rmtree(d, os.name in ('nt', 'cygwin')) - def write_file(self, path, content='xxx'): - """Writes a file in the given path. - - - path can be a string or a sequence. - """ - if isinstance(path, (list, tuple)): - path = os.path.join(*path) - f = open(path, 'w') - try: - f.write(content) - finally: - f.close() def mkdtemp(self): """Create a temporary directory that will be cleaned up. @@ -112,8 +119,7 @@ class TestShutil(unittest.TestCase): self.errorState = 0 os.mkdir(TESTFN) self.childpath = os.path.join(TESTFN, 'a') - f = open(self.childpath, 'w') - f.close() + support.create_empty_file(self.childpath) old_dir_mode = os.stat(TESTFN).st_mode old_child_mode = os.stat(self.childpath).st_mode # Make unwritable. @@ -161,81 +167,46 @@ class TestShutil(unittest.TestCase): def test_rmtree_dont_delete_file(self): # When called on a file instead of a directory, don't delete it. handle, path = tempfile.mkstemp() - os.fdopen(handle).close() + os.close(handle) self.assertRaises(OSError, shutil.rmtree, path) os.remove(path) - def _write_data(self, path, data): - f = open(path, "w") - f.write(data) - f.close() - def test_copytree_simple(self): - - def read_data(path): - f = open(path) - data = f.read() - f.close() - return data - src_dir = tempfile.mkdtemp() dst_dir = os.path.join(tempfile.mkdtemp(), 'destination') - self._write_data(os.path.join(src_dir, 'test.txt'), '123') + self.addCleanup(shutil.rmtree, src_dir) + self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) + write_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') - - try: - shutil.copytree(src_dir, dst_dir) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) - self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir'))) - self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir', - 'test.txt'))) - actual = read_data(os.path.join(dst_dir, 'test.txt')) - self.assertEqual(actual, '123') - actual = read_data(os.path.join(dst_dir, 'test_dir', 'test.txt')) - self.assertEqual(actual, '456') - finally: - for path in ( - os.path.join(src_dir, 'test.txt'), - os.path.join(dst_dir, 'test.txt'), - os.path.join(src_dir, 'test_dir', 'test.txt'), - os.path.join(dst_dir, 'test_dir', 'test.txt'), - ): - if os.path.exists(path): - os.remove(path) - for path in (src_dir, - os.path.dirname(dst_dir) - ): - if os.path.exists(path): - shutil.rmtree(path) + write_file((src_dir, 'test_dir', 'test.txt'), '456') + + shutil.copytree(src_dir, dst_dir) + self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) + self.assertTrue(os.path.isdir(os.path.join(dst_dir, 'test_dir'))) + self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test_dir', + 'test.txt'))) + actual = read_file((dst_dir, 'test.txt')) + self.assertEqual(actual, '123') + actual = read_file((dst_dir, 'test_dir', 'test.txt')) + self.assertEqual(actual, '456') def test_copytree_with_exclude(self): - - def read_data(path): - f = open(path) - data = f.read() - f.close() - return data - # creating data join = os.path.join exists = os.path.exists src_dir = tempfile.mkdtemp() try: dst_dir = join(tempfile.mkdtemp(), 'destination') - self._write_data(join(src_dir, 'test.txt'), '123') - self._write_data(join(src_dir, 'test.tmp'), '123') + write_file((src_dir, 'test.txt'), '123') + write_file((src_dir, 'test.tmp'), '123') os.mkdir(join(src_dir, 'test_dir')) - self._write_data(join(src_dir, 'test_dir', 'test.txt'), '456') + write_file((src_dir, 'test_dir', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2')) - self._write_data(join(src_dir, 'test_dir2', 'test.txt'), '456') + write_file((src_dir, 'test_dir2', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2', 'subdir')) os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) - self._write_data(join(src_dir, 'test_dir2', 'subdir', 'test.txt'), - '456') - self._write_data(join(src_dir, 'test_dir2', 'subdir2', 'test.py'), - '456') - + write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') + write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') # testing glob-like patterns try: @@ -243,21 +214,19 @@ class TestShutil(unittest.TestCase): shutil.copytree(src_dir, dst_dir, ignore=patterns) # checking the result: some elements should not be copied self.assertTrue(exists(join(dst_dir, 'test.txt'))) - self.assertTrue(not exists(join(dst_dir, 'test.tmp'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2'))) + self.assertFalse(exists(join(dst_dir, 'test.tmp'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2'))) finally: - if os.path.exists(dst_dir): - shutil.rmtree(dst_dir) + shutil.rmtree(dst_dir) try: patterns = shutil.ignore_patterns('*.tmp', 'subdir*') shutil.copytree(src_dir, dst_dir, ignore=patterns) # checking the result: some elements should not be copied - self.assertTrue(not exists(join(dst_dir, 'test.tmp'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir2'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir'))) + self.assertFalse(exists(join(dst_dir, 'test.tmp'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir'))) finally: - if os.path.exists(dst_dir): - shutil.rmtree(dst_dir) + shutil.rmtree(dst_dir) # testing callable-style try: @@ -276,13 +245,12 @@ class TestShutil(unittest.TestCase): shutil.copytree(src_dir, dst_dir, ignore=_filter) # checking the result: some elements should not be copied - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir2', - 'test.py'))) - self.assertTrue(not exists(join(dst_dir, 'test_dir2', 'subdir'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir2', + 'test.py'))) + self.assertFalse(exists(join(dst_dir, 'test_dir2', 'subdir'))) finally: - if os.path.exists(dst_dir): - shutil.rmtree(dst_dir) + shutil.rmtree(dst_dir) finally: shutil.rmtree(src_dir) shutil.rmtree(os.path.dirname(dst_dir)) @@ -377,9 +345,9 @@ class TestShutil(unittest.TestCase): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') - self._write_data(os.path.join(src_dir, 'test.txt'), '123') + write_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') + write_file((src_dir, 'test_dir', 'test.txt'), '456') copied = [] def _copy(src, dst): @@ -396,7 +364,7 @@ class TestShutil(unittest.TestCase): dst_dir = os.path.join(self.mkdtemp(), 'destination') os.symlink('IDONTEXIST', os.path.join(src_dir, 'test.txt')) os.mkdir(os.path.join(src_dir, 'test_dir')) - self._write_data(os.path.join(src_dir, 'test_dir', 'test.txt'), '456') + write_file((src_dir, 'test_dir', 'test.txt'), '456') self.assertRaises(Error, shutil.copytree, src_dir, dst_dir) # a dangling symlink is ignored with the proper flag @@ -412,7 +380,7 @@ class TestShutil(unittest.TestCase): def _copy_file(self, method): fname = 'test.txt' tmpdir = self.mkdtemp() - self.write_file([tmpdir, fname]) + write_file((tmpdir, fname), 'xxx') file1 = os.path.join(tmpdir, fname) tmpdir2 = self.mkdtemp() method(file1, tmpdir2) @@ -444,14 +412,14 @@ class TestShutil(unittest.TestCase): self.assertEqual(getattr(file1_stat, 'st_flags'), getattr(file2_stat, 'st_flags')) - @unittest.skipUnless(zlib, "requires zlib") + @requires_zlib def test_make_tarball(self): # creating something to tar tmpdir = self.mkdtemp() - self.write_file([tmpdir, 'file1'], 'xxx') - self.write_file([tmpdir, 'file2'], 'xxx') + write_file((tmpdir, 'file1'), 'xxx') + write_file((tmpdir, 'file2'), 'xxx') os.mkdir(os.path.join(tmpdir, 'sub')) - self.write_file([tmpdir, 'sub', 'file3'], 'xxx') + write_file((tmpdir, 'sub', 'file3'), 'xxx') tmpdir2 = self.mkdtemp() # force shutil to create the directory @@ -498,16 +466,16 @@ class TestShutil(unittest.TestCase): tmpdir = self.mkdtemp() dist = os.path.join(tmpdir, 'dist') os.mkdir(dist) - self.write_file([dist, 'file1'], 'xxx') - self.write_file([dist, 'file2'], 'xxx') + write_file((dist, 'file1'), 'xxx') + write_file((dist, 'file2'), 'xxx') os.mkdir(os.path.join(dist, 'sub')) - self.write_file([dist, 'sub', 'file3'], 'xxx') + write_file((dist, 'sub', 'file3'), 'xxx') os.mkdir(os.path.join(dist, 'sub2')) tmpdir2 = self.mkdtemp() base_name = os.path.join(tmpdir2, 'archive') return tmpdir, tmpdir2, base_name - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib @unittest.skipUnless(find_executable('tar') and find_executable('gzip'), 'Need the tar command to run') def test_tarfile_vs_tar(self): @@ -562,13 +530,13 @@ class TestShutil(unittest.TestCase): tarball = base_name + '.tar' self.assertTrue(os.path.exists(tarball)) - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib @unittest.skipUnless(ZIP_SUPPORT, 'Need zip support to run') def test_make_zipfile(self): # creating something to tar tmpdir = self.mkdtemp() - self.write_file([tmpdir, 'file1'], 'xxx') - self.write_file([tmpdir, 'file2'], 'xxx') + write_file((tmpdir, 'file1'), 'xxx') + write_file((tmpdir, 'file2'), 'xxx') tmpdir2 = self.mkdtemp() # force shutil to create the directory @@ -586,7 +554,7 @@ class TestShutil(unittest.TestCase): base_name = os.path.join(tmpdir, 'archive') self.assertRaises(ValueError, make_archive, base_name, 'xxx') - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib def test_make_archive_owner_group(self): # testing make_archive with owner and group, with various combinations # this works even if there's not gid/uid support @@ -614,7 +582,7 @@ class TestShutil(unittest.TestCase): self.assertTrue(os.path.exists(res)) - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") def test_tarfile_root_owner(self): tmpdir, tmpdir2, base_name = self._create_files() @@ -683,7 +651,7 @@ class TestShutil(unittest.TestCase): diff.append(file_) return diff - @unittest.skipUnless(zlib, "Requires zlib") + @requires_zlib def test_unpack_archive(self): formats = ['tar', 'gztar', 'zip'] if BZ2_SUPPORTED: @@ -734,6 +702,75 @@ class TestShutil(unittest.TestCase): unregister_unpack_format('Boo2') self.assertEqual(get_unpack_formats(), formats) + @unittest.skipUnless(hasattr(shutil, 'disk_usage'), + "disk_usage not available on this platform") + def test_disk_usage(self): + usage = shutil.disk_usage(os.getcwd()) + self.assertGreater(usage.total, 0) + self.assertGreater(usage.used, 0) + self.assertGreaterEqual(usage.free, 0) + self.assertGreaterEqual(usage.total, usage.used) + self.assertGreater(usage.total, usage.free) + + @unittest.skipUnless(UID_GID_SUPPORT, "Requires grp and pwd support") + @unittest.skipUnless(hasattr(os, 'chown'), 'requires os.chown') + def test_chown(self): + + # cleaned-up automatically by TestShutil.tearDown method + dirname = self.mkdtemp() + filename = tempfile.mktemp(dir=dirname) + write_file(filename, 'testing chown function') + + with self.assertRaises(ValueError): + shutil.chown(filename) + + with self.assertRaises(LookupError): + shutil.chown(filename, user='non-exising username') + + with self.assertRaises(LookupError): + shutil.chown(filename, group='non-exising groupname') + + with self.assertRaises(TypeError): + shutil.chown(filename, b'spam') + + with self.assertRaises(TypeError): + shutil.chown(filename, 3.14) + + uid = os.getuid() + gid = os.getgid() + + def check_chown(path, uid=None, gid=None): + s = os.stat(filename) + if uid is not None: + self.assertEqual(uid, s.st_uid) + if gid is not None: + self.assertEqual(gid, s.st_gid) + + shutil.chown(filename, uid, gid) + check_chown(filename, uid, gid) + shutil.chown(filename, uid) + check_chown(filename, uid) + shutil.chown(filename, user=uid) + check_chown(filename, uid) + shutil.chown(filename, group=gid) + check_chown(filename, gid=gid) + + shutil.chown(dirname, uid, gid) + check_chown(dirname, uid, gid) + shutil.chown(dirname, uid) + check_chown(dirname, uid) + shutil.chown(dirname, user=uid) + check_chown(dirname, uid) + shutil.chown(dirname, group=gid) + check_chown(dirname, gid=gid) + + user = pwd.getpwuid(uid)[0] + group = grp.getgrgid(gid)[0] + shutil.chown(filename, user, group) + check_chown(filename, uid, gid) + shutil.chown(dirname, user, group) + check_chown(dirname, uid, gid) + class TestMove(unittest.TestCase): @@ -965,8 +1002,7 @@ class TestCopyFile(unittest.TestCase): shutil.move(self.src_dir, dst_dir) self.assertTrue(os.path.isdir(dst_dir)) finally: - if os.path.exists(dst_dir): - os.rmdir(dst_dir) + os.rmdir(dst_dir) diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index 8df1bf0..fdeb4c2 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -1,17 +1,19 @@ -import errno +import unittest +from test import support +from contextlib import closing import gc -import os import pickle import select import signal +import struct import subprocess -import sys -import time import traceback -import unittest -from test import support -from contextlib import closing +import sys, os, time, errno from test.script_helper import assert_python_ok, spawn_python +try: + import threading +except ImportError: + threading = None if sys.platform in ('os2', 'riscos'): raise unittest.SkipTest("Can't test signal on %s" % sys.platform) @@ -57,15 +59,9 @@ class InterProcessSignalTests(unittest.TestCase): def handlerA(self, signum, frame): self.a_called = True - if support.verbose: - print("handlerA invoked from signal %s at:\n%s" % ( - signum, self.format_frame(frame, limit=1))) def handlerB(self, signum, frame): self.b_called = True - if support.verbose: - print ("handlerB invoked from signal %s at:\n%s" % ( - signum, self.format_frame(frame, limit=1))) raise HandlerBCalled(signum, self.format_frame(frame)) def wait(self, child): @@ -92,8 +88,6 @@ class InterProcessSignalTests(unittest.TestCase): # Let the sub-processes know who to send signals to. pid = os.getpid() - if support.verbose: - print("test runner's pid is", pid) child = ignoring_eintr(subprocess.Popen, ['kill', '-HUP', str(pid)]) if child: @@ -117,8 +111,6 @@ class InterProcessSignalTests(unittest.TestCase): except HandlerBCalled: self.assertTrue(self.b_called) self.assertFalse(self.a_called) - if support.verbose: - print("HandlerBCalled exception caught") child = ignoring_eintr(subprocess.Popen, ['kill', '-USR2', str(pid)]) if child: @@ -134,8 +126,7 @@ class InterProcessSignalTests(unittest.TestCase): # may return early. time.sleep(1) except KeyboardInterrupt: - if support.verbose: - print("KeyboardInterrupt (the alarm() went off)") + pass except: self.fail("Some other exception woke us from pause: %s" % traceback.format_exc()) @@ -191,7 +182,7 @@ class InterProcessSignalTests(unittest.TestCase): @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") -class BasicSignalTests(unittest.TestCase): +class PosixTests(unittest.TestCase): def trivial_signal_handler(self, *args): pass @@ -233,31 +224,44 @@ class WindowsSignalTests(unittest.TestCase): @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") class WakeupSignalTests(unittest.TestCase): - def check_wakeup(self, test_body): - # use a subprocess to have only one thread and to not change signal - # handling of the parent process + def check_wakeup(self, test_body, *signals, ordered=True): + # use a subprocess to have only one thread code = """if 1: import fcntl import os import signal + import struct + + signals = {!r} def handler(signum, frame): pass + def check_signum(signals): + data = os.read(read, len(signals)+1) + raised = struct.unpack('%uB' % len(data), data) + if not {!r}: + raised = set(raised) + signals = set(signals) + if raised != signals: + raise Exception("%r != %r" % (raised, signals)) + {} signal.signal(signal.SIGALRM, handler) read, write = os.pipe() - flags = fcntl.fcntl(write, fcntl.F_GETFL, 0) - flags = flags | os.O_NONBLOCK - fcntl.fcntl(write, fcntl.F_SETFL, flags) + for fd in (read, write): + flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0) + flags = flags | os.O_NONBLOCK + fcntl.fcntl(fd, fcntl.F_SETFL, flags) signal.set_wakeup_fd(write) test() + check_signum(signals) os.close(read) os.close(write) - """.format(test_body) + """.format(signals, ordered, test_body) assert_python_ok('-c', code) @@ -283,7 +287,7 @@ class WakeupSignalTests(unittest.TestCase): dt = after_time - mid_time if dt >= TIMEOUT_HALF: raise Exception("%s >= %s" % (dt, TIMEOUT_HALF)) - """) + """, signal.SIGALRM) def test_wakeup_fd_during(self): self.check_wakeup("""def test(): @@ -306,7 +310,32 @@ class WakeupSignalTests(unittest.TestCase): dt = after_time - before_time if dt >= TIMEOUT_HALF: raise Exception("%s >= %s" % (dt, TIMEOUT_HALF)) - """) + """, signal.SIGALRM) + + def test_signum(self): + self.check_wakeup("""def test(): + signal.signal(signal.SIGUSR1, handler) + os.kill(os.getpid(), signal.SIGUSR1) + os.kill(os.getpid(), signal.SIGALRM) + """, signal.SIGUSR1, signal.SIGALRM) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_pending(self): + self.check_wakeup("""def test(): + signum1 = signal.SIGUSR1 + signum2 = signal.SIGUSR2 + + signal.signal(signum1, handler) + signal.signal(signum2, handler) + + signal.pthread_sigmask(signal.SIG_BLOCK, (signum1, signum2)) + os.kill(os.getpid(), signum1) + os.kill(os.getpid(), signum2) + # Unblocking the 2 signals calls the C signal handler twice + signal.pthread_sigmask(signal.SIG_UNBLOCK, (signum1, signum2)) + """, signal.SIGUSR1, signal.SIGUSR2, ordered=False) + @unittest.skipIf(sys.platform == "win32", "Not valid on Windows") class SiginterruptTest(unittest.TestCase): @@ -316,9 +345,6 @@ class SiginterruptTest(unittest.TestCase): read is interrupted by the signal and raises an exception. Return False if it returns normally. """ - class Timeout(Exception): - pass - # use a subprocess to have only one thread, to have a timeout on the # blocking read and to not touch signal handling in this process code = """if 1: @@ -359,18 +385,8 @@ class SiginterruptTest(unittest.TestCase): # wait until the child process is loaded and has started first_line = process.stdout.readline() - # Wait the process with a timeout of 5 seconds - timeout = time.time() + 5.0 - while True: - if timeout < time.time(): - raise Timeout() - status = process.poll() - if status is not None: - break - time.sleep(0.1) - - stdout, stderr = process.communicate() - except Timeout: + stdout, stderr = process.communicate(timeout=5.0) + except subprocess.TimeoutExpired: process.kill() return False else: @@ -419,8 +435,6 @@ class ItimerTest(unittest.TestCase): def sig_alrm(self, *args): self.hndl_called = True - if support.verbose: - print("SIGALRM handler invoked", args) def sig_vtalrm(self, *args): self.hndl_called = True @@ -432,21 +446,13 @@ class ItimerTest(unittest.TestCase): elif self.hndl_count == 3: # disable ITIMER_VIRTUAL, this function shouldn't be called anymore signal.setitimer(signal.ITIMER_VIRTUAL, 0) - if support.verbose: - print("last SIGVTALRM handler call") self.hndl_count += 1 - if support.verbose: - print("SIGVTALRM handler invoked", args) - def sig_prof(self, *args): self.hndl_called = True signal.setitimer(signal.ITIMER_PROF, 0) - if support.verbose: - print("SIGPROF handler invoked", args) - def test_itimer_exc(self): # XXX I'm assuming -1 is an invalid itimer, but maybe some platform # defines it ? @@ -459,10 +465,7 @@ class ItimerTest(unittest.TestCase): def test_itimer_real(self): self.itimer = signal.ITIMER_REAL signal.setitimer(self.itimer, 1.0) - if support.verbose: - print("\ncall pause()...") signal.pause() - self.assertEqual(self.hndl_called, True) # Issue 3864, unknown if this affects earlier versions of freebsd also @@ -511,11 +514,361 @@ class ItimerTest(unittest.TestCase): # and the handler should have been called self.assertEqual(self.hndl_called, True) + +class PendingSignalsTests(unittest.TestCase): + """ + Test pthread_sigmask(), pthread_kill(), sigpending() and sigwait() + functions. + """ + @unittest.skipUnless(hasattr(signal, 'sigpending'), + 'need signal.sigpending()') + def test_sigpending_empty(self): + self.assertEqual(signal.sigpending(), set()) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + @unittest.skipUnless(hasattr(signal, 'sigpending'), + 'need signal.sigpending()') + def test_sigpending(self): + code = """if 1: + import os + import signal + + def handler(signum, frame): + 1/0 + + signum = signal.SIGUSR1 + signal.signal(signum, handler) + + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + os.kill(os.getpid(), signum) + pending = signal.sigpending() + if pending != {signum}: + raise Exception('%s != {%s}' % (pending, signum)) + try: + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + """ + assert_python_ok('-c', code) + + @unittest.skipUnless(hasattr(signal, 'pthread_kill'), + 'need signal.pthread_kill()') + def test_pthread_kill(self): + code = """if 1: + import signal + import threading + import sys + + signum = signal.SIGUSR1 + + def handler(signum, frame): + 1/0 + + signal.signal(signum, handler) + + if sys.platform == 'freebsd6': + # Issue #12392 and #12469: send a signal to the main thread + # doesn't work before the creation of the first thread on + # FreeBSD 6 + def noop(): + pass + thread = threading.Thread(target=noop) + thread.start() + thread.join() + + tid = threading.get_ident() + try: + signal.pthread_kill(tid, signum) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + """ + assert_python_ok('-c', code) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def wait_helper(self, blocked, test): + """ + test: body of the "def test(signum):" function. + blocked: number of the blocked signal + """ + code = '''if 1: + import signal + import sys + + def handler(signum, frame): + 1/0 + + %s + + blocked = %s + signum = signal.SIGALRM + + # child: block and wait the signal + try: + signal.signal(signum, handler) + signal.pthread_sigmask(signal.SIG_BLOCK, [blocked]) + + # Do the tests + test(signum) + + # The handler must not be called on unblock + try: + signal.pthread_sigmask(signal.SIG_UNBLOCK, [blocked]) + except ZeroDivisionError: + print("the signal handler has been called", + file=sys.stderr) + sys.exit(1) + except BaseException as err: + print("error: {}".format(err), file=sys.stderr) + sys.stderr.flush() + sys.exit(1) + ''' % (test.strip(), blocked) + + # sig*wait* must be called with the signal blocked: since the current + # process might have several threads running, use a subprocess to have + # a single thread. + assert_python_ok('-c', code) + + @unittest.skipUnless(hasattr(signal, 'sigwait'), + 'need signal.sigwait()') + def test_sigwait(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + signal.alarm(1) + received = signal.sigwait([signum]) + if received != signum: + raise Exception('received %s, not %s' % (received, signum)) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + signal.alarm(1) + info = signal.sigwaitinfo([signum]) + if info.si_signo != signum: + raise Exception("info.si_signo != %s" % signum) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + signal.alarm(1) + info = signal.sigtimedwait([signum], (10, 1000)) + if info.si_signo != signum: + raise Exception('info.si_signo != %s' % signum) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait_poll(self): + # check that polling with sigtimedwait works + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + import os + os.kill(os.getpid(), signum) + info = signal.sigtimedwait([signum], (0, 0)) + if info.si_signo != signum: + raise Exception('info.si_signo != %s' % signum) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait_timeout(self): + self.wait_helper(signal.SIGALRM, ''' + def test(signum): + received = signal.sigtimedwait([signum], (1, 0)) + if received is not None: + raise Exception("received=%r" % (received,)) + ''') + + @unittest.skipUnless(hasattr(signal, 'sigtimedwait'), + 'need signal.sigtimedwait()') + def test_sigtimedwait_negative_timeout(self): + signum = signal.SIGALRM + self.assertRaises(ValueError, signal.sigtimedwait, [signum], (-1, -1)) + self.assertRaises(ValueError, signal.sigtimedwait, [signum], (0, -1)) + self.assertRaises(ValueError, signal.sigtimedwait, [signum], (-1, 0)) + + @unittest.skipUnless(hasattr(signal, 'sigwaitinfo'), + 'need signal.sigwaitinfo()') + def test_sigwaitinfo_interrupted(self): + self.wait_helper(signal.SIGUSR1, ''' + def test(signum): + import errno + + hndl_called = True + def alarm_handler(signum, frame): + hndl_called = False + + signal.signal(signal.SIGALRM, alarm_handler) + signal.alarm(1) + try: + signal.sigwaitinfo([signal.SIGUSR1]) + except OSError as e: + if e.errno == errno.EINTR: + if not hndl_called: + raise Exception("SIGALRM handler not called") + else: + raise Exception("Expected EINTR to be raised by sigwaitinfo") + else: + raise Exception("Expected EINTR to be raised by sigwaitinfo") + ''') + + @unittest.skipUnless(hasattr(signal, 'sigwait'), + 'need signal.sigwait()') + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + @unittest.skipIf(threading is None, "test needs threading module") + def test_sigwait_thread(self): + # Check that calling sigwait() from a thread doesn't suspend the whole + # process. A new interpreter is spawned to avoid problems when mixing + # threads and fork(): only async-safe functions are allowed between + # fork() and exec(). + assert_python_ok("-c", """if True: + import os, threading, sys, time, signal + + # the default handler terminates the process + signum = signal.SIGUSR1 + + def kill_later(): + # wait until the main thread is waiting in sigwait() + time.sleep(1) + os.kill(os.getpid(), signum) + + # the signal must be blocked by all the threads + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + killer = threading.Thread(target=kill_later) + killer.start() + received = signal.sigwait([signum]) + if received != signum: + print("sigwait() received %s, not %s" % (received, signum), + file=sys.stderr) + sys.exit(1) + killer.join() + # unblock the signal, which should have been cleared by sigwait() + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + """) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_pthread_sigmask_arguments(self): + self.assertRaises(TypeError, signal.pthread_sigmask) + self.assertRaises(TypeError, signal.pthread_sigmask, 1) + self.assertRaises(TypeError, signal.pthread_sigmask, 1, 2, 3) + self.assertRaises(OSError, signal.pthread_sigmask, 1700, []) + + @unittest.skipUnless(hasattr(signal, 'pthread_sigmask'), + 'need signal.pthread_sigmask()') + def test_pthread_sigmask(self): + code = """if 1: + import signal + import os; import threading + + def handler(signum, frame): + 1/0 + + def kill(signum): + os.kill(os.getpid(), signum) + + def read_sigmask(): + return signal.pthread_sigmask(signal.SIG_BLOCK, []) + + signum = signal.SIGUSR1 + + # Install our signal handler + old_handler = signal.signal(signum, handler) + + # Unblock SIGUSR1 (and copy the old mask) to test our signal handler + old_mask = signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + try: + kill(signum) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + + # Block and then raise SIGUSR1. The signal is blocked: the signal + # handler is not called, and the signal is now pending + signal.pthread_sigmask(signal.SIG_BLOCK, [signum]) + kill(signum) + + # Check the new mask + blocked = read_sigmask() + if signum not in blocked: + raise Exception("%s not in %s" % (signum, blocked)) + if old_mask ^ blocked != {signum}: + raise Exception("%s ^ %s != {%s}" % (old_mask, blocked, signum)) + + # Unblock SIGUSR1 + try: + # unblock the pending signal calls immediatly the signal handler + signal.pthread_sigmask(signal.SIG_UNBLOCK, [signum]) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + try: + kill(signum) + except ZeroDivisionError: + pass + else: + raise Exception("ZeroDivisionError not raised") + + # Check the new mask + unblocked = read_sigmask() + if signum in unblocked: + raise Exception("%s in %s" % (signum, unblocked)) + if blocked ^ unblocked != {signum}: + raise Exception("%s ^ %s != {%s}" % (blocked, unblocked, signum)) + if old_mask != unblocked: + raise Exception("%s != %s" % (old_mask, unblocked)) + """ + assert_python_ok('-c', code) + + @unittest.skipIf(sys.platform == 'freebsd6', + "issue #12392: send a signal to the main thread doesn't work " + "before the creation of the first thread on FreeBSD 6") + @unittest.skipUnless(hasattr(signal, 'pthread_kill'), + 'need signal.pthread_kill()') + def test_pthread_kill_main_thread(self): + # Test that a signal can be sent to the main thread with pthread_kill() + # before any other thread has been created (see issue #12392). + code = """if True: + import threading + import signal + import sys + + def handler(signum, frame): + sys.exit(3) + + signal.signal(signal.SIGUSR1, handler) + signal.pthread_kill(threading.get_ident(), signal.SIGUSR1) + sys.exit(2) + """ + + with spawn_python('-c', code) as process: + stdout, stderr = process.communicate() + exitcode = process.wait() + if exitcode != 3: + raise Exception("Child error (exit code %s): %s" % + (exitcode, stdout)) + + def test_main(): try: - support.run_unittest(BasicSignalTests, InterProcessSignalTests, + support.run_unittest(PosixTests, InterProcessSignalTests, WakeupSignalTests, SiginterruptTest, - ItimerTest, WindowsSignalTests) + ItimerTest, WindowsSignalTests, + PendingSignalsTests) finally: support.reap_children() diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 2cb0d1a..05d97ef 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -72,6 +72,14 @@ class GeneralTests(unittest.TestCase): smtp = smtplib.SMTP(HOST, self.port) smtp.close() + def testSourceAddress(self): + mock_socket.reply_with(b"220 Hola mundo") + # connects + smtp = smtplib.SMTP(HOST, self.port, + source_address=('127.0.0.1',19876)) + self.assertEqual(smtp.source_address, ('127.0.0.1', 19876)) + smtp.close() + def testBasic2(self): mock_socket.reply_with(b"220 Hola mundo") # connects, include port in host name @@ -204,6 +212,20 @@ class DebuggingServerTests(unittest.TestCase): smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) smtp.quit() + def testSourceAddress(self): + # connect + port = support.find_unused_port() + try: + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=3, source_address=('127.0.0.1', port)) + self.assertEqual(smtp.source_address, ('127.0.0.1', port)) + self.assertEqual(smtp.local_hostname, 'localhost') + smtp.quit() + except IOError as e: + if e.errno == errno.EADDRINUSE: + self.skipTest("couldn't bind to port %d" % port) + raise + def testNOOP(self): smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', timeout=3) expected = (250, b'Ok') @@ -560,6 +582,9 @@ sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'], # Simulated SMTP channel & server class SimSMTPChannel(smtpd.SMTPChannel): + # For testing failures in QUIT when using the context manager API. + quit_response = None + def __init__(self, extra_features, *args, **kw): self._extrafeatures = ''.join( [ "250-{0}\r\n".format(x) for x in extra_features ]) @@ -610,19 +635,31 @@ class SimSMTPChannel(smtpd.SMTPChannel): else: self.push('550 No access for you!') + def smtp_QUIT(self, arg): + # args is ignored + if self.quit_response is None: + super(SimSMTPChannel, self).smtp_QUIT(arg) + else: + self.push(self.quit_response) + self.close_when_done() + def handle_error(self): raise class SimSMTPServer(smtpd.SMTPServer): + # For testing failures in QUIT when using the context manager API. + quit_response = None + def __init__(self, *args, **kw): self._extra_features = [] smtpd.SMTPServer.__init__(self, *args, **kw) def handle_accepted(self, conn, addr): - self._SMTPchannel = SimSMTPChannel(self._extra_features, - self, conn, addr) + self._SMTPchannel = SimSMTPChannel( + self._extra_features, self, conn, addr) + self._SMTPchannel.quit_response = self.quit_response def process_message(self, peer, mailfrom, rcpttos, data): pass @@ -752,6 +789,25 @@ class SMTPSimTests(unittest.TestCase): self.assertIn(sim_auth_credentials['cram-md5'], str(err)) smtp.close() + def test_with_statement(self): + with smtplib.SMTP(HOST, self.port) as smtp: + code, message = smtp.noop() + self.assertEqual(code, 250) + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo') + with smtplib.SMTP(HOST, self.port) as smtp: + smtp.close() + self.assertRaises(smtplib.SMTPServerDisconnected, smtp.send, b'foo') + + def test_with_statement_QUIT_failure(self): + self.serv.quit_response = '421 QUIT FAILED' + with self.assertRaises(smtplib.SMTPResponseException) as error: + with smtplib.SMTP(HOST, self.port) as smtp: + smtp.noop() + self.assertEqual(error.exception.smtp_code, 421) + self.assertEqual(error.exception.smtp_error, b'QUIT FAILED') + # We don't need to clean up self.serv.quit_response because a new + # server is always instantiated in the setUp(). + #TODO: add tests for correct AUTH method fallback now that the #test infrastructure can support it. diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index 0198ab6..86224ef 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -4,28 +4,60 @@ import unittest from test import support import smtplib +ssl = support.import_module("ssl") + support.requires("network") + +class SmtpTest(unittest.TestCase): + testServer = 'smtp.gmail.com' + remotePort = 25 + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) + + def test_connect_starttls(self): + support.get_attribute(smtplib, 'SMTP_SSL') + with support.transient_internet(self.testServer): + server = smtplib.SMTP(self.testServer, self.remotePort) + try: + server.starttls(context=self.context) + except smtplib.SMTPException as e: + if e.args[0] == 'STARTTLS extension not supported by server.': + unittest.skip(e.args[0]) + else: + raise + server.ehlo() + server.quit() + + class SmtpSSLTest(unittest.TestCase): testServer = 'smtp.gmail.com' remotePort = 465 + context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) def test_connect(self): support.get_attribute(smtplib, 'SMTP_SSL') with support.transient_internet(self.testServer): server = smtplib.SMTP_SSL(self.testServer, self.remotePort) - server.ehlo() - server.quit() + server.ehlo() + server.quit() def test_connect_default_port(self): support.get_attribute(smtplib, 'SMTP_SSL') with support.transient_internet(self.testServer): server = smtplib.SMTP_SSL(self.testServer) - server.ehlo() - server.quit() + server.ehlo() + server.quit() + + def test_connect_using_sslcontext(self): + support.get_attribute(smtplib, 'SMTP_SSL') + with support.transient_internet(self.testServer): + server = smtplib.SMTP_SSL(self.testServer, self.remotePort, context=self.context) + server.ehlo() + server.quit() + def test_main(): - support.run_unittest(SmtpSSLTest) + support.run_unittest(SmtpTest, SmtpSSLTest) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index f28a483..221a507 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -7,6 +7,8 @@ import errno import io import socket import select +import tempfile +import _testcapi import time import traceback import queue @@ -18,34 +20,15 @@ import contextlib from weakref import proxy import signal import math +import pickle +import struct try: import fcntl except ImportError: fcntl = False -def try_address(host, port=0, family=socket.AF_INET): - """Try to bind a socket on the given host:port and return True - if that has been possible.""" - try: - sock = socket.socket(family, socket.SOCK_STREAM) - sock.bind((host, port)) - except (socket.error, socket.gaierror): - return False - else: - sock.close() - return True - -def linux_version(): - try: - # platform.release() is something like '2.6.33.7-desktop-2mnb' - version_string = platform.release().split('-')[0] - return tuple(map(int, version_string.split('.'))) - except ValueError: - return 0, 0, 0 - HOST = support.HOST -MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf8') ## test unicode string and carriage return -SUPPORTS_IPV6 = socket.has_ipv6 and try_address('::1', family=socket.AF_INET6) +MSG = 'Michael Gilfix was here\u1234\r\n'.encode('utf-8') ## test unicode string and carriage return try: import _thread as thread @@ -54,6 +37,33 @@ except ImportError: thread = None threading = None +def _have_socket_can(): + """Check whether CAN sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + except (AttributeError, socket.error, OSError): + return False + else: + s.close() + return True + +def _have_socket_rds(): + """Check whether RDS sockets are supported on this host.""" + try: + s = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + except (AttributeError, OSError): + return False + else: + s.close() + return True + +HAVE_SOCKET_CAN = _have_socket_can() + +HAVE_SOCKET_RDS = _have_socket_rds() + +# Size in bytes of the int type +SIZEOF_INT = array.array("i").itemsize + class SocketTCPTest(unittest.TestCase): def setUp(self): @@ -75,6 +85,63 @@ class SocketUDPTest(unittest.TestCase): self.serv.close() self.serv = None +class ThreadSafeCleanupTestCase(unittest.TestCase): + """Subclass of unittest.TestCase with thread-safe cleanup methods. + + This subclass protects the addCleanup() and doCleanups() methods + with a recursive lock. + """ + + if threading: + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self._cleanup_lock = threading.RLock() + + def addCleanup(self, *args, **kwargs): + with self._cleanup_lock: + return super().addCleanup(*args, **kwargs) + + def doCleanups(self, *args, **kwargs): + with self._cleanup_lock: + return super().doCleanups(*args, **kwargs) + +class SocketCANTest(unittest.TestCase): + + """To be able to run this test, a `vcan0` CAN interface can be created with + the following commands: + # modprobe vcan + # ip link add dev vcan0 type vcan + # ifconfig vcan0 up + """ + interface = 'vcan0' + bufsize = 128 + + def setUp(self): + self.s = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + self.addCleanup(self.s.close) + try: + self.s.bind((self.interface,)) + except socket.error: + self.skipTest('network interface `%s` does not exist' % + self.interface) + + +class SocketRDSTest(unittest.TestCase): + + """To be able to run this test, the `rds` kernel module must be loaded: + # modprobe rds + """ + bufsize = 8192 + + def setUp(self): + self.serv = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + self.addCleanup(self.serv.close) + try: + self.port = support.bind_port(self.serv) + except OSError: + self.skipTest('unable to bind RDS socket') + + class ThreadableTest: """Threadable Test class @@ -132,6 +199,7 @@ class ThreadableTest: self.client_ready = threading.Event() self.done = threading.Event() self.queue = queue.Queue(1) + self.server_crashed = False # Do some munging to start the client test. methodname = self.id() @@ -141,8 +209,12 @@ class ThreadableTest: self.client_thread = thread.start_new_thread( self.clientRun, (test_method,)) - self.__setUp() - if not self.server_ready.is_set(): + try: + self.__setUp() + except: + self.server_crashed = True + raise + finally: self.server_ready.set() self.client_ready.wait() @@ -158,10 +230,16 @@ class ThreadableTest: self.server_ready.wait() self.clientSetUp() self.client_ready.set() + if self.server_crashed: + self.clientTearDown() + return if not hasattr(test_func, '__call__'): raise TypeError("test_func must be a callable function") try: test_func() + except unittest._ExpectedFailure: + # We deliberately ignore expected failures + pass except BaseException as e: self.queue.put(e) finally: @@ -202,6 +280,48 @@ class ThreadedUDPSocketTest(SocketUDPTest, ThreadableTest): self.cli = None ThreadableTest.clientTearDown(self) +class ThreadedCANSocketTest(SocketCANTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketCANTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) + try: + self.cli.bind((self.interface,)) + except socket.error: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + +class ThreadedRDSSocketTest(SocketRDSTest, ThreadableTest): + + def __init__(self, methodName='runTest'): + SocketRDSTest.__init__(self, methodName=methodName) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) + try: + # RDS sockets must be bound explicitly to send or receive data + self.cli.bind((HOST, 0)) + self.cli_addr = self.cli.getsockname() + except OSError: + # skipTest should not be called here, and will be called in the + # server instead + pass + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + class SocketConnectedTest(ThreadedTCPSocketTest): """Socket tests for client-server connection. @@ -257,6 +377,243 @@ class SocketPairTest(unittest.TestCase, ThreadableTest): ThreadableTest.clientTearDown(self) +# The following classes are used by the sendmsg()/recvmsg() tests. +# Combining, for instance, ConnectedStreamTestMixin and TCPTestBase +# gives a drop-in replacement for SocketConnectedTest, but different +# address families can be used, and the attributes serv_addr and +# cli_addr will be set to the addresses of the endpoints. + +class SocketTestBase(unittest.TestCase): + """A base class for socket tests. + + Subclasses must provide methods newSocket() to return a new socket + and bindSock(sock) to bind it to an unused address. + + Creates a socket self.serv and sets self.serv_addr to its address. + """ + + def setUp(self): + self.serv = self.newSocket() + self.bindServer() + + def bindServer(self): + """Bind server socket and set self.serv_addr to its address.""" + self.bindSock(self.serv) + self.serv_addr = self.serv.getsockname() + + def tearDown(self): + self.serv.close() + self.serv = None + + +class SocketListeningTestMixin(SocketTestBase): + """Mixin to listen on the server socket.""" + + def setUp(self): + super().setUp() + self.serv.listen(1) + + +class ThreadedSocketTestMixin(ThreadSafeCleanupTestCase, SocketTestBase, + ThreadableTest): + """Mixin to add client socket and allow client/server tests. + + Client socket is self.cli and its address is self.cli_addr. See + ThreadableTest for usage information. + """ + + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + ThreadableTest.__init__(self) + + def clientSetUp(self): + self.cli = self.newClientSocket() + self.bindClient() + + def newClientSocket(self): + """Return a new socket for use as client.""" + return self.newSocket() + + def bindClient(self): + """Bind client socket and set self.cli_addr to its address.""" + self.bindSock(self.cli) + self.cli_addr = self.cli.getsockname() + + def clientTearDown(self): + self.cli.close() + self.cli = None + ThreadableTest.clientTearDown(self) + + +class ConnectedStreamTestMixin(SocketListeningTestMixin, + ThreadedSocketTestMixin): + """Mixin to allow client/server stream tests with connected client. + + Server's socket representing connection to client is self.cli_conn + and client's connection to server is self.serv_conn. (Based on + SocketConnectedTest.) + """ + + def setUp(self): + super().setUp() + # Indicate explicitly we're ready for the client thread to + # proceed and then perform the blocking call to accept + self.serverExplicitReady() + conn, addr = self.serv.accept() + self.cli_conn = conn + + def tearDown(self): + self.cli_conn.close() + self.cli_conn = None + super().tearDown() + + def clientSetUp(self): + super().clientSetUp() + self.cli.connect(self.serv_addr) + self.serv_conn = self.cli + + def clientTearDown(self): + self.serv_conn.close() + self.serv_conn = None + super().clientTearDown() + + +class UnixSocketTestBase(SocketTestBase): + """Base class for Unix-domain socket tests.""" + + # This class is used for file descriptor passing tests, so we + # create the sockets in a private directory so that other users + # can't send anything that might be problematic for a privileged + # user running the tests. + + def setUp(self): + self.dir_path = tempfile.mkdtemp() + self.addCleanup(os.rmdir, self.dir_path) + super().setUp() + + def bindSock(self, sock): + path = tempfile.mktemp(dir=self.dir_path) + sock.bind(path) + self.addCleanup(support.unlink, path) + +class UnixStreamBase(UnixSocketTestBase): + """Base class for Unix-domain SOCK_STREAM tests.""" + + def newSocket(self): + return socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + + +class InetTestBase(SocketTestBase): + """Base class for IPv4 socket tests.""" + + host = HOST + + def setUp(self): + super().setUp() + self.port = self.serv_addr[1] + + def bindSock(self, sock): + support.bind_port(sock, host=self.host) + +class TCPTestBase(InetTestBase): + """Base class for TCP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM) + +class UDPTestBase(InetTestBase): + """Base class for UDP-over-IPv4 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + +class SCTPStreamBase(InetTestBase): + """Base class for SCTP tests in one-to-one (SOCK_STREAM) mode.""" + + def newSocket(self): + return socket.socket(socket.AF_INET, socket.SOCK_STREAM, + socket.IPPROTO_SCTP) + + +class Inet6TestBase(InetTestBase): + """Base class for IPv6 socket tests.""" + + # Don't use "localhost" here - it may not have an IPv6 address + # assigned to it by default (e.g. in /etc/hosts), and if someone + # has assigned it an IPv4-mapped address, then it's unlikely to + # work with the full IPv6 API. + host = "::1" + +class UDP6TestBase(Inet6TestBase): + """Base class for UDP-over-IPv6 tests.""" + + def newSocket(self): + return socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + + +# Test-skipping decorators for use with ThreadableTest. + +def skipWithClientIf(condition, reason): + """Skip decorated test if condition is true, add client_skip decorator. + + If the decorated object is not a class, sets its attribute + "client_skip" to a decorator which will return an empty function + if the test is to be skipped, or the original function if it is + not. This can be used to avoid running the client part of a + skipped test when using ThreadableTest. + """ + def client_pass(*args, **kwargs): + pass + def skipdec(obj): + retval = unittest.skip(reason)(obj) + if not isinstance(obj, type): + retval.client_skip = lambda f: client_pass + return retval + def noskipdec(obj): + if not (isinstance(obj, type) or hasattr(obj, "client_skip")): + obj.client_skip = lambda f: f + return obj + return skipdec if condition else noskipdec + + +def requireAttrs(obj, *attributes): + """Skip decorated test if obj is missing any of the given attributes. + + Sets client_skip attribute as skipWithClientIf() does. + """ + missing = [name for name in attributes if not hasattr(obj, name)] + return skipWithClientIf( + missing, "don't have " + ", ".join(name for name in missing)) + + +def requireSocket(*args): + """Skip decorated test if a socket cannot be created with given arguments. + + When an argument is given as a string, will use the value of that + attribute of the socket module, or skip the test if it doesn't + exist. Sets client_skip attribute as skipWithClientIf() does. + """ + err = None + missing = [obj for obj in args if + isinstance(obj, str) and not hasattr(socket, obj)] + if missing: + err = "don't have " + ", ".join(name for name in missing) + else: + callargs = [getattr(socket, obj) if isinstance(obj, str) else obj + for obj in args] + try: + s = socket.socket(*callargs) + except socket.error as e: + # XXX: check errno? + err = str(e) + else: + s.close() + return skipWithClientIf( + err is not None, + "can't create socket({0}): {1}".format( + ", ".join(str(o) for o in args), err)) + + ####################################################################### ## Begin Tests @@ -282,18 +639,13 @@ class GeneralModuleTests(unittest.TestCase): def testSocketError(self): # Testing socket module exceptions - def raise_error(*args, **kwargs): + msg = "Error raising socket exception (%s)." + with self.assertRaises(socket.error, msg=msg % 'socket.error'): raise socket.error - def raise_herror(*args, **kwargs): + with self.assertRaises(socket.error, msg=msg % 'socket.herror'): raise socket.herror - def raise_gaierror(*args, **kwargs): + with self.assertRaises(socket.error, msg=msg % 'socket.gaierror'): raise socket.gaierror - self.assertRaises(socket.error, raise_error, - "Error raising socket exception.") - self.assertRaises(socket.error, raise_herror, - "Error raising socket exception.") - self.assertRaises(socket.error, raise_gaierror, - "Error raising socket exception.") def testSendtoErrors(self): # Testing that sendto doens't masks failures. See #10169. @@ -369,6 +721,52 @@ class GeneralModuleTests(unittest.TestCase): if not fqhn in all_host_names: self.fail("Error testing host resolution mechanisms. (fqdn: %s, all: %s)" % (fqhn, repr(all_host_names))) + @unittest.skipUnless(hasattr(socket, 'sethostname'), "test needs socket.sethostname()") + @unittest.skipUnless(hasattr(socket, 'gethostname'), "test needs socket.gethostname()") + def test_sethostname(self): + oldhn = socket.gethostname() + try: + socket.sethostname('new') + except socket.error as e: + if e.errno == errno.EPERM: + self.skipTest("test should be run as root") + else: + raise + try: + # running test as root! + self.assertEqual(socket.gethostname(), 'new') + # Should work with bytes objects too + socket.sethostname(b'bar') + self.assertEqual(socket.gethostname(), 'bar') + finally: + socket.sethostname(oldhn) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInterfaceNameIndex(self): + interfaces = socket.if_nameindex() + for index, name in interfaces: + self.assertIsInstance(index, int) + self.assertIsInstance(name, str) + # interface indices are non-zero integers + self.assertGreater(index, 0) + _index = socket.if_nametoindex(name) + self.assertIsInstance(_index, int) + self.assertEqual(index, _index) + _name = socket.if_indextoname(index) + self.assertIsInstance(_name, str) + self.assertEqual(name, _name) + + @unittest.skipUnless(hasattr(socket, 'if_nameindex'), + 'socket.if_nameindex() not available.') + def testInvalidInterfaceNameIndex(self): + # test nonexistent interface index/name + self.assertRaises(socket.error, socket.if_indextoname, 0) + self.assertRaises(socket.error, socket.if_nametoindex, '_DEADBEEF') + # test with invalid values + self.assertRaises(TypeError, socket.if_nametoindex, 0) + self.assertRaises(TypeError, socket.if_indextoname, '_DEADBEEF') + def testRefCountGetNameInfo(self): # Testing reference count for getnameinfo if hasattr(sys, "getrefcount"): @@ -421,10 +819,8 @@ class GeneralModuleTests(unittest.TestCase): # Find one service that exists, then check all the related interfaces. # I've ordered this by protocols that have both a tcp and udp # protocol, at least for modern Linuxes. - if (sys.platform.startswith('linux') or - sys.platform.startswith('freebsd') or - sys.platform.startswith('netbsd') or - sys.platform == 'darwin'): + if (sys.platform.startswith(('freebsd', 'netbsd')) + or sys.platform in ('linux', 'darwin')): # avoid the 'echo' service on this platform, as there is an # assumption breaking non-standard port/protocol entry services = ('daytime', 'qotd', 'domain') @@ -654,7 +1050,7 @@ class GeneralModuleTests(unittest.TestCase): socket.getaddrinfo('localhost', 80) socket.getaddrinfo('127.0.0.1', 80) socket.getaddrinfo(None, 80) - if SUPPORTS_IPV6: + if support.IPV6_ENABLED: socket.getaddrinfo('::1', 80) # port can be a string service name such as "http", a numeric # port number or None @@ -774,7 +1170,13 @@ class GeneralModuleTests(unittest.TestCase): fp.close() self.assertEqual(repr(fp), "<_io.BufferedReader name=-1>") - def testListenBacklog0(self): + def test_pickle(self): + sock = socket.socket() + with sock: + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + self.assertRaises(TypeError, pickle.dumps, sock, protocol) + + def test_listen_backlog0(self): srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) srv.bind((HOST, 0)) # backlog = 0 @@ -782,6 +1184,222 @@ class GeneralModuleTests(unittest.TestCase): srv.close() +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +class BasicCANTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_CAN + socket.PF_CAN + socket.CAN_RAW + + def testCreateSocket(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + pass + + def testBindAny(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.bind(('', )) + + def testTooLongInterfaceName(self): + # most systems limit IFNAMSIZ to 16, take 1024 to be sure + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + self.assertRaisesRegex(socket.error, 'interface name too long', + s.bind, ('x' * 1024,)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_LOOPBACK"), + 'socket.CAN_RAW_LOOPBACK required for this test.') + def testLoopback(self): + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + for loopback in (0, 1): + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK, + loopback) + self.assertEqual(loopback, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_LOOPBACK)) + + @unittest.skipUnless(hasattr(socket, "CAN_RAW_FILTER"), + 'socket.CAN_RAW_FILTER required for this test.') + def testFilter(self): + can_id, can_mask = 0x200, 0x700 + can_filter = struct.pack("=II", can_id, can_mask) + with socket.socket(socket.PF_CAN, socket.SOCK_RAW, socket.CAN_RAW) as s: + s.setsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, can_filter) + self.assertEqual(can_filter, + s.getsockopt(socket.SOL_CAN_RAW, socket.CAN_RAW_FILTER, 8)) + + +@unittest.skipUnless(HAVE_SOCKET_CAN, 'SocketCan required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class CANTest(ThreadedCANSocketTest): + + """The CAN frame structure is defined in <linux/can.h>: + + struct can_frame { + canid_t can_id; /* 32 bit CAN_ID + EFF/RTR/ERR flags */ + __u8 can_dlc; /* data length code: 0 .. 8 */ + __u8 data[8] __attribute__((aligned(8))); + }; + """ + can_frame_fmt = "=IB3x8s" + + def __init__(self, methodName='runTest'): + ThreadedCANSocketTest.__init__(self, methodName=methodName) + + @classmethod + def build_can_frame(cls, can_id, data): + """Build a CAN frame.""" + can_dlc = len(data) + data = data.ljust(8, b'\x00') + return struct.pack(cls.can_frame_fmt, can_id, can_dlc, data) + + @classmethod + def dissect_can_frame(cls, frame): + """Dissect a CAN frame.""" + can_id, can_dlc, data = struct.unpack(cls.can_frame_fmt, frame) + return (can_id, can_dlc, data[:can_dlc]) + + def testSendFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + self.assertEqual(addr[0], self.interface) + self.assertEqual(addr[1], socket.AF_CAN) + + def _testSendFrame(self): + self.cf = self.build_can_frame(0x00, b'\x01\x02\x03\x04\x05') + self.cli.send(self.cf) + + def testSendMaxFrame(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf, cf) + + def _testSendMaxFrame(self): + self.cf = self.build_can_frame(0x00, b'\x07' * 8) + self.cli.send(self.cf) + + def testSendMultiFrames(self): + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf1, cf) + + cf, addr = self.s.recvfrom(self.bufsize) + self.assertEqual(self.cf2, cf) + + def _testSendMultiFrames(self): + self.cf1 = self.build_can_frame(0x07, b'\x44\x33\x22\x11') + self.cli.send(self.cf1) + + self.cf2 = self.build_can_frame(0x12, b'\x99\x22\x33') + self.cli.send(self.cf2) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +class BasicRDSTest(unittest.TestCase): + + def testCrucialConstants(self): + socket.AF_RDS + socket.PF_RDS + + def testCreateSocket(self): + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + pass + + def testSocketBufferSize(self): + bufsize = 16384 + with socket.socket(socket.PF_RDS, socket.SOCK_SEQPACKET, 0) as s: + s.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, bufsize) + s.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, bufsize) + + +@unittest.skipUnless(HAVE_SOCKET_RDS, 'RDS sockets required for this test.') +@unittest.skipUnless(thread, 'Threading required for this test.') +class RDSTest(ThreadedRDSSocketTest): + + def __init__(self, methodName='runTest'): + ThreadedRDSSocketTest.__init__(self, methodName=methodName) + + def setUp(self): + super().setUp() + self.evt = threading.Event() + + def testSendAndRecv(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + self.assertEqual(self.cli_addr, addr) + + def _testSendAndRecv(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testPeek(self): + data, addr = self.serv.recvfrom(self.bufsize, socket.MSG_PEEK) + self.assertEqual(self.data, data) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testPeek(self): + self.data = b'spam' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + @requireAttrs(socket.socket, 'recvmsg') + def testSendAndRecvMsg(self): + data, ancdata, msg_flags, addr = self.serv.recvmsg(self.bufsize) + self.assertEqual(self.data, data) + + @requireAttrs(socket.socket, 'sendmsg') + def _testSendAndRecvMsg(self): + self.data = b'hello ' * 10 + self.cli.sendmsg([self.data], (), 0, (HOST, self.port)) + + def testSendAndRecvMulti(self): + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data1, data) + + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data2, data) + + def _testSendAndRecvMulti(self): + self.data1 = b'bacon' + self.cli.sendto(self.data1, 0, (HOST, self.port)) + + self.data2 = b'egg' + self.cli.sendto(self.data2, 0, (HOST, self.port)) + + def testSelect(self): + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + data, addr = self.serv.recvfrom(self.bufsize) + self.assertEqual(self.data, data) + + def _testSelect(self): + self.data = b'select' + self.cli.sendto(self.data, 0, (HOST, self.port)) + + def testCongestion(self): + # wait until the sender is done + self.evt.wait() + + def _testCongestion(self): + # test the behavior in case of congestion + self.data = b'fill' + self.cli.setblocking(False) + try: + # try to lower the receiver's socket buffer size + self.cli.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 16384) + except OSError: + pass + with self.assertRaises(OSError) as cm: + try: + # fill the receiver's socket buffer + while True: + self.cli.sendto(self.data, 0, (HOST, self.port)) + finally: + # signal the receiver we're done + self.evt.set() + # sendto() should have failed with ENOBUFS + self.assertEqual(cm.exception.errno, errno.ENOBUFS) + # and we should have received a congestion notification through poll + r, w, x = select.select([self.serv], [], [], 3.0) + self.assertIn(self.serv, r) + + @unittest.skipUnless(thread, 'Threading required for this test.') class BasicTCPTest(SocketConnectedTest): @@ -920,6 +1538,1853 @@ class BasicUDPTest(ThreadedUDPSocketTest): def _testRecvFromNegative(self): self.cli.sendto(MSG, 0, (HOST, self.port)) + +# Tests for the sendmsg()/recvmsg() interface. Where possible, the +# same test code is used with different families and types of socket +# (e.g. stream, datagram), and tests using recvmsg() are repeated +# using recvmsg_into(). +# +# The generic test classes such as SendmsgTests and +# RecvmsgGenericTests inherit from SendrecvmsgBase and expect to be +# supplied with sockets cli_sock and serv_sock representing the +# client's and the server's end of the connection respectively, and +# attributes cli_addr and serv_addr holding their (numeric where +# appropriate) addresses. +# +# The final concrete test classes combine these with subclasses of +# SocketTestBase which set up client and server sockets of a specific +# type, and with subclasses of SendrecvmsgBase such as +# SendrecvmsgDgramBase and SendrecvmsgConnectedBase which map these +# sockets to cli_sock and serv_sock and override the methods and +# attributes of SendrecvmsgBase to fill in destination addresses if +# needed when sending, check for specific flags in msg_flags, etc. +# +# RecvmsgIntoMixin provides a version of doRecvmsg() implemented using +# recvmsg_into(). + +# XXX: like the other datagram (UDP) tests in this module, the code +# here assumes that datagram delivery on the local machine will be +# reliable. + +class SendrecvmsgBase(ThreadSafeCleanupTestCase): + # Base class for sendmsg()/recvmsg() tests. + + # Time in seconds to wait before considering a test failed, or + # None for no timeout. Not all tests actually set a timeout. + fail_timeout = 3.0 + + def setUp(self): + self.misc_event = threading.Event() + super().setUp() + + def sendToServer(self, msg): + # Send msg to the server. + return self.cli_sock.send(msg) + + # Tuple of alternative default arguments for sendmsg() when called + # via sendmsgToServer() (e.g. to include a destination address). + sendmsg_to_server_defaults = () + + def sendmsgToServer(self, *args): + # Call sendmsg() on self.cli_sock with the given arguments, + # filling in any arguments which are not supplied with the + # corresponding items of self.sendmsg_to_server_defaults, if + # any. + return self.cli_sock.sendmsg( + *(args + self.sendmsg_to_server_defaults[len(args):])) + + def doRecvmsg(self, sock, bufsize, *args): + # Call recvmsg() on sock with given arguments and return its + # result. Should be used for tests which can use either + # recvmsg() or recvmsg_into() - RecvmsgIntoMixin overrides + # this method with one which emulates it using recvmsg_into(), + # thus allowing the same test to be used for both methods. + result = sock.recvmsg(bufsize, *args) + self.registerRecvmsgResult(result) + return result + + def registerRecvmsgResult(self, result): + # Called by doRecvmsg() with the return value of recvmsg() or + # recvmsg_into(). Can be overridden to arrange cleanup based + # on the returned ancillary data, for instance. + pass + + def checkRecvmsgAddress(self, addr1, addr2): + # Called to compare the received address with the address of + # the peer. + self.assertEqual(addr1, addr2) + + # Flags that are normally unset in msg_flags + msg_flags_common_unset = 0 + for name in ("MSG_CTRUNC", "MSG_OOB"): + msg_flags_common_unset |= getattr(socket, name, 0) + + # Flags that are normally set + msg_flags_common_set = 0 + + # Flags set when a complete record has been received (e.g. MSG_EOR + # for SCTP) + msg_flags_eor_indicator = 0 + + # Flags set when a complete record has not been received + # (e.g. MSG_TRUNC for datagram sockets) + msg_flags_non_eor_indicator = 0 + + def checkFlags(self, flags, eor=None, checkset=0, checkunset=0, ignore=0): + # Method to check the value of msg_flags returned by recvmsg[_into](). + # + # Checks that all bits in msg_flags_common_set attribute are + # set in "flags" and all bits in msg_flags_common_unset are + # unset. + # + # The "eor" argument specifies whether the flags should + # indicate that a full record (or datagram) has been received. + # If "eor" is None, no checks are done; otherwise, checks + # that: + # + # * if "eor" is true, all bits in msg_flags_eor_indicator are + # set and all bits in msg_flags_non_eor_indicator are unset + # + # * if "eor" is false, all bits in msg_flags_non_eor_indicator + # are set and all bits in msg_flags_eor_indicator are unset + # + # If "checkset" and/or "checkunset" are supplied, they require + # the given bits to be set or unset respectively, overriding + # what the attributes require for those bits. + # + # If any bits are set in "ignore", they will not be checked, + # regardless of the other inputs. + # + # Will raise Exception if the inputs require a bit to be both + # set and unset, and it is not ignored. + + defaultset = self.msg_flags_common_set + defaultunset = self.msg_flags_common_unset + + if eor: + defaultset |= self.msg_flags_eor_indicator + defaultunset |= self.msg_flags_non_eor_indicator + elif eor is not None: + defaultset |= self.msg_flags_non_eor_indicator + defaultunset |= self.msg_flags_eor_indicator + + # Function arguments override defaults + defaultset &= ~checkunset + defaultunset &= ~checkset + + # Merge arguments with remaining defaults, and check for conflicts + checkset |= defaultset + checkunset |= defaultunset + inboth = checkset & checkunset & ~ignore + if inboth: + raise Exception("contradictory set, unset requirements for flags " + "{0:#x}".format(inboth)) + + # Compare with given msg_flags value + mask = (checkset | checkunset) & ~ignore + self.assertEqual(flags & mask, checkset & mask) + + +class RecvmsgIntoMixin(SendrecvmsgBase): + # Mixin to implement doRecvmsg() using recvmsg_into(). + + def doRecvmsg(self, sock, bufsize, *args): + buf = bytearray(bufsize) + result = sock.recvmsg_into([buf], *args) + self.registerRecvmsgResult(result) + self.assertGreaterEqual(result[0], 0) + self.assertLessEqual(result[0], bufsize) + return (bytes(buf[:result[0]]),) + result[1:] + + +class SendrecvmsgDgramFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for datagram sockets. + + @property + def msg_flags_non_eor_indicator(self): + return super().msg_flags_non_eor_indicator | socket.MSG_TRUNC + + +class SendrecvmsgSCTPFlagsBase(SendrecvmsgBase): + # Defines flags to be checked in msg_flags for SCTP sockets. + + @property + def msg_flags_eor_indicator(self): + return super().msg_flags_eor_indicator | socket.MSG_EOR + + +class SendrecvmsgConnectionlessBase(SendrecvmsgBase): + # Base class for tests on connectionless-mode sockets. Users must + # supply sockets on attributes cli and serv to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.serv + + @property + def cli_sock(self): + return self.cli + + @property + def sendmsg_to_server_defaults(self): + return ([], [], 0, self.serv_addr) + + def sendToServer(self, msg): + return self.cli_sock.sendto(msg, self.serv_addr) + + +class SendrecvmsgConnectedBase(SendrecvmsgBase): + # Base class for tests on connected sockets. Users must supply + # sockets on attributes serv_conn and cli_conn (representing the + # connections *to* the server and the client), to be mapped to + # cli_sock and serv_sock respectively. + + @property + def serv_sock(self): + return self.cli_conn + + @property + def cli_sock(self): + return self.serv_conn + + def checkRecvmsgAddress(self, addr1, addr2): + # Address is currently "unspecified" for a connected socket, + # so we don't examine it + pass + + +class SendrecvmsgServerTimeoutBase(SendrecvmsgBase): + # Base class to set a timeout on server's socket. + + def setUp(self): + super().setUp() + self.serv_sock.settimeout(self.fail_timeout) + + +class SendmsgTests(SendrecvmsgServerTimeoutBase): + # Tests for sendmsg() which can use any socket type and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsg(self): + # Send a simple message with sendmsg(). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG]), len(MSG)) + + def testSendmsgDataGenerator(self): + # Send from buffer obtained from a generator (not a sequence). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgDataGenerator(self): + self.assertEqual(self.sendmsgToServer((o for o in [MSG])), + len(MSG)) + + def testSendmsgAncillaryGenerator(self): + # Gather (empty) ancillary data from a generator. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgAncillaryGenerator(self): + self.assertEqual(self.sendmsgToServer([MSG], (o for o in [])), + len(MSG)) + + def testSendmsgArray(self): + # Send data from an array instead of the usual bytes object. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgArray(self): + self.assertEqual(self.sendmsgToServer([array.array("B", MSG)]), + len(MSG)) + + def testSendmsgGather(self): + # Send message data from more than one buffer (gather write). + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgGather(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + def testSendmsgBadArgs(self): + # Check that sendmsg() rejects invalid arguments. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadArgs(self): + self.assertRaises(TypeError, self.cli_sock.sendmsg) + self.assertRaises(TypeError, self.sendmsgToServer, + b"not in an iterable") + self.assertRaises(TypeError, self.sendmsgToServer, + object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG, object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], object()) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [], 0, object()) + self.sendToServer(b"done") + + def testSendmsgBadCmsg(self): + # Check that invalid ancillary data items are rejected. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgBadCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [object()]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(object(), 0, b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, object(), b"data")]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, object())]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0)]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b"data", 42)]) + self.sendToServer(b"done") + + @requireAttrs(socket, "CMSG_SPACE") + def testSendmsgBadMultiCmsg(self): + # Check that invalid ancillary data items are rejected when + # more than one item is present. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + @testSendmsgBadMultiCmsg.client_skip + def _testSendmsgBadMultiCmsg(self): + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [0, 0, b""]) + self.assertRaises(TypeError, self.sendmsgToServer, + [MSG], [(0, 0, b""), object()]) + self.sendToServer(b"done") + + def testSendmsgExcessCmsgReject(self): + # Check that sendmsg() rejects excess ancillary data items + # when the number that can be sent is limited. + self.assertEqual(self.serv_sock.recv(1000), b"done") + + def _testSendmsgExcessCmsgReject(self): + if not hasattr(socket, "CMSG_SPACE"): + # Can only send one item + with self.assertRaises(socket.error) as cm: + self.sendmsgToServer([MSG], [(0, 0, b""), (0, 0, b"")]) + self.assertIsNone(cm.exception.errno) + self.sendToServer(b"done") + + def testSendmsgAfterClose(self): + # Check that sendmsg() fails on a closed socket. + pass + + def _testSendmsgAfterClose(self): + self.cli_sock.close() + self.assertRaises(socket.error, self.sendmsgToServer, [MSG]) + + +class SendmsgStreamTests(SendmsgTests): + # Tests for sendmsg() which require a stream socket and do not + # involve recvmsg() or recvmsg_into(). + + def testSendmsgExplicitNoneAddr(self): + # Check that peer address can be specified as None. + self.assertEqual(self.serv_sock.recv(len(MSG)), MSG) + + def _testSendmsgExplicitNoneAddr(self): + self.assertEqual(self.sendmsgToServer([MSG], [], 0, None), len(MSG)) + + def testSendmsgTimeout(self): + # Check that timeout works with sendmsg(). + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + def _testSendmsgTimeout(self): + try: + self.cli_sock.settimeout(0.03) + with self.assertRaises(socket.timeout): + while True: + self.sendmsgToServer([b"a"*512]) + finally: + self.misc_event.set() + + # XXX: would be nice to have more tests for sendmsg flags argument. + + # Linux supports MSG_DONTWAIT when sending, but in general, it + # only works when receiving. Could add other platforms if they + # support it too. + @skipWithClientIf(sys.platform not in {"linux2"}, + "MSG_DONTWAIT not known to work on this platform when " + "sending") + def testSendmsgDontWait(self): + # Check that MSG_DONTWAIT in flags causes non-blocking behaviour. + self.assertEqual(self.serv_sock.recv(512), b"a"*512) + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @testSendmsgDontWait.client_skip + def _testSendmsgDontWait(self): + try: + with self.assertRaises(socket.error) as cm: + while True: + self.sendmsgToServer([b"a"*512], [], socket.MSG_DONTWAIT) + self.assertIn(cm.exception.errno, + (errno.EAGAIN, errno.EWOULDBLOCK)) + finally: + self.misc_event.set() + + +class SendmsgConnectionlessTests(SendmsgTests): + # Tests for sendmsg() which require a connectionless-mode + # (e.g. datagram) socket, and do not involve recvmsg() or + # recvmsg_into(). + + def testSendmsgNoDestAddr(self): + # Check that sendmsg() fails when no destination address is + # given for unconnected socket. + pass + + def _testSendmsgNoDestAddr(self): + self.assertRaises(socket.error, self.cli_sock.sendmsg, + [MSG]) + self.assertRaises(socket.error, self.cli_sock.sendmsg, + [MSG], [], 0, None) + + +class RecvmsgGenericTests(SendrecvmsgBase): + # Tests for recvmsg() which can also be emulated using + # recvmsg_into(), and can use any socket type. + + def testRecvmsg(self): + # Receive a simple message with recvmsg[_into](). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsg(self): + self.sendToServer(MSG) + + def testRecvmsgExplicitDefaults(self): + # Test recvmsg[_into]() with default arguments provided explicitly. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgExplicitDefaults(self): + self.sendToServer(MSG) + + def testRecvmsgShorter(self): + # Receive a message smaller than buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) + 42) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShorter(self): + self.sendToServer(MSG) + + # FreeBSD < 8 doesn't always set the MSG_TRUNC flag when a truncated + # datagram is received (issue #13001). + @support.requires_freebsd_version(8) + def testRecvmsgTrunc(self): + # Receive part of message, check for truncation indicators. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + @support.requires_freebsd_version(8) + def _testRecvmsgTrunc(self): + self.sendToServer(MSG) + + def testRecvmsgShortAncillaryBuf(self): + # Test ancillary data buffer too small to hold any ancillary data. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 1) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgShortAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgLongAncillaryBuf(self): + # Test large ancillary data buffer. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgLongAncillaryBuf(self): + self.sendToServer(MSG) + + def testRecvmsgAfterClose(self): + # Check that recvmsg[_into]() fails on a closed socket. + self.serv_sock.close() + self.assertRaises(socket.error, self.doRecvmsg, self.serv_sock, 1024) + + def _testRecvmsgAfterClose(self): + pass + + def testRecvmsgTimeout(self): + # Check that timeout works. + try: + self.serv_sock.settimeout(0.03) + self.assertRaises(socket.timeout, + self.doRecvmsg, self.serv_sock, len(MSG)) + finally: + self.misc_event.set() + + def _testRecvmsgTimeout(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + + @requireAttrs(socket, "MSG_PEEK") + def testRecvmsgPeek(self): + # Check that MSG_PEEK in flags enables examination of pending + # data without consuming it. + + # Receive part of data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3, 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG[:-3]) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + # Ignoring MSG_TRUNC here (so this test is the same for stream + # and datagram sockets). Some wording in POSIX seems to + # suggest that it needn't be set when peeking, but that may + # just be a slip. + self.checkFlags(flags, eor=False, + ignore=getattr(socket, "MSG_TRUNC", 0)) + + # Receive all data with MSG_PEEK. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 0, + socket.MSG_PEEK) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + # Check that the same data can still be received normally. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgPeek.client_skip + def _testRecvmsgPeek(self): + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + def testRecvmsgFromSendmsg(self): + # Test receiving with recvmsg[_into]() when message is sent + # using sendmsg(). + self.serv_sock.settimeout(self.fail_timeout) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, len(MSG)) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + @testRecvmsgFromSendmsg.client_skip + def _testRecvmsgFromSendmsg(self): + self.assertEqual(self.sendmsgToServer([MSG[:3], MSG[3:]]), len(MSG)) + + +class RecvmsgGenericStreamTests(RecvmsgGenericTests): + # Tests which require a stream socket and can use either recvmsg() + # or recvmsg_into(). + + def testRecvmsgEOF(self): + # Receive end-of-stream indicator (b"", peer socket closed). + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.assertEqual(msg, b"") + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=None) # Might not have end-of-record marker + + def _testRecvmsgEOF(self): + self.cli_sock.close() + + def testRecvmsgOverflow(self): + # Receive a message in more than one chunk. + seg1, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG) - 3) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=False) + + seg2, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, 1024) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + msg = seg1 + seg2 + self.assertEqual(msg, MSG) + + def _testRecvmsgOverflow(self): + self.sendToServer(MSG) + + +class RecvmsgTests(RecvmsgGenericTests): + # Tests for recvmsg() which can use any socket type. + + def testRecvmsgBadArgs(self): + # Check that recvmsg() rejects invalid arguments. + self.assertRaises(TypeError, self.serv_sock.recvmsg) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + -1, 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg, + len(MSG), -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + [bytearray(10)], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + object(), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg, + len(MSG), 0, object()) + + msg, ancdata, flags, addr = self.serv_sock.recvmsg(len(MSG), 0, 0) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgBadArgs(self): + self.sendToServer(MSG) + + +class RecvmsgIntoTests(RecvmsgIntoMixin, RecvmsgGenericTests): + # Tests for recvmsg_into() which can use any socket type. + + def testRecvmsgIntoBadArgs(self): + # Check that recvmsg_into() rejects invalid arguments. + buf = bytearray(len(MSG)) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + len(MSG), 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + buf, 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [object()], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [b"I'm not writable"], 0, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf, object()], 0, 0) + self.assertRaises(ValueError, self.serv_sock.recvmsg_into, + [buf], -1, 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], object(), 0) + self.assertRaises(TypeError, self.serv_sock.recvmsg_into, + [buf], 0, object()) + + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf], 0, 0) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoBadArgs(self): + self.sendToServer(MSG) + + def testRecvmsgIntoGenerator(self): + # Receive into buffer obtained from a generator (not a sequence). + buf = bytearray(len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + (o for o in [buf])) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf, bytearray(MSG)) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoGenerator(self): + self.sendToServer(MSG) + + def testRecvmsgIntoArray(self): + # Receive into an array rather than the usual bytearray. + buf = array.array("B", [0] * len(MSG)) + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into([buf]) + self.assertEqual(nbytes, len(MSG)) + self.assertEqual(buf.tobytes(), MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoArray(self): + self.sendToServer(MSG) + + def testRecvmsgIntoScatter(self): + # Receive into multiple buffers (scatter write). + b1 = bytearray(b"----") + b2 = bytearray(b"0123456789") + b3 = bytearray(b"--------------") + nbytes, ancdata, flags, addr = self.serv_sock.recvmsg_into( + [b1, memoryview(b2)[2:9], b3]) + self.assertEqual(nbytes, len(b"Mary had a little lamb")) + self.assertEqual(b1, bytearray(b"Mary")) + self.assertEqual(b2, bytearray(b"01 had a 9")) + self.assertEqual(b3, bytearray(b"little lamb---")) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True) + + def _testRecvmsgIntoScatter(self): + self.sendToServer(b"Mary had a little lamb") + + +class CmsgMacroTests(unittest.TestCase): + # Test the functions CMSG_LEN() and CMSG_SPACE(). Tests + # assumptions used by sendmsg() and recvmsg[_into](), which share + # code with these functions. + + # Match the definition in socketmodule.c + socklen_t_limit = min(0x7fffffff, _testcapi.INT_MAX) + + @requireAttrs(socket, "CMSG_LEN") + def testCMSG_LEN(self): + # Test CMSG_LEN() with various valid and invalid values, + # checking the assumptions used by recvmsg() and sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_LEN(0) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(socket.CMSG_LEN(0), array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_LEN(n) + # This is how recvmsg() calculates the data size + self.assertEqual(ret - socket.CMSG_LEN(0), n) + self.assertLessEqual(ret, self.socklen_t_limit) + + self.assertRaises(OverflowError, socket.CMSG_LEN, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_LEN, toobig) + self.assertRaises(OverflowError, socket.CMSG_LEN, sys.maxsize) + + @requireAttrs(socket, "CMSG_SPACE") + def testCMSG_SPACE(self): + # Test CMSG_SPACE() with various valid and invalid values, + # checking the assumptions used by sendmsg(). + toobig = self.socklen_t_limit - socket.CMSG_SPACE(1) + 1 + values = list(range(257)) + list(range(toobig - 257, toobig)) + + last = socket.CMSG_SPACE(0) + # struct cmsghdr has at least three members, two of which are ints + self.assertGreater(last, array.array("i").itemsize * 2) + for n in values: + ret = socket.CMSG_SPACE(n) + self.assertGreaterEqual(ret, last) + self.assertGreaterEqual(ret, socket.CMSG_LEN(n)) + self.assertGreaterEqual(ret, n + socket.CMSG_LEN(0)) + self.assertLessEqual(ret, self.socklen_t_limit) + last = ret + + self.assertRaises(OverflowError, socket.CMSG_SPACE, -1) + # sendmsg() shares code with these functions, and requires + # that it reject values over the limit. + self.assertRaises(OverflowError, socket.CMSG_SPACE, toobig) + self.assertRaises(OverflowError, socket.CMSG_SPACE, sys.maxsize) + + +class SCMRightsTest(SendrecvmsgServerTimeoutBase): + # Tests for file descriptor passing on Unix-domain sockets. + + # Invalid file descriptor value that's unlikely to evaluate to a + # real FD even if one of its bytes is replaced with a different + # value (which shouldn't actually happen). + badfd = -0x5555 + + def newFDs(self, n): + # Return a list of n file descriptors for newly-created files + # containing their list indices as ASCII numbers. + fds = [] + for i in range(n): + fd, path = tempfile.mkstemp() + self.addCleanup(os.unlink, path) + self.addCleanup(os.close, fd) + os.write(fd, str(i).encode()) + fds.append(fd) + return fds + + def checkFDs(self, fds): + # Check that the file descriptors in the given list contain + # their correct list indices as ASCII numbers. + for n, fd in enumerate(fds): + os.lseek(fd, 0, os.SEEK_SET) + self.assertEqual(os.read(fd, 1024), str(n).encode()) + + def registerRecvmsgResult(self, result): + self.addCleanup(self.closeRecvmsgFDs, result) + + def closeRecvmsgFDs(self, recvmsg_result): + # Close all file descriptors specified in the ancillary data + # of the given return value from recvmsg() or recvmsg_into(). + for cmsg_level, cmsg_type, cmsg_data in recvmsg_result[1]: + if (cmsg_level == socket.SOL_SOCKET and + cmsg_type == socket.SCM_RIGHTS): + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + for fd in fds: + os.close(fd) + + def createAndSendFDs(self, n): + # Send n new file descriptors created by newFDs() to the + # server, with the constant MSG as the non-ancillary data. + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(n)))]), + len(MSG)) + + def checkRecvmsgFDs(self, numfds, result, maxcmsgs=1, ignoreflags=0): + # Check that constant MSG was received with numfds file + # descriptors in a maximum of maxcmsgs control messages (which + # must contain only complete integers). By default, check + # that MSG_CTRUNC is unset, but ignore any flags in + # ignoreflags. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertIsInstance(ancdata, list) + self.assertLessEqual(len(ancdata), maxcmsgs) + fds = array.array("i") + for item in ancdata: + self.assertIsInstance(item, tuple) + cmsg_level, cmsg_type, cmsg_data = item + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data) % SIZEOF_INT, 0) + fds.frombytes(cmsg_data) + + self.assertEqual(len(fds), numfds) + self.checkFDs(fds) + + def testFDPassSimple(self): + # Pass a single FD (array read from bytes object). + self.checkRecvmsgFDs(1, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testFDPassSimple(self): + self.assertEqual( + self.sendmsgToServer( + [MSG], + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", self.newFDs(1)).tobytes())]), + len(MSG)) + + def testMultipleFDPass(self): + # Pass multiple FDs in a single array. + self.checkRecvmsgFDs(4, self.doRecvmsg(self.serv_sock, + len(MSG), 10240)) + + def _testMultipleFDPass(self): + self.createAndSendFDs(4) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassCMSG_SPACE(self): + # Test using CMSG_SPACE() to calculate ancillary buffer size. + self.checkRecvmsgFDs( + 4, self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(4 * SIZEOF_INT))) + + @testFDPassCMSG_SPACE.client_skip + def _testFDPassCMSG_SPACE(self): + self.createAndSendFDs(4) + + def testFDPassCMSG_LEN(self): + # Test using CMSG_LEN() to calculate ancillary buffer size. + self.checkRecvmsgFDs(1, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(4 * SIZEOF_INT)), + # RFC 3542 says implementations may set + # MSG_CTRUNC if there isn't enough space + # for trailing padding. + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassCMSG_LEN(self): + self.createAndSendFDs(1) + + # Issue #12958: The following test has problems on Mac OS X + @support.anticipate_failure(sys.platform == "darwin") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparate(self): + # Pass two FDs in two separate arrays. Arrays may be combined + # into a single control message by the OS. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), 10240), + maxcmsgs=2) + + @testFDPassSeparate.client_skip + @support.anticipate_failure(sys.platform == "darwin") + def _testFDPassSeparate(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + # Issue #12958: The following test has problems on Mac OS X + @support.anticipate_failure(sys.platform == "darwin") + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassSeparateMinSpace(self): + # Pass two FDs in two separate arrays, receiving them into the + # minimum space for two arrays. + self.checkRecvmsgFDs(2, + self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(SIZEOF_INT)), + maxcmsgs=2, ignoreflags=socket.MSG_CTRUNC) + + @testFDPassSeparateMinSpace.client_skip + @support.anticipate_failure(sys.platform == "darwin") + def _testFDPassSeparateMinSpace(self): + fd0, fd1 = self.newFDs(2) + self.assertEqual( + self.sendmsgToServer([MSG], [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0])), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]), + len(MSG)) + + def sendAncillaryIfPossible(self, msg, ancdata): + # Try to send msg and ancdata to server, but if the system + # call fails, just send msg with no ancillary data. + try: + nbytes = self.sendmsgToServer([msg], ancdata) + except socket.error as e: + # Check that it was the system call that failed + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer([msg]) + self.assertEqual(nbytes, len(msg)) + + def testFDPassEmpty(self): + # Try to pass an empty FD array. Can receive either no array + # or an empty array. + self.checkRecvmsgFDs(0, self.doRecvmsg(self.serv_sock, + len(MSG), 10240), + ignoreflags=socket.MSG_CTRUNC) + + def _testFDPassEmpty(self): + self.sendAncillaryIfPossible(MSG, [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + b"")]) + + def testFDPassPartialInt(self): + # Try to pass a truncated FD array. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 1) + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + def _testFDPassPartialInt(self): + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [self.badfd]).tobytes()[:-1])]) + + @requireAttrs(socket, "CMSG_SPACE") + def testFDPassPartialIntInMiddle(self): + # Try to pass two FD arrays, the first of which is truncated. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), 10240) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, ignore=socket.MSG_CTRUNC) + self.assertLessEqual(len(ancdata), 2) + fds = array.array("i") + # Arrays may have been combined in a single control message + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.assertLessEqual(len(fds), 2) + self.checkFDs(fds) + + @testFDPassPartialIntInMiddle.client_skip + def _testFDPassPartialIntInMiddle(self): + fd0, fd1 = self.newFDs(2) + self.sendAncillaryIfPossible( + MSG, + [(socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd0, self.badfd]).tobytes()[:-1]), + (socket.SOL_SOCKET, + socket.SCM_RIGHTS, + array.array("i", [fd1]))]) + + def checkTruncatedHeader(self, result, ignoreflags=0): + # Check that no ancillary data items are returned when data is + # truncated inside the cmsghdr structure. + msg, ancdata, flags, addr = result + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no buffer size + # is specified. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG)), + # BSD seems to set MSG_CTRUNC only + # if an item has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTruncNoBufSize(self): + self.createAndSendFDs(1) + + def testCmsgTrunc0(self): + # Check that no ancillary data is received when buffer size is 0. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 0), + ignoreflags=socket.MSG_CTRUNC) + + def _testCmsgTrunc0(self): + self.createAndSendFDs(1) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + def testCmsgTrunc1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), 1)) + + def _testCmsgTrunc1(self): + self.createAndSendFDs(1) + + def testCmsgTrunc2Int(self): + # The cmsghdr structure has at least three members, two of + # which are ints, so we still shouldn't see any ancillary + # data. + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + SIZEOF_INT * 2)) + + def _testCmsgTrunc2Int(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Minus1(self): + self.checkTruncatedHeader(self.doRecvmsg(self.serv_sock, len(MSG), + socket.CMSG_LEN(0) - 1)) + + def _testCmsgTruncLen0Minus1(self): + self.createAndSendFDs(1) + + # The following tests try to truncate the control message in the + # middle of the FD array. + + def checkTruncatedArray(self, ancbuf, maxdata, mindata=0): + # Check that file descriptor data is truncated to between + # mindata and maxdata bytes when received with buffer size + # ancbuf, and that any complete file descriptor numbers are + # valid. + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbuf) + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + if mindata == 0 and ancdata == []: + return + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.SOL_SOCKET) + self.assertEqual(cmsg_type, socket.SCM_RIGHTS) + self.assertGreaterEqual(len(cmsg_data), mindata) + self.assertLessEqual(len(cmsg_data), maxdata) + fds = array.array("i") + fds.frombytes(cmsg_data[: + len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) + self.checkFDs(fds) + + def testCmsgTruncLen0(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0), maxdata=0) + + def _testCmsgTruncLen0(self): + self.createAndSendFDs(1) + + def testCmsgTruncLen0Plus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(0) + 1, maxdata=1) + + def _testCmsgTruncLen0Plus1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(SIZEOF_INT), + maxdata=SIZEOF_INT) + + def _testCmsgTruncLen1(self): + self.createAndSendFDs(2) + + def testCmsgTruncLen2Minus1(self): + self.checkTruncatedArray(ancbuf=socket.CMSG_LEN(2 * SIZEOF_INT) - 1, + maxdata=(2 * SIZEOF_INT) - 1) + + def _testCmsgTruncLen2Minus1(self): + self.createAndSendFDs(2) + + +class RFC3542AncillaryTest(SendrecvmsgServerTimeoutBase): + # Test sendmsg() and recvmsg[_into]() using the ancillary data + # features of the RFC 3542 Advanced Sockets API for IPv6. + # Currently we can only handle certain data items (e.g. traffic + # class, hop limit, MTU discovery and fragmentation settings) + # without resorting to unportable means such as the struct module, + # but the tests here are aimed at testing the ancillary data + # handling in sendmsg() and recvmsg() rather than the IPv6 API + # itself. + + # Test value to use when setting hop limit of packet + hop_limit = 2 + + # Test value to use when setting traffic class of packet. + # -1 means "use kernel default". + traffic_class = -1 + + def ancillaryMapping(self, ancdata): + # Given ancillary data list ancdata, return a mapping from + # pairs (cmsg_level, cmsg_type) to corresponding cmsg_data. + # Check that no (level, type) pair appears more than once. + d = {} + for cmsg_level, cmsg_type, cmsg_data in ancdata: + self.assertNotIn((cmsg_level, cmsg_type), d) + d[(cmsg_level, cmsg_type)] = cmsg_data + return d + + def checkHopLimit(self, ancbufsize, maxhop=255, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space. Check that data is MSG, ancillary data is not + # truncated (but ignore any flags in ignoreflags), and hop + # limit is between 0 and maxhop inclusive. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + self.assertIsInstance(ancdata[0], tuple) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertIsInstance(cmsg_data, bytes) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimit(self): + # Test receiving the packet hop limit as ancillary data. + self.checkHopLimit(ancbufsize=10240) + + @testRecvHopLimit.client_skip + def _testRecvHopLimit(self): + # Need to wait until server has asked to receive ancillary + # data, as implementations are not required to buffer it + # otherwise. + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testRecvHopLimitCMSG_SPACE(self): + # Test receiving hop limit, using CMSG_SPACE to calculate buffer size. + self.checkHopLimit(ancbufsize=socket.CMSG_SPACE(SIZEOF_INT)) + + @testRecvHopLimitCMSG_SPACE.client_skip + def _testRecvHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Could test receiving into buffer sized using CMSG_LEN, but RFC + # 3542 says portable applications must provide space for trailing + # padding. Implementations may set MSG_CTRUNC if there isn't + # enough space for the padding. + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSetHopLimit(self): + # Test setting hop limit on outgoing packet and receiving it + # at the other end. + self.checkHopLimit(ancbufsize=10240, maxhop=self.hop_limit) + + @testSetHopLimit.client_skip + def _testSetHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + def checkTrafficClassAndHopLimit(self, ancbufsize, maxhop=255, + ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space. Check that data is MSG, ancillary + # data is not truncated (but ignore any flags in ignoreflags), + # and traffic class and hop limit are in range (hop limit no + # more than maxhop). + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkunset=socket.MSG_CTRUNC, + ignore=ignoreflags) + self.assertEqual(len(ancdata), 2) + ancmap = self.ancillaryMapping(ancdata) + + tcdata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_TCLASS)] + self.assertEqual(len(tcdata), SIZEOF_INT) + a = array.array("i") + a.frombytes(tcdata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + hldata = ancmap[(socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT)] + self.assertEqual(len(hldata), SIZEOF_INT) + a = array.array("i") + a.frombytes(hldata) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], maxhop) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimit(self): + # Test receiving traffic class and hop limit as ancillary data. + self.checkTrafficClassAndHopLimit(ancbufsize=10240) + + @testRecvTrafficClassAndHopLimit.client_skip + def _testRecvTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + # Test receiving traffic class and hop limit, using + # CMSG_SPACE() to calculate buffer size. + self.checkTrafficClassAndHopLimit( + ancbufsize=socket.CMSG_SPACE(SIZEOF_INT) * 2) + + @testRecvTrafficClassAndHopLimitCMSG_SPACE.client_skip + def _testRecvTrafficClassAndHopLimitCMSG_SPACE(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSetTrafficClassAndHopLimit(self): + # Test setting traffic class and hop limit on outgoing packet, + # and receiving them at the other end. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testSetTrafficClassAndHopLimit.client_skip + def _testSetTrafficClassAndHopLimit(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.assertEqual( + self.sendmsgToServer([MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]), + len(MSG)) + + @requireAttrs(socket.socket, "sendmsg") + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testOddCmsgSize(self): + # Try to send ancillary data with first item one byte too + # long. Fall back to sending with correct size if this fails, + # and check that second item was handled correctly. + self.checkTrafficClassAndHopLimit(ancbufsize=10240, + maxhop=self.hop_limit) + + @testOddCmsgSize.client_skip + def _testOddCmsgSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + try: + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class]).tobytes() + b"\x00"), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + except socket.error as e: + self.assertIsInstance(e.errno, int) + nbytes = self.sendmsgToServer( + [MSG], + [(socket.IPPROTO_IPV6, socket.IPV6_TCLASS, + array.array("i", [self.traffic_class])), + (socket.IPPROTO_IPV6, socket.IPV6_HOPLIMIT, + array.array("i", [self.hop_limit]))]) + self.assertEqual(nbytes, len(MSG)) + + # Tests for proper handling of truncated ancillary data + + def checkHopLimitTruncatedHeader(self, ancbufsize, ignoreflags=0): + # Receive hop limit into ancbufsize bytes of ancillary data + # space, which should be too small to contain the ancillary + # data header (if ancbufsize is None, pass no second argument + # to recvmsg()). Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and no ancillary data is + # returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + args = () if ancbufsize is None else (ancbufsize,) + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), *args) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.assertEqual(ancdata, []) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testCmsgTruncNoBufSize(self): + # Check that no ancillary data is received when no ancillary + # buffer size is provided. + self.checkHopLimitTruncatedHeader(ancbufsize=None, + # BSD seems to set + # MSG_CTRUNC only if an item + # has been partially + # received. + ignoreflags=socket.MSG_CTRUNC) + + @testCmsgTruncNoBufSize.client_skip + def _testCmsgTruncNoBufSize(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc0(self): + # Check that no ancillary data is received when ancillary + # buffer size is zero. + self.checkHopLimitTruncatedHeader(ancbufsize=0, + ignoreflags=socket.MSG_CTRUNC) + + @testSingleCmsgTrunc0.client_skip + def _testSingleCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + # Check that no ancillary data is returned for various non-zero + # (but still too small) buffer sizes. + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=1) + + @testSingleCmsgTrunc1.client_skip + def _testSingleCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTrunc2Int(self): + self.checkHopLimitTruncatedHeader(ancbufsize=2 * SIZEOF_INT) + + @testSingleCmsgTrunc2Int.client_skip + def _testSingleCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncLen0Minus1(self): + self.checkHopLimitTruncatedHeader(ancbufsize=socket.CMSG_LEN(0) - 1) + + @testSingleCmsgTruncLen0Minus1.client_skip + def _testSingleCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT") + def testSingleCmsgTruncInData(self): + # Test truncation of a control message inside its associated + # data. The message may be returned with its data truncated, + # or not returned at all. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + self.assertLessEqual(len(ancdata), 1) + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertEqual(cmsg_type, socket.IPV6_HOPLIMIT) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + @testSingleCmsgTruncInData.client_skip + def _testSingleCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + def checkTruncatedSecondHeader(self, ancbufsize, ignoreflags=0): + # Receive traffic class and hop limit into ancbufsize bytes of + # ancillary data space, which should be large enough to + # contain the first item, but too small to contain the header + # of the second. Check that data is MSG, MSG_CTRUNC is set + # (unless included in ignoreflags), and only one ancillary + # data item is returned. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg(self.serv_sock, + len(MSG), ancbufsize) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC, + ignore=ignoreflags) + + self.assertEqual(len(ancdata), 1) + cmsg_level, cmsg_type, cmsg_data = ancdata[0] + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + self.assertIn(cmsg_type, {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT}) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + # Try the above test with various buffer sizes. + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc0(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT), + ignoreflags=socket.MSG_CTRUNC) + + @testSecondCmsgTrunc0.client_skip + def _testSecondCmsgTrunc0(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + 1) + + @testSecondCmsgTrunc1.client_skip + def _testSecondCmsgTrunc1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTrunc2Int(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + 2 * SIZEOF_INT) + + @testSecondCmsgTrunc2Int.client_skip + def _testSecondCmsgTrunc2Int(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecondCmsgTruncLen0Minus1(self): + self.checkTruncatedSecondHeader(socket.CMSG_SPACE(SIZEOF_INT) + + socket.CMSG_LEN(0) - 1) + + @testSecondCmsgTruncLen0Minus1.client_skip + def _testSecondCmsgTruncLen0Minus1(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + @requireAttrs(socket, "CMSG_SPACE", "IPV6_RECVHOPLIMIT", "IPV6_HOPLIMIT", + "IPV6_RECVTCLASS", "IPV6_TCLASS") + def testSecomdCmsgTruncInData(self): + # Test truncation of the second of two control messages inside + # its associated data. + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVHOPLIMIT, 1) + self.serv_sock.setsockopt(socket.IPPROTO_IPV6, + socket.IPV6_RECVTCLASS, 1) + self.misc_event.set() + msg, ancdata, flags, addr = self.doRecvmsg( + self.serv_sock, len(MSG), + socket.CMSG_SPACE(SIZEOF_INT) + socket.CMSG_LEN(SIZEOF_INT) - 1) + + self.assertEqual(msg, MSG) + self.checkRecvmsgAddress(addr, self.cli_addr) + self.checkFlags(flags, eor=True, checkset=socket.MSG_CTRUNC) + + cmsg_types = {socket.IPV6_TCLASS, socket.IPV6_HOPLIMIT} + + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertEqual(len(cmsg_data), SIZEOF_INT) + a = array.array("i") + a.frombytes(cmsg_data) + self.assertGreaterEqual(a[0], 0) + self.assertLessEqual(a[0], 255) + + if ancdata: + cmsg_level, cmsg_type, cmsg_data = ancdata.pop(0) + self.assertEqual(cmsg_level, socket.IPPROTO_IPV6) + cmsg_types.remove(cmsg_type) + self.assertLess(len(cmsg_data), SIZEOF_INT) + + self.assertEqual(ancdata, []) + + @testSecomdCmsgTruncInData.client_skip + def _testSecomdCmsgTruncInData(self): + self.assertTrue(self.misc_event.wait(timeout=self.fail_timeout)) + self.sendToServer(MSG) + + +# Derive concrete test classes for different socket types. + +class SendrecvmsgUDPTestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDPTest(SendmsgConnectionlessTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDPTest(RecvmsgTests, SendrecvmsgUDPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDPTest(RecvmsgIntoTests, SendrecvmsgUDPTestBase): + pass + + +class SendrecvmsgUDP6TestBase(SendrecvmsgDgramFlagsBase, + SendrecvmsgConnectionlessBase, + ThreadedSocketTestMixin, UDP6TestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUDP6Test(SendmsgConnectionlessTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUDP6Test(RecvmsgTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUDP6Test(RecvmsgIntoTests, SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgRFC3542AncillaryUDP6Test(RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(socket.has_ipv6, "Python not built with IPv6 support") +@requireAttrs(socket, "IPPROTO_IPV6") +@requireSocket("AF_INET6", "SOCK_DGRAM") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoRFC3542AncillaryUDP6Test(RecvmsgIntoMixin, + RFC3542AncillaryTest, + SendrecvmsgUDP6TestBase): + pass + + +class SendrecvmsgTCPTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, TCPTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgTCPTest(SendmsgStreamTests, SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgTCPTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoTCPTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgTCPTestBase): + pass + + +class SendrecvmsgSCTPStreamTestBase(SendrecvmsgSCTPFlagsBase, + SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, SCTPStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgSCTPStreamTest(SendmsgStreamTests, SendrecvmsgSCTPStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCTPStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@requireSocket("AF_INET", "SOCK_STREAM", "IPPROTO_SCTP") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCTPStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgSCTPStreamTestBase): + pass + + +class SendrecvmsgUnixStreamTestBase(SendrecvmsgConnectedBase, + ConnectedStreamTestMixin, UnixStreamBase): + pass + +@requireAttrs(socket.socket, "sendmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class SendmsgUnixStreamTest(SendmsgStreamTests, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgUnixStreamTest(RecvmsgTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "recvmsg_into") +@requireAttrs(socket, "AF_UNIX") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoUnixStreamTest(RecvmsgIntoTests, RecvmsgGenericStreamTests, + SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgSCMRightsStreamTest(SCMRightsTest, SendrecvmsgUnixStreamTestBase): + pass + +@requireAttrs(socket.socket, "sendmsg", "recvmsg_into") +@requireAttrs(socket, "AF_UNIX", "SOL_SOCKET", "SCM_RIGHTS") +@unittest.skipUnless(thread, 'Threading required for this test.') +class RecvmsgIntoSCMRightsStreamTest(RecvmsgIntoMixin, SCMRightsTest, + SendrecvmsgUnixStreamTestBase): + pass + + +# Test interrupting the interruptible send/receive methods with a +# signal when a timeout is set. These tests avoid having multiple +# threads alive during the test so that the OS cannot deliver the +# signal to the wrong one. + +class InterruptedTimeoutBase(unittest.TestCase): + # Base class for interrupted send/receive tests. Installs an + # empty handler for SIGALRM and removes it on teardown, along with + # any scheduled alarms. + + def setUp(self): + super().setUp() + orig_alrm_handler = signal.signal(signal.SIGALRM, + lambda signum, frame: None) + self.addCleanup(signal.signal, signal.SIGALRM, orig_alrm_handler) + self.addCleanup(self.setAlarm, 0) + + # Timeout for socket operations + timeout = 4.0 + + # Provide setAlarm() method to schedule delivery of SIGALRM after + # given number of seconds, or cancel it if zero, and an + # appropriate time value to use. Use setitimer() if available. + if hasattr(signal, "setitimer"): + alarm_time = 0.05 + + def setAlarm(self, seconds): + signal.setitimer(signal.ITIMER_REAL, seconds) + else: + # Old systems may deliver the alarm up to one second early + alarm_time = 2 + + def setAlarm(self, seconds): + signal.alarm(seconds) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +class InterruptedRecvTimeoutTest(InterruptedTimeoutBase, UDPTestBase): + # Test interrupting the recv*() methods with signals when a + # timeout is set. + + def setUp(self): + super().setUp() + self.serv.settimeout(self.timeout) + + def checkInterruptedRecv(self, func, *args, **kwargs): + # Check that func(*args, **kwargs) raises socket.error with an + # errno of EINTR when interrupted by a signal. + self.setAlarm(self.alarm_time) + with self.assertRaises(socket.error) as cm: + func(*args, **kwargs) + self.assertNotIsInstance(cm.exception, socket.timeout) + self.assertEqual(cm.exception.errno, errno.EINTR) + + def testInterruptedRecvTimeout(self): + self.checkInterruptedRecv(self.serv.recv, 1024) + + def testInterruptedRecvIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recv_into, bytearray(1024)) + + def testInterruptedRecvfromTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom, 1024) + + def testInterruptedRecvfromIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvfrom_into, bytearray(1024)) + + @requireAttrs(socket.socket, "recvmsg") + def testInterruptedRecvmsgTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg, 1024) + + @requireAttrs(socket.socket, "recvmsg_into") + def testInterruptedRecvmsgIntoTimeout(self): + self.checkInterruptedRecv(self.serv.recvmsg_into, [bytearray(1024)]) + + +# Require siginterrupt() in order to ensure that system calls are +# interrupted by default. +@requireAttrs(signal, "siginterrupt") +@unittest.skipUnless(hasattr(signal, "alarm") or hasattr(signal, "setitimer"), + "Don't have signal.alarm or signal.setitimer") +@unittest.skipUnless(thread, 'Threading required for this test.') +class InterruptedSendTimeoutTest(InterruptedTimeoutBase, + ThreadSafeCleanupTestCase, + SocketListeningTestMixin, TCPTestBase): + # Test interrupting the interruptible send*() methods with signals + # when a timeout is set. + + def setUp(self): + super().setUp() + self.serv_conn = self.newSocket() + self.addCleanup(self.serv_conn.close) + # Use a thread to complete the connection, but wait for it to + # terminate before running the test, so that there is only one + # thread to accept the signal. + cli_thread = threading.Thread(target=self.doConnect) + cli_thread.start() + self.cli_conn, addr = self.serv.accept() + self.addCleanup(self.cli_conn.close) + cli_thread.join() + self.serv_conn.settimeout(self.timeout) + + def doConnect(self): + self.serv_conn.connect(self.serv_addr) + + def checkInterruptedSend(self, func, *args, **kwargs): + # Check that func(*args, **kwargs), run in a loop, raises + # socket.error with an errno of EINTR when interrupted by a + # signal. + with self.assertRaises(socket.error) as cm: + while True: + self.setAlarm(self.alarm_time) + func(*args, **kwargs) + self.assertNotIsInstance(cm.exception, socket.timeout) + self.assertEqual(cm.exception.errno, errno.EINTR) + + # Issue #12958: The following tests have problems on Mac OS X + @support.anticipate_failure(sys.platform == "darwin") + def testInterruptedSendTimeout(self): + self.checkInterruptedSend(self.serv_conn.send, b"a"*512) + + @support.anticipate_failure(sys.platform == "darwin") + def testInterruptedSendtoTimeout(self): + # Passing an actual address here as Python's wrapper for + # sendto() doesn't allow passing a zero-length one; POSIX + # requires that the address is ignored since the socket is + # connection-mode, however. + self.checkInterruptedSend(self.serv_conn.sendto, b"a"*512, + self.serv_addr) + + @support.anticipate_failure(sys.platform == "darwin") + @requireAttrs(socket.socket, "sendmsg") + def testInterruptedSendmsgTimeout(self): + self.checkInterruptedSend(self.serv_conn.sendmsg, [b"a"*512]) + + @unittest.skipUnless(thread, 'Threading required for this test.') class TCPCloserTest(ThreadedTCPSocketTest): @@ -996,11 +3461,8 @@ class NonBlockingTCPTests(ThreadedTCPSocketTest): pass if hasattr(socket, "SOCK_NONBLOCK"): + @support.requires_linux_version(2, 6, 28) def testInitNonBlocking(self): - v = linux_version() - if v < (2, 6, 28): - self.skipTest("Linux kernel 2.6.28 or higher required, not %s" - % ".".join(map(str, v))) # reinit server socket self.serv.close() self.serv = socket.socket(socket.AF_INET, socket.SOCK_STREAM | @@ -1102,7 +3564,7 @@ class FileObjectClassTestCase(SocketConnectedTest): """ bufsize = -1 # Use default buffer size - encoding = 'utf8' + encoding = 'utf-8' errors = 'strict' newline = None @@ -1283,7 +3745,7 @@ class FileObjectInterruptedTestCase(unittest.TestCase): @staticmethod def _raise_eintr(): - raise socket.error(errno.EINTR) + raise socket.error(errno.EINTR, "interrupted") def _textiowrap_mock_socket(self, mock, buffering=-1): raw = socket.SocketIO(mock, "r") @@ -1323,7 +3785,7 @@ class FileObjectInterruptedTestCase(unittest.TestCase): data = b'' else: data = '' - expecting = expecting.decode('utf8') + expecting = expecting.decode('utf-8') while len(data) != len(expecting): part = fo.read(size) if not part: @@ -1485,7 +3947,7 @@ class UnicodeReadFileObjectClassTestCase(FileObjectClassTestCase): """Tests for socket.makefile() in text mode (rather than binary)""" read_mode = 'r' - read_msg = MSG.decode('utf8') + read_msg = MSG.decode('utf-8') write_mode = 'wb' write_msg = MSG newline = '' @@ -1497,7 +3959,7 @@ class UnicodeWriteFileObjectClassTestCase(FileObjectClassTestCase): read_mode = 'rb' read_msg = MSG write_mode = 'w' - write_msg = MSG.decode('utf8') + write_msg = MSG.decode('utf-8') newline = '' @@ -1505,9 +3967,9 @@ class UnicodeReadWriteFileObjectClassTestCase(FileObjectClassTestCase): """Tests for socket.makefile() in text mode (rather than binary)""" read_mode = 'r' - read_msg = MSG.decode('utf8') + read_msg = MSG.decode('utf-8') write_mode = 'w' - write_msg = MSG.decode('utf8') + write_msg = MSG.decode('utf-8') newline = '' @@ -1979,11 +4441,8 @@ class ContextManagersTest(ThreadedTCPSocketTest): "SOCK_CLOEXEC not defined") @unittest.skipUnless(fcntl, "module fcntl not available") class CloexecConstantTest(unittest.TestCase): + @support.requires_linux_version(2, 6, 28) def test_SOCK_CLOEXEC(self): - v = linux_version() - if v < (2, 6, 28): - self.skipTest("Linux kernel 2.6.28 or higher required, not %s" - % ".".join(map(str, v))) with socket.socket(socket.AF_INET, socket.SOCK_STREAM | socket.SOCK_CLOEXEC) as s: self.assertTrue(s.type & socket.SOCK_CLOEXEC) @@ -2001,11 +4460,8 @@ class NonblockConstantTest(unittest.TestCase): self.assertFalse(s.type & socket.SOCK_NONBLOCK) self.assertEqual(s.gettimeout(), None) + @support.requires_linux_version(2, 6, 28) def test_SOCK_NONBLOCK(self): - v = linux_version() - if v < (2, 6, 28): - self.skipTest("Linux kernel 2.6.28 or higher required, not %s" - % ".".join(map(str, v))) # a lot of it seems silly and redundant, but I wanted to test that # changing back and forth worked ok with socket.socket(socket.AF_INET, @@ -2061,11 +4517,38 @@ def test_main(): ]) if hasattr(socket, "socketpair"): tests.append(BasicSocketPairTest) - if sys.platform == 'linux2': + if sys.platform == 'linux': tests.append(TestLinuxAbstractNamespace) if isTipcAvailable(): tests.append(TIPCTest) tests.append(TIPCThreadableTest) + tests.extend([BasicCANTest, CANTest]) + tests.extend([BasicRDSTest, RDSTest]) + tests.extend([ + CmsgMacroTests, + SendmsgUDPTest, + RecvmsgUDPTest, + RecvmsgIntoUDPTest, + SendmsgUDP6Test, + RecvmsgUDP6Test, + RecvmsgRFC3542AncillaryUDP6Test, + RecvmsgIntoRFC3542AncillaryUDP6Test, + RecvmsgIntoUDP6Test, + SendmsgTCPTest, + RecvmsgTCPTest, + RecvmsgIntoTCPTest, + SendmsgSCTPStreamTest, + RecvmsgSCTPStreamTest, + RecvmsgIntoSCTPStreamTest, + SendmsgUnixStreamTest, + RecvmsgUnixStreamTest, + RecvmsgIntoUnixStreamTest, + RecvmsgSCMRightsStreamTest, + RecvmsgIntoSCMRightsStreamTest, + # These are slow when setitimer() is not available + InterruptedRecvTimeoutTest, + InterruptedSendTimeoutTest, + ]) thread_info = support.threading_setup() support.run_unittest(*tests) diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py index f942d95..a2b4040 100644 --- a/Lib/test/test_ssl.py +++ b/Lib/test/test_ssl.py @@ -42,6 +42,9 @@ ONLYCERT = data_file("ssl_cert.pem") ONLYKEY = data_file("ssl_key.pem") BYTES_ONLYCERT = os.fsencode(ONLYCERT) BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" CAPATH = data_file("capath") BYTES_CAPATH = os.fsencode(CAPATH) @@ -103,6 +106,16 @@ class BasicSocketTests(unittest.TestCase): sys.stdout.write("\n RAND_status is %d (%s)\n" % (v, (v and "sufficient randomness") or "insufficient randomness")) + + data, is_cryptographic = ssl.RAND_pseudo_bytes(16) + self.assertEqual(len(data), 16) + self.assertEqual(is_cryptographic, v == 1) + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) + try: ssl.RAND_egd(1) except TypeError: @@ -337,6 +350,25 @@ class BasicSocketTests(unittest.TestCase): self.assertRaises(ValueError, ctx.wrap_socket, sock, True, server_hostname="some.hostname") + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s) + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + ss = ssl.wrap_socket(s, server_side=True, certfile=CERTFILE) + self.assertIsNone(ss.get_channel_binding("tls-unique")) + class ContextTests(unittest.TestCase): @skip_if_broken_ubuntu_ssl @@ -427,6 +459,60 @@ class ContextTests(unittest.TestCase): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) with self.assertRaisesRegex(ssl.SSLError, "key values mismatch"): ctx.load_cert_chain(SVN_PYTHON_ORG_ROOT_CERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegex(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegex(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegex(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegex(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegex(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) def test_load_verify_locations(self): ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1) @@ -533,13 +619,10 @@ class NetworkedTests(unittest.TestCase): try: s.do_handshake() break - except ssl.SSLError as err: - if err.args[0] == ssl.SSL_ERROR_WANT_READ: - select.select([s], [], [], 5.0) - elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: - select.select([], [s], [], 5.0) - else: - raise + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) # SSL established self.assertTrue(s.getpeercert()) finally: @@ -659,37 +742,39 @@ class NetworkedTests(unittest.TestCase): count += 1 s.do_handshake() break - except ssl.SSLError as err: - if err.args[0] == ssl.SSL_ERROR_WANT_READ: - select.select([s], [], []) - elif err.args[0] == ssl.SSL_ERROR_WANT_WRITE: - select.select([], [s], []) - else: - raise + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) s.close() if support.verbose: sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) def test_get_server_certificate(self): - with support.transient_internet("svn.python.org"): - pem = ssl.get_server_certificate(("svn.python.org", 443)) - if not pem: - self.fail("No server certificate on svn.python.org:443!") + def _test_get_server_certificate(host, port, cert=None): + with support.transient_internet(host): + pem = ssl.get_server_certificate((host, port)) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) - try: - pem = ssl.get_server_certificate(("svn.python.org", 443), ca_certs=CERTFILE) - except ssl.SSLError as x: - #should fail + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + self.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=cert) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) if support.verbose: - sys.stdout.write("%s\n" % x) - else: - self.fail("Got server certificate %s for svn.python.org!" % pem) + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) - pem = ssl.get_server_certificate(("svn.python.org", 443), ca_certs=SVN_PYTHON_ORG_ROOT_CERT) - if not pem: - self.fail("No server certificate on svn.python.org:443!") - if support.verbose: - sys.stdout.write("\nVerified certificate for svn.python.org:443 is\n%s\n" % pem) + _test_get_server_certificate('svn.python.org', 443, SVN_PYTHON_ORG_ROOT_CERT) + if support.IPV6_ENABLED: + _test_get_server_certificate('ipv6.google.com', 443) def test_ciphers(self): remote = ("svn.python.org", 443) @@ -837,6 +922,11 @@ else: self.sslconn = None if support.verbose and self.server.connectionchatty: sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") else: if (support.verbose and self.server.connectionchatty): @@ -934,12 +1024,11 @@ else: def _do_ssl_handshake(self): try: self.socket.do_handshake() - except ssl.SSLError as err: - if err.args[0] in (ssl.SSL_ERROR_WANT_READ, - ssl.SSL_ERROR_WANT_WRITE): - return - elif err.args[0] == ssl.SSL_ERROR_EOF: - return self.handle_close() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: raise except socket.error as err: if err.args[0] == errno.ECONNABORTED: @@ -1248,7 +1337,8 @@ else: t.join() @skip_if_broken_ubuntu_ssl - @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), "need SSLv2") + @unittest.skipUnless(hasattr(ssl, 'PROTOCOL_SSLv2'), + "OpenSSL is compiled without SSLv2 support") def test_protocol_sslv2(self): """Connecting to an SSLv2 server with various client options""" if support.verbose: @@ -1582,6 +1672,14 @@ else: # consume data s.read() + # Make sure sendmsg et al are disallowed to avoid + # inadvertent disclosure of data and/or corruption + # of the encrypted data stream + self.assertRaises(NotImplementedError, s.sendmsg, [b"data"]) + self.assertRaises(NotImplementedError, s.recvmsg, 100) + self.assertRaises(NotImplementedError, + s.recvmsg_into, bytearray(100)) + s.write(b"over\n") s.close() finally: @@ -1637,6 +1735,73 @@ else: t.join() server.close() + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + flag = threading.Event() + server.start(flag) + # wait for it to start + flag.wait() + # try to connect + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + try: + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got channel binding data: {0!r}\n" + .format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + s.close() + + # now, again + s = ssl.wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLSv1) + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write(" got another channel binding data: {0!r}\n" + .format(new_cb_data)) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + s.close() + finally: + server.stop() + server.join() def test_main(verbose=False): if support.verbose: diff --git a/Lib/test/test_string.py b/Lib/test/test_string.py index a352ee3..1615732 100644 --- a/Lib/test/test_string.py +++ b/Lib/test/test_string.py @@ -26,15 +26,38 @@ class ModuleTest(unittest.TestCase): self.assertEqual(string.capwords('\taBc\tDeF\t'), 'Abc Def') self.assertEqual(string.capwords('\taBc\tDeF\t', '\t'), '\tAbc\tDef\t') - def test_formatter(self): + def test_basic_formatter(self): fmt = string.Formatter() self.assertEqual(fmt.format("foo"), "foo") - self.assertEqual(fmt.format("foo{0}", "bar"), "foobar") self.assertEqual(fmt.format("foo{1}{0}-{1}", "bar", 6), "foo6bar-6") + + def test_conversion_specifiers(self): + fmt = string.Formatter() self.assertEqual(fmt.format("-{arg!r}-", arg='test'), "-'test'-") + self.assertEqual(fmt.format("{0!s}", 'test'), 'test') + self.assertRaises(ValueError, fmt.format, "{0!h}", 'test') + + def test_name_lookup(self): + fmt = string.Formatter() + class AnyAttr: + def __getattr__(self, attr): + return attr + x = AnyAttr() + self.assertEqual(fmt.format("{0.lumber}{0.jack}", x), 'lumberjack') + with self.assertRaises(AttributeError): + fmt.format("{0.lumber}{0.jack}", '') + + def test_index_lookup(self): + fmt = string.Formatter() + lookup = ["eggs", "and", "spam"] + self.assertEqual(fmt.format("{0[2]}{0[0]}", lookup), 'spameggs') + with self.assertRaises(IndexError): + fmt.format("{0[2]}{0[0]}", []) + with self.assertRaises(KeyError): + fmt.format("{0[2]}{0[0]}", {}) - # override get_value ############################################ + def test_override_get_value(self): class NamespaceFormatter(string.Formatter): def __init__(self, namespace={}): string.Formatter.__init__(self) @@ -54,7 +77,7 @@ class ModuleTest(unittest.TestCase): self.assertEqual(fmt.format("{greeting}, world!"), 'hello, world!') - # override format_field ######################################### + def test_override_format_field(self): class CallFormatter(string.Formatter): def format_field(self, value, format_spec): return format(value(), format_spec) @@ -63,18 +86,18 @@ class ModuleTest(unittest.TestCase): self.assertEqual(fmt.format('*{0}*', lambda : 'result'), '*result*') - # override convert_field ######################################## + def test_override_convert_field(self): class XFormatter(string.Formatter): def convert_field(self, value, conversion): if conversion == 'x': return None - return super(XFormatter, self).convert_field(value, conversion) + return super().convert_field(value, conversion) fmt = XFormatter() self.assertEqual(fmt.format("{0!r}:{0!x}", 'foo', 'foo'), "'foo':None") - # override parse ################################################ + def test_override_parse(self): class BarFormatter(string.Formatter): # returns an iterable that contains tuples of the form: # (literal_text, field_name, format_spec, conversion) @@ -90,7 +113,7 @@ class ModuleTest(unittest.TestCase): fmt = BarFormatter() self.assertEqual(fmt.format('*|+0:^10s|*', 'foo'), '* foo *') - # test all parameters used + def test_check_unused_args(self): class CheckAllUsedFormatter(string.Formatter): def check_unused_args(self, used_args, args, kwargs): # Track which arguments actually got used @@ -112,28 +135,13 @@ class ModuleTest(unittest.TestCase): self.assertRaises(ValueError, fmt.format, "{0}", 10, 20, i=100) self.assertRaises(ValueError, fmt.format, "{i}", 10, 20, i=100) - def test_vformat_assert(self): - cls = string.Formatter() - kwargs = { - "i": 100 - } - self.assertRaises(ValueError, cls._vformat, - cls.format, "{0}", kwargs, set(), -2) - - def test_convert_field(self): - cls = string.Formatter() - self.assertEqual(cls.format("{0!s}", 'foo'), 'foo') - self.assertRaises(ValueError, cls.format, "{0!h}", 'foo') - - def test_get_field(self): - cls = string.Formatter() - class MyClass: - name = 'lumberjack' - x = MyClass() - self.assertEqual(cls.format("{0.name}", x), 'lumberjack') - - lookup = ["eggs", "and", "spam"] - self.assertEqual(cls.format("{0[2]}", lookup), 'spam') + def test_vformat_recursion_limit(self): + fmt = string.Formatter() + args = () + kwargs = dict(i=100) + with self.assertRaises(ValueError) as err: + fmt._vformat("{i}", args, kwargs, set(), -1) + self.assertIn("recursion", str(err.exception)) def test_main(): diff --git a/Lib/test/test_strlit.py b/Lib/test/test_strlit.py index 9eb30e9..fb9cdbb 100644 --- a/Lib/test/test_strlit.py +++ b/Lib/test/test_strlit.py @@ -130,7 +130,7 @@ class TestLiterals(unittest.TestCase): self.assertRaises(SyntaxError, self.check_encoding, "utf-8", extra) def test_file_utf8(self): - self.check_encoding("utf8") + self.check_encoding("utf-8") def test_file_iso_8859_1(self): self.check_encoding("iso-8859-1") diff --git a/Lib/test/test_struct.py b/Lib/test/test_struct.py index 2ccaad2..dc75858 100644 --- a/Lib/test/test_struct.py +++ b/Lib/test/test_struct.py @@ -8,9 +8,19 @@ from test.support import run_unittest ISBIGENDIAN = sys.byteorder == "big" IS32BIT = sys.maxsize == 0x7fffffff -integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q' +integer_codes = 'b', 'B', 'h', 'H', 'i', 'I', 'l', 'L', 'q', 'Q', 'n', 'N' byteorders = '', '@', '=', '<', '>', '!' +def iter_integer_formats(byteorders=byteorders): + for code in integer_codes: + for byteorder in byteorders: + if (byteorder in ('', '@') and code in ('q', 'Q') and + not HAVE_LONG_LONG): + continue + if (byteorder not in ('', '@') and code in ('n', 'N')): + continue + yield code, byteorder + # Native 'q' packing isn't available on systems that don't have the C # long long type. try: @@ -141,14 +151,13 @@ class StructTest(unittest.TestCase): } # standard integer sizes - for code in integer_codes: - for byteorder in '=', '<', '>', '!': - format = byteorder+code - size = struct.calcsize(format) - self.assertEqual(size, expected_size[code]) + for code, byteorder in iter_integer_formats(('=', '<', '>', '!')): + format = byteorder+code + size = struct.calcsize(format) + self.assertEqual(size, expected_size[code]) # native integer sizes - native_pairs = 'bB', 'hH', 'iI', 'lL' + native_pairs = 'bB', 'hH', 'iI', 'lL', 'nN' if HAVE_LONG_LONG: native_pairs += 'qQ', for format_pair in native_pairs: @@ -166,9 +175,11 @@ class StructTest(unittest.TestCase): if HAVE_LONG_LONG: self.assertLessEqual(8, struct.calcsize('q')) self.assertLessEqual(struct.calcsize('l'), struct.calcsize('q')) + self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('i')) + self.assertGreaterEqual(struct.calcsize('n'), struct.calcsize('P')) def test_integers(self): - # Integer tests (bBhHiIlLqQ). + # Integer tests (bBhHiIlLqQnN). import binascii class IntTester(unittest.TestCase): @@ -182,11 +193,11 @@ class StructTest(unittest.TestCase): self.byteorder) self.bytesize = struct.calcsize(format) self.bitsize = self.bytesize * 8 - if self.code in tuple('bhilq'): + if self.code in tuple('bhilqn'): self.signed = True self.min_value = -(2**(self.bitsize-1)) self.max_value = 2**(self.bitsize-1) - 1 - elif self.code in tuple('BHILQ'): + elif self.code in tuple('BHILQN'): self.signed = False self.min_value = 0 self.max_value = 2**self.bitsize - 1 @@ -316,14 +327,23 @@ class StructTest(unittest.TestCase): struct.pack, self.format, obj) - for code in integer_codes: - for byteorder in byteorders: - if (byteorder in ('', '@') and code in ('q', 'Q') and - not HAVE_LONG_LONG): - continue + for code, byteorder in iter_integer_formats(): + format = byteorder+code + t = IntTester(format) + t.run() + + def test_nN_code(self): + # n and N don't exist in standard sizes + def assertStructError(func, *args, **kwargs): + with self.assertRaises(struct.error) as cm: + func(*args, **kwargs) + self.assertIn("bad char in struct format", str(cm.exception)) + for code in 'nN': + for byteorder in ('=', '<', '>', '!'): format = byteorder+code - t = IntTester(format) - t.run() + assertStructError(struct.calcsize, format) + assertStructError(struct.pack, format, 0) + assertStructError(struct.unpack, format, b"") def test_p_code(self): # Test p ("Pascal string") code. @@ -377,14 +397,10 @@ class StructTest(unittest.TestCase): self.assertRaises(OverflowError, struct.pack, ">f", big) def test_1530559(self): - for byteorder in '', '@', '=', '<', '>', '!': - for code in integer_codes: - if (byteorder in ('', '@') and code in ('q', 'Q') and - not HAVE_LONG_LONG): - continue - format = byteorder + code - self.assertRaises(struct.error, struct.pack, format, 1.0) - self.assertRaises(struct.error, struct.pack, format, 1.5) + for code, byteorder in iter_integer_formats(): + format = byteorder + code + self.assertRaises(struct.error, struct.pack, format, 1.0) + self.assertRaises(struct.error, struct.pack, format, 1.5) self.assertRaises(struct.error, struct.pack, 'P', 1.0) self.assertRaises(struct.error, struct.pack, 'P', 1.5) diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index ff76ec4..432d324 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -58,6 +58,8 @@ class BaseTestCase(unittest.TestCase): # shutdown time. That frustrates tests trying to check stderr produced # from a spawned Python process. actual = support.strip_python_stderr(stderr) + # strip_python_stderr also strips whitespace, so we do too. + expected = expected.strip() self.assertEqual(actual, expected, msg) @@ -69,6 +71,15 @@ class ProcessTestCase(BaseTestCase): "import sys; sys.exit(47)"]) self.assertEqual(rc, 47) + def test_call_timeout(self): + # call() function with timeout argument; we want to test that the child + # process gets killed when the timeout expires. If the child isn't + # killed, this call will deadlock since subprocess.call waits for the + # child. + self.assertRaises(subprocess.TimeoutExpired, subprocess.call, + [sys.executable, "-c", "while True: pass"], + timeout=0.1) + def test_check_call_zero(self): # check_call() function with zero return code rc = subprocess.check_call([sys.executable, "-c", @@ -111,6 +122,21 @@ class ProcessTestCase(BaseTestCase): self.fail("Expected ValueError when stdout arg supplied.") self.assertIn('stdout', c.exception.args[0]) + def test_check_output_timeout(self): + # check_output() function with timeout arg + with self.assertRaises(subprocess.TimeoutExpired) as c: + output = subprocess.check_output( + [sys.executable, "-c", + "import sys, time\n" + "sys.stdout.write('BDFL')\n" + "sys.stdout.flush()\n" + "time.sleep(3600)"], + # Some heavily loaded buildbots (sparc Debian 3.x) require + # this much time to start and print. + timeout=3) + self.fail("Expected TimeoutExpired.") + self.assertEqual(c.exception.output, b'BDFL') + def test_call_kwargs(self): # call() function with keyword args newenv = os.environ.copy() @@ -310,6 +336,31 @@ class ProcessTestCase(BaseTestCase): rc = subprocess.call([sys.executable, "-c", cmd], stdout=1) self.assertEqual(rc, 2) + def test_stdout_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'for i in range(10240):' + 'print("x" * 1024)'], + stdout=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdout, None) + + def test_stderr_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys\n' + 'for i in range(10240):' + 'sys.stderr.write("x" * 1024)'], + stderr=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stderr, None) + + def test_stdin_devnull(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys;' + 'sys.stdin.read(1)'], + stdin=subprocess.DEVNULL) + p.wait() + self.assertEqual(p.stdin, None) + def test_cwd(self): tmpdir = tempfile.gettempdir() # We cannot use os.path.realpath to canonicalize the path, @@ -398,6 +449,41 @@ class ProcessTestCase(BaseTestCase): self.assertEqual(stdout, b"banana") self.assertStderrEqual(stderr, b"pineapple") + def test_communicate_timeout(self): + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stderr.write("pineapple\\n");' + 'time.sleep(1);' + 'sys.stderr.write("pear\\n");' + 'sys.stdout.write(sys.stdin.read())'], + universal_newlines=True, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, "banana", + timeout=0.3) + # Make sure we can keep waiting for it, and that we get the whole output + # after it completes. + (stdout, stderr) = p.communicate() + self.assertEqual(stdout, "banana") + self.assertStderrEqual(stderr.encode(), b"pineapple\npear\n") + + def test_communicate_timeout_large_ouput(self): + # Test a expring timeout while the child is outputting lots of data. + p = subprocess.Popen([sys.executable, "-c", + 'import sys,os,time;' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));' + 'time.sleep(0.2);' + 'sys.stdout.write("a" * (64 * 1024));'], + stdout=subprocess.PIPE) + self.assertRaises(subprocess.TimeoutExpired, p.communicate, timeout=0.4) + (stdout, _) = p.communicate() + self.assertEqual(len(stdout), 4 * 64 * 1024) + # Test for the fd leak reported in http://bugs.python.org/issue2791. def test_communicate_pipe_fd_leak(self): for stdin_pipe in (False, True): @@ -434,24 +520,21 @@ class ProcessTestCase(BaseTestCase): # This test will probably deadlock rather than fail, if # communicate() does not work properly. x, y = os.pipe() - if mswindows: - pipe_buf = 512 - else: - pipe_buf = os.fpathconf(x, "PC_PIPE_BUF") os.close(x) os.close(y) p = subprocess.Popen([sys.executable, "-c", 'import sys,os;' 'sys.stdout.write(sys.stdin.read(47));' - 'sys.stderr.write("xyz"*%d);' - 'sys.stdout.write(sys.stdin.read())' % pipe_buf], + 'sys.stderr.write("x" * %d);' + 'sys.stdout.write(sys.stdin.read())' % + support.PIPE_MAX_SIZE], stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE) self.addCleanup(p.stdout.close) self.addCleanup(p.stderr.close) self.addCleanup(p.stdin.close) - string_to_write = b"abc"*pipe_buf + string_to_write = b"a" * support.PIPE_MAX_SIZE (stdout, stderr) = p.communicate(string_to_write) self.assertEqual(stdout, string_to_write) @@ -625,6 +708,15 @@ class ProcessTestCase(BaseTestCase): # Subsequent invocations should just return the returncode self.assertEqual(p.wait(), 0) + def test_wait_timeout(self): + p = subprocess.Popen([sys.executable, + "-c", "import time; time.sleep(0.1)"]) + with self.assertRaises(subprocess.TimeoutExpired) as c: + p.wait(timeout=0.01) + self.assertIn("0.01", str(c.exception)) # For coverage of __str__. + # Some heavily loaded buildbots (sparc Debian 3.x) require this much + # time to start. + self.assertEqual(p.wait(timeout=3), 0) def test_invalid_bufsize(self): # an invalid type of the bufsize argument should raise @@ -1220,6 +1312,11 @@ class POSIXProcessTestCase(BaseTestCase): exitcode = subprocess.call([abs_program, "-c", "pass"]) self.assertEqual(exitcode, 0) + # absolute bytes path as a string + cmd = b"'" + abs_program + b"' -c pass" + exitcode = subprocess.call(cmd, shell=True) + self.assertEqual(exitcode, 0) + # bytes program, unicode PATH env = os.environ.copy() env["PATH"] = path @@ -1375,7 +1472,7 @@ class POSIXProcessTestCase(BaseTestCase): stdout, stderr = p.communicate() self.assertEqual(0, p.returncode, "sigchild_ignore.py exited" " non-zero with this error:\n%s" % - stderr.decode('utf8')) + stderr.decode('utf-8')) def test_select_unbuffered(self): # Issue #11459: bufsize=0 should really set the pipes as @@ -1588,28 +1685,6 @@ class ProcessTestCaseNoPoll(ProcessTestCase): ProcessTestCase.tearDown(self) -@unittest.skipUnless(getattr(subprocess, '_posixsubprocess', False), - "_posixsubprocess extension module not found.") -class ProcessTestCasePOSIXPurePython(ProcessTestCase, POSIXProcessTestCase): - @classmethod - def setUpClass(cls): - global subprocess - assert subprocess._posixsubprocess - # Reimport subprocess while forcing _posixsubprocess to not exist. - with support.check_warnings(('.*_posixsubprocess .* not being used.*', - RuntimeWarning)): - subprocess = support.import_fresh_module( - 'subprocess', blocked=['_posixsubprocess']) - assert not subprocess._posixsubprocess - - @classmethod - def tearDownClass(cls): - global subprocess - # Reimport subprocess as it should be, restoring order to the universe. - subprocess = support.import_fresh_module('subprocess') - assert subprocess._posixsubprocess - - class HelperFunctionTests(unittest.TestCase): @unittest.skipIf(mswindows, "errno and EINTR make no sense on windows") def test_eintr_retry_call(self): @@ -1718,7 +1793,6 @@ def test_main(): unit_tests = (ProcessTestCase, POSIXProcessTestCase, Win32ProcessTestCase, - ProcessTestCasePOSIXPurePython, CommandTests, ProcessTestCaseNoPoll, HelperFunctionTests, diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index 914216d..298cae0 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -81,6 +81,16 @@ class TestSuper(unittest.TestCase): self.assertEqual(E().f(), 'AE') + def test___class___set(self): + # See issue #12370 + class X(A): + def f(self): + return super().f() + __class__ = 413 + x = X() + self.assertEqual(x.f(), 'A') + self.assertEqual(x.__class__, 413) + def test_main(): support.run_unittest(TestSuper) diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py new file mode 100644 index 0000000..394e210 --- /dev/null +++ b/Lib/test/test_support.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python + +import sys +import os +import unittest +import socket +import tempfile +import errno +from test import support + +TESTFN = support.TESTFN +TESTDIRN = os.path.basename(tempfile.mkdtemp(dir='.')) + + +class TestSupport(unittest.TestCase): + def setUp(self): + support.unlink(TESTFN) + support.rmtree(TESTDIRN) + tearDown = setUp + + def test_import_module(self): + support.import_module("ftplib") + self.assertRaises(unittest.SkipTest, support.import_module, "foo") + + def test_import_fresh_module(self): + support.import_fresh_module("ftplib") + + def test_get_attribute(self): + self.assertEqual(support.get_attribute(self, "test_get_attribute"), + self.test_get_attribute) + self.assertRaises(unittest.SkipTest, support.get_attribute, self, "foo") + + @unittest.skip("failing buildbots") + def test_get_original_stdout(self): + self.assertEqual(support.get_original_stdout(), sys.stdout) + + def test_unload(self): + import sched + self.assertIn("sched", sys.modules) + support.unload("sched") + self.assertNotIn("sched", sys.modules) + + def test_unlink(self): + with open(TESTFN, "w") as f: + pass + support.unlink(TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + support.unlink(TESTFN) + + def test_rmtree(self): + os.mkdir(TESTDIRN) + os.mkdir(os.path.join(TESTDIRN, TESTDIRN)) + support.rmtree(TESTDIRN) + self.assertFalse(os.path.exists(TESTDIRN)) + support.rmtree(TESTDIRN) + + def test_forget(self): + mod_filename = TESTFN + '.py' + with open(mod_filename, 'w') as f: + print('foo = 1', file=f) + sys.path.insert(0, os.curdir) + try: + mod = __import__(TESTFN) + self.assertIn(TESTFN, sys.modules) + + support.forget(TESTFN) + self.assertNotIn(TESTFN, sys.modules) + finally: + del sys.path[0] + support.unlink(mod_filename) + + def test_HOST(self): + s = socket.socket() + s.bind((support.HOST, 0)) + s.close() + + def test_find_unused_port(self): + port = support.find_unused_port() + s = socket.socket() + s.bind((support.HOST, port)) + s.close() + + def test_bind_port(self): + s = socket.socket() + support.bind_port(s) + s.listen(1) + s.close() + + def test_temp_cwd(self): + here = os.getcwd() + with support.temp_cwd(name=TESTFN): + self.assertEqual(os.path.basename(os.getcwd()), TESTFN) + self.assertFalse(os.path.exists(TESTFN)) + self.assertTrue(os.path.basename(os.getcwd()), here) + + def test_sortdict(self): + self.assertEqual(support.sortdict({3:3, 2:2, 1:1}), "{1: 1, 2: 2, 3: 3}") + + def test_make_bad_fd(self): + fd = support.make_bad_fd() + with self.assertRaises(OSError) as cm: + os.write(fd, b"foo") + self.assertEqual(cm.exception.errno, errno.EBADF) + + def test_check_syntax_error(self): + support.check_syntax_error(self, "def class") + self.assertRaises(AssertionError, support.check_syntax_error, self, "1") + + def test_CleanImport(self): + import importlib + with support.CleanImport("asyncore"): + importlib.import_module("asyncore") + + def test_DirsOnSysPath(self): + with support.DirsOnSysPath('foo', 'bar'): + self.assertIn("foo", sys.path) + self.assertIn("bar", sys.path) + self.assertNotIn("foo", sys.path) + self.assertNotIn("bar", sys.path) + + def test_captured_stdout(self): + with support.captured_stdout() as s: + print("hello") + self.assertEqual(s.getvalue(), "hello\n") + + def test_captured_stderr(self): + with support.captured_stderr() as s: + print("hello", file=sys.stderr) + self.assertEqual(s.getvalue(), "hello\n") + + def test_captured_stdin(self): + with support.captured_stdin() as s: + print("hello", file=sys.stdin) + self.assertEqual(s.getvalue(), "hello\n") + + def test_gc_collect(self): + support.gc_collect() + + def test_python_is_optimized(self): + self.assertIsInstance(support.python_is_optimized(), bool) + + def test_swap_attr(self): + class Obj: + x = 1 + obj = Obj() + with support.swap_attr(obj, "x", 5): + self.assertEqual(obj.x, 5) + self.assertEqual(obj.x, 1) + + def test_swap_item(self): + D = {"item":1} + with support.swap_item(D, "item", 5): + self.assertEqual(D["item"], 5) + self.assertEqual(D["item"], 1) + + # XXX -follows a list of untested API + # make_legacy_pyc + # is_resource_enabled + # requires + # fcmp + # umaks + # findfile + # check_warnings + # EnvironmentVarGuard + # TransientResource + # transient_internet + # run_with_locale + # set_memlimit + # bigmemtest + # precisionbigmemtest + # bigaddrspacetest + # requires_resource + # run_doctest + # threading_cleanup + # reap_threads + # reap_children + # strip_python_stderr + # args_from_interpreter_flags + # can_symlink + # skip_unless_symlink + + +def test_main(): + tests = [TestSupport] + support.run_unittest(*tests) + +if __name__ == '__main__': + test_main() diff --git a/Lib/test/test_sys.py b/Lib/test/test_sys.py index db27cdf..c99f4d7 100644 --- a/Lib/test/test_sys.py +++ b/Lib/test/test_sys.py @@ -303,6 +303,7 @@ class SysModuleTest(unittest.TestCase): self.assertEqual(sys.getdlopenflags(), oldflags+1) sys.setdlopenflags(oldflags) + @test.support.refcount_test def test_refcount(self): # n here must be a global in order for this test to pass while # tracing with a python function. Tracing calls PyFrame_FastToLocals @@ -342,7 +343,7 @@ class SysModuleTest(unittest.TestCase): # Test sys._current_frames() in a WITH_THREADS build. @test.support.reap_threads def current_frames_with_threads(self): - import threading, _thread + import threading import traceback # Spawn a thread that blocks at a known place. Then the main @@ -356,7 +357,7 @@ class SysModuleTest(unittest.TestCase): g456() def g456(): - thread_info.append(_thread.get_ident()) + thread_info.append(threading.get_ident()) entered_g.set() leave_g.wait() @@ -372,7 +373,7 @@ class SysModuleTest(unittest.TestCase): d = sys._current_frames() - main_id = _thread.get_ident() + main_id = threading.get_ident() self.assertIn(main_id, d) self.assertIn(thread_id, d) @@ -446,6 +447,7 @@ class SysModuleTest(unittest.TestCase): self.assertIsInstance(sys.maxsize, int) self.assertIsInstance(sys.maxunicode, int) + self.assertEqual(sys.maxunicode, 0x10FFFF) self.assertIsInstance(sys.platform, str) self.assertIsInstance(sys.prefix, str) self.assertIsInstance(sys.version, str) @@ -473,6 +475,14 @@ class SysModuleTest(unittest.TestCase): if not sys.platform.startswith('win'): self.assertIsInstance(sys.abiflags, str) + @unittest.skipUnless(hasattr(sys, 'thread_info'), + 'Threading required for this test.') + def test_thread_info(self): + info = sys.thread_info + self.assertEqual(len(info), 3) + self.assertIn(info.name, ('nt', 'os2', 'pthread', 'solaris', None)) + self.assertIn(info.lock, ('semaphore', 'mutex+cond', None)) + def test_43581(self): # Can't use sys.stdout, as this is a StringIO object when # the test runs under regrtest. @@ -500,7 +510,7 @@ class SysModuleTest(unittest.TestCase): def test_sys_flags(self): self.assertTrue(sys.flags) - attrs = ("debug", "division_warning", + attrs = ("debug", "inspect", "interactive", "optimize", "dont_write_bytecode", "no_user_site", "no_site", "ignore_environment", "verbose", "bytes_warning", "quiet") @@ -656,7 +666,7 @@ class SizeofTest(unittest.TestCase): return inner check(get_cell().__closure__[0], size(h + 'P')) # code - check(get_cell().__code__, size(h + '5i8Pi3P')) + check(get_cell().__code__, size(h + '5i9Pi3P')) # complex check(complex(0,1), size(h + '2d')) # method_descriptor (descriptor object) @@ -824,13 +834,35 @@ class SizeofTest(unittest.TestCase): class newstyleclass(object): pass check(newstyleclass, s) # unicode - usize = len('\0'.encode('unicode-internal')) - samples = ['', '1'*100] - # we need to test for both sizes, because we don't know if the string - # has been cached + # each tuple contains a string and its expected character size + # don't put any static strings here, as they may contain + # wchar_t or UTF-8 representations + samples = ['1'*100, '\xff'*50, + '\u0100'*40, '\uffff'*100, + '\U00010000'*30, '\U0010ffff'*100] + asciifields = h + "PPiP" + compactfields = asciifields + "PPP" + unicodefields = compactfields + "P" for s in samples: - basicsize = size(h + 'PPPiP') + usize * (len(s) + 1) - check(s, basicsize) + maxchar = ord(max(s)) + if maxchar < 128: + L = size(asciifields) + len(s) + 1 + elif maxchar < 256: + L = size(compactfields) + len(s) + 1 + elif maxchar < 65536: + L = size(compactfields) + 2*(len(s) + 1) + else: + L = size(compactfields) + 4*(len(s) + 1) + check(s, L) + # verify that the UTF-8 size is accounted for + s = chr(0x4000) # 4 bytes canonical representation + check(s, size(compactfields) + 4) + # compile() will trigger the generation of the UTF-8 + # representation as a side effect + compile(s, "<stdin>", "eval") + check(s, size(compactfields) + 4 + 4) + # TODO: add check that forces the presence of wchar_t representation + # TODO: add check that forces layout of unicodefields # weakref import weakref check(weakref.ref(int), size(h + '2Pl2P')) diff --git a/Lib/test/test_sys_settrace.py b/Lib/test/test_sys_settrace.py index cf3976c..578e95d 100644 --- a/Lib/test/test_sys_settrace.py +++ b/Lib/test/test_sys_settrace.py @@ -251,6 +251,7 @@ class TraceTestCase(unittest.TestCase): def setUp(self): self.using_gc = gc.isenabled() gc.disable() + self.addCleanup(sys.settrace, sys.gettrace()) def tearDown(self): if self.using_gc: @@ -389,6 +390,9 @@ class TraceTestCase(unittest.TestCase): class RaisingTraceFuncTestCase(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + def trace(self, frame, event, arg): """A trace function that raises an exception in response to a specific trace event.""" @@ -688,6 +692,10 @@ def no_jump_without_trace_function(): class JumpTestCase(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + sys.settrace(None) + def compare_jump_output(self, expected, received): if received != expected: self.fail( "Outputs don't match:\n" + @@ -739,6 +747,8 @@ class JumpTestCase(unittest.TestCase): def test_18_no_jump_to_non_integers(self): self.run_test(no_jump_to_non_integers) def test_19_no_jump_without_trace_function(self): + # Must set sys.settrace(None) in setUp(), else condition is not + # triggered. no_jump_without_trace_function() def test_20_large_function(self): diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index aabb6fa..8058d0e 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,25 +1,23 @@ -"""Tests for sysconfig.""" - import unittest import sys import os import subprocess import shutil -from copy import copy, deepcopy +from copy import copy from test.support import (run_unittest, TESTFN, unlink, captured_stdout, skip_unless_symlink) import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, - get_path, get_path_names, _INSTALL_SCHEMES, + get_path, get_path_names, _SCHEMES, _get_default_scheme, _expand_vars, get_scheme_names, get_config_var, _main) + class TestSysConfig(unittest.TestCase): def setUp(self): - """Make a copy of sys.path""" super(TestSysConfig, self).setUp() self.sys_path = sys.path[:] # patching os.uname @@ -39,10 +37,15 @@ class TestSysConfig(unittest.TestCase): self.isabs = os.path.isabs self.splitdrive = os.path.splitdrive self._config_vars = copy(sysconfig._CONFIG_VARS) - self.old_environ = deepcopy(os.environ) + self._added_envvars = [] + self._changed_envvars = [] + for var in ('MACOSX_DEPLOYMENT_TARGET', 'PATH'): + if var in os.environ: + self._changed_envvars.append((var, os.environ[var])) + else: + self._added_envvars.append(var) def tearDown(self): - """Restore sys.path""" sys.path[:] = self.sys_path self._cleanup_testfn() if self.uname is not None: @@ -57,13 +60,10 @@ class TestSysConfig(unittest.TestCase): os.path.isabs = self.isabs os.path.splitdrive = self.splitdrive sysconfig._CONFIG_VARS = copy(self._config_vars) - for key, value in self.old_environ.items(): - if os.environ.get(key) != value: - os.environ[key] = value - - for key in list(os.environ.keys()): - if key not in self.old_environ: - del os.environ[key] + for var, value in self._changed_envvars: + os.environ[var] = value + for var in self._added_envvars: + os.environ.pop(var, None) super(TestSysConfig, self).tearDown() @@ -81,27 +81,25 @@ class TestSysConfig(unittest.TestCase): shutil.rmtree(path) def test_get_path_names(self): - self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS) + self.assertEqual(get_path_names(), _SCHEMES.options('posix_prefix')) def test_get_paths(self): scheme = get_paths() default_scheme = _get_default_scheme() wanted = _expand_vars(default_scheme, None) - wanted = list(wanted.items()) - wanted.sort() - scheme = list(scheme.items()) - scheme.sort() + wanted = sorted(wanted.items()) + scheme = sorted(scheme.items()) self.assertEqual(scheme, wanted) def test_get_path(self): - # xxx make real tests here - for scheme in _INSTALL_SCHEMES: - for name in _INSTALL_SCHEMES[scheme]: + # XXX make real tests here + for scheme in _SCHEMES: + for name in _SCHEMES[scheme]: res = get_path(name, scheme) def test_get_config_vars(self): cvars = get_config_vars() - self.assertTrue(isinstance(cvars, dict)) + self.assertIsInstance(cvars, dict) self.assertTrue(cvars) def test_get_platform(self): @@ -135,8 +133,6 @@ class TestSysConfig(unittest.TestCase): ('Darwin Kernel Version 8.11.1: ' 'Wed Oct 10 18:23:28 PDT 2007; ' 'root:xnu-792.25.20~1/RELEASE_I386'), 'PowerPC')) - - get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = '10.3' get_config_vars()['CFLAGS'] = ('-fno-strict-aliasing -DNDEBUG -g ' @@ -151,7 +147,6 @@ class TestSysConfig(unittest.TestCase): finally: sys.maxsize = maxint - self._set_uname(('Darwin', 'macziade', '8.11.1', ('Darwin Kernel Version 8.11.1: ' 'Wed Oct 10 18:23:28 PDT 2007; ' @@ -209,9 +204,9 @@ class TestSysConfig(unittest.TestCase): get_config_vars()['CFLAGS'] = ('-arch %s -isysroot ' '/Developer/SDKs/MacOSX10.4u.sdk ' '-fno-strict-aliasing -fno-common ' - '-dynamic -DNDEBUG -g -O3'%(arch,)) + '-dynamic -DNDEBUG -g -O3' % arch) - self.assertEqual(get_platform(), 'macosx-10.4-%s'%(arch,)) + self.assertEqual(get_platform(), 'macosx-10.4-%s' % arch) # linux debian sarge os.name = 'posix' @@ -239,8 +234,8 @@ class TestSysConfig(unittest.TestCase): # On Windows, the EXE needs to know where pythonXY.dll is at so we have # to add the directory to the path. if sys.platform == "win32": - os.environ["Path"] = "{};{}".format( - os.path.dirname(sys.executable), os.environ["Path"]) + os.environ["PATH"] = "{};{}".format( + os.path.dirname(sys.executable), os.environ["PATH"]) # Issue 7880 def get(python): @@ -286,7 +281,6 @@ class TestSysConfig(unittest.TestCase): self.assertIn(ldflags, ldshared) - @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") def test_platform_in_subprocess(self): my_platform = sysconfig.get_platform() @@ -312,28 +306,29 @@ class TestSysConfig(unittest.TestCase): self.assertEqual(status, 0) self.assertEqual(my_platform, test_platform) - # Test with MACOSX_DEPLOYMENT_TARGET in the environment, and # using a value that is unlikely to be the default one. env = os.environ.copy() env['MACOSX_DEPLOYMENT_TARGET'] = '10.1' - p = subprocess.Popen([ - sys.executable, '-c', - 'import sysconfig; print(sysconfig.get_platform())', - ], - stdout=subprocess.PIPE, - stderr=open('/dev/null'), - env=env) - test_platform = p.communicate()[0].strip() - test_platform = test_platform.decode('utf-8') - status = p.wait() + with open('/dev/null') as dev_null: + p = subprocess.Popen([ + sys.executable, '-c', + 'import sysconfig; print(sysconfig.get_platform())', + ], + stdout=subprocess.PIPE, + stderr=dev_null, + env=env) + test_platform = p.communicate()[0].strip() + test_platform = test_platform.decode('utf-8') + status = p.wait() - self.assertEqual(status, 0) - self.assertEqual(my_platform, test_platform) + self.assertEqual(status, 0) + self.assertEqual(my_platform, test_platform) class MakefileTests(unittest.TestCase): + @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') def test_get_makefile_filename(self): diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 37cfbba..d346113 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -82,7 +82,7 @@ class UstarReadTest(ReadTest): def test_fileobj_iter(self): self.tar.extract("ustar/regtype", TEMPDIR) tarinfo = self.tar.getmember("ustar/regtype") - with open(os.path.join(TEMPDIR, "ustar/regtype"), "rU") as fobj1: + with open(os.path.join(TEMPDIR, "ustar/regtype"), "r") as fobj1: lines1 = fobj1.readlines() fobj2 = self.tar.extractfile(tarinfo) try: @@ -703,7 +703,7 @@ class GNUReadTest(LongnameTest): # Return True if the platform knows the st_blocks stat attribute and # uses st_blocks units of 512 bytes, and if the filesystem is able to # store holes in files. - if sys.platform == "linux2": + if sys.platform.startswith("linux"): # Linux evidentially has 512 byte st_blocks units. name = os.path.join(TEMPDIR, "sparse-test") with open(name, "wb") as fobj: @@ -893,7 +893,7 @@ class WriteTest(WriteTestBase): try: for name in ("foo", "bar", "baz"): name = os.path.join(tempdir, name) - open(name, "wb").close() + support.create_empty_file(name) exclude = os.path.isfile @@ -920,7 +920,7 @@ class WriteTest(WriteTestBase): try: for name in ("foo", "bar", "baz"): name = os.path.join(tempdir, name) - open(name, "wb").close() + support.create_empty_file(name) def filter(tarinfo): if os.path.basename(tarinfo.name) == "bar": @@ -959,7 +959,7 @@ class WriteTest(WriteTestBase): # and compare the stored name with the original. foo = os.path.join(TEMPDIR, "foo") if not dir: - open(foo, "w").close() + support.create_empty_file(foo) else: os.mkdir(foo) @@ -1319,7 +1319,7 @@ class UstarUnicodeTest(unittest.TestCase): self._test_unicode_filename("utf7") def test_utf8_filename(self): - self._test_unicode_filename("utf8") + self._test_unicode_filename("utf-8") def _test_unicode_filename(self, encoding): tar = tarfile.open(tmpname, "w", format=self.format, encoding=encoding, errors="strict") @@ -1398,7 +1398,7 @@ class GNUUnicodeTest(UstarUnicodeTest): def test_bad_pax_header(self): # Test for issue #8633. GNU tar <= 1.23 creates raw binary fields # without a hdrcharset=BINARY header. - for encoding, name in (("utf8", "pax/bad-pax-\udce4\udcf6\udcfc"), + for encoding, name in (("utf-8", "pax/bad-pax-\udce4\udcf6\udcfc"), ("iso8859-1", "pax/bad-pax-\xe4\xf6\xfc"),): with tarfile.open(tarname, encoding=encoding, errors="surrogateescape") as tar: try: @@ -1413,7 +1413,7 @@ class PAXUnicodeTest(UstarUnicodeTest): def test_binary_header(self): # Test a POSIX.1-2008 compatible header with a hdrcharset=BINARY field. - for encoding, name in (("utf8", "pax/hdrcharset-\udce4\udcf6\udcfc"), + for encoding, name in (("utf-8", "pax/hdrcharset-\udce4\udcf6\udcfc"), ("iso8859-1", "pax/hdrcharset-\xe4\xf6\xfc"),): with tarfile.open(tarname, encoding=encoding, errors="surrogateescape") as tar: try: diff --git a/Lib/test/test_telnetlib.py b/Lib/test/test_telnetlib.py index 87418f5..e58c10f 100644 --- a/Lib/test/test_telnetlib.py +++ b/Lib/test/test_telnetlib.py @@ -35,6 +35,7 @@ class GeneralTests(TestCase): def tearDown(self): self.thread.join() + del self.thread # Clear out any dangling Thread objects. def testBasic(self): # connects diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index f7f5bda..014fca0 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -20,7 +20,7 @@ has_spawnl = hasattr(os, 'spawnl') # TEST_FILES may need to be tweaked for systems depending on the maximum # number of files that can be opened at one time (see ulimit -n) -if sys.platform in ('openbsd3', 'openbsd4'): +if sys.platform.startswith('openbsd'): TEST_FILES = 48 else: TEST_FILES = 100 diff --git a/Lib/test/test_threaded_import.py b/Lib/test/test_threaded_import.py index 7791935..789920b 100644 --- a/Lib/test/test_threaded_import.py +++ b/Lib/test/test_threaded_import.py @@ -11,8 +11,8 @@ import sys import time import shutil import unittest -from test.support import verbose, import_module, run_unittest, TESTFN -thread = import_module('_thread') +from test.support import ( + verbose, import_module, run_unittest, TESTFN, reap_threads) threading = import_module('threading') def task(N, done, done_tasks, errors): @@ -30,7 +30,7 @@ def task(N, done, done_tasks, errors): except Exception as e: errors.append(e.with_traceback(None)) finally: - done_tasks.append(thread.get_ident()) + done_tasks.append(threading.get_ident()) finished = len(done_tasks) == N if finished: done.set() @@ -62,7 +62,7 @@ class Finder: def __init__(self): self.numcalls = 0 self.x = 0 - self.lock = thread.allocate_lock() + self.lock = threading.Lock() def find_module(self, name, path=None): # Simulate some thread-unsafe behaviour. If calls to find_module() @@ -113,7 +113,9 @@ class ThreadedImportTests(unittest.TestCase): done_tasks = [] done.clear() for i in range(N): - thread.start_new_thread(task, (N, done, done_tasks, errors,)) + t = threading.Thread(target=task, + args=(N, done, done_tasks, errors,)) + t.start() done.wait(60) self.assertFalse(errors) if verbose: @@ -203,6 +205,7 @@ class ThreadedImportTests(unittest.TestCase): self.assertEqual(set(results), {'a', 'b'}) +@reap_threads def test_main(): run_unittest(ThreadedImportTests) diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index ef15d10..da90e17 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -173,7 +173,7 @@ class ThreadTests(BaseTestCase): exception = ctypes.py_object(AsyncExc) # First check it works when setting the exception from the same thread. - tid = _thread.get_ident() + tid = threading.get_ident() try: result = set_async_exc(ctypes.c_long(tid), exception) @@ -202,7 +202,7 @@ class ThreadTests(BaseTestCase): class Worker(threading.Thread): def run(self): - self.id = _thread.get_ident() + self.id = threading.get_ident() self.finished = False try: @@ -407,6 +407,14 @@ class ThreadTests(BaseTestCase): t.daemon = True self.assertTrue('daemon' in repr(t)) + def test_deamon_param(self): + t = threading.Thread() + self.assertFalse(t.daemon) + t = threading.Thread(daemon=False) + self.assertFalse(t.daemon) + t = threading.Thread(daemon=True) + self.assertTrue(t.daemon) + class ThreadJoinOnShutdown(BaseTestCase): @@ -688,6 +696,10 @@ class ThreadingExceptionTests(BaseTestCase): thread.start() self.assertRaises(RuntimeError, setattr, thread, "daemon", True) + def test_releasing_unacquired_lock(self): + lock = threading.Lock() + self.assertRaises(RuntimeError, lock.release) + @unittest.skipUnless(sys.platform == 'darwin', 'test macosx problem') def test_recursion_limit(self): # Issue 9670 @@ -748,6 +760,7 @@ class BoundedSemaphoreTests(lock_tests.BoundedSemaphoreTests): class BarrierTests(lock_tests.BarrierTests): barriertype = staticmethod(threading.Barrier) + def test_main(): test.support.run_unittest(LockTests, PyRLockTests, CRLockTests, EventTests, ConditionAsRLockTests, ConditionTests, @@ -755,7 +768,7 @@ def test_main(): ThreadTests, ThreadJoinOnShutdown, ThreadingExceptionTests, - BarrierTests + BarrierTests, ) if __name__ == "__main__": diff --git a/Lib/test/test_threadsignals.py b/Lib/test/test_threadsignals.py index e0af31d..f975a75 100644 --- a/Lib/test/test_threadsignals.py +++ b/Lib/test/test_threadsignals.py @@ -14,10 +14,8 @@ if sys.platform[:3] in ('win', 'os2') or sys.platform=='riscos': process_pid = os.getpid() signalled_all=thread.allocate_lock() -# Issue #11223: Locks are implemented using a mutex and a condition variable of -# the pthread library on FreeBSD6. POSIX condition variables cannot be -# interrupted by signals (see pthread_cond_wait manual page). -USING_PTHREAD_COND = (sys.platform == 'freebsd6') +USING_PTHREAD_COND = (sys.thread_info.name == 'pthread' + and sys.thread_info.lock == 'mutex+cond') def registerSignals(for_usr1, for_usr2, for_alrm): usr1 = signal.signal(signal.SIGUSR1, for_usr1) diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 8499af8..f8db499 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -4,7 +4,13 @@ import unittest import locale import sysconfig import sys -import warnings +import platform + +# Max year is only limited by the size of C int. +SIZEOF_INT = sysconfig.get_config_var('SIZEOF_INT') or 4 +TIME_MAXYEAR = (1 << 8 * SIZEOF_INT - 1) - 1 +TIME_MINYEAR = -TIME_MAXYEAR - 1 + class TimeTestCase(unittest.TestCase): @@ -20,6 +26,27 @@ class TimeTestCase(unittest.TestCase): def test_clock(self): time.clock() + @unittest.skipUnless(hasattr(time, 'clock_gettime'), + 'need time.clock_gettime()') + def test_clock_realtime(self): + time.clock_gettime(time.CLOCK_REALTIME) + + @unittest.skipUnless(hasattr(time, 'clock_gettime'), + 'need time.clock_gettime()') + @unittest.skipUnless(hasattr(time, 'CLOCK_MONOTONIC'), + 'need time.CLOCK_MONOTONIC') + def test_clock_monotonic(self): + a = time.clock_gettime(time.CLOCK_MONOTONIC) + b = time.clock_gettime(time.CLOCK_MONOTONIC) + self.assertLessEqual(a, b) + + @unittest.skipUnless(hasattr(time, 'clock_getres'), + 'need time.clock_getres()') + def test_clock_getres(self): + res = time.clock_getres(time.CLOCK_REALTIME) + self.assertGreater(res, 0.0) + self.assertLessEqual(res, 1.0) + def test_conversions(self): self.assertEqual(time.ctime(self.t), time.asctime(time.localtime(self.t))) @@ -27,6 +54,8 @@ class TimeTestCase(unittest.TestCase): int(self.t)) def test_sleep(self): + self.assertRaises(ValueError, time.sleep, -2) + self.assertRaises(ValueError, time.sleep, -1) time.sleep(1.2) def test_strftime(self): @@ -47,28 +76,34 @@ class TimeTestCase(unittest.TestCase): with self.assertRaises(ValueError): time.strftime('%f') - def _bounds_checking(self, func=time.strftime): + def _bounds_checking(self, func): # Make sure that strftime() checks the bounds of the various parts - #of the time tuple (0 is valid for *all* values). + # of the time tuple (0 is valid for *all* values). # The year field is tested by other test cases above # Check month [1, 12] + zero support + func((1900, 0, 1, 0, 0, 0, 0, 1, -1)) + func((1900, 12, 1, 0, 0, 0, 0, 1, -1)) self.assertRaises(ValueError, func, (1900, -1, 1, 0, 0, 0, 0, 1, -1)) self.assertRaises(ValueError, func, (1900, 13, 1, 0, 0, 0, 0, 1, -1)) # Check day of month [1, 31] + zero support + func((1900, 1, 0, 0, 0, 0, 0, 1, -1)) + func((1900, 1, 31, 0, 0, 0, 0, 1, -1)) self.assertRaises(ValueError, func, (1900, 1, -1, 0, 0, 0, 0, 1, -1)) self.assertRaises(ValueError, func, (1900, 1, 32, 0, 0, 0, 0, 1, -1)) # Check hour [0, 23] + func((1900, 1, 1, 23, 0, 0, 0, 1, -1)) self.assertRaises(ValueError, func, (1900, 1, 1, -1, 0, 0, 0, 1, -1)) self.assertRaises(ValueError, func, (1900, 1, 1, 24, 0, 0, 0, 1, -1)) # Check minute [0, 59] + func((1900, 1, 1, 0, 59, 0, 0, 1, -1)) self.assertRaises(ValueError, func, (1900, 1, 1, 0, -1, 0, 0, 1, -1)) self.assertRaises(ValueError, func, @@ -78,15 +113,21 @@ class TimeTestCase(unittest.TestCase): (1900, 1, 1, 0, 0, -1, 0, 1, -1)) # C99 only requires allowing for one leap second, but Python's docs say # allow two leap seconds (0..61) + func((1900, 1, 1, 0, 0, 60, 0, 1, -1)) + func((1900, 1, 1, 0, 0, 61, 0, 1, -1)) self.assertRaises(ValueError, func, (1900, 1, 1, 0, 0, 62, 0, 1, -1)) # No check for upper-bound day of week; # value forced into range by a ``% 7`` calculation. # Start check at -2 since gettmarg() increments value before taking # modulo. + self.assertEqual(func((1900, 1, 1, 0, 0, 0, -1, 1, -1)), + func((1900, 1, 1, 0, 0, 0, +6, 1, -1))) self.assertRaises(ValueError, func, (1900, 1, 1, 0, 0, 0, -2, 1, -1)) # Check day of the year [1, 366] + zero support + func((1900, 1, 1, 0, 0, 0, 0, 0, -1)) + func((1900, 1, 1, 0, 0, 0, 0, 366, -1)) self.assertRaises(ValueError, func, (1900, 1, 1, 0, 0, 0, 0, -1, -1)) self.assertRaises(ValueError, func, @@ -96,12 +137,13 @@ class TimeTestCase(unittest.TestCase): self._bounds_checking(lambda tup: time.strftime('', tup)) def test_default_values_for_zero(self): - # Make sure that using all zeros uses the proper default values. - # No test for daylight savings since strftime() does not change output - # based on its value. + # Make sure that using all zeros uses the proper default + # values. No test for daylight savings since strftime() does + # not change output based on its value and no test for year + # because systems vary in their support for year 0. expected = "2000 01 01 00 00 00 1 001" with support.check_warnings(): - result = time.strftime("%Y %m %d %H %M %S %w %j", (0,)*9) + result = time.strftime("%Y %m %d %H %M %S %w %j", (2000,)+(0,)*8) self.assertEqual(expected, result) def test_strptime(self): @@ -128,11 +170,13 @@ class TimeTestCase(unittest.TestCase): time.asctime(time.gmtime(self.t)) # Max year is only limited by the size of C int. - sizeof_int = sysconfig.get_config_var('SIZEOF_INT') or 4 - bigyear = (1 << 8 * sizeof_int - 1) - 1 - asc = time.asctime((bigyear, 6, 1) + (0,)*6) - self.assertEqual(asc[-len(str(bigyear)):], str(bigyear)) - self.assertRaises(OverflowError, time.asctime, (bigyear + 1,) + (0,)*8) + for bigyear in TIME_MAXYEAR, TIME_MINYEAR: + asc = time.asctime((bigyear, 6, 1) + (0,) * 6) + self.assertEqual(asc[-len(str(bigyear)):], str(bigyear)) + self.assertRaises(OverflowError, time.asctime, + (TIME_MAXYEAR + 1,) + (0,) * 8) + self.assertRaises(OverflowError, time.asctime, + (TIME_MINYEAR - 1,) + (0,) * 8) self.assertRaises(TypeError, time.asctime, 0) self.assertRaises(TypeError, time.asctime, ()) self.assertRaises(TypeError, time.asctime, (0,) * 10) @@ -155,8 +199,8 @@ class TimeTestCase(unittest.TestCase): else: self.assertEqual(time.ctime(testval)[20:], str(year)) - @unittest.skipIf(not hasattr(time, "tzset"), - "time module has no attribute tzset") + @unittest.skipUnless(hasattr(time, "tzset"), + "time module has no attribute tzset") def test_tzset(self): from os import environ @@ -253,6 +297,34 @@ class TimeTestCase(unittest.TestCase): t1 = time.mktime(lt1) self.assertAlmostEqual(t1, t0, delta=0.2) + def test_mktime(self): + # Issue #1726687 + for t in (-2, -1, 0, 1): + try: + tt = time.localtime(t) + except (OverflowError, ValueError): + pass + else: + self.assertEqual(time.mktime(tt), t) + + # Issue #13309: passing extreme values to mktime() or localtime() + # borks the glibc's internal timezone data. + @unittest.skipUnless(platform.libc_ver()[0] != 'glibc', + "disabled because of a bug in glibc. Issue #13309") + def test_mktime_error(self): + # It may not be possible to reliably make mktime return error + # on all platfom. This will make sure that no other exception + # than OverflowError is raised for an extreme value. + tt = time.gmtime(self.t) + tzname = time.strftime('%Z', tt) + self.assertNotEqual(tzname, 'LMT') + try: + time.mktime((-1, 1, 1, 0, 0, 0, -1, -1, -1)) + except OverflowError: + pass + self.assertEqual(time.strftime('%Z', tt), tzname) + + class TestLocale(unittest.TestCase): def setUp(self): self.oldloc = locale.setlocale(locale.LC_ALL) @@ -271,19 +343,12 @@ class TestLocale(unittest.TestCase): class _BaseYearTest(unittest.TestCase): - accept2dyear = None - - def setUp(self): - self.saved_accept2dyear = time.accept2dyear - time.accept2dyear = self.accept2dyear - - def tearDown(self): - time.accept2dyear = self.saved_accept2dyear - def yearstr(self, y): raise NotImplementedError() class _TestAsctimeYear: + _format = '%d' + def yearstr(self, y): return time.asctime((y,) + (0,) * 8).split()[-1] @@ -293,114 +358,97 @@ class _TestAsctimeYear: self.assertEqual(self.yearstr(123456789), '123456789') class _TestStrftimeYear: - def yearstr(self, y): - return time.strftime('%Y', (y,) + (0,) * 8).split()[-1] - def test_large_year(self): + # Issue 13305: For years < 1000, the value is not always + # padded to 4 digits across platforms. The C standard + # assumes year >= 1900, so it does not specify the number + # of digits. + + if time.strftime('%Y', (1,) + (0,) * 8) == '0001': + _format = '%04d' + else: + _format = '%d' + + def yearstr(self, y): + return time.strftime('%Y', (y,) + (0,) * 8) + + def test_4dyear(self): + # Check that we can return the zero padded value. + if self._format == '%04d': + self.test_year('%04d') + else: + def year4d(y): + return time.strftime('%4Y', (y,) + (0,) * 8) + self.test_year('%04d', func=year4d) + + def skip_if_not_supported(y): + msg = "strftime() is limited to [1; 9999] with Visual Studio" # Check that it doesn't crash for year > 9999 try: - text = self.yearstr(12345) + time.strftime('%Y', (y,) + (0,) * 8) except ValueError: - # strftime() is limited to [1; 9999] with Visual Studio - return - self.assertEqual(text, '12345') - self.assertEqual(self.yearstr(123456789), '123456789') + cond = False + else: + cond = True + return unittest.skipUnless(cond, msg) -class _Test2dYear(_BaseYearTest): - accept2dyear = 1 + @skip_if_not_supported(10000) + def test_large_year(self): + return super().test_large_year() - def test_year(self): - with support.check_warnings(): - self.assertEqual(self.yearstr(0), '2000') - self.assertEqual(self.yearstr(69), '1969') - self.assertEqual(self.yearstr(68), '2068') - self.assertEqual(self.yearstr(99), '1999') + @skip_if_not_supported(0) + def test_negative(self): + return super().test_negative() + + del skip_if_not_supported - def test_invalid(self): - self.assertRaises(ValueError, self.yearstr, -1) - self.assertRaises(ValueError, self.yearstr, 100) - self.assertRaises(ValueError, self.yearstr, 999) class _Test4dYear(_BaseYearTest): - accept2dyear = 0 + _format = '%d' + + def test_year(self, fmt=None, func=None): + fmt = fmt or self._format + func = func or self.yearstr + self.assertEqual(func(1), fmt % 1) + self.assertEqual(func(68), fmt % 68) + self.assertEqual(func(69), fmt % 69) + self.assertEqual(func(99), fmt % 99) + self.assertEqual(func(999), fmt % 999) + self.assertEqual(func(9999), fmt % 9999) - def test_year(self): - self.assertIn(self.yearstr(1), ('1', '0001')) - self.assertIn(self.yearstr(68), ('68', '0068')) - self.assertIn(self.yearstr(69), ('69', '0069')) - self.assertIn(self.yearstr(99), ('99', '0099')) - self.assertIn(self.yearstr(999), ('999', '0999')) - self.assertEqual(self.yearstr(9999), '9999') + def test_large_year(self): + self.assertEqual(self.yearstr(12345), '12345') + self.assertEqual(self.yearstr(123456789), '123456789') + self.assertEqual(self.yearstr(TIME_MAXYEAR), str(TIME_MAXYEAR)) + self.assertRaises(OverflowError, self.yearstr, TIME_MAXYEAR + 1) def test_negative(self): - try: - text = self.yearstr(-1) - except ValueError: - # strftime() is limited to [1; 9999] with Visual Studio - return - self.assertIn(text, ('-1', '-001')) - + self.assertEqual(self.yearstr(-1), self._format % -1) self.assertEqual(self.yearstr(-1234), '-1234') self.assertEqual(self.yearstr(-123456), '-123456') + self.assertEqual(self.yearstr(-123456789), str(-123456789)) + self.assertEqual(self.yearstr(-1234567890), str(-1234567890)) + self.assertEqual(self.yearstr(TIME_MINYEAR + 1900), str(TIME_MINYEAR + 1900)) + # Issue #13312: it may return wrong value for year < TIME_MINYEAR + 1900 + # Skip the value test, but check that no error is raised + self.yearstr(TIME_MINYEAR) + # self.assertEqual(self.yearstr(TIME_MINYEAR), str(TIME_MINYEAR)) + self.assertRaises(OverflowError, self.yearstr, TIME_MINYEAR - 1) - def test_mktime(self): - # Issue #1726687 - for t in (-2, -1, 0, 1): - try: - tt = time.localtime(t) - except (OverflowError, ValueError): - pass - else: - self.assertEqual(time.mktime(tt), t) - # It may not be possible to reliably make mktime return error - # on all platfom. This will make sure that no other exception - # than OverflowError is raised for an extreme value. - try: - time.mktime((-1, 1, 1, 0, 0, 0, -1, -1, -1)) - except OverflowError: - pass - -class TestAsctimeAccept2dYear(_TestAsctimeYear, _Test2dYear): - pass - -class TestStrftimeAccept2dYear(_TestStrftimeYear, _Test2dYear): - pass - class TestAsctime4dyear(_TestAsctimeYear, _Test4dYear): pass class TestStrftime4dyear(_TestStrftimeYear, _Test4dYear): pass -class Test2dyearBool(_TestAsctimeYear, _Test2dYear): - accept2dyear = True - -class Test4dyearBool(_TestAsctimeYear, _Test4dYear): - accept2dyear = False - -class TestAccept2YearBad(_TestAsctimeYear, _BaseYearTest): - class X: - def __bool__(self): - raise RuntimeError('boo') - accept2dyear = X() - def test_2dyear(self): - pass - def test_invalid(self): - self.assertRaises(RuntimeError, self.yearstr, 200) - def test_main(): support.run_unittest( TimeTestCase, TestLocale, - TestAsctimeAccept2dYear, - TestStrftimeAccept2dYear, TestAsctime4dyear, - TestStrftime4dyear, - Test2dyearBool, - Test4dyearBool, - TestAccept2YearBad) + TestStrftime4dyear) if __name__ == "__main__": test_main() diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py index 9e9656c..af2bbf1 100644 --- a/Lib/test/test_tokenize.py +++ b/Lib/test/test_tokenize.py @@ -600,7 +600,7 @@ def roundtrip(f): f.close() tokens1 = [tok[:2] for tok in token_list] new_bytes = untokenize(tokens1) - readline = (line for line in new_bytes.splitlines(1)).__next__ + readline = (line for line in new_bytes.splitlines(keepends=True)).__next__ tokens2 = [tok[:2] for tok in tokenize(readline)] return tokens1 == tokens2 diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 461d1d8..d9bef38 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -102,6 +102,7 @@ class TracedClass(object): class TestLineCounts(unittest.TestCase): """White-box testing of line-counting, via runfunc""" def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) self.my_py_filename = fix_ext_py(__file__) @@ -192,6 +193,7 @@ class TestRunExecCounts(unittest.TestCase): """A simple sanity test of line-counting, via runctx (exec)""" def setUp(self): self.my_py_filename = fix_ext_py(__file__) + self.addCleanup(sys.settrace, sys.gettrace()) def test_exec_counts(self): self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) @@ -218,6 +220,7 @@ class TestRunExecCounts(unittest.TestCase): class TestFuncs(unittest.TestCase): """White-box testing of funcs tracing""" def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) self.tracer = Trace(count=0, trace=0, countfuncs=1) self.filemod = my_file_and_modname() @@ -242,6 +245,8 @@ class TestFuncs(unittest.TestCase): } self.assertEqual(self.tracer.results().calledfuncs, expected) + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'pre-existing trace function throws off measurements') def test_inst_method_calling(self): obj = TracedClass(20) self.tracer.runfunc(obj.inst_method_calling, 1) @@ -257,9 +262,12 @@ class TestFuncs(unittest.TestCase): class TestCallers(unittest.TestCase): """White-box testing of callers tracing""" def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) self.tracer = Trace(count=0, trace=0, countcallers=1) self.filemod = my_file_and_modname() + @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), + 'pre-existing trace function throws off measurements') def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) @@ -280,6 +288,9 @@ class TestCallers(unittest.TestCase): # Created separately for issue #3821 class TestCoverage(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + def tearDown(self): rmtree(TESTFN) unlink(TESTFN) diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index fd620f0..68a3219 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -8,8 +8,11 @@ Modified for Python 2.0 by Fredrik Lundh (fredrik@pythonware.com) """#" import unittest +import unicodedata from test import support +from http.client import HTTPException +from test.test_normalization import check_version class UnicodeNamesTest(unittest.TestCase): @@ -59,8 +62,6 @@ class UnicodeNamesTest(unittest.TestCase): ) def test_ascii_letters(self): - import unicodedata - for char in "".join(map(chr, range(ord("a"), ord("z")))): name = "LATIN SMALL LETTER %s" % char.upper() code = unicodedata.lookup(name) @@ -81,7 +82,6 @@ class UnicodeNamesTest(unittest.TestCase): self.checkletter("HANGUL SYLLABLE HWEOK", "\ud6f8") self.checkletter("HANGUL SYLLABLE HIH", "\ud7a3") - import unicodedata self.assertRaises(ValueError, unicodedata.name, "\ud7a4") def test_cjk_unified_ideographs(self): @@ -97,14 +97,11 @@ class UnicodeNamesTest(unittest.TestCase): self.checkletter("CJK UNIFIED IDEOGRAPH-2B81D", "\U0002B81D") def test_bmp_characters(self): - import unicodedata - count = 0 for code in range(0x10000): char = chr(code) name = unicodedata.name(char, None) if name is not None: self.assertEqual(unicodedata.lookup(name), char) - count += 1 def test_misc_symbols(self): self.checkletter("PILCROW SIGN", "\u00b6") @@ -112,8 +109,85 @@ class UnicodeNamesTest(unittest.TestCase): self.checkletter("HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK", "\uFF9F") self.checkletter("FULLWIDTH LATIN SMALL LETTER A", "\uFF41") + def test_aliases(self): + # Check that the aliases defined in the NameAliases.txt file work. + # This should be updated when new aliases are added or the file + # should be downloaded and parsed instead. See #12753. + aliases = [ + ('LATIN CAPITAL LETTER GHA', 0x01A2), + ('LATIN SMALL LETTER GHA', 0x01A3), + ('KANNADA LETTER LLLA', 0x0CDE), + ('LAO LETTER FO FON', 0x0E9D), + ('LAO LETTER FO FAY', 0x0E9F), + ('LAO LETTER RO', 0x0EA3), + ('LAO LETTER LO', 0x0EA5), + ('TIBETAN MARK BKA- SHOG GI MGO RGYAN', 0x0FD0), + ('YI SYLLABLE ITERATION MARK', 0xA015), + ('PRESENTATION FORM FOR VERTICAL RIGHT WHITE LENTICULAR BRACKET', 0xFE18), + ('BYZANTINE MUSICAL SYMBOL FTHORA SKLIRON CHROMA VASIS', 0x1D0C5) + ] + for alias, codepoint in aliases: + self.checkletter(alias, chr(codepoint)) + name = unicodedata.name(chr(codepoint)) + self.assertNotEqual(name, alias) + self.assertEqual(unicodedata.lookup(alias), + unicodedata.lookup(name)) + with self.assertRaises(KeyError): + unicodedata.ucd_3_2_0.lookup(alias) + + def test_aliases_names_in_pua_range(self): + # We are storing aliases in the PUA 15, but their names shouldn't leak + for cp in range(0xf0000, 0xf0100): + with self.assertRaises(ValueError) as cm: + unicodedata.name(chr(cp)) + self.assertEqual(str(cm.exception), 'no such name') + + def test_named_sequences_names_in_pua_range(self): + # We are storing named seq in the PUA 15, but their names shouldn't leak + for cp in range(0xf0100, 0xf0fff): + with self.assertRaises(ValueError) as cm: + unicodedata.name(chr(cp)) + self.assertEqual(str(cm.exception), 'no such name') + + def test_named_sequences_sample(self): + # Check a few named sequences. See #12753. + sequences = [ + ('LATIN SMALL LETTER R WITH TILDE', '\u0072\u0303'), + ('TAMIL SYLLABLE SAI', '\u0BB8\u0BC8'), + ('TAMIL SYLLABLE MOO', '\u0BAE\u0BCB'), + ('TAMIL SYLLABLE NNOO', '\u0BA3\u0BCB'), + ('TAMIL CONSONANT KSS', '\u0B95\u0BCD\u0BB7\u0BCD'), + ] + for seqname, codepoints in sequences: + self.assertEqual(unicodedata.lookup(seqname), codepoints) + with self.assertRaises(SyntaxError): + self.checkletter(seqname, None) + with self.assertRaises(KeyError): + unicodedata.ucd_3_2_0.lookup(seqname) + + def test_named_sequences_full(self): + # Check all the named sequences + url = ("http://www.unicode.org/Public/%s/ucd/NamedSequences.txt" % + unicodedata.unidata_version) + try: + testdata = support.open_urlresource(url, encoding="utf-8", + check=check_version) + except (IOError, HTTPException): + self.skipTest("Could not retrieve " + url) + self.addCleanup(testdata.close) + for line in testdata: + line = line.strip() + if not line or line.startswith('#'): + continue + seqname, codepoints = line.split(';') + codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split()) + self.assertEqual(unicodedata.lookup(seqname), codepoints) + with self.assertRaises(SyntaxError): + self.checkletter(seqname, None) + with self.assertRaises(KeyError): + unicodedata.ucd_3_2_0.lookup(seqname) + def test_errors(self): - import unicodedata self.assertRaises(TypeError, unicodedata.name) self.assertRaises(TypeError, unicodedata.name, 'xx') self.assertRaises(TypeError, unicodedata.lookup) diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py index 591a297..a669f8e 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_unicode.py @@ -5,17 +5,13 @@ Written by Marc-Andre Lemburg (mal@lemburg.com). (c) Copyright CNRI, All Rights Reserved. NO WARRANTY. """#" +import _string import codecs import struct import sys import unittest import warnings from test import support, string_tests -import _string - -# decorator to skip tests on narrow builds -requires_wide_build = unittest.skipIf(sys.maxunicode == 65535, - 'requires wide build') # Error handling (bad decoder return) def search_function(encoding): @@ -175,6 +171,15 @@ class UnicodeTest(string_tests.CommonTest, def test_find(self): string_tests.CommonTest.test_find(self) + # test implementation details of the memchr fast path + self.checkequal(100, 'a' * 100 + '\u0102', 'find', '\u0102') + self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0201') + self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0120') + self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0220') + self.checkequal(100, 'a' * 100 + '\U00100304', 'find', '\U00100304') + self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00100204') + self.checkequal(-1, 'a' * 100 + '\U00100304', 'find', '\U00102004') + # check mixed argument types self.checkequalnofix(0, 'abcdefghiabc', 'find', 'abc') self.checkequalnofix(9, 'abcdefghiabc', 'find', 'abc', 1) self.checkequalnofix(-1, 'abcdefghiabc', 'find', 'def', 4) @@ -184,6 +189,14 @@ class UnicodeTest(string_tests.CommonTest, def test_rfind(self): string_tests.CommonTest.test_rfind(self) + # test implementation details of the memrchr fast path + self.checkequal(0, '\u0102' + 'a' * 100 , 'rfind', '\u0102') + self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0201') + self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0120') + self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0220') + self.checkequal(0, '\U00100304' + 'a' * 100, 'rfind', '\U00100304') + self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00100204') + self.checkequal(-1, '\U00100304' + 'a' * 100, 'rfind', '\U00102004') # check mixed argument types self.checkequalnofix(9, 'abcdefghiabc', 'rfind', 'abc') self.checkequalnofix(12, 'abcdefghiabc', 'rfind', '') @@ -280,6 +293,12 @@ class UnicodeTest(string_tests.CommonTest, self.checkequalnofix('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1) self.assertRaises(TypeError, 'replace'.replace, "r", 42) + @support.cpython_only + def test_replace_id(self): + pattern = 'abc' + text = 'abc def' + self.assertIs(text.replace(pattern, pattern), text) + def test_bytes_comparison(self): with support.check_warnings(): warnings.simplefilter('ignore', BytesWarning) @@ -520,7 +539,6 @@ class UnicodeTest(string_tests.CommonTest, self.assertFalse(meth(s), '%a.%s() is False' % (s, meth_name)) - @requires_wide_build def test_lower(self): string_tests.CommonTest.test_lower(self) self.assertEqual('\U00010427'.lower(), '\U0001044F') @@ -531,7 +549,6 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('X\U00010427x\U0001044F'.lower(), 'x\U0001044Fx\U0001044F') - @requires_wide_build def test_upper(self): string_tests.CommonTest.test_upper(self) self.assertEqual('\U0001044F'.upper(), '\U00010427') @@ -542,7 +559,6 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('X\U00010427x\U0001044F'.upper(), 'X\U00010427X\U00010427') - @requires_wide_build def test_capitalize(self): string_tests.CommonTest.test_capitalize(self) self.assertEqual('\U0001044F'.capitalize(), '\U00010427') @@ -555,7 +571,6 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('X\U00010427x\U0001044F'.capitalize(), 'X\U0001044Fx\U0001044F') - @requires_wide_build def test_title(self): string_tests.MixinStrUnicodeUserStringTest.test_title(self) self.assertEqual('\U0001044F'.title(), '\U00010427') @@ -570,7 +585,6 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('X\U00010427x\U0001044F X\U00010427x\U0001044F'.title(), 'X\U0001044Fx\U0001044F X\U0001044Fx\U0001044F') - @requires_wide_build def test_swapcase(self): string_tests.CommonTest.test_swapcase(self) self.assertEqual('\U0001044F'.swapcase(), '\U00010427') @@ -776,7 +790,7 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('{0!s}'.format(G('data')), 'string is data') msg = 'object.__format__ with a non-empty format string is deprecated' - with support.check_warnings((msg, PendingDeprecationWarning)): + with support.check_warnings((msg, DeprecationWarning)): self.assertEqual('{0:^10}'.format(E('data')), ' E(data) ') self.assertEqual('{0:^10s}'.format(E('data')), ' E(data) ') self.assertEqual('{0:>15s}'.format(G('data')), ' string is data') @@ -1011,10 +1025,13 @@ class UnicodeTest(string_tests.CommonTest, class UnicodeSubclass(str): pass - self.assertEqual( - str(UnicodeSubclass('unicode subclass becomes unicode')), - 'unicode subclass becomes unicode' - ) + for text in ('ascii', '\xe9', '\u20ac', '\U0010FFFF'): + subclass = UnicodeSubclass(text) + self.assertEqual(str(subclass), text) + self.assertEqual(len(subclass), len(text)) + if text == 'ascii': + self.assertEqual(subclass.encode('ascii'), b'ascii') + self.assertEqual(subclass.encode('utf-8'), b'ascii') self.assertEqual( str('strings are converted to unicode'), @@ -1120,15 +1137,12 @@ class UnicodeTest(string_tests.CommonTest, def test_codecs_utf8(self): self.assertEqual(''.encode('utf-8'), b'') self.assertEqual('\u20ac'.encode('utf-8'), b'\xe2\x82\xac') - if sys.maxunicode == 65535: - self.assertEqual('\ud800\udc02'.encode('utf-8'), b'\xf0\x90\x80\x82') - self.assertEqual('\ud84d\udc56'.encode('utf-8'), b'\xf0\xa3\x91\x96') + self.assertEqual('\U00010002'.encode('utf-8'), b'\xf0\x90\x80\x82') + self.assertEqual('\U00023456'.encode('utf-8'), b'\xf0\xa3\x91\x96') self.assertEqual('\ud800'.encode('utf-8', 'surrogatepass'), b'\xed\xa0\x80') self.assertEqual('\udc00'.encode('utf-8', 'surrogatepass'), b'\xed\xb0\x80') - if sys.maxunicode == 65535: - self.assertEqual( - ('\ud800\udc02'*1000).encode('utf-8'), - b'\xf0\x90\x80\x82'*1000) + self.assertEqual(('\U00010002'*10).encode('utf-8'), + b'\xf0\x90\x80\x82'*10) self.assertEqual( '\u6b63\u78ba\u306b\u8a00\u3046\u3068\u7ffb\u8a33\u306f' '\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002\u4e00' @@ -1366,18 +1380,25 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual('hello'.encode('ascii'), b'hello') self.assertEqual('hello'.encode('utf-7'), b'hello') self.assertEqual('hello'.encode('utf-8'), b'hello') - self.assertEqual('hello'.encode('utf8'), b'hello') + self.assertEqual('hello'.encode('utf-8'), b'hello') self.assertEqual('hello'.encode('utf-16-le'), b'h\000e\000l\000l\000o\000') self.assertEqual('hello'.encode('utf-16-be'), b'\000h\000e\000l\000l\000o') self.assertEqual('hello'.encode('latin-1'), b'hello') + # Default encoding is utf-8 + self.assertEqual('\u2603'.encode(), b'\xe2\x98\x83') + # Roundtrip safety for BMP (just the first 1024 chars) for c in range(1024): u = chr(c) for encoding in ('utf-7', 'utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', 'raw_unicode_escape', 'unicode_escape', 'unicode_internal'): - self.assertEqual(str(u.encode(encoding),encoding), u) + with warnings.catch_warnings(): + # unicode-internal has been deprecated + warnings.simplefilter("ignore", DeprecationWarning) + + self.assertEqual(str(u.encode(encoding),encoding), u) # Roundtrip safety for BMP (just the first 256 chars) for c in range(256): @@ -1392,18 +1413,20 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual(str(u.encode(encoding),encoding), u) # Roundtrip safety for non-BMP (just a few chars) - u = '\U00010001\U00020002\U00030003\U00040004\U00050005' - for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', - #'raw_unicode_escape', - 'unicode_escape', 'unicode_internal'): - self.assertEqual(str(u.encode(encoding),encoding), u) + with warnings.catch_warnings(): + # unicode-internal has been deprecated + warnings.simplefilter("ignore", DeprecationWarning) + + u = '\U00010001\U00020002\U00030003\U00040004\U00050005' + for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be', + 'raw_unicode_escape', + 'unicode_escape', 'unicode_internal'): + self.assertEqual(str(u.encode(encoding),encoding), u) - # UTF-8 must be roundtrip safe for all UCS-2 code points - # This excludes surrogates: in the full range, there would be - # a surrogate pair (\udbff\udc00), which gets converted back - # to a non-BMP character (\U0010fc00) - u = ''.join(map(chr, list(range(0,0xd800)) + - list(range(0xe000,0x10000)))) + # UTF-8 must be roundtrip safe for all code points + # (except surrogates, which are forbidden). + u = ''.join(map(chr, list(range(0, 0xd800)) + + list(range(0xe000, 0x110000)))) for encoding in ('utf-8',): self.assertEqual(str(u.encode(encoding),encoding), u) @@ -1588,17 +1611,39 @@ class UnicodeTest(string_tests.CommonTest, return self.assertRaises(OverflowError, 't\tt\t'.expandtabs, sys.maxsize) + @support.cpython_only + def test_expandtabs_optimization(self): + s = 'abc' + self.assertIs(s.expandtabs(), s) + def test_raiseMemError(self): - # Ensure that the freelist contains a consistent object, even - # when a string allocation fails with a MemoryError. - # This used to crash the interpreter, - # or leak references when the number was smaller. - charwidth = 4 if sys.maxunicode >= 0x10000 else 2 - # Note: sys.maxsize is half of the actual max allocation because of - # the signedness of Py_ssize_t. - alloc = lambda: "a" * (sys.maxsize // charwidth * 2) - self.assertRaises(MemoryError, alloc) - self.assertRaises(MemoryError, alloc) + if struct.calcsize('P') == 8: + # 64 bits pointers + ascii_struct_size = 48 + compact_struct_size = 72 + else: + # 32 bits pointers + ascii_struct_size = 24 + compact_struct_size = 36 + + for char in ('a', '\xe9', '\u20ac', '\U0010ffff'): + code = ord(char) + if code < 0x100: + char_size = 1 # sizeof(Py_UCS1) + struct_size = ascii_struct_size + elif code < 0x10000: + char_size = 2 # sizeof(Py_UCS2) + struct_size = compact_struct_size + else: + char_size = 4 # sizeof(Py_UCS4) + struct_size = compact_struct_size + # Note: sys.maxsize is half of the actual max allocation because of + # the signedness of Py_ssize_t. Strings of maxlen-1 should in principle + # be allocatable, given enough memory. + maxlen = ((sys.maxsize - struct_size) // char_size) + alloc = lambda: char * maxlen + self.assertRaises(MemoryError, alloc) + self.assertRaises(MemoryError, alloc) def test_format_subclass(self): class S(str): @@ -1611,11 +1656,10 @@ class UnicodeTest(string_tests.CommonTest, # Test PyUnicode_FromFormat() def test_from_format(self): support.import_module('ctypes') - from ctypes import pythonapi, py_object, c_int - if sys.maxunicode == 65535: - name = "PyUnicodeUCS2_FromFormat" - else: - name = "PyUnicodeUCS4_FromFormat" + from ctypes import (pythonapi, py_object, + c_int, c_long, c_longlong, c_ssize_t, + c_uint, c_ulong, c_ulonglong, c_size_t) + name = "PyUnicode_FromFormat" _PyUnicode_FromFormat = getattr(pythonapi, name) _PyUnicode_FromFormat.restype = py_object @@ -1636,13 +1680,40 @@ class UnicodeTest(string_tests.CommonTest, 'string, got a non-ASCII byte: 0xe9$', PyUnicode_FromFormat, b'unicode\xe9=%s', 'ascii') + # test "%c" self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0xabcd)), '\uabcd') self.assertEqual(PyUnicode_FromFormat(b'%c', c_int(0x10ffff)), '\U0010ffff') - # other tests + # test "%" + self.assertEqual(PyUnicode_FromFormat(b'%'), '%') + self.assertEqual(PyUnicode_FromFormat(b'%%'), '%') + self.assertEqual(PyUnicode_FromFormat(b'%%s'), '%s') + self.assertEqual(PyUnicode_FromFormat(b'[%%]'), '[%]') + self.assertEqual(PyUnicode_FromFormat(b'%%%s', b'abc'), '%abc') + + # test integer formats (%i, %d, %u) + self.assertEqual(PyUnicode_FromFormat(b'%03i', c_int(10)), '010') + self.assertEqual(PyUnicode_FromFormat(b'%0.4i', c_int(10)), '0010') + self.assertEqual(PyUnicode_FromFormat(b'%i', c_int(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%li', c_long(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%lli', c_longlong(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%zi', c_ssize_t(-123)), '-123') + + self.assertEqual(PyUnicode_FromFormat(b'%d', c_int(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%ld', c_long(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%lld', c_longlong(-123)), '-123') + self.assertEqual(PyUnicode_FromFormat(b'%zd', c_ssize_t(-123)), '-123') + + self.assertEqual(PyUnicode_FromFormat(b'%u', c_uint(123)), '123') + self.assertEqual(PyUnicode_FromFormat(b'%lu', c_ulong(123)), '123') + self.assertEqual(PyUnicode_FromFormat(b'%llu', c_ulonglong(123)), '123') + self.assertEqual(PyUnicode_FromFormat(b'%zu', c_size_t(123)), '123') + + # test %A text = PyUnicode_FromFormat(b'%%A:%A', 'abc\xe9\uabcd\U0010ffff') self.assertEqual(text, r"%A:'abc\xe9\uabcd\U0010ffff'") + # test %V text = PyUnicode_FromFormat(b'repr=%V', 'abc', b'xyz') self.assertEqual(text, 'repr=abc') @@ -1656,6 +1727,13 @@ class UnicodeTest(string_tests.CommonTest, text = PyUnicode_FromFormat(b'repr=%V', None, b'abc\xff') self.assertEqual(text, 'repr=abc\ufffd') + # not supported: copy the raw format string. these tests are just here + # to check for crashs and should not be considered as specifications + self.assertEqual(PyUnicode_FromFormat(b'%1%s', b'abc'), '%s') + self.assertEqual(PyUnicode_FromFormat(b'%1abc'), '%1abc') + self.assertEqual(PyUnicode_FromFormat(b'%+i', c_int(10)), '%+i') + self.assertEqual(PyUnicode_FromFormat(b'%.%s', b'abc'), '%.%s') + # Test PyUnicode_AsWideChar() def test_aswidechar(self): from _testcapi import unicode_aswidechar @@ -1716,6 +1794,18 @@ class UnicodeTest(string_tests.CommonTest, self.assertEqual(size, nchar) self.assertEqual(wchar, nonbmp + '\0') + def test_subclass_add(self): + class S(str): + def __add__(self, o): + return "3" + self.assertEqual(S("4") + S("5"), "3") + class S(str): + def __iadd__(self, o): + return "3" + s = S("1") + s += "4" + self.assertEqual(s, "3") + class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): diff --git a/Lib/test/test_unicode_file.py b/Lib/test/test_unicode_file.py index 6c2011a..faa8da3 100644 --- a/Lib/test/test_unicode_file.py +++ b/Lib/test/test_unicode_file.py @@ -6,7 +6,7 @@ import unicodedata import unittest from test.support import (run_unittest, rmtree, - TESTFN_ENCODING, TESTFN_UNICODE, TESTFN_UNENCODABLE) + TESTFN_ENCODING, TESTFN_UNICODE, TESTFN_UNENCODABLE, create_empty_file) if not os.path.supports_unicode_filenames: try: @@ -56,16 +56,20 @@ class TestUnicodeFiles(unittest.TestCase): # Should be able to rename the file using either name. self.assertTrue(os.path.isfile(filename1)) # must exist. os.rename(filename1, filename2 + ".new") - self.assertTrue(os.path.isfile(filename1+".new")) + self.assertFalse(os.path.isfile(filename2)) + self.assertTrue(os.path.isfile(filename1 + '.new')) os.rename(filename1 + ".new", filename2) + self.assertFalse(os.path.isfile(filename1 + '.new')) self.assertTrue(os.path.isfile(filename2)) shutil.copy(filename1, filename2 + ".new") os.unlink(filename1 + ".new") # remove using equiv name. # And a couple of moves, one using each name. shutil.move(filename1, filename2 + ".new") - self.assertTrue(not os.path.exists(filename2)) + self.assertFalse(os.path.exists(filename2)) + self.assertTrue(os.path.exists(filename1 + '.new')) shutil.move(filename1 + ".new", filename2) + self.assertFalse(os.path.exists(filename2 + '.new')) self.assertTrue(os.path.exists(filename1)) # Note - due to the implementation of shutil.move, # it tries a rename first. This only fails on Windows when on @@ -73,10 +77,12 @@ class TestUnicodeFiles(unittest.TestCase): # So we test the shutil.copy2 function, which is the thing most # likely to fail. shutil.copy2(filename1, filename2 + ".new") + self.assertTrue(os.path.isfile(filename1 + '.new')) os.unlink(filename1 + ".new") + self.assertFalse(os.path.exists(filename2 + '.new')) def _do_directory(self, make_name, chdir_name): - cwd = os.getcwdb() + cwd = os.getcwd() if os.path.isdir(make_name): rmtree(make_name) os.mkdir(make_name) @@ -99,8 +105,7 @@ class TestUnicodeFiles(unittest.TestCase): # top-level 'test' functions would be if they could take params def _test_single(self, filename): remove_if_exists(filename) - f = open(filename, "w") - f.close() + create_empty_file(filename) try: self._do_single(filename) finally: diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 9744256..7715376 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -108,6 +108,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): self.assertEqual(self.db.digit('\u215b', None), None) self.assertEqual(self.db.digit('\u2468'), 9) self.assertEqual(self.db.digit('\U00020000', None), None) + self.assertEqual(self.db.digit('\U0001D7FD'), 7) self.assertRaises(TypeError, self.db.digit) self.assertRaises(TypeError, self.db.digit, 'xx') @@ -120,6 +121,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): self.assertEqual(self.db.numeric('\u2468'), 9.0) self.assertEqual(self.db.numeric('\ua627'), 7.0) self.assertEqual(self.db.numeric('\U00020000', None), None) + self.assertEqual(self.db.numeric('\U0001012A'), 9000) self.assertRaises(TypeError, self.db.numeric) self.assertRaises(TypeError, self.db.numeric, 'xx') @@ -131,6 +133,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): self.assertEqual(self.db.decimal('\u215b', None), None) self.assertEqual(self.db.decimal('\u2468', None), None) self.assertEqual(self.db.decimal('\U00020000', None), None) + self.assertEqual(self.db.decimal('\U0001D7FD'), 7) self.assertRaises(TypeError, self.db.decimal) self.assertRaises(TypeError, self.db.decimal, 'xx') @@ -141,6 +144,7 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): self.assertEqual(self.db.category('a'), 'Ll') self.assertEqual(self.db.category('A'), 'Lu') self.assertEqual(self.db.category('\U00020000'), 'Lo') + self.assertEqual(self.db.category('\U0001012A'), 'No') self.assertRaises(TypeError, self.db.category) self.assertRaises(TypeError, self.db.category, 'xx') @@ -308,14 +312,6 @@ class UnicodeMiscTest(UnicodeDatabaseTest): self.assertEqual(len(lines), 1, r"\u%.4x should not be a linebreak" % i) - def test_UCS4(self): - # unicodedata should work with code points outside the BMP - # even on a narrow Unicode build - self.assertEqual(self.db.category("\U0001012A"), "No") - self.assertEqual(self.db.numeric("\U0001012A"), 9000) - self.assertEqual(self.db.decimal("\U0001D7FD"), 7) - self.assertEqual(self.db.digit("\U0001D7FD"), 7) - def test_main(): test.support.run_unittest( UnicodeMiscTest, diff --git a/Lib/test/test_urllib.py b/Lib/test/test_urllib.py index 97781d7..d929a5e 100644 --- a/Lib/test/test_urllib.py +++ b/Lib/test/test_urllib.py @@ -301,7 +301,7 @@ class urlretrieve_FileTests(unittest.TestCase): def constructLocalFileUrl(self, filePath): filePath = os.path.abspath(filePath) try: - filePath.encode("utf8") + filePath.encode("utf-8") except UnicodeEncodeError: raise unittest.SkipTest("filePath is not encodable to utf8") return "file://%s" % urllib.request.pathname2url(filePath) @@ -1211,6 +1211,28 @@ class URLopener_Tests(unittest.TestCase): # self.assertEqual(ftp.ftp.sock.gettimeout(), 30) # ftp.close() +class RequestTests(unittest.TestCase): + """Unit tests for urllib.request.Request.""" + + def test_default_values(self): + Request = urllib.request.Request + request = Request("http://www.python.org") + self.assertEqual(request.get_method(), 'GET') + request = Request("http://www.python.org", {}) + self.assertEqual(request.get_method(), 'POST') + + def test_with_method_arg(self): + Request = urllib.request.Request + request = Request("http://www.python.org", method='HEAD') + self.assertEqual(request.method, 'HEAD') + self.assertEqual(request.get_method(), 'HEAD') + request = Request("http://www.python.org", {}, method='HEAD') + self.assertEqual(request.method, 'HEAD') + self.assertEqual(request.get_method(), 'HEAD') + request = Request("http://www.python.org", method='GET') + self.assertEqual(request.get_method(), 'GET') + request.method = 'HEAD' + self.assertEqual(request.get_method(), 'HEAD') def test_main(): @@ -1227,6 +1249,7 @@ def test_main(): Utility_Tests, URLopener_Tests, #FTPWrapperTests, + RequestTests, ) diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 83bb0a9..aa1854d 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -5,6 +5,7 @@ import os import io import socket import array +import sys import urllib.request # The proxy bypass method imported below has logic specific to the OSX @@ -18,6 +19,22 @@ import urllib.error # parse_keqv_list, parse_http_list, HTTPDigestAuthHandler class TrivialTests(unittest.TestCase): + + def test___all__(self): + # Verify which names are exposed + for module in 'request', 'response', 'parse', 'error', 'robotparser': + context = {} + exec('from urllib.%s import *' % module, context) + del context['__builtins__'] + if module == 'request' and os.name == 'nt': + u, p = context.pop('url2pathname'), context.pop('pathname2url') + self.assertEqual(u.__module__, 'nturl2path') + self.assertEqual(p.__module__, 'nturl2path') + for k, v in context.items(): + self.assertEqual(v.__module__, 'urllib.%s' % module, + "%r is exposed in 'urllib.%s' but defined in %r" % + (k, module, v.__module__)) + def test_trivial(self): # A couple trivial tests @@ -605,7 +622,7 @@ class OpenerDirectorTests(unittest.TestCase): def sanepathname2url(path): try: - path.encode("utf8") + path.encode("utf-8") except UnicodeEncodeError: raise unittest.SkipTest("path is not encodable to utf8") urlpath = urllib.request.pathname2url(path) @@ -1166,6 +1183,8 @@ class HandlerTests(unittest.TestCase): self.assertEqual(req.get_host(), "proxy.example.com:3128") self.assertEqual(req.get_header("Proxy-authorization"),"FooBar") + # TODO: This should be only for OSX + @unittest.skipUnless(sys.platform == 'darwin', "only relevant for OSX") def test_osx_proxy_bypass(self): bypass = { 'exclude_simple': False, @@ -1269,6 +1288,26 @@ class HandlerTests(unittest.TestCase): # _test_basic_auth called .open() twice) self.assertEqual(opener.recorded, ["digest", "basic"]*2) + def test_unsupported_auth_digest_handler(self): + opener = OpenerDirector() + # While using DigestAuthHandler + digest_auth_handler = urllib.request.HTTPDigestAuthHandler(None) + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: Kerberos\r\n\r\n') + opener.add_handler(digest_auth_handler) + opener.add_handler(http_handler) + self.assertRaises(ValueError,opener.open,"http://www.example.com") + + def test_unsupported_auth_basic_handler(self): + # While using BasicAuthHandler + opener = OpenerDirector() + basic_auth_handler = urllib.request.HTTPBasicAuthHandler(None) + http_handler = MockHTTPHandler( + 401, 'WWW-Authenticate: NTLM\r\n\r\n') + opener.add_handler(basic_auth_handler) + opener.add_handler(http_handler) + self.assertRaises(ValueError,opener.open,"http://www.example.com") + def _test_basic_auth(self, opener, auth_handler, auth_header, realm, http_handler, password_manager, request_url, protected_url): @@ -1306,6 +1345,7 @@ class HandlerTests(unittest.TestCase): self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) + class MiscTests(unittest.TestCase): def test_build_opener(self): diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index 868ed24..6381070 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -52,6 +52,12 @@ class UserListTest(list_tests.CommonTest): return str(key) + '!!!' self.assertEqual(next(iter(T((1,2)))), "0!!!") + def test_userlist_copy(self): + u = self.type2test([6, 8, 1, 9, 1]) + v = u.copy() + self.assertEqual(u, v) + self.assertEqual(type(u), type(v)) + def test_main(): support.run_unittest(UserListTest) diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index 7a8b932..d5d0c89 100755 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -17,11 +17,11 @@ class UserStringTest( # Overwrite the three testing methods, because UserString # can't cope with arguments propagated to UserString # (and we don't test with subclasses) - def checkequal(self, result, object, methodname, *args): + def checkequal(self, result, object, methodname, *args, **kwargs): result = self.fixtype(result) object = self.fixtype(object) # we don't fix the arguments, because UserString can't cope with it - realresult = getattr(object, methodname)(*args) + realresult = getattr(object, methodname)(*args, **kwargs) self.assertEqual( result, realresult diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 43fa656..7bc59ed 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -471,14 +471,14 @@ class TestUUID(TestCase): if pid == 0: os.close(fds[0]) value = uuid.uuid4() - os.write(fds[1], value.hex.encode('latin1')) + os.write(fds[1], value.hex.encode('latin-1')) os._exit(0) else: os.close(fds[1]) parent_value = uuid.uuid4().hex os.waitpid(pid, 0) - child_value = os.read(fds[0], 100).decode('latin1') + child_value = os.read(fds[0], 100).decode('latin-1') self.assertNotEqual(parent_value, child_value) diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py index 786e60b..bd06c8d 100644 --- a/Lib/test/test_wait3.py +++ b/Lib/test/test_wait3.py @@ -19,13 +19,16 @@ except AttributeError: class Wait3Test(ForkWait): def wait_impl(self, cpid): - for i in range(10): + # This many iterations can be required, since some previously run + # tests (e.g. test_ctypes) could have spawned a lot of children + # very quickly. + for i in range(30): # wait3() shouldn't hang, but some of the buildbots seem to hang # in the forking tests. This is an attempt to fix the problem. spid, status, rusage = os.wait3(os.WNOHANG) if spid == cpid: break - time.sleep(1.0) + time.sleep(0.1) self.assertEqual(spid, cpid) self.assertEqual(status, 0, "cause = %d, exit = %d" % (status&0xff, status>>8)) diff --git a/Lib/test/test_warnings.py b/Lib/test/test_warnings.py index 79be835..953b282 100644 --- a/Lib/test/test_warnings.py +++ b/Lib/test/test_warnings.py @@ -512,12 +512,11 @@ class _WarningsTests(BaseTest): def test_showwarning_not_callable(self): with original_warnings.catch_warnings(module=self.module): self.module.filterwarnings("always", category=UserWarning) - old_showwarning = self.module.showwarning + self.module.showwarning = print + with support.captured_output('stdout'): + self.module.warn('Warning!') self.module.showwarning = 23 - try: - self.assertRaises(TypeError, self.module.warn, "Warning!") - finally: - self.module.showwarning = old_showwarning + self.assertRaises(TypeError, self.module.warn, "Warning!") def test_show_warning_output(self): # With showarning() missing, make sure that output is okay. @@ -547,10 +546,13 @@ class _WarningsTests(BaseTest): globals_dict = globals() oldfile = globals_dict['__file__'] try: - with original_warnings.catch_warnings(module=self.module) as w: + catch = original_warnings.catch_warnings(record=True, + module=self.module) + with catch as w: self.module.filterwarnings("always", category=UserWarning) globals_dict['__file__'] = None original_warnings.warn('test', UserWarning) + self.assertTrue(len(w)) finally: globals_dict['__file__'] = oldfile diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index b7a996c..36b08cd 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -22,7 +22,7 @@ from xml.etree import ElementTree as ET SIMPLE_XMLFILE = findfile("simple.xml", subdir="xmltestdata") try: - SIMPLE_XMLFILE.encode("utf8") + SIMPLE_XMLFILE.encode("utf-8") except UnicodeEncodeError: raise unittest.SkipTest("filename is not encodable to utf8") SIMPLE_NS_XMLFILE = findfile("simple-ns.xml", subdir="xmltestdata") @@ -1256,8 +1256,8 @@ def processinginstruction(): >>> ET.tostring(ET.PI('test', '<testing&>')) b'<?test <testing&>?>' - >>> ET.tostring(ET.PI('test', '<testing&>\xe3'), 'latin1') - b"<?xml version='1.0' encoding='latin1'?>\\n<?test <testing&>\\xe3?>" + >>> ET.tostring(ET.PI('test', '<testing&>\xe3'), 'latin-1') + b"<?xml version='1.0' encoding='latin-1'?>\\n<?test <testing&>\\xe3?>" """ # @@ -1340,7 +1340,7 @@ def xinclude_loader(href, parse="xml", encoding=None): try: data = XINCLUDE[href] except KeyError: - raise IOError("resource not found") + raise OSError("resource not found") if parse == "xml": from xml.etree.ElementTree import XML return XML(data) @@ -1405,7 +1405,7 @@ def xinclude(): >>> document = xinclude_loader("C5.xml") >>> ElementInclude.include(document, xinclude_loader) Traceback (most recent call last): - IOError: resource not found + OSError: resource not found >>> # print(serialize(document)) # C5 """ @@ -1612,7 +1612,7 @@ def bug_xmltoolkit55(): class ExceptionFile: def read(self, x): - raise IOError + raise OSError def xmltoolkit60(): """ @@ -1620,7 +1620,7 @@ def xmltoolkit60(): Handle crash in stream source. >>> tree = ET.parse(ExceptionFile()) Traceback (most recent call last): - IOError + OSError """ diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index e97420b..76723fa 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -24,6 +24,8 @@ alist = [{'astring': 'foo@bar.baz.spam', 'ashortlong': 2, 'anotherlist': ['.zyx.41'], 'abase64': xmlrpclib.Binary(b"my dog has fleas"), + 'b64bytes': b"my dog has fleas", + 'b64bytearray': bytearray(b"my dog has fleas"), 'boolean': False, 'unicode': '\u4000\u6000\u8000', 'ukey\u4000': 'regular value', @@ -44,27 +46,54 @@ class XMLRPCTestCase(unittest.TestCase): def test_dump_bare_datetime(self): # This checks that an unwrapped datetime.date object can be handled # by the marshalling code. This can't be done via test_dump_load() - # since with use_datetime set to 1 the unmarshaller would create + # since with use_builtin_types set to 1 the unmarshaller would create # datetime objects for the 'datetime[123]' keys as well dt = datetime.datetime(2005, 2, 10, 11, 41, 23) + self.assertEqual(dt, xmlrpclib.DateTime('20050210T11:41:23')) s = xmlrpclib.dumps((dt,)) - (newdt,), m = xmlrpclib.loads(s, use_datetime=1) + + result, m = xmlrpclib.loads(s, use_builtin_types=True) + (newdt,) = result self.assertEqual(newdt, dt) - self.assertEqual(m, None) + self.assertIs(type(newdt), datetime.datetime) + self.assertIsNone(m) + + result, m = xmlrpclib.loads(s, use_builtin_types=False) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), xmlrpclib.DateTime) + self.assertIsNone(m) + + result, m = xmlrpclib.loads(s, use_datetime=True) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), datetime.datetime) + self.assertIsNone(m) + + result, m = xmlrpclib.loads(s, use_datetime=False) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), xmlrpclib.DateTime) + self.assertIsNone(m) - (newdt,), m = xmlrpclib.loads(s, use_datetime=0) - self.assertEqual(newdt, xmlrpclib.DateTime('20050210T11:41:23')) def test_datetime_before_1900(self): # same as before but with a date before 1900 dt = datetime.datetime(1, 2, 10, 11, 41, 23) + self.assertEqual(dt, xmlrpclib.DateTime('00010210T11:41:23')) s = xmlrpclib.dumps((dt,)) - (newdt,), m = xmlrpclib.loads(s, use_datetime=1) + + result, m = xmlrpclib.loads(s, use_builtin_types=True) + (newdt,) = result self.assertEqual(newdt, dt) - self.assertEqual(m, None) + self.assertIs(type(newdt), datetime.datetime) + self.assertIsNone(m) - (newdt,), m = xmlrpclib.loads(s, use_datetime=0) - self.assertEqual(newdt, xmlrpclib.DateTime('00010210T11:41:23')) + result, m = xmlrpclib.loads(s, use_builtin_types=False) + (newdt,) = result + self.assertEqual(newdt, dt) + self.assertIs(type(newdt), xmlrpclib.DateTime) + self.assertIsNone(m) def test_bug_1164912 (self): d = xmlrpclib.DateTime() @@ -133,6 +162,25 @@ class XMLRPCTestCase(unittest.TestCase): xmlrpclib.loads(strg)[0][0]) self.assertRaises(TypeError, xmlrpclib.dumps, (arg1,)) + def test_dump_bytes(self): + sample = b"my dog has fleas" + self.assertEqual(sample, xmlrpclib.Binary(sample)) + for type_ in bytes, bytearray, xmlrpclib.Binary: + value = type_(sample) + s = xmlrpclib.dumps((value,)) + + result, m = xmlrpclib.loads(s, use_builtin_types=True) + (newvalue,) = result + self.assertEqual(newvalue, sample) + self.assertIs(type(newvalue), bytes) + self.assertIsNone(m) + + result, m = xmlrpclib.loads(s, use_builtin_types=False) + (newvalue,) = result + self.assertEqual(newvalue, sample) + self.assertIs(type(newvalue), xmlrpclib.Binary) + self.assertIsNone(m) + def test_get_host_info(self): # see bug #3613, this raised a TypeError transp = xmlrpc.client.Transport() @@ -140,9 +188,6 @@ class XMLRPCTestCase(unittest.TestCase): ('host.tld', [('Authorization', 'Basic dXNlcg==')], {})) - def test_dump_bytes(self): - self.assertRaises(TypeError, xmlrpclib.dumps, (b"my dog has fleas",)) - def test_ssl_presence(self): try: import ssl diff --git a/Lib/test/test_xmlrpc_net.py b/Lib/test/test_xmlrpc_net.py index 5df79f0..b9853ed 100644 --- a/Lib/test/test_xmlrpc_net.py +++ b/Lib/test/test_xmlrpc_net.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 -import collections +import collections.abc import errno import socket import sys @@ -48,7 +48,7 @@ class CurrentTimeTest(unittest.TestCase): # Perform a minimal sanity check on the result, just to be sure # the request means what we think it means. - self.assertIsInstance(builders, collections.Sequence) + self.assertIsInstance(builders, collections.abc.Sequence) self.assertTrue([x for x in builders if "3.x" in x], builders) diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index bb0d79a..0b3a694 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -1,9 +1,3 @@ -# We can test part of the module without zlib. -try: - import zlib -except ImportError: - zlib = None - import io import os import sys @@ -19,7 +13,7 @@ from tempfile import TemporaryFile from random import randint, random from unittest import skipUnless -from test.support import TESTFN, run_unittest, findfile, unlink +from test.support import TESTFN, run_unittest, findfile, unlink, requires_zlib TESTFN2 = TESTFN + "2" TESTFNDIR = TESTFN + "d" @@ -269,44 +263,44 @@ class TestsWithSourceFile(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_iterlines_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_open_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_open_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_random_open_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_random_open_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readline_read_deflated(self): # Issue #7610: calls to readline() interleaved with calls to read(). for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_readline_read_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readline_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_readline_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readlines_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_readlines_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_iterlines_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_iterlines_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_low_compression(self): """Check for cases where compressed data is larger than original.""" # Create the ZIP archive @@ -377,7 +371,7 @@ class TestsWithSourceFile(unittest.TestCase): with open(TESTFN, "rb") as f: self.assertEqual(zipfp.read(TESTFN), f.read()) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_per_file_compression(self): """Check that files within a Zip archive can have different compression options.""" @@ -446,19 +440,18 @@ class TestsWithSourceFile(unittest.TestCase): # remove the test file subdirectories shutil.rmtree(os.path.join(os.getcwd(), 'ziptest2dir')) - def test_writestr_compression(self): + def test_writestr_compression_stored(self): zipfp = zipfile.ZipFile(TESTFN2, "w") zipfp.writestr("a.txt", "hello world", compress_type=zipfile.ZIP_STORED) - if zlib: - zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED) - info = zipfp.getinfo('a.txt') self.assertEqual(info.compress_type, zipfile.ZIP_STORED) - if zlib: - info = zipfp.getinfo('b.txt') - self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED) - + @requires_zlib + def test_writestr_compression_deflated(self): + zipfp = zipfile.ZipFile(TESTFN2, "w") + zipfp.writestr("b.txt", "hello world", compress_type=zipfile.ZIP_DEFLATED) + info = zipfp.getinfo('b.txt') + self.assertEqual(info.compress_type, zipfile.ZIP_DEFLATED) def zip_test_writestr_permissions(self, f, compression): # Make sure that writestr creates files with mode 0600, @@ -514,7 +507,12 @@ class TestsWithSourceFile(unittest.TestCase): self.assertRaises(ValueError, zipfp.write, TESTFN) - @skipUnless(zlib, "requires zlib") + + + + + + @requires_zlib def test_unicode_filenames(self): # bug #10801 fname = findfile('zip_cp437_header.zip') @@ -623,7 +621,7 @@ class TestZip64InSmallFiles(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_DEFLATED) @@ -983,7 +981,7 @@ class OtherTests(unittest.TestCase): def test_testzip_with_bad_crc_stored(self): self.check_testzip_with_bad_crc(zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_testzip_with_bad_crc_deflated(self): self.check_testzip_with_bad_crc(zipfile.ZIP_DEFLATED) @@ -1011,7 +1009,7 @@ class OtherTests(unittest.TestCase): def test_read_with_bad_crc_stored(self): self.check_read_with_bad_crc(zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_read_with_bad_crc_deflated(self): self.check_read_with_bad_crc(zipfile.ZIP_DEFLATED) @@ -1031,7 +1029,7 @@ class OtherTests(unittest.TestCase): def test_read_return_size_stored(self): self.check_read_return_size(zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_read_return_size_deflated(self): self.check_read_return_size(zipfile.ZIP_DEFLATED) @@ -1121,7 +1119,7 @@ class DecryptionTests(unittest.TestCase): self.zip2.setpassword(b"perl") self.assertRaises(RuntimeError, self.zip2.read, "zero") - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_good_password(self): self.zip.setpassword(b"python") self.assertEqual(self.zip.read("test.txt"), self.plain) @@ -1171,7 +1169,7 @@ class TestsWithRandomBinaryFiles(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_test(f, zipfile.ZIP_DEFLATED) @@ -1211,7 +1209,7 @@ class TestsWithRandomBinaryFiles(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_open_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_open_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_open_test(f, zipfile.ZIP_DEFLATED) @@ -1239,13 +1237,13 @@ class TestsWithRandomBinaryFiles(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_random_open_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_random_open_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.zip_random_open_test(f, zipfile.ZIP_DEFLATED) -@skipUnless(zlib, "requires zlib") +@requires_zlib class TestsWithMultipleOpens(unittest.TestCase): def setUp(self): # Create the ZIP archive @@ -1437,28 +1435,28 @@ class UniversalNewlineTests(unittest.TestCase): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.iterlines_test(f, zipfile.ZIP_STORED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_read_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.read_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readline_read_deflated(self): # Issue #7610: calls to readline() interleaved with calls to read(). for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.readline_read_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readline_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.readline_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_readlines_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.readlines_test(f, zipfile.ZIP_DEFLATED) - @skipUnless(zlib, "requires zlib") + @requires_zlib def test_iterlines_deflated(self): for f in (TESTFN2, TemporaryFile(), io.BytesIO()): self.iterlines_test(f, zipfile.ZIP_DEFLATED) diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index 0e7d73f..a8fb7ab 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -11,12 +11,6 @@ support.requires( 'test requires loads of disk-space bytes and a long time to run' ) -# We can test part of the module without zlib. -try: - import zlib -except ImportError: - zlib = None - import zipfile, os, unittest import time import sys @@ -24,7 +18,7 @@ import sys from io import StringIO from tempfile import TemporaryFile -from test.support import TESTFN, run_unittest +from test.support import TESTFN, run_unittest, requires_zlib TESTFN2 = TESTFN + "2" @@ -81,12 +75,12 @@ class TestsWithSourceFile(unittest.TestCase): for f in TemporaryFile(), TESTFN2: self.zipTest(f, zipfile.ZIP_STORED) - if zlib: - def testDeflated(self): - # Try the temp file first. If we do TESTFN2 first, then it hogs - # gigabytes of disk space for the duration of the test. - for f in TemporaryFile(), TESTFN2: - self.zipTest(f, zipfile.ZIP_DEFLATED) + @requires_zlib + def testDeflated(self): + # Try the temp file first. If we do TESTFN2 first, then it hogs + # gigabytes of disk space for the duration of the test. + for f in TemporaryFile(), TESTFN2: + self.zipTest(f, zipfile.ZIP_DEFLATED) def tearDown(self): for fname in TESTFN, TESTFN2: diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index df5ff9d..56141ef 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -9,12 +9,6 @@ import unittest from test import support from test.test_importhooks import ImportHooksBaseTestCase, test_src, test_co -# some tests can be ran even without zlib -try: - import zlib -except ImportError: - zlib = None - from zipfile import ZipFile, ZipInfo, ZIP_STORED, ZIP_DEFLATED import zipimport @@ -392,7 +386,7 @@ class UncompressedZipImportTestCase(ImportHooksBaseTestCase): os.remove(filename) -@unittest.skipUnless(zlib, "requires zlib") +@support.requires_zlib class CompressedZipImportTestCase(UncompressedZipImportTestCase): compression = ZIP_DEFLATED @@ -417,7 +411,7 @@ class BadFileZipImportTestCase(unittest.TestCase): def testEmptyFile(self): support.unlink(TESTMOD) - open(TESTMOD, 'w+').close() + support.create_empty_file(TESTMOD) self.assertZipFailure(TESTMOD) def testFileUnreadable(self): diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index a558d7d..0c93a8c 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -156,20 +156,19 @@ class ZipSupportTests(unittest.TestCase): test_zipped_doctest.test_DocTestRunner.verbose_flag, test_zipped_doctest.test_Example, test_zipped_doctest.test_debug, - test_zipped_doctest.test_pdb_set_trace, - test_zipped_doctest.test_pdb_set_trace_nested, test_zipped_doctest.test_testsource, test_zipped_doctest.test_trailing_space_in_test, test_zipped_doctest.test_DocTestSuite, test_zipped_doctest.test_DocTestFinder, ] - # These remaining tests are the ones which need access + # These tests are the ones which need access # to the data files, so we don't run them fail_due_to_missing_data_files = [ test_zipped_doctest.test_DocFileSuite, test_zipped_doctest.test_testfile, test_zipped_doctest.test_unittest_reportflags, ] + for obj in known_good_tests: _run_object_doctest(obj, test_zipped_doctest) finally: diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index dffa2ca..51a9d4b 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -13,6 +13,17 @@ except ImportError: mmap = None +class VersionTestCase(unittest.TestCase): + + def test_library_version(self): + # On the build system, ZLIB_RUNTIME_VERSION should match ZLIB_VERSION. + # ZLIB_RUNTIME_VERSION is the actual library version while ZLIB_VERSION + # is the version from the header file. On the build system, the headers + # should match with the library exactly. At runtime, only the first + # digit is required to match. + self.assertEqual(zlib.ZLIB_RUNTIME_VERSION, zlib.ZLIB_VERSION) + + class ChecksumTestCase(unittest.TestCase): # checksum test cases def test_crc32start(self): @@ -447,6 +458,26 @@ class CompressObjectTestCase(BaseCompressTestCase, unittest.TestCase): y += dco.flush() self.assertEqual(y, b'foo') + def test_decompress_eof(self): + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + self.assertFalse(dco.eof) + dco.decompress(x[:-5]) + self.assertFalse(dco.eof) + dco.decompress(x[-5:]) + self.assertTrue(dco.eof) + dco.flush() + self.assertTrue(dco.eof) + + def test_decompress_eof_incomplete_stream(self): + x = b'x\x9cK\xcb\xcf\x07\x00\x02\x82\x01E' # 'foo' + dco = zlib.decompressobj() + self.assertFalse(dco.eof) + dco.decompress(x[:-5]) + self.assertFalse(dco.eof) + dco.flush() + self.assertFalse(dco.eof) + if hasattr(zlib.compressobj(), "copy"): def test_compresscopy(self): # Test copying a compression object @@ -627,6 +658,7 @@ LAERTES def test_main(): support.run_unittest( + VersionTestCase, ChecksumTestCase, ChecksumBigBufferTestCase, ExceptionTestCase, diff --git a/Lib/test/threaded_import_hangers.py b/Lib/test/threaded_import_hangers.py index adf03e3..5484e60 100644 --- a/Lib/test/threaded_import_hangers.py +++ b/Lib/test/threaded_import_hangers.py @@ -35,8 +35,11 @@ for name, func, args in [ ("os.path.abspath", os.path.abspath, ('.',)), ]: - t = Worker(func, args) - t.start() - t.join(TIMEOUT) - if t.is_alive(): - errors.append("%s appeared to hang" % name) + try: + t = Worker(func, args) + t.start() + t.join(TIMEOUT) + if t.is_alive(): + errors.append("%s appeared to hang" % name) + finally: + del t |
