Merged revisions 57152-57220 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r57155 | brett.cannon | 2007-08-17 11:51:57 -0700 (Fri, 17 Aug 2007) | 2 lines

  Clarify the comment about setting the PYTHON variable for the Doc Makefile.
........
  r57156 | brett.cannon | 2007-08-17 11:59:58 -0700 (Fri, 17 Aug 2007) | 3 lines

  Remove news entry for test.test_support.guard_warnings_filter as it has been
  removed.
........
  r57158 | facundo.batista | 2007-08-17 12:16:44 -0700 (Fri, 17 Aug 2007) | 9 lines


  Added a flag (_send_traceback_header) to the SimpleXMLRPCServer class
  that allows sending back exception/stack trace information about
  internal server errors (this flag defaults to False to avoid sending
  such information unless explicitly enabled).  Added tests to verify
  behavior of this new feature (these tests are skipped on win32 because
  of problems with WSAEWOULDBLOCK). Renamed HTTPTestCase to
  SimpleServerTestCase. [GSoC - Alan McIntyre]
........
  r57160 | brett.cannon | 2007-08-17 13:16:15 -0700 (Fri, 17 Aug 2007) | 2 lines

  Make test_warnings re-entrant.
........
  r57175 | georg.brandl | 2007-08-17 23:05:56 -0700 (Fri, 17 Aug 2007) | 2 lines

  Fix PDB command descriptions.
........
  r57182 | brett.cannon | 2007-08-18 11:30:36 -0700 (Sat, 18 Aug 2007) | 3 lines

  Remove Mikael as the maintainer of BeOS at his request.  With no maintainer
  should the port be deprecated in 2.6?
........
  r57199 | georg.brandl | 2007-08-19 11:43:50 -0700 (Sun, 19 Aug 2007) | 2 lines

  Fix stray backticks.
........
  r57211 | hyeshik.chang | 2007-08-19 23:49:18 -0700 (Sun, 19 Aug 2007) | 2 lines

  Add cheot-ga-keut composed make-up sequence support in EUC-KR codec.
........
  r57216 | guido.van.rossum | 2007-08-20 08:18:04 -0700 (Mon, 20 Aug 2007) | 4 lines

  Add a hack (originally devised in a slightly different form by Thomas Wouters)
  to prevent spurious tracebacks when a daemon thread's cleanup happens to wake
  up when the world around it has already been destroyed.
........

7 files changed, 170 insertions, 13 deletions

diff --git a/Lib/SimpleXMLRPCServer.py b/Lib/SimpleXMLRPCServer.py
index 458d427..5f6e9d0 100644
--- a/Lib/SimpleXMLRPCServer.py
+++ b/Lib/SimpleXMLRPCServer.py
@@ -104,6 +104,7 @@ import SocketServer
 import BaseHTTPServer
 import sys
 import os
+import traceback
 try:
     import fcntl
 except ImportError:
@@ -451,9 +452,16 @@ class SimpleXMLRPCRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
             response = self.server._marshaled_dispatch(
                     data, getattr(self, '_dispatch', None)
                 )
-        except: # This should only happen if the module is buggy
+        except Exception as e: # This should only happen if the module is buggy
             # internal error, report as HTTP server error
             self.send_response(500)
+
+            # Send information about the exception if requested
+            if hasattr(self.server, '_send_traceback_header') and \
+                    self.server._send_traceback_header:
+                self.send_header("X-exception", str(e))
+                self.send_header("X-traceback", traceback.format_exc())
+
             self.end_headers()
         else:
             # got a valid XML RPC response
@@ -498,6 +506,12 @@ class SimpleXMLRPCServer(SocketServer.TCPServer,
 
     allow_reuse_address = True
 
+    # Warning: this is for debugging purposes only! Never set this to True in
+    # production code, as will be sending out sensitive information (exception
+    # and stack trace details) when exceptions are raised inside
+    # SimpleXMLRPCRequestHandler.do_POST
+    _send_traceback_header = False
+
     def __init__(self, addr, requestHandler=SimpleXMLRPCRequestHandler,
                  logRequests=True, allow_none=False, encoding=None, bind_and_activate=True):
         self.logRequests = logRequests
diff --git a/Lib/test/cjkencodings_test.py b/Lib/test/cjkencodings_test.py
index d5873c0..72f4bdc 100644
--- a/Lib/test/cjkencodings_test.py
+++ b/Lib/test/cjkencodings_test.py
@@ -376,13 +376,20 @@ b"\x20\xb9\xae\xb9\xfd\xb0\xfa\x20\xb5\xbf\xc0\xfb\x20\xc5\xb8\xc0"
 b"\xcc\xc7\xce\x2c\x20\xb1\xd7\xb8\xae\xb0\xed\x20\xc0\xce\xc5\xcd"
 b"\xc7\xc1\xb8\xae\xc6\xc3\x0a\xc8\xaf\xb0\xe6\xc0\xba\x20\xc6\xc4"
 b"\xc0\xcc\xbd\xe3\xc0\xbb\x20\xbd\xba\xc5\xa9\xb8\xb3\xc6\xc3\xb0"
-b"\xfa\x20\xbf\xa9\xb7\xc1\x20\xba\xd0\xbe\xdf\xbf\xa1\xbc\xad\xbf"
+b"\xfa\x20\xbf\xa9\xb7\xaf\x20\xba\xd0\xbe\xdf\xbf\xa1\xbc\xad\xbf"
 b"\xcd\x20\xb4\xeb\xba\xce\xba\xd0\xc0\xc7\x20\xc7\xc3\xb7\xa7\xc6"
 b"\xfb\xbf\xa1\xbc\xad\xc0\xc7\x20\xba\xfc\xb8\xa5\x0a\xbe\xd6\xc7"
 b"\xc3\xb8\xae\xc4\xc9\xc0\xcc\xbc\xc7\x20\xb0\xb3\xb9\xdf\xc0\xbb"
 b"\x20\xc7\xd2\x20\xbc\xf6\x20\xc0\xd6\xb4\xc2\x20\xc0\xcc\xbb\xf3"
 b"\xc0\xfb\xc0\xce\x20\xbe\xf0\xbe\xee\xb7\xce\x20\xb8\xb8\xb5\xe9"
-b"\xbe\xee\xc1\xdd\xb4\xcf\xb4\xd9\x2e\x0a\x0a",
+b"\xbe\xee\xc1\xdd\xb4\xcf\xb4\xd9\x2e\x0a\x0a\xa1\xd9\xc3\xb9\xb0"
+b"\xa1\xb3\xa1\x3a\x20\xb3\xaf\xbe\xc6\xb6\xf3\x20\xa4\xd4\xa4\xb6"
+b"\xa4\xd0\xa4\xd4\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4\xbe\xb1\x7e\x20"
+b"\xa4\xd4\xa4\xa4\xa4\xd2\xa4\xb7\xc5\xad\x21\x20\xa4\xd4\xa4\xa8"
+b"\xa4\xd1\xa4\xb7\xb1\xdd\xbe\xf8\xc0\xcc\x20\xc0\xfc\xa4\xd4\xa4"
+b"\xbe\xa4\xc8\xa4\xb2\xb4\xcf\xb4\xd9\x2e\x20\xa4\xd4\xa4\xb2\xa4"
+b"\xce\xa4\xaa\x2e\x20\xb1\xd7\xb7\xb1\xb0\xc5\x20\xa4\xd4\xa4\xb7"
+b"\xa4\xd1\xa4\xb4\xb4\xd9\x2e\x0a",
 b"\xe2\x97\x8e\x20\xed\x8c\x8c\xec\x9d\xb4\xec\x8d\xac\x28\x50\x79"
 b"\x74\x68\x6f\x6e\x29\xec\x9d\x80\x20\xeb\xb0\xb0\xec\x9a\xb0\xea"
 b"\xb8\xb0\x20\xec\x89\xbd\xea\xb3\xa0\x2c\x20\xea\xb0\x95\xeb\xa0"
@@ -404,7 +411,7 @@ b"\xb4\xed\x95\x91\x2c\x20\xea\xb7\xb8\xeb\xa6\xac\xea\xb3\xa0\x20"
 b"\xec\x9d\xb8\xed\x84\xb0\xed\x94\x84\xeb\xa6\xac\xed\x8c\x85\x0a"
 b"\xed\x99\x98\xea\xb2\xbd\xec\x9d\x80\x20\xed\x8c\x8c\xec\x9d\xb4"
 b"\xec\x8d\xac\xec\x9d\x84\x20\xec\x8a\xa4\xed\x81\xac\xeb\xa6\xbd"
-b"\xed\x8c\x85\xea\xb3\xbc\x20\xec\x97\xac\xeb\xa0\xa4\x20\xeb\xb6"
+b"\xed\x8c\x85\xea\xb3\xbc\x20\xec\x97\xac\xeb\x9f\xac\x20\xeb\xb6"
 b"\x84\xec\x95\xbc\xec\x97\x90\xec\x84\x9c\xec\x99\x80\x20\xeb\x8c"
 b"\x80\xeb\xb6\x80\xeb\xb6\x84\xec\x9d\x98\x20\xed\x94\x8c\xeb\x9e"
 b"\xab\xed\x8f\xbc\xec\x97\x90\xec\x84\x9c\xec\x9d\x98\x20\xeb\xb9"
@@ -413,7 +420,13 @@ b"\x80\xec\x9d\xb4\xec\x85\x98\x20\xea\xb0\x9c\xeb\xb0\x9c\xec\x9d"
 b"\x84\x20\xed\x95\xa0\x20\xec\x88\x98\x20\xec\x9e\x88\xeb\x8a\x94"
 b"\x20\xec\x9d\xb4\xec\x83\x81\xec\xa0\x81\xec\x9d\xb8\x20\xec\x96"
 b"\xb8\xec\x96\xb4\xeb\xa1\x9c\x20\xeb\xa7\x8c\xeb\x93\xa4\xec\x96"
-b"\xb4\xec\xa4\x8d\xeb\x8b\x88\xeb\x8b\xa4\x2e\x0a\x0a"),
+b"\xb4\xec\xa4\x8d\xeb\x8b\x88\xeb\x8b\xa4\x2e\x0a\x0a\xe2\x98\x86"
+b"\xec\xb2\xab\xea\xb0\x80\xeb\x81\x9d\x3a\x20\xeb\x82\xa0\xec\x95"
+b"\x84\xeb\x9d\xbc\x20\xec\x93\x94\xec\x93\x94\xec\x93\xa9\x7e\x20"
+b"\xeb\x8b\x81\xed\x81\xbc\x21\x20\xeb\x9c\xbd\xea\xb8\x88\xec\x97"
+b"\x86\xec\x9d\xb4\x20\xec\xa0\x84\xed\x99\xa5\xeb\x8b\x88\xeb\x8b"
+b"\xa4\x2e\x20\xeb\xb7\x81\x2e\x20\xea\xb7\xb8\xeb\x9f\xb0\xea\xb1"
+b"\xb0\x20\xec\x9d\x8e\xeb\x8b\xa4\x2e\x0a"),
 'gb18030': (
 b"\x50\x79\x74\x68\x6f\x6e\xa3\xa8\xc5\xc9\xc9\xad\xa3\xa9\xd3\xef"
 b"\xd1\xd4\xca\xc7\xd2\xbb\xd6\xd6\xb9\xa6\xc4\xdc\xc7\xbf\xb4\xf3"
diff --git a/Lib/test/test_codecencodings_kr.py b/Lib/test/test_codecencodings_kr.py
index 86663c7..18ed393 100644
--- a/Lib/test/test_codecencodings_kr.py
+++ b/Lib/test/test_codecencodings_kr.py
@@ -30,6 +30,24 @@ class Test_EUCKR(test_multibytecodec_support.TestBase, unittest.TestCase):
         (b"abc\x80\x80\xc1\xc4", "replace", "abc\ufffd\uc894"),
         (b"abc\x80\x80\xc1\xc4\xc8", "replace", "abc\ufffd\uc894\ufffd"),
         (b"abc\x80\x80\xc1\xc4", "ignore",  "abc\uc894"),
+
+        # composed make-up sequence errors
+        (b"\xa4\xd4", "strict", None),
+        (b"\xa4\xd4\xa4", "strict", None),
+        (b"\xa4\xd4\xa4\xb6", "strict", None),
+        (b"\xa4\xd4\xa4\xb6\xa4", "strict", None),
+        (b"\xa4\xd4\xa4\xb6\xa4\xd0", "strict", None),
+        (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4", "strict", None),
+        (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4", "strict", "\uc4d4"),
+        (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xd4x", "strict", "\uc4d4x"),
+        (b"a\xa4\xd4\xa4\xb6\xa4", "replace", "a\ufffd"),
+        (b"\xa4\xd4\xa3\xb6\xa4\xd0\xa4\xd4", "strict", None),
+        (b"\xa4\xd4\xa4\xb6\xa3\xd0\xa4\xd4", "strict", None),
+        (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa3\xd4", "strict", None),
+        (b"\xa4\xd4\xa4\xff\xa4\xd0\xa4\xd4", "replace", "\ufffd"),
+        (b"\xa4\xd4\xa4\xb6\xa4\xff\xa4\xd4", "replace", "\ufffd"),
+        (b"\xa4\xd4\xa4\xb6\xa4\xd0\xa4\xff", "replace", "\ufffd"),
+        (b"\xc1\xc4", "strict", "\uc894"),
     )
 
 class Test_JOHAB(test_multibytecodec_support.TestBase, unittest.TestCase):
diff --git a/Lib/test/test_codecmaps_kr.py b/Lib/test/test_codecmaps_kr.py
index 4a97611..f9c97e5 100644
--- a/Lib/test/test_codecmaps_kr.py
+++ b/Lib/test/test_codecmaps_kr.py
@@ -20,6 +20,10 @@ class TestEUCKRMap(test_multibytecodec_support.TestBase_Mapping,
     encoding = 'euc_kr'
     mapfileurl = 'http://people.freebsd.org/~perky/i18n/EUC-KR.TXT'
 
+    # A4D4 HANGUL FILLER indicates the begin of 8-bytes make-up sequence.
+    pass_enctest = [(b'\xa4\xd4', '\u3164')]
+    pass_dectest = [(b'\xa4\xd4', '\u3164')]
+
 
 class TestJOHABMap(test_multibytecodec_support.TestBase_Mapping,
                    unittest.TestCase):
diff --git a/Lib/test/test_warnings.py b/Lib/test/test_warnings.py
index 54d5dd2..6889e3f 100644
--- a/Lib/test/test_warnings.py
+++ b/Lib/test/test_warnings.py
@@ -1,5 +1,6 @@
 import warnings
 import os
+import sys
 import unittest
 from test import test_support
 
@@ -101,6 +102,10 @@ def test_main(verbose=None):
     # to test_main (regrtest -R).
     if '__warningregistry__' in globals():
         del globals()['__warningregistry__']
+    if hasattr(warning_tests, '__warningregistry__'):
+        del warning_tests.__warningregistry__
+    if hasattr(sys, '__warningregistry__'):
+        del sys.__warningregistry__
     test_support.run_unittest(TestModule)
 
 if __name__ == "__main__":
diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py
index 4a15de5..7dfabcf 100644
--- a/Lib/test/test_xmlrpc.py
+++ b/Lib/test/test_xmlrpc.py
@@ -6,6 +6,7 @@ import unittest
 import xmlrpclib
 import SimpleXMLRPCServer
 import threading
+import mimetools
 from test import test_support
 
 alist = [{'astring': 'foo@bar.baz.spam',
@@ -249,11 +250,9 @@ def http_server(evt, numrequests):
             '''This is the div function'''
             return x // y
 
-
-    serv = SimpleXMLRPCServer.SimpleXMLRPCServer(("localhost", 0),
-                    logRequests=False, bind_and_activate=False)
-
     try:
+        serv = SimpleXMLRPCServer.SimpleXMLRPCServer(("localhost", 0),
+                        logRequests=False, bind_and_activate=False)
         serv.socket.settimeout(3)
         serv.server_bind()
         global PORT
@@ -278,11 +277,15 @@ def http_server(evt, numrequests):
         evt.set()
 
 
-class HTTPTestCase(unittest.TestCase):
+class SimpleServerTestCase(unittest.TestCase):
     def setUp(self):
+        # enable traceback reporting
+        SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header = True
+
         self.evt = threading.Event()
-        # start server thread to handle just one request
-        threading.Thread(target=http_server, args=(self.evt,2)).start()
+        # start server thread to handle requests
+        serv_args = (self.evt, 2)
+        threading.Thread(target=http_server, args=serv_args).start()
 
         # wait for port to be assigned to server
         n = 1000
@@ -296,6 +299,9 @@ class HTTPTestCase(unittest.TestCase):
         # wait on the server thread to terminate
         self.evt.wait()
 
+        # disable traceback reporting
+        SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header = False
+
     def test_simple1(self):
         p = xmlrpclib.ServerProxy('http://localhost:%d' % PORT)
         self.assertEqual(p.pow(6,8), 6**8)
@@ -331,6 +337,82 @@ class HTTPTestCase(unittest.TestCase):
         self.assertEqual(div_result, 127//42)
 
 
+# This is a contrived way to make a failure occur on the server side
+# in order to test the _send_traceback_header flag on the server
+class FailingMessageClass(mimetools.Message):
+    def __getitem__(self, key):
+        key = key.lower()
+        if key == 'content-length':
+            return 'I am broken'
+        return mimetools.Message.__getitem__(self, key)
+
+
+class FailingServerTestCase(unittest.TestCase):
+    def setUp(self):
+        self.evt = threading.Event()
+        # start server thread to handle requests
+        serv_args = (self.evt, 2)
+        threading.Thread(target=http_server, args=serv_args).start()
+
+        # wait for port to be assigned to server
+        n = 1000
+        while n > 0 and PORT is None:
+            time.sleep(0.001)
+            n -= 1
+
+        time.sleep(0.5)
+
+    def tearDown(self):
+        # wait on the server thread to terminate
+        self.evt.wait()
+        # reset flag
+        SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header = False
+        # reset message class
+        SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.MessageClass = mimetools.Message
+
+    def test_basic(self):
+        # check that flag is false by default
+        flagval = SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header
+        self.assertEqual(flagval, False)
+
+        # test a call that won't fail just as a smoke test
+        p = xmlrpclib.ServerProxy('http://localhost:%d' % PORT)
+        self.assertEqual(p.pow(6,8), 6**8)
+
+    def test_fail_no_info(self):
+        # use the broken message class
+        SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.MessageClass = FailingMessageClass
+
+        try:
+            p = xmlrpclib.ServerProxy('http://localhost:%d' % PORT)
+            p.pow(6,8)
+        except xmlrpclib.ProtocolError as e:
+            # The two server-side error headers shouldn't be sent back in this case
+            self.assertTrue(e.headers.get("X-exception") is None)
+            self.assertTrue(e.headers.get("X-traceback") is None)
+        else:
+            self.fail('ProtocolError not raised')
+
+    def test_fail_with_info(self):
+        # use the broken message class
+        SimpleXMLRPCServer.SimpleXMLRPCRequestHandler.MessageClass = FailingMessageClass
+
+        # Check that errors in the server send back exception/traceback
+        # info when flag is set
+        SimpleXMLRPCServer.SimpleXMLRPCServer._send_traceback_header = True
+
+        try:
+            p = xmlrpclib.ServerProxy('http://localhost:%d' % PORT)
+            p.pow(6,8)
+        except xmlrpclib.ProtocolError as e:
+            # We should get error info in the response
+            expected_err = "invalid literal for int() with base 10: 'I am broken'"
+            self.assertEqual(e.headers.get("x-exception"), expected_err)
+            self.assertTrue(e.headers.get("x-traceback") is not None)
+        else:
+            self.fail('ProtocolError not raised')
+
+
 def test_main():
     xmlrpc_tests = [XMLRPCTestCase, HelperTestCase, DateTimeTestCase,
          BinaryTestCase, FaultTestCase]
@@ -340,7 +422,8 @@ def test_main():
     # run on Windows. This only happens on the first test to run, but it
     # fails every time and so these tests are skipped on win32 platforms.
     if sys.platform != 'win32':
-        xmlrpc_tests.append(HTTPTestCase)
+        xmlrpc_tests.append(SimpleServerTestCase)
+        xmlrpc_tests.append(FailingServerTestCase)
 
     test_support.run_unittest(*xmlrpc_tests)
 
diff --git a/Lib/threading.py b/Lib/threading.py
index cae2f77..4ba7252 100644
--- a/Lib/threading.py
+++ b/Lib/threading.py
@@ -445,6 +445,26 @@ class Thread(_Verbose):
             self.__target(*self.__args, **self.__kwargs)
 
     def __bootstrap(self):
+        # Wrapper around the real bootstrap code that ignores
+        # exceptions during interpreter cleanup.  Those typically
+        # happen when a daemon thread wakes up at an unfortunate
+        # moment, finds the world around it destroyed, and raises some
+        # random exception *** while trying to report the exception in
+        # __bootstrap_inner() below ***.  Those random exceptions
+        # don't help anybody, and they confuse users, so we suppress
+        # them.  We suppress them only when it appears that the world
+        # indeed has already been destroyed, so that exceptions in
+        # __bootstrap_inner() during normal business hours are properly
+        # reported.  Also, we only suppress them for daemonic threads;
+        # if a non-daemonic encounters this, something else is wrong.
+        try:
+            self.__bootstrap_inner()
+        except:
+            if self.__daemonic and _sys is None:
+                return
+            raise
+
+    def __bootstrap_inner(self):
         try:
             self.__started = True
             _active_limbo_lock.acquire()