From e266f25cf1f9ad28409f4c11c40595fb25f555da Mon Sep 17 00:00:00 2001 From: Senthil Kumaran Date: Sun, 24 May 2009 09:14:50 +0000 Subject: Fixed Issue1424152, urllib2 fails with HTTPS over Proxy. --- Doc/library/httplib.rst | 6 ++++++ Lib/httplib.py | 29 +++++++++++++++++++++++++++++ Lib/test/test_urllib2.py | 15 +++++++++++++++ Lib/urllib2.py | 16 +++++++++++++--- Misc/NEWS | 3 +++ 5 files changed, 66 insertions(+), 3 deletions(-) diff --git a/Doc/library/httplib.rst b/Doc/library/httplib.rst index e48c95c..9a5acc1 100644 --- a/Doc/library/httplib.rst +++ b/Doc/library/httplib.rst @@ -428,6 +428,12 @@ HTTPConnection Objects debug level is ``0``, meaning no debugging output is printed. +.. method:: HTTPConnection.set_tunnel(host,port=None) + + Set the host and the port for HTTP Connect Tunnelling. Normally used when + it is required to do HTTPS Conection through a proxy server. + + .. method:: HTTPConnection.connect() Connect to the server specified when the object was created. diff --git a/Lib/httplib.py b/Lib/httplib.py index 2e749ea..6fc5733 100644 --- a/Lib/httplib.py +++ b/Lib/httplib.py @@ -662,11 +662,18 @@ class HTTPConnection: self.__response = None self.__state = _CS_IDLE self._method = None + self._tunnel_host = None + self._tunnel_port = None self._set_hostport(host, port) if strict is not None: self.strict = strict + def set_tunnel(self, host, port=None): + """ Sets up the host and the port for the HTTP CONNECT Tunnelling.""" + self._tunnel_host = host + self._tunnel_port = port + def _set_hostport(self, host, port): if port is None: i = host.rfind(':') @@ -687,11 +694,30 @@ class HTTPConnection: def set_debuglevel(self, level): self.debuglevel = level + def _tunnel(self): + self._set_hostport(self._tunnel_host, self._tunnel_port) + self.send("CONNECT %s:%d HTTP/1.0\r\n\r\n" % (self.host, self.port)) + response = self.response_class(self.sock, strict = self.strict, + method = self._method) + (version, code, message) = response._read_status() + + if code != 200: + self.close() + raise socket.error, "Tunnel connection failed: %d %s" % (code, + message.strip()) + while True: + line = response.fp.readline() + if line == '\r\n': break + + def connect(self): """Connect to the host and port specified in __init__.""" self.sock = socket.create_connection((self.host,self.port), self.timeout) + if self._tunnel_host: + self._tunnel() + def close(self): """Close the connection to the HTTP server.""" if self.sock: @@ -1101,6 +1127,9 @@ else: "Connect to a host on a given (SSL) port." sock = socket.create_connection((self.host, self.port), self.timeout) + if self._tunnel_host: + self.sock = sock + self._tunnel() self.sock = ssl.wrap_socket(sock, self.key_file, self.cert_file) __all__.append("HTTPSConnection") diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index ff164c2..9edd7c2 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -939,6 +939,21 @@ class HandlerTests(unittest.TestCase): self.assertEqual([(handlers[0], "http_open")], [tup[0:2] for tup in o.calls]) + def test_proxy_https(self): + o = OpenerDirector() + ph = urllib2.ProxyHandler(dict(https='proxy.example.com:3128')) + o.add_handler(ph) + meth_spec = [ + [("https_open","return response")] + ] + handlers = add_ordered_mock_handlers(o, meth_spec) + req = Request("https://www.example.com/") + self.assertEqual(req.get_host(), "www.example.com") + r = o.open(req) + self.assertEqual(req.get_host(), "proxy.example.com:3128") + self.assertEqual([(handlers[0], "https_open")], + [tup[0:2] for tup in o.calls]) + def test_basic_auth(self, quote_char='"'): opener = OpenerDirector() password_manager = MockPasswordManager() diff --git a/Lib/urllib2.py b/Lib/urllib2.py index a2a7e66..03aaf79 100644 --- a/Lib/urllib2.py +++ b/Lib/urllib2.py @@ -192,6 +192,7 @@ class Request: # self.__r_type is what's left after doing the splittype self.host = None self.port = None + self._tunnel_host = None self.data = data self.headers = {} for key, value in headers.items(): @@ -252,8 +253,13 @@ class Request: return self.__r_host def set_proxy(self, host, type): - self.host, self.type = host, type - self.__r_host = self.__original + if self.type == 'https' and not self._tunnel_host: + self._tunnel_host = self.host + else: + self.type = type + self.__r_host = self.__original + + self.host = host def has_proxy(self): return self.__r_host == self.__original @@ -700,7 +706,7 @@ class ProxyHandler(BaseHandler): req.add_header('Proxy-authorization', 'Basic ' + creds) hostport = unquote(hostport) req.set_proxy(hostport, proxy_type) - if orig_type == proxy_type: + if orig_type == proxy_type or orig_type == 'https': # let other handlers take care of it return None else: @@ -1098,6 +1104,10 @@ class AbstractHTTPHandler(BaseHandler): headers["Connection"] = "close" headers = dict( (name.title(), val) for name, val in headers.items()) + + if req._tunnel_host: + h.set_tunnel(req._tunnel_host) + try: h.request(req.get_method(), req.get_selector(), req.data, headers) try: diff --git a/Misc/NEWS b/Misc/NEWS index deb8a0b..96d5044 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -302,6 +302,9 @@ Core and Builtins Library ------- +- Issue #1424152: Fix for httplib, urllib2 to support SSL while working through + proxy. Original patch by Christopher Li, changes made by Senthil Kumaran. + - Issue #1983: Fix functions taking or returning a process identifier to use the dedicated C type ``pid_t`` instead of a C ``int``. Some platforms have a process identifier type wider than the standard C integer type. -- cgit v0.12