Added an optional timeout parameter to function urllib2.urlopen,

with tests in test_urllib2net.py (must have network resource
enabled to execute them). Also modified test_urllib2.py because
testing mock classes must take it into acount. Docs are also
updated.

5 files changed, 78 insertions, 16 deletions

diff --git a/Doc/lib/liburllib2.tex b/Doc/lib/liburllib2.tex
index 0df7385..5547b15 100644
--- a/Doc/lib/liburllib2.tex
+++ b/Doc/lib/liburllib2.tex
@@ -14,7 +14,7 @@ authentication, redirections, cookies and more.
 
 The \module{urllib2} module defines the following functions:
 
-\begin{funcdesc}{urlopen}{url\optional{, data}}
+\begin{funcdesc}{urlopen}{url\optional{, data}\optional{, timeout}}
 Open the URL \var{url}, which can be either a string or a \class{Request}
 object.
 
@@ -27,6 +27,11 @@ parameter is provided.  \var{data} should be a buffer in the standard
 \function{urllib.urlencode()} function takes a mapping or sequence of
 2-tuples and returns a string in this format.
 
+The optional \var{timeout} parameter specifies a timeout in seconds for the
+connection attempt (if not specified, or passed as None, the global default
+timeout setting will be used). This actually only work for HTTP, HTTPS, FTP
+and FTPS connections.
+
 This function returns a file-like object with two additional methods:
 
 \begin{itemize}
@@ -351,12 +356,17 @@ that HTTP errors are a special case).
 \end{itemize}
 \end{methoddesc}
 
-\begin{methoddesc}[OpenerDirector]{open}{url\optional{, data}}
+\begin{methoddesc}[OpenerDirector]{open}{url\optional{, data}{\optional{, timeout}}}
 Open the given \var{url} (which can be a request object or a string),
 optionally passing the given \var{data}.
 Arguments, return values and exceptions raised are the same as those
 of \function{urlopen()} (which simply calls the \method{open()} method
-on the currently installed global \class{OpenerDirector}).
+on the currently installed global \class{OpenerDirector}).  The optional
+\var{timeout} parameter specifies a timeout in seconds for the connection 
+attempt (if not specified, or passed as None, the global default timeout 
+setting will be used; this actually only work for HTTP, HTTPS, FTP
+and FTPS connections).
+
 \end{methoddesc}
 
 \begin{methoddesc}[OpenerDirector]{error}{proto\optional{,
diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py
index 1a2986a..c9dcf5e 100644
--- a/Lib/test/test_urllib2.py
+++ b/Lib/test/test_urllib2.py
@@ -545,7 +545,7 @@ class HandlerTests(unittest.TestCase):
 
         class NullFTPHandler(urllib2.FTPHandler):
             def __init__(self, data): self.data = data
-            def connect_ftp(self, user, passwd, host, port, dirs):
+            def connect_ftp(self, user, passwd, host, port, dirs, timeout=None):
                 self.user, self.passwd = user, passwd
                 self.host, self.port = host, port
                 self.dirs = dirs
@@ -568,7 +568,9 @@ class HandlerTests(unittest.TestCase):
              "localhost", ftplib.FTP_PORT, "A",
              [], "baz.gif", None),  # XXX really this should guess image/gif
             ]:
-            r = h.ftp_open(Request(url))
+            req = Request(url)
+            req.timeout = None
+            r = h.ftp_open(req)
             # ftp authentication not yet implemented by FTPHandler
             self.assert_(h.user == h.passwd == "")
             self.assertEqual(h.host, socket.gethostbyname(host))
@@ -683,8 +685,9 @@ class HandlerTests(unittest.TestCase):
                 self.req_headers = []
                 self.data = None
                 self.raise_on_endheaders = False
-            def __call__(self, host):
+            def __call__(self, host, timeout=None):
                 self.host = host
+                self.timeout = timeout
                 return self
             def set_debuglevel(self, level):
                 self.level = level
@@ -707,6 +710,7 @@ class HandlerTests(unittest.TestCase):
         url = "http://example.com/"
         for method, data in [("GET", None), ("POST", "blah")]:
             req = Request(url, data, {"Foo": "bar"})
+            req.timeout = None
             req.add_unredirected_header("Spam", "eggs")
             http = MockHTTPClass()
             r = h.do_open(http, req)
diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py
index e76301e..c363140 100644
--- a/Lib/test/test_urllib2net.py
+++ b/Lib/test/test_urllib2net.py
@@ -267,6 +267,49 @@ class OtherNetworkTests(unittest.TestCase):
 
         return handlers
 
+class TimeoutTest(unittest.TestCase):
+    def test_http_basic(self):
+        u = urllib2.urlopen("http://www.python.org")
+        self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None)
+
+    def test_http_NoneWithdefault(self):
+        prev = socket.getdefaulttimeout()
+        socket.setdefaulttimeout(60)
+        try:
+            u = urllib2.urlopen("http://www.python.org", timeout=None)
+            self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 60)
+        finally:
+            socket.setdefaulttimeout(prev)
+
+    def test_http_Value(self):
+        u = urllib2.urlopen("http://www.python.org", timeout=120)
+        self.assertEqual(u.fp._sock.fp._sock.gettimeout(), 120)
+
+    def test_http_NoneNodefault(self):
+        u = urllib2.urlopen("http://www.python.org", timeout=None)
+        self.assertTrue(u.fp._sock.fp._sock.gettimeout() is None)
+
+    def test_ftp_basic(self):
+        u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/")
+        self.assertTrue(u.fp.fp._sock.gettimeout() is None)
+
+    def test_ftp_NoneWithdefault(self):
+        prev = socket.getdefaulttimeout()
+        socket.setdefaulttimeout(60)
+        try:
+            u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=None)
+            self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
+        finally:
+            socket.setdefaulttimeout(prev)
+
+    def test_ftp_NoneNodefault(self):
+        u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=None)
+        self.assertTrue(u.fp.fp._sock.gettimeout() is None)
+
+    def test_ftp_Value(self):
+        u = urllib2.urlopen("ftp://ftp.mirror.nl/pub/mirror/gnu/", timeout=60)
+        self.assertEqual(u.fp.fp._sock.gettimeout(), 60)
+
 
 def test_main():
     test_support.requires("network")
@@ -275,6 +318,7 @@ def test_main():
                               AuthTests,
                               OtherNetworkTests,
                               CloseSocketTest,
+                              TimeoutTest,
                               )
 
 if __name__ == "__main__":
diff --git a/Lib/urllib2.py b/Lib/urllib2.py
index fe32b2d..a1badb7 100644
--- a/Lib/urllib2.py
+++ b/Lib/urllib2.py
@@ -117,11 +117,11 @@ from urllib import localhost, url2pathname, getproxies
 __version__ = sys.version[:3]
 
 _opener = None
-def urlopen(url, data=None):
+def urlopen(url, data=None, timeout=None):
     global _opener
     if _opener is None:
         _opener = build_opener()
-    return _opener.open(url, data)
+    return _opener.open(url, data, timeout)
 
 def install_opener(opener):
     global _opener
@@ -355,7 +355,7 @@ class OpenerDirector:
             if result is not None:
                 return result
 
-    def open(self, fullurl, data=None):
+    def open(self, fullurl, data=None, timeout=None):
         # accept a URL or a Request object
         if isinstance(fullurl, basestring):
             req = Request(fullurl, data)
@@ -364,6 +364,7 @@ class OpenerDirector:
             if data is not None:
                 req.add_data(data)
 
+        req.timeout = timeout
         protocol = req.get_type()
 
         # pre-process request
@@ -1057,7 +1058,7 @@ class AbstractHTTPHandler(BaseHandler):
         if not host:
             raise URLError('no host given')
 
-        h = http_class(host) # will parse host:port
+        h = http_class(host, timeout=req.timeout) # will parse host:port
         h.set_debuglevel(self._debuglevel)
 
         headers = dict(req.headers)
@@ -1269,7 +1270,7 @@ class FTPHandler(BaseHandler):
         if dirs and not dirs[0]:
             dirs = dirs[1:]
         try:
-            fw = self.connect_ftp(user, passwd, host, port, dirs)
+            fw = self.connect_ftp(user, passwd, host, port, dirs, req.timeout)
             type = file and 'I' or 'D'
             for attr in attrs:
                 attr, value = splitvalue(attr)
@@ -1289,8 +1290,8 @@ class FTPHandler(BaseHandler):
         except ftplib.all_errors, msg:
             raise IOError, ('ftp error', msg), sys.exc_info()[2]
 
-    def connect_ftp(self, user, passwd, host, port, dirs):
-        fw = ftpwrapper(user, passwd, host, port, dirs)
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        fw = ftpwrapper(user, passwd, host, port, dirs, timeout)
 ##        fw.ftp.set_debuglevel(1)
         return fw
 
@@ -1310,12 +1311,12 @@ class CacheFTPHandler(FTPHandler):
     def setMaxConns(self, m):
         self.max_conns = m
 
-    def connect_ftp(self, user, passwd, host, port, dirs):
-        key = user, host, port, '/'.join(dirs)
+    def connect_ftp(self, user, passwd, host, port, dirs, timeout):
+        key = user, host, port, '/'.join(dirs), timeout
         if key in self.cache:
             self.timeout[key] = time.time() + self.delay
         else:
-            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs)
+            self.cache[key] = ftpwrapper(user, passwd, host, port, dirs, timeout)
             self.timeout[key] = time.time() + self.delay
         self.check_cache()
         return self.cache[key]
diff --git a/Misc/NEWS b/Misc/NEWS
index a627765..b786745 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -222,6 +222,9 @@ Core and builtins
 Library
 -------
 
+- The urlopen function of urllib2 now has an optional timeout parameter (note 
+  that it actually works with HTTP, HTTPS, FTP and FTPS connections).
+
 - In ftplib, the FTP.ntransfercmd method, when in passive mode, now uses
   the socket.create_connection function, using the timeout specified at
   connection time.