merge heads

4 files changed, 249 insertions, 214 deletions

diff --git a/Lib/gzip.py b/Lib/gzip.py
index 5bcfe61..b6656a9 100644
--- a/Lib/gzip.py
+++ b/Lib/gzip.py
@@ -413,8 +413,10 @@ class GzipFile(io.BufferedIOBase):
             if self.fileobj is None:
                 return b''
             try:
-                # 1024 is the same buffering heuristic used in read()
-                self._read(max(n, 1024))
+                # Ensure that we don't return b"" if we haven't reached EOF.
+                while self.extrasize == 0:
+                    # 1024 is the same buffering heuristic used in read()
+                    self._read(max(n, 1024))
             except EOFError:
                 pass
         offset = self.offset - self.extrastart
diff --git a/Lib/ipaddress.py b/Lib/ipaddress.py
index c6eea7f..c9338c5 100644
--- a/Lib/ipaddress.py
+++ b/Lib/ipaddress.py
@@ -497,6 +497,7 @@ class _IPAddressBase(_TotalOrderingMixin):
             prefixlen = self._prefixlen
         return self._string_from_ip_int(self._ip_int_from_prefix(prefixlen))
 
+
 class _BaseAddress(_IPAddressBase):
 
     """A generic IP object.
@@ -511,9 +512,6 @@ class _BaseAddress(_IPAddressBase):
             and '/' in str(address)):
             raise AddressValueError("Unexpected '/' in %r" % address)
 
-    def __index__(self):
-        return self._ip
-
     def __int__(self):
         return self._ip
 
@@ -571,12 +569,6 @@ class _BaseNetwork(_IPAddressBase):
     def __init__(self, address):
         self._cache = {}
 
-    def __index__(self):
-        return int(self.network_address) ^ self.prefixlen
-
-    def __int__(self):
-        return int(self.network_address)
-
     def __repr__(self):
         return '%s(%r)' % (self.__class__.__name__, str(self))
 
@@ -943,6 +935,76 @@ class _BaseNetwork(_IPAddressBase):
                                      strict=False)
         return t.__class__('%s/%d' % (t.network_address, t.prefixlen))
 
+    @property
+    def is_multicast(self):
+        """Test if the address is reserved for multicast use.
+
+        Returns:
+            A boolean, True if the address is a multicast address.
+            See RFC 2373 2.7 for details.
+
+        """
+        return (self.network_address.is_multicast and
+                self.broadcast_address.is_multicast)
+
+    @property
+    def is_reserved(self):
+        """Test if the address is otherwise IETF reserved.
+
+        Returns:
+            A boolean, True if the address is within one of the
+            reserved IPv6 Network ranges.
+
+        """
+        return (self.network_address.is_reserved and
+                self.broadcast_address.is_reserved)
+
+    @property
+    def is_link_local(self):
+        """Test if the address is reserved for link-local.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 4291.
+
+        """
+        return (self.network_address.is_link_local and
+                self.broadcast_address.is_link_local)
+
+    @property
+    def is_private(self):
+        """Test if this address is allocated for private networks.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 4193.
+
+        """
+        return (self.network_address.is_private and
+                self.broadcast_address.is_private)
+
+    @property
+    def is_unspecified(self):
+        """Test if the address is unspecified.
+
+        Returns:
+            A boolean, True if this is the unspecified address as defined in
+            RFC 2373 2.5.2.
+
+        """
+        return (self.network_address.is_unspecified and
+                self.broadcast_address.is_unspecified)
+
+    @property
+    def is_loopback(self):
+        """Test if the address is a loopback address.
+
+        Returns:
+            A boolean, True if the address is a loopback address as defined in
+            RFC 2373 2.5.3.
+
+        """
+        return (self.network_address.is_loopback and
+                self.broadcast_address.is_loopback)
+
 
 class _BaseV4:
 
@@ -1100,6 +1162,52 @@ class _BaseV4:
     def version(self):
         return self._version
 
+
+class IPv4Address(_BaseV4, _BaseAddress):
+
+    """Represent and manipulate single IPv4 Addresses."""
+
+    def __init__(self, address):
+
+        """
+        Args:
+            address: A string or integer representing the IP
+
+              Additionally, an integer can be passed, so
+              IPv4Address('192.0.2.1') == IPv4Address(3221225985).
+              or, more generally
+              IPv4Address(int(IPv4Address('192.0.2.1'))) ==
+                IPv4Address('192.0.2.1')
+
+        Raises:
+            AddressValueError: If ipaddress isn't a valid IPv4 address.
+
+        """
+        _BaseAddress.__init__(self, address)
+        _BaseV4.__init__(self, address)
+
+        # Efficient constructor from integer.
+        if isinstance(address, int):
+            self._check_int_address(address)
+            self._ip = address
+            return
+
+        # Constructing from a packed address
+        if isinstance(address, bytes):
+            self._check_packed_address(address, 4)
+            self._ip = struct.unpack('!I', address)[0]
+            return
+
+        # Assume input argument to be string or any object representation
+        # which converts into a formatted IP string.
+        addr_str = str(address)
+        self._ip = self._ip_int_from_string(addr_str)
+
+    @property
+    def packed(self):
+        """The binary representation of this address."""
+        return v4_int_to_packed(self._ip)
+
     @property
     def is_reserved(self):
         """Test if the address is otherwise IETF reserved.
@@ -1110,10 +1218,7 @@ class _BaseV4:
 
         """
         reserved_network = IPv4Network('240.0.0.0/4')
-        if isinstance(self, _BaseAddress):
-            return self in reserved_network
-        return (self.network_address in reserved_network and
-                self.broadcast_address in reserved_network)
+        return self in reserved_network
 
     @property
     def is_private(self):
@@ -1126,16 +1231,9 @@ class _BaseV4:
         private_10 = IPv4Network('10.0.0.0/8')
         private_172 = IPv4Network('172.16.0.0/12')
         private_192 = IPv4Network('192.168.0.0/16')
-        if isinstance(self, _BaseAddress):
-            return (self in private_10 or self in private_172 or
-                    self in private_192)
-        else:
-            return ((self.network_address in private_10 and
-                     self.broadcast_address in private_10) or
-                    (self.network_address in private_172 and
-                     self.broadcast_address in private_172) or
-                    (self.network_address in private_192 and
-                     self.broadcast_address in private_192))
+        return (self in private_10 or
+                self in private_172 or
+                self in private_192)
 
     @property
     def is_multicast(self):
@@ -1147,10 +1245,7 @@ class _BaseV4:
 
         """
         multicast_network = IPv4Network('224.0.0.0/4')
-        if isinstance(self, _BaseAddress):
-            return self in IPv4Network('224.0.0.0/4')
-        return (self.network_address in multicast_network and
-                self.broadcast_address in multicast_network)
+        return self in multicast_network
 
     @property
     def is_unspecified(self):
@@ -1162,10 +1257,7 @@ class _BaseV4:
 
         """
         unspecified_address = IPv4Address('0.0.0.0')
-        if isinstance(self, _BaseAddress):
-            return self == unspecified_address
-        return (self.network_address == self.broadcast_address ==
-                unspecified_address)
+        return self == unspecified_address
 
     @property
     def is_loopback(self):
@@ -1175,12 +1267,8 @@ class _BaseV4:
             A boolean, True if the address is a loopback per RFC 3330.
 
         """
-        loopback_address = IPv4Network('127.0.0.0/8')
-        if isinstance(self, _BaseAddress):
-            return self in loopback_address
-
-        return (self.network_address in loopback_address and
-                self.broadcast_address in loopback_address)
+        loopback_network = IPv4Network('127.0.0.0/8')
+        return self in loopback_network
 
     @property
     def is_link_local(self):
@@ -1191,56 +1279,7 @@ class _BaseV4:
 
         """
         linklocal_network = IPv4Network('169.254.0.0/16')
-        if isinstance(self, _BaseAddress):
-            return self in linklocal_network
-        return (self.network_address in linklocal_network and
-                self.broadcast_address in linklocal_network)
-
-
-class IPv4Address(_BaseV4, _BaseAddress):
-
-    """Represent and manipulate single IPv4 Addresses."""
-
-    def __init__(self, address):
-
-        """
-        Args:
-            address: A string or integer representing the IP
-
-              Additionally, an integer can be passed, so
-              IPv4Address('192.0.2.1') == IPv4Address(3221225985).
-              or, more generally
-              IPv4Address(int(IPv4Address('192.0.2.1'))) ==
-                IPv4Address('192.0.2.1')
-
-        Raises:
-            AddressValueError: If ipaddress isn't a valid IPv4 address.
-
-        """
-        _BaseAddress.__init__(self, address)
-        _BaseV4.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, int):
-            self._check_int_address(address)
-            self._ip = address
-            return
-
-        # Constructing from a packed address
-        if isinstance(address, bytes):
-            self._check_packed_address(address, 4)
-            self._ip = struct.unpack('!I', address)[0]
-            return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP string.
-        addr_str = str(address)
-        self._ip = self._ip_int_from_string(addr_str)
-
-    @property
-    def packed(self):
-        """The binary representation of this address."""
-        return v4_int_to_packed(self._ip)
+        return self in linklocal_network
 
 
 class IPv4Interface(IPv4Address):
@@ -1292,10 +1331,6 @@ class IPv4Interface(IPv4Address):
         return self._ip ^ self._prefixlen ^ int(self.network.network_address)
 
     @property
-    def prefixlen(self):
-        return self._prefixlen
-
-    @property
     def ip(self):
         return IPv4Address(self._ip)
 
@@ -1669,7 +1704,7 @@ class _BaseV6:
         hex_str = '%032x' % ip_int
         parts = [hex_str[x:x+4] for x in range(0, 32, 4)]
         if isinstance(self, (_BaseNetwork, IPv6Interface)):
-            return '%s/%d' % (':'.join(parts), self.prefixlen)
+            return '%s/%d' % (':'.join(parts), self._prefixlen)
         return ':'.join(parts)
 
     @property
@@ -1680,6 +1715,54 @@ class _BaseV6:
     def version(self):
         return self._version
 
+
+class IPv6Address(_BaseV6, _BaseAddress):
+
+    """Represent and manipulate single IPv6 Addresses."""
+
+    def __init__(self, address):
+        """Instantiate a new IPv6 address object.
+
+        Args:
+            address: A string or integer representing the IP
+
+              Additionally, an integer can be passed, so
+              IPv6Address('2001:db8::') ==
+                IPv6Address(42540766411282592856903984951653826560)
+              or, more generally
+              IPv6Address(int(IPv6Address('2001:db8::'))) ==
+                IPv6Address('2001:db8::')
+
+        Raises:
+            AddressValueError: If address isn't a valid IPv6 address.
+
+        """
+        _BaseAddress.__init__(self, address)
+        _BaseV6.__init__(self, address)
+
+        # Efficient constructor from integer.
+        if isinstance(address, int):
+            self._check_int_address(address)
+            self._ip = address
+            return
+
+        # Constructing from a packed address
+        if isinstance(address, bytes):
+            self._check_packed_address(address, 16)
+            tmp = struct.unpack('!QQ', address)
+            self._ip = (tmp[0] << 64) | tmp[1]
+            return
+
+        # Assume input argument to be string or any object representation
+        # which converts into a formatted IP string.
+        addr_str = str(address)
+        self._ip = self._ip_int_from_string(addr_str)
+
+    @property
+    def packed(self):
+        """The binary representation of this address."""
+        return v6_int_to_packed(self._ip)
+
     @property
     def is_multicast(self):
         """Test if the address is reserved for multicast use.
@@ -1690,10 +1773,7 @@ class _BaseV6:
 
         """
         multicast_network = IPv6Network('ff00::/8')
-        if isinstance(self, _BaseAddress):
-            return self in multicast_network
-        return (self.network_address in multicast_network and
-                self.broadcast_address in multicast_network)
+        return self in multicast_network
 
     @property
     def is_reserved(self):
@@ -1713,10 +1793,7 @@ class _BaseV6:
                              IPv6Network('F000::/5'), IPv6Network('F800::/6'),
                              IPv6Network('FE00::/9')]
 
-        if isinstance(self, _BaseAddress):
-            return any(self in x for x in reserved_networks)
-        return any(self.network_address in x and self.broadcast_address in x
-                   for x in reserved_networks)
+        return any(self in x for x in reserved_networks)
 
     @property
     def is_link_local(self):
@@ -1727,10 +1804,7 @@ class _BaseV6:
 
         """
         linklocal_network = IPv6Network('fe80::/10')
-        if isinstance(self, _BaseAddress):
-            return self in linklocal_network
-        return (self.network_address in linklocal_network and
-                self.broadcast_address in linklocal_network)
+        return self in linklocal_network
 
     @property
     def is_site_local(self):
@@ -1745,10 +1819,7 @@ class _BaseV6:
 
         """
         sitelocal_network = IPv6Network('fec0::/10')
-        if isinstance(self, _BaseAddress):
-            return self in sitelocal_network
-        return (self.network_address in sitelocal_network and
-                self.broadcast_address in sitelocal_network)
+        return self in sitelocal_network
 
     @property
     def is_private(self):
@@ -1759,10 +1830,29 @@ class _BaseV6:
 
         """
         private_network = IPv6Network('fc00::/7')
-        if isinstance(self, _BaseAddress):
-            return self in private_network
-        return (self.network_address in private_network and
-                self.broadcast_address in private_network)
+        return self in private_network
+
+    @property
+    def is_unspecified(self):
+        """Test if the address is unspecified.
+
+        Returns:
+            A boolean, True if this is the unspecified address as defined in
+            RFC 2373 2.5.2.
+
+        """
+        return self._ip == 0
+
+    @property
+    def is_loopback(self):
+        """Test if the address is a loopback address.
+
+        Returns:
+            A boolean, True if the address is a loopback address as defined in
+            RFC 2373 2.5.3.
+
+        """
+        return self._ip == 1
 
     @property
     def ipv4_mapped(self):
@@ -1805,85 +1895,6 @@ class _BaseV6:
             return None
         return IPv4Address((self._ip >> 80) & 0xFFFFFFFF)
 
-    @property
-    def is_unspecified(self):
-        """Test if the address is unspecified.
-
-        Returns:
-            A boolean, True if this is the unspecified address as defined in
-            RFC 2373 2.5.2.
-
-        """
-        if isinstance(self, (IPv6Network, IPv6Interface)):
-            return int(self.network_address) == 0 and getattr(
-                self, '_prefixlen', 128) == 128
-        return self._ip == 0
-
-    @property
-    def is_loopback(self):
-        """Test if the address is a loopback address.
-
-        Returns:
-            A boolean, True if the address is a loopback address as defined in
-            RFC 2373 2.5.3.
-
-        """
-        if isinstance(self, IPv6Network):
-            return int(self) == 1 and getattr(
-                self, '_prefixlen', 128) == 128
-        elif isinstance(self, IPv6Interface):
-            return int(self.network.network_address) == 1 and getattr(
-                self, '_prefixlen', 128) == 128
-        return self._ip == 1
-
-
-class IPv6Address(_BaseV6, _BaseAddress):
-
-    """Represent and manipulate single IPv6 Addresses."""
-
-    def __init__(self, address):
-        """Instantiate a new IPv6 address object.
-
-        Args:
-            address: A string or integer representing the IP
-
-              Additionally, an integer can be passed, so
-              IPv6Address('2001:db8::') ==
-                IPv6Address(42540766411282592856903984951653826560)
-              or, more generally
-              IPv6Address(int(IPv6Address('2001:db8::'))) ==
-                IPv6Address('2001:db8::')
-
-        Raises:
-            AddressValueError: If address isn't a valid IPv6 address.
-
-        """
-        _BaseAddress.__init__(self, address)
-        _BaseV6.__init__(self, address)
-
-        # Efficient constructor from integer.
-        if isinstance(address, int):
-            self._check_int_address(address)
-            self._ip = address
-            return
-
-        # Constructing from a packed address
-        if isinstance(address, bytes):
-            self._check_packed_address(address, 16)
-            tmp = struct.unpack('!QQ', address)
-            self._ip = (tmp[0] << 64) | tmp[1]
-            return
-
-        # Assume input argument to be string or any object representation
-        # which converts into a formatted IP string.
-        addr_str = str(address)
-        self._ip = self._ip_int_from_string(addr_str)
-
-    @property
-    def packed(self):
-        """The binary representation of this address."""
-        return v6_int_to_packed(self._ip)
-
 
 class IPv6Interface(IPv6Address):
 
@@ -1932,10 +1943,6 @@ class IPv6Interface(IPv6Address):
         return self._ip ^ self._prefixlen ^ int(self.network.network_address)
 
     @property
-    def prefixlen(self):
-        return self._prefixlen
-
-    @property
     def ip(self):
         return IPv6Address(self._ip)
 
@@ -1952,6 +1959,14 @@ class IPv6Interface(IPv6Address):
         return '%s/%s' % (self._string_from_ip_int(self._ip),
                           self.hostmask)
 
+    @property
+    def is_unspecified(self):
+        return self._ip == 0 and self.network.is_unspecified
+
+    @property
+    def is_loopback(self):
+        return self._ip == 1 and self.network.is_loopback
+
 
 class IPv6Network(_BaseV6, _BaseNetwork):
 
@@ -2060,3 +2075,18 @@ class IPv6Network(_BaseV6, _BaseNetwork):
         except ValueError:
             return False
         return 0 <= prefixlen <= self._max_prefixlen
+
+    @property
+    def is_site_local(self):
+        """Test if the address is reserved for site-local.
+
+        Note that the site-local address space has been deprecated by RFC 3879.
+        Use is_private to test if this address is in the space of unique local
+        addresses as defined by RFC 4193.
+
+        Returns:
+            A boolean, True if the address is reserved per RFC 3513 2.5.6.
+
+        """
+        return (self.network_address.is_site_local and
+                self.broadcast_address.is_site_local)
diff --git a/Lib/test/test_httpservers.py b/Lib/test/test_httpservers.py
index 1430ff2..171361f 100644
--- a/Lib/test/test_httpservers.py
+++ b/Lib/test/test_httpservers.py
@@ -313,6 +313,8 @@ class CGIHTTPServerTestCase(BaseTestCase):
     class request_handler(NoLogRequestHandler, CGIHTTPRequestHandler):
         pass
 
+    linesep = os.linesep.encode('ascii')
+
     def setUp(self):
         BaseTestCase.setUp(self)
         self.cwd = os.getcwd()
@@ -410,7 +412,7 @@ class CGIHTTPServerTestCase(BaseTestCase):
 
     def test_headers_and_content(self):
         res = self.request('/cgi-bin/file1.py')
-        self.assertEqual((b'Hello World\n', 'text/html', 200),
+        self.assertEqual((b'Hello World' + self.linesep, 'text/html', 200),
             (res.read(), res.getheader('Content-type'), res.status))
 
     def test_post(self):
@@ -419,7 +421,7 @@ class CGIHTTPServerTestCase(BaseTestCase):
         headers = {'Content-type' : 'application/x-www-form-urlencoded'}
         res = self.request('/cgi-bin/file2.py', 'POST', params, headers)
 
-        self.assertEqual(res.read(), b'1, python, 123456\n')
+        self.assertEqual(res.read(), b'1, python, 123456' + self.linesep)
 
     def test_invaliduri(self):
         res = self.request('/cgi-bin/invalid')
@@ -430,20 +432,20 @@ class CGIHTTPServerTestCase(BaseTestCase):
         headers = {b'Authorization' : b'Basic ' +
                    base64.b64encode(b'username:pass')}
         res = self.request('/cgi-bin/file1.py', 'GET', headers=headers)
-        self.assertEqual((b'Hello World\n', 'text/html', 200),
+        self.assertEqual((b'Hello World' + self.linesep, 'text/html', 200),
                 (res.read(), res.getheader('Content-type'), res.status))
 
     def test_no_leading_slash(self):
         # http://bugs.python.org/issue2254
         res = self.request('cgi-bin/file1.py')
-        self.assertEqual((b'Hello World\n', 'text/html', 200),
+        self.assertEqual((b'Hello World' + self.linesep, 'text/html', 200),
              (res.read(), res.getheader('Content-type'), res.status))
 
     def test_os_environ_is_not_altered(self):
         signature = "Test CGI Server"
         os.environ['SERVER_SOFTWARE'] = signature
         res = self.request('/cgi-bin/file1.py')
-        self.assertEqual((b'Hello World\n', 'text/html', 200),
+        self.assertEqual((b'Hello World' + self.linesep, 'text/html', 200),
                 (res.read(), res.getheader('Content-type'), res.status))
         self.assertEqual(os.environ['SERVER_SOFTWARE'], signature)
 
diff --git a/Lib/test/test_ipaddress.py b/Lib/test/test_ipaddress.py
index 619fa44..09b4a5e 100644
--- a/Lib/test/test_ipaddress.py
+++ b/Lib/test/test_ipaddress.py
@@ -7,6 +7,7 @@
 import unittest
 import re
 import contextlib
+import operator
 import ipaddress
 
 class BaseTestCase(unittest.TestCase):
@@ -72,6 +73,14 @@ class CommonTestMixin:
         with self.assertAddressError(re.escape(repr("1.0"))):
             self.factory(1.0)
 
+    def test_not_an_index_issue15559(self):
+        # Implementing __index__ makes for a very nasty interaction with the
+        # bytes constructor. Thus, we disallow implicit use as an integer
+        self.assertRaises(TypeError, operator.index, self.factory(1))
+        self.assertRaises(TypeError, hex, self.factory(1))
+        self.assertRaises(TypeError, bytes, self.factory(1))
+
+
 class CommonTestMixin_v4(CommonTestMixin):
 
     def test_leading_zeros(self):
@@ -599,7 +608,6 @@ class IpaddrUnitTest(unittest.TestCase):
         self.assertEqual(first, last)
         self.assertEqual(128, ipaddress._count_righthand_zero_bits(0, 128))
         self.assertEqual("IPv4Network('1.2.3.0/24')", repr(self.ipv4_network))
-        self.assertEqual('0x1020318', hex(self.ipv4_network))
 
     def testMissingAddressVersion(self):
         class Broken(ipaddress._BaseAddress):
@@ -639,8 +647,8 @@ class IpaddrUnitTest(unittest.TestCase):
 
         ipv4 = ipaddress.ip_network('1.2.3.4')
         ipv6 = ipaddress.ip_network('2001:658:22a:cafe:200:0:0:1')
-        self.assertEqual(ipv4, ipaddress.ip_network(int(ipv4)))
-        self.assertEqual(ipv6, ipaddress.ip_network(int(ipv6)))
+        self.assertEqual(ipv4, ipaddress.ip_network(int(ipv4.network_address)))
+        self.assertEqual(ipv6, ipaddress.ip_network(int(ipv6.network_address)))
 
         v6_int = 42540616829182469433547762482097946625
         self.assertEqual(self.ipv6_interface._ip,
@@ -723,8 +731,8 @@ class IpaddrUnitTest(unittest.TestCase):
                          '2001:658:22a:cafe:ffff:ffff:ffff:ffff')
 
     def testGetPrefixlen(self):
-        self.assertEqual(self.ipv4_interface.prefixlen, 24)
-        self.assertEqual(self.ipv6_interface.prefixlen, 64)
+        self.assertEqual(self.ipv4_interface.network.prefixlen, 24)
+        self.assertEqual(self.ipv6_interface.network.prefixlen, 64)
 
     def testGetSupernet(self):
         self.assertEqual(self.ipv4_network.supernet().prefixlen, 23)
@@ -1545,13 +1553,6 @@ class IpaddrUnitTest(unittest.TestCase):
         self.assertEqual(42540616829182469433547762482097946625,
                          int(self.ipv6_address))
 
-    def testHexRepresentation(self):
-        self.assertEqual(hex(0x1020304),
-                         hex(self.ipv4_address))
-
-        self.assertEqual(hex(0x20010658022ACAFE0200000000000001),
-                         hex(self.ipv6_address))
-
     def testForceVersion(self):
         self.assertEqual(ipaddress.ip_network(1).version, 4)
         self.assertEqual(ipaddress.IPv6Network(1).version, 6)