Upgrade to ctypes version 0.9.9.7.

Summary of changes:

- support for 'variable sized' data
- support for anonymous structure/union fields
- fix severe bug with certain arrays or structures containing more than 256 fields

8 files changed, 308 insertions, 30 deletions

diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py
index f2ddbaa..042bc47 100644
--- a/Lib/ctypes/__init__.py
+++ b/Lib/ctypes/__init__.py
@@ -1,9 +1,8 @@
 """create and manipulate C data types in Python"""
 
 import os as _os, sys as _sys
-from itertools import chain as _chain
 
-__version__ = "0.9.9.6"
+__version__ = "0.9.9.7"
 
 from _ctypes import Union, Structure, Array
 from _ctypes import _Pointer
@@ -111,7 +110,7 @@ if _os.name in ("nt", "ce"):
 elif _os.name == "posix":
     from _ctypes import dlopen as _dlopen
 
-from _ctypes import sizeof, byref, addressof, alignment
+from _ctypes import sizeof, byref, addressof, alignment, resize
 from _ctypes import _SimpleCData
 
 class py_object(_SimpleCData):
@@ -293,7 +292,7 @@ class CDLL(object):
         return "<%s '%s', handle %x at %x>" % \
                (self.__class__.__name__, self._name,
                 (self._handle & (_sys.maxint*2 + 1)),
-                id(self))
+                id(self) & (_sys.maxint*2 + 1))
 
     def __getattr__(self, name):
         if name.startswith('__') and name.endswith('__'):
@@ -419,12 +418,10 @@ def PYFUNCTYPE(restype, *argtypes):
         _restype_ = restype
         _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI
     return CFunctionType
-_cast = PYFUNCTYPE(py_object, c_void_p, py_object)(_cast_addr)
 
+_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr)
 def cast(obj, typ):
-    result = _cast(obj, typ)
-    result.__keepref = obj
-    return result
+    return _cast(obj, obj, typ)
 
 _string_at = CFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr)
 def string_at(ptr, size=0):
diff --git a/Lib/ctypes/test/test_anon.py b/Lib/ctypes/test/test_anon.py
new file mode 100644
index 0000000..62e2ce7
--- /dev/null
+++ b/Lib/ctypes/test/test_anon.py
@@ -0,0 +1,60 @@
+import unittest
+from ctypes import *
+
+class AnonTest(unittest.TestCase):
+
+    def test_anon(self):
+        class ANON(Union):
+            _fields_ = [("a", c_int),
+                        ("b", c_int)]
+
+        class Y(Structure):
+            _fields_ = [("x", c_int),
+                        ("_", ANON),
+                        ("y", c_int)]
+            _anonymous_ = ["_"]
+
+        self.failUnlessEqual(Y.a.offset, sizeof(c_int))
+        self.failUnlessEqual(Y.b.offset, sizeof(c_int))
+
+        self.failUnlessEqual(ANON.a.offset, 0)
+        self.failUnlessEqual(ANON.b.offset, 0)
+
+    def test_anon_nonseq(self):
+        # TypeError: _anonymous_ must be a sequence
+        self.failUnlessRaises(TypeError,
+                              lambda: type(Structure)("Name",
+                                                      (Structure,),
+                                                      {"_fields_": [], "_anonymous_": 42}))
+
+    def test_anon_nonmember(self):
+        # AttributeError: type object 'Name' has no attribute 'x'
+        self.failUnlessRaises(AttributeError,
+                              lambda: type(Structure)("Name",
+                                                      (Structure,),
+                                                      {"_fields_": [],
+                                                       "_anonymous_": ["x"]}))
+
+    def test_nested(self):
+        class ANON_S(Structure):
+            _fields_ = [("a", c_int)]
+
+        class ANON_U(Union):
+            _fields_ = [("_", ANON_S),
+                        ("b", c_int)]
+            _anonymous_ = ["_"]
+
+        class Y(Structure):
+            _fields_ = [("x", c_int),
+                        ("_", ANON_U),
+                        ("y", c_int)]
+            _anonymous_ = ["_"]
+        
+        self.failUnlessEqual(Y.x.offset, 0)
+        self.failUnlessEqual(Y.a.offset, sizeof(c_int))
+        self.failUnlessEqual(Y.b.offset, sizeof(c_int))
+        self.failUnlessEqual(Y._.offset, sizeof(c_int))
+        self.failUnlessEqual(Y.y.offset, sizeof(c_int) * 2)
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/Lib/ctypes/test/test_cast.py b/Lib/ctypes/test/test_cast.py
index 821ce3f..09e928f 100644
--- a/Lib/ctypes/test/test_cast.py
+++ b/Lib/ctypes/test/test_cast.py
@@ -30,17 +30,32 @@ class Test(unittest.TestCase):
         ptr = cast(address, POINTER(c_int))
         self.failUnlessEqual([ptr[i] for i in range(3)], [42, 17, 2])
 
-
-    def test_ptr2array(self):
-        array = (c_int * 3)(42, 17, 2)
-
-        from sys import getrefcount
-
-        before = getrefcount(array)
-        ptr = cast(array, POINTER(c_int))
-        self.failUnlessEqual(getrefcount(array), before + 1)
-        del ptr
-        self.failUnlessEqual(getrefcount(array), before)
+    def test_p2a_objects(self):
+        array = (c_char_p * 5)()
+        self.failUnlessEqual(array._objects, None)
+        array[0] = "foo bar"
+        self.failUnlessEqual(array._objects, {'0': "foo bar"})
+
+        p = cast(array, POINTER(c_char_p))
+        # array and p share a common _objects attribute
+        self.failUnless(p._objects is array._objects)
+        self.failUnlessEqual(array._objects, {'0': "foo bar", id(array): array})
+        p[0] = "spam spam"
+        self.failUnlessEqual(p._objects, {'0': "spam spam", id(array): array})
+        self.failUnless(array._objects is p._objects)
+        p[1] = "foo bar"
+        self.failUnlessEqual(p._objects, {'1': 'foo bar', '0': "spam spam", id(array): array})
+        self.failUnless(array._objects is p._objects)
+
+    def test_other(self):
+        p = cast((c_int * 4)(1, 2, 3, 4), POINTER(c_int))
+        self.failUnlessEqual(p[:4], [1,2, 3, 4])
+        c_int()
+        self.failUnlessEqual(p[:4], [1, 2, 3, 4])
+        p[2] = 96
+        self.failUnlessEqual(p[:4], [1, 2, 96, 4])
+        c_int()
+        self.failUnlessEqual(p[:4], [1, 2, 96, 4])
 
 if __name__ == "__main__":
     unittest.main()
diff --git a/Lib/ctypes/test/test_keeprefs.py b/Lib/ctypes/test/test_keeprefs.py
index 7318f29..80b6ca2 100644
--- a/Lib/ctypes/test/test_keeprefs.py
+++ b/Lib/ctypes/test/test_keeprefs.py
@@ -61,6 +61,8 @@ class StructureTestCase(unittest.TestCase):
         r.ul.x = 22
         r.ul.y = 44
         self.assertEquals(r._objects, {'0': {}})
+        r.lr = POINT()
+        self.assertEquals(r._objects, {'0': {}, '1': {}})
 
 class ArrayTestCase(unittest.TestCase):
     def test_cint_array(self):
@@ -86,9 +88,10 @@ class ArrayTestCase(unittest.TestCase):
         self.assertEquals(x._objects, {'1': {}})
 
 class PointerTestCase(unittest.TestCase):
-    def X_test_p_cint(self):
-        x = pointer(c_int(42))
-        print x._objects
+    def test_p_cint(self):
+        i = c_int(42)
+        x = pointer(i)
+        self.failUnlessEqual(x._objects, {'1': i})
 
 class DeletePointerTestCase(unittest.TestCase):
     def X_test(self):
diff --git a/Lib/ctypes/test/test_objects.py b/Lib/ctypes/test/test_objects.py
new file mode 100644
index 0000000..e49e435
--- /dev/null
+++ b/Lib/ctypes/test/test_objects.py
@@ -0,0 +1,66 @@
+r'''
+This tests the '_objects' attribute of ctypes instances.  '_objects'
+holds references to objects that must be kept alive as long as the
+ctypes instance, to make sure that the memory buffer is valid.
+
+WARNING: The '_objects' attribute is exposed ONLY for debugging ctypes itself,
+it MUST NEVER BE MODIFIED!
+
+'_objects' is initialized to a dictionary on first use, before that it
+is None.
+
+Here is an array of string pointers:
+
+>>> from ctypes import *
+>>> array = (c_char_p * 5)()
+>>> print array._objects
+None
+>>>
+
+The memory block stores pointers to strings, and the strings itself
+assigned from Python must be kept.
+
+>>> array[4] = 'foo bar'
+>>> array._objects
+{'4': 'foo bar'}
+>>> array[4]
+'foo bar'
+>>>
+
+It gets more complicated when the ctypes instance itself is contained
+in a 'base' object.
+
+>>> class X(Structure):
+...     _fields_ = [("x", c_int), ("y", c_int), ("array", c_char_p * 5)]
+...
+>>> x = X()
+>>> print x._objects
+None
+>>>
+
+The'array' attribute of the 'x' object shares part of the memory buffer
+of 'x' ('_b_base_' is either None, or the root object owning the memory block):
+
+>>> print x.array._b_base_ # doctest: +ELLIPSIS
+<ctypes.test.test_objects.X object at 0x...>
+>>>
+
+>>> x.array[0] = 'spam spam spam'
+>>> x._objects
+{'0:2': 'spam spam spam'}
+>>> x.array._b_base_._objects
+{'0:2': 'spam spam spam'}
+>>>
+
+'''
+
+import unittest, doctest
+
+import ctypes.test.test_objects
+
+class TestCase(unittest.TestCase):
+    def test(self):
+        doctest.testmod(ctypes.test.test_objects)
+
+if __name__ == '__main__':
+    doctest.testmod(ctypes.test.test_objects)
diff --git a/Lib/ctypes/test/test_slicing.py b/Lib/ctypes/test/test_slicing.py
index 08c811e..511c3d3 100644
--- a/Lib/ctypes/test/test_slicing.py
+++ b/Lib/ctypes/test/test_slicing.py
@@ -35,7 +35,7 @@ class SlicesTestCase(unittest.TestCase):
         self.assertRaises(ValueError, setslice, a, 0, 5, range(32))
 
     def test_char_ptr(self):
-        s = "abcdefghijklmnopqrstuvwxyz\0"
+        s = "abcdefghijklmnopqrstuvwxyz"
 
         dll = CDLL(_ctypes_test.__file__)
         dll.my_strdup.restype = POINTER(c_char)
@@ -50,9 +50,31 @@ class SlicesTestCase(unittest.TestCase):
 
         dll.my_strdup.restype = POINTER(c_byte)
         res = dll.my_strdup(s)
-        self.failUnlessEqual(res[:len(s)-1], range(ord("a"), ord("z")+1))
+        self.failUnlessEqual(res[:len(s)], range(ord("a"), ord("z")+1))
         dll.my_free(res)
 
+    def test_char_ptr_with_free(self):
+        dll = CDLL(_ctypes_test.__file__)
+        s = "abcdefghijklmnopqrstuvwxyz"
+
+        class allocated_c_char_p(c_char_p):
+            pass
+
+        dll.my_free.restype = None
+        def errcheck(result, func, args):
+            retval = result.value
+            dll.my_free(result)
+            return retval
+
+        dll.my_strdup.restype = allocated_c_char_p
+        dll.my_strdup.errcheck = errcheck
+        try:
+            res = dll.my_strdup(s)
+            self.failUnlessEqual(res, s)
+        finally:
+            del dll.my_strdup.errcheck
+
+
     def test_char_array(self):
         s = "abcdefghijklmnopqrstuvwxyz\0"
 
diff --git a/Lib/ctypes/test/test_structures.py b/Lib/ctypes/test/test_structures.py
index 49f064b..bb56a61 100644
--- a/Lib/ctypes/test/test_structures.py
+++ b/Lib/ctypes/test/test_structures.py
@@ -138,8 +138,8 @@ class StructureTestCase(unittest.TestCase):
         self.failUnlessEqual(X.y.size, sizeof(c_char))
 
         # readonly
-        self.assertRaises(AttributeError, setattr, X.x, "offset", 92)
-        self.assertRaises(AttributeError, setattr, X.x, "size", 92)
+        self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92)
+        self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92)
 
         class X(Union):
             _fields_ = [("x", c_int),
@@ -152,8 +152,8 @@ class StructureTestCase(unittest.TestCase):
         self.failUnlessEqual(X.y.size, sizeof(c_char))
 
         # readonly
-        self.assertRaises(AttributeError, setattr, X.x, "offset", 92)
-        self.assertRaises(AttributeError, setattr, X.x, "size", 92)
+        self.assertRaises((TypeError, AttributeError), setattr, X.x, "offset", 92)
+        self.assertRaises((TypeError, AttributeError), setattr, X.x, "size", 92)
 
         # XXX Should we check nested data types also?
         # offset is always relative to the class...
@@ -298,7 +298,7 @@ class StructureTestCase(unittest.TestCase):
                                  "expected string or Unicode object, int found")
         else:
             self.failUnlessEqual(msg,
-                                 "(Phone) TypeError: "
+                                 "(Phone) exceptions.TypeError: "
                                  "expected string or Unicode object, int found")
 
         cls, msg = self.get_except(Person, "Someone", ("a", "b", "c"))
@@ -307,7 +307,7 @@ class StructureTestCase(unittest.TestCase):
             self.failUnlessEqual(msg,
                                  "(Phone) <type 'exceptions.ValueError'>: too many initializers")
         else:
-            self.failUnlessEqual(msg, "(Phone) ValueError: too many initializers")
+            self.failUnlessEqual(msg, "(Phone) exceptions.ValueError: too many initializers")
 
 
     def get_except(self, func, *args):
diff --git a/Lib/ctypes/test/test_varsize_struct.py b/Lib/ctypes/test/test_varsize_struct.py
new file mode 100644
index 0000000..aa8930d
--- /dev/null
+++ b/Lib/ctypes/test/test_varsize_struct.py
@@ -0,0 +1,115 @@
+from ctypes import *
+import unittest
+
+class VarSizeTest(unittest.TestCase):
+    def test_resize(self):
+        class X(Structure):
+            _fields_ = [("item", c_int),
+                        ("array", c_int * 1)]
+
+        self.failUnlessEqual(sizeof(X), sizeof(c_int) * 2)
+        x = X()
+        x.item = 42
+        x.array[0] = 100
+        self.failUnlessEqual(sizeof(x), sizeof(c_int) * 2)
+
+        # make room for one additional item
+        new_size = sizeof(X) + sizeof(c_int) * 1
+        resize(x, new_size)
+        self.failUnlessEqual(sizeof(x), new_size)
+        self.failUnlessEqual((x.item, x.array[0]), (42, 100))
+
+        # make room for 10 additional items
+        new_size = sizeof(X) + sizeof(c_int) * 9
+        resize(x, new_size)
+        self.failUnlessEqual(sizeof(x), new_size)
+        self.failUnlessEqual((x.item, x.array[0]), (42, 100))
+
+        # make room for one additional item
+        new_size = sizeof(X) + sizeof(c_int) * 1
+        resize(x, new_size)
+        self.failUnlessEqual(sizeof(x), new_size)
+        self.failUnlessEqual((x.item, x.array[0]), (42, 100))
+
+    def test_array_invalid_length(self):
+        # cannot create arrays with non-positive size
+        self.failUnlessRaises(ValueError, lambda: c_int * -1)
+        self.failUnlessRaises(ValueError, lambda: c_int * -3)
+
+    def test_zerosized_array(self):
+        array = (c_int * 0)()
+        # accessing elements of zero-sized arrays raise IndexError
+        self.failUnlessRaises(IndexError, array.__setitem__, 0, None)
+        self.failUnlessRaises(IndexError, array.__getitem__, 0)
+        self.failUnlessRaises(IndexError, array.__setitem__, 1, None)
+        self.failUnlessRaises(IndexError, array.__getitem__, 1)
+        self.failUnlessRaises(IndexError, array.__setitem__, -1, None)
+        self.failUnlessRaises(IndexError, array.__getitem__, -1)
+
+    def test_varsized_array(self):
+        array = (c_int * 20)(20, 21, 22, 23, 24, 25, 26, 27, 28, 29)
+
+        # no range checking is done on arrays with size == 1
+        varsize_array = (c_int * 1).from_address(addressof(array))
+
+        # __getitem__
+        self.failUnlessEqual(varsize_array[0], 20)
+        self.failUnlessEqual(varsize_array[1], 21)
+        self.failUnlessEqual(varsize_array[2], 22)
+        self.failUnlessEqual(varsize_array[3], 23)
+        self.failUnlessEqual(varsize_array[4], 24)
+        self.failUnlessEqual(varsize_array[5], 25)
+        self.failUnlessEqual(varsize_array[6], 26)
+        self.failUnlessEqual(varsize_array[7], 27)
+        self.failUnlessEqual(varsize_array[8], 28)
+        self.failUnlessEqual(varsize_array[9], 29)
+
+        # still, normal sequence of length one behaviour:
+        self.failUnlessEqual(varsize_array[-1], 20)
+        self.failUnlessRaises(IndexError, lambda: varsize_array[-2])
+        # except for this one, which will raise MemoryError
+        self.failUnlessRaises(MemoryError, lambda: varsize_array[:])
+
+        # __setitem__
+        varsize_array[0] = 100
+        varsize_array[1] = 101
+        varsize_array[2] = 102
+        varsize_array[3] = 103
+        varsize_array[4] = 104
+        varsize_array[5] = 105
+        varsize_array[6] = 106
+        varsize_array[7] = 107
+        varsize_array[8] = 108
+        varsize_array[9] = 109
+
+        for i in range(10):
+            self.failUnlessEqual(varsize_array[i], i + 100)
+            self.failUnlessEqual(array[i], i + 100)
+
+        # __getslice__
+        self.failUnlessEqual(varsize_array[0:10], range(100, 110))
+        self.failUnlessEqual(varsize_array[1:9], range(101, 109))
+        self.failUnlessEqual(varsize_array[1:-1], [])
+
+        # __setslice__
+        varsize_array[0:10] = range(1000, 1010)
+        self.failUnlessEqual(varsize_array[0:10], range(1000, 1010))
+
+        varsize_array[1:9] = range(1001, 1009)
+        self.failUnlessEqual(varsize_array[1:9], range(1001, 1009))
+
+    def test_vararray_is_sane(self):
+        array = (c_int * 15)(20, 21, 22, 23, 24, 25, 26, 27, 28, 29)
+
+        varsize_array = (c_int * 1).from_address(addressof(array))
+        varsize_array[:] = [1, 2, 3, 4, 5]
+
+        self.failUnlessEqual(array[:], [1, 2, 3, 4, 5, 25, 26, 27, 28, 29, 0, 0, 0, 0, 0])
+        self.failUnlessEqual(varsize_array[0:10], [1, 2, 3, 4, 5, 25, 26, 27, 28, 29])
+
+        array[:5] = [10, 11, 12, 13, 14]
+        self.failUnlessEqual(array[:], [10, 11, 12, 13, 14, 25, 26, 27, 28, 29, 0, 0, 0, 0, 0])
+        self.failUnlessEqual(varsize_array[0:10], [10, 11, 12, 13, 14, 25, 26, 27, 28, 29])
+        
+if __name__ == "__main__":
+    unittest.main()