Merging the py3k-pep3137 branch back into the py3k branch.

No detailed change log; just check out the change log for the py3k-pep3137
branch.  The most obvious changes:

  - str8 renamed to bytes (PyString at the C level);
  - bytes renamed to buffer (PyBytes at the C level);
  - PyString and PyUnicode are no longer compatible.

I.e. we now have an immutable bytes type and a mutable bytes type.

The behavior of PyString was modified quite a bit, to make it more
bytes-like.  Some changes are still on the to-do list.

1 files changed, 244 insertions, 220 deletions

diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index 0b1cf5a..6db3572 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -5,6 +5,8 @@ import copy_reg
 
 from test.test_support import TestFailed, TESTFN, run_with_locale
 
+from pickle import bytes_types
+
 # Tests that try a number of pickle protocols should have a
 #     for proto in protocols:
 # kind of outer loop.
@@ -87,149 +89,137 @@ class use_metaclass(object, metaclass=metaclass):
 # DATA0 .. DATA2 are the pickles we expect under the various protocols, for
 # the object returned by create_data().
 
-# break into multiple strings to avoid confusing font-lock-mode
-DATA0 = b"""(lp1
-I0
-aL1L
-aF2
-ac__builtin__
-complex
-p2
-""" + \
-b"""(F3
-F0
-tRp3
-aI1
-aI-1
-aI255
-aI-255
-aI-256
-aI65535
-aI-65535
-aI-65536
-aI2147483647
-aI-2147483647
-aI-2147483648
-a""" + \
-b"""(S'abc'
-p4
-g4
-""" + \
-b"""(i__main__
-C
-p5
-""" + \
-b"""(dp6
-S'foo'
-p7
-I1
-sS'bar'
-p8
-I2
-sbg5
-tp9
-ag9
-aI5
-a.
-"""
-
-# Disassembly of DATA0.
+DATA0 = (
+    b'(lp0\nL0\naL1\naF2.0\nac'
+    b'__builtin__\ncomplex\n'
+    b'p1\n(F3.0\nF0.0\ntp2\nRp'
+    b'3\naL1\naL-1\naL255\naL-'
+    b'255\naL-256\naL65535\na'
+    b'L-65535\naL-65536\naL2'
+    b'147483647\naL-2147483'
+    b'647\naL-2147483648\na('
+    b'Vabc\np4\ng4\nccopy_reg'
+    b'\n_reconstructor\np5\n('
+    b'c__main__\nC\np6\nc__bu'
+    b'iltin__\nobject\np7\nNt'
+    b'p8\nRp9\n(dp10\nVfoo\np1'
+    b'1\nL1\nsVbar\np12\nL2\nsb'
+    b'g9\ntp13\nag13\naL5\na.'
+)
+
+# Disassembly of DATA0
 DATA0_DIS = """\
     0: (    MARK
     1: l        LIST       (MARK at 0)
-    2: p    PUT        1
-    5: I    INT        0
+    2: p    PUT        0
+    5: L    LONG       0
     8: a    APPEND
-    9: L    LONG       1L
-   13: a    APPEND
-   14: F    FLOAT      2.0
-   17: a    APPEND
-   18: c    GLOBAL     '__builtin__ complex'
-   39: p    PUT        2
-   42: (    MARK
-   43: F        FLOAT      3.0
-   46: F        FLOAT      0.0
-   49: t        TUPLE      (MARK at 42)
-   50: R    REDUCE
-   51: p    PUT        3
-   54: a    APPEND
-   55: I    INT        1
-   58: a    APPEND
-   59: I    INT        -1
-   63: a    APPEND
-   64: I    INT        255
-   69: a    APPEND
-   70: I    INT        -255
-   76: a    APPEND
-   77: I    INT        -256
-   83: a    APPEND
-   84: I    INT        65535
+    9: L    LONG       1
+   12: a    APPEND
+   13: F    FLOAT      2.0
+   18: a    APPEND
+   19: c    GLOBAL     '__builtin__ complex'
+   40: p    PUT        1
+   43: (    MARK
+   44: F        FLOAT      3.0
+   49: F        FLOAT      0.0
+   54: t        TUPLE      (MARK at 43)
+   55: p    PUT        2
+   58: R    REDUCE
+   59: p    PUT        3
+   62: a    APPEND
+   63: L    LONG       1
+   66: a    APPEND
+   67: L    LONG       -1
+   71: a    APPEND
+   72: L    LONG       255
+   77: a    APPEND
+   78: L    LONG       -255
+   84: a    APPEND
+   85: L    LONG       -256
    91: a    APPEND
-   92: I    INT        -65535
-  100: a    APPEND
-  101: I    INT        -65536
-  109: a    APPEND
-  110: I    INT        2147483647
-  122: a    APPEND
-  123: I    INT        -2147483647
-  136: a    APPEND
-  137: I    INT        -2147483648
-  150: a    APPEND
-  151: (    MARK
-  152: S        STRING     'abc'
-  159: p        PUT        4
-  162: g        GET        4
-  165: (        MARK
-  166: i            INST       '__main__ C' (MARK at 165)
-  178: p        PUT        5
-  181: (        MARK
-  182: d            DICT       (MARK at 181)
-  183: p        PUT        6
-  186: S        STRING     'foo'
-  193: p        PUT        7
-  196: I        INT        1
-  199: s        SETITEM
-  200: S        STRING     'bar'
-  207: p        PUT        8
-  210: I        INT        2
-  213: s        SETITEM
-  214: b        BUILD
-  215: g        GET        5
-  218: t        TUPLE      (MARK at 151)
-  219: p    PUT        9
-  222: a    APPEND
-  223: g    GET        9
-  226: a    APPEND
-  227: I    INT        5
-  230: a    APPEND
-  231: .    STOP
+   92: L    LONG       65535
+   99: a    APPEND
+  100: L    LONG       -65535
+  108: a    APPEND
+  109: L    LONG       -65536
+  117: a    APPEND
+  118: L    LONG       2147483647
+  130: a    APPEND
+  131: L    LONG       -2147483647
+  144: a    APPEND
+  145: L    LONG       -2147483648
+  158: a    APPEND
+  159: (    MARK
+  160: V        UNICODE    'abc'
+  165: p        PUT        4
+  168: g        GET        4
+  171: c        GLOBAL     'copy_reg _reconstructor'
+  196: p        PUT        5
+  199: (        MARK
+  200: c            GLOBAL     '__main__ C'
+  212: p            PUT        6
+  215: c            GLOBAL     '__builtin__ object'
+  235: p            PUT        7
+  238: N            NONE
+  239: t            TUPLE      (MARK at 199)
+  240: p        PUT        8
+  243: R        REDUCE
+  244: p        PUT        9
+  247: (        MARK
+  248: d            DICT       (MARK at 247)
+  249: p        PUT        10
+  253: V        UNICODE    'foo'
+  258: p        PUT        11
+  262: L        LONG       1
+  265: s        SETITEM
+  266: V        UNICODE    'bar'
+  271: p        PUT        12
+  275: L        LONG       2
+  278: s        SETITEM
+  279: b        BUILD
+  280: g        GET        9
+  283: t        TUPLE      (MARK at 159)
+  284: p    PUT        13
+  288: a    APPEND
+  289: g    GET        13
+  293: a    APPEND
+  294: L    LONG       5
+  297: a    APPEND
+  298: .    STOP
 highest protocol among opcodes = 0
 """
 
-DATA1 = (b']q\x01(K\x00L1L\nG@\x00\x00\x00\x00\x00\x00\x00'
-         b'c__builtin__\ncomplex\nq\x02(G@\x08\x00\x00\x00\x00\x00'
-         b'\x00G\x00\x00\x00\x00\x00\x00\x00\x00tRq\x03K\x01J\xff\xff'
-         b'\xff\xffK\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xff'
-         b'J\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00'
-         b'\x00\x80J\x00\x00\x00\x80(U\x03abcq\x04h\x04(c__main__\n'
-         b'C\nq\x05oq\x06}q\x07(U\x03fooq\x08K\x01U\x03barq\tK\x02ubh'
-         b'\x06tq\nh\nK\x05e.'
-        )
-
-# Disassembly of DATA1.
+DATA1 = (
+    b']q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c__'
+    b'builtin__\ncomplex\nq\x01'
+    b'(G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00t'
+    b'q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xffJ'
+    b'\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff\xff'
+    b'\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00ab'
+    b'cq\x04h\x04ccopy_reg\n_reco'
+    b'nstructor\nq\x05(c__main'
+    b'__\nC\nq\x06c__builtin__\n'
+    b'object\nq\x07Ntq\x08Rq\t}q\n('
+    b'X\x03\x00\x00\x00fooq\x0bK\x01X\x03\x00\x00\x00bar'
+    b'q\x0cK\x02ubh\ttq\rh\rK\x05e.'
+)
+
+# Disassembly of DATA1
 DATA1_DIS = """\
     0: ]    EMPTY_LIST
-    1: q    BINPUT     1
+    1: q    BINPUT     0
     3: (    MARK
     4: K        BININT1    0
-    6: L        LONG       1L
-   10: G        BINFLOAT   2.0
-   19: c        GLOBAL     '__builtin__ complex'
-   40: q        BINPUT     2
-   42: (        MARK
-   43: G            BINFLOAT   3.0
-   52: G            BINFLOAT   0.0
-   61: t            TUPLE      (MARK at 42)
+    6: K        BININT1    1
+    8: G        BINFLOAT   2.0
+   17: c        GLOBAL     '__builtin__ complex'
+   38: q        BINPUT     1
+   40: (        MARK
+   41: G            BINFLOAT   3.0
+   50: G            BINFLOAT   0.0
+   59: t            TUPLE      (MARK at 40)
+   60: q        BINPUT     2
    62: R        REDUCE
    63: q        BINPUT     3
    65: K        BININT1    1
@@ -244,97 +234,110 @@ DATA1_DIS = """\
   102: J        BININT     -2147483647
   107: J        BININT     -2147483648
   112: (        MARK
-  113: U            SHORT_BINSTRING 'abc'
-  118: q            BINPUT     4
-  120: h            BINGET     4
-  122: (            MARK
-  123: c                GLOBAL     '__main__ C'
-  135: q                BINPUT     5
-  137: o                OBJ        (MARK at 122)
-  138: q            BINPUT     6
-  140: }            EMPTY_DICT
-  141: q            BINPUT     7
-  143: (            MARK
-  144: U                SHORT_BINSTRING 'foo'
-  149: q                BINPUT     8
-  151: K                BININT1    1
-  153: U                SHORT_BINSTRING 'bar'
-  158: q                BINPUT     9
-  160: K                BININT1    2
-  162: u                SETITEMS   (MARK at 143)
-  163: b            BUILD
-  164: h            BINGET     6
-  166: t            TUPLE      (MARK at 112)
-  167: q        BINPUT     10
-  169: h        BINGET     10
-  171: K        BININT1    5
-  173: e        APPENDS    (MARK at 3)
-  174: .    STOP
+  113: X            BINUNICODE 'abc'
+  121: q            BINPUT     4
+  123: h            BINGET     4
+  125: c            GLOBAL     'copy_reg _reconstructor'
+  150: q            BINPUT     5
+  152: (            MARK
+  153: c                GLOBAL     '__main__ C'
+  165: q                BINPUT     6
+  167: c                GLOBAL     '__builtin__ object'
+  187: q                BINPUT     7
+  189: N                NONE
+  190: t                TUPLE      (MARK at 152)
+  191: q            BINPUT     8
+  193: R            REDUCE
+  194: q            BINPUT     9
+  196: }            EMPTY_DICT
+  197: q            BINPUT     10
+  199: (            MARK
+  200: X                BINUNICODE 'foo'
+  208: q                BINPUT     11
+  210: K                BININT1    1
+  212: X                BINUNICODE 'bar'
+  220: q                BINPUT     12
+  222: K                BININT1    2
+  224: u                SETITEMS   (MARK at 199)
+  225: b            BUILD
+  226: h            BINGET     9
+  228: t            TUPLE      (MARK at 112)
+  229: q        BINPUT     13
+  231: h        BINGET     13
+  233: K        BININT1    5
+  235: e        APPENDS    (MARK at 3)
+  236: .    STOP
 highest protocol among opcodes = 1
 """
 
-DATA2 = (b'\x80\x02]q\x01(K\x00\x8a\x01\x01G@\x00\x00\x00\x00\x00\x00\x00'
-         b'c__builtin__\ncomplex\nq\x02G@\x08\x00\x00\x00\x00\x00\x00G\x00'
-         b'\x00\x00\x00\x00\x00\x00\x00\x86Rq\x03K\x01J\xff\xff\xff\xffK'
-         b'\xffJ\x01\xff\xff\xffJ\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xff'
-         b'J\x00\x00\xff\xffJ\xff\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00'
-         b'\x80(U\x03abcq\x04h\x04(c__main__\nC\nq\x05oq\x06}q\x07(U\x03foo'
-         b'q\x08K\x01U\x03barq\tK\x02ubh\x06tq\nh\nK\x05e.')
-
-# Disassembly of DATA2.
+DATA2 = (
+    b'\x80\x02]q\x00(K\x00K\x01G@\x00\x00\x00\x00\x00\x00\x00c'
+    b'__builtin__\ncomplex\n'
+    b'q\x01G@\x08\x00\x00\x00\x00\x00\x00G\x00\x00\x00\x00\x00\x00\x00\x00'
+    b'\x86q\x02Rq\x03K\x01J\xff\xff\xff\xffK\xffJ\x01\xff\xff\xff'
+    b'J\x00\xff\xff\xffM\xff\xffJ\x01\x00\xff\xffJ\x00\x00\xff\xffJ\xff'
+    b'\xff\xff\x7fJ\x01\x00\x00\x80J\x00\x00\x00\x80(X\x03\x00\x00\x00a'
+    b'bcq\x04h\x04c__main__\nC\nq\x05'
+    b')\x81q\x06}q\x07(X\x03\x00\x00\x00fooq\x08K\x01'
+    b'X\x03\x00\x00\x00barq\tK\x02ubh\x06tq\nh'
+    b'\nK\x05e.'
+)
+
+# Disassembly of DATA2
 DATA2_DIS = """\
     0: \x80 PROTO      2
     2: ]    EMPTY_LIST
-    3: q    BINPUT     1
+    3: q    BINPUT     0
     5: (    MARK
     6: K        BININT1    0
-    8: \x8a     LONG1      1L
-   11: G        BINFLOAT   2.0
-   20: c        GLOBAL     '__builtin__ complex'
-   41: q        BINPUT     2
-   43: G        BINFLOAT   3.0
-   52: G        BINFLOAT   0.0
-   61: \x86     TUPLE2
-   62: R        REDUCE
-   63: q        BINPUT     3
-   65: K        BININT1    1
-   67: J        BININT     -1
-   72: K        BININT1    255
-   74: J        BININT     -255
-   79: J        BININT     -256
-   84: M        BININT2    65535
-   87: J        BININT     -65535
-   92: J        BININT     -65536
-   97: J        BININT     2147483647
-  102: J        BININT     -2147483647
-  107: J        BININT     -2147483648
-  112: (        MARK
-  113: U            SHORT_BINSTRING 'abc'
-  118: q            BINPUT     4
-  120: h            BINGET     4
-  122: (            MARK
-  123: c                GLOBAL     '__main__ C'
-  135: q                BINPUT     5
-  137: o                OBJ        (MARK at 122)
-  138: q            BINPUT     6
-  140: }            EMPTY_DICT
-  141: q            BINPUT     7
-  143: (            MARK
-  144: U                SHORT_BINSTRING 'foo'
-  149: q                BINPUT     8
-  151: K                BININT1    1
-  153: U                SHORT_BINSTRING 'bar'
-  158: q                BINPUT     9
-  160: K                BININT1    2
-  162: u                SETITEMS   (MARK at 143)
-  163: b            BUILD
-  164: h            BINGET     6
-  166: t            TUPLE      (MARK at 112)
-  167: q        BINPUT     10
-  169: h        BINGET     10
-  171: K        BININT1    5
-  173: e        APPENDS    (MARK at 5)
-  174: .    STOP
+    8: K        BININT1    1
+   10: G        BINFLOAT   2.0
+   19: c        GLOBAL     '__builtin__ complex'
+   40: q        BINPUT     1
+   42: G        BINFLOAT   3.0
+   51: G        BINFLOAT   0.0
+   60: \x86     TUPLE2
+   61: q        BINPUT     2
+   63: R        REDUCE
+   64: q        BINPUT     3
+   66: K        BININT1    1
+   68: J        BININT     -1
+   73: K        BININT1    255
+   75: J        BININT     -255
+   80: J        BININT     -256
+   85: M        BININT2    65535
+   88: J        BININT     -65535
+   93: J        BININT     -65536
+   98: J        BININT     2147483647
+  103: J        BININT     -2147483647
+  108: J        BININT     -2147483648
+  113: (        MARK
+  114: X            BINUNICODE 'abc'
+  122: q            BINPUT     4
+  124: h            BINGET     4
+  126: c            GLOBAL     '__main__ C'
+  138: q            BINPUT     5
+  140: )            EMPTY_TUPLE
+  141: \x81         NEWOBJ
+  142: q            BINPUT     6
+  144: }            EMPTY_DICT
+  145: q            BINPUT     7
+  147: (            MARK
+  148: X                BINUNICODE 'foo'
+  156: q                BINPUT     8
+  158: K                BININT1    1
+  160: X                BINUNICODE 'bar'
+  168: q                BINPUT     9
+  170: K                BININT1    2
+  172: u                SETITEMS   (MARK at 147)
+  173: b            BUILD
+  174: h            BINGET     6
+  176: t            TUPLE      (MARK at 113)
+  177: q        BINPUT     10
+  179: h        BINGET     10
+  181: K        BININT1    5
+  183: e        APPENDS    (MARK at 5)
+  184: .    STOP
 highest protocol among opcodes = 2
 """
 
@@ -393,11 +396,14 @@ class AbstractPickleTests(unittest.TestCase):
             got = self.loads(s)
             self.assertEqual(expected, got)
 
-    def test_load_from_canned_string(self):
-        expected = self._testdata
-        for canned in DATA0, DATA1, DATA2:
-            got = self.loads(canned)
-            self.assertEqual(expected, got)
+    def test_load_from_data0(self):
+        self.assertEqual(self._testdata, self.loads(DATA0))
+
+    def test_load_from_data1(self):
+        self.assertEqual(self._testdata, self.loads(DATA1))
+
+    def test_load_from_data2(self):
+        self.assertEqual(self._testdata, self.loads(DATA2))
 
     # There are gratuitous differences between pickles produced by
     # pickle and cPickle, largely because cPickle starts PUT indices at
@@ -762,7 +768,7 @@ class AbstractPickleTests(unittest.TestCase):
         x = dict.fromkeys(range(n))
         for proto in protocols:
             s = self.dumps(x, proto)
-            assert isinstance(s, bytes)
+            assert isinstance(s, bytes_types)
             y = self.loads(s)
             self.assertEqual(x, y)
             num_setitems = count_opcode(pickle.SETITEMS, s)
@@ -996,3 +1002,21 @@ class AbstractPersistentPicklerTests(unittest.TestCase):
         self.assertEqual(self.loads(self.dumps(L, 1)), L)
         self.assertEqual(self.id_count, 5)
         self.assertEqual(self.load_count, 5)
+
+if __name__ == "__main__":
+    # Print some stuff that can be used to rewrite DATA{0,1,2}
+    from pickletools import dis
+    x = create_data()
+    for i in range(3):
+        p = pickle.dumps(x, i)
+        print("DATA{0} = (".format(i))
+        for j in range(0, len(p), 20):
+            b = bytes(p[j:j+20])
+            print("    {0!r}".format(b))
+        print(")")
+        print()
+        print("# Disassembly of DATA{0}".format(i))
+        print("DATA{0}_DIS = \"\"\"\\".format(i))
+        dis(p)
+        print("\"\"\"")
+        print()