- A new pickle protocol (protocol 3) is added with explicit support

  for bytes.  This is the default protocol.  It intentionally cannot
  be unpickled by Python 2.x.

- When a pickle	written	by Python 2.x contains an (8-bit) str
  instance, this is now decoded to a (Unicode) str instance.  The
  encoding used to do this defaults to ASCII, but can be overridden
  via two new keyword arguments to the Unpickler class.  Previously
  this would create bytes instances, which is usually wrong: str
  instances are often used to pickle attribute names etc., and text is
  more common than binary data anyway.

3 files changed, 22 insertions, 13 deletions

diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index 5ac2bdc..0230f3c 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -490,6 +490,12 @@ class AbstractPickleTests(unittest.TestCase):
                 u2 = self.loads(p)
                 self.assertEqual(u2, u)
 
+    def test_bytes(self):
+        for proto in protocols:
+            for u in b'', b'xyz', b'xyz'*100:
+                p = self.dumps(u)
+                self.assertEqual(self.loads(p), u)
+
     def test_ints(self):
         import sys
         for proto in protocols:
@@ -532,8 +538,8 @@ class AbstractPickleTests(unittest.TestCase):
 
     @run_with_locale('LC_ALL', 'de_DE', 'fr_FR')
     def test_float_format(self):
-        # make sure that floats are formatted locale independent
-        self.assertEqual(self.dumps(1.2)[0:3], b'F1.')
+        # make sure that floats are formatted locale independent with proto 0
+        self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.')
 
     def test_reduce(self):
         pass
@@ -624,6 +630,12 @@ class AbstractPickleTests(unittest.TestCase):
                            (2, 2): pickle.TUPLE2,
                            (2, 3): pickle.TUPLE3,
                            (2, 4): pickle.TUPLE,
+
+                           (3, 0): pickle.EMPTY_TUPLE,
+                           (3, 1): pickle.TUPLE1,
+                           (3, 2): pickle.TUPLE2,
+                           (3, 3): pickle.TUPLE3,
+                           (3, 4): pickle.TUPLE,
                           }
         a = ()
         b = (1,)
@@ -643,14 +655,17 @@ class AbstractPickleTests(unittest.TestCase):
         expected_opcode = {(0, None): pickle.NONE,
                            (1, None): pickle.NONE,
                            (2, None): pickle.NONE,
+                           (3, None): pickle.NONE,
 
                            (0, True): pickle.INT,
                            (1, True): pickle.INT,
                            (2, True): pickle.NEWTRUE,
+                           (3, True): pickle.NEWTRUE,
 
                            (0, False): pickle.INT,
                            (1, False): pickle.INT,
                            (2, False): pickle.NEWFALSE,
+                           (3, False): pickle.NEWFALSE,
                           }
         for proto in protocols:
             for x in None, False, True:
@@ -955,7 +970,7 @@ class AbstractPickleModuleTests(unittest.TestCase):
 
     def test_highest_protocol(self):
         # Of course this needs to be changed when HIGHEST_PROTOCOL changes.
-        self.assertEqual(self.module.HIGHEST_PROTOCOL, 2)
+        self.assertEqual(self.module.HIGHEST_PROTOCOL, 3)
 
     def test_callapi(self):
         from io import BytesIO
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index aa09a6a..67d83c7c 100644
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -12,23 +12,19 @@ class PickleTests(AbstractPickleTests, AbstractPickleModuleTests):
     module = pickle
     error = KeyError
 
-    def dumps(self, arg, proto=0, fast=0):
-        # Ignore fast
+    def dumps(self, arg, proto=None):
         return pickle.dumps(arg, proto)
 
     def loads(self, buf):
-        # Ignore fast
         return pickle.loads(buf)
 
 class PicklerTests(AbstractPickleTests):
 
     error = KeyError
 
-    def dumps(self, arg, proto=0, fast=0):
+    def dumps(self, arg, proto=None):
         f = io.BytesIO()
         p = pickle.Pickler(f, proto)
-        if fast:
-            p.fast = fast
         p.dump(arg)
         f.seek(0)
         return bytes(f.read())
@@ -40,14 +36,12 @@ class PicklerTests(AbstractPickleTests):
 
 class PersPicklerTests(AbstractPersistentPicklerTests):
 
-    def dumps(self, arg, proto=0, fast=0):
+    def dumps(self, arg, proto=None):
         class PersPickler(pickle.Pickler):
             def persistent_id(subself, obj):
                 return self.persistent_id(obj)
         f = io.BytesIO()
         p = PersPickler(f, proto)
-        if fast:
-            p.fast = fast
         p.dump(arg)
         f.seek(0)
         return f.read()
diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py
index 3e5b35a..932dcd1 100644
--- a/Lib/test/test_pickletools.py
+++ b/Lib/test/test_pickletools.py
@@ -6,7 +6,7 @@ from test.pickletester import AbstractPickleModuleTests
 
 class OptimizedPickleTests(AbstractPickleTests, AbstractPickleModuleTests):
 
-    def dumps(self, arg, proto=0, fast=0):
+    def dumps(self, arg, proto=None):
         return pickletools.optimize(pickle.dumps(arg, proto))
 
     def loads(self, buf):