Issue #9410: Various optimizations to the pickle module, leading to

speedups up to 4x (depending on the benchmark).  Mostly ported from
Unladen Swallow; initial patch by Alexandre Vassalotti.

3 files changed, 29 insertions, 7 deletions

diff --git a/Lib/pickle.py b/Lib/pickle.py
index 8732508..aca8fd1 100644
--- a/Lib/pickle.py
+++ b/Lib/pickle.py
@@ -1287,12 +1287,6 @@ def decode_long(data):
     """
     return int.from_bytes(data, byteorder='little', signed=True)
 
-# Use the faster _pickle if possible
-try:
-    from _pickle import *
-except ImportError:
-    Pickler, Unpickler = _Pickler, _Unpickler
-
 # Shorthands
 
 def dump(obj, file, protocol=None, *, fix_imports=True):
@@ -1316,6 +1310,12 @@ def loads(s, *, fix_imports=True, encoding="ASCII", errors="strict"):
     return Unpickler(file, fix_imports=fix_imports,
                      encoding=encoding, errors=errors).load()
 
+# Use the faster _pickle if possible
+try:
+    from _pickle import *
+except ImportError:
+    Pickler, Unpickler = _Pickler, _Unpickler
+
 # Doctest
 def _test():
     import doctest
diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py
index 70e5ad0..33c85dc 100644
--- a/Lib/test/pickletester.py
+++ b/Lib/test/pickletester.py
@@ -1068,6 +1068,15 @@ class AbstractPickleTests(unittest.TestCase):
         dumped = self.dumps(set([3]), 2)
         self.assertEqual(dumped, DATA6)
 
+    def test_large_pickles(self):
+        # Test the correctness of internal buffering routines when handling
+        # large data.
+        for proto in protocols:
+            data = (1, b'x' * (256 * 1024))
+            dumped = self.dumps(data, proto)
+            loaded = self.loads(dumped)
+            self.assertEqual(loaded, data)
+
 
 # Test classes for reduce_ex
 
diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py
index bb681bf..7f642c8 100644
--- a/Lib/test/test_pickle.py
+++ b/Lib/test/test_pickle.py
@@ -37,6 +37,18 @@ class PyPicklerTests(AbstractPickleTests):
         return u.load()
 
 
+class InMemoryPickleTests(AbstractPickleTests):
+
+    pickler = pickle._Pickler
+    unpickler = pickle._Unpickler
+
+    def dumps(self, arg, proto=None):
+        return pickle.dumps(arg, proto)
+
+    def loads(self, buf):
+        return pickle.loads(buf)
+
+
 class PyPersPicklerTests(AbstractPersistentPicklerTests):
 
     pickler = pickle._Pickler
@@ -95,7 +107,8 @@ def test_main():
         tests.extend([CPicklerTests, CPersPicklerTests,
                       CDumpPickle_LoadPickle, DumpPickle_CLoadPickle,
                       PyPicklerUnpicklerObjectTests,
-                      CPicklerUnpicklerObjectTests])
+                      CPicklerUnpicklerObjectTests,
+                      InMemoryPickleTests])
     support.run_unittest(*tests)
     support.run_doctest(pickle)