diff options
| author | Jeff Allen <ja.py@farowl.co.uk> | 2023-12-24 09:43:44 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2023-12-24 09:43:44 (GMT) |
| commit | 08398631a0298dcf785ee7bd0e26c7844823ce59 (patch) | |
| tree | f669f700f6f6b5b91d59825045c00f7dbae4e408 | |
| parent | 894f0e573d9eb49cd5864c44328f10a731852dab (diff) | |
| download | cpython-08398631a0298dcf785ee7bd0e26c7844823ce59.zip cpython-08398631a0298dcf785ee7bd0e26c7844823ce59.tar.gz cpython-08398631a0298dcf785ee7bd0e26c7844823ce59.tar.bz2 | |
gh-113028: Correctly memoize str in pickle when escapes added (GH-113436)
This fixes a divergence between the Python and C implementations of pickle
for protocol 0, such that it pickle.py fails to re-use the first pickled
representation of strings involving characters that have to be escaped.
| -rw-r--r-- | Lib/pickle.py | 14 | ||||
| -rw-r--r-- | Lib/test/pickletester.py | 8 | ||||
| -rw-r--r-- | Misc/NEWS.d/next/Library/2023-12-23-16-51-17.gh-issue-113028.3Jmdoj.rst | 6 |
3 files changed, 21 insertions, 7 deletions
diff --git a/Lib/pickle.py b/Lib/pickle.py index 4f5ad5b..988c088 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -857,13 +857,13 @@ class _Pickler: else: self.write(BINUNICODE + pack("<I", n) + encoded) else: - obj = obj.replace("\\", "\\u005c") - obj = obj.replace("\0", "\\u0000") - obj = obj.replace("\n", "\\u000a") - obj = obj.replace("\r", "\\u000d") - obj = obj.replace("\x1a", "\\u001a") # EOF on DOS - self.write(UNICODE + obj.encode('raw-unicode-escape') + - b'\n') + # Escape what raw-unicode-escape doesn't, but memoize the original. + tmp = obj.replace("\\", "\\u005c") + tmp = tmp.replace("\0", "\\u0000") + tmp = tmp.replace("\n", "\\u000a") + tmp = tmp.replace("\r", "\\u000d") + tmp = tmp.replace("\x1a", "\\u001a") # EOF on DOS + self.write(UNICODE + tmp.encode('raw-unicode-escape') + b'\n') self.memoize(obj) dispatch[str] = save_str diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index fd446c8..74b82ca 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -1825,6 +1825,14 @@ class AbstractPickleTests: t2 = self.loads(p) self.assert_is_copy(t, t2) + def test_unicode_memoization(self): + # Repeated str is re-used (even when escapes added). + for proto in protocols: + for s in '', 'xyz', 'xyz\n', 'x\\yz', 'x\xa1yz\r': + p = self.dumps((s, s), proto) + s1, s2 = self.loads(p) + self.assertIs(s1, s2) + def test_bytes(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: diff --git a/Misc/NEWS.d/next/Library/2023-12-23-16-51-17.gh-issue-113028.3Jmdoj.rst b/Misc/NEWS.d/next/Library/2023-12-23-16-51-17.gh-issue-113028.3Jmdoj.rst new file mode 100644 index 0000000..5f66d6a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2023-12-23-16-51-17.gh-issue-113028.3Jmdoj.rst @@ -0,0 +1,6 @@ +When a second reference to a string appears in the input to :mod:`pickle`, +and the Python implementation is in use, +we are guaranteed that a single copy gets pickled +and a single object is shared when reloaded. +Previously, in protocol 0, when a string contained certain characters +(e.g. newline) it resulted in duplicate objects. |