From 887ff8e37e238fbce18c647e588283904f38ab24 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 20 Apr 2020 21:22:50 +0200 Subject: [3.8] bpo-40330: Fix utf-8 size check in ShareableList (GH-19606) (GH-19625) The item size must be checked after encoding to bytes, not before. Automerge-Triggered-By: @pitrou. (cherry picked from commit eba9f6155df59c9beed97fb5764c9f01dd941af0) Co-authored-by: Antoine Pitrou --- Lib/multiprocessing/shared_memory.py | 10 ++++++---- Lib/test/_test_multiprocessing.py | 16 ++++++++++++++-- .../Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst | 2 ++ 3 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 184e367..f92eb01 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -433,9 +433,12 @@ class ShareableList: if not isinstance(value, (str, bytes)): new_format = self._types_mapping[type(value)] + encoded_value = value else: - if len(value) > self._allocated_bytes[position]: - raise ValueError("exceeds available storage for existing str") + encoded_value = (value.encode(_encoding) + if isinstance(value, str) else value) + if len(encoded_value) > self._allocated_bytes[position]: + raise ValueError("bytes/str item exceeds available storage") if current_format[-1] == "s": new_format = current_format else: @@ -448,8 +451,7 @@ class ShareableList: new_format, value ) - value = value.encode(_encoding) if isinstance(value, str) else value - struct.pack_into(new_format, self.shm.buf, offset, value) + struct.pack_into(new_format, self.shm.buf, offset, encoded_value) def __reduce__(self): return partial(self.__class__, name=self.shm.name), () diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index e64a8e9..d5336e4 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -3964,9 +3964,21 @@ class _TestSharedMemory(BaseTestCase): sl[4] = 'some' # Change type at a given position. self.assertEqual(sl[4], 'some') self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q') - with self.assertRaises(ValueError): - sl[4] = 'far too many' # Exceeds available storage. + with self.assertRaisesRegex(ValueError, + "exceeds available storage"): + sl[4] = 'far too many' self.assertEqual(sl[4], 'some') + sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data + self.assertEqual(sl[0], 'encodés') + self.assertEqual(sl[1], b'HoWdY') # no spillage + with self.assertRaisesRegex(ValueError, + "exceeds available storage"): + sl[0] = 'encodées' # Exactly 9 bytes of UTF-8 data + self.assertEqual(sl[1], b'HoWdY') + with self.assertRaisesRegex(ValueError, + "exceeds available storage"): + sl[1] = b'123456789' + self.assertEqual(sl[1], b'HoWdY') # Exercise count(). with warnings.catch_warnings(): diff --git a/Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst b/Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst new file mode 100644 index 0000000..98cb62f --- /dev/null +++ b/Misc/NEWS.d/next/Library/2020-04-19-17-31-29.bpo-40330.DGjoIS.rst @@ -0,0 +1,2 @@ +In :meth:`ShareableList.__setitem__`, check the size of a new string item +after encoding it to utf-8, not before. -- cgit v0.12