Movatterモバイル変換

[0]ホーム
URL:

Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly

 Sign up
Appearance settings

bpo-44170: Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList#26328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.

Already on GitHub?Sign in to your account

Open
junnplus wants to merge2 commits intopython:main
base:main
Choose a base branch
Loading
fromjunnplus:bpo44170-multibyte-fix
Open
Show file tree
Hide file tree
Changes fromall commits
Commits
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletionsLib/multiprocessing/shared_memory.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -293,14 +293,21 @@ def _extract_recreation_code(value):
else:
return 3 # NoneType

@staticmethod
def _encode_value(value):
if not isinstance(value, str):
return value
else:
return value.encode(_encoding)

def __init__(self, sequence=None, *, name=None):
if name is None or sequence is not None:
sequence = sequence or ()
_formats = [
self._types_mapping[type(item)]
if not isinstance(item, (str, bytes))
else self._types_mapping[type(item)] % (
self._alignment * (len(item) // self._alignment + 1),
self._alignment * (len(self._encode_value(item)) // self._alignment + 1),
)
for item in sequence
]
Expand DownExpand Up@@ -341,7 +348,7 @@ def __init__(self, sequence=None, *, name=None):
"".join(_formats),
self.shm.buf,
self._offset_data_start,
*(v.encode(_enc) if isinstance(v, str) else v for v in sequence)
*(self._encode_value(v) for v in sequence)
)
struct.pack_into(
self._format_packing_metainfo,
Expand DownExpand Up@@ -451,9 +458,8 @@ def __setitem__(self, position, value):
else:
allocated_length = self._allocated_offsets[position + 1] - item_offset

encoded_value = (value.encode(_encoding)
if isinstance(value, str) else value)
if len(encoded_value) > allocated_length:
encoded_value = self._encode_value(value)
if len(encoded_value) >= allocated_length:
Copy link
Author

@junnplusjunnplusMay 24, 2021•
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others.Learn more.

>>>frommultiprocessing.shared_memoryimportShareableList>>>s1=ShareableList(['1234567'])>>>s1.format'8s'>>>s2=ShareableList(['12345678'])>>>s2.format'16s'>>>s1[0]='12345678'# Is this behavior expected?>>>s1.format'8s'>>>s3=ShareableList(s1)>>>s3.format'16s'

raise ValueError("bytes/str item exceeds available storage")
if current_format[-1] == "s":
new_format = current_format
Expand Down
24 changes: 12 additions & 12 deletionsLib/test/_test_multiprocessing.py
View file
Open in desktop
Original file line numberDiff line numberDiff line change
Expand Up@@ -3997,7 +3997,7 @@ def test_shared_memory_SharedMemoryManager_basics(self):

def test_shared_memory_ShareableList_basics(self):
sl = shared_memory.ShareableList(
['howdy', b'HoWdY', -273.154, 100, None, True, 42]
['howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥']
)
self.addCleanup(sl.shm.unlink)

Expand All@@ -4007,22 +4007,22 @@ def test_shared_memory_ShareableList_basics(self):

# Index Out of Range (get)
with self.assertRaises(IndexError):
sl[7]
sl[8]

# Index Out of Range (set)
with self.assertRaises(IndexError):
sl[7] = 2
sl[8] = 2

# Assign value without format change (str -> str)
current_format = sl._get_packing_format(0)
sl[0] = 'howdy'
self.assertEqual(current_format, sl._get_packing_format(0))

# Verify attributes are readable.
self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q')
self.assertEqual(sl.format, '8s8sdqxxxxxx?xxxxxxxx?q16s')

# Exercise len().
self.assertEqual(len(sl),7)
self.assertEqual(len(sl),8)

# Exercise index().
with warnings.catch_warnings():
Expand All@@ -4034,30 +4034,30 @@ def test_shared_memory_ShareableList_basics(self):

# Exercise retrieving individual values.
self.assertEqual(sl[0], 'howdy')
self.assertEqual(sl[-2], True)
self.assertEqual(sl[-3], True)
self.assertEqual(sl[-1], '💥 💥')

# Exercise iterability.
self.assertEqual(
tuple(sl),
('howdy', b'HoWdY', -273.154, 100, None, True, 42)
('howdy', b'HoWdY', -273.154, 100, None, True, 42, '💥 💥')
)

# Exercise modifying individual values.
sl[3] = 42
self.assertEqual(sl[3], 42)
sl[4] = 'some' # Change type at a given position.
self.assertEqual(sl[4], 'some')
self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q')
self.assertEqual(sl.format, '8s8sdq8sxxxxxxx?q16s')
with self.assertRaisesRegex(ValueError,
"exceeds available storage"):
sl[4] = 'far too many'
self.assertEqual(sl[4], 'some')
sl[0] = 'encodés' # Exactly 8 bytes of UTF-8 data
self.assertEqual(sl[0], 'encodés')
self.assertEqual(sl[1], b'HoWdY') # no spillage
sl[0] = 'encodé'
self.assertEqual(sl[0], 'encodé') # no spillage
with self.assertRaisesRegex(ValueError,
"exceeds available storage"):
sl[0] = 'encodées' # Exactly9 bytes of UTF-8 data
sl[0] = 'encodés' # Exactly8 bytes of UTF-8 data
self.assertEqual(sl[1], b'HoWdY')
with self.assertRaisesRegex(ValueError,
"exceeds available storage"):
Expand Down
View file
Open in desktop
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
Fix UnicodeDecodeError with multibyte utf8 characters in ShareableList.
[8]ページ先頭
©2009-2026 Movatter.jp