From 55ca7788a98fe8ae9e43e66a30a02385c8cb7b31 Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Thu, 26 Jun 2025 21:49:55 +1200 Subject: [PATCH 1/3] gh-135862: fix asyncio socket partial writes of non-1d-binary arrays The :mod:`asyncio` code is writing binary data to a :class:`socket.socket`, and advancing through the buffer after a partial write by doing ``memoryview(data)[n:]``, where ``n`` is the number of bytes written. This assumes the :class:`memoryview` is a one dimensional buffer of bytes, which is not the case e.g. for a :class:`memoryview` of an :class:`array.array` of non-bytes, or an ``ndarray`` with more than one dimension. Partial writes of such :class:`memoryview`s will corrupt the stream, repeating or omitting some data. When creating a :class:`memoryview` from a :class:`memoryview`, first convert it to a one-dimensional array of bytes before taking the remaining slice, thus avoiding these issues. Add an assertion that the remaining data is bytes, to guard against possible regressions if additional types of data are allowed in the future. --- Lib/asyncio/selector_events.py | 6 ++- Lib/test/test_asyncio/test_selector_events.py | 40 ++++++++++++++++++- ...-06-26-22-12-13.gh-issue-135862.wYKSKx.rst | 2 + 3 files changed, 46 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-06-26-22-12-13.gh-issue-135862.wYKSKx.rst diff --git a/Lib/asyncio/selector_events.py b/Lib/asyncio/selector_events.py index 6ad84044adf146..cbe011571acef5 100644 --- a/Lib/asyncio/selector_events.py +++ b/Lib/asyncio/selector_events.py @@ -1077,7 +1077,11 @@ def write(self, data): self._fatal_error(exc, 'Fatal write error on socket transport') return else: - data = memoryview(data)[n:] + if isinstance(data, memoryview): + data = data.cast('c')[n:] + else: + data = memoryview(data)[n:] + assert(data.itemsize == 1) if not data: return # Not all was written; register write handler. diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index aab6a779170eb9..c8f884418b63a2 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -1,5 +1,6 @@ """Tests for selector_events.py""" +import array import collections import selectors import socket @@ -13,6 +14,11 @@ except ImportError: ssl = None +try: + from _testbuffer import * +except ImportError: + ndarray = None + import asyncio from asyncio.selector_events import (BaseSelectorEventLoop, _SelectorDatagramTransport, @@ -768,7 +774,39 @@ def test_write_partial_memoryview(self): transport.write(data) self.loop.assert_writer(7, transport._write_ready) - self.assertEqual(list_to_buffer([b'ta']), transport._buffer) + self.assertEqualBufferContents(b'ta', transport._buffer) + + def test_write_partial_nonbyte_array_memview_write(self): + arr = array.array('l', [-1, 1]) + data = memoryview(arr) + + self.sock.send.return_value = 8 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + remainder = memoryview(array.array('l', [1])) + self.assertEqualBufferContents(remainder.tobytes(), transport._buffer) + + @unittest.skipUnless(ndarray, 'ndarray object required for this test') + def test_write_partial_ndarray_memview_write(self): + items = (-2, -1, 1, 2) + arr = ndarray(items, format='l', shape=(1, 2, 2)) + data = memoryview(arr) + + self.sock.send.return_value = 16 + + transport = self.socket_transport() + transport.write(data) + + self.loop.assert_writer(7, transport._write_ready) + remainder = memoryview(array.array('l', (1, 2))) + self.assertEqualBufferContents(remainder.tobytes(), transport._buffer) + + def assertEqualBufferContents(self, expected, buffer): + self.assertEqual(len(buffer), 1) + self.assertEqual(expected, buffer[0].tobytes()) def test_write_partial_none(self): data = b'data' diff --git a/Misc/NEWS.d/next/Library/2025-06-26-22-12-13.gh-issue-135862.wYKSKx.rst b/Misc/NEWS.d/next/Library/2025-06-26-22-12-13.gh-issue-135862.wYKSKx.rst new file mode 100644 index 00000000000000..c3f4749095699d --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-06-26-22-12-13.gh-issue-135862.wYKSKx.rst @@ -0,0 +1,2 @@ +Fix bug where partial writes of :class:`memoryview` s of non-byte or 2+ +dimensional arrays would write remaining binary data incorrectly. From 1aea9678e8b360bbea82fb8ecb94029e24a901f6 Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Thu, 26 Jun 2025 23:26:59 +1200 Subject: [PATCH 2/3] Don't assume the byte size of arrays of longs Hopefully this will fix the windows compile failures... --- Lib/test/test_asyncio/test_selector_events.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index c8f884418b63a2..309c23f7a14af3 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -780,7 +780,7 @@ def test_write_partial_nonbyte_array_memview_write(self): arr = array.array('l', [-1, 1]) data = memoryview(arr) - self.sock.send.return_value = 8 + self.sock.send.return_value = len(data) * data.itemsize // 2 transport = self.socket_transport() transport.write(data) @@ -795,7 +795,7 @@ def test_write_partial_ndarray_memview_write(self): arr = ndarray(items, format='l', shape=(1, 2, 2)) data = memoryview(arr) - self.sock.send.return_value = 16 + self.sock.send.return_value = len(data) * data.itemsize // 2 transport = self.socket_transport() transport.write(data) From 8ba5a979540448316d567a03167c68500335944e Mon Sep 17 00:00:00 2001 From: Duane Griffin Date: Thu, 26 Jun 2025 23:37:02 +1200 Subject: [PATCH 3/3] Fix byte size calculation --- Lib/test/test_asyncio/test_selector_events.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_asyncio/test_selector_events.py b/Lib/test/test_asyncio/test_selector_events.py index 309c23f7a14af3..21233d1f38031a 100644 --- a/Lib/test/test_asyncio/test_selector_events.py +++ b/Lib/test/test_asyncio/test_selector_events.py @@ -780,7 +780,7 @@ def test_write_partial_nonbyte_array_memview_write(self): arr = array.array('l', [-1, 1]) data = memoryview(arr) - self.sock.send.return_value = len(data) * data.itemsize // 2 + self.sock.send.return_value = len(arr) * data.itemsize // 2 transport = self.socket_transport() transport.write(data) @@ -795,7 +795,7 @@ def test_write_partial_ndarray_memview_write(self): arr = ndarray(items, format='l', shape=(1, 2, 2)) data = memoryview(arr) - self.sock.send.return_value = len(data) * data.itemsize // 2 + self.sock.send.return_value = arr.nbytes // 2 transport = self.socket_transport() transport.write(data)