From e240327ee3fc5793815bda8f288a60a4415d1fd7 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 15:09:42 -0800 Subject: [PATCH 01/23] gh-129559: Add `bytearray.resize()` Add `bytearray.resize()` which wraps `PyByteArray_Resize` --- Doc/library/stdtypes.rst | 7 ++++ Lib/test/test_bytes.py | 27 ++++++++++++ ...-02-01-14-55-33.gh-issue-129559.hQCeAz.rst | 3 ++ Objects/bytearrayobject.c | 28 ++++++++++++- Objects/clinic/bytearrayobject.c.h | 41 ++++++++++++++++++- 5 files changed, 104 insertions(+), 2 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2025-02-01-14-55-33.gh-issue-129559.hQCeAz.rst diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 6050784264707b..bcf805a6e46b53 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2841,6 +2841,13 @@ objects. optional *sep* and *bytes_per_sep* parameters to insert separators between bytes in the hex output. + .. method:: resize(size) + + Resize the :class:`bytearray` to contain size bytes with a NULL byte + following. + + .. versionadded:: next + Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be a bytearray object of length 1. (This contrasts with text strings, where diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 7bb1ab38aa4fdf..9a52c789bb3768 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1359,6 +1359,33 @@ def by(s): b = by("Hello, world") self.assertEqual(re.findall(br"\w+", b), [by("Hello"), by("world")]) + def test_resize(self): + ba = bytearray(b'abcdef') + self.assertIsNone(ba.resize(3)) + self.assertEqual(ba, bytearray(b'abc')) + self.assertIsNone(ba.resize(10)) + self.assertEqual(len(ba), 10) + self.assertEqual(ba[:3], bytearray(b'abc')) + self.assertIsNone(ba.resize(2**20)) + self.assertEqual(len(ba), 2**20) + self.assertEqual(ba[:3], bytearray(b'abc')) + self.assertIsNone(ba.resize(0)) + self.assertEqual(ba, bytearray()) + + ba = ByteArraySubclass(b'abcdef') + self.assertIsNone(ba.resize(3)) + self.assertEqual(ba, bytearray(b'abc')) + + # Check arguments + self.assertRaises(TypeError, lambda: bytearray().resize()) + self.assertRaises(TypeError, lambda: bytearray().resize(10, 10)) + + self.assertRaises(BufferError, lambda: bytearray().resize(-1)) + self.assertRaises(BufferError, lambda: bytearray().resize(-200)) + self.assertRaises(MemoryError, lambda: bytearray().resize(sys.maxsize)) + self.assertRaises(MemoryError, lambda: bytearray(1000).resize(sys.maxsize)) + + def test_setitem(self): def setitem_as_mapping(b, i, val): b[i] = val diff --git a/Misc/NEWS.d/next/Library/2025-02-01-14-55-33.gh-issue-129559.hQCeAz.rst b/Misc/NEWS.d/next/Library/2025-02-01-14-55-33.gh-issue-129559.hQCeAz.rst new file mode 100644 index 00000000000000..bc56b77a73835a --- /dev/null +++ b/Misc/NEWS.d/next/Library/2025-02-01-14-55-33.gh-issue-129559.hQCeAz.rst @@ -0,0 +1,3 @@ +Add :meth:`bytearray.resize` method to :class:`bytearray` wrapping +:c:func:`PyByteArray_Resize` so :class:`bytearray` can be efficiently +resized in place. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 21584332e0e443..32730512825d48 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -184,7 +184,12 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) assert(self != NULL); assert(PyByteArray_Check(self)); assert(logical_offset <= alloc); - assert(requested_size >= 0); + + if (requested_size < 0) { + PyErr_Format(PyExc_BufferError, + "Can only resize to positive sizes, got %zd.", requested_size); + return -1; + } if (requested_size == Py_SIZE(self)) { return 0; @@ -1388,6 +1393,26 @@ bytearray_removesuffix_impl(PyByteArrayObject *self, Py_buffer *suffix) } +/*[clinic input] +bytearray.resize + size: Py_ssize_t + New size to resize to.. + / +Resize the internal buffer of bytearray to len. +[clinic start generated code]*/ + +static PyObject * +bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) +/*[clinic end generated code: output=f73524922990b2d9 input=75fd4d17c4aa47d3]*/ +{ + int result = PyByteArray_Resize((PyObject *)self, size); + if (result == -1) { + return NULL; + } + Py_RETURN_NONE; +} + + /*[clinic input] bytearray.translate @@ -2361,6 +2386,7 @@ static PyMethodDef bytearray_methods[] = { BYTEARRAY_REPLACE_METHODDEF BYTEARRAY_REMOVEPREFIX_METHODDEF BYTEARRAY_REMOVESUFFIX_METHODDEF + BYTEARRAY_RESIZE_METHODDEF BYTEARRAY_REVERSE_METHODDEF BYTEARRAY_RFIND_METHODDEF BYTEARRAY_RINDEX_METHODDEF diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h index 91cf5363e639d1..03b5a8a516cc09 100644 --- a/Objects/clinic/bytearrayobject.c.h +++ b/Objects/clinic/bytearrayobject.c.h @@ -565,6 +565,45 @@ bytearray_removesuffix(PyObject *self, PyObject *arg) return return_value; } +PyDoc_STRVAR(bytearray_resize__doc__, +"resize($self, size, /)\n" +"--\n" +"\n" +"Resize the internal buffer of bytearray to len.\n" +"\n" +" size\n" +" New size to resize to.."); + +#define BYTEARRAY_RESIZE_METHODDEF \ + {"resize", (PyCFunction)bytearray_resize, METH_O, bytearray_resize__doc__}, + +static PyObject * +bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size); + +static PyObject * +bytearray_resize(PyObject *self, PyObject *arg) +{ + PyObject *return_value = NULL; + Py_ssize_t size; + + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(arg); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + size = ival; + } + return_value = bytearray_resize_impl((PyByteArrayObject *)self, size); + +exit: + return return_value; +} + PyDoc_STRVAR(bytearray_translate__doc__, "translate($self, table, /, delete=b\'\')\n" "--\n" @@ -1623,4 +1662,4 @@ bytearray_sizeof(PyObject *self, PyObject *Py_UNUSED(ignored)) { return bytearray_sizeof_impl((PyByteArrayObject *)self); } -/*[clinic end generated code: output=bc8bec8514102bf3 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=41bb67a8a181e733 input=a9049054013a1b77]*/ From 925fcbd7634c6d01bdf2eedd75efc80b402734e6 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 15:46:21 -0800 Subject: [PATCH 02/23] Add versionchanged note to c-api around len behavior --- Doc/c-api/bytearray.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 9045689a6be567..ee8926b0bf5755 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -75,6 +75,10 @@ Direct API functions Resize the internal buffer of *bytearray* to *len*. + versionchanged:: 3.14 + A negative *len* will now result in a failure with ``-1`` return and a + exeception set. + Macros ^^^^^^ From 29c04ad30362e0c0a291f484be2ffd7321a79eb1 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 15:52:52 -0800 Subject: [PATCH 03/23] Add argument formatting to stdtypes.rst --- Doc/library/stdtypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index bcf805a6e46b53..72a2505c11729c 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2843,7 +2843,7 @@ objects. .. method:: resize(size) - Resize the :class:`bytearray` to contain size bytes with a NULL byte + Resize the :class:`bytearray` to contain *size* bytes with a NULL byte following. .. versionadded:: next From d15ef67cd8db3044f7e7ebb25c46b09caf72f362 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 16:28:18 -0800 Subject: [PATCH 04/23] Add buffer error to c api test --- Lib/test/test_capi/test_bytearray.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index 39099f6b82240f..29a87b9dd8a2f4 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -151,6 +151,8 @@ def test_resize(self): self.assertEqual(resize(ba, 3), 0) self.assertEqual(ba, bytearray(b'abc')) + self.assertRaises(BufferError, resize, bytearray(), -1) + self.assertRaises(BufferError, resize, bytearray(), -200) self.assertRaises(MemoryError, resize, bytearray(), PY_SSIZE_T_MAX) self.assertRaises(MemoryError, resize, bytearray(1000), PY_SSIZE_T_MAX) From d199032248c666c194a106ae32492532d7404de1 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 16:40:51 -0800 Subject: [PATCH 05/23] Remove case that no longer crashes --- Lib/test/test_capi/test_bytearray.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index 29a87b9dd8a2f4..d90c600c28ebfa 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -156,7 +156,6 @@ def test_resize(self): self.assertRaises(MemoryError, resize, bytearray(), PY_SSIZE_T_MAX) self.assertRaises(MemoryError, resize, bytearray(1000), PY_SSIZE_T_MAX) - # CRASHES resize(bytearray(b'abc'), -1) # CRASHES resize(b'abc', 0) # CRASHES resize(object(), 0) # CRASHES resize(NULL, 0) From 18829a2a1f86bee390da331faa851cc23103ad73 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 17:16:17 -0800 Subject: [PATCH 06/23] Fix versionchanged --- Doc/c-api/bytearray.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index ee8926b0bf5755..32bd3a2df89b20 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -75,7 +75,7 @@ Direct API functions Resize the internal buffer of *bytearray* to *len*. - versionchanged:: 3.14 + .. versionchanged:: next A negative *len* will now result in a failure with ``-1`` return and a exeception set. From df779e2b420d064cf1ecbedc5515c68bc087a60b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 17:37:29 -0800 Subject: [PATCH 07/23] Fix doc warnings --- Doc/c-api/bytearray.rst | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 32bd3a2df89b20..8e36264c0537e8 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -76,8 +76,9 @@ Direct API functions Resize the internal buffer of *bytearray* to *len*. .. versionchanged:: next - A negative *len* will now result in a failure with ``-1`` return and a - exeception set. + A negative *len* will now result in a failure with ``-1`` return and a + exeception set. + Macros ^^^^^^ From facc91f82c0189fbcc030bedb6003d3a481c8605 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 18:11:38 -0800 Subject: [PATCH 08/23] Fix requested size check range --- Objects/bytearrayobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 32730512825d48..995b08221dab93 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -185,7 +185,7 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) assert(PyByteArray_Check(self)); assert(logical_offset <= alloc); - if (requested_size < 0) { + if (requested_size <= 0) { PyErr_Format(PyExc_BufferError, "Can only resize to positive sizes, got %zd.", requested_size); return -1; From d8b9faf5a8f718bb4d345c15762916e3e0807c6a Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 19:03:31 -0800 Subject: [PATCH 09/23] 0 is a fine size --- Objects/bytearrayobject.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 995b08221dab93..32730512825d48 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -185,7 +185,7 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) assert(PyByteArray_Check(self)); assert(logical_offset <= alloc); - if (requested_size <= 0) { + if (requested_size < 0) { PyErr_Format(PyExc_BufferError, "Can only resize to positive sizes, got %zd.", requested_size); return -1; From 7429cf46b251ec57c1338e282d896cf46354cd61 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sat, 1 Feb 2025 19:11:31 -0800 Subject: [PATCH 10/23] Fix grammar --- Doc/c-api/bytearray.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 8e36264c0537e8..bbec3e846d87ba 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -76,7 +76,7 @@ Direct API functions Resize the internal buffer of *bytearray* to *len*. .. versionchanged:: next - A negative *len* will now result in a failure with ``-1`` return and a + A negative *len* will now result in a failure with ``-1`` return and an exeception set. From c1831160f108d7a960f2eb9d1cba281d128c6d3e Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sun, 2 Feb 2025 01:46:45 -0800 Subject: [PATCH 11/23] Add NULL byte tests, include set bytes in docs --- Doc/library/stdtypes.rst | 8 ++++++-- Lib/test/test_bytes.py | 7 +++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 72a2505c11729c..b94a456ac448cf 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2843,8 +2843,12 @@ objects. .. method:: resize(size) - Resize the :class:`bytearray` to contain *size* bytes with a NULL byte - following. + Resize the :class:`bytearray` using :ref:`Memory Management ` APIs + to contain *size* bytes. Data from the existing :class:`bytearray` will be + kept up to *size*. Any new bytes (the :class:`bytearray` grew) will be + unchanged from the memory allocator except one byte beyond the requested + *size* which will be set to NULL to ensure the :class:`bytearray` is + always NULL terminated. .. versionadded:: next diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 9a52c789bb3768..fb3ac724c6c012 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1365,10 +1365,13 @@ def test_resize(self): self.assertEqual(ba, bytearray(b'abc')) self.assertIsNone(ba.resize(10)) self.assertEqual(len(ba), 10) - self.assertEqual(ba[:3], bytearray(b'abc')) + # Check bytes are intact after resize including the NULL from .resize(3) + self.assertEqual(ba[:4], bytearray(b'abc\0')) self.assertIsNone(ba.resize(2**20)) self.assertEqual(len(ba), 2**20) - self.assertEqual(ba[:3], bytearray(b'abc')) + self.assertEqual(ba[:4], bytearray(b'abc\0')) + # Check for NULL from .resize(10) + self.assertEqual(ba[10], 0) self.assertIsNone(ba.resize(0)) self.assertEqual(ba, bytearray()) From dd46a85097814161cb0e225be6490b979c88ce8b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sun, 2 Feb 2025 09:55:55 -0800 Subject: [PATCH 12/23] Update Doc/library/stdtypes.rst Co-authored-by: Victorien <65306057+Viicos@users.noreply.github.com> --- Doc/library/stdtypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index b94a456ac448cf..ded39e4cdeb055 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2850,7 +2850,7 @@ objects. *size* which will be set to NULL to ensure the :class:`bytearray` is always NULL terminated. - .. versionadded:: next + .. versionadded:: next Since bytearray objects are sequences of integers (akin to a list), for a bytearray object *b*, ``b[0]`` will be an integer, while ``b[0:1]`` will be From 69bbdc15c78bdd171aece7a405b0228489fecace Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Sun, 2 Feb 2025 12:03:40 -0800 Subject: [PATCH 13/23] Always null new bytes --- Doc/library/stdtypes.rst | 8 ++++---- Lib/test/test_bytes.py | 14 +++++++++----- Objects/bytearrayobject.c | 5 +++++ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index ded39e4cdeb055..c7da9325a5f1a2 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2845,10 +2845,10 @@ objects. Resize the :class:`bytearray` using :ref:`Memory Management ` APIs to contain *size* bytes. Data from the existing :class:`bytearray` will be - kept up to *size*. Any new bytes (the :class:`bytearray` grew) will be - unchanged from the memory allocator except one byte beyond the requested - *size* which will be set to NULL to ensure the :class:`bytearray` is - always NULL terminated. + kept up to *size*. If :class:`bytearray` needs to grow, all new bytes will + be zeroed / set to the NULL byte (``b"\0"``). One byte beyond *size* will + always be set to NULL to ensure the :class:`bytearray` is always NULL + terminated regardless of contents. .. versionadded:: next diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index fb3ac724c6c012..f2ffbf132311bf 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1365,15 +1365,19 @@ def test_resize(self): self.assertEqual(ba, bytearray(b'abc')) self.assertIsNone(ba.resize(10)) self.assertEqual(len(ba), 10) - # Check bytes are intact after resize including the NULL from .resize(3) - self.assertEqual(ba[:4], bytearray(b'abc\0')) + # Bytes beyond set values must be cleared. + self.assertEqual(ba, bytearray(b'abc\0\0\0\0\0\0\0')) + expected_bytearray = bytearray(b'abcdeabcde') + ba[3:10] = b'deabcde' + self.assertEqual(ba, expected_bytearray) self.assertIsNone(ba.resize(2**20)) self.assertEqual(len(ba), 2**20) - self.assertEqual(ba[:4], bytearray(b'abc\0')) - # Check for NULL from .resize(10) - self.assertEqual(ba[10], 0) + self.assertEqual(ba[:10], expected_bytearray) + self.assertEqual(ba[10:], bytearray(b'\0' * (2 ** 20 - 10))) self.assertIsNone(ba.resize(0)) self.assertEqual(ba, bytearray()) + self.assertIsNone(ba.resize(10)) + self.assertEqual(ba, bytearray(b'\0' * 10)) ba = ByteArraySubclass(b'abcdef') self.assertIsNone(ba.resize(3)) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 32730512825d48..c9b19bb6475a01 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1405,10 +1405,15 @@ static PyObject * bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) /*[clinic end generated code: output=f73524922990b2d9 input=75fd4d17c4aa47d3]*/ { + Py_ssize_t start_size = PyByteArray_GET_SIZE(self); int result = PyByteArray_Resize((PyObject *)self, size); if (result == -1) { return NULL; } + // Set new bytes to provide consistent / safer behavior in Python version. + if (size > start_size) { + memset(PyByteArray_AS_STRING(self) + start_size, 0, size-start_size); + } Py_RETURN_NONE; } From ec4aa3da41b1e9478a43cab8b203ff3f1b5d751f Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 4 Feb 2025 10:23:42 -0800 Subject: [PATCH 14/23] Apply suggestions from code review Co-authored-by: Peter Bierma Co-authored-by: Victor Stinner --- Doc/c-api/bytearray.rst | 5 ++--- Doc/library/stdtypes.rst | 8 ++------ Lib/test/test_bytes.py | 10 ++++------ Objects/bytearrayobject.c | 4 ++-- 4 files changed, 10 insertions(+), 17 deletions(-) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index bbec3e846d87ba..9fecaef9a3fe6d 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -73,11 +73,10 @@ Direct API functions .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) - Resize the internal buffer of *bytearray* to *len*. + Resize the internal buffer of *bytearray* to *len*. Failure is a ``-1`` return with an exception set. .. versionchanged:: next - A negative *len* will now result in a failure with ``-1`` return and an - exeception set. + A negative *len* will now result in a failure. Macros diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index c7da9325a5f1a2..26e64f6c9ec725 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2843,12 +2843,8 @@ objects. .. method:: resize(size) - Resize the :class:`bytearray` using :ref:`Memory Management ` APIs - to contain *size* bytes. Data from the existing :class:`bytearray` will be - kept up to *size*. If :class:`bytearray` needs to grow, all new bytes will - be zeroed / set to the NULL byte (``b"\0"``). One byte beyond *size* will - always be set to NULL to ensure the :class:`bytearray` is always NULL - terminated regardless of contents. + Resize the :class:`bytearray` to contain *size* bytes. + If :class:`bytearray` needs to grow, all new bytes will be set to null bytes. .. versionadded:: next diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index f2ffbf132311bf..1d58593293e892 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1367,13 +1367,11 @@ def test_resize(self): self.assertEqual(len(ba), 10) # Bytes beyond set values must be cleared. self.assertEqual(ba, bytearray(b'abc\0\0\0\0\0\0\0')) - expected_bytearray = bytearray(b'abcdeabcde') - ba[3:10] = b'deabcde' - self.assertEqual(ba, expected_bytearray) + ba[3:10] = b'defghij' + self.assertEqual(ba, bytearray(b'abcdefghij')) self.assertIsNone(ba.resize(2**20)) self.assertEqual(len(ba), 2**20) - self.assertEqual(ba[:10], expected_bytearray) - self.assertEqual(ba[10:], bytearray(b'\0' * (2 ** 20 - 10))) + self.assertEqual(ba, bytearray(b'abcdefghij' + b'\0' * (2 ** 20 - 10))) self.assertIsNone(ba.resize(0)) self.assertEqual(ba, bytearray()) self.assertIsNone(ba.resize(10)) @@ -1385,7 +1383,7 @@ def test_resize(self): # Check arguments self.assertRaises(TypeError, lambda: bytearray().resize()) - self.assertRaises(TypeError, lambda: bytearray().resize(10, 10)) + self.assertRaises(TypeError, bytearray().resize, 10, 10) self.assertRaises(BufferError, lambda: bytearray().resize(-1)) self.assertRaises(BufferError, lambda: bytearray().resize(-200)) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index c9b19bb6475a01..448e2801aa9238 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1407,12 +1407,12 @@ bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) { Py_ssize_t start_size = PyByteArray_GET_SIZE(self); int result = PyByteArray_Resize((PyObject *)self, size); - if (result == -1) { + if (result < 0) { return NULL; } // Set new bytes to provide consistent / safer behavior in Python version. if (size > start_size) { - memset(PyByteArray_AS_STRING(self) + start_size, 0, size-start_size); + memset(PyByteArray_AS_STRING(self) + start_size, 0, size - start_size); } Py_RETURN_NONE; } From 336299ada6e30fbbf90783e0de56ad08eda5ba3d Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 4 Feb 2025 10:34:32 -0800 Subject: [PATCH 15/23] Update Objects/bytearrayobject.c Co-authored-by: Victor Stinner --- Objects/bytearrayobject.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 448e2801aa9238..82699f1276a71e 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -186,8 +186,8 @@ PyByteArray_Resize(PyObject *self, Py_ssize_t requested_size) assert(logical_offset <= alloc); if (requested_size < 0) { - PyErr_Format(PyExc_BufferError, - "Can only resize to positive sizes, got %zd.", requested_size); + PyErr_Format(PyExc_ValueError, + "Can only resize to positive sizes, got %zd", requested_size); return -1; } From 1a0e1570903014f3588f16b2971033343a7f2aeb Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 4 Feb 2025 13:03:40 -0800 Subject: [PATCH 16/23] tweak docs, fixup return exception --- Doc/library/stdtypes.rst | 2 ++ Lib/test/test_bytes.py | 12 ++++++------ Lib/test/test_capi/test_bytearray.py | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 26e64f6c9ec725..97f93c83b58b53 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2846,6 +2846,8 @@ objects. Resize the :class:`bytearray` to contain *size* bytes. If :class:`bytearray` needs to grow, all new bytes will be set to null bytes. + This is equivalent to ``self += b'\0' * size`` + .. versionadded:: next Since bytearray objects are sequences of integers (akin to a list), for a diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 1d58593293e892..394de2fcc66b98 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1382,13 +1382,13 @@ def test_resize(self): self.assertEqual(ba, bytearray(b'abc')) # Check arguments - self.assertRaises(TypeError, lambda: bytearray().resize()) - self.assertRaises(TypeError, bytearray().resize, 10, 10) + self.assertRaises(TypeError, bytearray().resize) + self.assertRaises(TypeError, bytearray().resize, (10, 10)) - self.assertRaises(BufferError, lambda: bytearray().resize(-1)) - self.assertRaises(BufferError, lambda: bytearray().resize(-200)) - self.assertRaises(MemoryError, lambda: bytearray().resize(sys.maxsize)) - self.assertRaises(MemoryError, lambda: bytearray(1000).resize(sys.maxsize)) + self.assertRaises(ValueError, bytearray().resize, -1) + self.assertRaises(ValueError, bytearray().resize, -200) + self.assertRaises(MemoryError, bytearray().resize, sys.maxsize) + self.assertRaises(MemoryError, bytearray(1000).resize, sys.maxsize) def test_setitem(self): diff --git a/Lib/test/test_capi/test_bytearray.py b/Lib/test/test_capi/test_bytearray.py index d90c600c28ebfa..323e0d2a5acdcb 100644 --- a/Lib/test/test_capi/test_bytearray.py +++ b/Lib/test/test_capi/test_bytearray.py @@ -151,8 +151,8 @@ def test_resize(self): self.assertEqual(resize(ba, 3), 0) self.assertEqual(ba, bytearray(b'abc')) - self.assertRaises(BufferError, resize, bytearray(), -1) - self.assertRaises(BufferError, resize, bytearray(), -200) + self.assertRaises(ValueError, resize, bytearray(), -1) + self.assertRaises(ValueError, resize, bytearray(), -200) self.assertRaises(MemoryError, resize, bytearray(), PY_SSIZE_T_MAX) self.assertRaises(MemoryError, resize, bytearray(1000), PY_SSIZE_T_MAX) From cf89ec16f0c808ec8904b1e0ad43ee304f498e20 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 4 Feb 2025 13:09:28 -0800 Subject: [PATCH 17/23] Update test_resize_forbidden to use bytearray.resize --- Lib/test/test_bytes.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 394de2fcc66b98..875a1409e2d240 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1747,17 +1747,18 @@ def test_resize_forbidden(self): # if it wouldn't reallocate the underlying buffer. # Furthermore, no destructive changes to the buffer may be applied # before raising the error. - b = bytearray(range(10)) + b = bytearray(10) v = memoryview(b) - def resize(n): + def manual_resize(n): b[1:-1] = range(n + 1, 2*n - 1) - resize(10) + b.resize(10) orig = b[:] - self.assertRaises(BufferError, resize, 11) + self.assertRaises(BufferError, b.resize, 11) + self.assertRaises(BufferError, manual_resize, 11) self.assertEqual(b, orig) - self.assertRaises(BufferError, resize, 9) + self.assertRaises(BufferError, b.resize, 9) self.assertEqual(b, orig) - self.assertRaises(BufferError, resize, 0) + self.assertRaises(BufferError, b.resize, 0) self.assertEqual(b, orig) # Other operations implying resize self.assertRaises(BufferError, b.pop, 0) From ddc7b0916e4daa3a4b8f7c72a3044e6e3e4b1128 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 4 Feb 2025 22:47:36 -0800 Subject: [PATCH 18/23] doc tweaks --- Doc/c-api/bytearray.rst | 2 +- Doc/library/stdtypes.rst | 16 +++++++++++++--- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index 9fecaef9a3fe6d..c88cfb30ae0809 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -76,7 +76,7 @@ Direct API functions Resize the internal buffer of *bytearray* to *len*. Failure is a ``-1`` return with an exception set. .. versionchanged:: next - A negative *len* will now result in a failure. + A negative *len* will now result in an exception being set and -1 returned. Macros diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 97f93c83b58b53..da29933634d384 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2843,10 +2843,20 @@ objects. .. method:: resize(size) - Resize the :class:`bytearray` to contain *size* bytes. - If :class:`bytearray` needs to grow, all new bytes will be set to null bytes. + Resize the :class:`bytearray` to contain *size* bytes. *size* must be + greater than or equal to 0. - This is equivalent to ``self += b'\0' * size`` + If the :class:`bytearray` needs to shrink bytes beyond *size* are truncated. + + If the :class:`bytearray` needs to grow all new bytes, those beyond *size*, + will be set to null bytes. + + + This is equivalent to: + >>> if len(self) > size: + >>> del self[size:] + >>> else: + >>> self += b'\0' * (size - len(self)) .. versionadded:: next From a49374be4607f0e22d7e546dc520073ba9d5df29 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 4 Feb 2025 22:58:27 -0800 Subject: [PATCH 19/23] Fix indentation of equivalent code --- Doc/library/stdtypes.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index da29933634d384..2588823055d97a 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2853,10 +2853,11 @@ objects. This is equivalent to: - >>> if len(self) > size: - >>> del self[size:] - >>> else: - >>> self += b'\0' * (size - len(self)) + + >>> if len(self) > size: + >>> del self[size:] + >>> else: + >>> self += b'\0' * (size - len(self)) .. versionadded:: next From 298d052b48ae2d2ad308716e9cbc84a28dfc5661 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Tue, 4 Feb 2025 23:29:00 -0800 Subject: [PATCH 20/23] Fix doctest --- Doc/library/stdtypes.rst | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 2588823055d97a..aab992a4e533a5 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2854,10 +2854,20 @@ objects. This is equivalent to: - >>> if len(self) > size: - >>> del self[size:] - >>> else: - >>> self += b'\0' * (size - len(self)) + >>> def resize(ba, size): + ... if len(ba) > size: + ... del ba[size:] + ... else: + ... ba += b'\0' * (size - len(ba)) + >>> + >>> shrink = bytearray(5) + >>> resize(shrink, 0) + >>> shrink + bytearray(b'') + >>> grow = bytearray(2) + >>> resize(grow, 7) + >>> grow + bytearray(b'\x00\x00\x00\x00\x00\x00\x00') .. versionadded:: next From 2f6b0a3c5368a5adb223f609ec8caaaccfdac86b Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 5 Feb 2025 09:42:58 -0800 Subject: [PATCH 21/23] Apply suggestions from code review Co-authored-by: Victor Stinner --- Doc/c-api/bytearray.rst | 3 ++- Doc/library/stdtypes.rst | 4 ++-- Lib/test/test_bytes.py | 6 ++++++ .../Library/2025-02-01-14-55-33.gh-issue-129559.hQCeAz.rst | 3 +-- Objects/bytearrayobject.c | 2 +- 5 files changed, 12 insertions(+), 6 deletions(-) diff --git a/Doc/c-api/bytearray.rst b/Doc/c-api/bytearray.rst index c88cfb30ae0809..15295096a710c8 100644 --- a/Doc/c-api/bytearray.rst +++ b/Doc/c-api/bytearray.rst @@ -73,7 +73,8 @@ Direct API functions .. c:function:: int PyByteArray_Resize(PyObject *bytearray, Py_ssize_t len) - Resize the internal buffer of *bytearray* to *len*. Failure is a ``-1`` return with an exception set. + Resize the internal buffer of *bytearray* to *len*. + Failure is a ``-1`` return with an exception set. .. versionchanged:: next A negative *len* will now result in an exception being set and -1 returned. diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index aab992a4e533a5..021facff498ec4 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2846,9 +2846,9 @@ objects. Resize the :class:`bytearray` to contain *size* bytes. *size* must be greater than or equal to 0. - If the :class:`bytearray` needs to shrink bytes beyond *size* are truncated. + If the :class:`bytearray` needs to shrink, bytes beyond *size* are truncated. - If the :class:`bytearray` needs to grow all new bytes, those beyond *size*, + If the :class:`bytearray` needs to grow, all new bytes, those beyond *size*, will be set to null bytes. diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 875a1409e2d240..979746c3344bda 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1363,20 +1363,26 @@ def test_resize(self): ba = bytearray(b'abcdef') self.assertIsNone(ba.resize(3)) self.assertEqual(ba, bytearray(b'abc')) + self.assertIsNone(ba.resize(10)) self.assertEqual(len(ba), 10) # Bytes beyond set values must be cleared. self.assertEqual(ba, bytearray(b'abc\0\0\0\0\0\0\0')) + ba[3:10] = b'defghij' self.assertEqual(ba, bytearray(b'abcdefghij')) + self.assertIsNone(ba.resize(2**20)) self.assertEqual(len(ba), 2**20) self.assertEqual(ba, bytearray(b'abcdefghij' + b'\0' * (2 ** 20 - 10))) + self.assertIsNone(ba.resize(0)) self.assertEqual(ba, bytearray()) + self.assertIsNone(ba.resize(10)) self.assertEqual(ba, bytearray(b'\0' * 10)) + # Subclass ba = ByteArraySubclass(b'abcdef') self.assertIsNone(ba.resize(3)) self.assertEqual(ba, bytearray(b'abc')) diff --git a/Misc/NEWS.d/next/Library/2025-02-01-14-55-33.gh-issue-129559.hQCeAz.rst b/Misc/NEWS.d/next/Library/2025-02-01-14-55-33.gh-issue-129559.hQCeAz.rst index bc56b77a73835a..f08d47b63a84b7 100644 --- a/Misc/NEWS.d/next/Library/2025-02-01-14-55-33.gh-issue-129559.hQCeAz.rst +++ b/Misc/NEWS.d/next/Library/2025-02-01-14-55-33.gh-issue-129559.hQCeAz.rst @@ -1,3 +1,2 @@ -Add :meth:`bytearray.resize` method to :class:`bytearray` wrapping -:c:func:`PyByteArray_Resize` so :class:`bytearray` can be efficiently +Add :meth:`bytearray.resize` method so :class:`bytearray` can be efficiently resized in place. diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c index 82699f1276a71e..6133d30f49930a 100644 --- a/Objects/bytearrayobject.c +++ b/Objects/bytearrayobject.c @@ -1410,7 +1410,7 @@ bytearray_resize_impl(PyByteArrayObject *self, Py_ssize_t size) if (result < 0) { return NULL; } - // Set new bytes to provide consistent / safer behavior in Python version. + // Set new bytes to null bytes if (size > start_size) { memset(PyByteArray_AS_STRING(self) + start_size, 0, size - start_size); } From bf00f334a6264b874a52b1d16b1f5cd85d17a615 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 5 Feb 2025 09:54:10 -0800 Subject: [PATCH 22/23] Update doctest per review --- Doc/library/stdtypes.rst | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/Doc/library/stdtypes.rst b/Doc/library/stdtypes.rst index 021facff498ec4..4a15e27f82a160 100644 --- a/Doc/library/stdtypes.rst +++ b/Doc/library/stdtypes.rst @@ -2859,15 +2859,17 @@ objects. ... del ba[size:] ... else: ... ba += b'\0' * (size - len(ba)) - >>> - >>> shrink = bytearray(5) - >>> resize(shrink, 0) - >>> shrink - bytearray(b'') - >>> grow = bytearray(2) - >>> resize(grow, 7) - >>> grow - bytearray(b'\x00\x00\x00\x00\x00\x00\x00') + + Examples: + + >>> shrink = bytearray(b'abc') + >>> shrink.resize(1) + >>> (shrink, len(shrink)) + (bytearray(b'a'), 1) + >>> grow = bytearray(b'abc') + >>> grow.resize(5) + >>> (grow, len(grow)) + (bytearray(b'abc\x00\x00'), 5) .. versionadded:: next From 8df7b026768d4debec4127a922a73c7df5d75193 Mon Sep 17 00:00:00 2001 From: Cody Maloney Date: Wed, 5 Feb 2025 09:55:34 -0800 Subject: [PATCH 23/23] Fix whitespace in test_bytes --- Lib/test/test_bytes.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 979746c3344bda..18d619eb6239a1 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -1363,22 +1363,22 @@ def test_resize(self): ba = bytearray(b'abcdef') self.assertIsNone(ba.resize(3)) self.assertEqual(ba, bytearray(b'abc')) - + self.assertIsNone(ba.resize(10)) self.assertEqual(len(ba), 10) # Bytes beyond set values must be cleared. self.assertEqual(ba, bytearray(b'abc\0\0\0\0\0\0\0')) - + ba[3:10] = b'defghij' self.assertEqual(ba, bytearray(b'abcdefghij')) - - self.assertIsNone(ba.resize(2**20)) + + self.assertIsNone(ba.resize(2 ** 20)) self.assertEqual(len(ba), 2**20) self.assertEqual(ba, bytearray(b'abcdefghij' + b'\0' * (2 ** 20 - 10))) - + self.assertIsNone(ba.resize(0)) self.assertEqual(ba, bytearray()) - + self.assertIsNone(ba.resize(10)) self.assertEqual(ba, bytearray(b'\0' * 10))