From e601da9f0251ea99ae51f6513257eb9e398ce108 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:36:04 -0700 Subject: [PATCH] gh-160181: Don't modify a hashed bytes buffer in place in io.BytesIO (GH-160182) Treat the internal buffer as shared once its hash is cached, so the next modification copies it, as gh-158219 did for bytearray. ANT-2026-IZN6JY41 (cherry picked from commit fd657b067473fdf549dfab4761a739e03894a659) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/test/test_memoryio.py | 46 +++++++++++++++++++ ...-10-22-00-32-51.gh-issue-160181.nnEeg1.rst | 3 ++ Modules/_io/bytesio.c | 18 ++++++-- 3 files changed, 63 insertions(+), 4 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 8ba08e629bb397..0f4da8c1bd60da 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -952,6 +952,52 @@ def test_cow_mutable(self): memio = self.ioclass(ba) self.assertEqual(sys.getrefcount(ba), old_rc) + def test_cow_hashed(self): + # A buffer which was exposed to Python code and hashed must not be + # modified in place, even if BytesIO holds the only reference to it: + # it would keep the cached hash of its old contents. + def write(memio): + memio.seek(0) + memio.write(b"efgh") + def getbuffer(memio): + with memio.getbuffer() as view: + view[:] = b"efgh" + def truncate(memio): + memio.truncate(2) + memio.seek(0) + memio.write(b"efgh") + def getvalue(memio): + return memio.getvalue() + def read(memio): + memio.seek(0) + return memio.read() + def peek(memio): + memio.seek(0) + return memio.peek(4) + for mutation in write, getbuffer, truncate: + for expose in getvalue, read, peek: + with self.subTest(mutation=mutation.__name__, + expose=expose.__name__): + memio = self.ioclass() + memio.write(b"abcd") + # Shrink the overallocated buffer to the exact size, + # so read() and peek() can return it without copying. + memio.getvalue() + hash(expose(memio)) + mutation(memio) + self.assertEqual(memio.getvalue(), b"efgh") + self.assertEqual(hash(memio.getvalue()), hash(b"efgh")) + self.assertIn(memio.getvalue(), {b"efgh"}) + + with self.subTest(mutation=mutation.__name__, expose="__init__"): + initial = bytes(bytearray(b"abcd")) + hash(initial) + memio = self.ioclass(initial) + del initial + mutation(memio) + self.assertEqual(memio.getvalue(), b"efgh") + self.assertEqual(hash(memio.getvalue()), hash(b"efgh")) + def test_write_with_export(self): memio = self.ioclass(b"abcd") memio.seek(2) diff --git a/Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst b/Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst new file mode 100644 index 00000000000000..5644b99cce1720 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-00-32-51.gh-issue-160181.nnEeg1.rst @@ -0,0 +1,3 @@ +Fix :meth:`io.BytesIO.getvalue` sometimes returning a :class:`bytes` object +with the hash of earlier contents after the buffer was written to, which made +:class:`dict` and :class:`set` lookups with it fail. diff --git a/Modules/_io/bytesio.c b/Modules/_io/bytesio.c index fe114ada59845c..5cd367767b7b82 100644 --- a/Modules/_io/bytesio.c +++ b/Modules/_io/bytesio.c @@ -38,8 +38,8 @@ typedef struct { #define bytesiobuf_CAST(op) ((bytesiobuf *)(op)) /* The bytesio object can be in three states: - * Py_REFCNT(buf) == 1, exports == 0. - * Py_REFCNT(buf) > 1. exports == 0, + * Py_REFCNT(buf) == 1, the hash of buf is not cached, exports == 0. + * Py_REFCNT(buf) > 1 or the hash of buf is cached. exports == 0, first modification or export causes the internal buffer copying. * exports > 0. Py_REFCNT(buf) == 1, any modifications are forbidden. */ @@ -75,10 +75,19 @@ check_exports(bytesio *self) return NULL; \ } +/* If the hash value is cached, the internal bytes object was already used in + Python somehow (ex: as a dictionary key). Modifying it in place would leave + the hash of the old contents, even if nothing else references it anymore. */ +#define HASHED_BUF(self) \ + (_PyBytes_GET_CACHED_HASH((PyBytesObject *)(self)->buf) != -1) + #ifdef Py_GIL_DISABLED -#define SHARED_BUF(self) ((self)->buf_shared || !_PyObject_IsUniquelyReferenced((self)->buf)) +#define SHARED_BUF(self) \ + ((self)->buf_shared || !_PyObject_IsUniquelyReferenced((self)->buf) \ + || HASHED_BUF(self)) #else -#define SHARED_BUF(self) (!_PyObject_IsUniquelyReferenced((self)->buf)) +#define SHARED_BUF(self) \ + (!_PyObject_IsUniquelyReferenced((self)->buf) || HASHED_BUF(self)) #endif static inline void @@ -267,6 +276,7 @@ write_bytes_lock_held(bytesio *self, PyObject *b) goto done; } } + assert(_PyBytes_IsMutable(self->buf)); if (self->pos > self->string_size) { /* In case of overseek, pad with null bytes the buffer region between