From f74be850a2e55e7d93c5c3e027522164a6eca0b5 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" <68491+gpshead@users.noreply.github.com> Date: Sat, 31 Oct 2026 16:09:23 -0700 Subject: [PATCH 1/3] gh-160324: Keep `io.StringIO` state after `MemoryError` (GH-160325) Allocate before consuming the accumulated writes in realize() and make_intermediate(), and reset the size when the contents are lost, so that later calls do not index outside the internal buffer. Shrink the buffer of the accumulating writer before finishing it, so that getvalue(), read() and the other methods leave the contents unchanged when they fail with MemoryError instead of dropping them. ANT-2026-MPHGX2FL (cherry picked from commit 83b047ef80836c24672cf3e2e807e68efed7d495) Co-authored-by: Gregory P. Smith <68491+gpshead@users.noreply.github.com> --- Lib/test/test_io/test_memoryio.py | 33 +++++++++ ...-10-22-08-15-56.gh-issue-160324.sIoRlz.rst | 3 + Modules/_io/stringio.c | 71 ++++++++++++++----- 3 files changed, 90 insertions(+), 17 deletions(-) create mode 100644 Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst diff --git a/Lib/test/test_io/test_memoryio.py b/Lib/test/test_io/test_memoryio.py index f70a589f45d47a9..70e5304c9f11eaf 100644 --- a/Lib/test/test_io/test_memoryio.py +++ b/Lib/test/test_io/test_memoryio.py @@ -1073,6 +1073,39 @@ def __str__(self): memio2.write(MyStr("world")) self.assertEqual(memio2.getvalue(), "hello world") + @support.nomemtest + def test_memory_error_consistent_state(self): + # A MemoryError while copying the accumulated writes into the + # internal buffer must leave the size of the stream consistent with + # the buffer, so that using the object again does not read or write + # outside it. + piece = "0123456789" * 10 + text = piece * 50 + calls = [("readline",), ("read", 5), ("read",), ("getvalue",), + ("truncate", 3), ("write", "ab"), ("__next__",)] + for name, *args in calls: + for n in range(3): + with self.subTest(method=name, n=n): + memio = self.ioclass() + for _ in range(50): + memio.write(piece) + memio.seek(0) + method = getattr(memio, name) + try: + with support.inject_memory_error_cm(n, n + 1): + method(*args) + except MemoryError: + pass + else: + continue + # The contents are unchanged. + self.assertEqual(memio.getvalue(), text) + self.assertEqual(memio.seek(0, 2), len(text)) + memio.seek(0) + self.assertEqual(memio.readline(), text) + memio.write("!") + self.assertEqual(memio.getvalue(), text + "!") + class CStringIOPickleTest(PyStringIOPickleTest): UnsupportedOperation = io.UnsupportedOperation diff --git a/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst b/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst new file mode 100644 index 000000000000000..d54069128e2c0eb --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-10-22-08-15-56.gh-issue-160324.sIoRlz.rst @@ -0,0 +1,3 @@ +Fix a crash or garbage data when an :class:`io.StringIO` object is used again +after one of its methods failed with :exc:`MemoryError`. The object now keeps +its contents. diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 08a3f7e16e4d8a3..32b9f1c65a617b8 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -1,6 +1,7 @@ #include "Python.h" #include // offsetof() #include "pycore_object.h" +#include "pycore_unicodeobject.h" // _PyUnicode_ResizeCompact() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "_iomodule.h" @@ -129,25 +130,52 @@ resize_buffer(stringio *self, size_t size) return -1; } +/* Shrink the buffer of the writer to its contents, so that the following + PyUnicodeWriter_Finish() has nothing to allocate and cannot fail. + If this fails, the writer is unchanged. */ +static int +shrink_writer(stringio *self) +{ + _PyUnicodeWriter *writer = (_PyUnicodeWriter *)self->writer; + if (writer->readonly || writer->buffer == NULL + || writer->pos == 0 || writer->pos == writer->size) + { + return 0; + } + PyObject *buffer = _PyUnicode_ResizeCompact(writer->buffer, writer->pos); + if (buffer == NULL) { + return -1; + } + writer->buffer = NULL; + _PyUnicodeWriter_SetBuffer(writer, buffer); + return 0; +} + static PyObject * make_intermediate(stringio *self) { - PyObject *intermediate = PyUnicodeWriter_Finish(self->writer); - self->writer = NULL; - self->state = STATE_REALIZED; - if (intermediate == NULL) + /* Create the new writer and shrink the old one first: if this fails, + the object keeps its contents. */ + PyUnicodeWriter *writer = PyUnicodeWriter_Create(0); + if (writer == NULL) return NULL; + if (shrink_writer(self) < 0) { + PyUnicodeWriter_Discard(writer); + return NULL; + } - self->writer = PyUnicodeWriter_Create(0); - if (self->writer == NULL) { - Py_DECREF(intermediate); + PyObject *intermediate = PyUnicodeWriter_Finish(self->writer); + self->writer = writer; + if (intermediate == NULL) { + /* The contents are lost, the new writer is empty. */ + self->string_size = 0; return NULL; } if (PyUnicodeWriter_WriteStr(self->writer, intermediate)) { Py_DECREF(intermediate); + self->string_size = 0; return NULL; } - self->state = STATE_ACCUMULATING; return intermediate; } @@ -160,23 +188,30 @@ realize(stringio *self) if (self->state == STATE_REALIZED) return 0; assert(self->state == STATE_ACCUMULATING); + + /* Allocate the internal buffer and shrink the writer first: if this + fails, the object stays in the accumulating state and keeps its + contents. */ + if (resize_buffer(self, self->string_size) < 0) + return -1; + if (shrink_writer(self) < 0) + return -1; self->state = STATE_REALIZED; intermediate = PyUnicodeWriter_Finish(self->writer); self->writer = NULL; - if (intermediate == NULL) + if (intermediate == NULL) { + /* The contents are lost, the internal buffer contains nothing. */ + self->string_size = 0; return -1; + } - /* Append the intermediate string to the internal buffer. - The length should be equal to the current cursor position. - */ + /* Copy the intermediate string to the internal buffer. */ len = PyUnicode_GET_LENGTH(intermediate); - if (resize_buffer(self, len) < 0) { - Py_DECREF(intermediate); - return -1; - } + assert(len == self->string_size); if (!PyUnicode_AsUCS4(intermediate, self->buf, len, 0)) { Py_DECREF(intermediate); + self->string_size = 0; return -1; } @@ -344,7 +379,9 @@ _io_StringIO_read_impl(stringio *self, Py_ssize_t size) /* Optimization for seek(0); read() */ if (self->state == STATE_ACCUMULATING && self->pos == 0 && size == n) { PyObject *result = make_intermediate(self); - self->pos = self->string_size; + if (result != NULL) { + self->pos = self->string_size; + } return result; } From fd2c162a40923ad594256b67545f1c30cd3083f6 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Mon, 9 Nov 2026 00:29:50 +0000 Subject: [PATCH 2/3] Avoid _PyUnicodeWriter_SetBuffer(); it does not exist in 3.15 Update the writer fields directly instead. --- Modules/_io/stringio.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 32b9f1c65a617b8..4c4dc402b04d5aa 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -146,8 +146,11 @@ shrink_writer(stringio *self) if (buffer == NULL) { return -1; } - writer->buffer = NULL; - _PyUnicodeWriter_SetBuffer(writer, buffer); + writer->buffer = buffer; + writer->data = PyUnicode_DATA(buffer); + writer->kind = PyUnicode_KIND(buffer); + writer->maxchar = PyUnicode_MAX_CHAR_VALUE(buffer); + writer->size = PyUnicode_GET_LENGTH(buffer); return 0; } From b2e108cb2100a572a05d41f570acd5909eed52c1 Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith" Date: Mon, 9 Nov 2026 02:15:25 +0000 Subject: [PATCH 3/3] Do not copy the intermediate string in make_intermediate() In 3.15, writing a string to a new PyUnicodeWriter copies it, so a MemoryError there lost the contents of the StringIO. Disable overallocation for this write so the writer shares the string (copy-on-write) instead, as it does in main. --- Modules/_io/stringio.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Modules/_io/stringio.c b/Modules/_io/stringio.c index 4c4dc402b04d5aa..a2e38f09390fcbb 100644 --- a/Modules/_io/stringio.c +++ b/Modules/_io/stringio.c @@ -174,7 +174,12 @@ make_intermediate(stringio *self) self->string_size = 0; return NULL; } - if (PyUnicodeWriter_WriteStr(self->writer, intermediate)) { + /* Without overallocation, writing to an empty writer shares the string + (copy-on-write) instead of copying it, so it does not allocate. */ + ((_PyUnicodeWriter *)writer)->overallocate = 0; + int res = PyUnicodeWriter_WriteStr(writer, intermediate); + ((_PyUnicodeWriter *)writer)->overallocate = 1; + if (res < 0) { Py_DECREF(intermediate); self->string_size = 0; return NULL;