Skip to content

Commit d561f84

Browse files
bpo-31829: Make protocol 0 pickles be loadable in text mode in Python 2. (GH-11859)
Escape ``\r``, ``\0`` and ``\x1a`` (end-of-file on Windows) in Unicode strings. (cherry picked from commit 38ab7d4) Co-authored-by: Serhiy Storchaka <[email protected]>
1 parent 534136a commit d561f84

File tree

4 files changed

+25
-5
lines changed

4 files changed

+25
-5
lines changed

Lib/pickle.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -749,7 +749,10 @@ def save_str(self, obj):
749749
self.write(BINUNICODE + pack("<I", n) + encoded)
750750
else:
751751
obj = obj.replace("\\", "\\u005c")
752+
obj = obj.replace("\0", "\\u0000")
752753
obj = obj.replace("\n", "\\u000a")
754+
obj = obj.replace("\r", "\\u000d")
755+
obj = obj.replace("\x1a", "\\u001a") # EOF on DOS
753756
self.write(UNICODE + obj.encode('raw-unicode-escape') +
754757
b'\n')
755758
self.memoize(obj)

Lib/test/pickletester.py

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2709,22 +2709,20 @@ def __getattr__(self, key):
27092709
class AbstractPickleModuleTests(unittest.TestCase):
27102710

27112711
def test_dump_closed_file(self):
2712-
import os
27132712
f = open(TESTFN, "wb")
27142713
try:
27152714
f.close()
27162715
self.assertRaises(ValueError, self.dump, 123, f)
27172716
finally:
2718-
os.remove(TESTFN)
2717+
support.unlink(TESTFN)
27192718

27202719
def test_load_closed_file(self):
2721-
import os
27222720
f = open(TESTFN, "wb")
27232721
try:
27242722
f.close()
27252723
self.assertRaises(ValueError, self.dump, 123, f)
27262724
finally:
2727-
os.remove(TESTFN)
2725+
support.unlink(TESTFN)
27282726

27292727
def test_load_from_and_dump_to_file(self):
27302728
stream = io.BytesIO()
@@ -2748,6 +2746,19 @@ def test_callapi(self):
27482746
self.Pickler(f, -1)
27492747
self.Pickler(f, protocol=-1)
27502748

2749+
def test_dump_text_file(self):
2750+
f = open(TESTFN, "w")
2751+
try:
2752+
for proto in protocols:
2753+
self.assertRaises(TypeError, self.dump, 123, f, proto)
2754+
finally:
2755+
f.close()
2756+
support.unlink(TESTFN)
2757+
2758+
def test_incomplete_input(self):
2759+
s = io.BytesIO(b"X''.")
2760+
self.assertRaises((EOFError, struct.error, pickle.UnpicklingError), self.load, s)
2761+
27512762
def test_bad_init(self):
27522763
# Test issue3664 (pickle can segfault from a badly initialized Pickler).
27532764
# Override initialization without calling __init__() of the superclass.
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
``\r``, ``\0`` and ``\x1a`` (end-of-file on Windows) are now escaped in
2+
protocol 0 pickles of Unicode strings. This allows to load them without loss
3+
from files open in text mode in Python 2.

Modules/_pickle.c

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2337,7 +2337,10 @@ raw_unicode_escape(PyObject *obj)
23372337
*p++ = Py_hexdigits[ch & 15];
23382338
}
23392339
/* Map 16-bit characters, '\\' and '\n' to '\uxxxx' */
2340-
else if (ch >= 256 || ch == '\\' || ch == '\n') {
2340+
else if (ch >= 256 ||
2341+
ch == '\\' || ch == 0 || ch == '\n' || ch == '\r' ||
2342+
ch == 0x1a)
2343+
{
23412344
/* -1: subtract 1 preallocated byte */
23422345
p = _PyBytesWriter_Prepare(&writer, p, 6-1);
23432346
if (p == NULL)

0 commit comments

Comments
 (0)