Skip to content

Commit 4bd6b0c

Browse files
skirpichevskirpichev
committed
gh-127936: convert marshal module to use import/export API for ints (PEP 757)
1 parent 24b147a commit 4bd6b0c

File tree

1 file changed

+157
-75
lines changed

1 file changed

+157
-75
lines changed

‎Python/marshal.c

Lines changed: 157 additions & 75 deletions
Original file line numberDiff line numberDiff line change
@@ -240,10 +240,6 @@ w_short_pstring(const void *s, Py_ssize_t n, WFILE *p)
240240
#define PyLong_MARSHAL_SHIFT 15
241241
#define PyLong_MARSHAL_BASE ((short)1 << PyLong_MARSHAL_SHIFT)
242242
#define PyLong_MARSHAL_MASK (PyLong_MARSHAL_BASE - 1)
243-
#if PyLong_SHIFT % PyLong_MARSHAL_SHIFT != 0
244-
#error "PyLong_SHIFT must be a multiple of PyLong_MARSHAL_SHIFT"
245-
#endif
246-
#define PyLong_MARSHAL_RATIO (PyLong_SHIFT / PyLong_MARSHAL_SHIFT)
247243

248244
#define W_TYPE(t, p) do { \
249245
w_byte((t) | flag, (p)); \
@@ -252,47 +248,101 @@ w_short_pstring(const void *s, Py_ssize_t n, WFILE *p)
252248
static PyObject *
253249
_PyMarshal_WriteObjectToString(PyObject *x, int version, int allow_code);
254250

251+
#define _r_digits(bs) \
252+
static void \
253+
_r_digits##bs(const uint##bs##_t *digits, Py_ssize_t n, uint8_t negative, \
254+
Py_ssize_t marshal_ratio, WFILE *p) \
255+
{ \
256+
/* set l to number of base PyLong_MARSHAL_BASE digits */ \
257+
Py_ssize_t l = (n - 1)*marshal_ratio; \
258+
uint##bs##_t d = digits[n - 1]; \
259+
\
260+
assert(d != 0); /* a PyLong is always normalized */ \
261+
do { \
262+
d >>= PyLong_MARSHAL_SHIFT; \
263+
l++; \
264+
} while (d != 0); \
265+
if (l > SIZE32_MAX) { \
266+
p->depth--; \
267+
p->error = WFERR_UNMARSHALLABLE; \
268+
return; \
269+
} \
270+
w_long((long)(negative ? -l : l), p); \
271+
\
272+
for (Py_ssize_t i = 0; i < n - 1; i++) { \
273+
d = digits[i]; \
274+
for (Py_ssize_t j = 0; j < marshal_ratio; j++) { \
275+
w_short(d & PyLong_MARSHAL_MASK, p); \
276+
d >>= PyLong_MARSHAL_SHIFT; \
277+
} \
278+
assert (d == 0); \
279+
} \
280+
d = digits[n - 1]; \
281+
do { \
282+
w_short(d & PyLong_MARSHAL_MASK, p); \
283+
d >>= PyLong_MARSHAL_SHIFT; \
284+
} while (d != 0); \
285+
}
286+
_r_digits(16)
287+
_r_digits(32)
288+
255289
static void
256290
w_PyLong(const PyLongObject *ob, char flag, WFILE *p)
257291
{
258-
Py_ssize_t i, j, n, l;
259-
digit d;
260-
261292
W_TYPE(TYPE_LONG, p);
262293
if (_PyLong_IsZero(ob)) {
263294
w_long((long)0, p);
264295
return;
265296
}
266297

267-
/* set l to number of base PyLong_MARSHAL_BASE digits */
268-
n = _PyLong_DigitCount(ob);
269-
l = (n-1) * PyLong_MARSHAL_RATIO;
270-
d = ob->long_value.ob_digit[n-1];
271-
assert(d != 0); /* a PyLong is always normalized */
272-
do {
273-
d >>= PyLong_MARSHAL_SHIFT;
274-
l++;
275-
} while (d != 0);
276-
if (l > SIZE32_MAX) {
298+
PyLongExport long_export;
299+
300+
if (PyLong_Export((PyObject *)ob, &long_export) < 0) {
277301
p->depth--;
278302
p->error = WFERR_UNMARSHALLABLE;
279303
return;
280304
}
281-
w_long((long)(_PyLong_IsNegative(ob) ? -l : l), p);
305+
if (!long_export.digits) {
306+
int8_t sign = long_export.value < 0 ? -1 : 1;
307+
uint64_t abs_value = Py_ABS(long_export.value);
308+
uint64_t d = abs_value;
309+
long l = 0;
282310

283-
for (i=0; i < n-1; i++) {
284-
d = ob->long_value.ob_digit[i];
285-
for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
311+
/* set l to number of base PyLong_MARSHAL_BASE digits */
312+
do {
313+
d >>= PyLong_MARSHAL_SHIFT;
314+
l += sign;
315+
} while (d);
316+
w_long(l, p);
317+
d = abs_value;
318+
do {
286319
w_short(d & PyLong_MARSHAL_MASK, p);
287320
d >>= PyLong_MARSHAL_SHIFT;
288-
}
289-
assert (d == 0);
321+
} while (d);
322+
return;
323+
}
324+
325+
const PyLongLayout *layout = PyLong_GetNativeLayout();
326+
Py_ssize_t marshal_ratio = layout->bits_per_digit/PyLong_MARSHAL_SHIFT;
327+
328+
/* must be a multiple of PyLong_MARSHAL_SHIFT */
329+
assert(layout->bits_per_digit % PyLong_MARSHAL_SHIFT == 0);
330+
331+
/* other assumptions on PyLongObject internals */
332+
assert(layout->bits_per_digit <= 32);
333+
assert(layout->digits_order == -1);
334+
assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1));
335+
assert(layout->digit_size == 2 || layout->digit_size == 4);
336+
337+
if (layout->digit_size == 4) {
338+
_r_digits32(long_export.digits, long_export.ndigits,
339+
long_export.negative, marshal_ratio, p);
340+
}
341+
else {
342+
_r_digits16(long_export.digits, long_export.ndigits,
343+
long_export.negative, marshal_ratio, p);
290344
}
291-
d = ob->long_value.ob_digit[n-1];
292-
do {
293-
w_short(d & PyLong_MARSHAL_MASK, p);
294-
d >>= PyLong_MARSHAL_SHIFT;
295-
} while (d != 0);
345+
PyLong_FreeExport(&long_export);
296346
}
297347

298348
static void
@@ -875,17 +925,60 @@ r_long64(RFILE *p)
875925
1 /* signed */);
876926
}
877927

928+
#define _w_digits(bs) \
929+
static int \
930+
_w_digits##bs(uint##bs##_t *digits, Py_ssize_t size, \
931+
Py_ssize_t marshal_ratio, \
932+
int shorts_in_top_digit, RFILE *p) \
933+
{ \
934+
int md; \
935+
uint##bs##_t d; \
936+
\
937+
for (Py_ssize_t i = 0; i < size - 1; i++) { \
938+
d = 0; \
939+
for (Py_ssize_t j = 0; j < marshal_ratio; j++) { \
940+
md = r_short(p); \
941+
if (md < 0 || md > PyLong_MARSHAL_BASE) { \
942+
goto bad_digit; \
943+
} \
944+
d += (uint##bs##_t)md << j*PyLong_MARSHAL_SHIFT; \
945+
} \
946+
digits[i] = d; \
947+
} \
948+
\
949+
d = 0; \
950+
for (Py_ssize_t j = 0; j < shorts_in_top_digit; j++) { \
951+
md = r_short(p); \
952+
if (md < 0 || md > PyLong_MARSHAL_BASE) { \
953+
goto bad_digit; \
954+
} \
955+
/* topmost marshal digit should be nonzero */ \
956+
if (md == 0 && j == shorts_in_top_digit - 1) { \
957+
PyErr_SetString(PyExc_ValueError, \
958+
"bad marshal data (unnormalized long data)"); \
959+
return -1; \
960+
} \
961+
d += (uint##bs##_t)md << j*PyLong_MARSHAL_SHIFT; \
962+
} \
963+
assert(!PyErr_Occurred()); \
964+
/* top digit should be nonzero, else the resulting PyLong won't be \
965+
normalized */ \
966+
digits[size - 1] = d; \
967+
return 0; \
968+
bad_digit: \
969+
if (!PyErr_Occurred()) { \
970+
PyErr_SetString(PyExc_ValueError, \
971+
"bad marshal data (digit out of range in long)"); \
972+
} \
973+
return -1; \
974+
}
975+
_w_digits(32)
976+
_w_digits(16)
977+
878978
static PyObject *
879979
r_PyLong(RFILE *p)
880980
{
881-
PyLongObject *ob;
882-
long n, size, i;
883-
int j, md, shorts_in_top_digit;
884-
digit d;
885-
886-
n = r_long(p);
887-
if (n == 0)
888-
return (PyObject *)_PyLong_New(0);
981+
long n = r_long(p);
889982
if (n == -1 && PyErr_Occurred()) {
890983
return NULL;
891984
}
@@ -895,51 +988,40 @@ r_PyLong(RFILE *p)
895988
return NULL;
896989
}
897990

898-
size = 1 + (Py_ABS(n) - 1) / PyLong_MARSHAL_RATIO;
899-
shorts_in_top_digit = 1 + (Py_ABS(n) - 1) % PyLong_MARSHAL_RATIO;
900-
ob = _PyLong_New(size);
901-
if (ob == NULL)
902-
return NULL;
991+
const PyLongLayout *layout = PyLong_GetNativeLayout();
992+
Py_ssize_t marshal_ratio = layout->bits_per_digit/PyLong_MARSHAL_SHIFT;
903993

904-
_PyLong_SetSignAndDigitCount(ob, n < 0 ? -1 : 1, size);
994+
/* must be a multiple of PyLong_MARSHAL_SHIFT */
995+
assert(layout->bits_per_digit % PyLong_MARSHAL_SHIFT == 0);
905996

906-
for (i = 0; i < size-1; i++) {
907-
d = 0;
908-
for (j=0; j < PyLong_MARSHAL_RATIO; j++) {
909-
md = r_short(p);
910-
if (md < 0 || md > PyLong_MARSHAL_BASE)
911-
goto bad_digit;
912-
d += (digit)md << j*PyLong_MARSHAL_SHIFT;
913-
}
914-
ob->long_value.ob_digit[i] = d;
997+
/* other assumptions on PyLongObject internals */
998+
assert(layout->bits_per_digit <= 32);
999+
assert(layout->digits_order == -1);
1000+
assert(layout->digit_endianness == (PY_LITTLE_ENDIAN ? -1 : 1));
1001+
assert(layout->digit_size == 2 || layout->digit_size == 4);
1002+
1003+
Py_ssize_t size = 1 + (Py_ABS(n) - 1) / marshal_ratio;
1004+
int shorts_in_top_digit = 1 + (Py_ABS(n) - 1) % marshal_ratio;
1005+
void *digits;
1006+
PyLongWriter *writer = PyLongWriter_Create(n < 0, size, &digits);
1007+
1008+
if (writer == NULL) {
1009+
return NULL;
9151010
}
9161011

917-
d = 0;
918-
for (j=0; j < shorts_in_top_digit; j++) {
919-
md = r_short(p);
920-
if (md < 0 || md > PyLong_MARSHAL_BASE)
921-
goto bad_digit;
922-
/* topmost marshal digit should be nonzero */
923-
if (md == 0 && j == shorts_in_top_digit - 1) {
924-
Py_DECREF(ob);
925-
PyErr_SetString(PyExc_ValueError,
926-
"bad marshal data (unnormalized long data)");
927-
return NULL;
928-
}
929-
d += (digit)md << j*PyLong_MARSHAL_SHIFT;
1012+
int ret;
1013+
1014+
if (layout->digit_size == 4) {
1015+
ret = _w_digits32(digits, size, marshal_ratio, shorts_in_top_digit, p);
9301016
}
931-
assert(!PyErr_Occurred());
932-
/* top digit should be nonzero, else the resulting PyLong won't be
933-
normalized */
934-
ob->long_value.ob_digit[size-1] = d;
935-
return (PyObject *)ob;
936-
bad_digit:
937-
Py_DECREF(ob);
938-
if (!PyErr_Occurred()) {
939-
PyErr_SetString(PyExc_ValueError,
940-
"bad marshal data (digit out of range in long)");
1017+
else {
1018+
ret = _w_digits16(digits, size, marshal_ratio, shorts_in_top_digit, p);
1019+
}
1020+
if (ret < 0) {
1021+
PyLongWriter_Discard(writer);
1022+
return NULL;
9411023
}
942-
return NULL;
1024+
return PyLongWriter_Finish(writer);
9431025
}
9441026

9451027
static double

0 commit comments

Comments
 (0)