From 6f0f3069acab852b3c5f32afb81cf00316cf46dc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 1 Dec 2018 01:17:28 +0100 Subject: [PATCH 1/6] bpo-9566: Fix compiler warnings in peephole.c Ensure that opcode arguments are smaller than UINT_MAX. --- Python/peephole.c | 58 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 41 insertions(+), 17 deletions(-) diff --git a/Python/peephole.c b/Python/peephole.c index 77d134939f8de9..6801272d173565 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -152,6 +152,13 @@ fold_tuple_on_constants(_Py_CODEUNIT *codestr, Py_ssize_t codelen, PyTuple_SET_ITEM(newconst, i, constant); } + Py_ssize_t index = PyList_GET_SIZE(consts); + if ((size_t)index >= UINT_MAX - 1) { + Py_DECREF(newconst); + PyErr_SetString(PyExc_OverflowError, "too many constants"); + return -1; + } + /* Append folded constant onto consts */ if (PyList_Append(consts, newconst)) { Py_DECREF(newconst); @@ -160,7 +167,7 @@ fold_tuple_on_constants(_Py_CODEUNIT *codestr, Py_ssize_t codelen, Py_DECREF(newconst); return copy_op_arg(codestr, c_start, LOAD_CONST, - PyList_GET_SIZE(consts)-1, opcode_end); + (unsigned int)index, opcode_end); } static unsigned int * @@ -221,7 +228,7 @@ PyObject * PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, PyObject *lnotab_obj) { - Py_ssize_t h, i, nexti, op_start, codelen, tgt; + Py_ssize_t h, i, nexti, op_start, tgt; unsigned int j, nops; unsigned char opcode, nextop; _Py_CODEUNIT *codestr = NULL; @@ -249,17 +256,22 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, the peephole optimizer doesn't modify line numbers. */ assert(PyBytes_Check(code)); - codelen = PyBytes_GET_SIZE(code); - assert(codelen % sizeof(_Py_CODEUNIT) == 0); + Py_ssize_t codesize = PyBytes_GET_SIZE(code); + assert(codesize % sizeof(_Py_CODEUNIT) == 0); + Py_ssize_t codelen = codesize / sizeof(_Py_CODEUNIT); + if ((size_t)codelen > INT_MAX) { + /* Python assembler is limited to INT_MAX: see assembler.a_offset in + compile.c. */ + goto exitUnchanged; + } /* Make a modifiable copy of the code string */ - codestr = (_Py_CODEUNIT *)PyMem_Malloc(codelen); + codestr = (_Py_CODEUNIT *)PyMem_Malloc(codesize); if (codestr == NULL) { PyErr_NoMemory(); goto exitError; } - memcpy(codestr, PyBytes_AS_STRING(code), codelen); - codelen /= sizeof(_Py_CODEUNIT); + memcpy(codestr, PyBytes_AS_STRING(code), codesize); blocks = markblocks(codestr, codelen); if (blocks == NULL) @@ -353,11 +365,16 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, stack effect */ h = set_arg(codestr, i, get_arg(codestr, tgt)); } else { + Py_ssize_t arg = (tgt + 1); + /* cannot overflow: codelen <= INT_MAX */ + assert((size_t)arg <= UINT_MAX / sizeof(_Py_CODEUNIT)); + arg *= sizeof(_Py_CODEUNIT); + /* The second jump is not taken if the first is (so jump past it), and all conditional jumps pop their argument when they're not taken (so change the first jump to pop its argument when it's taken). */ - h = set_arg(codestr, i, (tgt + 1) * sizeof(_Py_CODEUNIT)); + h = set_arg(codestr, i, (unsigned int)arg); j = opcode == JUMP_IF_TRUE_OR_POP ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE; } @@ -383,17 +400,20 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, codestr[op_start] = PACKOPARG(RETURN_VALUE, 0); fill_nops(codestr, op_start + 1, i + 1); } else if (UNCONDITIONAL_JUMP(_Py_OPCODE(codestr[tgt]))) { - j = GETJUMPTGT(codestr, tgt); + Py_ssize_t arg = GETJUMPTGT(codestr, tgt); if (opcode == JUMP_FORWARD) { /* JMP_ABS can go backwards */ opcode = JUMP_ABSOLUTE; } else if (!ABSOLUTE_JUMP(opcode)) { - if ((Py_ssize_t)j < i + 1) { + if (arg < i + 1) { break; /* No backward relative jumps */ } - j -= i + 1; /* Calc relative jump addr */ + arg -= i + 1; /* Calc relative jump addr */ } - j *= sizeof(_Py_CODEUNIT); - copy_op_arg(codestr, op_start, opcode, j, i + 1); + /* cannot overflow: codelen <= INT_MAX */ + assert((size_t)arg <= (UINT_MAX / sizeof(_Py_CODEUNIT))); + arg *= sizeof(_Py_CODEUNIT); + copy_op_arg(codestr, op_start, opcode, + (unsigned int)arg, i + 1); } break; @@ -427,9 +447,11 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, /* Fixup lnotab */ for (i = 0, nops = 0; i < codelen; i++) { - assert(i - nops <= INT_MAX); + Py_ssize_t block = i - nops; + /* cannot overflow: codelen <= INT_MAX */ + assert(block <= UINT_MAX); /* original code offset => new code offset */ - blocks[i] = i - nops; + blocks[i] = (unsigned int)block; if (_Py_OPCODE(codestr[i]) == NOP) nops++; } @@ -477,10 +499,12 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, break; } nexti = i - op_start + 1; - if (instrsize(j) > nexti) + if (instrsize(j) > nexti) { goto exitUnchanged; + } + assert(nexti <= INT_MAX); /* If instrsize(j) < nexti, we'll emit EXTENDED_ARG 0 */ - write_op_arg(codestr + h, opcode, j, nexti); + write_op_arg(codestr + h, opcode, j, (int)nexti); h += nexti; } assert(h + (Py_ssize_t)nops == codelen); From b097f369fa44b38a962fca2d897b8e4726122024 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 6 Dec 2018 12:12:53 +0100 Subject: [PATCH 2/6] Fix issues with signed vs unsigned --- Python/peephole.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Python/peephole.c b/Python/peephole.c index 6801272d173565..31823ee43e32ff 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -259,7 +259,7 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, Py_ssize_t codesize = PyBytes_GET_SIZE(code); assert(codesize % sizeof(_Py_CODEUNIT) == 0); Py_ssize_t codelen = codesize / sizeof(_Py_CODEUNIT); - if ((size_t)codelen > INT_MAX) { + if (codelen > INT_MAX) { /* Python assembler is limited to INT_MAX: see assembler.a_offset in compile.c. */ goto exitUnchanged; @@ -400,17 +400,17 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, codestr[op_start] = PACKOPARG(RETURN_VALUE, 0); fill_nops(codestr, op_start + 1, i + 1); } else if (UNCONDITIONAL_JUMP(_Py_OPCODE(codestr[tgt]))) { - Py_ssize_t arg = GETJUMPTGT(codestr, tgt); + size_t arg = GETJUMPTGT(codestr, tgt); if (opcode == JUMP_FORWARD) { /* JMP_ABS can go backwards */ opcode = JUMP_ABSOLUTE; } else if (!ABSOLUTE_JUMP(opcode)) { - if (arg < i + 1) { + if (arg < (size_t)(i + 1)) { break; /* No backward relative jumps */ } arg -= i + 1; /* Calc relative jump addr */ } /* cannot overflow: codelen <= INT_MAX */ - assert((size_t)arg <= (UINT_MAX / sizeof(_Py_CODEUNIT))); + assert(arg <= (UINT_MAX / sizeof(_Py_CODEUNIT))); arg *= sizeof(_Py_CODEUNIT); copy_op_arg(codestr, op_start, opcode, (unsigned int)arg, i + 1); @@ -447,13 +447,14 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, /* Fixup lnotab */ for (i = 0, nops = 0; i < codelen; i++) { - Py_ssize_t block = i - nops; + size_t block = (size_t)i - nops; /* cannot overflow: codelen <= INT_MAX */ assert(block <= UINT_MAX); /* original code offset => new code offset */ blocks[i] = (unsigned int)block; - if (_Py_OPCODE(codestr[i]) == NOP) + if (_Py_OPCODE(codestr[i]) == NOP) { nops++; + } } cum_orig_offset = 0; last_offset = 0; From 0133edbc752ae48ac9f46f97c9169b9a29cdc4e3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 7 Dec 2018 16:49:34 +0100 Subject: [PATCH 3/6] Move comment --- Python/peephole.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Python/peephole.c b/Python/peephole.c index 31823ee43e32ff..c86981e6ae3eca 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -365,15 +365,14 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, stack effect */ h = set_arg(codestr, i, get_arg(codestr, tgt)); } else { - Py_ssize_t arg = (tgt + 1); - /* cannot overflow: codelen <= INT_MAX */ - assert((size_t)arg <= UINT_MAX / sizeof(_Py_CODEUNIT)); - arg *= sizeof(_Py_CODEUNIT); - /* The second jump is not taken if the first is (so jump past it), and all conditional jumps pop their argument when they're not taken (so change the first jump to pop its argument when it's taken). */ + Py_ssize_t arg = (tgt + 1); + /* cannot overflow: codelen <= INT_MAX */ + assert((size_t)arg <= UINT_MAX / sizeof(_Py_CODEUNIT)); + arg *= sizeof(_Py_CODEUNIT); h = set_arg(codestr, i, (unsigned int)arg); j = opcode == JUMP_IF_TRUE_OR_POP ? POP_JUMP_IF_TRUE : POP_JUMP_IF_FALSE; From 0afc9186ba6a0a15c6b56641a6975bacce3a11e1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 7 Dec 2018 17:00:16 +0100 Subject: [PATCH 4/6] Add ilen variable --- Python/peephole.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Python/peephole.c b/Python/peephole.c index c86981e6ae3eca..bea229af648368 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -498,14 +498,14 @@ PyCode_Optimize(PyObject *code, PyObject* consts, PyObject *names, j *= sizeof(_Py_CODEUNIT); break; } - nexti = i - op_start + 1; - if (instrsize(j) > nexti) { + Py_ssize_t ilen = i - op_start + 1; + if (instrsize(j) > ilen) { goto exitUnchanged; } - assert(nexti <= INT_MAX); - /* If instrsize(j) < nexti, we'll emit EXTENDED_ARG 0 */ - write_op_arg(codestr + h, opcode, j, (int)nexti); - h += nexti; + assert(ilen <= INT_MAX); + /* If instrsize(j) < ilen, we'll emit EXTENDED_ARG 0 */ + write_op_arg(codestr + h, opcode, j, (int)ilen); + h += ilen; } assert(h + (Py_ssize_t)nops == codelen); From 341e474d64da96e27cdda83411f9204f3772a266 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 7 Dec 2018 17:03:57 +0100 Subject: [PATCH 5/6] Add "#if SIZEOF_SIZE_T > SIZEOF_INT" --- Python/peephole.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Python/peephole.c b/Python/peephole.c index bea229af648368..2b193e2d359d81 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -152,12 +152,14 @@ fold_tuple_on_constants(_Py_CODEUNIT *codestr, Py_ssize_t codelen, PyTuple_SET_ITEM(newconst, i, constant); } +#if SIZEOF_SIZE_T > SIZEOF_INT Py_ssize_t index = PyList_GET_SIZE(consts); if ((size_t)index >= UINT_MAX - 1) { Py_DECREF(newconst); PyErr_SetString(PyExc_OverflowError, "too many constants"); return -1; } +#endif /* Append folded constant onto consts */ if (PyList_Append(consts, newconst)) { From 7a97ab6f663b39acd5df0920e04f53fec23588dc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 7 Dec 2018 17:21:38 +0100 Subject: [PATCH 6/6] Fix #if --- Python/peephole.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Python/peephole.c b/Python/peephole.c index 2b193e2d359d81..cc244aa433ee18 100644 --- a/Python/peephole.c +++ b/Python/peephole.c @@ -152,8 +152,8 @@ fold_tuple_on_constants(_Py_CODEUNIT *codestr, Py_ssize_t codelen, PyTuple_SET_ITEM(newconst, i, constant); } -#if SIZEOF_SIZE_T > SIZEOF_INT Py_ssize_t index = PyList_GET_SIZE(consts); +#if SIZEOF_SIZE_T > SIZEOF_INT if ((size_t)index >= UINT_MAX - 1) { Py_DECREF(newconst); PyErr_SetString(PyExc_OverflowError, "too many constants");