Forward-port of r52136,52138: a review of overflow-detecting code.

* unified the way intobject, longobject and mystrtoul handle
  values around -sys.maxint-1.

* in general, trying to entierely avoid overflows in any computation
  involving signed ints or longs is extremely involved.  Fixed a few
  simple cases where a compiler might be too clever (but that's all
  guesswork).

* more overflow checks against bad data in marshal.c.

* 2.5 specific: fixed a number of places that were still confusing int
  and Py_ssize_t.  Some of them could potentially have caused
  "real-world" breakage.

* list.pop(x): fixing overflow issues on x was messy.  I just reverted
  to PyArg_ParseTuple("n"), which does the right thing.  (An obscure
  test was trying to give a Decimal to list.pop()... doesn't make
  sense any more IMHO)

* trying to write a few tests...

1 files changed, 47 insertions, 24 deletions

diff --git a/Objects/longobject.c b/Objects/longobject.c
index e32c42566e..4d886cd724 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -193,6 +193,18 @@ PyLong_FromDouble(double dval)
 	return (PyObject *)v;
 }
 
+/* Checking for overflow in PyLong_AsLong is a PITA since C doesn't define
+ * anything about what happens when a signed integer operation overflows,
+ * and some compilers think they're doing you a favor by being "clever"
+ * then.  The bit pattern for the largest postive signed long is
+ * (unsigned long)LONG_MAX, and for the smallest negative signed long
+ * it is abs(LONG_MIN), which we could write -(unsigned long)LONG_MIN.
+ * However, some other compilers warn about applying unary minus to an
+ * unsigned operand.  Hence the weird "0-".
+ */
+#define PY_ABS_LONG_MIN		(0-(unsigned long)LONG_MIN)
+#define PY_ABS_SSIZE_T_MIN	(0-(size_t)PY_SSIZE_T_MIN)
+
 /* Get a C long int from a long int object.
    Returns -1 and sets an error condition if overflow occurs. */
 
@@ -225,14 +237,16 @@ PyLong_AsLong(PyObject *vv)
 		if ((x >> SHIFT) != prev)
 			goto overflow;
 	}
-	/* Haven't lost any bits, but if the sign bit is set we're in
-	 * trouble *unless* this is the min negative number.  So,
-	 * trouble iff sign bit set && (positive || some bit set other
-	 * than the sign bit).
-	 */
-	if ((long)x < 0 && (sign > 0 || (x << 1) != 0))
-		goto overflow;
-	return (long)x * sign;
+	/* Haven't lost any bits, but casting to long requires extra care
+	 * (see comment above).
+         */
+	if (x <= (unsigned long)LONG_MAX) {
+		return (long)x * sign;
+	}
+	else if (sign < 0 && x == PY_ABS_LONG_MIN) {
+		return LONG_MIN;
+	}
+	/* else overflow */
 
  overflow:
 	PyErr_SetString(PyExc_OverflowError,
@@ -268,14 +282,16 @@ _PyLong_AsSsize_t(PyObject *vv) {
 		if ((x >> SHIFT) != prev)
 			goto overflow;
 	}
-	/* Haven't lost any bits, but if the sign bit is set we're in
-	 * trouble *unless* this is the min negative number.  So,
-	 * trouble iff sign bit set && (positive || some bit set other
-	 * than the sign bit).
+	/* Haven't lost any bits, but casting to a signed type requires
+	 * extra care (see comment above).
 	 */
-	if ((Py_ssize_t)x < 0 && (sign > 0 || (x << 1) != 0))
-		goto overflow;
-	return (Py_ssize_t)x * sign;
+	if (x <= (size_t)PY_SSIZE_T_MAX) {
+		return (Py_ssize_t)x * sign;
+	}
+	else if (sign < 0 && x == PY_ABS_SSIZE_T_MIN) {
+		return PY_SSIZE_T_MIN;
+	}
+	/* else overflow */
 
  overflow:
 	PyErr_SetString(PyExc_OverflowError,
@@ -1167,7 +1183,7 @@ long_format(PyObject *aa, int base, int addL)
 {
 	register PyLongObject *a = (PyLongObject *)aa;
 	PyStringObject *str;
-	Py_ssize_t i;
+	Py_ssize_t i, j, sz;
 	Py_ssize_t size_a;
 	char *p;
 	int bits;
@@ -1187,11 +1203,18 @@ long_format(PyObject *aa, int base, int addL)
 		++bits;
 		i >>= 1;
 	}
-	i = 5 + (addL ? 1 : 0) + (size_a*SHIFT + bits-1) / bits;
-	str = (PyStringObject *) PyString_FromStringAndSize((char *)0, i);
+	i = 5 + (addL ? 1 : 0);
+	j = size_a*SHIFT + bits-1;
+	sz = i + j / bits;
+	if (j / SHIFT < size_a || sz < i) {
+		PyErr_SetString(PyExc_OverflowError,
+				"long is too large to format");
+		return NULL;
+	}
+	str = (PyStringObject *) PyString_FromStringAndSize((char *)0, sz);
 	if (str == NULL)
 		return NULL;
-	p = PyString_AS_STRING(str) + i;
+	p = PyString_AS_STRING(str) + sz;
 	*p = '\0';
         if (addL)
                 *--p = 'L';
@@ -1305,7 +1328,7 @@ long_format(PyObject *aa, int base, int addL)
 		} while ((*q++ = *p++) != '\0');
 		q--;
 		_PyString_Resize((PyObject **)&str,
-				 (int) (q - PyString_AS_STRING(str)));
+				 (Py_ssize_t) (q - PyString_AS_STRING(str)));
 	}
 	return (PyObject *)str;
 }
@@ -1363,14 +1386,14 @@ long_from_binary_base(char **str, int base)
 	while (_PyLong_DigitValue[Py_CHARMASK(*p)] < base)
 		++p;
 	*str = p;
-	n = (p - start) * bits_per_char;
-	if (n / bits_per_char != p - start) {
+	/* n <- # of Python digits needed, = ceiling(n/SHIFT). */
+	n = (p - start) * bits_per_char + SHIFT - 1;
+	if (n / bits_per_char < p - start) {
 		PyErr_SetString(PyExc_ValueError,
 				"long string too large to convert");
 		return NULL;
 	}
-	/* n <- # of Python digits needed, = ceiling(n/SHIFT). */
-	n = (n + SHIFT - 1) / SHIFT;
+	n = n / SHIFT;
 	z = _PyLong_New(n);
 	if (z == NULL)
 		return NULL;