Marc-Andre's third try at this bulk patch seems to work (except that

his copy of test_contains.py seems to be broken -- the lines he
deleted were already absent).  Checkin messages:


New Unicode support for int(), float(), complex() and long().

- new APIs PyInt_FromUnicode() and PyLong_FromUnicode()
- added support for Unicode to PyFloat_FromString()
- new encoding API PyUnicode_EncodeDecimal() which converts
  Unicode to a decimal char* string (used in the above new
  APIs)
- shortcuts for calls like int(<int object>) and float(<float obj>)
- tests for all of the above

Unicode compares and contains checks:
- comparing Unicode and non-string types now works; TypeErrors
  are masked, all other errors such as ValueError during
  Unicode coercion are passed through (note that PyUnicode_Compare
  does not implement the masking -- PyObject_Compare does this)
- contains now works for non-string types too; TypeErrors are
  masked and 0 returned; all other errors are passed through

Better testing support for the standard codecs.

Misc minor enhancements, such as an alias dbcs for the mbcs codec.

Changes:
- PyLong_FromString() now applies the same error checks as
  does PyInt_FromString(): trailing garbage is reported
  as error and not longer silently ignored. The only characters
  which may be trailing the digits are 'L' and 'l' -- these
  are still silently ignored.
- string.ato?() now directly interface to int(), long() and
  float(). The error strings are now a little different, but
  the type still remains the same. These functions are now
  ready to get declared obsolete ;-)
- PyNumber_Int() now also does a check for embedded NULL chars
  in the input string; PyNumber_Long() already did this (and
  still does)

Followed by:

Looks like I've gone a step too far there... (and test_contains.py
seem to have a bug too).

I've changed back to reporting all errors in PyUnicode_Contains()
and added a few more test cases to test_contains.py (plus corrected
the join() NameError).

5 files changed, 183 insertions, 53 deletions

diff --git a/Objects/abstract.c b/Objects/abstract.c
index 7202b43..410b80b 100644
--- a/Objects/abstract.c
+++ b/Objects/abstract.c
@@ -726,6 +726,27 @@ PyNumber_Absolute(o)
 	return type_error("bad operand type for abs()");
 }
 
+/* Add a check for embedded NULL-bytes in the argument. */
+static PyObject *
+int_from_string(s, len)
+	const char *s;
+	int len;
+{
+	char *end;
+	PyObject *x;
+
+	x = PyInt_FromString((char*)s, &end, 10);
+	if (x == NULL)
+		return NULL;
+	if (end != s + len) {
+		PyErr_SetString(PyExc_ValueError,
+				"null byte in argument for int()");
+		Py_DECREF(x);
+		return NULL;
+	}
+	return x;
+}
+
 PyObject *
 PyNumber_Int(o)
 	PyObject *o;
@@ -736,69 +757,42 @@ PyNumber_Int(o)
 
 	if (o == NULL)
 		return null_error();
+	if (PyInt_Check(o)) {
+		Py_INCREF(o);
+		return o;
+	}
 	if (PyString_Check(o))
-		return PyInt_FromString(PyString_AS_STRING(o), NULL, 10);
+		return int_from_string(PyString_AS_STRING(o), 
+				       PyString_GET_SIZE(o));
+	if (PyUnicode_Check(o))
+		return PyInt_FromUnicode(PyUnicode_AS_UNICODE(o),
+					 PyUnicode_GET_SIZE(o),
+					 10);
 	m = o->ob_type->tp_as_number;
 	if (m && m->nb_int)
 		return m->nb_int(o);
 	if (!PyObject_AsCharBuffer(o, &buffer, &buffer_len))
-		return PyInt_FromString((char*)buffer, NULL, 10);
+		return int_from_string((char*)buffer, buffer_len);
 
 	return type_error("object can't be converted to int");
 }
 
-/* There are two C API functions for converting a string to a long,
- * PyNumber_Long() and PyLong_FromString().  Both are used in builtin_long, 
- * reachable from Python with the built-in function long().
- *
- * The difference is this: PyNumber_Long will raise an exception when the
- * string cannot be converted to a long.  The most common situation is
- * where a float string is passed in; this raises a ValueError.
- * PyLong_FromString does not raise an exception; it silently truncates the 
- * float to an integer.
- *
- * You can see the different behavior from Python with the following:
- *
- * long('9.5')
- * => ValueError: invalid literal for long(): 9.5
- *
- * long('9.5', 10)
- * => 9L
- *
- * The first example ends up calling PyNumber_Long(), while the second one
- * calls PyLong_FromString().
- */
+/* Add a check for embedded NULL-bytes in the argument. */
 static PyObject *
 long_from_string(s, len)
 	const char *s;
 	int len;
 {
-	const char *start;
 	char *end;
 	PyObject *x;
-	char buffer[256]; /* For errors */
 
-	start = s;
-	while (*s && isspace(Py_CHARMASK(*s)))
-		s++;
 	x = PyLong_FromString((char*)s, &end, 10);
-	if (x == NULL) {
-		if (PyErr_ExceptionMatches(PyExc_ValueError))
-			goto bad;
-		return NULL;
-	}
-	while (*end && isspace(Py_CHARMASK(*end)))
-		end++;
-	if (*end != '\0') {
-  bad:
-		sprintf(buffer, "invalid literal for long(): %.200s", s);
-		PyErr_SetString(PyExc_ValueError, buffer);
-		Py_XDECREF(x);
+	if (x == NULL)
 		return NULL;
-	}
-	else if (end != start + len) {
+	if (end != s + len) {
 		PyErr_SetString(PyExc_ValueError,
 				"null byte in argument for long()");
+		Py_DECREF(x);
 		return NULL;
 	}
 	return x;
@@ -814,6 +808,10 @@ PyNumber_Long(o)
 
 	if (o == NULL)
 		return null_error();
+	if (PyLong_Check(o)) {
+		Py_INCREF(o);
+		return o;
+	}
 	if (PyString_Check(o))
 		/* need to do extra error checking that PyLong_FromString() 
 		 * doesn't do.  In particular long('9.5') must raise an
@@ -821,6 +819,11 @@ PyNumber_Long(o)
 		 */
 		return long_from_string(PyString_AS_STRING(o),
 					PyString_GET_SIZE(o));
+	if (PyUnicode_Check(o))
+		/* The above check is done in PyLong_FromUnicode(). */
+		return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o),
+					  PyUnicode_GET_SIZE(o),
+					  10);
 	m = o->ob_type->tp_as_number;
 	if (m && m->nb_long)
 		return m->nb_long(o);
@@ -838,6 +841,10 @@ PyNumber_Float(o)
 
 	if (o == NULL)
 		return null_error();
+	if (PyFloat_Check(o)) {
+		Py_INCREF(o);
+		return o;
+	}
 	if (!PyString_Check(o)) {
 		m = o->ob_type->tp_as_number;
 		if (m && m->nb_float)
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index fb1acdc..77ef8d0 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -164,6 +164,22 @@ PyFloat_FromString(v, pend)
 		s = PyString_AS_STRING(v);
 		len = PyString_GET_SIZE(v);
 	}
+	else if (PyUnicode_Check(v)) {
+		char s_buffer[256];
+
+		if (PyUnicode_GET_SIZE(v) >= sizeof(s_buffer)) {
+			PyErr_SetString(PyExc_ValueError,
+				 "float() literal too large to convert");
+			return NULL;
+		}
+		if (PyUnicode_EncodeDecimal(PyUnicode_AS_UNICODE(v), 
+					    PyUnicode_GET_SIZE(v),
+					    s_buffer, 
+					    NULL))
+			return NULL;
+		s = s_buffer;
+		len = strlen(s);
+	}
 	else if (PyObject_AsCharBuffer(v, &s, &len)) {
 		PyErr_SetString(PyExc_TypeError,
 				"float() needs a string argument");
diff --git a/Objects/intobject.c b/Objects/intobject.c
index 59c84ad..0c8eefc 100644
--- a/Objects/intobject.c
+++ b/Objects/intobject.c
@@ -261,6 +261,24 @@ PyInt_FromString(s, pend, base)
 	return PyInt_FromLong(x);
 }
 
+PyObject *
+PyInt_FromUnicode(s, length, base)
+	Py_UNICODE *s;
+	int length;
+	int base;
+{
+	char buffer[256];
+	
+	if (length >= sizeof(buffer)) {
+		PyErr_SetString(PyExc_ValueError,
+				"int() literal too large to convert");
+		return NULL;
+	}
+	if (PyUnicode_EncodeDecimal(s, length, buffer, NULL))
+		return NULL;
+	return PyInt_FromString(buffer, NULL, base);
+}
+
 /* Methods */
 
 /* ARGSUSED */
diff --git a/Objects/longobject.c b/Objects/longobject.c
index bdf4774..d5c2f5f 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -724,7 +724,7 @@ PyLong_FromString(str, pend, base)
 	int base;
 {
 	int sign = 1;
-	char *start;
+	char *start, *orig_str = str;
 	PyLongObject *z;
 	
 	if ((base != 0 && base < 2) || base > 36) {
@@ -772,17 +772,44 @@ PyLong_FromString(str, pend, base)
 	}
 	if (z == NULL)
 		return NULL;
-	if (str == start) {
-		PyErr_SetString(PyExc_ValueError,
-				"no digits in long int constant");
-		Py_DECREF(z);
-		return NULL;
-	}
+	if (str == start)
+		goto onError;
 	if (sign < 0 && z != NULL && z->ob_size != 0)
 		z->ob_size = -(z->ob_size);
+	if (*str == 'L' || *str == 'l')
+		str++;
+	while (*str && isspace(Py_CHARMASK(*str)))
+		str++;
+	if (*str != '\0')
+		goto onError;
 	if (pend)
 		*pend = str;
 	return (PyObject *) z;
+
+ onError:
+	PyErr_Format(PyExc_ValueError, 
+		     "invalid literal for long(): %.200s", orig_str);
+	Py_XDECREF(z);
+	return NULL;
+}
+
+PyObject *
+PyLong_FromUnicode(u, length, base)
+	Py_UNICODE *u;
+	int length;
+	int base;
+{
+	char buffer[256];
+
+	if (length >= sizeof(buffer)) {
+		PyErr_SetString(PyExc_ValueError,
+				"long() literal too large to convert");
+		return NULL;
+	}
+	if (PyUnicode_EncodeDecimal(u, length, buffer, NULL))
+		return NULL;
+
+	return PyLong_FromString(buffer, NULL, base);
 }
 
 static PyLongObject *x_divrem
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 9c35e2d..e751bc4 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -329,8 +329,14 @@ PyObject *PyUnicode_FromObject(register PyObject *obj)
 	s = PyString_AS_STRING(obj);
 	len = PyString_GET_SIZE(obj);
     }
-    else if (PyObject_AsCharBuffer(obj, &s, &len))
+    else if (PyObject_AsCharBuffer(obj, &s, &len)) {
+	/* Overwrite the error message with something more useful in
+	   case of a TypeError. */
+	if (PyErr_ExceptionMatches(PyExc_TypeError))
+	    PyErr_SetString(PyExc_TypeError,
+			    "coercing to Unicode: need string or charbuffer");
 	return NULL;
+    }
     if (len == 0) {
 	Py_INCREF(unicode_empty);
 	return (PyObject *)unicode_empty;
@@ -1923,6 +1929,60 @@ PyObject *PyUnicode_Translate(PyObject *str,
     return NULL;
 }
     
+/* --- Decimal Encoder ---------------------------------------------------- */
+
+int PyUnicode_EncodeDecimal(Py_UNICODE *s,
+			    int length,
+			    char *output,
+			    const char *errors)
+{
+    Py_UNICODE *p, *end;
+
+    if (output == NULL) {
+	PyErr_BadArgument();
+	return -1;
+    }
+
+    p = s;
+    end = s + length;
+    while (p < end) {
+	register Py_UNICODE ch = *p++;
+	int decimal;
+	
+	if (Py_UNICODE_ISSPACE(ch)) {
+	    *output++ = ' ';
+	    continue;
+	}
+	decimal = Py_UNICODE_TODECIMAL(ch);
+	if (decimal >= 0) {
+	    *output++ = '0' + decimal;
+	    continue;
+	}
+	if (0 < ch < 256) {
+	    *output++ = ch;
+	    continue;
+	}
+	/* All other characters are considered invalid */
+	if (errors == NULL || strcmp(errors, "strict") == 0) {
+	    PyErr_SetString(PyExc_ValueError,
+			    "invalid decimal Unicode string");
+	    goto onError;
+	}
+	else if (strcmp(errors, "ignore") == 0)
+	    continue;
+	else if (strcmp(errors, "replace") == 0) {
+	    *output++ = '?';
+	    continue;
+	}
+    }
+    /* 0-terminate the output string */
+    *output++ = '\0';
+    return 0;
+
+ onError:
+    return -1;
+}
+
 /* --- Helpers ------------------------------------------------------------ */
 
 static 
@@ -2811,12 +2871,14 @@ int PyUnicode_Contains(PyObject *container,
     register Py_UNICODE ch;
 
     /* Coerce the two arguments */
-    u = (PyUnicodeObject *)PyUnicode_FromObject(container);
-    if (u == NULL)
-	goto onError;
     v = (PyUnicodeObject *)PyUnicode_FromObject(element);
     if (v == NULL)
 	goto onError;
+    u = (PyUnicodeObject *)PyUnicode_FromObject(container);
+    if (u == NULL) {
+	Py_DECREF(v);
+	goto onError;
+    }
 
     /* Check v in u */
     if (PyUnicode_GET_SIZE(v) != 1) {