Marc-Andre's third try at this bulk patch seems to work (except that

his copy of test_contains.py seems to be broken -- the lines he
deleted were already absent).  Checkin messages:


New Unicode support for int(), float(), complex() and long().

- new APIs PyInt_FromUnicode() and PyLong_FromUnicode()
- added support for Unicode to PyFloat_FromString()
- new encoding API PyUnicode_EncodeDecimal() which converts
  Unicode to a decimal char* string (used in the above new
  APIs)
- shortcuts for calls like int(<int object>) and float(<float obj>)
- tests for all of the above

Unicode compares and contains checks:
- comparing Unicode and non-string types now works; TypeErrors
  are masked, all other errors such as ValueError during
  Unicode coercion are passed through (note that PyUnicode_Compare
  does not implement the masking -- PyObject_Compare does this)
- contains now works for non-string types too; TypeErrors are
  masked and 0 returned; all other errors are passed through

Better testing support for the standard codecs.

Misc minor enhancements, such as an alias dbcs for the mbcs codec.

Changes:
- PyLong_FromString() now applies the same error checks as
  does PyInt_FromString(): trailing garbage is reported
  as error and not longer silently ignored. The only characters
  which may be trailing the digits are 'L' and 'l' -- these
  are still silently ignored.
- string.ato?() now directly interface to int(), long() and
  float(). The error strings are now a little different, but
  the type still remains the same. These functions are now
  ready to get declared obsolete ;-)
- PyNumber_Int() now also does a check for embedded NULL chars
  in the input string; PyNumber_Long() already did this (and
  still does)

Followed by:

Looks like I've gone a step too far there... (and test_contains.py
seem to have a bug too).

I've changed back to reporting all errors in PyUnicode_Contains()
and added a few more test cases to test_contains.py (plus corrected
the join() NameError).

2 files changed, 61 insertions, 13 deletions

diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 90af888..3016d56 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -449,17 +449,44 @@ complex_from_string(v)
 	PyObject *v;
 {
 	extern double strtod Py_PROTO((const char *, char **));
-	char *s, *start, *end;
+	const char *s, *start;
+	char *end;
 	double x=0.0, y=0.0, z;
 	int got_re=0, got_im=0, done=0;
 	int digit_or_dot;
 	int sw_error=0;
 	int sign;
 	char buffer[256]; /* For errors */
+	int len;
+
+	if (PyString_Check(v)) {
+		s = PyString_AS_STRING(v);
+		len = PyString_GET_SIZE(v);
+	}
+	else if (PyUnicode_Check(v)) {
+		char s_buffer[256];
 
-	start = s = PyString_AS_STRING(v);
+		if (PyUnicode_GET_SIZE(v) >= sizeof(s_buffer)) {
+			PyErr_SetString(PyExc_ValueError,
+				 "complex() literal too large to convert");
+			return NULL;
+		}
+		if (PyUnicode_EncodeDecimal(PyUnicode_AS_UNICODE(v), 
+					    PyUnicode_GET_SIZE(v),
+					    s_buffer, 
+					    NULL))
+			return NULL;
+		s = s_buffer;
+		len = strlen(s);
+	}
+	else if (PyObject_AsCharBuffer(v, &s, &len)) {
+		PyErr_SetString(PyExc_TypeError,
+				"complex() needs a string first argument");
+		return NULL;
+	}
 
 	/* position on first nonblank */
+	start = s;
 	while (*s && isspace(Py_CHARMASK(*s)))
 		s++;
 	if (s[0] == '\0') {
@@ -475,7 +502,7 @@ complex_from_string(v)
 		switch (*s) {
 
 		case '\0':
-			if (s-start != PyString_GET_SIZE(v)) {
+			if (s-start != len) {
 				PyErr_SetString(
 					PyExc_ValueError,
 					"null byte in argument for complex()");
@@ -584,7 +611,7 @@ builtin_complex(self, args)
 	i = NULL;
 	if (!PyArg_ParseTuple(args, "O|O:complex", &r, &i))
 		return NULL;
-	if (PyString_Check(r))
+	if (PyString_Check(r) || PyUnicode_Check(r))
 		return complex_from_string(r);
 	if ((nbr = r->ob_type->tp_as_number) == NULL ||
 	    nbr->nb_float == NULL ||
@@ -1289,12 +1316,17 @@ builtin_int(self, args)
 		return NULL;
 	if (base == -909)
 		return PyNumber_Int(v);
-	else if (!PyString_Check(v)) {
+	else if (PyString_Check(v))
+		return PyInt_FromString(PyString_AS_STRING(v), NULL, base);
+	else if (PyUnicode_Check(v))
+		return PyInt_FromUnicode(PyUnicode_AS_UNICODE(v),
+					 PyUnicode_GET_SIZE(v),
+					 base);
+	else {
 		PyErr_SetString(PyExc_TypeError,
 				"can't convert non-string with explicit base");
 		return NULL;
 	}
-	return PyInt_FromString(PyString_AS_STRING(v), NULL, base);
 }
 
 static char int_doc[] =
@@ -1319,12 +1351,17 @@ builtin_long(self, args)
 		return NULL;
 	if (base == -909)
 		return PyNumber_Long(v);
-	else if (!PyString_Check(v)) {
+	else if (PyString_Check(v))
+		return PyLong_FromString(PyString_AS_STRING(v), NULL, base);
+	else if (PyUnicode_Check(v))
+		return PyLong_FromUnicode(PyUnicode_AS_UNICODE(v),
+					  PyUnicode_GET_SIZE(v),
+					  base);
+	else {
 		PyErr_SetString(PyExc_TypeError,
 				"can't convert non-string with explicit base");
 		return NULL;
 	}
-	return PyLong_FromString(PyString_AS_STRING(v), NULL, base);
 }
 
 static char long_doc[] =
diff --git a/Python/codecs.c b/Python/codecs.c
index b9a6461..4df389a 100644
--- a/Python/codecs.c
+++ b/Python/codecs.c
@@ -84,8 +84,11 @@ int PyCodec_Register(PyObject *search_function)
     return -1;
 }
 
+/* Convert a string to a normalized Python string: all characters are
+   converted to lower case, spaces are replaced with underscores. */
+
 static
-PyObject *lowercasestring(const char *string)
+PyObject *normalizestring(const char *string)
 {
     register int i;
     int len = strlen(string);
@@ -96,8 +99,14 @@ PyObject *lowercasestring(const char *string)
     if (v == NULL)
 	return NULL;
     p = PyString_AS_STRING(v);
-    for (i = 0; i < len; i++)
-	p[i] = tolower(string[i]);
+    for (i = 0; i < len; i++) {
+        register char ch = string[i];
+        if (ch == ' ')
+            ch = '-';
+        else
+            ch = tolower(ch);
+	p[i] = ch;
+    }
     return v;
 }
 
@@ -132,8 +141,10 @@ PyObject *_PyCodec_Lookup(const char *encoding)
 	    goto onError;
     }
 
-    /* Convert the encoding to a lower-cased Python string */
-    v = lowercasestring(encoding);
+    /* Convert the encoding to a normalized Python string: all
+       characters are converted to lower case, spaces and hypens are
+       replaced with underscores. */
+    v = normalizestring(encoding);
     if (v == NULL)
 	goto onError;
     PyString_InternInPlace(&v);