summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--Include/Python.h2
-rw-r--r--Include/intobject.h1
-rw-r--r--Include/longobject.h1
-rw-r--r--Include/unicodeobject.h35
-rw-r--r--Lib/encodings/__init__.py6
-rw-r--r--Lib/encodings/aliases.py3
-rw-r--r--Lib/string.py39
-rw-r--r--Lib/test/output/test_unicode1
-rw-r--r--Lib/test/test_b1.py37
-rw-r--r--Lib/test/test_unicode.py101
-rw-r--r--Objects/abstract.c91
-rw-r--r--Objects/floatobject.c16
-rw-r--r--Objects/intobject.c18
-rw-r--r--Objects/longobject.c41
-rw-r--r--Objects/unicodeobject.c70
-rw-r--r--Python/bltinmodule.c53
-rw-r--r--Python/codecs.c21
17 files changed, 421 insertions, 115 deletions
diff --git a/Include/Python.h b/Include/Python.h
index 4179b2e..044ad5f 100644
--- a/Include/Python.h
+++ b/Include/Python.h
@@ -72,6 +72,7 @@ PERFORMANCE OF THIS SOFTWARE.
#include "pydebug.h"
+#include "unicodeobject.h"
#include "intobject.h"
#include "longobject.h"
#include "floatobject.h"
@@ -92,7 +93,6 @@ PERFORMANCE OF THIS SOFTWARE.
#include "cobject.h"
#include "traceback.h"
#include "sliceobject.h"
-#include "unicodeobject.h"
#include "codecs.h"
#include "pyerrors.h"
diff --git a/Include/intobject.h b/Include/intobject.h
index 781a4b5..507a9e2 100644
--- a/Include/intobject.h
+++ b/Include/intobject.h
@@ -60,6 +60,7 @@ extern DL_IMPORT(PyTypeObject) PyInt_Type;
#define PyInt_Check(op) ((op)->ob_type == &PyInt_Type)
extern DL_IMPORT(PyObject *) PyInt_FromString Py_PROTO((char*, char**, int));
+extern DL_IMPORT(PyObject *) PyInt_FromUnicode Py_PROTO((Py_UNICODE*, int, int));
extern DL_IMPORT(PyObject *) PyInt_FromLong Py_PROTO((long));
extern DL_IMPORT(long) PyInt_AsLong Py_PROTO((PyObject *));
extern DL_IMPORT(long) PyInt_GetMax Py_PROTO((void));
diff --git a/Include/longobject.h b/Include/longobject.h
index 89f2aa3..ca1adb7 100644
--- a/Include/longobject.h
+++ b/Include/longobject.h
@@ -82,6 +82,7 @@ extern DL_IMPORT(unsigned LONG_LONG) PyLong_AsUnsignedLongLong Py_PROTO((PyObjec
#endif /* HAVE_LONG_LONG */
DL_IMPORT(PyObject *) PyLong_FromString Py_PROTO((char *, char **, int));
+DL_IMPORT(PyObject *) PyLong_FromUnicode Py_PROTO((Py_UNICODE*, int, int));
#ifdef __cplusplus
}
diff --git a/Include/unicodeobject.h b/Include/unicodeobject.h
index cfc8126..e1f5914 100644
--- a/Include/unicodeobject.h
+++ b/Include/unicodeobject.h
@@ -358,7 +358,7 @@ extern DL_IMPORT(PyObject*) PyUnicode_EncodeUTF8(
/* --- UTF-16 Codecs ------------------------------------------------------ */
-/* Decodes length bytes from a UTF-16 encoded buffer string and return
+/* Decodes length bytes from a UTF-16 encoded buffer string and returns
the corresponding Unicode object.
errors (if non-NULL) defines the error handling. It defaults
@@ -397,7 +397,7 @@ extern DL_IMPORT(PyObject*) PyUnicode_AsUTF16String(
);
/* Returns a Python string object holding the UTF-16 encoded value of
- the Unicode data in s.
+ the Unicode data.
If byteorder is not 0, output is written according to the following
byte order:
@@ -587,6 +587,37 @@ extern DL_IMPORT(PyObject*) PyUnicode_EncodeMBCS(
#endif /* MS_WIN32 */
+/* --- Decimal Encoder ---------------------------------------------------- */
+
+/* Takes a Unicode string holding a decimal value and writes it into
+ an output buffer using standard ASCII digit codes.
+
+ The output buffer has to provide at least length+1 bytes of storage
+ area. The output string is 0-terminated.
+
+ The encoder converts whitespace to ' ', decimal characters to their
+ corresponding ASCII digit and all other Latin-1 characters except
+ \0 as-is. Characters outside this range (Unicode ordinals 1-256)
+ are treated as errors. This includes embedded NULL bytes.
+
+ Error handling is defined by the errors argument:
+
+ NULL or "strict": raise a ValueError
+ "ignore": ignore the wrong characters (these are not copied to the
+ output buffer)
+ "replace": replaces illegal characters with '?'
+
+ Returns 0 on success, -1 on failure.
+
+*/
+
+extern DL_IMPORT(int) PyUnicode_EncodeDecimal(
+ Py_UNICODE *s, /* Unicode buffer */
+ int length, /* Number of Py_UNICODE chars to encode */
+ char *output, /* Output buffer; must have size >= length */
+ const char *errors /* error handling */
+ );
+
/* --- Methods & Slots ----------------------------------------------------
These are capable of handling Unicode objects and strings on input
diff --git a/Lib/encodings/__init__.py b/Lib/encodings/__init__.py
index cd5876e..fa3119a 100644
--- a/Lib/encodings/__init__.py
+++ b/Lib/encodings/__init__.py
@@ -4,8 +4,8 @@
directory.
Codec modules must have names corresponding to standard lower-case
- encoding names. Hyphens are automatically converted to
- underscores, e.g. 'utf-8' is looked up as module utf_8.
+ encoding names with hyphens mapped to underscores, e.g. 'utf-8' is
+ implemented by the module 'utf_8.py'.
Each codec module must export the following interface:
@@ -40,7 +40,7 @@ def search_function(encoding):
return entry
# Import the module
- modname = string.replace(encoding,'-','_')
+ modname = string.replace(encoding, '-', '_')
modname = aliases.aliases.get(modname,modname)
try:
mod = __import__(modname,globals(),locals(),'*')
diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py
index d99b38e..1214cf9 100644
--- a/Lib/encodings/aliases.py
+++ b/Lib/encodings/aliases.py
@@ -54,4 +54,7 @@ aliases = {
'macroman': 'mac_roman',
'macturkish': 'mac_turkish',
+ # MBCS
+ 'dbcs': 'mbcs',
+
}
diff --git a/Lib/string.py b/Lib/string.py
index 5eb5697..5f90723 100644
--- a/Lib/string.py
+++ b/Lib/string.py
@@ -196,14 +196,11 @@ def atof(s):
Return the floating point number represented by the string s.
"""
- if type(s) == _StringType:
- return _float(s)
- else:
- raise TypeError('argument 1: expected string, %s found' %
- type(s).__name__)
+ return _float(s)
+
# Convert string to integer
-def atoi(*args):
+def atoi(s , base=10):
"""atoi(s [,base]) -> int
Return the integer represented by the string s in the given
@@ -214,23 +211,11 @@ def atoi(*args):
accepted.
"""
- try:
- s = args[0]
- except IndexError:
- raise TypeError('function requires at least 1 argument: %d given' %
- len(args))
- # Don't catch type error resulting from too many arguments to int(). The
- # error message isn't compatible but the error type is, and this function
- # is complicated enough already.
- if type(s) == _StringType:
- return _apply(_int, args)
- else:
- raise TypeError('argument 1: expected string, %s found' %
- type(s).__name__)
+ return _int(s, base)
# Convert string to long integer
-def atol(*args):
+def atol(s, base=10):
"""atol(s [,base]) -> long
Return the long integer represented by the string s in the
@@ -242,19 +227,7 @@ def atol(*args):
unless base is 0.
"""
- try:
- s = args[0]
- except IndexError:
- raise TypeError('function requires at least 1 argument: %d given' %
- len(args))
- # Don't catch type error resulting from too many arguments to long(). The
- # error message isn't compatible but the error type is, and this function
- # is complicated enough already.
- if type(s) == _StringType:
- return _apply(_long, args)
- else:
- raise TypeError('argument 1: expected string, %s found' %
- type(s).__name__)
+ return _long(s, base)
# Left-justify a string
diff --git a/Lib/test/output/test_unicode b/Lib/test/output/test_unicode
index 88e8624..15ffdc1 100644
--- a/Lib/test/output/test_unicode
+++ b/Lib/test/output/test_unicode
@@ -3,3 +3,4 @@ Testing Unicode comparisons... done.
Testing Unicode contains method... done.
Testing Unicode formatting strings... done.
Testing builtin codecs... done.
+Testing standard mapping codecs... 0-127... 128-255... done.
diff --git a/Lib/test/test_b1.py b/Lib/test/test_b1.py
index d9c6320..6a89d22 100644
--- a/Lib/test/test_b1.py
+++ b/Lib/test/test_b1.py
@@ -95,6 +95,7 @@ if complex(0.0, 3.14j) <> -3.14+0j: raise TestFailed, 'complex(0.0, 3.14j)'
if complex(0j, 3.14) <> 3.14j: raise TestFailed, 'complex(0j, 3.14)'
if complex(0.0, 3.14) <> 3.14j: raise TestFailed, 'complex(0.0, 3.14)'
if complex(" 3.14+J ") <> 3.14+1j: raise TestFailed, 'complex(" 3.14+J )"'
+if complex(u" 3.14+J ") <> 3.14+1j: raise TestFailed, 'complex(u" 3.14+J )"'
class Z:
def __complex__(self): return 3.14j
z = Z()
@@ -208,6 +209,9 @@ if float(3.14) <> 3.14: raise TestFailed, 'float(3.14)'
if float(314) <> 314.0: raise TestFailed, 'float(314)'
if float(314L) <> 314.0: raise TestFailed, 'float(314L)'
if float(" 3.14 ") <> 3.14: raise TestFailed, 'float(" 3.14 ")'
+if float(u" 3.14 ") <> 3.14: raise TestFailed, 'float(u" 3.14 ")'
+if float(u" \u0663.\u0661\u0664 ") <> 3.14:
+ raise TestFailed, 'float(u" \u0663.\u0661\u0664 ")'
print 'getattr'
import sys
@@ -254,6 +258,9 @@ if int(3.9) <> 3: raise TestFailed, 'int(3.9)'
if int(-3.9) <> -3: raise TestFailed, 'int(-3.9)'
if int(3.5) <> 3: raise TestFailed, 'int(3.5)'
if int(-3.5) <> -3: raise TestFailed, 'int(-3.5)'
+# Different base:
+if int("10",16) <> 16L: raise TestFailed, 'int("10",16)'
+if int(u"10",16) <> 16L: raise TestFailed, 'int(u"10",16)'
# Test conversion fron strings and various anomalies
L = [
('0', 0),
@@ -267,9 +274,28 @@ L = [
('314 ', 314),
(' \t\t 314 \t\t ', 314),
(`sys.maxint`, sys.maxint),
+ (' 1x', ValueError),
+ (' 1 ', 1),
+ (' 1\02 ', ValueError),
('', ValueError),
(' ', ValueError),
(' \t\t ', ValueError),
+ (u'0', 0),
+ (u'1', 1),
+ (u'9', 9),
+ (u'10', 10),
+ (u'99', 99),
+ (u'100', 100),
+ (u'314', 314),
+ (u' 314', 314),
+ (u'\u0663\u0661\u0664 ', 314),
+ (u' \t\t 314 \t\t ', 314),
+ (u' 1x', ValueError),
+ (u' 1 ', 1),
+ (u' 1\02 ', ValueError),
+ (u'', ValueError),
+ (u' ', ValueError),
+ (u' \t\t ', ValueError),
]
for s, v in L:
for sign in "", "+", "-":
@@ -349,10 +375,17 @@ if long(3.9) <> 3L: raise TestFailed, 'long(3.9)'
if long(-3.9) <> -3L: raise TestFailed, 'long(-3.9)'
if long(3.5) <> 3L: raise TestFailed, 'long(3.5)'
if long(-3.5) <> -3L: raise TestFailed, 'long(-3.5)'
+if long("-3") <> -3L: raise TestFailed, 'long("-3")'
+if long(u"-3") <> -3L: raise TestFailed, 'long(u"-3")'
+# Different base:
+if long("10",16) <> 16L: raise TestFailed, 'long("10",16)'
+if long(u"10",16) <> 16L: raise TestFailed, 'long(u"10",16)'
# Check conversions from string (same test set as for int(), and then some)
LL = [
('1' + '0'*20, 10L**20),
('1' + '0'*100, 10L**100),
+ (u'1' + u'0'*20, 10L**20),
+ (u'1' + u'0'*100, 10L**100),
]
for s, v in L + LL:
for sign in "", "+", "-":
@@ -363,11 +396,11 @@ for s, v in L + LL:
vv = -v
try:
if long(ss) != long(vv):
- raise TestFailed, "int(%s)" % `ss`
+ raise TestFailed, "long(%s)" % `ss`
except v:
pass
except ValueError, e:
- raise TestFailed, "int(%s) raised ValueError: %s" % (`ss`, e)
+ raise TestFailed, "long(%s) raised ValueError: %s" % (`ss`, e)
print 'map'
if map(None, 'hello world') <> ['h','e','l','l','o',' ','w','o','r','l','d']:
diff --git a/Lib/test/test_unicode.py b/Lib/test/test_unicode.py
index f90887a..5c0a063 100644
--- a/Lib/test/test_unicode.py
+++ b/Lib/test/test_unicode.py
@@ -221,15 +221,23 @@ test('translate', u"abababc", u'iiix', {ord('a'):None, ord('b'):ord('i'), ord('c
# Contains:
print 'Testing Unicode contains method...',
-assert ('a' in 'abdb') == 1
-assert ('a' in 'bdab') == 1
-assert ('a' in 'bdaba') == 1
-assert ('a' in 'bdba') == 1
+assert ('a' in u'abdb') == 1
+assert ('a' in u'bdab') == 1
+assert ('a' in u'bdaba') == 1
+assert ('a' in u'bdba') == 1
assert ('a' in u'bdba') == 1
assert (u'a' in u'bdba') == 1
assert (u'a' in u'bdb') == 0
assert (u'a' in 'bdb') == 0
assert (u'a' in 'bdba') == 1
+assert (u'a' in ('a',1,None)) == 1
+assert (u'a' in (1,None,'a')) == 1
+assert (u'a' in (1,None,u'a')) == 1
+assert ('a' in ('a',1,None)) == 1
+assert ('a' in (1,None,'a')) == 1
+assert ('a' in (1,None,u'a')) == 1
+assert ('a' in ('x',1,u'y')) == 0
+assert ('a' in ('x',1,None)) == 0
print 'done.'
# Formatting:
@@ -270,11 +278,88 @@ for encoding in ('utf-8', 'utf-16', 'utf-16-le', 'utf-16-be',
assert unicode(u.encode(encoding),encoding) == u
u = u''.join(map(unichr, range(256)))
-for encoding in ('latin-1',):
- assert unicode(u.encode(encoding),encoding) == u
+for encoding in (
+ 'latin-1',
+ ):
+ try:
+ assert unicode(u.encode(encoding),encoding) == u
+ except AssertionError:
+ print '*** codec "%s" failed round-trip' % encoding
+ except ValueError,why:
+ print '*** codec for "%s" failed: %s' % (encoding, why)
u = u''.join(map(unichr, range(128)))
-for encoding in ('ascii',):
- assert unicode(u.encode(encoding),encoding) == u
+for encoding in (
+ 'ascii',
+ ):
+ try:
+ assert unicode(u.encode(encoding),encoding) == u
+ except AssertionError:
+ print '*** codec "%s" failed round-trip' % encoding
+ except ValueError,why:
+ print '*** codec for "%s" failed: %s' % (encoding, why)
+
+print 'done.'
+
+print 'Testing standard mapping codecs...',
+
+print '0-127...',
+s = ''.join(map(chr, range(128)))
+for encoding in (
+ 'cp037', 'cp1026',
+ 'cp437', 'cp500', 'cp737', 'cp775', 'cp850',
+ 'cp852', 'cp855', 'cp860', 'cp861', 'cp862',
+ 'cp863', 'cp865', 'cp866',
+ 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
+ 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6',
+ 'iso8859_7', 'iso8859_9', 'koi8_r', 'latin_1',
+ 'mac_cyrillic', 'mac_latin2',
+
+ 'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
+ 'cp1256', 'cp1257', 'cp1258',
+ 'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
+
+ 'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
+ 'cp1006', 'cp875', 'iso8859_8',
+
+ ### These have undefined mappings:
+ #'cp424',
+
+ ):
+ try:
+ assert unicode(s,encoding).encode(encoding) == s
+ except AssertionError:
+ print '*** codec "%s" failed round-trip' % encoding
+ except ValueError,why:
+ print '*** codec for "%s" failed: %s' % (encoding, why)
+
+print '128-255...',
+s = ''.join(map(chr, range(128,256)))
+for encoding in (
+ 'cp037', 'cp1026',
+ 'cp437', 'cp500', 'cp737', 'cp775', 'cp850',
+ 'cp852', 'cp855', 'cp860', 'cp861', 'cp862',
+ 'cp863', 'cp865', 'cp866',
+ 'iso8859_10', 'iso8859_13', 'iso8859_14', 'iso8859_15',
+ 'iso8859_2', 'iso8859_3', 'iso8859_4', 'iso8859_5', 'iso8859_6',
+ 'iso8859_7', 'iso8859_9', 'koi8_r', 'latin_1',
+ 'mac_cyrillic', 'mac_latin2',
+
+ ### These have undefined mappings:
+ #'cp1250', 'cp1251', 'cp1252', 'cp1253', 'cp1254', 'cp1255',
+ #'cp1256', 'cp1257', 'cp1258',
+ #'cp424', 'cp856', 'cp857', 'cp864', 'cp869', 'cp874',
+ #'mac_greek', 'mac_iceland','mac_roman', 'mac_turkish',
+
+ ### These fail the round-trip:
+ #'cp1006', 'cp875', 'iso8859_8',
+
+ ):
+ try:
+ assert unicode(s,encoding).encode(encoding) == s
+ except AssertionError:
+ print '*** codec "%s" failed round-trip' % encoding
+ except ValueError,why:
+ print '*** codec for "%s" failed: %s' % (encoding, why)
print 'done.'
diff --git a/Objects/abstract.c b/Objects/abstract.c
index 7202b43..410b80b 100644
--- a/Objects/abstract.c
+++ b/Objects/abstract.c
@@ -726,6 +726,27 @@ PyNumber_Absolute(o)
return type_error("bad operand type for abs()");
}
+/* Add a check for embedded NULL-bytes in the argument. */
+static PyObject *
+int_from_string(s, len)
+ const char *s;
+ int len;
+{
+ char *end;
+ PyObject *x;
+
+ x = PyInt_FromString((char*)s, &end, 10);
+ if (x == NULL)
+ return NULL;
+ if (end != s + len) {
+ PyErr_SetString(PyExc_ValueError,
+ "null byte in argument for int()");
+ Py_DECREF(x);
+ return NULL;
+ }
+ return x;
+}
+
PyObject *
PyNumber_Int(o)
PyObject *o;
@@ -736,69 +757,42 @@ PyNumber_Int(o)
if (o == NULL)
return null_error();
+ if (PyInt_Check(o)) {
+ Py_INCREF(o);
+ return o;
+ }
if (PyString_Check(o))
- return PyInt_FromString(PyString_AS_STRING(o), NULL, 10);
+ return int_from_string(PyString_AS_STRING(o),
+ PyString_GET_SIZE(o));
+ if (PyUnicode_Check(o))
+ return PyInt_FromUnicode(PyUnicode_AS_UNICODE(o),
+ PyUnicode_GET_SIZE(o),
+ 10);
m = o->ob_type->tp_as_number;
if (m && m->nb_int)
return m->nb_int(o);
if (!PyObject_AsCharBuffer(o, &buffer, &buffer_len))
- return PyInt_FromString((char*)buffer, NULL, 10);
+ return int_from_string((char*)buffer, buffer_len);
return type_error("object can't be converted to int");
}
-/* There are two C API functions for converting a string to a long,
- * PyNumber_Long() and PyLong_FromString(). Both are used in builtin_long,
- * reachable from Python with the built-in function long().
- *
- * The difference is this: PyNumber_Long will raise an exception when the
- * string cannot be converted to a long. The most common situation is
- * where a float string is passed in; this raises a ValueError.
- * PyLong_FromString does not raise an exception; it silently truncates the
- * float to an integer.
- *
- * You can see the different behavior from Python with the following:
- *
- * long('9.5')
- * => ValueError: invalid literal for long(): 9.5
- *
- * long('9.5', 10)
- * => 9L
- *
- * The first example ends up calling PyNumber_Long(), while the second one
- * calls PyLong_FromString().
- */
+/* Add a check for embedded NULL-bytes in the argument. */
static PyObject *
long_from_string(s, len)
const char *s;
int len;
{
- const char *start;
char *end;
PyObject *x;
- char buffer[256]; /* For errors */
- start = s;
- while (*s && isspace(Py_CHARMASK(*s)))
- s++;
x = PyLong_FromString((char*)s, &end, 10);
- if (x == NULL) {
- if (PyErr_ExceptionMatches(PyExc_ValueError))
- goto bad;
- return NULL;
- }
- while (*end && isspace(Py_CHARMASK(*end)))
- end++;
- if (*end != '\0') {
- bad:
- sprintf(buffer, "invalid literal for long(): %.200s", s);
- PyErr_SetString(PyExc_ValueError, buffer);
- Py_XDECREF(x);
+ if (x == NULL)
return NULL;
- }
- else if (end != start + len) {
+ if (end != s + len) {
PyErr_SetString(PyExc_ValueError,
"null byte in argument for long()");
+ Py_DECREF(x);
return NULL;
}
return x;
@@ -814,6 +808,10 @@ PyNumber_Long(o)
if (o == NULL)
return null_error();
+ if (PyLong_Check(o)) {
+ Py_INCREF(o);
+ return o;
+ }
if (PyString_Check(o))
/* need to do extra error checking that PyLong_FromString()
* doesn't do. In particular long('9.5') must raise an
@@ -821,6 +819,11 @@ PyNumber_Long(o)
*/
return long_from_string(PyString_AS_STRING(o),
PyString_GET_SIZE(o));
+ if (PyUnicode_Check(o))
+ /* The above check is done in PyLong_FromUnicode(). */
+ return PyLong_FromUnicode(PyUnicode_AS_UNICODE(o),
+ PyUnicode_GET_SIZE(o),
+ 10);
m = o->ob_type->tp_as_number;
if (m && m->nb_long)
return m->nb_long(o);
@@ -838,6 +841,10 @@ PyNumber_Float(o)
if (o == NULL)
return null_error();
+ if (PyFloat_Check(o)) {
+ Py_INCREF(o);
+ return o;
+ }
if (!PyString_Check(o)) {
m = o->ob_type->tp_as_number;
if (m && m->nb_float)
diff --git a/Objects/floatobject.c b/Objects/floatobject.c
index fb1acdc..77ef8d0 100644
--- a/Objects/floatobject.c
+++ b/Objects/floatobject.c
@@ -164,6 +164,22 @@ PyFloat_FromString(v, pend)
s = PyString_AS_STRING(v);
len = PyString_GET_SIZE(v);
}
+ else if (PyUnicode_Check(v)) {
+ char s_buffer[256];
+
+ if (PyUnicode_GET_SIZE(v) >= sizeof(s_buffer)) {
+ PyErr_SetString(PyExc_ValueError,
+ "float() literal too large to convert");
+ return NULL;
+ }
+ if (PyUnicode_EncodeDecimal(PyUnicode_AS_UNICODE(v),
+ PyUnicode_GET_SIZE(v),
+ s_buffer,
+ NULL))
+ return NULL;
+ s = s_buffer;
+ len = strlen(s);
+ }
else if (PyObject_AsCharBuffer(v, &s, &len)) {
PyErr_SetString(PyExc_TypeError,
"float() needs a string argument");
diff --git a/Objects/intobject.c b/Objects/intobject.c
index 59c84ad..0c8eefc 100644
--- a/Objects/intobject.c
+++ b/Objects/intobject.c
@@ -261,6 +261,24 @@ PyInt_FromString(s, pend, base)
return PyInt_FromLong(x);
}
+PyObject *
+PyInt_FromUnicode(s, length, base)
+ Py_UNICODE *s;
+ int length;
+ int base;
+{
+ char buffer[256];
+
+ if (length >= sizeof(buffer)) {
+ PyErr_SetString(PyExc_ValueError,
+ "int() literal too large to convert");
+ return NULL;
+ }
+ if (PyUnicode_EncodeDecimal(s, length, buffer, NULL))
+ return NULL;
+ return PyInt_FromString(buffer, NULL, base);
+}
+
/* Methods */
/* ARGSUSED */
diff --git a/Objects/longobject.c b/Objects/longobject.c
index bdf4774..d5c2f5f 100644
--- a/Objects/longobject.c
+++ b/Objects/longobject.c
@@ -724,7 +724,7 @@ PyLong_FromString(str, pend, base)
int base;
{
int sign = 1;
- char *start;
+ char *start, *orig_str = str;
PyLongObject *z;
if ((base != 0 && base < 2) || base > 36) {
@@ -772,17 +772,44 @@ PyLong_FromString(str, pend, base)
}
if (z == NULL)
return NULL;
- if (str == start) {
- PyErr_SetString(PyExc_ValueError,
- "no digits in long int constant");
- Py_DECREF(z);
- return NULL;
- }
+ if (str == start)
+ goto onError;
if (sign < 0 && z != NULL && z->ob_size != 0)
z->ob_size = -(z->ob_size);
+ if (*str == 'L' || *str == 'l')
+ str++;
+ while (*str && isspace(Py_CHARMASK(*str)))
+ str++;
+ if (*str != '\0')
+ goto onError;
if (pend)
*pend = str;
return (PyObject *) z;
+
+ onError:
+ PyErr_Format(PyExc_ValueError,
+ "invalid literal for long(): %.200s", orig_str);
+ Py_XDECREF(z);
+ return NULL;
+}
+
+PyObject *
+PyLong_FromUnicode(u, length, base)
+ Py_UNICODE *u;
+ int length;
+ int base;
+{
+ char buffer[256];
+
+ if (length >= sizeof(buffer)) {
+ PyErr_SetString(PyExc_ValueError,
+ "long() literal too large to convert");
+ return NULL;
+ }
+ if (PyUnicode_EncodeDecimal(u, length, buffer, NULL))
+ return NULL;
+
+ return PyLong_FromString(buffer, NULL, base);
}
static PyLongObject *x_divrem
diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c
index 9c35e2d..e751bc4 100644
--- a/Objects/unicodeobject.c
+++ b/Objects/unicodeobject.c
@@ -329,8 +329,14 @@ PyObject *PyUnicode_FromObject(register PyObject *obj)
s = PyString_AS_STRING(obj);
len = PyString_GET_SIZE(obj);
}
- else if (PyObject_AsCharBuffer(obj, &s, &len))
+ else if (PyObject_AsCharBuffer(obj, &s, &len)) {
+ /* Overwrite the error message with something more useful in
+ case of a TypeError. */
+ if (PyErr_ExceptionMatches(PyExc_TypeError))
+ PyErr_SetString(PyExc_TypeError,
+ "coercing to Unicode: need string or charbuffer");
return NULL;
+ }
if (len == 0) {
Py_INCREF(unicode_empty);
return (PyObject *)unicode_empty;
@@ -1923,6 +1929,60 @@ PyObject *PyUnicode_Translate(PyObject *str,
return NULL;
}
+/* --- Decimal Encoder ---------------------------------------------------- */
+
+int PyUnicode_EncodeDecimal(Py_UNICODE *s,
+ int length,
+ char *output,
+ const char *errors)
+{
+ Py_UNICODE *p, *end;
+
+ if (output == NULL) {
+ PyErr_BadArgument();
+ return -1;
+ }
+
+ p = s;
+ end = s + length;
+ while (p < end) {
+ register Py_UNICODE ch = *p++;
+ int decimal;
+
+ if (Py_UNICODE_ISSPACE(ch)) {
+ *output++ = ' ';
+ continue;
+ }
+ decimal = Py_UNICODE_TODECIMAL(ch);
+ if (decimal >= 0) {
+ *output++ = '0' + decimal;
+ continue;
+ }
+ if (0 < ch < 256) {
+ *output++ = ch;
+ continue;
+ }
+ /* All other characters are considered invalid */
+ if (errors == NULL || strcmp(errors, "strict") == 0) {
+ PyErr_SetString(PyExc_ValueError,
+ "invalid decimal Unicode string");
+ goto onError;
+ }
+ else if (strcmp(errors, "ignore") == 0)
+ continue;
+ else if (strcmp(errors, "replace") == 0) {
+ *output++ = '?';
+ continue;
+ }
+ }
+ /* 0-terminate the output string */
+ *output++ = '\0';
+ return 0;
+
+ onError:
+ return -1;
+}
+
/* --- Helpers ------------------------------------------------------------ */
static
@@ -2811,12 +2871,14 @@ int PyUnicode_Contains(PyObject *container,
register Py_UNICODE ch;
/* Coerce the two arguments */
- u = (PyUnicodeObject *)PyUnicode_FromObject(container);
- if (u == NULL)
- goto onError;
v = (PyUnicodeObject *)PyUnicode_FromObject(element);
if (v == NULL)
goto onError;
+ u = (PyUnicodeObject *)PyUnicode_FromObject(container);
+ if (u == NULL) {
+ Py_DECREF(v);
+ goto onError;
+ }
/* Check v in u */
if (PyUnicode_GET_SIZE(v) != 1) {
diff --git a/Python/bltinmodule.c b/Python/bltinmodule.c
index 90af888..3016d56 100644
--- a/Python/bltinmodule.c
+++ b/Python/bltinmodule.c
@@ -449,17 +449,44 @@ complex_from_string(v)
PyObject *v;
{
extern double strtod Py_PROTO((const char *, char **));
- char *s, *start, *end;
+ const char *s, *start;
+ char *end;
double x=0.0, y=0.0, z;
int got_re=0, got_im=0, done=0;
int digit_or_dot;
int sw_error=0;
int sign;
char buffer[256]; /* For errors */
+ int len;
+
+ if (PyString_Check(v)) {
+ s = PyString_AS_STRING(v);
+ len = PyString_GET_SIZE(v);
+ }
+ else if (PyUnicode_Check(v)) {
+ char s_buffer[256];
- start = s = PyString_AS_STRING(v);
+ if (PyUnicode_GET_SIZE(v) >= sizeof(s_buffer)) {
+ PyErr_SetString(PyExc_ValueError,
+ "complex() literal too large to convert");
+ return NULL;
+ }
+ if (PyUnicode_EncodeDecimal(PyUnicode_AS_UNICODE(v),
+ PyUnicode_GET_SIZE(v),
+ s_buffer,
+ NULL))
+ return NULL;
+ s = s_buffer;
+ len = strlen(s);
+ }
+ else if (PyObject_AsCharBuffer(v, &s, &len)) {
+ PyErr_SetString(PyExc_TypeError,
+ "complex() needs a string first argument");
+ return NULL;
+ }
/* position on first nonblank */
+ start = s;
while (*s && isspace(Py_CHARMASK(*s)))
s++;
if (s[0] == '\0') {
@@ -475,7 +502,7 @@ complex_from_string(v)
switch (*s) {
case '\0':
- if (s-start != PyString_GET_SIZE(v)) {
+ if (s-start != len) {
PyErr_SetString(
PyExc_ValueError,
"null byte in argument for complex()");
@@ -584,7 +611,7 @@ builtin_complex(self, args)
i = NULL;
if (!PyArg_ParseTuple(args, "O|O:complex", &r, &i))
return NULL;
- if (PyString_Check(r))
+ if (PyString_Check(r) || PyUnicode_Check(r))
return complex_from_string(r);
if ((nbr = r->ob_type->tp_as_number) == NULL ||
nbr->nb_float == NULL ||
@@ -1289,12 +1316,17 @@ builtin_int(self, args)
return NULL;
if (base == -909)
return PyNumber_Int(v);
- else if (!PyString_Check(v)) {
+ else if (PyString_Check(v))
+ return PyInt_FromString(PyString_AS_STRING(v), NULL, base);
+ else if (PyUnicode_Check(v))
+ return PyInt_FromUnicode(PyUnicode_AS_UNICODE(v),
+ PyUnicode_GET_SIZE(v),
+ base);
+ else {
PyErr_SetString(PyExc_TypeError,
"can't convert non-string with explicit base");
return NULL;
}
- return PyInt_FromString(PyString_AS_STRING(v), NULL, base);
}
static char int_doc[] =
@@ -1319,12 +1351,17 @@ builtin_long(self, args)
return NULL;
if (base == -909)
return PyNumber_Long(v);
- else if (!PyString_Check(v)) {
+ else if (PyString_Check(v))
+ return PyLong_FromString(PyString_AS_STRING(v), NULL, base);
+ else if (PyUnicode_Check(v))
+ return PyLong_FromUnicode(PyUnicode_AS_UNICODE(v),
+ PyUnicode_GET_SIZE(v),
+ base);
+ else {
PyErr_SetString(PyExc_TypeError,
"can't convert non-string with explicit base");
return NULL;
}
- return PyLong_FromString(PyString_AS_STRING(v), NULL, base);
}
static char long_doc[] =
diff --git a/Python/codecs.c b/Python/codecs.c
index b9a6461..4df389a 100644
--- a/Python/codecs.c
+++ b/Python/codecs.c
@@ -84,8 +84,11 @@ int PyCodec_Register(PyObject *search_function)
return -1;
}
+/* Convert a string to a normalized Python string: all characters are
+ converted to lower case, spaces are replaced with underscores. */
+
static
-PyObject *lowercasestring(const char *string)
+PyObject *normalizestring(const char *string)
{
register int i;
int len = strlen(string);
@@ -96,8 +99,14 @@ PyObject *lowercasestring(const char *string)
if (v == NULL)
return NULL;
p = PyString_AS_STRING(v);
- for (i = 0; i < len; i++)
- p[i] = tolower(string[i]);
+ for (i = 0; i < len; i++) {
+ register char ch = string[i];
+ if (ch == ' ')
+ ch = '-';
+ else
+ ch = tolower(ch);
+ p[i] = ch;
+ }
return v;
}
@@ -132,8 +141,10 @@ PyObject *_PyCodec_Lookup(const char *encoding)
goto onError;
}
- /* Convert the encoding to a lower-cased Python string */
- v = lowercasestring(encoding);
+ /* Convert the encoding to a normalized Python string: all
+ characters are converted to lower case, spaces and hypens are
+ replaced with underscores. */
+ v = normalizestring(encoding);
if (v == NULL)
goto onError;
PyString_InternInPlace(&v);
generated by cgit v0.12 at 2024-11-22 21:30:59 (GMT)