Merged revisions 62734,62736,62748,62769 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r62734 | brett.cannon | 2008-05-05 22:21:38 +0200 (Mon, 05 May 2008) | 5 lines

  Add the 'json' package. Code taken from simplejson 1.9 and contributed by Bob
  Ippolito.

  Closes issue #2750.
........
  r62736 | georg.brandl | 2008-05-05 22:53:39 +0200 (Mon, 05 May 2008) | 2 lines

  Fix JSON module docs.
........
  r62748 | benjamin.peterson | 2008-05-06 04:51:10 +0200 (Tue, 06 May 2008) | 2 lines

  PEP 8 nits in json package
........
  r62769 | christian.heimes | 2008-05-06 18:18:41 +0200 (Tue, 06 May 2008) | 2 lines

  Intern static string
  Use float constructors instead of magic code for float constants
........

1 files changed, 620 insertions, 0 deletions

diff --git a/Modules/_json.c b/Modules/_json.c
new file mode 100644
index 0000000..39d39cb
--- /dev/null
+++ b/Modules/_json.c
@@ -0,0 +1,620 @@
+#include "Python.h"
+
+#define DEFAULT_ENCODING "utf-8"
+#define S_CHAR(c) (c >= ' ' && c <= '~' && c != '\\' && c != '"')
+#define MIN_EXPANSION 6
+
+#ifdef Py_UNICODE_WIDE
+#define MAX_EXPANSION (2 * MIN_EXPANSION)
+#else
+#define MAX_EXPANSION MIN_EXPANSION
+#endif
+
+static Py_ssize_t
+ascii_escape_char(Py_UNICODE c, char *output, Py_ssize_t chars)
+{
+    Py_UNICODE x;
+    output[chars++] = '\\';
+    switch (c) {
+        case '\\': output[chars++] = (char)c; break;
+        case '"': output[chars++] = (char)c; break;
+        case '\b': output[chars++] = 'b'; break;
+        case '\f': output[chars++] = 'f'; break;
+        case '\n': output[chars++] = 'n'; break;
+        case '\r': output[chars++] = 'r'; break;
+        case '\t': output[chars++] = 't'; break;
+        default:
+#ifdef Py_UNICODE_WIDE
+            if (c >= 0x10000) {
+                /* UTF-16 surrogate pair */
+                Py_UNICODE v = c - 0x10000;
+                c = 0xd800 | ((v >> 10) & 0x3ff);
+                output[chars++] = 'u';
+                x = (c & 0xf000) >> 12;
+                output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10);
+                x = (c & 0x0f00) >> 8;
+                output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10);
+                x = (c & 0x00f0) >> 4;
+                output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10);
+                x = (c & 0x000f);
+                output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10);
+                c = 0xdc00 | (v & 0x3ff);
+                output[chars++] = '\\';
+            }
+#endif
+            output[chars++] = 'u';
+            x = (c & 0xf000) >> 12;
+            output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10);
+            x = (c & 0x0f00) >> 8;
+            output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10);
+            x = (c & 0x00f0) >> 4;
+            output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10);
+            x = (c & 0x000f);
+            output[chars++] = (x < 10) ? '0' + x : 'a' + (x - 10);
+    }
+    return chars;
+}
+
+static PyObject *
+ascii_escape_unicode(PyObject *pystr)
+{
+    Py_ssize_t i;
+    Py_ssize_t input_chars;
+    Py_ssize_t output_size;
+    Py_ssize_t chars;
+    PyObject *rval;
+    char *output;
+    Py_UNICODE *input_unicode;
+
+    input_chars = PyUnicode_GET_SIZE(pystr);
+    input_unicode = PyUnicode_AS_UNICODE(pystr);
+    /* One char input can be up to 6 chars output, estimate 4 of these */
+    output_size = 2 + (MIN_EXPANSION * 4) + input_chars;
+    rval = PyString_FromStringAndSize(NULL, output_size);
+    if (rval == NULL) {
+        return NULL;
+    }
+    output = PyString_AS_STRING(rval);
+    chars = 0;
+    output[chars++] = '"';
+    for (i = 0; i < input_chars; i++) {
+        Py_UNICODE c = input_unicode[i];
+        if (S_CHAR(c)) {
+            output[chars++] = (char)c;
+        }
+	else {
+            chars = ascii_escape_char(c, output, chars);
+        }
+        if (output_size - chars < (1 + MAX_EXPANSION)) {
+            /* There's more than four, so let's resize by a lot */
+            output_size *= 2;
+            /* This is an upper bound */
+            if (output_size > 2 + (input_chars * MAX_EXPANSION)) {
+                output_size = 2 + (input_chars * MAX_EXPANSION);
+            }
+            if (_PyString_Resize(&rval, output_size) == -1) {
+                return NULL;
+            }
+            output = PyString_AS_STRING(rval);
+        }
+    }
+    output[chars++] = '"';
+    if (_PyString_Resize(&rval, chars) == -1) {
+        return NULL;
+    }
+    return rval;
+}
+
+static PyObject *
+ascii_escape_str(PyObject *pystr)
+{
+    Py_ssize_t i;
+    Py_ssize_t input_chars;
+    Py_ssize_t output_size;
+    Py_ssize_t chars;
+    PyObject *rval;
+    char *output;
+    char *input_str;
+
+    input_chars = PyString_GET_SIZE(pystr);
+    input_str = PyString_AS_STRING(pystr);
+    /* One char input can be up to 6 chars output, estimate 4 of these */
+    output_size = 2 + (MIN_EXPANSION * 4) + input_chars;
+    rval = PyString_FromStringAndSize(NULL, output_size);
+    if (rval == NULL) {
+        return NULL;
+    }
+    output = PyString_AS_STRING(rval);
+    chars = 0;
+    output[chars++] = '"';
+    for (i = 0; i < input_chars; i++) {
+        Py_UNICODE c = (Py_UNICODE)input_str[i];
+        if (S_CHAR(c)) {
+            output[chars++] = (char)c;
+        }
+	else if (c > 0x7F) {
+            /* We hit a non-ASCII character, bail to unicode mode */
+            PyObject *uni;
+            Py_DECREF(rval);
+            uni = PyUnicode_DecodeUTF8(input_str, input_chars, "strict");
+            if (uni == NULL) {
+                return NULL;
+            }
+            rval = ascii_escape_unicode(uni);
+            Py_DECREF(uni);
+            return rval;
+        }
+	else {
+            chars = ascii_escape_char(c, output, chars);
+        }
+        /* An ASCII char can't possibly expand to a surrogate! */
+        if (output_size - chars < (1 + MIN_EXPANSION)) {
+            /* There's more than four, so let's resize by a lot */
+            output_size *= 2;
+            if (output_size > 2 + (input_chars * MIN_EXPANSION)) {
+                output_size = 2 + (input_chars * MIN_EXPANSION);
+            }
+            if (_PyString_Resize(&rval, output_size) == -1) {
+                return NULL;
+            }
+            output = PyString_AS_STRING(rval);
+        }
+    }
+    output[chars++] = '"';
+    if (_PyString_Resize(&rval, chars) == -1) {
+        return NULL;
+    }
+    return rval;
+}
+
+void
+raise_errmsg(char *msg, PyObject *s, Py_ssize_t end)
+{
+    static PyObject *errmsg_fn = NULL;
+    PyObject *pymsg;
+    if (errmsg_fn == NULL) {
+        PyObject *decoder = PyImport_ImportModule("json.decoder");
+        if (decoder == NULL)
+            return;
+        errmsg_fn = PyObject_GetAttrString(decoder, "errmsg");
+        if (errmsg_fn == NULL)
+            return;
+        Py_XDECREF(decoder);
+    }
+    pymsg = PyObject_CallFunction(errmsg_fn, "(zOn)", msg, s, end);
+    PyErr_SetObject(PyExc_ValueError, pymsg);
+    Py_DECREF(pymsg);
+/*
+
+def linecol(doc, pos):
+    lineno = doc.count('\n', 0, pos) + 1
+    if lineno == 1:
+        colno = pos
+    else:
+        colno = pos - doc.rindex('\n', 0, pos)
+    return lineno, colno
+
+def errmsg(msg, doc, pos, end=None):
+    lineno, colno = linecol(doc, pos)
+    if end is None:
+        return '%s: line %d column %d (char %d)' % (msg, lineno, colno, pos)
+    endlineno, endcolno = linecol(doc, end)
+    return '%s: line %d column %d - line %d column %d (char %d - %d)' % (
+        msg, lineno, colno, endlineno, endcolno, pos, end)
+
+*/
+}
+
+static PyObject *
+join_list_unicode(PyObject *lst)
+{
+    static PyObject *ustr = NULL;
+    static PyObject *joinstr = NULL;
+    if (ustr == NULL) {
+        Py_UNICODE c = 0;
+        ustr = PyUnicode_FromUnicode(&c, 0);
+    }
+    if (joinstr == NULL) {
+        joinstr = PyUnicode_InternFromString("join");
+    }
+    if (joinstr == NULL || ustr == NULL) {
+        return NULL;
+    }
+    return PyObject_CallMethodObjArgs(ustr, joinstr, lst, NULL);
+}
+
+static PyObject *
+scanstring_str(PyObject *pystr, Py_ssize_t end, char *encoding, int strict)
+{
+    PyObject *rval;
+    Py_ssize_t len = PyString_GET_SIZE(pystr);
+    Py_ssize_t begin = end - 1;
+    Py_ssize_t next = begin;
+    char *buf = PyString_AS_STRING(pystr);
+    Py_buffer info;
+    PyObject *chunks = PyList_New(0);
+    if (chunks == NULL) {
+        goto bail;
+    }
+    while (1) {
+        /* Find the end of the string or the next escape */
+        Py_UNICODE c = 0;
+        PyObject *chunk = NULL;
+        for (next = end; next < len; next++) {
+            c = buf[next];
+            if (c == '"' || c == '\\') {
+                break;
+            }
+            else if (strict && c <= 0x1f) {
+                raise_errmsg("Invalid control character at", pystr, begin);
+                goto bail;
+            }
+        }
+        if (!(c == '"' || c == '\\')) {
+            raise_errmsg("Unterminated string starting at", pystr, begin);
+            goto bail;
+        }
+        /* Pick up this chunk if it's not zero length */
+        if (next != end) {
+            if (PyBuffer_FillInfo(&info, &buf[end], next - end, 1, 0) < 0) {
+                goto bail;
+            }
+            PyObject *strchunk = PyMemoryView_FromMemory(&info);
+            if (strchunk == NULL) {
+                goto bail;
+            }
+            chunk = PyUnicode_FromEncodedObject(strchunk, encoding, NULL);
+            Py_DECREF(strchunk);
+            if (chunk == NULL) {
+                goto bail;
+            }
+            if (PyList_Append(chunks, chunk)) {
+                goto bail;
+            }
+            Py_DECREF(chunk);
+        }
+        next++;
+        if (c == '"') {
+            end = next;
+            break;
+        }
+        if (next == len) {
+            raise_errmsg("Unterminated string starting at", pystr, begin);
+            goto bail;
+        }
+        c = buf[next];
+        if (c != 'u') {
+            /* Non-unicode backslash escapes */
+            end = next + 1;
+            switch (c) {
+                case '"': break;
+                case '\\': break;
+                case '/': break;
+                case 'b': c = '\b'; break;
+                case 'f': c = '\f'; break;
+                case 'n': c = '\n'; break;
+                case 'r': c = '\r'; break;
+                case 't': c = '\t'; break;
+                default: c = 0;
+            }
+            if (c == 0) {
+                raise_errmsg("Invalid \\escape", pystr, end - 2);
+                goto bail;
+            }
+        }
+        else {
+            c = 0;
+            next++;
+            end = next + 4;
+            if (end >= len) {
+                raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1);
+                goto bail;
+            }
+            /* Decode 4 hex digits */
+            for (; next < end; next++) {
+                Py_ssize_t shl = (end - next - 1) << 2;
+                Py_UNICODE digit = buf[next];
+                switch (digit) {
+                    case '0': case '1': case '2': case '3': case '4':
+                    case '5': case '6': case '7': case '8': case '9':
+                        c |= (digit - '0') << shl; break;
+                    case 'a': case 'b': case 'c': case 'd': case 'e':
+                    case 'f':
+                        c |= (digit - 'a' + 10) << shl; break;
+                    case 'A': case 'B': case 'C': case 'D': case 'E':
+                    case 'F':
+                        c |= (digit - 'A' + 10) << shl; break;
+                    default:
+                        raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
+                        goto bail;
+                }
+            }
+#ifdef Py_UNICODE_WIDE
+            /* Surrogate pair */
+            if (c >= 0xd800 && c <= 0xdbff) {
+                Py_UNICODE c2 = 0;
+                if (end + 6 >= len) {
+                    raise_errmsg("Invalid \\uXXXX\\uXXXX surrogate pair", pystr,
+                        end - 5);
+                }
+                if (buf[next++] != '\\' || buf[next++] != 'u') {
+                    raise_errmsg("Invalid \\uXXXX\\uXXXX surrogate pair", pystr,
+                        end - 5);
+                }
+                end += 6;
+                /* Decode 4 hex digits */
+                for (; next < end; next++) {
+                    Py_ssize_t shl = (end - next - 1) << 2;
+                    Py_UNICODE digit = buf[next];
+                    switch (digit) {
+                        case '0': case '1': case '2': case '3': case '4':
+                        case '5': case '6': case '7': case '8': case '9':
+                            c2 |= (digit - '0') << shl; break;
+                        case 'a': case 'b': case 'c': case 'd': case 'e':
+                        case 'f':
+                            c2 |= (digit - 'a' + 10) << shl; break;
+                        case 'A': case 'B': case 'C': case 'D': case 'E':
+                        case 'F':
+                            c2 |= (digit - 'A' + 10) << shl; break;
+                        default:
+                            raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
+                            goto bail;
+                    }
+                }
+                c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
+            }
+#endif
+        }
+        chunk = PyUnicode_FromUnicode(&c, 1);
+        if (chunk == NULL) {
+            goto bail;
+        }
+        if (PyList_Append(chunks, chunk)) {
+            goto bail;
+        }
+        Py_DECREF(chunk);
+    }
+
+    rval = join_list_unicode(chunks);
+    if (rval == NULL) {
+        goto bail;
+    }
+    Py_DECREF(chunks);
+    chunks = NULL;
+    return Py_BuildValue("(Nn)", rval, end);
+bail:
+    Py_XDECREF(chunks);
+    return NULL;
+}
+
+
+static PyObject *
+scanstring_unicode(PyObject *pystr, Py_ssize_t end, int strict)
+{
+    PyObject *rval;
+    Py_ssize_t len = PyUnicode_GET_SIZE(pystr);
+    Py_ssize_t begin = end - 1;
+    Py_ssize_t next = begin;
+    const Py_UNICODE *buf = PyUnicode_AS_UNICODE(pystr);
+    PyObject *chunks = PyList_New(0);
+    if (chunks == NULL) {
+        goto bail;
+    }
+    while (1) {
+        /* Find the end of the string or the next escape */
+        Py_UNICODE c = 0;
+        PyObject *chunk = NULL;
+        for (next = end; next < len; next++) {
+            c = buf[next];
+            if (c == '"' || c == '\\') {
+                break;
+            }
+            else if (strict && c <= 0x1f) {
+                raise_errmsg("Invalid control character at", pystr, begin);
+                goto bail;
+            }
+        }
+        if (!(c == '"' || c == '\\')) {
+            raise_errmsg("Unterminated string starting at", pystr, begin);
+            goto bail;
+        }
+        /* Pick up this chunk if it's not zero length */
+        if (next != end) {
+            chunk = PyUnicode_FromUnicode(&buf[end], next - end);
+            if (chunk == NULL) {
+                goto bail;
+            }
+            if (PyList_Append(chunks, chunk)) {
+                goto bail;
+            }
+            Py_DECREF(chunk);
+        }
+        next++;
+        if (c == '"') {
+            end = next;
+            break;
+        }
+        if (next == len) {
+            raise_errmsg("Unterminated string starting at", pystr, begin);
+            goto bail;
+        }
+        c = buf[next];
+        if (c != 'u') {
+            /* Non-unicode backslash escapes */
+            end = next + 1;
+            switch (c) {
+                case '"': break;
+                case '\\': break;
+                case '/': break;
+                case 'b': c = '\b'; break;
+                case 'f': c = '\f'; break;
+                case 'n': c = '\n'; break;
+                case 'r': c = '\r'; break;
+                case 't': c = '\t'; break;
+                default: c = 0;
+            }
+            if (c == 0) {
+                raise_errmsg("Invalid \\escape", pystr, end - 2);
+                goto bail;
+            }
+        }
+        else {
+            c = 0;
+            next++;
+            end = next + 4;
+            if (end >= len) {
+                raise_errmsg("Invalid \\uXXXX escape", pystr, next - 1);
+                goto bail;
+            }
+            /* Decode 4 hex digits */
+            for (; next < end; next++) {
+                Py_ssize_t shl = (end - next - 1) << 2;
+                Py_UNICODE digit = buf[next];
+                switch (digit) {
+                    case '0': case '1': case '2': case '3': case '4':
+                    case '5': case '6': case '7': case '8': case '9':
+                        c |= (digit - '0') << shl; break;
+                    case 'a': case 'b': case 'c': case 'd': case 'e':
+                    case 'f':
+                        c |= (digit - 'a' + 10) << shl; break;
+                    case 'A': case 'B': case 'C': case 'D': case 'E':
+                    case 'F':
+                        c |= (digit - 'A' + 10) << shl; break;
+                    default:
+                        raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
+                        goto bail;
+                }
+            }
+#ifdef Py_UNICODE_WIDE
+            /* Surrogate pair */
+            if (c >= 0xd800 && c <= 0xdbff) {
+                Py_UNICODE c2 = 0;
+                if (end + 6 >= len) {
+                    raise_errmsg("Invalid \\uXXXX\\uXXXX surrogate pair", pystr,
+                        end - 5);
+                }
+                if (buf[next++] != '\\' || buf[next++] != 'u') {
+                    raise_errmsg("Invalid \\uXXXX\\uXXXX surrogate pair", pystr,
+                        end - 5);
+                }
+                end += 6;
+                /* Decode 4 hex digits */
+                for (; next < end; next++) {
+                    Py_ssize_t shl = (end - next - 1) << 2;
+                    Py_UNICODE digit = buf[next];
+                    switch (digit) {
+                        case '0': case '1': case '2': case '3': case '4':
+                        case '5': case '6': case '7': case '8': case '9':
+                            c2 |= (digit - '0') << shl; break;
+                        case 'a': case 'b': case 'c': case 'd': case 'e':
+                        case 'f':
+                            c2 |= (digit - 'a' + 10) << shl; break;
+                        case 'A': case 'B': case 'C': case 'D': case 'E':
+                        case 'F':
+                            c2 |= (digit - 'A' + 10) << shl; break;
+                        default:
+                            raise_errmsg("Invalid \\uXXXX escape", pystr, end - 5);
+                            goto bail;
+                    }
+                }
+                c = 0x10000 + (((c - 0xd800) << 10) | (c2 - 0xdc00));
+            }
+#endif
+        }
+        chunk = PyUnicode_FromUnicode(&c, 1);
+        if (chunk == NULL) {
+            goto bail;
+        }
+        if (PyList_Append(chunks, chunk)) {
+            goto bail;
+        }
+        Py_DECREF(chunk);
+    }
+
+    rval = join_list_unicode(chunks);
+    if (rval == NULL) {
+        goto bail;
+    }
+    Py_DECREF(chunks);
+    chunks = NULL;
+    return Py_BuildValue("(Nn)", rval, end);
+bail:
+    Py_XDECREF(chunks);
+    return NULL;
+}
+
+PyDoc_STRVAR(pydoc_scanstring,
+"scanstring(str_or_bytes, end, encoding) -> (bytes, end)\n");
+
+static PyObject *
+py_scanstring(PyObject* self, PyObject *args)
+{
+    PyObject *pystr;
+    Py_ssize_t end;
+    char *encoding = NULL;
+    int strict = 0;
+    if (!PyArg_ParseTuple(args, "On|zi:scanstring", &pystr, &end, &encoding, &strict)) {
+        return NULL;
+    }
+    if (encoding == NULL) {
+        encoding = DEFAULT_ENCODING;
+    }
+    if (PyString_Check(pystr)) {
+        return scanstring_str(pystr, end, encoding, strict);
+    }
+    else if (PyUnicode_Check(pystr)) {
+        return scanstring_unicode(pystr, end, strict);
+    }
+    else {
+        PyErr_Format(PyExc_TypeError, 
+                     "first argument must be a string or bytes, not %.80s",
+                     Py_TYPE(pystr)->tp_name);
+        return NULL;
+    }
+}
+
+PyDoc_STRVAR(pydoc_encode_basestring_ascii,
+"encode_basestring_ascii(str_or_bytes) -> bytes\n");
+
+static PyObject *
+py_encode_basestring_ascii(PyObject* self, PyObject *pystr)
+{
+    PyObject *rval;
+    /* METH_O */
+    if (PyString_Check(pystr)) {
+        rval = ascii_escape_str(pystr);
+    }
+    else if (PyUnicode_Check(pystr)) {
+        rval = ascii_escape_unicode(pystr);
+    }
+    else {
+        PyErr_Format(PyExc_TypeError, 
+                     "first argument must be a string or unicode, not %.80s",
+                     Py_TYPE(pystr)->tp_name);
+        return NULL;
+    }
+    if (PyString_Check(rval)) {
+        PyObject *urval = PyUnicode_DecodeASCII(PyString_AS_STRING(rval), PyString_GET_SIZE(rval), NULL);
+        Py_DECREF(rval);
+        return urval;
+    }
+    return rval;
+}
+
+static PyMethodDef json_methods[] = {
+    {"encode_basestring_ascii", (PyCFunction)py_encode_basestring_ascii,
+     METH_O, pydoc_encode_basestring_ascii},
+    {"scanstring", (PyCFunction)py_scanstring, METH_VARARGS,
+     pydoc_scanstring},
+    {NULL, NULL, 0, NULL}
+};
+
+PyDoc_STRVAR(module_doc,
+"json speedups\n");
+
+void
+init_json(void)
+{
+    PyObject *m;
+    m = Py_InitModule3("_json", json_methods, module_doc);
+}