diff options
Diffstat (limited to 'Parser/tokenizer.c')
| -rw-r--r-- | Parser/tokenizer.c | 23 |
1 files changed, 11 insertions, 12 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 5b3fd9e..099f6df 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -646,7 +646,7 @@ decode_str(const char *str, struct tok_state *tok) "unknown encoding: %s", tok->enc); return error_ret(tok); } - str = PyBytes_AsString(utf8); + str = PyString_AS_STRING(utf8); } assert(tok->decoding_buffer == NULL); tok->decoding_buffer = utf8; /* CAUTION */ @@ -765,8 +765,8 @@ tok_nextc(register struct tok_state *tok) tok->done = E_DECODE; return EOF; } - buflen = PyBytes_Size(u); - buf = PyBytes_AsString(u); + buflen = PyString_GET_SIZE(u); + buf = PyString_AS_STRING(u); if (!buf) { Py_DECREF(u); tok->done = E_DECODE; @@ -1550,7 +1550,7 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int* offset) #else static PyObject * dec_utf8(const char *enc, const char *text, size_t len) { - PyObject *ret = NULL; + PyObject *ret = NULL; PyObject *unicode_text = PyUnicode_DecodeUTF8(text, len, "replace"); if (unicode_text) { ret = PyUnicode_AsEncodedString(unicode_text, enc, "replace"); @@ -1560,7 +1560,7 @@ dec_utf8(const char *enc, const char *text, size_t len) { PyErr_Clear(); } else { - assert(PyBytes_Check(ret)); + assert(PyString_Check(ret)); } return ret; } @@ -1573,8 +1573,8 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset) /* convert source to original encondig */ PyObject *lineobj = dec_utf8(tok->encoding, tok->buf, len); if (lineobj != NULL) { - int linelen = PyBytes_GET_SIZE(lineobj); - const char *line = PyBytes_AS_STRING(lineobj); + int linelen = PyString_GET_SIZE(lineobj); + const char *line = PyString_AS_STRING(lineobj); text = PyObject_MALLOC(linelen + 1); if (text != NULL && line != NULL) { if (linelen) @@ -1582,19 +1582,18 @@ PyTokenizer_RestoreEncoding(struct tok_state* tok, int len, int *offset) text[linelen] = '\0'; } Py_DECREF(lineobj); - + /* adjust error offset */ if (*offset > 1) { - PyObject *offsetobj = dec_utf8(tok->encoding, + PyObject *offsetobj = dec_utf8(tok->encoding, tok->buf, *offset-1); if (offsetobj) { - *offset = 1 + - PyBytes_GET_SIZE(offsetobj); + *offset = 1 + Py_Size(offsetobj); Py_DECREF(offsetobj); } } - + } } return text; |