diff options
Diffstat (limited to 'Modules/cjkcodecs/cjkcodecs.h')
-rw-r--r-- | Modules/cjkcodecs/cjkcodecs.h | 578 |
1 files changed, 289 insertions, 289 deletions
diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h index e630671..ab0682a 100644 --- a/Modules/cjkcodecs/cjkcodecs.h +++ b/Modules/cjkcodecs/cjkcodecs.h @@ -13,12 +13,12 @@ /* a unicode "undefined" codepoint */ -#define UNIINV 0xFFFE +#define UNIINV 0xFFFE /* internal-use DBCS codepoints which aren't used by any charsets */ -#define NOCHAR 0xFFFF -#define MULTIC 0xFFFE -#define DBCINV 0xFFFD +#define NOCHAR 0xFFFF +#define MULTIC 0xFFFE +#define DBCINV 0xFFFD /* shorter macros to save source size of mapping tables */ #define U UNIINV @@ -27,94 +27,94 @@ #define D DBCINV struct dbcs_index { - const ucs2_t *map; - unsigned char bottom, top; + const ucs2_t *map; + unsigned char bottom, top; }; typedef struct dbcs_index decode_map; struct widedbcs_index { - const ucs4_t *map; - unsigned char bottom, top; + const ucs4_t *map; + unsigned char bottom, top; }; typedef struct widedbcs_index widedecode_map; struct unim_index { - const DBCHAR *map; - unsigned char bottom, top; + const DBCHAR *map; + unsigned char bottom, top; }; typedef struct unim_index encode_map; struct unim_index_bytebased { - const unsigned char *map; - unsigned char bottom, top; + const unsigned char *map; + unsigned char bottom, top; }; struct dbcs_map { - const char *charset; - const struct unim_index *encmap; - const struct dbcs_index *decmap; + const char *charset; + const struct unim_index *encmap; + const struct dbcs_index *decmap; }; struct pair_encodemap { - ucs4_t uniseq; - DBCHAR code; + ucs4_t uniseq; + DBCHAR code; }; static const MultibyteCodec *codec_list; static const struct dbcs_map *mapping_list; -#define CODEC_INIT(encoding) \ - static int encoding##_codec_init(const void *config) - -#define ENCODER_INIT(encoding) \ - static int encoding##_encode_init( \ - MultibyteCodec_State *state, const void *config) -#define ENCODER(encoding) \ - static Py_ssize_t encoding##_encode( \ - MultibyteCodec_State *state, const void *config, \ - const Py_UNICODE **inbuf, Py_ssize_t inleft, \ - unsigned char **outbuf, Py_ssize_t outleft, int flags) -#define ENCODER_RESET(encoding) \ - static Py_ssize_t encoding##_encode_reset( \ - MultibyteCodec_State *state, const void *config, \ - unsigned char **outbuf, Py_ssize_t outleft) - -#define DECODER_INIT(encoding) \ - static int encoding##_decode_init( \ - MultibyteCodec_State *state, const void *config) -#define DECODER(encoding) \ - static Py_ssize_t encoding##_decode( \ - MultibyteCodec_State *state, const void *config, \ - const unsigned char **inbuf, Py_ssize_t inleft, \ - Py_UNICODE **outbuf, Py_ssize_t outleft) -#define DECODER_RESET(encoding) \ - static Py_ssize_t encoding##_decode_reset( \ - MultibyteCodec_State *state, const void *config) +#define CODEC_INIT(encoding) \ + static int encoding##_codec_init(const void *config) + +#define ENCODER_INIT(encoding) \ + static int encoding##_encode_init( \ + MultibyteCodec_State *state, const void *config) +#define ENCODER(encoding) \ + static Py_ssize_t encoding##_encode( \ + MultibyteCodec_State *state, const void *config, \ + const Py_UNICODE **inbuf, Py_ssize_t inleft, \ + unsigned char **outbuf, Py_ssize_t outleft, int flags) +#define ENCODER_RESET(encoding) \ + static Py_ssize_t encoding##_encode_reset( \ + MultibyteCodec_State *state, const void *config, \ + unsigned char **outbuf, Py_ssize_t outleft) + +#define DECODER_INIT(encoding) \ + static int encoding##_decode_init( \ + MultibyteCodec_State *state, const void *config) +#define DECODER(encoding) \ + static Py_ssize_t encoding##_decode( \ + MultibyteCodec_State *state, const void *config, \ + const unsigned char **inbuf, Py_ssize_t inleft, \ + Py_UNICODE **outbuf, Py_ssize_t outleft) +#define DECODER_RESET(encoding) \ + static Py_ssize_t encoding##_decode_reset( \ + MultibyteCodec_State *state, const void *config) #if Py_UNICODE_SIZE == 4 -#define UCS4INVALID(code) \ - if ((code) > 0xFFFF) \ - return 1; +#define UCS4INVALID(code) \ + if ((code) > 0xFFFF) \ + return 1; #else -#define UCS4INVALID(code) \ - if (0) ; +#define UCS4INVALID(code) \ + if (0) ; #endif -#define NEXT_IN(i) \ - (*inbuf) += (i); \ - (inleft) -= (i); -#define NEXT_OUT(o) \ - (*outbuf) += (o); \ - (outleft) -= (o); -#define NEXT(i, o) \ - NEXT_IN(i) NEXT_OUT(o) - -#define REQUIRE_INBUF(n) \ - if (inleft < (n)) \ - return MBERR_TOOFEW; -#define REQUIRE_OUTBUF(n) \ - if (outleft < (n)) \ - return MBERR_TOOSMALL; +#define NEXT_IN(i) \ + (*inbuf) += (i); \ + (inleft) -= (i); +#define NEXT_OUT(o) \ + (*outbuf) += (o); \ + (outleft) -= (o); +#define NEXT(i, o) \ + NEXT_IN(i) NEXT_OUT(o) + +#define REQUIRE_INBUF(n) \ + if (inleft < (n)) \ + return MBERR_TOOFEW; +#define REQUIRE_OUTBUF(n) \ + if (outleft < (n)) \ + return MBERR_TOOSMALL; #define IN1 ((*inbuf)[0]) #define IN2 ((*inbuf)[1]) @@ -126,289 +126,289 @@ static const struct dbcs_map *mapping_list; #define OUT3(c) ((*outbuf)[2]) = (c); #define OUT4(c) ((*outbuf)[3]) = (c); -#define WRITE1(c1) \ - REQUIRE_OUTBUF(1) \ - (*outbuf)[0] = (c1); -#define WRITE2(c1, c2) \ - REQUIRE_OUTBUF(2) \ - (*outbuf)[0] = (c1); \ - (*outbuf)[1] = (c2); -#define WRITE3(c1, c2, c3) \ - REQUIRE_OUTBUF(3) \ - (*outbuf)[0] = (c1); \ - (*outbuf)[1] = (c2); \ - (*outbuf)[2] = (c3); -#define WRITE4(c1, c2, c3, c4) \ - REQUIRE_OUTBUF(4) \ - (*outbuf)[0] = (c1); \ - (*outbuf)[1] = (c2); \ - (*outbuf)[2] = (c3); \ - (*outbuf)[3] = (c4); +#define WRITE1(c1) \ + REQUIRE_OUTBUF(1) \ + (*outbuf)[0] = (c1); +#define WRITE2(c1, c2) \ + REQUIRE_OUTBUF(2) \ + (*outbuf)[0] = (c1); \ + (*outbuf)[1] = (c2); +#define WRITE3(c1, c2, c3) \ + REQUIRE_OUTBUF(3) \ + (*outbuf)[0] = (c1); \ + (*outbuf)[1] = (c2); \ + (*outbuf)[2] = (c3); +#define WRITE4(c1, c2, c3, c4) \ + REQUIRE_OUTBUF(4) \ + (*outbuf)[0] = (c1); \ + (*outbuf)[1] = (c2); \ + (*outbuf)[2] = (c3); \ + (*outbuf)[3] = (c4); #if Py_UNICODE_SIZE == 2 -# define WRITEUCS4(c) \ - REQUIRE_OUTBUF(2) \ - (*outbuf)[0] = 0xd800 + (((c) - 0x10000) >> 10); \ - (*outbuf)[1] = 0xdc00 + (((c) - 0x10000) & 0x3ff); \ - NEXT_OUT(2) +# define WRITEUCS4(c) \ + REQUIRE_OUTBUF(2) \ + (*outbuf)[0] = 0xd800 + (((c) - 0x10000) >> 10); \ + (*outbuf)[1] = 0xdc00 + (((c) - 0x10000) & 0x3ff); \ + NEXT_OUT(2) #else -# define WRITEUCS4(c) \ - REQUIRE_OUTBUF(1) \ - **outbuf = (Py_UNICODE)(c); \ - NEXT_OUT(1) +# define WRITEUCS4(c) \ + REQUIRE_OUTBUF(1) \ + **outbuf = (Py_UNICODE)(c); \ + NEXT_OUT(1) #endif -#define _TRYMAP_ENC(m, assi, val) \ - ((m)->map != NULL && (val) >= (m)->bottom && \ - (val)<= (m)->top && ((assi) = (m)->map[(val) - \ - (m)->bottom]) != NOCHAR) -#define TRYMAP_ENC_COND(charset, assi, uni) \ - _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff) -#define TRYMAP_ENC(charset, assi, uni) \ - if TRYMAP_ENC_COND(charset, assi, uni) - -#define _TRYMAP_DEC(m, assi, val) \ - ((m)->map != NULL && (val) >= (m)->bottom && \ - (val)<= (m)->top && ((assi) = (m)->map[(val) - \ - (m)->bottom]) != UNIINV) -#define TRYMAP_DEC(charset, assi, c1, c2) \ - if _TRYMAP_DEC(&charset##_decmap[c1], assi, c2) - -#define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val) \ - ((m)->map != NULL && (val) >= (m)->bottom && \ - (val)<= (m)->top && \ - ((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \ - (((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \ - (((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1)) -#define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni) \ - if _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \ - assplane, asshi, asslo, (uni) & 0xff) -#define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2) \ - if _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2) +#define _TRYMAP_ENC(m, assi, val) \ + ((m)->map != NULL && (val) >= (m)->bottom && \ + (val)<= (m)->top && ((assi) = (m)->map[(val) - \ + (m)->bottom]) != NOCHAR) +#define TRYMAP_ENC_COND(charset, assi, uni) \ + _TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff) +#define TRYMAP_ENC(charset, assi, uni) \ + if TRYMAP_ENC_COND(charset, assi, uni) + +#define _TRYMAP_DEC(m, assi, val) \ + ((m)->map != NULL && (val) >= (m)->bottom && \ + (val)<= (m)->top && ((assi) = (m)->map[(val) - \ + (m)->bottom]) != UNIINV) +#define TRYMAP_DEC(charset, assi, c1, c2) \ + if _TRYMAP_DEC(&charset##_decmap[c1], assi, c2) + +#define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val) \ + ((m)->map != NULL && (val) >= (m)->bottom && \ + (val)<= (m)->top && \ + ((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \ + (((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \ + (((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1)) +#define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni) \ + if _TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \ + assplane, asshi, asslo, (uni) & 0xff) +#define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2) \ + if _TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2) #if Py_UNICODE_SIZE == 2 -#define DECODE_SURROGATE(c) \ - if (c >> 10 == 0xd800 >> 10) { /* high surrogate */ \ - REQUIRE_INBUF(2) \ - if (IN2 >> 10 == 0xdc00 >> 10) { /* low surrogate */ \ - c = 0x10000 + ((ucs4_t)(c - 0xd800) << 10) + \ - ((ucs4_t)(IN2) - 0xdc00); \ - } \ - } -#define GET_INSIZE(c) ((c) > 0xffff ? 2 : 1) +#define DECODE_SURROGATE(c) \ + if (c >> 10 == 0xd800 >> 10) { /* high surrogate */ \ + REQUIRE_INBUF(2) \ + if (IN2 >> 10 == 0xdc00 >> 10) { /* low surrogate */ \ + c = 0x10000 + ((ucs4_t)(c - 0xd800) << 10) + \ + ((ucs4_t)(IN2) - 0xdc00); \ + } \ + } +#define GET_INSIZE(c) ((c) > 0xffff ? 2 : 1) #else #define DECODE_SURROGATE(c) {;} -#define GET_INSIZE(c) 1 +#define GET_INSIZE(c) 1 #endif #define BEGIN_MAPPINGS_LIST static const struct dbcs_map _mapping_list[] = { #define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL}, #define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap}, #define MAPPING_ENCDEC(enc) {#enc, (void*)enc##_encmap, (void*)enc##_decmap}, -#define END_MAPPINGS_LIST \ - {"", NULL, NULL} }; \ - static const struct dbcs_map *mapping_list = \ - (const struct dbcs_map *)_mapping_list; +#define END_MAPPINGS_LIST \ + {"", NULL, NULL} }; \ + static const struct dbcs_map *mapping_list = \ + (const struct dbcs_map *)_mapping_list; #define BEGIN_CODECS_LIST static const MultibyteCodec _codec_list[] = { -#define _STATEFUL_METHODS(enc) \ - enc##_encode, \ - enc##_encode_init, \ - enc##_encode_reset, \ - enc##_decode, \ - enc##_decode_init, \ - enc##_decode_reset, -#define _STATELESS_METHODS(enc) \ - enc##_encode, NULL, NULL, \ - enc##_decode, NULL, NULL, -#define CODEC_STATEFUL(enc) { \ - #enc, NULL, NULL, \ - _STATEFUL_METHODS(enc) \ +#define _STATEFUL_METHODS(enc) \ + enc##_encode, \ + enc##_encode_init, \ + enc##_encode_reset, \ + enc##_decode, \ + enc##_decode_init, \ + enc##_decode_reset, +#define _STATELESS_METHODS(enc) \ + enc##_encode, NULL, NULL, \ + enc##_decode, NULL, NULL, +#define CODEC_STATEFUL(enc) { \ + #enc, NULL, NULL, \ + _STATEFUL_METHODS(enc) \ }, -#define CODEC_STATELESS(enc) { \ - #enc, NULL, NULL, \ - _STATELESS_METHODS(enc) \ +#define CODEC_STATELESS(enc) { \ + #enc, NULL, NULL, \ + _STATELESS_METHODS(enc) \ }, -#define CODEC_STATELESS_WINIT(enc) { \ - #enc, NULL, \ - enc##_codec_init, \ - _STATELESS_METHODS(enc) \ +#define CODEC_STATELESS_WINIT(enc) { \ + #enc, NULL, \ + enc##_codec_init, \ + _STATELESS_METHODS(enc) \ }, -#define END_CODECS_LIST \ - {"", NULL,} }; \ - static const MultibyteCodec *codec_list = \ - (const MultibyteCodec *)_codec_list; +#define END_CODECS_LIST \ + {"", NULL,} }; \ + static const MultibyteCodec *codec_list = \ + (const MultibyteCodec *)_codec_list; static PyObject * getmultibytecodec(void) { - static PyObject *cofunc = NULL; - - if (cofunc == NULL) { - PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec"); - if (mod == NULL) - return NULL; - cofunc = PyObject_GetAttrString(mod, "__create_codec"); - Py_DECREF(mod); - } - return cofunc; + static PyObject *cofunc = NULL; + + if (cofunc == NULL) { + PyObject *mod = PyImport_ImportModuleNoBlock("_multibytecodec"); + if (mod == NULL) + return NULL; + cofunc = PyObject_GetAttrString(mod, "__create_codec"); + Py_DECREF(mod); + } + return cofunc; } static PyObject * getcodec(PyObject *self, PyObject *encoding) { - PyObject *codecobj, *r, *cofunc; - const MultibyteCodec *codec; - const char *enc; - - if (!PyUnicode_Check(encoding)) { - PyErr_SetString(PyExc_TypeError, - "encoding name must be a string."); - return NULL; - } - enc = _PyUnicode_AsString(encoding); - if (enc == NULL) - return NULL; - - cofunc = getmultibytecodec(); - if (cofunc == NULL) - return NULL; - - for (codec = codec_list; codec->encoding[0]; codec++) - if (strcmp(codec->encoding, enc) == 0) - break; - - if (codec->encoding[0] == '\0') { - PyErr_SetString(PyExc_LookupError, - "no such codec is supported."); - return NULL; - } - - codecobj = PyCapsule_New((void *)codec, PyMultibyteCodec_CAPSULE_NAME, NULL); - if (codecobj == NULL) - return NULL; - - r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL); - Py_DECREF(codecobj); - - return r; + PyObject *codecobj, *r, *cofunc; + const MultibyteCodec *codec; + const char *enc; + + if (!PyUnicode_Check(encoding)) { + PyErr_SetString(PyExc_TypeError, + "encoding name must be a string."); + return NULL; + } + enc = _PyUnicode_AsString(encoding); + if (enc == NULL) + return NULL; + + cofunc = getmultibytecodec(); + if (cofunc == NULL) + return NULL; + + for (codec = codec_list; codec->encoding[0]; codec++) + if (strcmp(codec->encoding, enc) == 0) + break; + + if (codec->encoding[0] == '\0') { + PyErr_SetString(PyExc_LookupError, + "no such codec is supported."); + return NULL; + } + + codecobj = PyCapsule_New((void *)codec, PyMultibyteCodec_CAPSULE_NAME, NULL); + if (codecobj == NULL) + return NULL; + + r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL); + Py_DECREF(codecobj); + + return r; } static struct PyMethodDef __methods[] = { - {"getcodec", (PyCFunction)getcodec, METH_O, ""}, - {NULL, NULL}, + {"getcodec", (PyCFunction)getcodec, METH_O, ""}, + {NULL, NULL}, }; static int register_maps(PyObject *module) { - const struct dbcs_map *h; - - for (h = mapping_list; h->charset[0] != '\0'; h++) { - char mhname[256] = "__map_"; - int r; - strcpy(mhname + sizeof("__map_") - 1, h->charset); - r = PyModule_AddObject(module, mhname, - PyCapsule_New((void *)h, PyMultibyteCodec_CAPSULE_NAME, NULL)); - if (r == -1) - return -1; - } - return 0; + const struct dbcs_map *h; + + for (h = mapping_list; h->charset[0] != '\0'; h++) { + char mhname[256] = "__map_"; + int r; + strcpy(mhname + sizeof("__map_") - 1, h->charset); + r = PyModule_AddObject(module, mhname, + PyCapsule_New((void *)h, PyMultibyteCodec_CAPSULE_NAME, NULL)); + if (r == -1) + return -1; + } + return 0; } #ifdef USING_BINARY_PAIR_SEARCH static DBCHAR find_pairencmap(ucs2_t body, ucs2_t modifier, - const struct pair_encodemap *haystack, int haystacksize) + const struct pair_encodemap *haystack, int haystacksize) { - int pos, min, max; - ucs4_t value = body << 16 | modifier; - - min = 0; - max = haystacksize; - - for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) - if (value < haystack[pos].uniseq) { - if (max == pos) break; - else max = pos; - } - else if (value > haystack[pos].uniseq) { - if (min == pos) break; - else min = pos; - } - else - break; - - if (value == haystack[pos].uniseq) - return haystack[pos].code; - else - return DBCINV; + int pos, min, max; + ucs4_t value = body << 16 | modifier; + + min = 0; + max = haystacksize; + + for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1) + if (value < haystack[pos].uniseq) { + if (max == pos) break; + else max = pos; + } + else if (value > haystack[pos].uniseq) { + if (min == pos) break; + else min = pos; + } + else + break; + + if (value == haystack[pos].uniseq) + return haystack[pos].code; + else + return DBCINV; } #endif #ifdef USING_IMPORTED_MAPS #define IMPORT_MAP(locale, charset, encmap, decmap) \ - importmap("_codecs_" #locale, "__map_" #charset, \ - (const void**)encmap, (const void**)decmap) + importmap("_codecs_" #locale, "__map_" #charset, \ + (const void**)encmap, (const void**)decmap) static int importmap(const char *modname, const char *symbol, - const void **encmap, const void **decmap) + const void **encmap, const void **decmap) { - PyObject *o, *mod; - - mod = PyImport_ImportModule((char *)modname); - if (mod == NULL) - return -1; - - o = PyObject_GetAttrString(mod, (char*)symbol); - if (o == NULL) - goto errorexit; - else if (!PyCapsule_IsValid(o, PyMultibyteCodec_CAPSULE_NAME)) { - PyErr_SetString(PyExc_ValueError, - "map data must be a Capsule."); - goto errorexit; - } - else { - struct dbcs_map *map; - map = PyCapsule_GetPointer(o, PyMultibyteCodec_CAPSULE_NAME); - if (encmap != NULL) - *encmap = map->encmap; - if (decmap != NULL) - *decmap = map->decmap; - Py_DECREF(o); - } - - Py_DECREF(mod); - return 0; + PyObject *o, *mod; + + mod = PyImport_ImportModule((char *)modname); + if (mod == NULL) + return -1; + + o = PyObject_GetAttrString(mod, (char*)symbol); + if (o == NULL) + goto errorexit; + else if (!PyCapsule_IsValid(o, PyMultibyteCodec_CAPSULE_NAME)) { + PyErr_SetString(PyExc_ValueError, + "map data must be a Capsule."); + goto errorexit; + } + else { + struct dbcs_map *map; + map = PyCapsule_GetPointer(o, PyMultibyteCodec_CAPSULE_NAME); + if (encmap != NULL) + *encmap = map->encmap; + if (decmap != NULL) + *decmap = map->decmap; + Py_DECREF(o); + } + + Py_DECREF(mod); + return 0; errorexit: - Py_DECREF(mod); - return -1; + Py_DECREF(mod); + return -1; } #endif -#define I_AM_A_MODULE_FOR(loc) \ - static struct PyModuleDef __module = { \ - PyModuleDef_HEAD_INIT, \ - "_codecs_"#loc, \ - NULL, \ - 0, \ - __methods, \ - NULL, \ - NULL, \ - NULL, \ - NULL \ - }; \ - PyObject* \ - PyInit__codecs_##loc(void) \ - { \ - PyObject *m = PyModule_Create(&__module); \ - if (m != NULL) \ - (void)register_maps(m); \ - return m; \ - } +#define I_AM_A_MODULE_FOR(loc) \ + static struct PyModuleDef __module = { \ + PyModuleDef_HEAD_INIT, \ + "_codecs_"#loc, \ + NULL, \ + 0, \ + __methods, \ + NULL, \ + NULL, \ + NULL, \ + NULL \ + }; \ + PyObject* \ + PyInit__codecs_##loc(void) \ + { \ + PyObject *m = PyModule_Create(&__module); \ + if (m != NULL) \ + (void)register_maps(m); \ + return m; \ + } #endif |