Bring CJKCodecs 1.1 into trunk. This completely reorganizes source

and installed layouts to make maintenance simple and easy.  And it
also adds four new codecs; big5hkscs, euc-jis-2004, shift-jis-2004
and iso2022-jp-2004.

1 files changed, 399 insertions, 0 deletions

diff --git a/Modules/cjkcodecs/cjkcodecs.h b/Modules/cjkcodecs/cjkcodecs.h
new file mode 100644
index 0000000..5fb05a8
--- /dev/null
+++ b/Modules/cjkcodecs/cjkcodecs.h
@@ -0,0 +1,399 @@
+/*
+ * cjkcodecs.h: common header for cjkcodecs
+ *
+ * Written by Hye-Shik Chang <perky@FreeBSD.org>
+ * $CJKCodecs: cjkcodecs.h,v 1.5 2004/07/06 17:05:24 perky Exp $
+ */
+
+#ifndef _CJKCODECS_H_
+#define _CJKCODECS_H_
+
+#include "Python.h"
+#include "multibytecodec.h"
+
+
+#define UNIINV	Py_UNICODE_REPLACEMENT_CHARACTER
+#define NOCHAR	0xFFFF
+#define MULTIC	0xFFFE
+#define DBCINV	0xFFFD
+
+/* shorter macros to save source size of mapping tables */
+#define U UNIINV
+#define N NOCHAR
+#define M MULTIC
+#define D DBCINV
+
+struct dbcs_index {
+	const ucs2_t *map;
+	unsigned char bottom, top;
+};
+typedef struct dbcs_index decode_map;
+
+struct widedbcs_index {
+	const ucs4_t *map;
+	unsigned char bottom, top;
+};
+typedef struct widedbcs_index widedecode_map;
+
+struct unim_index {
+	const DBCHAR *map;
+	unsigned char bottom, top;
+};
+typedef struct unim_index encode_map;
+
+struct unim_index_bytebased {
+	const unsigned char *map;
+	unsigned char bottom, top;
+};
+
+struct dbcs_map {
+	const char *charset;
+	const struct unim_index *encmap;
+	const struct dbcs_index *decmap;
+};
+
+struct pair_encodemap {
+	ucs4_t uniseq;
+	DBCHAR code;
+};
+
+static const MultibyteCodec codec_list[];
+static const struct dbcs_map mapping_list[];
+
+#define CODEC_INIT(encoding)						\
+	static int encoding##_codec_init(const void *config)
+
+#define ENCODER_INIT(encoding)						\
+	static int encoding##_encode_init(				\
+		MultibyteCodec_State *state, const void *config)
+#define ENCODER(encoding)						\
+	static int encoding##_encode(					\
+		MultibyteCodec_State *state, const void *config,	\
+		const Py_UNICODE **inbuf, size_t inleft,		\
+		unsigned char **outbuf, size_t outleft, int flags)
+#define ENCODER_RESET(encoding)						\
+	static int encoding##_encode_reset(				\
+		MultibyteCodec_State *state, const void *config,	\
+		unsigned char **outbuf, size_t outleft)
+
+#define DECODER_INIT(encoding)						\
+	static int encoding##_decode_init(				\
+		MultibyteCodec_State *state, const void *config)
+#define DECODER(encoding)						\
+	static int encoding##_decode(					\
+		MultibyteCodec_State *state, const void *config,	\
+		const unsigned char **inbuf, size_t inleft,		\
+		Py_UNICODE **outbuf, size_t outleft)
+#define DECODER_RESET(encoding)						\
+	static int encoding##_decode_reset(				\
+		MultibyteCodec_State *state, const void *config)
+
+#if Py_UNICODE_SIZE == 4
+#define UCS4INVALID(code)	\
+	if ((code) > 0xFFFF)	\
+	return 1;
+#else
+#define UCS4INVALID(code)	\
+	if (0) ;
+#endif
+
+#define NEXT_IN(i)				\
+	(*inbuf) += (i);			\
+	(inleft) -= (i);
+#define NEXT_OUT(o)				\
+	(*outbuf) += (o);			\
+	(outleft) -= (o);
+#define NEXT(i, o)				\
+	NEXT_IN(i) NEXT_OUT(o)
+
+#define REQUIRE_INBUF(n)			\
+	if (inleft < (n))			\
+		return MBERR_TOOFEW;
+#define REQUIRE_OUTBUF(n)			\
+	if (outleft < (n))			\
+		return MBERR_TOOSMALL;
+
+#define IN1 ((*inbuf)[0])
+#define IN2 ((*inbuf)[1])
+#define IN3 ((*inbuf)[2])
+#define IN4 ((*inbuf)[3])
+
+#define OUT1(c) ((*outbuf)[0]) = (c);
+#define OUT2(c) ((*outbuf)[1]) = (c);
+#define OUT3(c) ((*outbuf)[2]) = (c);
+#define OUT4(c) ((*outbuf)[3]) = (c);
+
+#define WRITE1(c1)		\
+	REQUIRE_OUTBUF(1)	\
+	(*outbuf)[0] = (c1);
+#define WRITE2(c1, c2)		\
+	REQUIRE_OUTBUF(2)	\
+	(*outbuf)[0] = (c1);	\
+	(*outbuf)[1] = (c2);
+#define WRITE3(c1, c2, c3)	\
+	REQUIRE_OUTBUF(3)	\
+	(*outbuf)[0] = (c1);	\
+	(*outbuf)[1] = (c2);	\
+	(*outbuf)[2] = (c3);
+#define WRITE4(c1, c2, c3, c4)	\
+	REQUIRE_OUTBUF(4)	\
+	(*outbuf)[0] = (c1);	\
+	(*outbuf)[1] = (c2);	\
+	(*outbuf)[2] = (c3);	\
+	(*outbuf)[3] = (c4);
+
+#if Py_UNICODE_SIZE == 2
+# define WRITEUCS4(c)						\
+	REQUIRE_OUTBUF(2)					\
+	(*outbuf)[0] = 0xd800 + (((c) - 0x10000) >> 10);	\
+	(*outbuf)[1] = 0xdc00 + (((c) - 0x10000) & 0x3ff);	\
+	NEXT_OUT(2)
+#else
+# define WRITEUCS4(c)						\
+	REQUIRE_OUTBUF(1)					\
+	**outbuf = (Py_UNICODE)(c);				\
+	NEXT_OUT(1)
+#endif
+
+#define _TRYMAP_ENC(m, assi, val)				\
+	if ((m)->map != NULL && (val) >= (m)->bottom &&		\
+	    (val)<= (m)->top && ((assi) = (m)->map[(val) -	\
+	    (m)->bottom]) != NOCHAR)
+#define TRYMAP_ENC(charset, assi, uni)				\
+	_TRYMAP_ENC(&charset##_encmap[(uni) >> 8], assi, (uni) & 0xff)
+#define _TRYMAP_DEC(m, assi, val)				\
+	if ((m)->map != NULL && (val) >= (m)->bottom &&		\
+	    (val)<= (m)->top && ((assi) = (m)->map[(val) -	\
+	    (m)->bottom]) != UNIINV)
+#define TRYMAP_DEC(charset, assi, c1, c2)			\
+	_TRYMAP_DEC(&charset##_decmap[c1], assi, c2)
+
+#define _TRYMAP_ENC_MPLANE(m, assplane, asshi, asslo, val)	\
+	if ((m)->map != NULL && (val) >= (m)->bottom &&		\
+	    (val)<= (m)->top &&					\
+	    ((assplane) = (m)->map[((val) - (m)->bottom)*3]) != 0 && \
+	    (((asshi) = (m)->map[((val) - (m)->bottom)*3 + 1]), 1) && \
+	    (((asslo) = (m)->map[((val) - (m)->bottom)*3 + 2]), 1))
+#define TRYMAP_ENC_MPLANE(charset, assplane, asshi, asslo, uni)	\
+	_TRYMAP_ENC_MPLANE(&charset##_encmap[(uni) >> 8], \
+			   assplane, asshi, asslo, (uni) & 0xff)
+#define TRYMAP_DEC_MPLANE(charset, assi, plane, c1, c2)		\
+	_TRYMAP_DEC(&charset##_decmap[plane][c1], assi, c2)
+
+#if Py_UNICODE_SIZE == 2
+#define DECODE_SURROGATE(c)					\
+	if (c >> 10 == 0xd800 >> 10) { /* high surrogate */	\
+		REQUIRE_INBUF(2)				\
+		if (IN2 >> 10 == 0xdc00 >> 10) { /* low surrogate */ \
+		    c = 0x10000 + ((ucs4_t)(c - 0xd800) << 10) + \
+			((ucs4_t)(IN2) - 0xdc00);		\
+		}						\
+	}
+#define GET_INSIZE(c)	((c) > 0xffff ? 2 : 1)
+#else
+#define DECODE_SURROGATE(c) {;}
+#define GET_INSIZE(c)	1
+#endif
+
+#define BEGIN_MAPPINGS_LIST static const struct dbcs_map mapping_list[] = {
+#define MAPPING_ENCONLY(enc) {#enc, (void*)enc##_encmap, NULL},
+#define MAPPING_DECONLY(enc) {#enc, NULL, (void*)enc##_decmap},
+#define MAPPING_ENCDEC(enc) {#enc, (void*)enc##_encmap, (void*)enc##_decmap},
+#define END_MAPPINGS_LIST {"", NULL, NULL} };
+
+#define BEGIN_CODECS_LIST static const MultibyteCodec codec_list[] = {
+#define _STATEFUL_METHODS(enc)		\
+	enc##_encode,			\
+	enc##_encode_init,		\
+	enc##_encode_reset,		\
+	enc##_decode,			\
+	enc##_decode_init,		\
+	enc##_decode_reset,
+#define _STATELESS_METHODS(enc)		\
+	enc##_encode, NULL, NULL,	\
+	enc##_decode, NULL, NULL,
+#define CODEC_STATEFUL(enc) {		\
+	#enc, NULL, NULL,		\
+	_STATEFUL_METHODS(enc)		\
+},
+#define CODEC_STATELESS(enc) {		\
+	#enc, NULL, NULL,		\
+	_STATELESS_METHODS(enc)		\
+},
+#define CODEC_STATELESS_WINIT(enc) {	\
+	#enc, NULL,			\
+	enc##_codec_init,		\
+	_STATELESS_METHODS(enc)		\
+},
+#define END_CODECS_LIST {"", NULL,} };
+
+static PyObject *
+getmultibytecodec(void)
+{
+	static PyObject *cofunc = NULL;
+
+	if (cofunc == NULL) {
+		PyObject *mod = PyImport_ImportModule("_multibytecodec");
+		if (mod == NULL)
+			return NULL;
+		cofunc = PyObject_GetAttrString(mod, "__create_codec");
+		Py_DECREF(mod);
+	}
+	return cofunc;
+}
+
+static PyObject *
+getcodec(PyObject *self, PyObject *encoding)
+{
+	PyObject *codecobj, *r, *cofunc;
+	const MultibyteCodec *codec;
+	const char *enc;
+#ifdef NO_METH_O
+	PyObject *args = encoding;
+
+        if (!PyArg_ParseTuple(args, "O:getcodec", &encoding))
+	        return NULL;
+#endif
+
+	if (!PyString_Check(encoding)) {
+		PyErr_SetString(PyExc_TypeError,
+				"encoding name must be a string.");
+		return NULL;
+	}
+
+	cofunc = getmultibytecodec();
+	if (cofunc == NULL)
+		return NULL;
+
+	enc = PyString_AS_STRING(encoding);
+	for (codec = codec_list; codec->encoding[0]; codec++)
+		if (strcmp(codec->encoding, enc) == 0)
+			break;
+
+	if (codec->encoding[0] == '\0') {
+		PyErr_SetString(PyExc_LookupError,
+				"no such codec is supported.");
+		return NULL;
+	}
+
+	codecobj = PyCObject_FromVoidPtr((void *)codec, NULL);
+	if (codecobj == NULL)
+		return NULL;
+
+#if PY_VERSION_HEX >= 0x02020000
+	r = PyObject_CallFunctionObjArgs(cofunc, codecobj, NULL);
+#else
+	r = PyObject_CallFunction(cofunc, "O", codecobj);
+#endif
+	Py_DECREF(codecobj);
+
+	return r;
+}
+
+static struct PyMethodDef __methods[] = {
+#ifndef NO_METH_O
+	{"getcodec", (PyCFunction)getcodec, METH_O, ""},
+#else
+	{"getcodec", (PyCFunction)getcodec, METH_VARARGS, ""},
+#endif
+	{NULL, NULL},
+};
+
+static int
+register_maps(PyObject *module)
+{
+	const struct dbcs_map *h;
+
+	for (h = mapping_list; h->charset[0] != '\0'; h++) {
+		char mhname[256] = "__map_";
+		int r;
+		strcpy(mhname + sizeof("__map_") - 1, h->charset);
+		r = PyModule_AddObject(module, mhname,
+				PyCObject_FromVoidPtr((void *)h, NULL));
+		if (r == -1)
+			return -1;
+	}
+	return 0;
+}
+
+#ifdef USING_BINARY_PAIR_SEARCH
+static DBCHAR
+find_pairencmap(ucs2_t body, ucs2_t modifier,
+		const struct pair_encodemap *haystack, int haystacksize)
+{
+	int pos, min, max;
+	ucs4_t value = body << 16 | modifier;
+
+	min = 0;
+	max = haystacksize;
+
+	for (pos = haystacksize >> 1; min != max; pos = (min + max) >> 1)
+		if (value < haystack[pos].uniseq) {
+			if (max == pos) break;
+			else max = pos;
+		}
+		else if (value > haystack[pos].uniseq) {
+			if (min == pos) break;
+			else min = pos;
+		}
+		else
+			break;
+
+		if (value == haystack[pos].uniseq)
+			return haystack[pos].code;
+		else
+			return DBCINV;
+}
+#endif
+
+#ifdef USING_IMPORTED_MAPS
+#define IMPORT_MAP(locale, charset, encmap, decmap) \
+	importmap("_codecs_" #locale, "__map_" #charset, \
+		  (const void**)encmap, (const void**)decmap)
+
+static int
+importmap(const char *modname, const char *symbol,
+	  const void **encmap, const void **decmap)
+{
+	PyObject *o, *mod;
+
+	mod = PyImport_ImportModule((char *)modname);
+	if (mod == NULL)
+		return -1;
+
+	o = PyObject_GetAttrString(mod, (char*)symbol);
+	if (o == NULL)
+		goto errorexit;
+	else if (!PyCObject_Check(o)) {
+		PyErr_SetString(PyExc_ValueError,
+				"map data must be a CObject.");
+		goto errorexit;
+	}
+	else {
+		struct dbcs_map *map;
+		map = PyCObject_AsVoidPtr(o);
+		if (encmap != NULL)
+			*encmap = map->encmap;
+		if (decmap != NULL)
+			*decmap = map->decmap;
+		Py_DECREF(o);
+	}
+
+	Py_DECREF(mod);
+	return 0;
+
+errorexit:
+	Py_DECREF(mod);
+	return -1;
+}
+#endif
+
+#define I_AM_A_MODULE_FOR(loc)						\
+	void								\
+	init_codecs_##loc(void)						\
+	{								\
+		PyObject *m = Py_InitModule("_codecs_" #loc, __methods);\
+		(void)register_maps(m);					\
+	}
+
+#endif