Merge p3yk branch with the trunk up to revision 45595. This breaks a fair

number of tests, all because of the codecs/_multibytecodecs issue described
here (it's not a Py3K issue, just something Py3K discovers):
http://mail.python.org/pipermail/python-dev/2006-April/064051.html

Hye-Shik Chang promised to look for a fix, so no need to fix it here. The
tests that are expected to break are:

test_codecencodings_cn
test_codecencodings_hk
test_codecencodings_jp
test_codecencodings_kr
test_codecencodings_tw
test_codecs
test_multibytecodec

This merge fixes an actual test failure (test_weakref) in this branch,
though, so I believe merging is the right thing to do anyway.

1 files changed, 83 insertions, 52 deletions

diff --git a/Python/compile.c b/Python/compile.c
index b92fb62..12b190a 100644
--- a/Python/compile.c
+++ b/Python/compile.c
@@ -58,8 +58,9 @@ struct instr {
 };
 
 typedef struct basicblock_ {
-	/* next block in the list of blocks for a unit (don't confuse with
-	 * b_next) */
+    /* Each basicblock in a compilation unit is linked via b_list in the
+       reverse order that the block are allocated.  b_list points to the next
+       block, not to be confused with b_next, which is next by control flow. */
 	struct basicblock_ *b_list;
 	/* number of instructions used */
 	int b_iused;
@@ -114,7 +115,9 @@ struct compiler_unit {
 	PyObject *u_private;	/* for private name mangling */
 
 	int u_argcount;	   /* number of arguments for block */ 
-	basicblock *u_blocks; /* pointer to list of blocks */
+    /* Pointer to the most recently allocated block.  By following b_list
+       members, you can reach all early allocated blocks. */
+	basicblock *u_blocks;
 	basicblock *u_curblock; /* pointer to current block */
 	int u_tmpname;	   /* temporary variables for list comps */
 
@@ -194,19 +197,19 @@ static PyCodeObject *assemble(struct compiler *, int addNone);
 static PyObject *__doc__;
 
 PyObject *
-_Py_Mangle(PyObject *private, PyObject *ident)
+_Py_Mangle(PyObject *privateobj, PyObject *ident)
 {
 	/* Name mangling: __private becomes _classname__private.
 	   This is independent from how the name is used. */
 	const char *p, *name = PyString_AsString(ident);
 	char *buffer;
 	size_t nlen, plen;
-	if (private == NULL || name == NULL || name[0] != '_' ||
+	if (privateobj == NULL || name == NULL || name[0] != '_' ||
             name[1] != '_') {
 		Py_INCREF(ident);
 		return ident;
 	}
-	p = PyString_AsString(private);
+	p = PyString_AsString(privateobj);
 	nlen = strlen(name);
 	if (name[nlen-1] == '_' && name[nlen-2] == '_') {
 		Py_INCREF(ident);
@@ -311,7 +314,7 @@ compiler_free(struct compiler *c)
 	if (c->c_st)
 		PySymtable_Free(c->c_st);
 	if (c->c_future)
-		PyMem_Free(c->c_future);
+		PyObject_Free(c->c_future);
 	Py_DECREF(c->c_stack);
 }
 
@@ -319,7 +322,9 @@ static PyObject *
 list2dict(PyObject *list)
 {
 	Py_ssize_t i, n;
-	PyObject *v, *k, *dict = PyDict_New();
+	PyObject *v, *k;
+	PyObject *dict = PyDict_New();
+	if (!dict) return NULL;
 
 	n = PyList_Size(list);
 	for (i = 0; i < n; i++) {
@@ -602,7 +607,7 @@ fold_unaryops_on_constants(unsigned char *codestr, PyObject *consts)
 static unsigned int *
 markblocks(unsigned char *code, int len)
 {
-	unsigned int *blocks = PyMem_Malloc(len*sizeof(int));
+	unsigned int *blocks = (unsigned int *)PyMem_Malloc(len*sizeof(int));
 	int i,j, opcode, blockcnt = 0;
 
 	if (blocks == NULL)
@@ -683,10 +688,11 @@ optimize_code(PyObject *code, PyObject* consts, PyObject *names,
 		goto exitUnchanged;
 
 	/* Make a modifiable copy of the code string */
-	codestr = PyMem_Malloc(codelen);
+	codestr = (unsigned char *)PyMem_Malloc(codelen);
 	if (codestr == NULL)
 		goto exitUnchanged;
-	codestr = memcpy(codestr, PyString_AS_STRING(code), codelen);
+	codestr = (unsigned char *)memcpy(codestr, 
+                                        PyString_AS_STRING(code), codelen);
 
 	/* Verify that RETURN_VALUE terminates the codestring.	This allows
 	   the various transformation patterns to look ahead several
@@ -697,7 +703,7 @@ optimize_code(PyObject *code, PyObject* consts, PyObject *names,
 		goto exitUnchanged;
 
 	/* Mapping to new jump targets after NOPs are removed */
-	addrmap = PyMem_Malloc(codelen * sizeof(int));
+	addrmap = (int *)PyMem_Malloc(codelen * sizeof(int));
 	if (addrmap == NULL)
 		goto exitUnchanged;
 
@@ -1077,7 +1083,8 @@ compiler_enter_scope(struct compiler *c, identifier name, void *key,
 {
 	struct compiler_unit *u;
 
-	u = PyObject_Malloc(sizeof(struct compiler_unit));
+	u = (struct compiler_unit *)PyObject_Malloc(sizeof(
+                                                struct compiler_unit));
 	if (!u) {
 		PyErr_NoMemory();
 		return 0;
@@ -1187,7 +1194,7 @@ compiler_new_block(struct compiler *c)
 		return NULL;
 	}
 	memset((void *)b, 0, sizeof(basicblock));
-	assert (b->b_next == NULL);
+    /* Extend the singly linked list of blocks with new block. */
 	b->b_list = u->u_blocks;
 	u->u_blocks = b;
 	return b;
@@ -1233,8 +1240,8 @@ compiler_next_instr(struct compiler *c, basicblock *b)
 {
 	assert(b != NULL);
 	if (b->b_instr == NULL) {
-		b->b_instr = PyObject_Malloc(sizeof(struct instr) *
-					     DEFAULT_BLOCK_SIZE);
+		b->b_instr = (struct instr *)PyObject_Malloc(
+                                 sizeof(struct instr) * DEFAULT_BLOCK_SIZE);
 		if (b->b_instr == NULL) {
 			PyErr_NoMemory();
 			return -1;
@@ -1252,7 +1259,8 @@ compiler_next_instr(struct compiler *c, basicblock *b)
 			return -1;
 		}
 		b->b_ialloc <<= 1;
-		b->b_instr = PyObject_Realloc((void *)b->b_instr, newsize);
+		b->b_instr = (struct instr *)PyObject_Realloc(
+                                                (void *)b->b_instr, newsize);
 		if (b->b_instr == NULL)
 			return -1;
 		memset((char *)b->b_instr + oldsize, 0, newsize - oldsize);
@@ -1260,6 +1268,13 @@ compiler_next_instr(struct compiler *c, basicblock *b)
 	return b->b_iused++;
 }
 
+/* Set the i_lineno member of the instruction at offse off if the
+   line number for the current expression/statement (?) has not
+   already been set.  If it has been set, the call has no effect.
+
+   Every time a new node is b
+   */
+
 static void
 compiler_set_lineno(struct compiler *c, int off)
 {
@@ -1600,7 +1615,6 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
 	off = compiler_next_instr(c, c->u->u_curblock);
 	if (off < 0)
 		return 0;
-	compiler_set_lineno(c, off);
 	i = &c->u->u_curblock->b_instr[off];
 	i->i_opcode = opcode;
 	i->i_target = b;
@@ -1609,6 +1623,7 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
 		i->i_jabs = 1;
 	else
 		i->i_jrel = 1;
+	compiler_set_lineno(c, off);
 	return 1;
 }
 
@@ -1695,7 +1710,7 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
 	int _i; \
 	asdl_seq *seq = (SEQ); /* avoid variable capture */ \
 	for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \
-		TYPE ## _ty elt = asdl_seq_GET(seq, _i); \
+		TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \
 		if (!compiler_visit_ ## TYPE((C), elt)) \
 			return 0; \
 	} \
@@ -1705,7 +1720,7 @@ compiler_addop_j(struct compiler *c, int opcode, basicblock *b, int absolute)
 	int _i; \
 	asdl_seq *seq = (SEQ); /* avoid variable capture */ \
 	for (_i = 0; _i < asdl_seq_LEN(seq); _i++) { \
-		TYPE ## _ty elt = asdl_seq_GET(seq, _i); \
+		TYPE ## _ty elt = (TYPE ## _ty)asdl_seq_GET(seq, _i); \
 		if (!compiler_visit_ ## TYPE((C), elt)) { \
 			compiler_exit_scope(c); \
 			return 0; \
@@ -1731,7 +1746,7 @@ compiler_body(struct compiler *c, asdl_seq *stmts)
 
 	if (!asdl_seq_LEN(stmts))
 		return 1;
-	st = asdl_seq_GET(stmts, 0);
+	st = (stmt_ty)asdl_seq_GET(stmts, 0);
 	if (compiler_isdocstring(st)) {
 		i = 1;
 		VISIT(c, expr, st->v.Expr.value);
@@ -1739,7 +1754,7 @@ compiler_body(struct compiler *c, asdl_seq *stmts)
 			return 0;
 	}
 	for (; i < asdl_seq_LEN(stmts); i++)
-	    VISIT(c, stmt, asdl_seq_GET(stmts, i));
+	    VISIT(c, stmt, (stmt_ty)asdl_seq_GET(stmts, i));
 	return 1;
 }
 
@@ -1765,7 +1780,8 @@ compiler_mod(struct compiler *c, mod_ty mod)
 		break;
 	case Interactive_kind:
 		c->c_interactive = 1;
-		VISIT_SEQ_IN_SCOPE(c, stmt, mod->v.Interactive.body);
+		VISIT_SEQ_IN_SCOPE(c, stmt, 
+                                        mod->v.Interactive.body);
 		break;
 	case Expression_kind:
 		VISIT_IN_SCOPE(c, expr, mod->v.Expression.body);
@@ -1882,7 +1898,7 @@ compiler_decorators(struct compiler *c, asdl_seq* decos)
 		return 1;
 
 	for (i = 0; i < asdl_seq_LEN(decos); i++) {
-		VISIT(c, expr, asdl_seq_GET(decos, i));
+		VISIT(c, expr, (expr_ty)asdl_seq_GET(decos, i));
 	}
 	return 1;
 }
@@ -1894,7 +1910,7 @@ compiler_arguments(struct compiler *c, arguments_ty args)
 	int n = asdl_seq_LEN(args->args);
 	/* Correctly handle nested argument lists */
 	for (i = 0; i < n; i++) {
-		expr_ty arg = asdl_seq_GET(args->args, i);
+		expr_ty arg = (expr_ty)asdl_seq_GET(args->args, i);
 		if (arg->kind == Tuple_kind) {
 			PyObject *id = PyString_FromFormat(".%d", i);
 			if (id == NULL) {
@@ -1931,7 +1947,7 @@ compiler_function(struct compiler *c, stmt_ty s)
 				  s->lineno))
 		return 0;
 
-	st = asdl_seq_GET(s->v.FunctionDef.body, 0);
+	st = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, 0);
 	docstring = compiler_isdocstring(st);
 	if (docstring)
 	    first_const = st->v.Expr.value->v.Str.s;
@@ -1947,7 +1963,7 @@ compiler_function(struct compiler *c, stmt_ty s)
 	n = asdl_seq_LEN(s->v.FunctionDef.body);
 	/* if there was a docstring, we need to skip the first statement */
 	for (i = docstring; i < n; i++) {
-		stmt_ty s2 = asdl_seq_GET(s->v.FunctionDef.body, i);
+		stmt_ty s2 = (stmt_ty)asdl_seq_GET(s->v.FunctionDef.body, i);
 		if (i == 0 && s2->kind == Expr_kind &&
 		    s2->v.Expr.value->kind == Str_kind)
 			continue;
@@ -2221,7 +2237,7 @@ compiler_while(struct compiler *c, stmt_ty s)
 		ADDOP(c, POP_BLOCK);
 	}
 	compiler_pop_fblock(c, LOOP, loop);
-	if (orelse != NULL)
+	if (orelse != NULL) /* what if orelse is just pass? */
 		VISIT_SEQ(c, stmt, s->v.While.orelse);
 	compiler_use_next_block(c, end);
 
@@ -2375,10 +2391,12 @@ compiler_try_except(struct compiler *c, stmt_ty s)
 	n = asdl_seq_LEN(s->v.TryExcept.handlers);
 	compiler_use_next_block(c, except);
 	for (i = 0; i < n; i++) {
-		excepthandler_ty handler = asdl_seq_GET(
+		excepthandler_ty handler = (excepthandler_ty)asdl_seq_GET(
 						s->v.TryExcept.handlers, i);
 		if (!handler->type && i < n-1)
 		    return compiler_error(c, "default 'except:' must be last");
+        c->u->u_lineno_set = false;
+        c->u->u_lineno = handler->lineno;
 		except = compiler_new_block(c);
 		if (except == NULL)
 			return 0;
@@ -2453,7 +2471,7 @@ compiler_import(struct compiler *c, stmt_ty s)
 	int i, n = asdl_seq_LEN(s->v.Import.names);
 
 	for (i = 0; i < n; i++) {
-		alias_ty alias = asdl_seq_GET(s->v.Import.names, i);
+		alias_ty alias = (alias_ty)asdl_seq_GET(s->v.Import.names, i);
 		int r;
 		PyObject *level;
 
@@ -2508,7 +2526,7 @@ compiler_from_import(struct compiler *c, stmt_ty s)
 
 	/* build up the names */
 	for (i = 0; i < n; i++) {
-		alias_ty alias = asdl_seq_GET(s->v.ImportFrom.names, i);
+		alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i);
 		Py_INCREF(alias->name);
 		PyTuple_SET_ITEM(names, i, alias->name);
 	}
@@ -2531,7 +2549,7 @@ compiler_from_import(struct compiler *c, stmt_ty s)
 	Py_DECREF(names);
 	ADDOP_NAME(c, IMPORT_NAME, s->v.ImportFrom.module, names);
 	for (i = 0; i < n; i++) {
-		alias_ty alias = asdl_seq_GET(s->v.ImportFrom.names, i);
+		alias_ty alias = (alias_ty)asdl_seq_GET(s->v.ImportFrom.names, i);
 		identifier store_name;
 
 		if (i == 0 && *PyString_AS_STRING(alias->name) == '*') {
@@ -2592,8 +2610,10 @@ compiler_visit_stmt(struct compiler *c, stmt_ty s)
 {
 	int i, n;
 
+    /* Always assign a lineno to the next instruction for a stmt. */
 	c->u->u_lineno = s->lineno;
 	c->u->u_lineno_set = false;
+
 	switch (s->kind) {
 	case FunctionDef_kind:
 		return compiler_function(c, s);
@@ -2962,11 +2982,11 @@ compiler_boolop(struct compiler *c, expr_ty e)
 	s = e->v.BoolOp.values;
 	n = asdl_seq_LEN(s) - 1;
 	for (i = 0; i < n; ++i) {
-		VISIT(c, expr, asdl_seq_GET(s, i));
+		VISIT(c, expr, (expr_ty)asdl_seq_GET(s, i));
 		ADDOP_JREL(c, jumpi, end);
 		ADDOP(c, POP_TOP)
 	}
-	VISIT(c, expr, asdl_seq_GET(s, n));
+	VISIT(c, expr, (expr_ty)asdl_seq_GET(s, n));
 	compiler_use_next_block(c, end);
 	return 1;
 }
@@ -3013,24 +3033,25 @@ compiler_compare(struct compiler *c, expr_ty e)
 		cleanup = compiler_new_block(c);
 		if (cleanup == NULL)
 		    return 0;
-		VISIT(c, expr, asdl_seq_GET(e->v.Compare.comparators, 0));
+		VISIT(c, expr, 
+                        (expr_ty)asdl_seq_GET(e->v.Compare.comparators, 0));
 	}
 	for (i = 1; i < n; i++) {
 		ADDOP(c, DUP_TOP);
 		ADDOP(c, ROT_THREE);
-		/* XXX We're casting a void* to cmpop_ty in the next stmt. */
 		ADDOP_I(c, COMPARE_OP,
-			cmpop((cmpop_ty)asdl_seq_GET(e->v.Compare.ops, i - 1)));
+			cmpop((cmpop_ty)(asdl_seq_GET(
+                                                  e->v.Compare.ops, i - 1))));
 		ADDOP_JREL(c, JUMP_IF_FALSE, cleanup);
 		NEXT_BLOCK(c);
 		ADDOP(c, POP_TOP);
 		if (i < (n - 1))
-		    VISIT(c, expr, asdl_seq_GET(e->v.Compare.comparators, i));
+		    VISIT(c, expr, 
+                            (expr_ty)asdl_seq_GET(e->v.Compare.comparators, i));
 	}
-	VISIT(c, expr, asdl_seq_GET(e->v.Compare.comparators, n - 1));
+	VISIT(c, expr, (expr_ty)asdl_seq_GET(e->v.Compare.comparators, n - 1));
 	ADDOP_I(c, COMPARE_OP,
-		/* XXX We're casting a void* to cmpop_ty in the next stmt. */
-	       cmpop((cmpop_ty)asdl_seq_GET(e->v.Compare.ops, n - 1)));
+	       cmpop((cmpop_ty)(asdl_seq_GET(e->v.Compare.ops, n - 1))));
 	if (n > 1) {
 		basicblock *end = compiler_new_block(c);
 		if (end == NULL)
@@ -3043,6 +3064,7 @@ compiler_compare(struct compiler *c, expr_ty e)
 	}
 	return 1;
 }
+#undef CMPCAST
 
 static int
 compiler_call(struct compiler *c, expr_ty e)
@@ -3102,7 +3124,7 @@ compiler_listcomp_generator(struct compiler *c, PyObject *tmpname,
 		anchor == NULL)
 	    return 0;
 
-	l = asdl_seq_GET(generators, gen_index);
+	l = (comprehension_ty)asdl_seq_GET(generators, gen_index);
 	VISIT(c, expr, l->iter);
 	ADDOP(c, GET_ITER);
 	compiler_use_next_block(c, start);
@@ -3113,7 +3135,7 @@ compiler_listcomp_generator(struct compiler *c, PyObject *tmpname,
 	/* XXX this needs to be cleaned up...a lot! */
 	n = asdl_seq_LEN(l->ifs);
 	for (i = 0; i < n; i++) {
-		expr_ty e = asdl_seq_GET(l->ifs, i);
+		expr_ty e = (expr_ty)asdl_seq_GET(l->ifs, i);
 		VISIT(c, expr, e);
 		ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup);
 		NEXT_BLOCK(c);
@@ -3198,7 +3220,7 @@ compiler_genexp_generator(struct compiler *c,
 	    anchor == NULL || end == NULL)
 		return 0;
 
-	ge = asdl_seq_GET(generators, gen_index);
+	ge = (comprehension_ty)asdl_seq_GET(generators, gen_index);
 	ADDOP_JREL(c, SETUP_LOOP, end);
 	if (!compiler_push_fblock(c, LOOP, start))
 		return 0;
@@ -3221,7 +3243,7 @@ compiler_genexp_generator(struct compiler *c,
 	/* XXX this needs to be cleaned up...a lot! */
 	n = asdl_seq_LEN(ge->ifs);
 	for (i = 0; i < n; i++) {
-		expr_ty e = asdl_seq_GET(ge->ifs, i);
+		expr_ty e = (expr_ty)asdl_seq_GET(ge->ifs, i);
 		VISIT(c, expr, e);
 		ADDOP_JREL(c, JUMP_IF_FALSE, if_cleanup);
 		NEXT_BLOCK(c);
@@ -3462,6 +3484,9 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
 {
 	int i, n;
 
+    /* If expr e has a different line number than the last expr/stmt,
+       set a new line number for the next instruction.
+       */
 	if (e->lineno > c->u->u_lineno) {
 		c->u->u_lineno = e->lineno;
 		c->u->u_lineno_set = false;
@@ -3490,9 +3515,11 @@ compiler_visit_expr(struct compiler *c, expr_ty e)
 		   It wants the stack to look like (value) (dict) (key) */
 		for (i = 0; i < n; i++) {
 			ADDOP(c, DUP_TOP);
-			VISIT(c, expr, asdl_seq_GET(e->v.Dict.values, i));
+			VISIT(c, expr, 
+                                (expr_ty)asdl_seq_GET(e->v.Dict.values, i));
 			ADDOP(c, ROT_TWO);
-			VISIT(c, expr, asdl_seq_GET(e->v.Dict.keys, i));
+			VISIT(c, expr, 
+                                (expr_ty)asdl_seq_GET(e->v.Dict.keys, i));
 			ADDOP(c, STORE_SUBSCR);
 		}
 		break;
@@ -3859,7 +3886,8 @@ compiler_visit_slice(struct compiler *c, slice_ty s, expr_context_ty ctx)
 		if (ctx != AugStore) {
 			int i, n = asdl_seq_LEN(s->v.ExtSlice.dims);
 			for (i = 0; i < n; i++) {
-				slice_ty sub = asdl_seq_GET(s->v.ExtSlice.dims, i);
+				slice_ty sub = (slice_ty)asdl_seq_GET(
+                                        s->v.ExtSlice.dims, i);
 				if (!compiler_visit_nested_slice(c, sub, ctx))
 					return 0;
 			}
@@ -4048,7 +4076,7 @@ assemble_lnotab(struct assembler *a, struct instr *i)
 {
 	int d_bytecode, d_lineno;
 	int len;
-	char *lnotab;
+	unsigned char *lnotab;
 
 	d_bytecode = a->a_offset - a->a_lineno_off;
 	d_lineno = i->i_lineno - a->a_lineno;
@@ -4071,7 +4099,8 @@ assemble_lnotab(struct assembler *a, struct instr *i)
 			if (_PyString_Resize(&a->a_lnotab, len) < 0)
 				return 0;
 		}
-		lnotab = PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
+		lnotab = (unsigned char *)
+			   PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
 		for (j = 0; j < ncodes; j++) {
 			*lnotab++ = 255;
 			*lnotab++ = 0;
@@ -4092,7 +4121,8 @@ assemble_lnotab(struct assembler *a, struct instr *i)
 			if (_PyString_Resize(&a->a_lnotab, len) < 0)
 				return 0;
 		}
-		lnotab = PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
+		lnotab = (unsigned char *)
+			   PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
 		*lnotab++ = 255;
 		*lnotab++ = d_bytecode;
 		d_bytecode = 0;
@@ -4109,7 +4139,8 @@ assemble_lnotab(struct assembler *a, struct instr *i)
 		if (_PyString_Resize(&a->a_lnotab, len * 2) < 0)
 			return 0;
 	}
-	lnotab = PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
+	lnotab = (unsigned char *)
+			PyString_AS_STRING(a->a_lnotab) + a->a_lnotab_off;
 
 	a->a_lnotab_off += 2;
 	if (d_bytecode) {