Merge p3yk branch with the trunk up to revision 45595. This breaks a fair

number of tests, all because of the codecs/_multibytecodecs issue described
here (it's not a Py3K issue, just something Py3K discovers):
http://mail.python.org/pipermail/python-dev/2006-April/064051.html

Hye-Shik Chang promised to look for a fix, so no need to fix it here. The
tests that are expected to break are:

test_codecencodings_cn
test_codecencodings_hk
test_codecencodings_jp
test_codecencodings_kr
test_codecencodings_tw
test_codecs
test_multibytecodec

This merge fixes an actual test failure (test_weakref) in this branch,
though, so I believe merging is the right thing to do anyway.

11 files changed, 158 insertions, 126 deletions

diff --git a/Parser/Python.asdl b/Parser/Python.asdl
index 4397d89..00de381 100644
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -98,8 +98,11 @@ module Python version "$Revision$"
 	comprehension = (expr target, expr iter, expr* ifs)
 
 	-- not sure what to call the first argument for raise and except
-
-	excepthandler = (expr? type, expr? name, stmt* body)
+	-- TODO(jhylton): Figure out if there is a better way to handle
+	--                lineno and col_offset fields, particularly when
+        --                ast is exposed to Python.
+	excepthandler = (expr? type, expr? name, stmt* body, int lineno,
+	                 int col_offset)
 
 	arguments = (expr* args, identifier? vararg, 
 		     identifier? kwarg, expr* defaults)
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index ad2209d..b6d9830 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -155,8 +155,10 @@ class StructVisitor(EmitVisitor):
             type = sum.types[i]
             enum.append("%s_kind=%d" % (type.name, i + 1))
 
+        emit("enum _%(name)s_kind {" + ", ".join(enum) + "};")
+
         emit("struct _%(name)s {")
-        emit("enum { " + ", ".join(enum) + " } kind;", depth + 1)
+        emit("enum _%(name)s_kind kind;", depth + 1)
         emit("union {", depth + 1)
         for t in sum.types:
             self.visit(t, depth + 2)
@@ -186,7 +188,10 @@ class StructVisitor(EmitVisitor):
         ctype = get_c_type(field.type)
         name = field.name
         if field.seq:
-            self.emit("asdl_seq *%(name)s;" % locals(), depth)
+            if field.type.value in ('cmpop',):
+                self.emit("asdl_int_seq *%(name)s;" % locals(), depth)
+            else:
+                self.emit("asdl_seq *%(name)s;" % locals(), depth)
         else:
             self.emit("%(ctype)s %(name)s;" % locals(), depth)
 
@@ -232,7 +237,10 @@ class PrototypeVisitor(EmitVisitor):
                 name = f.name
             # XXX should extend get_c_type() to handle this
             if f.seq:
-                ctype = "asdl_seq *"
+                if f.type.value in ('cmpop',):
+                    ctype = "asdl_int_seq *"
+                else:
+                    ctype = "asdl_seq *"
             else:
                 ctype = get_c_type(f.type)
             args.append((ctype, name, f.opt or f.seq))
@@ -276,7 +284,7 @@ class FunctionVisitor(PrototypeVisitor):
         emit("%s p;" % ctype, 1)
         for argtype, argname, opt in args:
             # XXX hack alert: false is allowed for a bool
-            if not opt and not argtype == "bool":
+            if not opt and not (argtype == "bool" or argtype == "int"):
                 emit("if (!%s) {" % argname, 1)
                 emit("PyErr_SetString(PyExc_ValueError,", 2)
                 msg = "field %s is required for %s" % (argname, name)
@@ -413,10 +421,10 @@ static PyTypeObject* make_type(char *type, PyTypeObject* base, char**fields, int
 
 static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
 {
-    int i;
+    int i, result;
     PyObject *s, *l = PyList_New(num_fields);
     if (!l) return 0;
-    for(i=0; i < num_fields; i++) {
+    for(i = 0; i < num_fields; i++) {
         s = PyString_FromString(attrs[i]);
         if (!s) {
             Py_DECREF(l);
@@ -424,7 +432,9 @@ static int add_attributes(PyTypeObject* type, char**attrs, int num_fields)
         }
         PyList_SET_ITEM(l, i, s);
     }
-    return PyObject_SetAttrString((PyObject*)type, "_attributes", l) >=0;
+    result = PyObject_SetAttrString((PyObject*)type, "_attributes", l) >= 0;
+    Py_DECREF(l);
+    return result;
 }
 
 static PyObject* ast2obj_list(asdl_seq *seq, PyObject* (*func)(void*))
@@ -465,9 +475,9 @@ static PyObject* ast2obj_int(bool b)
 }
 """, 0, reflow=False)
 
-        self.emit("static int initialized;", 0)
         self.emit("static int init_types(void)",0)
         self.emit("{", 0)
+        self.emit("static int initialized;", 1)
         self.emit("if (initialized) return 1;", 1)
         self.emit('AST_type = make_type("AST", &PyBaseObject_Type, NULL, 0);', 1)
         for dfn in mod.dfns:
@@ -543,7 +553,7 @@ class ASTModuleVisitor(PickleVisitor):
         self.addObj(cons.name)
 
     def addObj(self, name):
-        self.emit('if(PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1)
+        self.emit('if (PyDict_SetItemString(d, "%s", (PyObject*)%s_type) < 0) return;' % (name, name), 1)
 
 _SPECIALIZED_SEQUENCES = ('stmt', 'expr')
 
@@ -677,8 +687,8 @@ class ObjVisitor(PickleVisitor):
                 self.emit("if (!value) goto failed;", depth+1)
                 self.emit("for(i = 0; i < n; i++)", depth+1)
                 # This cannot fail, so no need for error handling
-                self.emit("PyList_SET_ITEM(value, i, ast2obj_%s((%s_ty)asdl_seq_GET(%s, i)));" %
-                                (field.type, field.type, value), depth+2, reflow=False)
+                self.emit("PyList_SET_ITEM(value, i, ast2obj_cmpop((cmpop_ty)asdl_seq_GET(%s, i)));" % value,
+                          depth+2, reflow=False)
                 self.emit("}", depth)
             else:
                 self.emit("value = ast2obj_list(%s, ast2obj_%s);" % (value, field.type), depth)
@@ -716,39 +726,35 @@ def main(srcfile):
         sys.exit(1)
     if INC_DIR:
         p = "%s/%s-ast.h" % (INC_DIR, mod.name)
-    else:
-        p = "%s-ast.h" % mod.name
-    f = open(p, "wb")
-    print >> f, auto_gen_msg
-    print >> f, '#include "asdl.h"\n'
-    c = ChainOfVisitors(TypeDefVisitor(f),
-                        StructVisitor(f),
-                        PrototypeVisitor(f),
-                        )
-    c.visit(mod)
-    print >>f, "PyObject* PyAST_mod2obj(mod_ty t);"
-    f.close()
+        f = open(p, "wb")
+        print >> f, auto_gen_msg
+        print >> f, '#include "asdl.h"\n'
+        c = ChainOfVisitors(TypeDefVisitor(f),
+                            StructVisitor(f),
+                            PrototypeVisitor(f),
+                            )
+        c.visit(mod)
+        print >>f, "PyObject* PyAST_mod2obj(mod_ty t);"
+        f.close()
 
     if SRC_DIR:
-        p = "%s/%s-ast.c" % (SRC_DIR, mod.name)
-    else:
-        p = "%s-ast.c" % mod.name
-    f = open(p, "wb")
-    print >> f, auto_gen_msg
-    print >> f, '#include "Python.h"'
-    print >> f, '#include "%s-ast.h"' % mod.name
-    print >> f
-    print >>f, "static PyTypeObject* AST_type;"
-    v = ChainOfVisitors(
-                        PyTypesDeclareVisitor(f),
-                        PyTypesVisitor(f),
-                        FunctionVisitor(f),
-                        ObjVisitor(f),
-                        ASTModuleVisitor(f),
-                        PartingShots(f),
-                        )
-    v.visit(mod)
-    f.close()
+        p = os.path.join(SRC_DIR, str(mod.name) + "-ast.c")
+        f = open(p, "wb")
+        print >> f, auto_gen_msg
+        print >> f, '#include "Python.h"'
+        print >> f, '#include "%s-ast.h"' % mod.name
+        print >> f
+        print >>f, "static PyTypeObject* AST_type;"
+        v = ChainOfVisitors(
+            PyTypesDeclareVisitor(f),
+            PyTypesVisitor(f),
+            FunctionVisitor(f),
+            ObjVisitor(f),
+            ASTModuleVisitor(f),
+            PartingShots(f),
+            )
+        v.visit(mod)
+        f.close()
 
 if __name__ == "__main__":
     import sys
@@ -757,6 +763,9 @@ if __name__ == "__main__":
     INC_DIR = ''
     SRC_DIR = ''
     opts, args = getopt.getopt(sys.argv[1:], "h:c:")
+    if len(opts) != 1:
+        print "Must specify exactly one output file"
+        sys.exit(1)
     for o, v in opts:
         if o == '-h':
             INC_DIR = v
@@ -764,4 +773,5 @@ if __name__ == "__main__":
             SRC_DIR = v
     if len(args) != 1:
         print "Must specify single input file"
+        sys.exit(1)
     main(args[0])
diff --git a/Parser/bitset.c b/Parser/bitset.c
index 3834e19..b5543b8 100644
--- a/Parser/bitset.c
+++ b/Parser/bitset.c
@@ -8,7 +8,7 @@ bitset
 newbitset(int nbits)
 {
 	int nbytes = NBYTES(nbits);
-	bitset ss = PyMem_NEW(BYTE, nbytes);
+	bitset ss = (char *)PyObject_MALLOC(sizeof(BYTE) *  nbytes);
 	
 	if (ss == NULL)
 		Py_FatalError("no mem for bitset");
@@ -22,7 +22,7 @@ newbitset(int nbits)
 void
 delbitset(bitset ss)
 {
-	PyMem_DEL(ss);
+	PyObject_FREE(ss);
 }
 
 int
diff --git a/Parser/firstsets.c b/Parser/firstsets.c
index 0f4e09d..00467b3 100644
--- a/Parser/firstsets.c
+++ b/Parser/firstsets.c
@@ -59,7 +59,7 @@ calcfirstset(grammar *g, dfa *d)
 	nbits = g->g_ll.ll_nlabels;
 	result = newbitset(nbits);
 	
-	sym = PyMem_NEW(int, 1);
+	sym = (int *)PyObject_MALLOC(sizeof(int));
 	if (sym == NULL)
 		Py_FatalError("no mem for new sym in calcfirstset");
 	nsyms = 1;
@@ -73,7 +73,8 @@ calcfirstset(grammar *g, dfa *d)
 				break;
 		}
 		if (j >= nsyms) { /* New label */
-			PyMem_RESIZE(sym, int, nsyms + 1);
+			sym = (int *)PyObject_REALLOC(sym, 
+                                                sizeof(int) * (nsyms + 1));
 			if (sym == NULL)
 				Py_FatalError(
 				    "no mem to resize sym in calcfirstset");
@@ -108,5 +109,5 @@ calcfirstset(grammar *g, dfa *d)
 		printf(" }\n");
 	}
 
-	PyMem_FREE(sym);
+	PyObject_FREE(sym);
 }
diff --git a/Parser/grammar.c b/Parser/grammar.c
index d8e3897..b0dafe7 100644
--- a/Parser/grammar.c
+++ b/Parser/grammar.c
@@ -20,7 +20,7 @@ newgrammar(int start)
 {
 	grammar *g;
 	
-	g = PyMem_NEW(grammar, 1);
+	g = (grammar *)PyObject_MALLOC(sizeof(grammar));
 	if (g == NULL)
 		Py_FatalError("no mem for new grammar");
 	g->g_ndfas = 0;
@@ -37,7 +37,8 @@ adddfa(grammar *g, int type, char *name)
 {
 	dfa *d;
 	
-	PyMem_RESIZE(g->g_dfa, dfa, g->g_ndfas + 1);
+	g->g_dfa = (dfa *)PyObject_REALLOC(g->g_dfa, 
+                                            sizeof(dfa) * (g->g_ndfas + 1));
 	if (g->g_dfa == NULL)
 		Py_FatalError("no mem to resize dfa in adddfa");
 	d = &g->g_dfa[g->g_ndfas++];
@@ -55,7 +56,8 @@ addstate(dfa *d)
 {
 	state *s;
 	
-	PyMem_RESIZE(d->d_state, state, d->d_nstates + 1);
+	d->d_state = (state *)PyObject_REALLOC(d->d_state,
+				      sizeof(state) * (d->d_nstates + 1));
 	if (d->d_state == NULL)
 		Py_FatalError("no mem to resize state in addstate");
 	s = &d->d_state[d->d_nstates++];
@@ -78,7 +80,7 @@ addarc(dfa *d, int from, int to, int lbl)
 	assert(0 <= to && to < d->d_nstates);
 	
 	s = &d->d_state[from];
-	PyMem_RESIZE(s->s_arc, arc, s->s_narcs + 1);
+	s->s_arc = (arc *)PyObject_REALLOC(s->s_arc, sizeof(arc) * (s->s_narcs + 1));
 	if (s->s_arc == NULL)
 		Py_FatalError("no mem to resize arc list in addarc");
 	a = &s->s_arc[s->s_narcs++];
@@ -97,7 +99,8 @@ addlabel(labellist *ll, int type, char *str)
 			strcmp(ll->ll_label[i].lb_str, str) == 0)
 			return i;
 	}
-	PyMem_RESIZE(ll->ll_label, label, ll->ll_nlabels + 1);
+	ll->ll_label = (label *)PyObject_REALLOC(ll->ll_label,
+					sizeof(label) * (ll->ll_nlabels + 1));
 	if (ll->ll_label == NULL)
 		Py_FatalError("no mem to resize labellist in addlabel");
 	lb = &ll->ll_label[ll->ll_nlabels++];
@@ -195,7 +198,7 @@ translabel(grammar *g, label *lb)
 				name_len = p - src;
 			else
 				name_len = strlen(src);
-			dest = malloc(name_len + 1);
+			dest = (char *)malloc(name_len + 1);
 			strncpy(dest, src, name_len);
 			dest[name_len] = '\0';
 			free(lb->lb_str);
diff --git a/Parser/myreadline.c b/Parser/myreadline.c
index a932a87..32a1088 100644
--- a/Parser/myreadline.c
+++ b/Parser/myreadline.c
@@ -111,7 +111,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
 	size_t n;
 	char *p;
 	n = 100;
-	if ((p = PyMem_MALLOC(n)) == NULL)
+	if ((p = (char *)PyMem_MALLOC(n)) == NULL)
 		return NULL;
 	fflush(sys_stdout);
 #ifndef RISCOS
@@ -141,7 +141,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
 	n = strlen(p);
 	while (n > 0 && p[n-1] != '\n') {
 		size_t incr = n+2;
-		p = PyMem_REALLOC(p, n + incr);
+		p = (char *)PyMem_REALLOC(p, n + incr);
 		if (p == NULL)
 			return NULL;
 		if (incr > INT_MAX) {
@@ -151,7 +151,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
 			break;
 		n += strlen(p+n);
 	}
-	return PyMem_REALLOC(p, n+1);
+	return (char *)PyMem_REALLOC(p, n+1);
 }
 
 
diff --git a/Parser/node.c b/Parser/node.c
index 7ed6c0e..97f887a 100644
--- a/Parser/node.c
+++ b/Parser/node.c
@@ -62,7 +62,7 @@ fancy_roundup(int n)
  * Win98).
  *
  * In a run of compileall across the 2.3a0 Lib directory, Andrew MacIntyre
- * reported that, with this scheme, 89% of PyMem_RESIZE calls in
+ * reported that, with this scheme, 89% of PyObject_REALLOC calls in
  * PyNode_AddChild passed 1 for the size, and 9% passed 4.  So this usually
  * wastes very little memory, but is very effective at sidestepping
  * platform-realloc disasters on vulnernable platforms.
diff --git a/Parser/parser.c b/Parser/parser.c
index 213410c..45302ed 100644
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -75,7 +75,7 @@ PyParser_New(grammar *g, int start)
 	
 	if (!g->g_accel)
 		PyGrammar_AddAccelerators(g);
-	ps = PyMem_NEW(parser_state, 1);
+	ps = (parser_state *)PyMem_MALLOC(sizeof(parser_state));
 	if (ps == NULL)
 		return NULL;
 	ps->p_grammar = g;
@@ -84,7 +84,7 @@ PyParser_New(grammar *g, int start)
 #endif
 	ps->p_tree = PyNode_New(start);
 	if (ps->p_tree == NULL) {
-		PyMem_DEL(ps);
+		PyMem_FREE(ps);
 		return NULL;
 	}
 	s_reset(&ps->p_stack);
@@ -98,7 +98,7 @@ PyParser_Delete(parser_state *ps)
 	/* NB If you want to save the parse tree,
 	   you must set p_tree to NULL before calling delparser! */
 	PyNode_Free(ps->p_tree);
-	PyMem_DEL(ps);
+	PyMem_FREE(ps);
 }
 
 
diff --git a/Parser/pgen.c b/Parser/pgen.c
index e643d33..dfe7cac 100644
--- a/Parser/pgen.c
+++ b/Parser/pgen.c
@@ -49,7 +49,8 @@ addnfastate(nfa *nf)
 {
 	nfastate *st;
 	
-	PyMem_RESIZE(nf->nf_state, nfastate, nf->nf_nstates + 1);
+	nf->nf_state = (nfastate *)PyObject_REALLOC(nf->nf_state, 
+                                    sizeof(nfastate) * (nf->nf_nstates + 1));
 	if (nf->nf_state == NULL)
 		Py_FatalError("out of mem");
 	st = &nf->nf_state[nf->nf_nstates++];
@@ -65,7 +66,8 @@ addnfaarc(nfa *nf, int from, int to, int lbl)
 	nfaarc *ar;
 	
 	st = &nf->nf_state[from];
-	PyMem_RESIZE(st->st_arc, nfaarc, st->st_narcs + 1);
+	st->st_arc = (nfaarc *)PyObject_REALLOC(st->st_arc,
+				      sizeof(nfaarc) * (st->st_narcs + 1));
 	if (st->st_arc == NULL)
 		Py_FatalError("out of mem");
 	ar = &st->st_arc[st->st_narcs++];
@@ -79,7 +81,7 @@ newnfa(char *name)
 	nfa *nf;
 	static int type = NT_OFFSET; /* All types will be disjunct */
 	
-	nf = PyMem_NEW(nfa, 1);
+	nf = (nfa *)PyObject_MALLOC(sizeof(nfa));
 	if (nf == NULL)
 		Py_FatalError("no mem for new nfa");
 	nf->nf_type = type++;
@@ -104,7 +106,7 @@ newnfagrammar(void)
 {
 	nfagrammar *gr;
 	
-	gr = PyMem_NEW(nfagrammar, 1);
+	gr = (nfagrammar *)PyObject_MALLOC(sizeof(nfagrammar));
 	if (gr == NULL)
 		Py_FatalError("no mem for new nfa grammar");
 	gr->gr_nnfas = 0;
@@ -121,7 +123,8 @@ addnfa(nfagrammar *gr, char *name)
 	nfa *nf;
 	
 	nf = newnfa(name);
-	PyMem_RESIZE(gr->gr_nfa, nfa *, gr->gr_nnfas + 1);
+	gr->gr_nfa = (nfa **)PyObject_REALLOC(gr->gr_nfa,
+				      sizeof(nfa) * (gr->gr_nnfas + 1));
 	if (gr->gr_nfa == NULL)
 		Py_FatalError("out of mem");
 	gr->gr_nfa[gr->gr_nnfas++] = nf;
@@ -361,7 +364,7 @@ typedef struct _ss_arc {
 typedef struct _ss_state {
 	bitset	ss_ss;
 	int	ss_narcs;
-	ss_arc	*ss_arc;
+	struct _ss_arc	*ss_arc;
 	int	ss_deleted;
 	int	ss_finish;
 	int	ss_rename;
@@ -392,7 +395,7 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
 	
 	ss = newbitset(nbits);
 	addclosure(ss, nf, nf->nf_start);
-	xx_state = PyMem_NEW(ss_state, 1);
+	xx_state = (ss_state *)PyObject_MALLOC(sizeof(ss_state));
 	if (xx_state == NULL)
 		Py_FatalError("no mem for xx_state in makedfa");
 	xx_nstates = 1;
@@ -411,6 +414,7 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
 
 	/* For each unmarked state... */
 	for (istate = 0; istate < xx_nstates; ++istate) {
+		size_t size;
 		yy = &xx_state[istate];
 		ss = yy->ss_ss;
 		/* For all its states... */
@@ -430,8 +434,9 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
 						goto found;
 				}
 				/* Add new arc for this state */
-				PyMem_RESIZE(yy->ss_arc, ss_arc,
-					     yy->ss_narcs + 1);
+				size = sizeof(ss_arc) * (yy->ss_narcs + 1);
+				yy->ss_arc = (ss_arc *)PyObject_REALLOC(
+                                                            yy->ss_arc, size);
 				if (yy->ss_arc == NULL)
 					Py_FatalError("out of mem");
 				zz = &yy->ss_arc[yy->ss_narcs++];
@@ -453,7 +458,9 @@ makedfa(nfagrammar *gr, nfa *nf, dfa *d)
 					goto done;
 				}
 			}
-			PyMem_RESIZE(xx_state, ss_state, xx_nstates + 1);
+			size = sizeof(ss_state) * (xx_nstates + 1);
+			xx_state = (ss_state *)PyObject_REALLOC(xx_state, 
+                                                                    size);
 			if (xx_state == NULL)
 				Py_FatalError("out of mem");
 			zz->sa_arrow = xx_nstates;
diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c
index 695e2b7..fc27a2c 100644
--- a/Parser/pgenmain.c
+++ b/Parser/pgenmain.c
@@ -104,7 +104,7 @@ getgrammar(char *filename)
 					putc(' ', stderr);
 			}
 			fprintf(stderr, "^\n");
-			PyMem_DEL(err.text);
+			PyObject_FREE(err.text);
 		}
 		Py_Exit(1);
 	}
@@ -136,7 +136,7 @@ char *
 PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
 {
 	size_t n = 1000;
-	char *p = PyMem_MALLOC(n);
+	char *p = (char *)PyMem_MALLOC(n);
 	char *q;
 	if (p == NULL)
 		return NULL;
@@ -149,7 +149,7 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
 	n = strlen(p);
 	if (n > 0 && p[n-1] != '\n')
 		p[n-1] = '\n';
-	return PyMem_REALLOC(p, n+1);
+	return (char *)PyMem_REALLOC(p, n+1);
 }
 
 /* No-nonsense fgets */
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 0631ca3..d9dcc41 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -105,7 +105,8 @@ char *_PyParser_TokenNames[] = {
 static struct tok_state *
 tok_new(void)
 {
-	struct tok_state *tok = PyMem_NEW(struct tok_state, 1);
+	struct tok_state *tok = (struct tok_state *)PyMem_MALLOC(
+                                                sizeof(struct tok_state));
 	if (tok == NULL)
 		return NULL;
 	tok->buf = tok->cur = tok->end = tok->inp = tok->start = NULL;
@@ -163,7 +164,7 @@ error_ret(struct tok_state *tok) /* XXX */
 {
 	tok->decoding_erred = 1;
 	if (tok->fp != NULL && tok->buf != NULL) /* see PyTokenizer_Free */
-		PyMem_DEL(tok->buf);
+		PyMem_FREE(tok->buf);
 	tok->buf = NULL;
 	return NULL;		/* as if it were EOF */
 }
@@ -171,7 +172,7 @@ error_ret(struct tok_state *tok) /* XXX */
 static char *
 new_string(const char *s, Py_ssize_t len)
 {
-	char* result = PyMem_NEW(char, len + 1);
+	char* result = (char *)PyMem_MALLOC(len + 1);
 	if (result != NULL) {
 		memcpy(result, s, len);
 		result[len] = '\0';
@@ -236,7 +237,7 @@ get_coding_spec(const char *s, Py_ssize_t size)
 				char* r = new_string(begin, t - begin);
 				char* q = get_normal_name(r);
 				if (r != q) {
-					PyMem_DEL(r);
+					PyMem_FREE(r);
 					r = new_string(q, strlen(q));
 				}
 				return r;
@@ -277,18 +278,18 @@ check_coding_spec(const char* line, Py_ssize_t size, struct tok_state *tok,
 					tok->decoding_state = -1;
 				}
 				else
-					PyMem_DEL(cs);
+					PyMem_FREE(cs);
 #else
                                 /* Without Unicode support, we cannot
                                    process the coding spec. Since there
                                    won't be any Unicode literals, that
                                    won't matter. */
-				PyMem_DEL(cs);
+				PyMem_FREE(cs);
 #endif
 			}
 		} else {	/* then, compare cs with BOM */
 			r = (strcmp(tok->encoding, cs) == 0);
-			PyMem_DEL(cs);
+			PyMem_FREE(cs);
 		}
 	}
 	if (!r) {
@@ -334,7 +335,7 @@ check_bom(int get_char(struct tok_state *),
 		return 1;
 	}
 	if (tok->encoding != NULL)
-		PyMem_DEL(tok->encoding);
+		PyMem_FREE(tok->encoding);
 	tok->encoding = new_string("utf-8", 5);	/* resulting is in utf-8 */
 	return 1;
   NON_BOM:
@@ -345,7 +346,7 @@ check_bom(int get_char(struct tok_state *),
 
 /* Read a line of text from TOK into S, using the stream in TOK.
    Return NULL on failure, else S.
-   
+
    On entry, tok->decoding_buffer will be one of:
      1) NULL: need to call tok->decoding_readline to get a new line
      2) PyUnicodeObject *: decoding_feof has called tok->decoding_readline and
@@ -354,7 +355,7 @@ check_bom(int get_char(struct tok_state *),
            (in the s buffer) to copy entire contents of the line read
            by tok->decoding_readline.  tok->decoding_buffer has the overflow.
            In this case, fp_readl is called in a loop (with an expanded buffer)
-           until the buffer ends with a '\n' (or until the end of the file is 
+           until the buffer ends with a '\n' (or until the end of the file is
            reached): see tok_nextc and its calls to decoding_fgets.
 */
 
@@ -470,7 +471,7 @@ decoding_fgets(char *s, int size, struct tok_state *tok)
 			break;
 		} else if (tok->decoding_state > 0) {
 			/* We want a 'raw' read. */
-			line = Py_UniversalNewlineFgets(s, size, 
+			line = Py_UniversalNewlineFgets(s, size,
 							tok->fp, NULL);
 			break;
 		} else {
@@ -502,11 +503,11 @@ decoding_fgets(char *s, int size, struct tok_state *tok)
 		char buf[500];
 		/* Need to add 1 to the line number, since this line
 		   has not been counted, yet.  */
-		sprintf(buf, 
+		sprintf(buf,
 			"Non-ASCII character '\\x%.2x' "
 			"in file %.200s on line %i, "
 			"but no encoding declared; "
-			"see http://www.python.org/peps/pep-0263.html for details", 
+			"see http://www.python.org/peps/pep-0263.html for details",
 			badchar, tok->filename, tok->lineno + 1);
 		PyErr_SetString(PyExc_SyntaxError, buf);
 		return error_ret(tok);
@@ -537,13 +538,15 @@ decoding_feof(struct tok_state *tok)
 
 /* Fetch a byte from TOK, using the string buffer. */
 
-static int buf_getc(struct tok_state *tok) {
+static int
+buf_getc(struct tok_state *tok) {
 	return Py_CHARMASK(*tok->str++);
 }
 
 /* Unfetch a byte from TOK, using the string buffer. */
 
-static void buf_ungetc(int c, struct tok_state *tok) {
+static void
+buf_ungetc(int c, struct tok_state *tok) {
 	tok->str--;
 	assert(Py_CHARMASK(*tok->str) == c);	/* tok->cur may point to read-only segment */
 }
@@ -551,7 +554,8 @@ static void buf_ungetc(int c, struct tok_state *tok) {
 /* Set the readline function for TOK to ENC. For the string-based
    tokenizer, this means to just record the encoding. */
 
-static int buf_setreadl(struct tok_state *tok, const char* enc) {
+static int
+buf_setreadl(struct tok_state *tok, const char* enc) {
 	tok->enc = enc;
 	return 1;
 }
@@ -653,7 +657,7 @@ PyTokenizer_FromFile(FILE *fp, char *ps1, char *ps2)
 	struct tok_state *tok = tok_new();
 	if (tok == NULL)
 		return NULL;
-	if ((tok->buf = PyMem_NEW(char, BUFSIZ)) == NULL) {
+	if ((tok->buf = (char *)PyMem_MALLOC(BUFSIZ)) == NULL) {
 		PyTokenizer_Free(tok);
 		return NULL;
 	}
@@ -672,14 +676,14 @@ void
 PyTokenizer_Free(struct tok_state *tok)
 {
 	if (tok->encoding != NULL)
-		PyMem_DEL(tok->encoding);
+		PyMem_FREE(tok->encoding);
 #ifndef PGEN
 	Py_XDECREF(tok->decoding_readline);
 	Py_XDECREF(tok->decoding_buffer);
 #endif
 	if (tok->fp != NULL && tok->buf != NULL)
-		PyMem_DEL(tok->buf);
-	PyMem_DEL(tok);
+		PyMem_FREE(tok->buf);
+	PyMem_FREE(tok);
 }
 
 #if !defined(PGEN) && defined(Py_USING_UNICODE)
@@ -711,7 +715,9 @@ tok_stdin_decode(struct tok_state *tok, char **inp)
 	if (utf8 == NULL)
 		goto error_clear;
 
-	converted = new_string(PyString_AsString(utf8), PyString_Size(utf8));
+	assert(PyString_Check(utf8));
+	converted = new_string(PyString_AS_STRING(utf8),
+			       PyString_GET_SIZE(utf8));
 	Py_DECREF(utf8);
 	if (converted == NULL)
 		goto error_nomem;
@@ -719,7 +725,7 @@ tok_stdin_decode(struct tok_state *tok, char **inp)
 	PyMem_FREE(*inp);
 	*inp = converted;
 	if (tok->encoding != NULL)
-		PyMem_DEL(tok->encoding);
+		PyMem_FREE(tok->encoding);
 	tok->encoding = new_string(encoding, strlen(encoding));
 	if (tok->encoding == NULL)
 		goto error_nomem;
@@ -770,38 +776,38 @@ tok_nextc(register struct tok_state *tok)
 			return Py_CHARMASK(*tok->cur++);
 		}
 		if (tok->prompt != NULL) {
-			char *new = PyOS_Readline(stdin, stdout, tok->prompt);
+			char *newtok = PyOS_Readline(stdin, stdout, tok->prompt);
 			if (tok->nextprompt != NULL)
 				tok->prompt = tok->nextprompt;
-			if (new == NULL)
+			if (newtok == NULL)
 				tok->done = E_INTR;
-			else if (*new == '\0') {
-				PyMem_FREE(new);
+			else if (*newtok == '\0') {
+				PyMem_FREE(newtok);
 				tok->done = E_EOF;
 			}
 #if !defined(PGEN) && defined(Py_USING_UNICODE)
-			else if (tok_stdin_decode(tok, &new) != 0)
-				PyMem_FREE(new);
+			else if (tok_stdin_decode(tok, &newtok) != 0)
+				PyMem_FREE(newtok);
 #endif
 			else if (tok->start != NULL) {
 				size_t start = tok->start - tok->buf;
 				size_t oldlen = tok->cur - tok->buf;
-				size_t newlen = oldlen + strlen(new);
+				size_t newlen = oldlen + strlen(newtok);
 				char *buf = tok->buf;
-				PyMem_RESIZE(buf, char, newlen+1);
+				buf = (char *)PyMem_REALLOC(buf, newlen+1);
 				tok->lineno++;
 				if (buf == NULL) {
-					PyMem_DEL(tok->buf);
+					PyMem_FREE(tok->buf);
 					tok->buf = NULL;
-					PyMem_FREE(new);
+					PyMem_FREE(newtok);
 					tok->done = E_NOMEM;
 					return EOF;
 				}
 				tok->buf = buf;
 				tok->cur = tok->buf + oldlen;
 				tok->line_start = tok->cur;
-				strcpy(tok->buf + oldlen, new);
-				PyMem_FREE(new);
+				strcpy(tok->buf + oldlen, newtok);
+				PyMem_FREE(newtok);
 				tok->inp = tok->buf + newlen;
 				tok->end = tok->inp + 1;
 				tok->start = tok->buf + start;
@@ -809,8 +815,8 @@ tok_nextc(register struct tok_state *tok)
 			else {
 				tok->lineno++;
 				if (tok->buf != NULL)
-					PyMem_DEL(tok->buf);
-				tok->buf = new;
+					PyMem_FREE(tok->buf);
+				tok->buf = newtok;
 				tok->line_start = tok->buf;
 				tok->cur = tok->buf;
 				tok->line_start = tok->buf;
@@ -824,7 +830,8 @@ tok_nextc(register struct tok_state *tok)
 			char *pt;
 			if (tok->start == NULL) {
 				if (tok->buf == NULL) {
-					tok->buf = PyMem_NEW(char, BUFSIZ);
+					tok->buf = (char *)
+						PyMem_MALLOC(BUFSIZ);
 					if (tok->buf == NULL) {
 						tok->done = E_NOMEM;
 						return EOF;
@@ -859,7 +866,8 @@ tok_nextc(register struct tok_state *tok)
 				Py_ssize_t curvalid = tok->inp - tok->buf;
 				Py_ssize_t newsize = curvalid + BUFSIZ;
 				char *newbuf = tok->buf;
-				PyMem_RESIZE(newbuf, char, newsize);
+				newbuf = (char *)PyMem_REALLOC(newbuf,
+							       newsize);
 				if (newbuf == NULL) {
 					tok->done = E_NOMEM;
 					tok->cur = tok->inp;
@@ -1182,9 +1190,9 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 			}
 		}
 	}
-	
+
 	tok->start = tok->cur;
-	
+
 	/* Return pending indents/dedents */
 	if (tok->pendin != 0) {
 		if (tok->pendin < 0) {
@@ -1196,27 +1204,27 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 			return INDENT;
 		}
 	}
-	
+
  again:
 	tok->start = NULL;
 	/* Skip spaces */
 	do {
 		c = tok_nextc(tok);
 	} while (c == ' ' || c == '\t' || c == '\014');
-	
+
 	/* Set start of current token */
 	tok->start = tok->cur - 1;
-	
+
 	/* Skip comment */
 	if (c == '#')
 		while (c != EOF && c != '\n')
 			c = tok_nextc(tok);
-	
+
 	/* Check for EOF and errors now */
 	if (c == EOF) {
 		return tok->done == E_EOF ? ENDMARKER : ERRORTOKEN;
 	}
-	
+
 	/* Identifier (most frequent token!) */
 	if (isalpha(c) || c == '_') {
 		/* Process r"", u"" and ur"" */
@@ -1244,7 +1252,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 		*p_end = tok->cur;
 		return NAME;
 	}
-	
+
 	/* Newline */
 	if (c == '\n') {
 		tok->atbol = 1;
@@ -1255,7 +1263,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
                 tok->cont_line = 0;
 		return NEWLINE;
 	}
-	
+
 	/* Period or number starting with period? */
 	if (c == '.') {
 		c = tok_nextc(tok);
@@ -1418,7 +1426,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 		*p_end = tok->cur;
 		return STRING;
 	}
-	
+
 	/* Line continuation */
 	if (c == '\\') {
 		c = tok_nextc(tok);
@@ -1430,7 +1438,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
                 tok->cont_line = 1;
 		goto again; /* Read next line */
 	}
-	
+
 	/* Check for two-character token */
 	{
 		int c2 = tok_nextc(tok);
@@ -1449,7 +1457,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 		}
 		tok_backup(tok, c2);
 	}
-	
+
 	/* Keep track of parentheses nesting level */
 	switch (c) {
 	case '(':
@@ -1463,7 +1471,7 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
 		tok->level--;
 		break;
 	}
-	
+
 	/* Punctuation character */
 	*p_start = tok->start;
 	*p_end = tok->cur;