* pythonrun.c: Print exception type+arg *after* stack trace instead of

  before it.
* ceval.c, object.c: moved testbool() to object.c (now extern visible)
* stringobject.c: fix bugs in and rationalize string resize in formatstring()
* tokenizer.[ch]: fix non-working code for lines longer than BUFSIZ

7 files changed, 95 insertions, 81 deletions

diff --git a/Objects/object.c b/Objects/object.c
index 9a8d4b3..e28158e 100644
--- a/Objects/object.c
+++ b/Objects/object.c
@@ -213,6 +213,29 @@ setattr(v, name, w)
 	}
 }
 
+/* Test a value used as condition, e.g., in a for or if statement.
+   Return -1 if an error occurred */
+
+int
+testbool(v)
+	object *v;
+{
+	int res;
+	if (v == None)
+		res = 0;
+	else if (v->ob_type->tp_as_number != NULL)
+		res = (*v->ob_type->tp_as_number->nb_nonzero)(v);
+	else if (v->ob_type->tp_as_mapping != NULL)
+		res = (*v->ob_type->tp_as_mapping->mp_length)(v);
+	else if (v->ob_type->tp_as_sequence != NULL)
+		res = (*v->ob_type->tp_as_sequence->sq_length)(v);
+	else
+		res = 1;
+	if (res > 0)
+		res = 1;
+	return res;
+}
+
 
 /*
 NoObject is usable as a non-NULL undefined value, used by the macro None.
diff --git a/Objects/stringobject.c b/Objects/stringobject.c
index 5c7345d..33fe485 100644
--- a/Objects/stringobject.c
+++ b/Objects/stringobject.c
@@ -491,13 +491,13 @@ formatstring(format, args)
 		err_badcall();
 		return NULL;
 	}
-	reslen = rescnt = 100;
+	fmt = getstringvalue(format);
+	fmtcnt = getstringsize(format);
+	reslen = rescnt = fmtcnt + 100;
 	result = newsizedstringobject((char *)NULL, reslen);
 	if (result == NULL)
 		return NULL;
 	res = getstringvalue(result);
-	fmt = getstringvalue(format);
-	fmtcnt = getstringsize(format);
 	if (is_tupleobject(args)) {
 		arglen = gettuplesize(args);
 		argidx = 0;
@@ -509,12 +509,11 @@ formatstring(format, args)
 	while (--fmtcnt >= 0) {
 		if (*fmt != '%') {
 			if (--rescnt < 0) {
-				rescnt = reslen;
-				reslen = reslen * 2; /* Maybe less when big? */
+				rescnt = fmtcnt + 100;
+				reslen += rescnt;
 				if (resizestring(&result, reslen) < 0)
 					return NULL;
-				res = getstringvalue(result) + rescnt;
-				rescnt = reslen - rescnt;
+				res = getstringvalue(result) + reslen - rescnt;
 			}
 			*res++ = *fmt++;
 		}
@@ -692,12 +691,12 @@ formatstring(format, args)
 			if (width < len)
 				width = len;
 			if (rescnt < width + (sign != '\0')) {
-				rescnt = reslen;
-				reslen = reslen + width + 100;
+				reslen -= rescnt;
+				rescnt = width + fmtcnt + 100;
+				reslen += rescnt;
 				if (resizestring(&result, reslen) < 0)
 					return NULL;
-				res = getstringvalue(result) + rescnt;
-				rescnt = reslen - rescnt;
+				res = getstringvalue(result) + reslen - rescnt;
 			}
 			if (sign) {
 				*res++ = sign;
diff --git a/Objects/xxobject.c b/Objects/xxobject.c
index 2d03cfd..64e0228 100644
--- a/Objects/xxobject.c
+++ b/Objects/xxobject.c
@@ -137,4 +137,5 @@ static typeobject Xxtype = {
 	0,			/*tp_as_number*/
 	0,			/*tp_as_sequence*/
 	0,			/*tp_as_mapping*/
+	0,			/*tp_hash*/
 };
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 22c20b0..38a9e9a 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -170,31 +170,15 @@ static int
 tok_nextc(tok)
 	register struct tok_state *tok;
 {
-	if (tok->done != E_OK)
-		return EOF;
-	
 	for (;;) {
-		if (tok->cur < tok->inp)
-			return *tok->cur++;
+		if (tok->cur != tok->inp)
+			return *tok->cur++; /* Fast path */
+		if (tok->done != E_OK)
+			return EOF;
 		if (tok->fp == NULL) {
 			tok->done = E_EOF;
 			return EOF;
 		}
-		if (tok->inp > tok->buf && tok->inp[-1] == '\n')
-			tok->inp = tok->buf;
-		if (tok->inp == tok->end) {
-			int n = tok->end - tok->buf;
-			char *new = tok->buf;
-			RESIZE(new, char, n+n);
-			if (new == NULL) {
-				fprintf(stderr, "tokenizer out of mem\n");
-				tok->done = E_NOMEM;
-				return EOF;
-			}
-			tok->buf = new;
-			tok->inp = tok->buf + n;
-			tok->end = tok->inp + n;
-		}
 #ifdef USE_READLINE
 		if (tok->prompt != NULL) {
 			extern char *readline PROTO((char *prompt));
@@ -211,46 +195,71 @@ tok_nextc(tok)
 			(void) intrcheck(); /* Clear pending interrupt */
 			if (tok->nextprompt != NULL)
 				tok->prompt = tok->nextprompt;
-				/* XXX different semantics w/o readline()! */
 			if (tok->buf == NULL) {
 				tok->done = E_EOF;
 			}
 			else {
-				unsigned int n = strlen(tok->buf);
-				if (n > 0)
+				tok->end = strchr(tok->buf, '\0');
+				if (tok->end > tok->buf)
 					add_history(tok->buf);
-				/* Append the '\n' that readline()
-				   doesn't give us, for the tokenizer... */
-				tok->buf = realloc(tok->buf, n+2);
-				if (tok->buf == NULL)
-					tok->done = E_NOMEM;
-				else {
-					tok->end = tok->buf + n;
-					*tok->end++ = '\n';
-					*tok->end = '\0';
-					tok->inp = tok->end;
-					tok->cur = tok->buf;
-				}
+				/* Replace trailing '\n' by '\0'
+				   (we don't need a '\0', but the
+				   tokenizer wants a '\n'...) */
+				*tok->end++ = '\n';
+				tok->inp = tok->end;
+				tok->cur = tok->buf;
 			}
 		}
 		else
 #endif
 		{
-			tok->cur = tok->inp;
-			if (tok->prompt != NULL && tok->inp == tok->buf) {
+			if (tok->prompt != NULL) {
 				fprintf(stderr, "%s", tok->prompt);
-				tok->prompt = tok->nextprompt;
+				if (tok->nextprompt != NULL)
+					tok->prompt = tok->nextprompt;
 			}
-			tok->done = fgets_intr(tok->inp,
-				(int)(tok->end - tok->inp), tok->fp);
+			if (tok->buf == NULL) {
+				tok->buf = NEW(char, BUFSIZ);
+				if (tok->buf == NULL) {
+					tok->done = E_NOMEM;
+					return EOF;
+				}
+				tok->end = tok->buf + BUFSIZ;
+			}
+			tok->done = fgets_intr(tok->buf,
+				(int)(tok->end - tok->buf), tok->fp);
+			tok->inp = strchr(tok->buf, '\0');
+			/* Read until '\n' or EOF */
+			while (tok->inp+1==tok->end && tok->inp[-1]!='\n') {
+				int curvalid = tok->inp - tok->buf;
+				int cursize = tok->end - tok->buf;
+				int newsize = cursize + BUFSIZ;
+				char *newbuf = tok->buf;
+				RESIZE(newbuf, char, newsize);
+				if (newbuf == NULL) {
+					tok->done = E_NOMEM;
+					tok->cur = tok->inp;
+					return EOF;
+				}
+				tok->buf = newbuf;
+				tok->inp = tok->buf + curvalid;
+				tok->end = tok->buf + newsize;
+				if (fgets_intr(tok->inp,
+					       (int)(tok->end - tok->inp),
+					       tok->fp) != E_OK)
+					break;
+				tok->inp = strchr(tok->inp, '\0');
+			}
+			tok->cur = tok->buf;
 		}
 		if (tok->done != E_OK) {
 			if (tok->prompt != NULL)
 				fprintf(stderr, "\n");
+			tok->cur = tok->inp;
 			return EOF;
 		}
-		tok->inp = strchr(tok->inp, '\0');
 	}
+	/*NOTREACHED*/
 }
 
 
@@ -390,6 +399,7 @@ tok_get(tok, p_start, p_end)
 				if (tok->indent+1 >= MAXINDENT) {
 					fprintf(stderr, "excessive indent\n");
 					tok->done = E_TOKEN;
+					tok->cur = tok->inp;
 					return ERRORTOKEN;
 				}
 				tok->pendin++;
@@ -405,6 +415,7 @@ tok_get(tok, p_start, p_end)
 				if (col != tok->indstack[tok->indent]) {
 					fprintf(stderr, "inconsistent dedent\n");
 					tok->done = E_TOKEN;
+					tok->cur = tok->inp;
 					return ERRORTOKEN;
 				}
 			}
@@ -558,6 +569,7 @@ tok_get(tok, p_start, p_end)
 			c = tok_nextc(tok);
 			if (c == '\n' || c == EOF) {
 				tok->done = E_TOKEN;
+				tok->cur = tok->inp;
 				return ERRORTOKEN;
 			}
 			if (c == '\\') {
@@ -565,6 +577,7 @@ tok_get(tok, p_start, p_end)
 				*p_end = tok->cur;
 				if (c == '\n' || c == EOF) {
 					tok->done = E_TOKEN;
+					tok->cur = tok->inp;
 					return ERRORTOKEN;
 				}
 				continue;
@@ -581,6 +594,7 @@ tok_get(tok, p_start, p_end)
 		c = tok_nextc(tok);
 		if (c != '\n') {
 			tok->done = E_TOKEN;
+			tok->cur = tok->inp;
 			return ERRORTOKEN;
 		}
 		tok->lineno++;
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index 636295a..9abc598 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -31,12 +31,13 @@ OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 /* Tokenizer state */
 struct tok_state {
 	/* Input state; buf <= cur <= inp <= end */
-	/* NB an entire token must fit in the buffer */
-	char *buf;	/* Input buffer */
+	/* NB an entire line is held in the buffer */
+	char *buf;	/* Input buffer, or NULL; malloc'ed if fp != NULL */
 	char *cur;	/* Next character in buffer */
 	char *inp;	/* End of data in buffer */
-	char *end;	/* End of input buffer */
-	int done;	/* 0 normally, 1 at EOF, -1 after error */
+	char *end;	/* End of input buffer if buf != NULL */
+	int done;	/* E_OK normally, E_EOF at EOF, otherwise error code */
+	/* NB If done != E_OK, cur must be == inp!!! */
 	FILE *fp;	/* Rest of input; NULL if tokenizing a string */
 	int tabsize;	/* Tab spacing */
 	int indent;	/* Current indentation index */
diff --git a/Python/ceval.c b/Python/ceval.c
index 935ce52..44239e6 100644
--- a/Python/ceval.c
+++ b/Python/ceval.c
@@ -58,7 +58,6 @@ static int prtrace PROTO((object *, char *));
 static void call_exc_trace PROTO((object **, object**, frameobject *));
 static int call_trace
 	PROTO((object **, object **, frameobject *, char *, object *));
-static int testbool PROTO((object *));
 static object *add PROTO((object *, object *));
 static object *sub PROTO((object *, object *));
 static object *mul PROTO((object *, object *));
@@ -1612,29 +1611,6 @@ flushline()
 }
 
 
-/* Test a value used as condition, e.g., in a for or if statement.
-   Return -1 if an error occurred */
-
-static int
-testbool(v)
-	object *v;
-{
-	int res;
-	if (v == None)
-		res = 0;
-	else if (v->ob_type->tp_as_number != NULL)
-		res = (*v->ob_type->tp_as_number->nb_nonzero)(v);
-	else if (v->ob_type->tp_as_mapping != NULL)
-		res = (*v->ob_type->tp_as_mapping->mp_length)(v);
-	else if (v->ob_type->tp_as_sequence != NULL)
-		res = (*v->ob_type->tp_as_sequence->sq_length)(v);
-	else
-		res = 1;
-	if (res > 0)
-		res = 1;
-	return res;
-}
-
 static object *
 or(v, w)
 	object *v, *w;
diff --git a/Python/pythonrun.c b/Python/pythonrun.c
index 98008b4..3898d13 100644
--- a/Python/pythonrun.c
+++ b/Python/pythonrun.c
@@ -245,6 +245,7 @@ print_error()
 	if (f == NULL)
 		fprintf(stderr, "lost sys.stderr\n");
 	else {
+		printtraceback(f);
 		if (writeobject(exception, f, PRINT_RAW) != 0)
 			err_clear();
 		if (v != NULL && v != None) {
@@ -253,7 +254,6 @@ print_error()
 				err_clear();
 		}
 		writestring("\n", f);
-		printtraceback(f);
 	}
 	XDECREF(exception);
 	XDECREF(v);