Merged revisions 78603 via svnmerge from

svn+ssh://pythondev@svn.python.org/python/trunk

........
  r78603 | victor.stinner | 2010-03-03 00:20:02 +0100 (mer., 03 mars 2010) | 5 lines

  Issue #7820: The parser tokenizer restores all bytes in the right if the BOM
  check fails.

  Fix an assertion in pydebug mode.
........

1 files changed, 28 insertions, 23 deletions

diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 5a1c268..50b0ff1 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -318,46 +318,51 @@ check_bom(int get_char(struct tok_state *),
 	  int set_readline(struct tok_state *, const char *),
 	  struct tok_state *tok)
 {
-	int ch = get_char(tok);
+	int ch1, ch2, ch3;
+	ch1 = get_char(tok);
 	tok->decoding_state = STATE_RAW;
-	if (ch == EOF) {
+	if (ch1 == EOF) {
 		return 1;
-	} else if (ch == 0xEF) {
-		ch = get_char(tok); 
-		if (ch != 0xBB) {
-			unget_char(ch, tok);
-			unget_char(0xEF, tok);
-			/* any token beginning with '\xEF' is a bad token */
+	} else if (ch1 == 0xEF) {
+		ch2 = get_char(tok);
+		if (ch2 != 0xBB) {
+			unget_char(ch2, tok);
+			unget_char(ch1, tok);
 			return 1;
 		}
-		ch = get_char(tok); 
-		if (ch != 0xBF) {
-			unget_char(ch, tok);
-			unget_char(0xBB, tok);
-			unget_char(0xEF, tok);
-			/* any token beginning with '\xEF' is a bad token */
+		ch3 = get_char(tok);
+		if (ch3 != 0xBF) {
+			unget_char(ch3, tok);
+			unget_char(ch2, tok);
+			unget_char(ch1, tok);
 			return 1;
 		}
 #if 0
 	/* Disable support for UTF-16 BOMs until a decision
 	   is made whether this needs to be supported.  */
-	} else if (ch == 0xFE) {
-		ch = get_char(tok);
-		if (ch != 0xFF)
-			goto NON_BOM;
+	} else if (ch1 == 0xFE) {
+		ch2 = get_char(tok);
+		if (ch2 != 0xFF) {
+			unget_char(ch2, tok);
+			unget_char(ch1, tok);
+			return 1;
+		}
 		if (!set_readline(tok, "utf-16-be"))
 			return 0;
 		tok->decoding_state = STATE_NORMAL;
-	} else if (ch == 0xFF) {
-		ch = get_char(tok);
-		if (ch != 0xFE)
-			goto NON_BOM;
+	} else if (ch1 == 0xFF) {
+		ch2 = get_char(tok);
+		if (ch2 != 0xFE) {
+			unget_char(ch2, tok);
+			unget_char(ch1, tok);
+			return 1;
+		}
 		if (!set_readline(tok, "utf-16-le"))
 			return 0;
 		tok->decoding_state = STATE_NORMAL;
 #endif
 	} else {
-		unget_char(ch, tok);
+		unget_char(ch1, tok);
 		return 1;
 	}
 	if (tok->encoding != NULL)