diff options
Diffstat (limited to 'Parser')
| -rw-r--r-- | Parser/tokenizer.c | 34 |
1 files changed, 23 insertions, 11 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index cc142a7..c1a6014 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -195,20 +195,26 @@ get_normal_name(char *s) /* for utf-8 and latin-1 */ int i; for (i = 0; i < 12; i++) { int c = s[i]; - if (c == '\0') break; - else if (c == '_') buf[i] = '-'; - else buf[i] = tolower(c); + if (c == '\0') + break; + else if (c == '_') + buf[i] = '-'; + else + buf[i] = tolower(c); } buf[i] = '\0'; if (strcmp(buf, "utf-8") == 0 || - strncmp(buf, "utf-8-", 6) == 0) return "utf-8"; + strncmp(buf, "utf-8-", 6) == 0) + return "utf-8"; else if (strcmp(buf, "latin-1") == 0 || strcmp(buf, "iso-8859-1") == 0 || strcmp(buf, "iso-latin-1") == 0 || strncmp(buf, "latin-1-", 8) == 0 || strncmp(buf, "iso-8859-1-", 11) == 0 || - strncmp(buf, "iso-latin-1-", 12) == 0) return "iso-8859-1"; - else return s; + strncmp(buf, "iso-latin-1-", 12) == 0) + return "iso-8859-1"; + else + return s; } /* Return the coding spec in S, or NULL if none is found. */ @@ -334,12 +340,18 @@ check_bom(int get_char(struct tok_state *), /* Disable support for UTF-16 BOMs until a decision is made whether this needs to be supported. */ } else if (ch == 0xFE) { - ch = get_char(tok); if (ch != 0xFF) goto NON_BOM; - if (!set_readline(tok, "utf-16-be")) return 0; + ch = get_char(tok); + if (ch != 0xFF) + goto NON_BOM; + if (!set_readline(tok, "utf-16-be")) + return 0; tok->decoding_state = STATE_NORMAL; } else if (ch == 0xFF) { - ch = get_char(tok); if (ch != 0xFE) goto NON_BOM; - if (!set_readline(tok, "utf-16-le")) return 0; + ch = get_char(tok); + if (ch != 0xFE) + goto NON_BOM; + if (!set_readline(tok, "utf-16-le")) + return 0; tok->decoding_state = STATE_NORMAL; #endif } else { @@ -980,7 +992,7 @@ tok_backup(register struct tok_state *tok, register int c) { if (c != EOF) { if (--tok->cur < tok->buf) - Py_FatalError("tok_backup: begin of buffer"); + Py_FatalError("tok_backup: beginning of buffer"); if (*tok->cur != c) *tok->cur = c; } |