diff options
| author | Guido van Rossum <guido@python.org> | 2000-03-10 22:56:54 (GMT) |
|---|---|---|
| committer | Guido van Rossum <guido@python.org> | 2000-03-10 22:56:54 (GMT) |
| commit | 86016cb4828a73c28ea3ae7ed3c76996216c03b3 (patch) | |
| tree | 550a83742d883d92e8cf26bb2de652bf1e8937a4 /Parser/tokenizer.c | |
| parent | 4aa1e63e4c955ec7d2777685b62c68d3aebf760b (diff) | |
| download | cpython-86016cb4828a73c28ea3ae7ed3c76996216c03b3.zip cpython-86016cb4828a73c28ea3ae7ed3c76996216c03b3.tar.gz cpython-86016cb4828a73c28ea3ae7ed3c76996216c03b3.tar.bz2 | |
Marc-Andre Lemburg: add new string token types u"..." and ur"..."
(Unicode and raw Unicode).
Diffstat (limited to 'Parser/tokenizer.c')
| -rw-r--r-- | Parser/tokenizer.c | 10 |
1 files changed, 10 insertions, 0 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c index 08b6ee9..ff564ee 100644 --- a/Parser/tokenizer.c +++ b/Parser/tokenizer.c @@ -591,12 +591,22 @@ PyTokenizer_Get(tok, p_start, p_end) /* Identifier (most frequent token!) */ if (isalpha(c) || c == '_') { + /* Process r"", u"" and ur"" */ switch (c) { case 'r': case 'R': c = tok_nextc(tok); if (c == '"' || c == '\'') goto letter_quote; + break; + case 'u': + case 'U': + c = tok_nextc(tok); + if (c == 'r' || c == 'R') + c = tok_nextc(tok); + if (c == '"' || c == '\'') + goto letter_quote; + break; } while (isalnum(c) || c == '_') { c = tok_nextc(tok); |