summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGuido van Rossum <guido@python.org>2000-03-10 22:56:54 (GMT)
committerGuido van Rossum <guido@python.org>2000-03-10 22:56:54 (GMT)
commit86016cb4828a73c28ea3ae7ed3c76996216c03b3 (patch)
tree550a83742d883d92e8cf26bb2de652bf1e8937a4
parent4aa1e63e4c955ec7d2777685b62c68d3aebf760b (diff)
downloadcpython-86016cb4828a73c28ea3ae7ed3c76996216c03b3.zip
cpython-86016cb4828a73c28ea3ae7ed3c76996216c03b3.tar.gz
cpython-86016cb4828a73c28ea3ae7ed3c76996216c03b3.tar.bz2
Marc-Andre Lemburg: add new string token types u"..." and ur"..."
(Unicode and raw Unicode).
-rw-r--r--Parser/tokenizer.c10
1 files changed, 10 insertions, 0 deletions
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 08b6ee9..ff564ee 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -591,12 +591,22 @@ PyTokenizer_Get(tok, p_start, p_end)
/* Identifier (most frequent token!) */
if (isalpha(c) || c == '_') {
+ /* Process r"", u"" and ur"" */
switch (c) {
case 'r':
case 'R':
c = tok_nextc(tok);
if (c == '"' || c == '\'')
goto letter_quote;
+ break;
+ case 'u':
+ case 'U':
+ c = tok_nextc(tok);
+ if (c == 'r' || c == 'R')
+ c = tok_nextc(tok);
+ if (c == '"' || c == '\'')
+ goto letter_quote;
+ break;
}
while (isalnum(c) || c == '_') {
c = tok_nextc(tok);