diff options
Diffstat (limited to 'Doc/library/re.rst')
| -rw-r--r-- | Doc/library/re.rst | 3 |
1 files changed, 3 insertions, 0 deletions
diff --git a/Doc/library/re.rst b/Doc/library/re.rst index c627f1a..423540c 100644 --- a/Doc/library/re.rst +++ b/Doc/library/re.rst @@ -1300,6 +1300,7 @@ successive matches:: Token = collections.namedtuple('Token', 'typ value line column') def tokenize(s): + keywords = {'IF', 'THEN', 'FOR', 'NEXT', 'GOSUB', 'RETURN'} tok_spec = [ ('NUMBER', r'\d+(\.\d*)?'), # Integer or decimal number ('ASSIGN', r':='), # Assignment operator @@ -1320,6 +1321,8 @@ successive matches:: line_start = pos line += 1 elif typ != 'SKIP': + if typ == 'ID' and val in keywords: + typ = val yield Token(typ, mo.group(typ), line, mo.start()-line_start) pos = mo.end() mo = gettok(s, pos) |