diff options
Diffstat (limited to 'Tools/i18n')
| -rwxr-xr-x | Tools/i18n/pygettext.py | 4 |
1 files changed, 3 insertions, 1 deletions
diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index bdf52e0..69a19ef 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -631,7 +631,9 @@ def main(): try: eater.set_filename(filename) try: - tokenize.tokenize(fp.readline, eater) + tokens = tokenize.generate_tokens(fp.readline) + for _token in tokens: + eater(*_token) except tokenize.TokenError as e: print('%s: %s, line %d, column %d' % ( e.args[0], filename, e.args[1][0], e.args[1][1]), |