diff options
Diffstat (limited to 'Tools/scripts')
| -rwxr-xr-x | Tools/scripts/checkappend.py | 4 | ||||
| -rwxr-xr-x | Tools/scripts/reindent.py | 4 |
2 files changed, 6 insertions, 2 deletions
diff --git a/Tools/scripts/checkappend.py b/Tools/scripts/checkappend.py index 8953285..4c74ee5 100755 --- a/Tools/scripts/checkappend.py +++ b/Tools/scripts/checkappend.py @@ -103,7 +103,9 @@ class AppendChecker: def run(self): try: - tokenize.tokenize(self.file.readline, self.tokeneater) + tokens = tokenize.generate_tokens(self.file.readline) + for _token in tokens: + self.tokeneater(*_token) except tokenize.TokenError as msg: errprint("%r: Token Error: %s" % (self.fname, msg)) self.nerrors = self.nerrors + 1 diff --git a/Tools/scripts/reindent.py b/Tools/scripts/reindent.py index 981f63a..9a55198 100755 --- a/Tools/scripts/reindent.py +++ b/Tools/scripts/reindent.py @@ -173,7 +173,9 @@ class Reindenter: self.stats = [] def run(self): - tokenize.tokenize(self.getline, self.tokeneater) + tokens = tokenize.generate_tokens(self.getline) + for _token in tokens: + self.tokeneater(*_token) # Remove trailing empty lines. lines = self.lines while lines and lines[-1] == "\n": |