From 15bd9efd01e44087664e78bf766865a6d2e06626 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sun, 24 Jan 2021 01:23:17 -0800 Subject: bpo-43014: Improve performance of tokenize.tokenize by 20-30% --- Lib/tokenize.py | 2 ++ Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst | 1 + 2 files changed, 3 insertions(+) create mode 100644 Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 1aee21b..42c1f10 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -27,6 +27,7 @@ __credits__ = ('GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, ' from builtins import open as _builtin_open from codecs import lookup, BOM_UTF8 import collections +import functools from io import TextIOWrapper import itertools as _itertools import re @@ -95,6 +96,7 @@ def _all_string_prefixes(): result.add(''.join(u)) return result +@functools.lru_cache def _compile(expr): return re.compile(expr, re.UNICODE) diff --git a/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst new file mode 100644 index 0000000..02898e4 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2021-01-24-00-37-40.bpo-43014.BVPhEr.rst @@ -0,0 +1 @@ +Improve performance of :mod:`tokenize` by 20-30%. Patch by Anthony Sottile. -- cgit v0.12