From 145fa86598b850e9ee0efed413dec9ec5e4431b1 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Date: Mon, 15 Jun 2020 04:31:05 +0100 Subject: [3.9] Include soft keywords in keyword.py (GH-20877). (GH-20880) (cherry picked from commit 78319e373d57cd4da67660f888aa7092efbd6f24) Co-authored-by: Pablo Galindo --- Lib/keyword.py | 7 ++++++- Tools/peg_generator/pegen/c_generator.py | 2 ++ Tools/peg_generator/pegen/keywordgen.py | 19 ++++++++++++------- 3 files changed, 20 insertions(+), 8 deletions(-) diff --git a/Lib/keyword.py b/Lib/keyword.py index a4db67e..59fcfb0 100644 --- a/Lib/keyword.py +++ b/Lib/keyword.py @@ -13,7 +13,7 @@ the python source tree and run: Alternatively, you can run 'make regen-keyword'. """ -__all__ = ["iskeyword", "kwlist"] +__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"] kwlist = [ 'False', @@ -54,4 +54,9 @@ kwlist = [ 'yield' ] +softkwlist = [ + +] + iskeyword = frozenset(kwlist).__contains__ +issoftkeyword = frozenset(softkwlist).__contains__ diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py index ce1d6bb..58a44fb 100644 --- a/Tools/peg_generator/pegen/c_generator.py +++ b/Tools/peg_generator/pegen/c_generator.py @@ -105,6 +105,7 @@ class CCallMakerVisitor(GrammarVisitor): self.non_exact_tokens = non_exact_tokens self.cache: Dict[Any, FunctionCall] = {} self.keyword_cache: Dict[str, int] = {} + self.soft_keywords: Set[str] = set() def keyword_helper(self, keyword: str) -> FunctionCall: if keyword not in self.keyword_cache: @@ -119,6 +120,7 @@ class CCallMakerVisitor(GrammarVisitor): ) def soft_keyword_helper(self, value: str) -> FunctionCall: + self.soft_keywords.add(value.replace('"', "")) return FunctionCall( assigned_variable="_keyword", function="_PyPegen_expect_soft_keyword", diff --git a/Tools/peg_generator/pegen/keywordgen.py b/Tools/peg_generator/pegen/keywordgen.py index 279c34b..fa57770 100644 --- a/Tools/peg_generator/pegen/keywordgen.py +++ b/Tools/peg_generator/pegen/keywordgen.py @@ -21,13 +21,18 @@ the python source tree and run: Alternatively, you can run 'make regen-keyword'. """ -__all__ = ["iskeyword", "kwlist"] +__all__ = ["iskeyword", "issoftkeyword", "kwlist", "softkwlist"] kwlist = [ - {keywords} +{keywords} +] + +softkwlist = [ +{soft_keywords} ] iskeyword = frozenset(kwlist).__contains__ +issoftkeyword = frozenset(softkwlist).__contains__ '''.lstrip() EXTRA_KEYWORDS = ["async", "await"] @@ -61,12 +66,12 @@ def main(): gen.collect_todo() with args.keyword_file as thefile: - all_keywords = sorted( - list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS - ) + all_keywords = sorted(list(gen.callmakervisitor.keyword_cache.keys()) + EXTRA_KEYWORDS) + all_soft_keywords = sorted(gen.callmakervisitor.soft_keywords) - keywords = ",\n ".join(map(repr, all_keywords)) - thefile.write(TEMPLATE.format(keywords=keywords)) + keywords = "" if not all_keywords else " " + ",\n ".join(map(repr, all_keywords)) + soft_keywords = "" if not all_soft_keywords else " " + ",\n ".join(map(repr, all_soft_keywords)) + thefile.write(TEMPLATE.format(keywords=keywords, soft_keywords=soft_keywords)) if __name__ == "__main__": -- cgit v0.12