diff options
| author | Pablo Galindo Salgado <Pablogsal@gmail.com> | 2024-12-15 23:17:01 (GMT) |
|---|---|---|
| committer | GitHub <noreply@github.com> | 2024-12-15 23:17:01 (GMT) |
| commit | 47c5a0f307cff3ed477528536e8de095c0752efa (patch) | |
| tree | 3dcf9219dbbfef2aee35611269c1e982c5135786 | |
| parent | b74c8f58e875d909ce6b5b9dbcddd6d8331d2081 (diff) | |
| download | cpython-47c5a0f307cff3ed477528536e8de095c0752efa.zip cpython-47c5a0f307cff3ed477528536e8de095c0752efa.tar.gz cpython-47c5a0f307cff3ed477528536e8de095c0752efa.tar.bz2 | |
gh-125588: Allow to regenerate the parser with Python < 3.12 (#127969)
Signed-off-by: Pablo Galindo <pablogsal@gmail.com>
| -rw-r--r-- | Tools/peg_generator/pegen/parser.py | 6 | ||||
| -rw-r--r-- | Tools/peg_generator/pegen/parser_generator.py | 6 |
2 files changed, 9 insertions, 3 deletions
diff --git a/Tools/peg_generator/pegen/parser.py b/Tools/peg_generator/pegen/parser.py index 692eb9e..a987d30 100644 --- a/Tools/peg_generator/pegen/parser.py +++ b/Tools/peg_generator/pegen/parser.py @@ -207,7 +207,7 @@ class Parser: @memoize def fstring_start(self) -> Optional[tokenize.TokenInfo]: - FSTRING_START = getattr(token, "FSTRING_START") + FSTRING_START = getattr(token, "FSTRING_START", None) if not FSTRING_START: return None tok = self._tokenizer.peek() @@ -217,7 +217,7 @@ class Parser: @memoize def fstring_middle(self) -> Optional[tokenize.TokenInfo]: - FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE") + FSTRING_MIDDLE = getattr(token, "FSTRING_MIDDLE", None) if not FSTRING_MIDDLE: return None tok = self._tokenizer.peek() @@ -227,7 +227,7 @@ class Parser: @memoize def fstring_end(self) -> Optional[tokenize.TokenInfo]: - FSTRING_END = getattr(token, "FSTRING_END") + FSTRING_END = getattr(token, "FSTRING_END", None) if not FSTRING_END: return None tok = self._tokenizer.peek() diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py index b42b12c..6ce0649 100644 --- a/Tools/peg_generator/pegen/parser_generator.py +++ b/Tools/peg_generator/pegen/parser_generator.py @@ -1,3 +1,4 @@ +import sys import ast import contextlib import re @@ -75,6 +76,11 @@ class RuleCheckingVisitor(GrammarVisitor): def __init__(self, rules: Dict[str, Rule], tokens: Set[str]): self.rules = rules self.tokens = tokens + # If python < 3.12 add the virtual fstring tokens + if sys.version_info < (3, 12): + self.tokens.add("FSTRING_START") + self.tokens.add("FSTRING_END") + self.tokens.add("FSTRING_MIDDLE") def visit_NameLeaf(self, node: NameLeaf) -> None: if node.value not in self.rules and node.value not in self.tokens: |