summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorPablo Galindo <Pablogsal@gmail.com>2020-05-21 21:09:43 (GMT)
committerGitHub <noreply@github.com>2020-05-21 21:09:43 (GMT)
commit7178b1bb68aa14c35338b7b31ea5d1cbf76fb6a4 (patch)
tree90f515295c4e58ec6c936f6ac9647a2b982d40df
parent275d7e1080d0007a82965d1ac510abb0ae8d7821 (diff)
downloadcpython-7178b1bb68aa14c35338b7b31ea5d1cbf76fb6a4.zip
cpython-7178b1bb68aa14c35338b7b31ea5d1cbf76fb6a4.tar.gz
cpython-7178b1bb68aa14c35338b7b31ea5d1cbf76fb6a4.tar.bz2
[3.9] Fix typing problems reported by mypy in pegen (GH-20297) (GH-20300)
(cherry picked from commit d10fef35c6ce8a3193b974be7e8c8304b1146153) Co-authored-by: Pablo Galindo <Pablogsal@gmail.com>
-rw-r--r--Tools/peg_generator/pegen/c_generator.py51
-rw-r--r--Tools/peg_generator/pegen/parser_generator.py4
2 files changed, 27 insertions, 28 deletions
diff --git a/Tools/peg_generator/pegen/c_generator.py b/Tools/peg_generator/pegen/c_generator.py
index c93b348..586f28c 100644
--- a/Tools/peg_generator/pegen/c_generator.py
+++ b/Tools/peg_generator/pegen/c_generator.py
@@ -1,5 +1,5 @@
import ast
-from dataclasses import dataclass
+from dataclasses import field, dataclass
import re
from typing import Any, Dict, IO, Optional, List, Text, Tuple, Set
from enum import Enum
@@ -25,6 +25,7 @@ from pegen.grammar import (
)
from pegen.parser_generator import ParserGenerator
+
EXTENSION_PREFIX = """\
#include "pegen.h"
@@ -63,7 +64,7 @@ BASE_NODETYPES = {
@dataclass
class FunctionCall:
function: str
- arguments: Optional[List[Any]] = None
+ arguments: List[Any] = field(default_factory=list)
assigned_variable: Optional[str] = None
return_type: Optional[str] = None
nodetype: Optional[NodeTypes] = None
@@ -94,7 +95,7 @@ class CCallMakerVisitor(GrammarVisitor):
self.gen = parser_generator
self.exact_tokens = exact_tokens
self.non_exact_tokens = non_exact_tokens
- self.cache: Dict[Any, Any] = {}
+ self.cache: Dict[Any, FunctionCall] = {}
self.keyword_cache: Dict[str, int] = {}
def keyword_helper(self, keyword: str) -> FunctionCall:
@@ -171,7 +172,7 @@ class CCallMakerVisitor(GrammarVisitor):
if node in self.cache:
return self.cache[node]
if can_we_inline(node):
- self.cache[node] = self.visit(node.alts[0].items[0])
+ self.cache[node] = self.generate_call(node.alts[0].items[0])
else:
name = self.gen.name_node(node)
self.cache[node] = FunctionCall(
@@ -183,13 +184,13 @@ class CCallMakerVisitor(GrammarVisitor):
return self.cache[node]
def visit_NamedItem(self, node: NamedItem) -> FunctionCall:
- call = self.visit(node.item)
+ call = self.generate_call(node.item)
if node.name:
call.assigned_variable = node.name
return call
def lookahead_call_helper(self, node: Lookahead, positive: int) -> FunctionCall:
- call = self.visit(node.node)
+ call = self.generate_call(node.node)
if call.nodetype == NodeTypes.NAME_TOKEN:
return FunctionCall(
function=f"_PyPegen_lookahead_with_name",
@@ -217,7 +218,7 @@ class CCallMakerVisitor(GrammarVisitor):
return self.lookahead_call_helper(node, 0)
def visit_Opt(self, node: Opt) -> FunctionCall:
- call = self.visit(node.node)
+ call = self.generate_call(node.node)
return FunctionCall(
assigned_variable="_opt_var",
function=call.function,
@@ -266,7 +267,7 @@ class CCallMakerVisitor(GrammarVisitor):
return self.cache[node]
def visit_Group(self, node: Group) -> FunctionCall:
- return self.visit(node.rhs)
+ return self.generate_call(node.rhs)
def visit_Cut(self, node: Cut) -> FunctionCall:
return FunctionCall(
@@ -276,6 +277,9 @@ class CCallMakerVisitor(GrammarVisitor):
nodetype=NodeTypes.CUT_OPERATOR,
)
+ def generate_call(self, node: Any) -> FunctionCall:
+ return super().visit(node)
+
class CParserGenerator(ParserGenerator, GrammarVisitor):
def __init__(
@@ -317,17 +321,13 @@ class CParserGenerator(ParserGenerator, GrammarVisitor):
self.print(f"goto {goto_target};")
self.print(f"}}")
- def out_of_memory_return(
- self,
- expr: str,
- cleanup_code: Optional[str] = None,
- ) -> None:
+ def out_of_memory_return(self, expr: str, cleanup_code: Optional[str] = None,) -> None:
self.print(f"if ({expr}) {{")
with self.indent():
if cleanup_code is not None:
self.print(cleanup_code)
self.print("p->error_indicator = 1;")
- self.print("PyErr_NoMemory();");
+ self.print("PyErr_NoMemory();")
self.print("return NULL;")
self.print(f"}}")
@@ -484,10 +484,7 @@ class CParserGenerator(ParserGenerator, GrammarVisitor):
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
self._set_up_token_start_metadata_extraction()
self.visit(
- rhs,
- is_loop=False,
- is_gather=node.is_gather(),
- rulename=node.name,
+ rhs, is_loop=False, is_gather=node.is_gather(), rulename=node.name,
)
if self.debug:
self.print('fprintf(stderr, "Fail at %d: {node.name}\\n", p->mark);')
@@ -518,10 +515,7 @@ class CParserGenerator(ParserGenerator, GrammarVisitor):
if any(alt.action and "EXTRA" in alt.action for alt in rhs.alts):
self._set_up_token_start_metadata_extraction()
self.visit(
- rhs,
- is_loop=True,
- is_gather=node.is_gather(),
- rulename=node.name,
+ rhs, is_loop=True, is_gather=node.is_gather(), rulename=node.name,
)
if is_repeat1:
self.print("if (_n == 0 || p->error_indicator) {")
@@ -567,7 +561,7 @@ class CParserGenerator(ParserGenerator, GrammarVisitor):
self.print("}")
def visit_NamedItem(self, node: NamedItem) -> None:
- call = self.callmakervisitor.visit(node)
+ call = self.callmakervisitor.generate_call(node)
if call.assigned_variable:
call.assigned_variable = self.dedupe(call.assigned_variable)
self.print(call)
@@ -674,7 +668,9 @@ class CParserGenerator(ParserGenerator, GrammarVisitor):
self.print("if (_n == _children_capacity) {")
with self.indent():
self.print("_children_capacity *= 2;")
- self.print("void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));")
+ self.print(
+ "void **_new_children = PyMem_Realloc(_children, _children_capacity*sizeof(void *));"
+ )
self.out_of_memory_return(f"!_new_children")
self.print("_children = _new_children;")
self.print("}")
@@ -721,5 +717,8 @@ class CParserGenerator(ParserGenerator, GrammarVisitor):
return types
def add_var(self, node: NamedItem) -> Tuple[Optional[str], Optional[str]]:
- call = self.callmakervisitor.visit(node.item)
- return self.dedupe(node.name if node.name else call.assigned_variable), call.return_type
+ call = self.callmakervisitor.generate_call(node.item)
+ name = node.name if node.name else call.assigned_variable
+ if name is not None:
+ name = self.dedupe(name)
+ return name, call.return_type
diff --git a/Tools/peg_generator/pegen/parser_generator.py b/Tools/peg_generator/pegen/parser_generator.py
index 689022b..364eccb 100644
--- a/Tools/peg_generator/pegen/parser_generator.py
+++ b/Tools/peg_generator/pegen/parser_generator.py
@@ -27,7 +27,7 @@ class RuleCheckingVisitor(GrammarVisitor):
# TODO: Add line/col info to (leaf) nodes
raise GrammarError(f"Dangling reference to rule {node.value!r}")
- def visit_NamedItem(self, node: NameLeaf) -> None:
+ def visit_NamedItem(self, node: NamedItem) -> None:
if node.name and node.name.startswith("_"):
raise GrammarError(f"Variable names cannot start with underscore: '{node.name}'")
self.visit(node.item)
@@ -57,7 +57,7 @@ class ParserGenerator:
self.all_rules: Dict[str, Rule] = {} # Rules + temporal rules
self._local_variable_stack: List[List[str]] = []
- def validate_rule_names(self):
+ def validate_rule_names(self) -> None:
for rule in self.rules:
if rule.startswith("_"):
raise GrammarError(f"Rule names cannot start with underscore: '{rule}'")