summaryrefslogtreecommitdiffstats
path: root/Parser/pgen/grammar.py
diff options
context:
space:
mode:
Diffstat (limited to 'Parser/pgen/grammar.py')
-rw-r--r--Parser/pgen/grammar.py147
1 files changed, 0 insertions, 147 deletions
diff --git a/Parser/pgen/grammar.py b/Parser/pgen/grammar.py
deleted file mode 100644
index ce40e16..0000000
--- a/Parser/pgen/grammar.py
+++ /dev/null
@@ -1,147 +0,0 @@
-import collections
-
-
-class Grammar:
- """Pgen parsing tables class.
-
- The instance variables are as follows:
-
- symbol2number -- a dict mapping symbol names to numbers. Symbol
- numbers are always 256 or higher, to distinguish
- them from token numbers, which are between 0 and
- 255 (inclusive).
-
- number2symbol -- a dict mapping numbers to symbol names;
- these two are each other's inverse.
-
- states -- a list of DFAs, where each DFA is a list of
- states, each state is a list of arcs, and each
- arc is a (i, j) pair where i is a label and j is
- a state number. The DFA number is the index into
- this list. (This name is slightly confusing.)
- Final states are represented by a special arc of
- the form (0, j) where j is its own state number.
-
- dfas -- a dict mapping symbol numbers to (DFA, first)
- pairs, where DFA is an item from the states list
- above, and first is a set of tokens that can
- begin this grammar rule.
-
- labels -- a list of (x, y) pairs where x is either a token
- number or a symbol number, and y is either None
- or a string; the strings are keywords. The label
- number is the index in this list; label numbers
- are used to mark state transitions (arcs) in the
- DFAs.
-
- start -- the number of the grammar's start symbol.
-
- keywords -- a dict mapping keyword strings to arc labels.
-
- tokens -- a dict mapping token numbers to arc labels.
-
- """
-
- def __init__(self):
- self.symbol2number = collections.OrderedDict()
- self.number2symbol = collections.OrderedDict()
- self.states = []
- self.dfas = collections.OrderedDict()
- self.labels = [(0, "EMPTY")]
- self.keywords = collections.OrderedDict()
- self.tokens = collections.OrderedDict()
- self.symbol2label = collections.OrderedDict()
- self.start = 256
-
- def produce_graminit_h(self, writer):
- writer("/* Generated by Parser/pgen */\n\n")
- for number, symbol in self.number2symbol.items():
- writer("#define {} {}\n".format(symbol, number))
-
- def produce_graminit_c(self, writer):
- writer("/* Generated by Parser/pgen */\n\n")
-
- writer('#include "exports.h"\n')
- writer('#include "grammar.h"\n')
- writer("Py_EXPORTED_SYMBOL grammar _PyParser_Grammar;\n")
-
- self.print_dfas(writer)
- self.print_labels(writer)
-
- writer("Py_EXPORTED_SYMBOL grammar _PyParser_Grammar = {\n")
- writer(" {n_dfas},\n".format(n_dfas=len(self.dfas)))
- writer(" dfas,\n")
- writer(" {{{n_labels}, labels}},\n".format(n_labels=len(self.labels)))
- writer(" {start_number}\n".format(start_number=self.start))
- writer("};\n")
-
- def print_labels(self, writer):
- writer(
- "static const label labels[{n_labels}] = {{\n".format(
- n_labels=len(self.labels)
- )
- )
- for label, name in self.labels:
- label_name = '"{}"'.format(name) if name is not None else 0
- writer(
- " {{{label}, {label_name}}},\n".format(
- label=label, label_name=label_name
- )
- )
- writer("};\n")
-
- def print_dfas(self, writer):
- self.print_states(writer)
- writer("static const dfa dfas[{}] = {{\n".format(len(self.dfas)))
- for dfaindex, dfa_elem in enumerate(self.dfas.items()):
- symbol, (dfa, first_sets) = dfa_elem
- writer(
- ' {{{dfa_symbol}, "{symbol_name}", '.format(
- dfa_symbol=symbol, symbol_name=self.number2symbol[symbol]
- )
- + "{n_states}, states_{dfa_index},\n".format(
- n_states=len(dfa), dfa_index=dfaindex
- )
- + ' "'
- )
-
- bitset = bytearray((len(self.labels) >> 3) + 1)
- for token in first_sets:
- bitset[token >> 3] |= 1 << (token & 7)
- for byte in bitset:
- writer("\\%03o" % (byte & 0xFF))
- writer('"},\n')
- writer("};\n")
-
- def print_states(self, write):
- for dfaindex, dfa in enumerate(self.states):
- self.print_arcs(write, dfaindex, dfa)
- write(
- "static state states_{dfa_index}[{n_states}] = {{\n".format(
- dfa_index=dfaindex, n_states=len(dfa)
- )
- )
- for stateindex, state in enumerate(dfa):
- narcs = len(state)
- write(
- " {{{n_arcs}, arcs_{dfa_index}_{state_index}}},\n".format(
- n_arcs=narcs, dfa_index=dfaindex, state_index=stateindex
- )
- )
- write("};\n")
-
- def print_arcs(self, write, dfaindex, states):
- for stateindex, state in enumerate(states):
- narcs = len(state)
- write(
- "static const arc arcs_{dfa_index}_{state_index}[{n_arcs}] = {{\n".format(
- dfa_index=dfaindex, state_index=stateindex, n_arcs=narcs
- )
- )
- for a, b in state:
- write(
- " {{{from_label}, {to_state}}},\n".format(
- from_label=a, to_state=b
- )
- )
- write("};\n")