summaryrefslogtreecommitdiffstats
path: root/Parser
diff options
context:
space:
mode:
Diffstat (limited to 'Parser')
-rw-r--r--Parser/Python.asdl12
-rw-r--r--Parser/asdl.py33
-rwxr-xr-xParser/asdl_c.py69
-rw-r--r--Parser/grammar.c10
-rw-r--r--Parser/grammar1.c8
-rw-r--r--Parser/myreadline.c48
-rw-r--r--Parser/node.c4
-rw-r--r--Parser/parser.c36
-rw-r--r--Parser/parsetok.c93
-rw-r--r--Parser/pgenmain.c3
-rw-r--r--Parser/tokenizer.c29
-rw-r--r--Parser/tokenizer.h6
12 files changed, 245 insertions, 106 deletions
diff --git a/Parser/Python.asdl b/Parser/Python.asdl
index c24d840..debd89e 100644
--- a/Parser/Python.asdl
+++ b/Parser/Python.asdl
@@ -1,4 +1,4 @@
--- ASDL's five builtin types are identifier, int, string, bytes, object
+-- ASDL's six builtin types are identifier, int, string, bytes, object, singleton
module Python
{
@@ -69,8 +69,8 @@ module Python
| Num(object n) -- a number as a PyObject.
| Str(string s) -- need to specify raw, unicode, etc?
| Bytes(bytes s)
+ | NameConstant(singleton value)
| Ellipsis
- -- other literals? bools?
-- the following expression can appear in assignment context
| Attribute(expr value, identifier attr, expr_context ctx)
@@ -103,11 +103,11 @@ module Python
excepthandler = ExceptHandler(expr? type, identifier? name, stmt* body)
attributes (int lineno, int col_offset)
- arguments = (arg* args, identifier? vararg, expr? varargannotation,
- arg* kwonlyargs, identifier? kwarg,
- expr? kwargannotation, expr* defaults,
- expr* kw_defaults)
+ arguments = (arg* args, arg? vararg, arg* kwonlyargs, expr* kw_defaults,
+ arg? kwarg, expr* defaults)
+
arg = (identifier arg, expr? annotation)
+ attributes (int lineno, int col_offset)
-- keyword arguments supplied to call
keyword = (identifier arg, expr value)
diff --git a/Parser/asdl.py b/Parser/asdl.py
index 08aa05b..fc1b16c 100644
--- a/Parser/asdl.py
+++ b/Parser/asdl.py
@@ -16,8 +16,9 @@ import traceback
import spark
-def output(string):
- sys.stdout.write(string + "\n")
+def output(*strings):
+ for s in strings:
+ sys.stdout.write(str(s) + "\n")
class Token(object):
@@ -158,11 +159,19 @@ class ASDLParser(spark.GenericParser, object):
msg="expected attributes, found %s" % id)
return Sum(sum, attributes)
- def p_product(self, info):
+ def p_product_0(self, info):
" product ::= ( fields ) "
_0, fields, _1 = info
return Product(fields)
+ def p_product_1(self, info):
+ " product ::= ( fields ) Id ( fields ) "
+ _0, fields, _1, id, _2, attributes, _3 = info
+ if id.value != "attributes":
+ raise ASDLSyntaxError(id.lineno,
+ msg="expected attributes, found %s" % id)
+ return Product(fields, attributes)
+
def p_sum_0(self, constructor):
" sum ::= constructor "
return [constructor[0]]
@@ -222,7 +231,7 @@ class ASDLParser(spark.GenericParser, object):
" field ::= Id ? "
return Field(type[0], opt=True)
-builtin_types = ("identifier", "string", "bytes", "int", "object")
+builtin_types = ("identifier", "string", "bytes", "int", "object", "singleton")
# below is a collection of classes to capture the AST of an AST :-)
# not sure if any of the methods are useful yet, but I'm adding them
@@ -289,11 +298,15 @@ class Sum(AST):
return "Sum(%s, %s)" % (self.types, self.attributes)
class Product(AST):
- def __init__(self, fields):
+ def __init__(self, fields, attributes=None):
self.fields = fields
+ self.attributes = attributes or []
def __repr__(self):
- return "Product(%s)" % self.fields
+ if self.attributes is None:
+ return "Product(%s)" % self.fields
+ else:
+ return "Product(%s, %s)" % (self.fields, self.attributes)
class VisitorBase(object):
@@ -385,7 +398,11 @@ def parse(file):
scanner = ASDLScanner()
parser = ASDLParser()
- buf = open(file).read()
+ f = open(file)
+ try:
+ buf = f.read()
+ finally:
+ f.close()
tokens = scanner.tokenize(buf)
try:
return parser.parse(tokens)
@@ -416,4 +433,4 @@ if __name__ == "__main__":
output("Check failed")
else:
for dfn in mod.dfns:
- output(dfn.type)
+ output(dfn.name, dfn.value)
diff --git a/Parser/asdl_c.py b/Parser/asdl_c.py
index 4b84e0f..d6086e6 100755
--- a/Parser/asdl_c.py
+++ b/Parser/asdl_c.py
@@ -209,6 +209,11 @@ class StructVisitor(EmitVisitor):
self.emit("struct _%(name)s {" % locals(), depth)
for f in product.fields:
self.visit(f, depth + 1)
+ for field in product.attributes:
+ # rudimentary attribute handling
+ type = str(field.type)
+ assert type in asdl.builtin_types, type
+ self.emit("%s %s;" % (type, field.name), depth + 1);
self.emit("};", depth)
self.emit("", depth)
@@ -493,7 +498,11 @@ class Obj2ModVisitor(PickleVisitor):
def visitField(self, field, name, sum=None, prod=None, depth=0):
ctype = get_c_type(field.type)
- self.emit("if (_PyObject_HasAttrId(obj, &PyId_%s)) {" % field.name, depth)
+ if field.opt:
+ check = "exists_not_none(obj, &PyId_%s)" % (field.name,)
+ else:
+ check = "_PyObject_HasAttrId(obj, &PyId_%s)" % (field.name,)
+ self.emit("if (%s) {" % (check,), depth, reflow=False)
self.emit("int res;", depth+1)
if field.seq:
self.emit("Py_ssize_t len;", depth+1)
@@ -510,9 +519,9 @@ class Obj2ModVisitor(PickleVisitor):
self.emit("}", depth+1)
self.emit("len = PyList_GET_SIZE(tmp);", depth+1)
if self.isSimpleType(field):
- self.emit("%s = asdl_int_seq_new(len, arena);" % field.name, depth+1)
+ self.emit("%s = _Py_asdl_int_seq_new(len, arena);" % field.name, depth+1)
else:
- self.emit("%s = asdl_seq_new(len, arena);" % field.name, depth+1)
+ self.emit("%s = _Py_asdl_seq_new(len, arena);" % field.name, depth+1)
self.emit("if (%s == NULL) goto failed;" % field.name, depth+1)
self.emit("for (i = 0; i < len; i++) {", depth+1)
self.emit("%s value;" % ctype, depth+2)
@@ -558,6 +567,13 @@ class PyTypesDeclareVisitor(PickleVisitor):
def visitProduct(self, prod, name):
self.emit("static PyTypeObject *%s_type;" % name, 0)
self.emit("static PyObject* ast2obj_%s(void*);" % name, 0)
+ if prod.attributes:
+ for a in prod.attributes:
+ self.emit_identifier(a.name)
+ self.emit("static char *%s_attributes[] = {" % name, 0)
+ for a in prod.attributes:
+ self.emit('"%s",' % a.name, 1)
+ self.emit("};", 0)
if prod.fields:
for f in prod.fields:
self.emit_identifier(f.name)
@@ -819,6 +835,7 @@ static PyObject* ast2obj_object(void *o)
Py_INCREF((PyObject*)o);
return (PyObject*)o;
}
+#define ast2obj_singleton ast2obj_object
#define ast2obj_identifier ast2obj_object
#define ast2obj_string ast2obj_object
#define ast2obj_bytes ast2obj_object
@@ -830,6 +847,17 @@ static PyObject* ast2obj_int(long b)
/* Conversion Python -> AST */
+static int obj2ast_singleton(PyObject *obj, PyObject** out, PyArena* arena)
+{
+ if (obj != Py_None && obj != Py_True && obj != Py_False) {
+ PyErr_SetString(PyExc_ValueError,
+ "AST singleton must be True, False, or None");
+ return 1;
+ }
+ *out = obj;
+ return 0;
+}
+
static int obj2ast_object(PyObject* obj, PyObject** out, PyArena* arena)
{
if (obj == Py_None)
@@ -904,6 +932,19 @@ static int add_ast_fields(void)
return 0;
}
+static int exists_not_none(PyObject *obj, _Py_Identifier *id)
+{
+ int isnone;
+ PyObject *attr = _PyObject_GetAttrId(obj, id);
+ if (!attr) {
+ PyErr_Clear();
+ return 0;
+ }
+ isnone = attr == Py_None;
+ Py_DECREF(attr);
+ return !isnone;
+}
+
""", 0, reflow=False)
self.emit("static int init_types(void)",0)
@@ -925,6 +966,11 @@ static int add_ast_fields(void)
self.emit('%s_type = make_type("%s", &AST_type, %s, %d);' %
(name, name, fields, len(prod.fields)), 1)
self.emit("if (!%s_type) return 0;" % name, 1)
+ if prod.attributes:
+ self.emit("if (!add_attributes(%s_type, %s_attributes, %d)) return 0;" %
+ (name, name, len(prod.attributes)), 1)
+ else:
+ self.emit("if (!add_attributes(%s_type, NULL, 0)) return 0;" % name, 1)
def visitSum(self, sum, name):
self.emit('%s_type = make_type("%s", &AST_type, NULL, 0);' %
@@ -968,7 +1014,7 @@ class ASTModuleVisitor(PickleVisitor):
self.emit("if (!m) return NULL;", 1)
self.emit("d = PyModule_GetDict(m);", 1)
self.emit('if (PyDict_SetItemString(d, "AST", (PyObject*)&AST_type) < 0) return NULL;', 1)
- self.emit('if (PyModule_AddIntConstant(m, "PyCF_ONLY_AST", PyCF_ONLY_AST) < 0)', 1)
+ self.emit('if (PyModule_AddIntMacro(m, PyCF_ONLY_AST) < 0)', 1)
self.emit("return NULL;", 2)
for dfn in mod.dfns:
self.visit(dfn)
@@ -1080,6 +1126,12 @@ class ObjVisitor(PickleVisitor):
self.emit("if (!result) return NULL;", 1)
for field in prod.fields:
self.visitField(field, name, 1, True)
+ for a in prod.attributes:
+ self.emit("value = ast2obj_%s(o->%s);" % (a.type, a.name), 1)
+ self.emit("if (!value) goto failed;", 1)
+ self.emit('if (_PyObject_SetAttrId(result, &PyId_%s, value) < 0)' % a.name, 1)
+ self.emit('goto failed;', 2)
+ self.emit('Py_DECREF(value);', 1)
self.func_end()
def visitConstructor(self, cons, enum, name):
@@ -1142,7 +1194,8 @@ class PartingShots(StaticVisitor):
CODE = """
PyObject* PyAST_mod2obj(mod_ty t)
{
- init_types();
+ if (!init_types())
+ return NULL;
return ast2obj_mod(t);
}
@@ -1160,7 +1213,8 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
assert(0 <= mode && mode <= 2);
- init_types();
+ if (!init_types())
+ return NULL;
isinstance = PyObject_IsInstance(ast, req_type[mode]);
if (isinstance == -1)
@@ -1178,7 +1232,8 @@ mod_ty PyAST_obj2mod(PyObject* ast, PyArena* arena, int mode)
int PyAST_Check(PyObject* obj)
{
- init_types();
+ if (!init_types())
+ return -1;
return PyObject_IsInstance(obj, (PyObject*)&AST_type);
}
"""
diff --git a/Parser/grammar.c b/Parser/grammar.c
index f2a25ca..b598294 100644
--- a/Parser/grammar.c
+++ b/Parser/grammar.c
@@ -29,7 +29,7 @@ newgrammar(int start)
}
dfa *
-adddfa(grammar *g, int type, char *name)
+adddfa(grammar *g, int type, const char *name)
{
dfa *d;
@@ -63,7 +63,7 @@ addstate(dfa *d)
s->s_upper = 0;
s->s_accel = NULL;
s->s_accept = 0;
- return s - d->d_state;
+ return Py_SAFE_DOWNCAST(s - d->d_state, Py_intptr_t, int);
}
void
@@ -85,7 +85,7 @@ addarc(dfa *d, int from, int to, int lbl)
}
int
-addlabel(labellist *ll, int type, char *str)
+addlabel(labellist *ll, int type, const char *str)
{
int i;
label *lb;
@@ -105,13 +105,13 @@ addlabel(labellist *ll, int type, char *str)
if (Py_DebugFlag)
printf("Label @ %8p, %d: %s\n", ll, ll->ll_nlabels,
PyGrammar_LabelRepr(lb));
- return lb - ll->ll_label;
+ return Py_SAFE_DOWNCAST(lb - ll->ll_label, Py_intptr_t, int);
}
/* Same, but rather dies than adds */
int
-findlabel(labellist *ll, int type, char *str)
+findlabel(labellist *ll, int type, const char *str)
{
int i;
diff --git a/Parser/grammar1.c b/Parser/grammar1.c
index 440ba44..30a6f07 100644
--- a/Parser/grammar1.c
+++ b/Parser/grammar1.c
@@ -9,9 +9,9 @@
/* Return the DFA for the given type */
dfa *
-PyGrammar_FindDFA(grammar *g, register int type)
+PyGrammar_FindDFA(grammar *g, int type)
{
- register dfa *d;
+ dfa *d;
#if 1
/* Massive speed-up */
d = &g->g_dfa[type - NT_OFFSET];
@@ -19,7 +19,7 @@ PyGrammar_FindDFA(grammar *g, register int type)
return d;
#else
/* Old, slow version */
- register int i;
+ int i;
for (i = g->g_ndfas, d = g->g_dfa; --i >= 0; d++) {
if (d->d_type == type)
@@ -30,7 +30,7 @@ PyGrammar_FindDFA(grammar *g, register int type)
#endif
}
-char *
+const char *
PyGrammar_LabelRepr(label *lb)
{
static char buf[100];
diff --git a/Parser/myreadline.c b/Parser/myreadline.c
index 9f1fc1e..28c7b6d 100644
--- a/Parser/myreadline.c
+++ b/Parser/myreadline.c
@@ -15,10 +15,6 @@
#include "windows.h"
#endif /* MS_WINDOWS */
-#ifdef __VMS
-extern char* vms__StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt);
-#endif
-
PyThreadState* _PyOS_ReadlineTState;
@@ -68,7 +64,7 @@ my_fgets(char *buf, int len, FILE *fp)
*/
if (GetLastError()==ERROR_OPERATION_ABORTED) {
hInterruptEvent = _PyOS_SigintEvent();
- switch (WaitForSingleObject(hInterruptEvent, 10)) {
+ switch (WaitForSingleObjectEx(hInterruptEvent, 10, FALSE)) {
case WAIT_OBJECT_0:
ResetEvent(hInterruptEvent);
return 1; /* Interrupt */
@@ -109,22 +105,26 @@ my_fgets(char *buf, int len, FILE *fp)
/* Readline implementation using fgets() */
char *
-PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
+PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
{
size_t n;
char *p, *pr;
+
n = 100;
- if ((p = (char *)PyMem_MALLOC(n)) == NULL)
+ p = (char *)PyMem_RawMalloc(n);
+ if (p == NULL)
return NULL;
+
fflush(sys_stdout);
if (prompt)
fprintf(stderr, "%s", prompt);
fflush(stderr);
+
switch (my_fgets(p, (int)n, sys_stdin)) {
case 0: /* Normal case */
break;
case 1: /* Interrupt */
- PyMem_FREE(p);
+ PyMem_RawFree(p);
return NULL;
case -1: /* EOF */
case -2: /* Error */
@@ -136,13 +136,13 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
while (n > 0 && p[n-1] != '\n') {
size_t incr = n+2;
if (incr > INT_MAX) {
- PyMem_FREE(p);
+ PyMem_RawFree(p);
PyErr_SetString(PyExc_OverflowError, "input line too long");
return NULL;
}
- pr = (char *)PyMem_REALLOC(p, n + incr);
+ pr = (char *)PyMem_RawRealloc(p, n + incr);
if (pr == NULL) {
- PyMem_FREE(p);
+ PyMem_RawFree(p);
PyErr_NoMemory();
return NULL;
}
@@ -151,9 +151,9 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
break;
n += strlen(p+n);
}
- pr = (char *)PyMem_REALLOC(p, n+1);
+ pr = (char *)PyMem_RawRealloc(p, n+1);
if (pr == NULL) {
- PyMem_FREE(p);
+ PyMem_RawFree(p);
PyErr_NoMemory();
return NULL;
}
@@ -166,15 +166,16 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
Note: Python expects in return a buffer allocated with PyMem_Malloc. */
-char *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, char *);
+char *(*PyOS_ReadlineFunctionPointer)(FILE *, FILE *, const char *);
/* Interface used by tokenizer.c and bltinmodule.c */
char *
-PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
+PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
{
- char *rv;
+ char *rv, *res;
+ size_t len;
if (_PyOS_ReadlineTState == PyThreadState_GET()) {
PyErr_SetString(PyExc_RuntimeError,
@@ -184,11 +185,7 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
if (PyOS_ReadlineFunctionPointer == NULL) {
-#ifdef __VMS
- PyOS_ReadlineFunctionPointer = vms__StdioReadline;
-#else
PyOS_ReadlineFunctionPointer = PyOS_StdioReadline;
-#endif
}
#ifdef WITH_THREAD
@@ -221,5 +218,14 @@ PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
_PyOS_ReadlineTState = NULL;
- return rv;
+ if (rv == NULL)
+ return NULL;
+
+ len = strlen(rv) + 1;
+ res = PyMem_Malloc(len);
+ if (res != NULL)
+ memcpy(res, rv, len);
+ PyMem_RawFree(rv);
+
+ return res;
}
diff --git a/Parser/node.c b/Parser/node.c
index 1e4f0da..564bd91 100644
--- a/Parser/node.c
+++ b/Parser/node.c
@@ -76,7 +76,7 @@ fancy_roundup(int n)
int
-PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offset)
+PyNode_AddChild(node *n1, int type, char *str, int lineno, int col_offset)
{
const int nch = n1->n_nchildren;
int current_capacity;
@@ -91,7 +91,7 @@ PyNode_AddChild(register node *n1, int type, char *str, int lineno, int col_offs
if (current_capacity < 0 || required_capacity < 0)
return E_OVERFLOW;
if (current_capacity < required_capacity) {
- if (required_capacity > PY_SIZE_MAX / sizeof(node)) {
+ if ((size_t)required_capacity > PY_SIZE_MAX / sizeof(node)) {
return E_NOMEM;
}
n = n1->n_child;
diff --git a/Parser/parser.c b/Parser/parser.c
index b505fe0..56ec514 100644
--- a/Parser/parser.c
+++ b/Parser/parser.c
@@ -35,9 +35,9 @@ s_reset(stack *s)
#define s_empty(s) ((s)->s_top == &(s)->s_base[MAXSTACK])
static int
-s_push(register stack *s, dfa *d, node *parent)
+s_push(stack *s, dfa *d, node *parent)
{
- register stackentry *top;
+ stackentry *top;
if (s->s_top == s->s_base) {
fprintf(stderr, "s_push: parser stack overflow\n");
return E_NOMEM;
@@ -52,7 +52,7 @@ s_push(register stack *s, dfa *d, node *parent)
#ifdef Py_DEBUG
static void
-s_pop(register stack *s)
+s_pop(stack *s)
{
if (s_empty(s))
Py_FatalError("s_pop: parser stack underflow -- FATAL");
@@ -105,7 +105,7 @@ PyParser_Delete(parser_state *ps)
/* PARSER STACK OPERATIONS */
static int
-shift(register stack *s, int type, char *str, int newstate, int lineno, int col_offset)
+shift(stack *s, int type, char *str, int newstate, int lineno, int col_offset)
{
int err;
assert(!s_empty(s));
@@ -117,10 +117,10 @@ shift(register stack *s, int type, char *str, int newstate, int lineno, int col_
}
static int
-push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
+push(stack *s, int type, dfa *d, int newstate, int lineno, int col_offset)
{
int err;
- register node *n;
+ node *n;
n = s->s_top->s_parent;
assert(!s_empty(s));
err = PyNode_AddChild(n, type, (char *)NULL, lineno, col_offset);
@@ -134,15 +134,15 @@ push(register stack *s, int type, dfa *d, int newstate, int lineno, int col_offs
/* PARSER PROPER */
static int
-classify(parser_state *ps, int type, char *str)
+classify(parser_state *ps, int type, const char *str)
{
grammar *g = ps->p_grammar;
- register int n = g->g_ll.ll_nlabels;
+ int n = g->g_ll.ll_nlabels;
if (type == NAME) {
- register char *s = str;
- register label *l = g->g_ll.ll_label;
- register int i;
+ const char *s = str;
+ label *l = g->g_ll.ll_label;
+ int i;
for (i = n; i > 0; i--, l++) {
if (l->lb_type != NAME || l->lb_str == NULL ||
l->lb_str[0] != s[0] ||
@@ -165,8 +165,8 @@ classify(parser_state *ps, int type, char *str)
}
{
- register label *l = g->g_ll.ll_label;
- register int i;
+ label *l = g->g_ll.ll_label;
+ int i;
for (i = n; i > 0; i--, l++) {
if (l->lb_type == type && l->lb_str == NULL) {
D(printf("It's a token we know\n"));
@@ -225,10 +225,10 @@ future_hack(parser_state *ps)
#endif /* future keyword */
int
-PyParser_AddToken(register parser_state *ps, register int type, char *str,
+PyParser_AddToken(parser_state *ps, int type, char *str,
int lineno, int col_offset, int *expected_ret)
{
- register int ilabel;
+ int ilabel;
int err;
D(printf("Token %s/'%s' ... ", _PyParser_TokenNames[type], str));
@@ -241,15 +241,15 @@ PyParser_AddToken(register parser_state *ps, register int type, char *str,
/* Loop until the token is shifted or an error occurred */
for (;;) {
/* Fetch the current dfa and state */
- register dfa *d = ps->p_stack.s_top->s_dfa;
- register state *s = &d->d_state[ps->p_stack.s_top->s_state];
+ dfa *d = ps->p_stack.s_top->s_dfa;
+ state *s = &d->d_state[ps->p_stack.s_top->s_state];
D(printf(" DFA '%s', state %d:",
d->d_name, ps->p_stack.s_top->s_state));
/* Check accelerator */
if (s->s_lower <= ilabel && ilabel < s->s_upper) {
- register int x = s->s_accel[ilabel - s->s_lower];
+ int x = s->s_accel[ilabel - s->s_lower];
if (x != -1) {
if (x & (1<<7)) {
/* Push non-terminal */
diff --git a/Parser/parsetok.c b/Parser/parsetok.c
index 7beb735..629dee5 100644
--- a/Parser/parsetok.c
+++ b/Parser/parsetok.c
@@ -13,7 +13,7 @@
/* Forward */
static node *parsetok(struct tok_state *, grammar *, int, perrdetail *, int *);
-static int initerr(perrdetail *err_ret, const char* filename);
+static int initerr(perrdetail *err_ret, PyObject * filename);
/* Parse input coming from a string. Return error code, print some errors. */
node *
@@ -41,9 +41,9 @@ PyParser_ParseStringFlagsFilename(const char *s, const char *filename,
}
node *
-PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
- grammar *g, int start,
- perrdetail *err_ret, int *flags)
+PyParser_ParseStringObject(const char *s, PyObject *filename,
+ grammar *g, int start,
+ perrdetail *err_ret, int *flags)
{
struct tok_state *tok;
int exec_input = start == file_input;
@@ -67,11 +67,35 @@ PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename,
return parsetok(tok, g, start, err_ret, flags);
}
+node *
+PyParser_ParseStringFlagsFilenameEx(const char *s, const char *filename_str,
+ grammar *g, int start,
+ perrdetail *err_ret, int *flags)
+{
+ node *n;
+ PyObject *filename = NULL;
+#ifndef PGEN
+ if (filename_str != NULL) {
+ filename = PyUnicode_DecodeFSDefault(filename_str);
+ if (filename == NULL) {
+ err_ret->error = E_ERROR;
+ return NULL;
+ }
+ }
+#endif
+ n = PyParser_ParseStringObject(s, filename, g, start, err_ret, flags);
+#ifndef PGEN
+ Py_XDECREF(filename);
+#endif
+ return n;
+}
+
/* Parse input coming from a file. Return error code, print some errors. */
node *
PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
- char *ps1, char *ps2, perrdetail *err_ret)
+ const char *ps1, const char *ps2,
+ perrdetail *err_ret)
{
return PyParser_ParseFileFlags(fp, filename, NULL,
g, start, ps1, ps2, err_ret, 0);
@@ -80,7 +104,8 @@ PyParser_ParseFile(FILE *fp, const char *filename, grammar *g, int start,
node *
PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
grammar *g, int start,
- char *ps1, char *ps2, perrdetail *err_ret, int flags)
+ const char *ps1, const char *ps2,
+ perrdetail *err_ret, int flags)
{
int iflags = flags;
return PyParser_ParseFileFlagsEx(fp, filename, enc, g, start, ps1,
@@ -88,16 +113,17 @@ PyParser_ParseFileFlags(FILE *fp, const char *filename, const char *enc,
}
node *
-PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
- const char *enc, grammar *g, int start,
- char *ps1, char *ps2, perrdetail *err_ret, int *flags)
+PyParser_ParseFileObject(FILE *fp, PyObject *filename,
+ const char *enc, grammar *g, int start,
+ const char *ps1, const char *ps2,
+ perrdetail *err_ret, int *flags)
{
struct tok_state *tok;
if (initerr(err_ret, filename) < 0)
return NULL;
- if ((tok = PyTokenizer_FromFile(fp, (char *)enc, ps1, ps2)) == NULL) {
+ if ((tok = PyTokenizer_FromFile(fp, enc, ps1, ps2)) == NULL) {
err_ret->error = E_NOMEM;
return NULL;
}
@@ -108,6 +134,31 @@ PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
return parsetok(tok, g, start, err_ret, flags);
}
+node *
+PyParser_ParseFileFlagsEx(FILE *fp, const char *filename,
+ const char *enc, grammar *g, int start,
+ const char *ps1, const char *ps2,
+ perrdetail *err_ret, int *flags)
+{
+ node *n;
+ PyObject *fileobj = NULL;
+#ifndef PGEN
+ if (filename != NULL) {
+ fileobj = PyUnicode_DecodeFSDefault(filename);
+ if (fileobj == NULL) {
+ err_ret->error = E_ERROR;
+ return NULL;
+ }
+ }
+#endif
+ n = PyParser_ParseFileObject(fp, fileobj, enc, g,
+ start, ps1, ps2, err_ret, flags);
+#ifndef PGEN
+ Py_XDECREF(fileobj);
+#endif
+ return n;
+}
+
#ifdef PY_PARSER_REQUIRES_FUTURE_KEYWORD
#if 0
static char with_msg[] =
@@ -138,7 +189,6 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
int started = 0;
if ((ps = PyParser_New(g, start)) == NULL) {
- fprintf(stderr, "no mem for new parser\n");
err_ret->error = E_NOMEM;
PyTokenizer_Free(tok);
return NULL;
@@ -178,7 +228,6 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
len = b - a; /* XXX this may compute NULL - NULL */
str = (char *) PyObject_MALLOC(len + 1);
if (str == NULL) {
- fprintf(stderr, "no mem for next token\n");
err_ret->error = E_NOMEM;
break;
}
@@ -205,7 +254,8 @@ parsetok(struct tok_state *tok, grammar *g, int start, perrdetail *err_ret,
}
#endif
if (a >= tok->line_start)
- col_offset = a - tok->line_start;
+ col_offset = Py_SAFE_DOWNCAST(a - tok->line_start,
+ Py_intptr_t, int);
else
col_offset = -1;
@@ -308,7 +358,7 @@ done:
}
static int
-initerr(perrdetail *err_ret, const char *filename)
+initerr(perrdetail *err_ret, PyObject *filename)
{
err_ret->error = E_OK;
err_ret->lineno = 0;
@@ -317,13 +367,16 @@ initerr(perrdetail *err_ret, const char *filename)
err_ret->token = -1;
err_ret->expected = -1;
#ifndef PGEN
- if (filename)
- err_ret->filename = PyUnicode_DecodeFSDefault(filename);
- else
+ if (filename) {
+ Py_INCREF(filename);
+ err_ret->filename = filename;
+ }
+ else {
err_ret->filename = PyUnicode_FromString("<string>");
- if (err_ret->filename == NULL) {
- err_ret->error = E_ERROR;
- return -1;
+ if (err_ret->filename == NULL) {
+ err_ret->error = E_ERROR;
+ return -1;
+ }
}
#endif
return 0;
diff --git a/Parser/pgenmain.c b/Parser/pgenmain.c
index 52b8380..017a4f9 100644
--- a/Parser/pgenmain.c
+++ b/Parser/pgenmain.c
@@ -113,6 +113,7 @@ getgrammar(char *filename)
Py_Exit(1);
}
g = pgen(n);
+ PyNode_Free(n);
if (g == NULL) {
printf("Bad grammar.\n");
Py_Exit(1);
@@ -137,7 +138,7 @@ Py_FatalError(const char *msg)
/* No-nonsense my_readline() for tokenizer.c */
char *
-PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, char *prompt)
+PyOS_Readline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
{
size_t n = 1000;
char *p = (char *)PyMem_MALLOC(n);
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index 8530723..c32a3bf 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -31,7 +31,7 @@
|| c == '_'\
|| (c >= 128))
-extern char *PyOS_Readline(FILE *, FILE *, char *);
+extern char *PyOS_Readline(FILE *, FILE *, const char *);
/* Return malloc'ed string including trailing \n;
empty malloc'ed string for EOF;
NULL if interrupted */
@@ -47,7 +47,7 @@ static void tok_backup(struct tok_state *tok, int c);
/* Token names */
-char *_PyParser_TokenNames[] = {
+const char *_PyParser_TokenNames[] = {
"ENDMARKER",
"NAME",
"NUMBER",
@@ -681,7 +681,8 @@ translate_into_utf8(const char* str, const char* enc) {
static char *
translate_newlines(const char *s, int exec_input, struct tok_state *tok) {
- int skip_next_lf = 0, needed_length = strlen(s) + 2, final_length;
+ int skip_next_lf = 0;
+ size_t needed_length = strlen(s) + 2, final_length;
char *buf, *current;
char c = '\0';
buf = PyMem_MALLOC(needed_length);
@@ -789,7 +790,7 @@ PyTokenizer_FromString(const char *str, int exec_input)
struct tok_state *tok = tok_new();
if (tok == NULL)
return NULL;
- str = (char *)decode_str(str, exec_input, tok);
+ str = decode_str(str, exec_input, tok);
if (str == NULL) {
PyTokenizer_Free(tok);
return NULL;
@@ -832,7 +833,8 @@ PyTokenizer_FromUTF8(const char *str, int exec_input)
/* Set up tokenizer for file */
struct tok_state *
-PyTokenizer_FromFile(FILE *fp, char* enc, char *ps1, char *ps2)
+PyTokenizer_FromFile(FILE *fp, const char* enc,
+ const char *ps1, const char *ps2)
{
struct tok_state *tok = tok_new();
if (tok == NULL)
@@ -883,7 +885,7 @@ PyTokenizer_Free(struct tok_state *tok)
/* Get next char, updating state; error code goes into tok->done */
static int
-tok_nextc(register struct tok_state *tok)
+tok_nextc(struct tok_state *tok)
{
for (;;) {
if (tok->cur != tok->inp) {
@@ -1080,7 +1082,7 @@ tok_nextc(register struct tok_state *tok)
/* Back-up one character */
static void
-tok_backup(register struct tok_state *tok, register int c)
+tok_backup(struct tok_state *tok, int c)
{
if (c != EOF) {
if (--tok->cur < tok->buf)
@@ -1310,9 +1312,9 @@ verify_identifier(struct tok_state *tok)
/* Get next token, after space stripping etc. */
static int
-tok_get(register struct tok_state *tok, char **p_start, char **p_end)
+tok_get(struct tok_state *tok, char **p_start, char **p_end)
{
- register int c;
+ int c;
int blankline, nonascii;
*p_start = *p_end = NULL;
@@ -1322,8 +1324,8 @@ tok_get(register struct tok_state *tok, char **p_start, char **p_end)
/* Get indentation level */
if (tok->atbol) {
- register int col = 0;
- register int altcol = 0;
+ int col = 0;
+ int altcol = 0;
tok->atbol = 0;
for (;;) {
c = tok_nextc(tok);
@@ -1739,10 +1741,15 @@ PyTokenizer_FindEncodingFilename(int fd, PyObject *filename)
FILE *fp;
char *p_start =NULL , *p_end =NULL , *encoding = NULL;
+#ifndef PGEN
+ fd = _Py_dup(fd);
+#else
fd = dup(fd);
+#endif
if (fd < 0) {
return NULL;
}
+
fp = fdopen(fd, "r");
if (fp == NULL) {
return NULL;
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
index ed1f3aa..1ce6eeb 100644
--- a/Parser/tokenizer.h
+++ b/Parser/tokenizer.h
@@ -35,7 +35,7 @@ struct tok_state {
int indstack[MAXINDENT]; /* Stack of indents */
int atbol; /* Nonzero if at begin of new line */
int pendin; /* Pending indents (if > 0) or dedents (if < 0) */
- char *prompt, *nextprompt; /* For interactive prompting */
+ const char *prompt, *nextprompt; /* For interactive prompting */
int lineno; /* Current line number */
int level; /* () [] {} Parentheses nesting level */
/* Used to allow free continuations inside them */
@@ -69,8 +69,8 @@ struct tok_state {
extern struct tok_state *PyTokenizer_FromString(const char *, int);
extern struct tok_state *PyTokenizer_FromUTF8(const char *, int);
-extern struct tok_state *PyTokenizer_FromFile(FILE *, char*,
- char *, char *);
+extern struct tok_state *PyTokenizer_FromFile(FILE *, const char*,
+ const char *, const char *);
extern void PyTokenizer_Free(struct tok_state *);
extern int PyTokenizer_Get(struct tok_state *, char **, char **);
extern char * PyTokenizer_RestoreEncoding(struct tok_state* tok,
generated by cgit v0.12 at 2025-03-03 02:24:21 (GMT)