diff options
| -rw-r--r-- | Lib/test/test_parser.py | 38 | ||||
| -rw-r--r-- | Modules/parsermodule.c | 45 |
2 files changed, 68 insertions, 15 deletions
diff --git a/Lib/test/test_parser.py b/Lib/test/test_parser.py index 8aa1657..bddec16 100644 --- a/Lib/test/test_parser.py +++ b/Lib/test/test_parser.py @@ -183,6 +183,44 @@ class RoundtripLegalSyntaxTestCase(unittest.TestCase): def test_assert(self): self.check_suite("assert alo < ahi and blo < bhi\n") + def test_position(self): + # An absolutely minimal test of position information. Better + # tests would be a big project. + code = "def f(x):\n return x + 1\n" + st1 = parser.suite(code) + st2 = st1.totuple(line_info=1, col_info=1) + + def walk(tree): + node_type = tree[0] + next = tree[1] + if isinstance(next, tuple): + for elt in tree[1:]: + for x in walk(elt): + yield x + else: + yield tree + + terminals = list(walk(st2)) + self.assertEqual([ + (1, 'def', 1, 0), + (1, 'f', 1, 4), + (7, '(', 1, 5), + (1, 'x', 1, 6), + (8, ')', 1, 7), + (11, ':', 1, 8), + (4, '', 1, 9), + (5, '', 2, -1), + (1, 'return', 2, 4), + (1, 'x', 2, 11), + (14, '+', 2, 13), + (2, '1', 2, 15), + (4, '', 2, 16), + (6, '', 2, -1), + (4, '', 2, -1), + (0, '', 2, -1)], + terminals) + + # # Second, we take *invalid* trees and make sure we get ParserError # rejections for them. diff --git a/Modules/parsermodule.c b/Modules/parsermodule.c index e33197e..23364fe 100644 --- a/Modules/parsermodule.c +++ b/Modules/parsermodule.c @@ -74,7 +74,8 @@ static PyObject* node2tuple(node *n, /* node to convert */ SeqMaker mkseq, /* create sequence */ SeqInserter addelem, /* func. to add elem. in seq. */ - int lineno) /* include line numbers? */ + int lineno, /* include line numbers? */ + int col_offset) /* include column offsets? */ { if (n == NULL) { Py_INCREF(Py_None); @@ -95,7 +96,7 @@ node2tuple(node *n, /* node to convert */ } (void) addelem(v, 0, w); for (i = 0; i < NCH(n); i++) { - w = node2tuple(CHILD(n, i), mkseq, addelem, lineno); + w = node2tuple(CHILD(n, i), mkseq, addelem, lineno, col_offset); if (w == NULL) { Py_DECREF(v); return ((PyObject*) NULL); @@ -108,12 +109,14 @@ node2tuple(node *n, /* node to convert */ return (v); } else if (ISTERMINAL(TYPE(n))) { - PyObject *result = mkseq(2 + lineno); + PyObject *result = mkseq(2 + lineno + col_offset); if (result != NULL) { (void) addelem(result, 0, PyInt_FromLong(TYPE(n))); (void) addelem(result, 1, PyString_FromString(STR(n))); if (lineno == 1) (void) addelem(result, 2, PyInt_FromLong(n->n_lineno)); + if (col_offset == 1) + (void) addelem(result, 3, PyInt_FromLong(n->n_col_offset)); } return (result); } @@ -289,29 +292,35 @@ static PyObject* parser_st2tuple(PyST_Object *self, PyObject *args, PyObject *kw) { PyObject *line_option = 0; + PyObject *col_option = 0; PyObject *res = 0; int ok; - static char *keywords[] = {"ast", "line_info", NULL}; + static char *keywords[] = {"ast", "line_info", "col_info", NULL}; if (self == NULL) { - ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O:st2tuple", keywords, - &PyST_Type, &self, &line_option); + ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2tuple", keywords, + &PyST_Type, &self, &line_option, + &col_option); } else - ok = PyArg_ParseTupleAndKeywords(args, kw, "|O:totuple", &keywords[1], - &line_option); + ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:totuple", &keywords[1], + &line_option, &col_option); if (ok != 0) { int lineno = 0; + int col_offset = 0; if (line_option != NULL) { lineno = (PyObject_IsTrue(line_option) != 0) ? 1 : 0; } + if (col_option != NULL) { + col_offset = (PyObject_IsTrue(col_option) != 0) ? 1 : 0; + } /* * Convert ST into a tuple representation. Use Guido's function, * since it's known to work already. */ res = node2tuple(((PyST_Object*)self)->st_node, - PyTuple_New, PyTuple_SetItem, lineno); + PyTuple_New, PyTuple_SetItem, lineno, col_offset); } return (res); } @@ -327,28 +336,34 @@ static PyObject* parser_st2list(PyST_Object *self, PyObject *args, PyObject *kw) { PyObject *line_option = 0; + PyObject *col_option = 0; PyObject *res = 0; int ok; - static char *keywords[] = {"ast", "line_info", NULL}; + static char *keywords[] = {"ast", "line_info", "col_info", NULL}; if (self == NULL) - ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|O:st2list", keywords, - &PyST_Type, &self, &line_option); + ok = PyArg_ParseTupleAndKeywords(args, kw, "O!|OO:st2list", keywords, + &PyST_Type, &self, &line_option, + &col_option); else - ok = PyArg_ParseTupleAndKeywords(args, kw, "|O:tolist", &keywords[1], - &line_option); + ok = PyArg_ParseTupleAndKeywords(args, kw, "|OO:tolist", &keywords[1], + &line_option, &col_option); if (ok) { int lineno = 0; + int col_offset = 0; if (line_option != 0) { lineno = PyObject_IsTrue(line_option) ? 1 : 0; } + if (col_option != NULL) { + col_offset = (PyObject_IsTrue(col_option) != 0) ? 1 : 0; + } /* * Convert ST into a tuple representation. Use Guido's function, * since it's known to work already. */ res = node2tuple(self->st_node, - PyList_New, PyList_SetItem, lineno); + PyList_New, PyList_SetItem, lineno, col_offset); } return (res); } |