summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorGregory P. Smith ext:(%20%5BGoogle%20Inc.%5D) <greg@krypto.org>2016-09-10 01:18:52 (GMT)
committerGregory P. Smith ext:(%20%5BGoogle%20Inc.%5D) <greg@krypto.org>2016-09-10 01:18:52 (GMT)
commit28325749c01815097aa2bc06508004a1f894c279 (patch)
tree83219772233a4f8b3d8db1df9bfb6b573fe39bfe
parentdbdf029a5575f6e6ec0140260236963ed7d2c2be (diff)
downloadcpython-28325749c01815097aa2bc06508004a1f894c279.zip
cpython-28325749c01815097aa2bc06508004a1f894c279.tar.gz
cpython-28325749c01815097aa2bc06508004a1f894c279.tar.bz2
Issue #25969: Update the lib2to3 grammar to handle the unpacking
generalizations added in 3.5.
-rw-r--r--Lib/lib2to3/Grammar.txt23
-rw-r--r--Lib/lib2to3/fixes/fix_apply.py11
-rw-r--r--Lib/lib2to3/fixes/fix_intern.py10
-rw-r--r--Lib/lib2to3/fixes/fix_reload.py11
-rw-r--r--Lib/lib2to3/tests/test_fixers.py4
-rw-r--r--Lib/lib2to3/tests/test_parser.py35
-rw-r--r--Misc/NEWS3
7 files changed, 91 insertions, 6 deletions
diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt
index c954669..88f7884 100644
--- a/Lib/lib2to3/Grammar.txt
+++ b/Lib/lib2to3/Grammar.txt
@@ -138,15 +138,26 @@ subscript: test | [test] ':' [test] [sliceop]
sliceop: ':' [test]
exprlist: (expr|star_expr) (',' (expr|star_expr))* [',']
testlist: test (',' test)* [',']
-dictsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) |
- (test (comp_for | (',' test)* [','])) )
+dictsetmaker: ( ((test ':' test | '**' expr)
+ (comp_for | (',' (test ':' test | '**' expr))* [','])) |
+ ((test | star_expr)
+ (comp_for | (',' (test | star_expr))* [','])) )
classdef: 'class' NAME ['(' [arglist] ')'] ':' suite
-arglist: (argument ',')* (argument [',']
- |'*' test (',' argument)* [',' '**' test]
- |'**' test)
-argument: test [comp_for] | test '=' test # Really [keyword '='] test
+arglist: argument (',' argument)* [',']
+
+# "test '=' test" is really "keyword '=' test", but we have no such token.
+# These need to be in a single rule to avoid grammar that is ambiguous
+# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr,
+# we explicitly match '*' here, too, to give it proper precedence.
+# Illegal combinations and orderings are blocked in ast.c:
+# multiple (test comp_for) arguements are blocked; keyword unpackings
+# that precede iterable unpackings are blocked; etc.
+argument: ( test [comp_for] |
+ test '=' test |
+ '**' expr |
+ star_expr )
comp_iter: comp_for | comp_if
comp_for: 'for' exprlist 'in' testlist_safe [comp_iter]
diff --git a/Lib/lib2to3/fixes/fix_apply.py b/Lib/lib2to3/fixes/fix_apply.py
index 81a46dc..826ec8c 100644
--- a/Lib/lib2to3/fixes/fix_apply.py
+++ b/Lib/lib2to3/fixes/fix_apply.py
@@ -34,6 +34,17 @@ class FixApply(fixer_base.BaseFix):
func = results["func"]
args = results["args"]
kwds = results.get("kwds")
+ # I feel like we should be able to express this logic in the
+ # PATTERN above but I don't know how to do it so...
+ if args:
+ if args.type == self.syms.star_expr:
+ return # Make no change.
+ if (args.type == self.syms.argument and
+ args.children[0].value == '**'):
+ return # Make no change.
+ if kwds and (kwds.type == self.syms.argument and
+ kwds.children[0].value == '**'):
+ return # Make no change.
prefix = node.prefix
func = func.clone()
if (func.type not in (token.NAME, syms.atom) and
diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py
index fb2973c..a852330 100644
--- a/Lib/lib2to3/fixes/fix_intern.py
+++ b/Lib/lib2to3/fixes/fix_intern.py
@@ -25,6 +25,16 @@ class FixIntern(fixer_base.BaseFix):
"""
def transform(self, node, results):
+ if results:
+ # I feel like we should be able to express this logic in the
+ # PATTERN above but I don't know how to do it so...
+ obj = results['obj']
+ if obj:
+ if obj.type == self.syms.star_expr:
+ return # Make no change.
+ if (obj.type == self.syms.argument and
+ obj.children[0].value == '**'):
+ return # Make no change.
names = ('sys', 'intern')
new = ImportAndCall(node, results, names)
touch_import(None, 'sys', node)
diff --git a/Lib/lib2to3/fixes/fix_reload.py b/Lib/lib2to3/fixes/fix_reload.py
index 1855357..b717273 100644
--- a/Lib/lib2to3/fixes/fix_reload.py
+++ b/Lib/lib2to3/fixes/fix_reload.py
@@ -22,6 +22,17 @@ class FixReload(fixer_base.BaseFix):
"""
def transform(self, node, results):
+ if results:
+ # I feel like we should be able to express this logic in the
+ # PATTERN above but I don't know how to do it so...
+ obj = results['obj']
+ print('obj:', repr(obj))
+ if obj:
+ if obj.type == self.syms.star_expr:
+ return # Make no change.
+ if (obj.type == self.syms.argument and
+ obj.children[0].value == '**'):
+ return # Make no change.
names = ('imp', 'reload')
new = ImportAndCall(node, results, names)
touch_import(None, 'imp', node)
diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py
index e69cf32..640dcef 100644
--- a/Lib/lib2to3/tests/test_fixers.py
+++ b/Lib/lib2to3/tests/test_fixers.py
@@ -260,6 +260,10 @@ class Test_apply(FixerTestCase):
s = """apply(f, *args)"""
self.unchanged(s)
+ def test_unchanged_6b(self):
+ s = """apply(f, **kwds)"""
+ self.unchanged(s)
+
def test_unchanged_7(self):
s = """apply(func=f, args=args, kwds=kwds)"""
self.unchanged(s)
diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py
index 0b2ca8b..e4a0194 100644
--- a/Lib/lib2to3/tests/test_parser.py
+++ b/Lib/lib2to3/tests/test_parser.py
@@ -208,6 +208,41 @@ class TestRaiseChanges(GrammarTest):
self.invalid_syntax("raise E from")
+# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292
+# and Lib/test/text_parser.py test_list_displays, test_set_displays,
+# test_dict_displays, test_argument_unpacking, ... changes.
+class TestUnpackingGeneralizations(GrammarTest):
+ def test_mid_positional_star(self):
+ self.validate("""func(1, *(2, 3), 4)""")
+
+ def test_double_star_dict_literal(self):
+ self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""")
+
+ def test_double_star_dict_literal_after_keywords(self):
+ self.validate("""func(spam='fried', **{'eggs':'scrambled'})""")
+
+ def test_list_display(self):
+ self.validate("""[*{2}, 3, *[4]]""")
+
+ def test_set_display(self):
+ self.validate("""{*{2}, 3, *[4]}""")
+
+ def test_dict_display_1(self):
+ self.validate("""{**{}}""")
+
+ def test_dict_display_2(self):
+ self.validate("""{**{}, 3:4, **{5:6, 7:8}}""")
+
+ def test_argument_unpacking_1(self):
+ self.validate("""f(a, *b, *c, d)""")
+
+ def test_argument_unpacking_2(self):
+ self.validate("""f(**a, **b)""")
+
+ def test_argument_unpacking_3(self):
+ self.validate("""f(2, *a, *b, **b, **c, **d)""")
+
+
# Adapted from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef
class TestFunctionAnnotations(GrammarTest):
def test_1(self):
diff --git a/Misc/NEWS b/Misc/NEWS
index fcc27ba..bce1688 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -65,6 +65,9 @@ Core and Builtins
Library
-------
+- Issue #25969: Update the lib2to3 grammar to handle the unpacking
+ generalizations added in 3.5.
+
- Issue #27932: Fixes memory leak in platform.win32_ver()
- Issue #14977: mailcap now respects the order of the lines in the mailcap