From 88c00132ccdced2645e78e0059eab93590adecbd Mon Sep 17 00:00:00 2001 From: "Gregory P. Smith ext:(%20%5BGoogle%20Inc.%5D)" Date: Fri, 9 Sep 2016 18:32:52 -0700 Subject: Issue #25969: Update the lib2to3 grammar to handle the unpacking generalizations added in 3.5. --- Lib/lib2to3/Grammar.txt | 23 +++++++--- Lib/lib2to3/fixes/fix_apply.py | 11 +++++ Lib/lib2to3/fixes/fix_intern.py | 10 +++++ Lib/lib2to3/tests/test_fixers.py | 96 ++-------------------------------------- Lib/lib2to3/tests/test_parser.py | 35 +++++++++++++++ Misc/NEWS | 3 ++ 6 files changed, 80 insertions(+), 98 deletions(-) diff --git a/Lib/lib2to3/Grammar.txt b/Lib/lib2to3/Grammar.txt index e667bcd..e151bac 100644 --- a/Lib/lib2to3/Grammar.txt +++ b/Lib/lib2to3/Grammar.txt @@ -136,15 +136,26 @@ subscript: test | [test] ':' [test] [sliceop] sliceop: ':' [test] exprlist: (expr|star_expr) (',' (expr|star_expr))* [','] testlist: test (',' test)* [','] -dictsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | - (test (comp_for | (',' test)* [','])) ) +dictsetmaker: ( ((test ':' test | '**' expr) + (comp_for | (',' (test ':' test | '**' expr))* [','])) | + ((test | star_expr) + (comp_for | (',' (test | star_expr))* [','])) ) classdef: 'class' NAME ['(' [arglist] ')'] ':' suite -arglist: (argument ',')* (argument [','] - |'*' test (',' argument)* [',' '**' test] - |'**' test) -argument: test [comp_for] | test '=' test # Really [keyword '='] test +arglist: argument (',' argument)* [','] + +# "test '=' test" is really "keyword '=' test", but we have no such token. +# These need to be in a single rule to avoid grammar that is ambiguous +# to our LL(1) parser. Even though 'test' includes '*expr' in star_expr, +# we explicitly match '*' here, too, to give it proper precedence. +# Illegal combinations and orderings are blocked in ast.c: +# multiple (test comp_for) arguements are blocked; keyword unpackings +# that precede iterable unpackings are blocked; etc. +argument: ( test [comp_for] | + test '=' test | + '**' expr | + star_expr ) comp_iter: comp_for | comp_if comp_for: 'for' exprlist 'in' testlist_safe [comp_iter] diff --git a/Lib/lib2to3/fixes/fix_apply.py b/Lib/lib2to3/fixes/fix_apply.py index a7dc3a0..1a465c2 100644 --- a/Lib/lib2to3/fixes/fix_apply.py +++ b/Lib/lib2to3/fixes/fix_apply.py @@ -34,6 +34,17 @@ class FixApply(fixer_base.BaseFix): func = results["func"] args = results["args"] kwds = results.get("kwds") + # I feel like we should be able to express this logic in the + # PATTERN above but I don't know how to do it so... + if args: + if args.type == self.syms.star_expr: + return # Make no change. + if (args.type == self.syms.argument and + args.children[0].value == '**'): + return # Make no change. + if kwds and (kwds.type == self.syms.argument and + kwds.children[0].value == '**'): + return # Make no change. prefix = node.prefix func = func.clone() if (func.type not in (token.NAME, syms.atom) and diff --git a/Lib/lib2to3/fixes/fix_intern.py b/Lib/lib2to3/fixes/fix_intern.py index e7bb505..285c126 100644 --- a/Lib/lib2to3/fixes/fix_intern.py +++ b/Lib/lib2to3/fixes/fix_intern.py @@ -26,6 +26,16 @@ class FixIntern(fixer_base.BaseFix): """ def transform(self, node, results): + if results: + # I feel like we should be able to express this logic in the + # PATTERN above but I don't know how to do it so... + obj = results['obj'] + if obj: + if obj.type == self.syms.star_expr: + return # Make no change. + if (obj.type == self.syms.argument and + obj.children[0].value == '**'): + return # Make no change. syms = self.syms obj = results["obj"].clone() if obj.type == syms.arglist: diff --git a/Lib/lib2to3/tests/test_fixers.py b/Lib/lib2to3/tests/test_fixers.py index 6fa603f..b0e60fe 100644 --- a/Lib/lib2to3/tests/test_fixers.py +++ b/Lib/lib2to3/tests/test_fixers.py @@ -260,6 +260,10 @@ class Test_apply(FixerTestCase): s = """apply(f, *args)""" self.unchanged(s) + def test_unchanged_6b(self): + s = """apply(f, **kwds)""" + self.unchanged(s) + def test_unchanged_7(self): s = """apply(func=f, args=args, kwds=kwds)""" self.unchanged(s) @@ -2861,98 +2865,6 @@ class Test_unicode(FixerTestCase): a = f + """r'\\\\\\u20ac\\U0001d121\\\\u20ac'""" self.check(b, a) -class Test_callable(FixerTestCase): - fixer = "callable" - - def test_prefix_preservation(self): - b = """callable( x)""" - a = """import collections\nisinstance( x, collections.Callable)""" - self.check(b, a) - - b = """if callable(x): pass""" - a = """import collections -if isinstance(x, collections.Callable): pass""" - self.check(b, a) - - def test_callable_call(self): - b = """callable(x)""" - a = """import collections\nisinstance(x, collections.Callable)""" - self.check(b, a) - - def test_global_import(self): - b = """ -def spam(foo): - callable(foo)"""[1:] - a = """ -import collections -def spam(foo): - isinstance(foo, collections.Callable)"""[1:] - self.check(b, a) - - b = """ -import collections -def spam(foo): - callable(foo)"""[1:] - # same output if it was already imported - self.check(b, a) - - b = """ -from collections import * -def spam(foo): - callable(foo)"""[1:] - a = """ -from collections import * -import collections -def spam(foo): - isinstance(foo, collections.Callable)"""[1:] - self.check(b, a) - - b = """ -do_stuff() -do_some_other_stuff() -assert callable(do_stuff)"""[1:] - a = """ -import collections -do_stuff() -do_some_other_stuff() -assert isinstance(do_stuff, collections.Callable)"""[1:] - self.check(b, a) - - b = """ -if isinstance(do_stuff, Callable): - assert callable(do_stuff) - do_stuff(do_stuff) - if not callable(do_stuff): - exit(1) - else: - assert callable(do_stuff) -else: - assert not callable(do_stuff)"""[1:] - a = """ -import collections -if isinstance(do_stuff, Callable): - assert isinstance(do_stuff, collections.Callable) - do_stuff(do_stuff) - if not isinstance(do_stuff, collections.Callable): - exit(1) - else: - assert isinstance(do_stuff, collections.Callable) -else: - assert not isinstance(do_stuff, collections.Callable)"""[1:] - self.check(b, a) - - def test_callable_should_not_change(self): - a = """callable(*x)""" - self.unchanged(a) - - a = """callable(x, y)""" - self.unchanged(a) - - a = """callable(x, kw=y)""" - self.unchanged(a) - - a = """callable()""" - self.unchanged(a) class Test_filter(FixerTestCase): fixer = "filter" diff --git a/Lib/lib2to3/tests/test_parser.py b/Lib/lib2to3/tests/test_parser.py index cf484a1..ebf8441 100644 --- a/Lib/lib2to3/tests/test_parser.py +++ b/Lib/lib2to3/tests/test_parser.py @@ -154,6 +154,41 @@ class TestRaiseChanges(GrammarTest): self.invalid_syntax("raise E from") +# Modelled after Lib/test/test_grammar.py:TokenTests.test_funcdef issue2292 +# and Lib/test/text_parser.py test_list_displays, test_set_displays, +# test_dict_displays, test_argument_unpacking, ... changes. +class TestUnpackingGeneralizations(GrammarTest): + def test_mid_positional_star(self): + self.validate("""func(1, *(2, 3), 4)""") + + def test_double_star_dict_literal(self): + self.validate("""func(**{'eggs':'scrambled', 'spam':'fried'})""") + + def test_double_star_dict_literal_after_keywords(self): + self.validate("""func(spam='fried', **{'eggs':'scrambled'})""") + + def test_list_display(self): + self.validate("""[*{2}, 3, *[4]]""") + + def test_set_display(self): + self.validate("""{*{2}, 3, *[4]}""") + + def test_dict_display_1(self): + self.validate("""{**{}}""") + + def test_dict_display_2(self): + self.validate("""{**{}, 3:4, **{5:6, 7:8}}""") + + def test_argument_unpacking_1(self): + self.validate("""f(a, *b, *c, d)""") + + def test_argument_unpacking_2(self): + self.validate("""f(**a, **b)""") + + def test_argument_unpacking_3(self): + self.validate("""f(2, *a, *b, **b, **c, **d)""") + + # Adaptated from Python 3's Lib/test/test_grammar.py:GrammarTests.testFuncdef class TestFunctionAnnotations(GrammarTest): def test_1(self): diff --git a/Misc/NEWS b/Misc/NEWS index 5f3eccb..f6acbea 100644 --- a/Misc/NEWS +++ b/Misc/NEWS @@ -42,6 +42,9 @@ Core and Builtins Library ------- +- Issue #25969: Update the lib2to3 grammar to handle the unpacking + generalizations added in 3.5. + - Issue #24594: Validates persist parameter when opening MSI database - Issue #27570: Avoid zero-length memcpy() etc calls with null source -- cgit v0.12