diff options
| -rw-r--r-- | Lib/test/test_tools/test_com2ann.py | 260 | ||||
| -rw-r--r-- | Tools/parser/com2ann.py | 308 |
2 files changed, 0 insertions, 568 deletions
diff --git a/Lib/test/test_tools/test_com2ann.py b/Lib/test/test_tools/test_com2ann.py deleted file mode 100644 index 2731f82..0000000 --- a/Lib/test/test_tools/test_com2ann.py +++ /dev/null @@ -1,260 +0,0 @@ -"""Tests for the com2ann.py script in the Tools/parser directory.""" - -import unittest -import test.support -import os -import re - -from test.test_tools import basepath, toolsdir, skip_if_missing - -skip_if_missing() - -parser_path = os.path.join(toolsdir, "parser") - -with test.support.DirsOnSysPath(parser_path): - from com2ann import * - -class BaseTestCase(unittest.TestCase): - - def check(self, code, expected, n=False, e=False): - self.assertEqual(com2ann(code, - drop_None=n, drop_Ellipsis=e, silent=True), - expected) - -class SimpleTestCase(BaseTestCase): - # Tests for basic conversions - - def test_basics(self): - self.check("z = 5", "z = 5") - self.check("z: int = 5", "z: int = 5") - self.check("z = 5 # type: int", "z: int = 5") - self.check("z = 5 # type: int # comment", - "z: int = 5 # comment") - - def test_type_ignore(self): - self.check("foobar = foobaz() #type: ignore", - "foobar = foobaz() #type: ignore") - self.check("a = 42 #type: ignore #comment", - "a = 42 #type: ignore #comment") - - def test_complete_tuple(self): - self.check("t = 1, 2, 3 # type: Tuple[int, ...]", - "t: Tuple[int, ...] = (1, 2, 3)") - self.check("t = 1, # type: Tuple[int]", - "t: Tuple[int] = (1,)") - self.check("t = (1, 2, 3) # type: Tuple[int, ...]", - "t: Tuple[int, ...] = (1, 2, 3)") - - def test_drop_None(self): - self.check("x = None # type: int", - "x: int", True) - self.check("x = None # type: int # another", - "x: int # another", True) - self.check("x = None # type: int # None", - "x: int # None", True) - - def test_drop_Ellipsis(self): - self.check("x = ... # type: int", - "x: int", False, True) - self.check("x = ... # type: int # another", - "x: int # another", False, True) - self.check("x = ... # type: int # ...", - "x: int # ...", False, True) - - def test_newline(self): - self.check("z = 5 # type: int\r\n", "z: int = 5\r\n") - self.check("z = 5 # type: int # comment\x85", - "z: int = 5 # comment\x85") - - def test_wrong(self): - self.check("#type : str", "#type : str") - self.check("x==y #type: bool", "x==y #type: bool") - - def test_pattern(self): - for line in ["#type: int", " # type: str[:] # com"]: - self.assertTrue(re.search(TYPE_COM, line)) - for line in ["", "#", "# comment", "#type", "type int:"]: - self.assertFalse(re.search(TYPE_COM, line)) - -class BigTestCase(BaseTestCase): - # Tests for really crazy formatting, to be sure - # that script works reasonably in extreme situations - - def test_crazy(self): - self.maxDiff = None - self.check(crazy_code, big_result, False, False) - self.check(crazy_code, big_result_ne, True, True) - -crazy_code = """\ -# -*- coding: utf-8 -*- # this should not be spoiled -''' -Docstring here -''' - -import testmod -x = 5 #type : int # this one is OK -ttt \\ - = \\ - 1.0, \\ - 2.0, \\ - 3.0, #type: Tuple[float, float, float] -with foo(x==1) as f: #type: str - print(f) - -for i, j in my_inter(x=1): # type: ignore - i + j # type: int # what about this - -x = y = z = 1 # type: int -x, y, z = [], [], [] # type: (List[int], List[int], List[str]) -class C: - - - l[f(x - =1)] = [ - - 1, - 2, - ] # type: List[int] - - - (C.x[1]) = \\ - 42 == 5# type: bool -lst[...] = \\ - ((\\ -...)) # type: int # comment .. - -y = ... # type: int # comment ... -z = ... -#type: int - - -#DONE placement of annotation after target rather than before = - -TD.x[1] \\ - = 0 == 5# type: bool - -TD.y[1] =5 == 5# type: bool # one more here -F[G(x == y, - -# hm... - - z)]\\ - = None # type: OMG[int] # comment: None -x = None#type:int #comment : None""" - -big_result = """\ -# -*- coding: utf-8 -*- # this should not be spoiled -''' -Docstring here -''' - -import testmod -x: int = 5 # this one is OK -ttt: Tuple[float, float, float] \\ - = \\ - (1.0, \\ - 2.0, \\ - 3.0,) -with foo(x==1) as f: #type: str - print(f) - -for i, j in my_inter(x=1): # type: ignore - i + j # type: int # what about this - -x = y = z = 1 # type: int -x, y, z = [], [], [] # type: (List[int], List[int], List[str]) -class C: - - - l[f(x - =1)]: List[int] = [ - - 1, - 2, - ] - - - (C.x[1]): bool = \\ - 42 == 5 -lst[...]: int = \\ - ((\\ -...)) # comment .. - -y: int = ... # comment ... -z = ... -#type: int - - -#DONE placement of annotation after target rather than before = - -TD.x[1]: bool \\ - = 0 == 5 - -TD.y[1]: bool =5 == 5 # one more here -F[G(x == y, - -# hm... - - z)]: OMG[int]\\ - = None # comment: None -x: int = None #comment : None""" - -big_result_ne = """\ -# -*- coding: utf-8 -*- # this should not be spoiled -''' -Docstring here -''' - -import testmod -x: int = 5 # this one is OK -ttt: Tuple[float, float, float] \\ - = \\ - (1.0, \\ - 2.0, \\ - 3.0,) -with foo(x==1) as f: #type: str - print(f) - -for i, j in my_inter(x=1): # type: ignore - i + j # type: int # what about this - -x = y = z = 1 # type: int -x, y, z = [], [], [] # type: (List[int], List[int], List[str]) -class C: - - - l[f(x - =1)]: List[int] = [ - - 1, - 2, - ] - - - (C.x[1]): bool = \\ - 42 == 5 -lst[...]: int \\ - \\ - # comment .. - -y: int # comment ... -z = ... -#type: int - - -#DONE placement of annotation after target rather than before = - -TD.x[1]: bool \\ - = 0 == 5 - -TD.y[1]: bool =5 == 5 # one more here -F[G(x == y, - -# hm... - - z)]: OMG[int]\\ - # comment: None -x: int #comment : None""" - -if __name__ == '__main__': - unittest.main() diff --git a/Tools/parser/com2ann.py b/Tools/parser/com2ann.py deleted file mode 100644 index b5868f4..0000000 --- a/Tools/parser/com2ann.py +++ /dev/null @@ -1,308 +0,0 @@ -"""Helper module to tranlate 3.5 type comments to 3.6 variable annotations.""" -import re -import os -import ast -import argparse -import tokenize -from collections import defaultdict -from textwrap import dedent -from io import BytesIO - -__all__ = ['com2ann', 'TYPE_COM'] - -TYPE_COM = re.compile(r'\s*#\s*type\s*:.*$', flags=re.DOTALL) -TRAIL_OR_COM = re.compile(r'\s*$|\s*#.*$', flags=re.DOTALL) - - -class _Data: - """Internal class describing global data on file.""" - def __init__(self, lines, tokens): - self.lines = lines - self.tokens = tokens - ttab = defaultdict(list) # maps line number to token numbers - for i, tok in enumerate(tokens): - ttab[tok.start[0]].append(i) - self.ttab = ttab - self.success = [] # list of lines where type comments where processed - self.fail = [] # list of lines where type comments where rejected - - -def skip_blank(d, lno): - while d.lines[lno].strip() == '': - lno += 1 - return lno - - -def find_start(d, lcom): - """Find first char of the assignment target.""" - i = d.ttab[lcom + 1][-2] # index of type comment token in tokens list - while ((d.tokens[i].exact_type != tokenize.NEWLINE) and - (d.tokens[i].exact_type != tokenize.ENCODING)): - i -= 1 - lno = d.tokens[i].start[0] - return skip_blank(d, lno) - - -def check_target(stmt): - if len(stmt.body): - assign = stmt.body[0] - else: - return False - if isinstance(assign, ast.Assign) and len(assign.targets) == 1: - targ = assign.targets[0] - else: - return False - if (isinstance(targ, ast.Name) or isinstance(targ, ast.Attribute) - or isinstance(targ, ast.Subscript)): - return True - return False - - -def find_eq(d, lstart): - """Find equal sign starting from lstart taking care about d[f(x=1)] = 5.""" - col = pars = 0 - lno = lstart - while d.lines[lno][col] != '=' or pars != 0: - ch = d.lines[lno][col] - if ch in '([{': - pars += 1 - elif ch in ')]}': - pars -= 1 - if ch == '#' or col == len(d.lines[lno])-1: - lno = skip_blank(d, lno+1) - col = 0 - else: - col += 1 - return lno, col - - -def find_val(d, poseq): - """Find position of first char of assignment value starting from poseq.""" - lno, col = poseq - while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'): - if col == len(d.lines[lno])-1: - lno += 1 - col = 0 - else: - col += 1 - return lno, col - - -def find_targ(d, poseq): - """Find position of last char of target (annotation goes here).""" - lno, col = poseq - while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'): - if col == 0: - lno -= 1 - col = len(d.lines[lno])-1 - else: - col -= 1 - return lno, col+1 - - -def trim(new_lines, string, ltarg, poseq, lcom, ccom): - """Remove None or Ellipsis from assignment value. - - Also remove parens if one has (None), (...) etc. - string -- 'None' or '...' - ltarg -- line where last char of target is located - poseq -- position of equal sign - lcom, ccom -- position of type comment - """ - nopars = lambda s: s.replace('(', '').replace(')', '') - leq, ceq = poseq - end = ccom if leq == lcom else len(new_lines[leq]) - subline = new_lines[leq][:ceq] - if leq == ltarg: - subline = subline.rstrip() - new_lines[leq] = subline + (new_lines[leq][end:] if leq == lcom - else new_lines[leq][ceq+1:end]) - - for lno in range(leq+1,lcom): - new_lines[lno] = nopars(new_lines[lno]) - - if lcom != leq: - subline = nopars(new_lines[lcom][:ccom]).replace(string, '') - if (not subline.isspace()): - subline = subline.rstrip() - new_lines[lcom] = subline + new_lines[lcom][ccom:] - - -def _com2ann(d, drop_None, drop_Ellipsis): - new_lines = d.lines[:] - for lcom, line in enumerate(d.lines): - match = re.search(TYPE_COM, line) - if match: - # strip " # type : annotation \n" -> "annotation \n" - tp = match.group().lstrip()[1:].lstrip()[4:].lstrip()[1:].lstrip() - submatch = re.search(TRAIL_OR_COM, tp) - subcom = '' - if submatch and submatch.group(): - subcom = submatch.group() - tp = tp[:submatch.start()] - if tp == 'ignore': - continue - ccom = match.start() - if not any(d.tokens[i].exact_type == tokenize.COMMENT - for i in d.ttab[lcom + 1]): - d.fail.append(lcom) - continue # type comment inside string - lstart = find_start(d, lcom) - stmt_str = dedent(''.join(d.lines[lstart:lcom+1])) - try: - stmt = ast.parse(stmt_str) - except SyntaxError: - d.fail.append(lcom) - continue # for or with statements - if not check_target(stmt): - d.fail.append(lcom) - continue - - d.success.append(lcom) - val = stmt.body[0].value - - # writing output now - poseq = find_eq(d, lstart) - lval, cval = find_val(d, poseq) - ltarg, ctarg = find_targ(d, poseq) - - op_par = '' - cl_par = '' - if isinstance(val, ast.Tuple): - if d.lines[lval][cval] != '(': - op_par = '(' - cl_par = ')' - # write the comment first - new_lines[lcom] = d.lines[lcom][:ccom].rstrip() + cl_par + subcom - ccom = len(d.lines[lcom][:ccom].rstrip()) - - string = False - if isinstance(val, ast.Tuple): - # t = 1, 2 -> t = (1, 2); only latter is allowed with annotation - free_place = int(new_lines[lval][cval-2:cval] == ' ') - new_lines[lval] = (new_lines[lval][:cval-free_place] + - op_par + new_lines[lval][cval:]) - elif isinstance(val, ast.Ellipsis) and drop_Ellipsis: - string = '...' - elif (isinstance(val, ast.NameConstant) and - val.value is None and drop_None): - string = 'None' - if string: - trim(new_lines, string, ltarg, poseq, lcom, ccom) - - # finally write an annotation - new_lines[ltarg] = (new_lines[ltarg][:ctarg] + - ': ' + tp + new_lines[ltarg][ctarg:]) - return ''.join(new_lines) - - -def com2ann(code, *, drop_None=False, drop_Ellipsis=False, silent=False): - """Translate type comments to type annotations in code. - - Take code as string and return this string where:: - - variable = value # type: annotation # real comment - - is translated to:: - - variable: annotation = value # real comment - - For unsupported syntax cases, the type comments are - left intact. If drop_None is True or if drop_Ellipsis - is True translate correcpondingly:: - - variable = None # type: annotation - variable = ... # type: annotation - - into:: - - variable: annotation - - The tool tries to preserve code formatting as much as - possible, but an exact translation is not guarateed. - A summary of translated comments id printed by default. - """ - try: - ast.parse(code) # we want to work only with file without syntax errors - except SyntaxError: - return None - lines = code.splitlines(keepends=True) - rl = BytesIO(code.encode('utf-8')).readline - tokens = list(tokenize.tokenize(rl)) - - data = _Data(lines, tokens) - new_code = _com2ann(data, drop_None, drop_Ellipsis) - - if not silent: - if data.success: - print('Comments translated on lines:', - ', '.join(str(lno+1) for lno in data.success)) - if data.fail: - print('Comments rejected on lines:', - ', '.join(str(lno+1) for lno in data.fail)) - if not data.success and not data.fail: - print('No type comments found') - - return new_code - - -def translate_file(infile, outfile, dnone, dell, silent): - try: - descr = tokenize.open(infile) - except SyntaxError: - print("Cannot open", infile) - return - with descr as f: - code = f.read() - enc = f.encoding - if not silent: - print('File:', infile) - new_code = com2ann(code, drop_None=dnone, - drop_Ellipsis=dell, - silent=silent) - if new_code is None: - print("SyntaxError in", infile) - return - with open(outfile, 'wb') as f: - f.write((new_code).encode(enc)) - - -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("-o", "--outfile", - help="output file, will be overwritten if exists,\n" - "defaults to input file") - parser.add_argument("infile", - help="input file or directory for translation, must\n" - "contain no syntax errors, for directory\n" - "the outfile is ignored and translation is\n" - "made in place") - parser.add_argument("-s", "--silent", - help="Do not print summary for line numbers of\n" - "translated and rejected comments", - action="store_true") - parser.add_argument("-n", "--drop-none", - help="drop any None as assignment value during\n" - "translation if it is annotated by a type coment", - action="store_true") - parser.add_argument("-e", "--drop-ellipsis", - help="drop any Ellipsis (...) as assignment value during\n" - "translation if it is annotated by a type coment", - action="store_true") - args = parser.parse_args() - if args.outfile is None: - args.outfile = args.infile - - if os.path.isfile(args.infile): - translate_file(args.infile, args.outfile, - args.drop_none, args.drop_ellipsis, args.silent) - else: - for root, dirs, files in os.walk(args.infile): - for afile in files: - _, ext = os.path.splitext(afile) - if ext == '.py' or ext == '.pyi': - fname = os.path.join(root, afile) - translate_file(fname, fname, - args.drop_none, args.drop_ellipsis, - args.silent) |