Issue #28038: Remove Tools/parser/com2ann.py and its unit test.

Development is moving to https://github.com/ilevkivskyi/com2ann

2 files changed, 0 insertions, 568 deletions

diff --git a/Lib/test/test_tools/test_com2ann.py b/Lib/test/test_tools/test_com2ann.py
deleted file mode 100644
index 2731f82..0000000
--- a/Lib/test/test_tools/test_com2ann.py
+++ /dev/null
@@ -1,260 +0,0 @@
-"""Tests for the com2ann.py script in the Tools/parser directory."""
-
-import unittest
-import test.support
-import os
-import re
-
-from test.test_tools import basepath, toolsdir, skip_if_missing
-
-skip_if_missing()
-
-parser_path = os.path.join(toolsdir, "parser")
-
-with test.support.DirsOnSysPath(parser_path):
-    from com2ann import *
-
-class BaseTestCase(unittest.TestCase):
-
-    def check(self, code, expected, n=False, e=False):
-        self.assertEqual(com2ann(code,
-                         drop_None=n, drop_Ellipsis=e, silent=True),
-                         expected)
-
-class SimpleTestCase(BaseTestCase):
-    # Tests for basic conversions
-
-    def test_basics(self):
-        self.check("z = 5", "z = 5")
-        self.check("z: int = 5", "z: int = 5")
-        self.check("z = 5 # type: int", "z: int = 5")
-        self.check("z = 5 # type: int # comment",
-                   "z: int = 5 # comment")
-
-    def test_type_ignore(self):
-        self.check("foobar = foobaz() #type: ignore",
-                   "foobar = foobaz() #type: ignore")
-        self.check("a = 42 #type: ignore #comment",
-                   "a = 42 #type: ignore #comment")
-
-    def test_complete_tuple(self):
-        self.check("t = 1, 2, 3 # type: Tuple[int, ...]",
-                   "t: Tuple[int, ...] = (1, 2, 3)")
-        self.check("t = 1, # type: Tuple[int]",
-                   "t: Tuple[int] = (1,)")
-        self.check("t = (1, 2, 3) # type: Tuple[int, ...]",
-                   "t: Tuple[int, ...] = (1, 2, 3)")
-
-    def test_drop_None(self):
-        self.check("x = None # type: int",
-                   "x: int", True)
-        self.check("x = None # type: int # another",
-                   "x: int # another", True)
-        self.check("x = None # type: int # None",
-                   "x: int # None", True)
-
-    def test_drop_Ellipsis(self):
-        self.check("x = ... # type: int",
-                   "x: int", False, True)
-        self.check("x = ... # type: int # another",
-                   "x: int # another", False, True)
-        self.check("x = ... # type: int # ...",
-                   "x: int # ...", False, True)
-
-    def test_newline(self):
-        self.check("z = 5 # type: int\r\n", "z: int = 5\r\n")
-        self.check("z = 5 # type: int # comment\x85",
-                   "z: int = 5 # comment\x85")
-
-    def test_wrong(self):
-        self.check("#type : str", "#type : str")
-        self.check("x==y #type: bool", "x==y #type: bool")
-
-    def test_pattern(self):
-        for line in ["#type: int", "  # type:  str[:] # com"]:
-            self.assertTrue(re.search(TYPE_COM, line))
-        for line in ["", "#", "# comment", "#type", "type int:"]:
-            self.assertFalse(re.search(TYPE_COM, line))
-
-class BigTestCase(BaseTestCase):
-    # Tests for really crazy formatting, to be sure
-    # that script works reasonably in extreme situations
-
-    def test_crazy(self):
-        self.maxDiff = None
-        self.check(crazy_code, big_result, False, False)
-        self.check(crazy_code, big_result_ne, True, True)
-
-crazy_code = """\
-# -*- coding: utf-8 -*- # this should not be spoiled
-'''
-Docstring here
-'''
-
-import testmod
-x = 5 #type    : int # this one is OK
-ttt \\
-    = \\
-        1.0, \\
-        2.0, \\
-        3.0, #type: Tuple[float, float, float]
-with foo(x==1) as f: #type: str
-    print(f)
-
-for i, j in my_inter(x=1): # type: ignore
-    i + j # type: int # what about this
-
-x = y = z = 1 # type: int
-x, y, z = [], [], []  # type: (List[int], List[int], List[str])
-class C:
-
-
-    l[f(x
-        =1)] = [
-
-         1,
-         2,
-         ]  # type: List[int]
-
-
-    (C.x[1]) = \\
-        42 == 5# type: bool
-lst[...] = \\
-    ((\\
-...)) # type: int # comment ..
-
-y = ... # type: int # comment ...
-z = ...
-#type: int
-
-
-#DONE placement of annotation after target rather than before =
-
-TD.x[1]  \\
-    = 0 == 5# type: bool
-
-TD.y[1] =5 == 5# type: bool # one more here
-F[G(x == y,
-
-# hm...
-
-    z)]\\
-      = None # type: OMG[int] # comment: None
-x = None#type:int   #comment : None"""
-
-big_result = """\
-# -*- coding: utf-8 -*- # this should not be spoiled
-'''
-Docstring here
-'''
-
-import testmod
-x: int = 5 # this one is OK
-ttt: Tuple[float, float, float] \\
-    = \\
-       (1.0, \\
-        2.0, \\
-        3.0,)
-with foo(x==1) as f: #type: str
-    print(f)
-
-for i, j in my_inter(x=1): # type: ignore
-    i + j # type: int # what about this
-
-x = y = z = 1 # type: int
-x, y, z = [], [], []  # type: (List[int], List[int], List[str])
-class C:
-
-
-    l[f(x
-        =1)]: List[int] = [
-
-         1,
-         2,
-         ]
-
-
-    (C.x[1]): bool = \\
-        42 == 5
-lst[...]: int = \\
-    ((\\
-...)) # comment ..
-
-y: int = ... # comment ...
-z = ...
-#type: int
-
-
-#DONE placement of annotation after target rather than before =
-
-TD.x[1]: bool  \\
-    = 0 == 5
-
-TD.y[1]: bool =5 == 5 # one more here
-F[G(x == y,
-
-# hm...
-
-    z)]: OMG[int]\\
-      = None # comment: None
-x: int = None   #comment : None"""
-
-big_result_ne = """\
-# -*- coding: utf-8 -*- # this should not be spoiled
-'''
-Docstring here
-'''
-
-import testmod
-x: int = 5 # this one is OK
-ttt: Tuple[float, float, float] \\
-    = \\
-       (1.0, \\
-        2.0, \\
-        3.0,)
-with foo(x==1) as f: #type: str
-    print(f)
-
-for i, j in my_inter(x=1): # type: ignore
-    i + j # type: int # what about this
-
-x = y = z = 1 # type: int
-x, y, z = [], [], []  # type: (List[int], List[int], List[str])
-class C:
-
-
-    l[f(x
-        =1)]: List[int] = [
-
-         1,
-         2,
-         ]
-
-
-    (C.x[1]): bool = \\
-        42 == 5
-lst[...]: int \\
-    \\
- # comment ..
-
-y: int # comment ...
-z = ...
-#type: int
-
-
-#DONE placement of annotation after target rather than before =
-
-TD.x[1]: bool  \\
-    = 0 == 5
-
-TD.y[1]: bool =5 == 5 # one more here
-F[G(x == y,
-
-# hm...
-
-    z)]: OMG[int]\\
-       # comment: None
-x: int   #comment : None"""
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/Tools/parser/com2ann.py b/Tools/parser/com2ann.py
deleted file mode 100644
index b5868f4..0000000
--- a/Tools/parser/com2ann.py
+++ /dev/null
@@ -1,308 +0,0 @@
-"""Helper module to tranlate 3.5 type comments to 3.6 variable annotations."""
-import re
-import os
-import ast
-import argparse
-import tokenize
-from collections import defaultdict
-from textwrap import dedent
-from io import BytesIO
-
-__all__ = ['com2ann', 'TYPE_COM']
-
-TYPE_COM = re.compile(r'\s*#\s*type\s*:.*$', flags=re.DOTALL)
-TRAIL_OR_COM = re.compile(r'\s*$|\s*#.*$', flags=re.DOTALL)
-
-
-class _Data:
-    """Internal class describing global data on file."""
-    def __init__(self, lines, tokens):
-        self.lines = lines
-        self.tokens = tokens
-        ttab = defaultdict(list) # maps line number to token numbers
-        for i, tok in enumerate(tokens):
-            ttab[tok.start[0]].append(i)
-        self.ttab = ttab
-        self.success = [] # list of lines where type comments where processed
-        self.fail = [] # list of lines where type comments where rejected
-
-
-def skip_blank(d, lno):
-    while d.lines[lno].strip() == '':
-        lno += 1
-    return lno
-
-
-def find_start(d, lcom):
-    """Find first char of the assignment target."""
-    i = d.ttab[lcom + 1][-2] # index of type comment token in tokens list
-    while ((d.tokens[i].exact_type != tokenize.NEWLINE) and
-           (d.tokens[i].exact_type != tokenize.ENCODING)):
-        i -= 1
-    lno = d.tokens[i].start[0]
-    return skip_blank(d, lno)
-
-
-def check_target(stmt):
-    if len(stmt.body):
-        assign = stmt.body[0]
-    else:
-        return False
-    if isinstance(assign, ast.Assign) and len(assign.targets) == 1:
-        targ = assign.targets[0]
-    else:
-        return False
-    if (isinstance(targ, ast.Name) or isinstance(targ, ast.Attribute)
-        or isinstance(targ, ast.Subscript)):
-        return True
-    return False
-
-
-def find_eq(d, lstart):
-    """Find equal sign starting from lstart taking care about d[f(x=1)] = 5."""
-    col = pars = 0
-    lno = lstart
-    while d.lines[lno][col] != '=' or pars != 0:
-        ch = d.lines[lno][col]
-        if ch in '([{':
-            pars += 1
-        elif ch in ')]}':
-            pars -= 1
-        if ch == '#' or col == len(d.lines[lno])-1:
-            lno = skip_blank(d, lno+1)
-            col = 0
-        else:
-            col += 1
-    return lno, col
-
-
-def find_val(d, poseq):
-    """Find position of first char of assignment value starting from poseq."""
-    lno, col = poseq
-    while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'):
-        if col == len(d.lines[lno])-1:
-            lno += 1
-            col = 0
-        else:
-            col += 1
-    return lno, col
-
-
-def find_targ(d, poseq):
-    """Find position of last char of target (annotation goes here)."""
-    lno, col = poseq
-    while (d.lines[lno][col].isspace() or d.lines[lno][col] in '=\\'):
-        if col == 0:
-            lno -= 1
-            col = len(d.lines[lno])-1
-        else:
-            col -= 1
-    return lno, col+1
-
-
-def trim(new_lines, string, ltarg, poseq, lcom, ccom):
-    """Remove None or Ellipsis from assignment value.
-
-    Also remove parens if one has (None), (...) etc.
-    string -- 'None' or '...'
-    ltarg -- line where last char of target is located
-    poseq -- position of equal sign
-    lcom, ccom -- position of type comment
-    """
-    nopars = lambda s: s.replace('(', '').replace(')', '')
-    leq, ceq = poseq
-    end = ccom if leq == lcom else len(new_lines[leq])
-    subline = new_lines[leq][:ceq]
-    if leq == ltarg:
-        subline = subline.rstrip()
-    new_lines[leq] = subline + (new_lines[leq][end:] if leq == lcom
-                                else new_lines[leq][ceq+1:end])
-
-    for lno in range(leq+1,lcom):
-        new_lines[lno] = nopars(new_lines[lno])
-
-    if lcom != leq:
-        subline = nopars(new_lines[lcom][:ccom]).replace(string, '')
-        if (not subline.isspace()):
-            subline = subline.rstrip()
-        new_lines[lcom] = subline + new_lines[lcom][ccom:]
-
-
-def _com2ann(d, drop_None, drop_Ellipsis):
-    new_lines = d.lines[:]
-    for lcom, line in enumerate(d.lines):
-        match = re.search(TYPE_COM, line)
-        if match:
-            # strip " #  type  :  annotation  \n" -> "annotation  \n"
-            tp = match.group().lstrip()[1:].lstrip()[4:].lstrip()[1:].lstrip()
-            submatch = re.search(TRAIL_OR_COM, tp)
-            subcom = ''
-            if submatch and submatch.group():
-                subcom = submatch.group()
-                tp = tp[:submatch.start()]
-            if tp == 'ignore':
-                continue
-            ccom = match.start()
-            if not any(d.tokens[i].exact_type == tokenize.COMMENT
-                   for i in d.ttab[lcom + 1]):
-                d.fail.append(lcom)
-                continue # type comment inside string
-            lstart = find_start(d, lcom)
-            stmt_str = dedent(''.join(d.lines[lstart:lcom+1]))
-            try:
-                stmt = ast.parse(stmt_str)
-            except SyntaxError:
-                d.fail.append(lcom)
-                continue # for or with statements
-            if not check_target(stmt):
-                d.fail.append(lcom)
-                continue
-
-            d.success.append(lcom)
-            val = stmt.body[0].value
-
-            # writing output now
-            poseq = find_eq(d, lstart)
-            lval, cval = find_val(d, poseq)
-            ltarg, ctarg = find_targ(d, poseq)
-
-            op_par = ''
-            cl_par = ''
-            if isinstance(val, ast.Tuple):
-                if d.lines[lval][cval] != '(':
-                    op_par = '('
-                    cl_par = ')'
-            # write the comment first
-            new_lines[lcom] = d.lines[lcom][:ccom].rstrip() + cl_par + subcom
-            ccom = len(d.lines[lcom][:ccom].rstrip())
-
-            string = False
-            if isinstance(val, ast.Tuple):
-            # t = 1, 2 -> t = (1, 2); only latter is allowed with annotation
-                free_place = int(new_lines[lval][cval-2:cval] == '  ')
-                new_lines[lval] = (new_lines[lval][:cval-free_place] +
-                                       op_par + new_lines[lval][cval:])
-            elif isinstance(val, ast.Ellipsis) and drop_Ellipsis:
-                string = '...'
-            elif (isinstance(val, ast.NameConstant) and
-                        val.value is None and drop_None):
-                string = 'None'
-            if string:
-                trim(new_lines, string, ltarg, poseq, lcom, ccom)
-
-            # finally write an annotation
-            new_lines[ltarg] = (new_lines[ltarg][:ctarg] +
-                              ': ' + tp + new_lines[ltarg][ctarg:])
-    return ''.join(new_lines)
-
-
-def com2ann(code, *, drop_None=False, drop_Ellipsis=False, silent=False):
-    """Translate type comments to type annotations in code.
-
-    Take code as string and return this string where::
-
-      variable = value # type: annotation # real comment
-
-    is translated to::
-
-      variable: annotation = value # real comment
-
-    For unsupported syntax cases, the type comments are
-    left intact. If drop_None is True or if drop_Ellipsis
-    is True translate correcpondingly::
-
-      variable = None # type: annotation
-      variable = ... # type: annotation
-
-    into::
-
-      variable: annotation
-
-    The tool tries to preserve code formatting as much as
-    possible, but an exact translation is not guarateed.
-    A summary of translated comments id printed by default.
-    """
-    try:
-        ast.parse(code) # we want to work only with file without syntax errors
-    except SyntaxError:
-        return None
-    lines = code.splitlines(keepends=True)
-    rl = BytesIO(code.encode('utf-8')).readline
-    tokens = list(tokenize.tokenize(rl))
-
-    data = _Data(lines, tokens)
-    new_code = _com2ann(data, drop_None, drop_Ellipsis)
-
-    if not silent:
-        if data.success:
-            print('Comments translated on lines:',
-                  ', '.join(str(lno+1) for lno in data.success))
-        if data.fail:
-            print('Comments rejected on lines:',
-                  ', '.join(str(lno+1) for lno in data.fail))
-        if not data.success and not data.fail:
-            print('No type comments found')
-
-    return new_code
-
-
-def translate_file(infile, outfile, dnone, dell, silent):
-    try:
-        descr = tokenize.open(infile)
-    except SyntaxError:
-        print("Cannot open", infile)
-        return
-    with descr as f:
-        code = f.read()
-        enc = f.encoding
-    if not silent:
-        print('File:', infile)
-    new_code = com2ann(code, drop_None=dnone,
-                             drop_Ellipsis=dell,
-                             silent=silent)
-    if new_code is None:
-        print("SyntaxError in", infile)
-        return
-    with open(outfile, 'wb') as f:
-        f.write((new_code).encode(enc))
-
-
-if __name__ == '__main__':
-
-    parser = argparse.ArgumentParser(description=__doc__)
-    parser.add_argument("-o", "--outfile",
-                        help="output file, will be overwritten if exists,\n"
-                             "defaults to input file")
-    parser.add_argument("infile",
-                        help="input file or directory for translation, must\n"
-                             "contain no syntax errors, for directory\n"
-                             "the outfile is ignored and translation is\n"
-                             "made in place")
-    parser.add_argument("-s", "--silent",
-                        help="Do not print summary for line numbers of\n"
-                             "translated and rejected comments",
-                        action="store_true")
-    parser.add_argument("-n", "--drop-none",
-                   help="drop any None as assignment value during\n"
-                        "translation if it is annotated by a type coment",
-                   action="store_true")
-    parser.add_argument("-e", "--drop-ellipsis",
-                   help="drop any Ellipsis (...) as assignment value during\n"
-                        "translation if it is annotated by a type coment",
-                   action="store_true")
-    args = parser.parse_args()
-    if args.outfile is None:
-        args.outfile = args.infile
-
-    if os.path.isfile(args.infile):
-        translate_file(args.infile, args.outfile,
-                       args.drop_none, args.drop_ellipsis, args.silent)
-    else:
-        for root, dirs, files in os.walk(args.infile):
-            for afile in files:
-                _, ext = os.path.splitext(afile)
-                if ext == '.py' or ext == '.pyi':
-                    fname = os.path.join(root, afile)
-                    translate_file(fname, fname,
-                                   args.drop_none, args.drop_ellipsis,
-                                   args.silent)