Issue #25317: Converted doctests in test_tokenize to unittests.

1 files changed, 295 insertions, 251 deletions

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 01ef839..fd9486b 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1,20 +1,42 @@
-doctests = """
-Tests for the tokenize module.
+from test import test_support
+from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
+                     STRING, ENDMARKER, tok_name, Untokenizer, tokenize)
+from StringIO import StringIO
+import os
+from unittest import TestCase
+
+
+class TokenizeTest(TestCase):
+    # Tests for the tokenize module.
 
-    >>> import glob, random, sys
+    # The tests can be really simple. Given a small fragment of source
+    # code, print out a table with tokens. The ENDMARKER is omitted for
+    # brevity.
 
-The tests can be really simple. Given a small fragment of source
-code, print out a table with tokens. The ENDMARKER is omitted for
-brevity.
+    def check_tokenize(self, s, expected):
+        # Format the tokens in s in a table format.
+        # The ENDMARKER is omitted.
+        result = []
+        f = StringIO(s)
+        for type, token, start, end, line in generate_tokens(f.readline):
+            if type == ENDMARKER:
+                break
+            type = tok_name[type]
+            result.append("    %(type)-10.10s %(token)-13.13r %(start)s %(end)s" %
+                          locals())
+        self.assertEqual(result,
+                         expected.rstrip().splitlines())
 
-    >>> dump_tokens("1 + 1")
+
+    def test_basic(self):
+        self.check_tokenize("1 + 1", """\
     NUMBER     '1'           (1, 0) (1, 1)
     OP         '+'           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
-
-    >>> dump_tokens("if False:\\n"
-    ...             "    # NL\\n"
-    ...             "    True = False # NEWLINE\\n")
+    """)
+        self.check_tokenize("if False:\n"
+                            "    # NL\n"
+                            "    True = False # NEWLINE\n", """\
     NAME       'if'          (1, 0) (1, 2)
     NAME       'False'       (1, 3) (1, 8)
     OP         ':'           (1, 8) (1, 9)
@@ -28,122 +50,48 @@ brevity.
     COMMENT    '# NEWLINE'   (3, 17) (3, 26)
     NEWLINE    '\\n'          (3, 26) (3, 27)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
 
-    >>> indent_error_file = \"""
-    ... def k(x):
-    ...     x += 2
-    ...   x += 5
-    ... \"""
-
-    >>> for tok in generate_tokens(StringIO(indent_error_file).readline): pass
-    Traceback (most recent call last):
-        ...
-    IndentationError: unindent does not match any outer indentation level
-
-Test roundtrip for `untokenize`. `f` is an open file or a string. The source
-code in f is tokenized, converted back to source code via tokenize.untokenize(),
-and tokenized again from the latter. The test fails if the second tokenization
-doesn't match the first.
-
-    >>> def roundtrip(f):
-    ...     if isinstance(f, str): f = StringIO(f)
-    ...     token_list = list(generate_tokens(f.readline))
-    ...     f.close()
-    ...     tokens1 = [tok[:2] for tok in token_list]
-    ...     new_text = untokenize(tokens1)
-    ...     readline = iter(new_text.splitlines(1)).next
-    ...     tokens2 = [tok[:2] for tok in generate_tokens(readline)]
-    ...     return tokens1 == tokens2
-    ...
-
-There are some standard formatting practices that are easy to get right.
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    print x\\n")
-    True
-
-    >>> roundtrip("# This is a comment\\n# This also")
-    True
-
-Some people use different formatting conventions, which makes
-untokenize a little trickier. Note that this test involves trailing
-whitespace after the colon. Note that we use hex escapes to make the
-two trailing blanks apperant in the expected output.
-
-    >>> roundtrip("if x == 1 : \\n"
-    ...           "  print x\\n")
-    True
-
-    >>> f = test_support.findfile("tokenize_tests" + os.extsep + "txt")
-    >>> roundtrip(open(f))
-    True
-
-    >>> roundtrip("if x == 1:\\n"
-    ...           "    # A comment by itself.\\n"
-    ...           "    print x # Comment here, too.\\n"
-    ...           "    # Another comment.\\n"
-    ...           "after_if = True\\n")
-    True
-
-    >>> roundtrip("if (x # The comments need to go in the right place\\n"
-    ...           "    == 1):\\n"
-    ...           "    print 'x==1'\\n")
-    True
-
-    >>> roundtrip("class Test: # A comment here\\n"
-    ...           "  # A comment with weird indent\\n"
-    ...           "  after_com = 5\\n"
-    ...           "  def x(m): return m*5 # a one liner\\n"
-    ...           "  def y(m): # A whitespace after the colon\\n"
-    ...           "     return y*4 # 3-space indent\\n")
-    True
-
-Some error-handling code
-
-    >>> roundtrip("try: import somemodule\\n"
-    ...           "except ImportError: # comment\\n"
-    ...           "    print 'Can not import' # comment2\\n"
-    ...           "else:   print 'Loaded'\\n")
-    True
-
-Balancing continuation
-
-    >>> roundtrip("a = (3,4, \\n"
-    ...           "5,6)\\n"
-    ...           "y = [3, 4,\\n"
-    ...           "5]\\n"
-    ...           "z = {'a': 5,\\n"
-    ...           "'b':15, 'c':True}\\n"
-    ...           "x = len(y) + 5 - a[\\n"
-    ...           "3] - a[2]\\n"
-    ...           "+ len(z) - z[\\n"
-    ...           "'b']\\n")
-    True
-
-Ordinary integers and binary operators
-
-    >>> dump_tokens("0xff <= 255")
+        indent_error_file = """\
+def k(x):
+    x += 2
+  x += 5
+"""
+        with self.assertRaisesRegexp(IndentationError,
+                                     "unindent does not match any "
+                                     "outer indentation level"):
+            for tok in generate_tokens(StringIO(indent_error_file).readline):
+                pass
+
+    def test_int(self):
+        # Ordinary integers and binary operators
+        self.check_tokenize("0xff <= 255", """\
     NUMBER     '0xff'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0b10 <= 255")
+    """)
+        self.check_tokenize("0b10 <= 255", """\
     NUMBER     '0b10'        (1, 0) (1, 4)
     OP         '<='          (1, 5) (1, 7)
     NUMBER     '255'         (1, 8) (1, 11)
-    >>> dump_tokens("0o123 <= 0123")
+    """)
+        self.check_tokenize("0o123 <= 0123", """\
     NUMBER     '0o123'       (1, 0) (1, 5)
     OP         '<='          (1, 6) (1, 8)
     NUMBER     '0123'        (1, 9) (1, 13)
-    >>> dump_tokens("01234567 > ~0x15")
+    """)
+        self.check_tokenize("01234567 > ~0x15", """\
     NUMBER     '01234567'    (1, 0) (1, 8)
     OP         '>'           (1, 9) (1, 10)
     OP         '~'           (1, 11) (1, 12)
     NUMBER     '0x15'        (1, 12) (1, 16)
-    >>> dump_tokens("2134568 != 01231515")
+    """)
+        self.check_tokenize("2134568 != 01231515", """\
     NUMBER     '2134568'     (1, 0) (1, 7)
     OP         '!='          (1, 8) (1, 10)
     NUMBER     '01231515'    (1, 11) (1, 19)
-    >>> dump_tokens("(-124561-1) & 0200000000")
+    """)
+        self.check_tokenize("(-124561-1) & 0200000000", """\
     OP         '('           (1, 0) (1, 1)
     OP         '-'           (1, 1) (1, 2)
     NUMBER     '124561'      (1, 2) (1, 8)
@@ -152,78 +100,93 @@ Ordinary integers and binary operators
     OP         ')'           (1, 10) (1, 11)
     OP         '&'           (1, 12) (1, 13)
     NUMBER     '0200000000'  (1, 14) (1, 24)
-    >>> dump_tokens("0xdeadbeef != -1")
+    """)
+        self.check_tokenize("0xdeadbeef != -1", """\
     NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
     OP         '!='          (1, 11) (1, 13)
     OP         '-'           (1, 14) (1, 15)
     NUMBER     '1'           (1, 15) (1, 16)
-    >>> dump_tokens("0xdeadc0de & 012345")
+    """)
+        self.check_tokenize("0xdeadc0de & 012345", """\
     NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
     OP         '&'           (1, 11) (1, 12)
     NUMBER     '012345'      (1, 13) (1, 19)
-    >>> dump_tokens("0xFF & 0x15 | 1234")
+    """)
+        self.check_tokenize("0xFF & 0x15 | 1234", """\
     NUMBER     '0xFF'        (1, 0) (1, 4)
     OP         '&'           (1, 5) (1, 6)
     NUMBER     '0x15'        (1, 7) (1, 11)
     OP         '|'           (1, 12) (1, 13)
     NUMBER     '1234'        (1, 14) (1, 18)
+    """)
 
-Long integers
-
-    >>> dump_tokens("x = 0L")
+    def test_long(self):
+        # Long integers
+        self.check_tokenize("x = 0L", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0L'          (1, 4) (1, 6)
-    >>> dump_tokens("x = 0xfffffffffff")
+    """)
+        self.check_tokenize("x = 0xfffffffffff", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '0xffffffffff (1, 4) (1, 17)
-    >>> dump_tokens("x = 123141242151251616110l")
+    """)
+        self.check_tokenize("x = 123141242151251616110l", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '123141242151 (1, 4) (1, 26)
-    >>> dump_tokens("x = -15921590215012591L")
+    """)
+        self.check_tokenize("x = -15921590215012591L", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     OP         '-'           (1, 4) (1, 5)
     NUMBER     '159215902150 (1, 5) (1, 23)
+    """)
 
-Floating point numbers
-
-    >>> dump_tokens("x = 3.14159")
+    def test_float(self):
+        # Floating point numbers
+        self.check_tokenize("x = 3.14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 314159.")
+    """)
+        self.check_tokenize("x = 314159.", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '314159.'     (1, 4) (1, 11)
-    >>> dump_tokens("x = .314159")
+    """)
+        self.check_tokenize("x = .314159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '.314159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3e14159")
+    """)
+        self.check_tokenize("x = 3e14159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3e14159'     (1, 4) (1, 11)
-    >>> dump_tokens("x = 3E123")
+    """)
+        self.check_tokenize("x = 3E123", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3E123'       (1, 4) (1, 9)
-    >>> dump_tokens("x+y = 3e-1230")
+    """)
+        self.check_tokenize("x+y = 3e-1230", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '+'           (1, 1) (1, 2)
     NAME       'y'           (1, 2) (1, 3)
     OP         '='           (1, 4) (1, 5)
     NUMBER     '3e-1230'     (1, 6) (1, 13)
-    >>> dump_tokens("x = 3.14e159")
+    """)
+        self.check_tokenize("x = 3.14e159", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '3.14e159'    (1, 4) (1, 12)
+    """)
 
-String literals
-
-    >>> dump_tokens("x = ''; y = \\\"\\\"")
+    def test_string(self):
+        # String literals
+        self.check_tokenize("x = ''; y = \"\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "''"          (1, 4) (1, 6)
@@ -231,7 +194,8 @@ String literals
     NAME       'y'           (1, 8) (1, 9)
     OP         '='           (1, 10) (1, 11)
     STRING     '""'          (1, 12) (1, 14)
-    >>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
+    """)
+        self.check_tokenize("x = '\"'; y = \"'\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '\\'"\\''       (1, 4) (1, 7)
@@ -239,25 +203,29 @@ String literals
     NAME       'y'           (1, 9) (1, 10)
     OP         '='           (1, 11) (1, 12)
     STRING     '"\\'"'        (1, 13) (1, 16)
-    >>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
+    """)
+        self.check_tokenize("x = \"doesn't \"shrink\", does it\"", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     '"doesn\\'t "' (1, 4) (1, 14)
     NAME       'shrink'      (1, 14) (1, 20)
     STRING     '", does it"' (1, 20) (1, 31)
-    >>> dump_tokens("x = u'abc' + U'ABC'")
+    """)
+        self.check_tokenize("x = u'abc' + U'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "u'abc'"      (1, 4) (1, 10)
     OP         '+'           (1, 11) (1, 12)
     STRING     "U'ABC'"      (1, 13) (1, 19)
-    >>> dump_tokens('y = u"ABC" + U"ABC"')
+    """)
+        self.check_tokenize('y = u"ABC" + U"ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     'u"ABC"'      (1, 4) (1, 10)
     OP         '+'           (1, 11) (1, 12)
     STRING     'U"ABC"'      (1, 13) (1, 19)
-    >>> dump_tokens("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'")
+    """)
+        self.check_tokenize("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     "ur'abc'"     (1, 4) (1, 11)
@@ -267,7 +235,8 @@ String literals
     STRING     "uR'ABC'"     (1, 24) (1, 31)
     OP         '+'           (1, 32) (1, 33)
     STRING     "UR'ABC'"     (1, 34) (1, 41)
-    >>> dump_tokens('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"')
+    """)
+        self.check_tokenize('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"', """\
     NAME       'y'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     STRING     'ur"abc"'     (1, 4) (1, 11)
@@ -278,15 +247,18 @@ String literals
     OP         '+'           (1, 32) (1, 33)
     STRING     'UR"ABC"'     (1, 34) (1, 41)
 
-    >>> dump_tokens("b'abc' + B'abc'")
+    """)
+        self.check_tokenize("b'abc' + B'abc'", """\
     STRING     "b'abc'"      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     "B'abc'"      (1, 9) (1, 15)
-    >>> dump_tokens('b"abc" + B"abc"')
+    """)
+        self.check_tokenize('b"abc" + B"abc"', """\
     STRING     'b"abc"'      (1, 0) (1, 6)
     OP         '+'           (1, 7) (1, 8)
     STRING     'B"abc"'      (1, 9) (1, 15)
-    >>> dump_tokens("br'abc' + bR'abc' + Br'abc' + BR'abc'")
+    """)
+        self.check_tokenize("br'abc' + bR'abc' + Br'abc' + BR'abc'", """\
     STRING     "br'abc'"     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     "bR'abc'"     (1, 10) (1, 17)
@@ -294,7 +266,8 @@ String literals
     STRING     "Br'abc'"     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     "BR'abc'"     (1, 30) (1, 37)
-    >>> dump_tokens('br"abc" + bR"abc" + Br"abc" + BR"abc"')
+    """)
+        self.check_tokenize('br"abc" + bR"abc" + Br"abc" + BR"abc"', """\
     STRING     'br"abc"'     (1, 0) (1, 7)
     OP         '+'           (1, 8) (1, 9)
     STRING     'bR"abc"'     (1, 10) (1, 17)
@@ -302,10 +275,10 @@ String literals
     STRING     'Br"abc"'     (1, 20) (1, 27)
     OP         '+'           (1, 28) (1, 29)
     STRING     'BR"abc"'     (1, 30) (1, 37)
+    """)
 
-Operators
-
-    >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
+    def test_function(self):
+        self.check_tokenize("def d22(a, b, c=2, d=2, *k): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd22'         (1, 4) (1, 7)
     OP         '('           (1, 7) (1, 8)
@@ -326,7 +299,8 @@ Operators
     OP         ')'           (1, 26) (1, 27)
     OP         ':'           (1, 27) (1, 28)
     NAME       'pass'        (1, 29) (1, 33)
-    >>> dump_tokens("def d01v_(a=1, *k, **w): pass")
+    """)
+        self.check_tokenize("def d01v_(a=1, *k, **w): pass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'd01v_'       (1, 4) (1, 9)
     OP         '('           (1, 9) (1, 10)
@@ -342,11 +316,12 @@ Operators
     OP         ')'           (1, 22) (1, 23)
     OP         ':'           (1, 23) (1, 24)
     NAME       'pass'        (1, 25) (1, 29)
+    """)
 
-Comparison
-
-    >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
-    ...             "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
+    def test_comparison(self):
+        # Comparison
+        self.check_tokenize("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
+                            "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass", """\
     NAME       'if'          (1, 0) (1, 2)
     NUMBER     '1'           (1, 3) (1, 4)
     OP         '<'           (1, 5) (1, 6)
@@ -379,10 +354,11 @@ Comparison
     NUMBER     '1'           (1, 81) (1, 82)
     OP         ':'           (1, 82) (1, 83)
     NAME       'pass'        (1, 84) (1, 88)
+    """)
 
-Shift
-
-    >>> dump_tokens("x = 1 << 1 >> 5")
+    def test_shift(self):
+        # Shift
+        self.check_tokenize("x = 1 << 1 >> 5", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -390,10 +366,11 @@ Shift
     NUMBER     '1'           (1, 9) (1, 10)
     OP         '>>'          (1, 11) (1, 13)
     NUMBER     '5'           (1, 14) (1, 15)
+    """)
 
-Additive
-
-    >>> dump_tokens("x = 1 - y + 15 - 01 + 0x124 + z + a[5]")
+    def test_additive(self):
+        # Additive
+        self.check_tokenize("x = 1 - y + 15 - 01 + 0x124 + z + a[5]", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -412,10 +389,11 @@ Additive
     OP         '['           (1, 35) (1, 36)
     NUMBER     '5'           (1, 36) (1, 37)
     OP         ']'           (1, 37) (1, 38)
+    """)
 
-Multiplicative
-
-    >>> dump_tokens("x = 1//1*1/5*12%0x12")
+    def test_multiplicative(self):
+        # Multiplicative
+        self.check_tokenize("x = 1//1*1/5*12%0x12", """\
     NAME       'x'           (1, 0) (1, 1)
     OP         '='           (1, 2) (1, 3)
     NUMBER     '1'           (1, 4) (1, 5)
@@ -429,10 +407,11 @@ Multiplicative
     NUMBER     '12'          (1, 13) (1, 15)
     OP         '%'           (1, 15) (1, 16)
     NUMBER     '0x12'        (1, 16) (1, 20)
+    """)
 
-Unary
-
-    >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
+    def test_unary(self):
+        # Unary
+        self.check_tokenize("~1 ^ 1 & 1 |1 ^ -1", """\
     OP         '~'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '^'           (1, 3) (1, 4)
@@ -444,7 +423,8 @@ Unary
     OP         '^'           (1, 14) (1, 15)
     OP         '-'           (1, 16) (1, 17)
     NUMBER     '1'           (1, 17) (1, 18)
-    >>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
+    """)
+        self.check_tokenize("-1*1/1+1*1//1 - ---1**1", """\
     OP         '-'           (1, 0) (1, 1)
     NUMBER     '1'           (1, 1) (1, 2)
     OP         '*'           (1, 2) (1, 3)
@@ -464,10 +444,12 @@ Unary
     NUMBER     '1'           (1, 19) (1, 20)
     OP         '**'          (1, 20) (1, 22)
     NUMBER     '1'           (1, 22) (1, 23)
+    """)
 
-Selector
-
-    >>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
+    def test_selector(self):
+        # Selector
+        self.check_tokenize("import sys, time\n"
+                            "x = sys.modules['time'].time()", """\
     NAME       'import'      (1, 0) (1, 6)
     NAME       'sys'         (1, 7) (1, 10)
     OP         ','           (1, 10) (1, 11)
@@ -485,10 +467,12 @@ Selector
     NAME       'time'        (2, 24) (2, 28)
     OP         '('           (2, 28) (2, 29)
     OP         ')'           (2, 29) (2, 30)
+    """)
 
-Methods
-
-    >>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
+    def test_method(self):
+        # Methods
+        self.check_tokenize("@staticmethod\n"
+                            "def foo(x,y): pass", """\
     OP         '@'           (1, 0) (1, 1)
     NAME       'staticmethod (1, 1) (1, 13)
     NEWLINE    '\\n'          (1, 13) (1, 14)
@@ -501,41 +485,13 @@ Methods
     OP         ')'           (2, 11) (2, 12)
     OP         ':'           (2, 12) (2, 13)
     NAME       'pass'        (2, 14) (2, 18)
+    """)
 
-Backslash means line continuation, except for comments
-
-    >>> roundtrip("x=1+\\\\n"
-    ...           "1\\n"
-    ...           "# This is a comment\\\\n"
-    ...           "# This also\\n")
-    True
-    >>> roundtrip("# Comment \\\\nx = 0")
-    True
-
-Two string literals on the same line
-
-    >>> roundtrip("'' ''")
-    True
-
-Test roundtrip on random python modules.
-pass the '-ucpu' option to process the full directory.
-
-    >>>
-    >>> tempdir = os.path.dirname(f) or os.curdir
-    >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
-
-    >>> if not test_support.is_resource_enabled("cpu"):
-    ...     testfiles = random.sample(testfiles, 10)
-    ...
-    >>> for testfile in testfiles:
-    ...     if not roundtrip(open(testfile)):
-    ...         print "Roundtrip failed for file %s" % testfile
-    ...         break
-    ... else: True
-    True
-
-Evil tabs
-    >>> dump_tokens("def f():\\n\\tif x\\n        \\tpass")
+    def test_tabs(self):
+        # Evil tabs
+        self.check_tokenize("def f():\n"
+                            "\tif x\n"
+                            "        \tpass", """\
     NAME       'def'         (1, 0) (1, 3)
     NAME       'f'           (1, 4) (1, 5)
     OP         '('           (1, 5) (1, 6)
@@ -550,56 +506,16 @@ Evil tabs
     NAME       'pass'        (3, 9) (3, 13)
     DEDENT     ''            (4, 0) (4, 0)
     DEDENT     ''            (4, 0) (4, 0)
+    """)
 
-Pathological whitespace (http://bugs.python.org/issue16152)
-    >>> dump_tokens("@          ")
+    def test_pathological_trailing_whitespace(self):
+        # Pathological whitespace (http://bugs.python.org/issue16152)
+        self.check_tokenize("@          ", """\
     OP         '@'           (1, 0) (1, 1)
-"""
-
+    """)
 
-from test import test_support
-from tokenize import (untokenize, generate_tokens, NUMBER, NAME, OP,
-                     STRING, ENDMARKER, tok_name, Untokenizer, tokenize)
-from StringIO import StringIO
-import os
-from unittest import TestCase
 
-def dump_tokens(s):
-    """Print out the tokens in s in a table format.
-
-    The ENDMARKER is omitted.
-    """
-    f = StringIO(s)
-    for type, token, start, end, line in generate_tokens(f.readline):
-        if type == ENDMARKER:
-            break
-        type = tok_name[type]
-        print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
-
-# This is an example from the docs, set up as a doctest.
 def decistmt(s):
-    """Substitute Decimals for floats in a string of statements.
-
-    >>> from decimal import Decimal
-    >>> s = 'print +21.3e-5*-.1234/81.7'
-    >>> decistmt(s)
-    "print +Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')"
-
-    The format of the exponent is inherited from the platform C library.
-    Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
-    we're only showing 12 digits, and the 13th isn't close to 5, the
-    rest of the output should be platform-independent.
-
-    >>> exec(s) #doctest: +ELLIPSIS
-    -3.21716034272e-0...7
-
-    Output from calculations with Decimal should be identical across all
-    platforms.
-
-    >>> exec(decistmt(s))
-    -3.217160342717258261933904529E-7
-    """
-
     result = []
     g = generate_tokens(StringIO(s).readline)   # tokenize the string
     for toknum, tokval, _, _, _  in g:
@@ -614,6 +530,27 @@ def decistmt(s):
             result.append((toknum, tokval))
     return untokenize(result)
 
+class TestMisc(TestCase):
+
+    def test_decistmt(self):
+        # Substitute Decimals for floats in a string of statements.
+        # This is an example from the docs.
+
+        from decimal import Decimal
+        s = '+21.3e-5*-.1234/81.7'
+        self.assertEqual(decistmt(s),
+                         "+Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')")
+
+        # The format of the exponent is inherited from the platform C library.
+        # Known cases are "e-007" (Windows) and "e-07" (not Windows).  Since
+        # we're only showing 12 digits, and the 13th isn't close to 5, the
+        # rest of the output should be platform-independent.
+        self.assertRegexpMatches(str(eval(s)), '-3.21716034272e-0+7')
+
+        # Output from calculations with Decimal should be identical across all
+        # platforms.
+        self.assertEqual(eval(decistmt(s)), Decimal('-3.217160342717258261933904529E-7'))
+
 
 class UntokenizeTest(TestCase):
 
@@ -651,6 +588,115 @@ class UntokenizeTest(TestCase):
 
 
 class TestRoundtrip(TestCase):
+
+    def check_roundtrip(self, f):
+        """
+        Test roundtrip for `untokenize`. `f` is an open file or a string.
+        The source code in f is tokenized, converted back to source code
+        via tokenize.untokenize(), and tokenized again from the latter.
+        The test fails if the second tokenization doesn't match the first.
+        """
+        if isinstance(f, str): f = StringIO(f)
+        token_list = list(generate_tokens(f.readline))
+        f.close()
+        tokens1 = [tok[:2] for tok in token_list]
+        new_text = untokenize(tokens1)
+        readline = iter(new_text.splitlines(1)).next
+        tokens2 = [tok[:2] for tok in generate_tokens(readline)]
+        self.assertEqual(tokens2, tokens1)
+
+    def test_roundtrip(self):
+        # There are some standard formatting practices that are easy to get right.
+
+        self.check_roundtrip("if x == 1:\n"
+                             "    print(x)\n")
+
+        # There are some standard formatting practices that are easy to get right.
+
+        self.check_roundtrip("if x == 1:\n"
+                             "    print x\n")
+        self.check_roundtrip("# This is a comment\n"
+                             "# This also")
+
+        # Some people use different formatting conventions, which makes
+        # untokenize a little trickier. Note that this test involves trailing
+        # whitespace after the colon. Note that we use hex escapes to make the
+        # two trailing blanks apperant in the expected output.
+
+        self.check_roundtrip("if x == 1 : \n"
+                             "  print x\n")
+        fn = test_support.findfile("tokenize_tests" + os.extsep + "txt")
+        with open(fn) as f:
+            self.check_roundtrip(f)
+        self.check_roundtrip("if x == 1:\n"
+                             "    # A comment by itself.\n"
+                             "    print x # Comment here, too.\n"
+                             "    # Another comment.\n"
+                             "after_if = True\n")
+        self.check_roundtrip("if (x # The comments need to go in the right place\n"
+                             "    == 1):\n"
+                             "    print 'x==1'\n")
+        self.check_roundtrip("class Test: # A comment here\n"
+                             "  # A comment with weird indent\n"
+                             "  after_com = 5\n"
+                             "  def x(m): return m*5 # a one liner\n"
+                             "  def y(m): # A whitespace after the colon\n"
+                             "     return y*4 # 3-space indent\n")
+
+        # Some error-handling code
+
+        self.check_roundtrip("try: import somemodule\n"
+                             "except ImportError: # comment\n"
+                             "    print 'Can not import' # comment2\n"
+                             "else:   print 'Loaded'\n")
+
+    def test_continuation(self):
+        # Balancing continuation
+        self.check_roundtrip("a = (3,4, \n"
+                             "5,6)\n"
+                             "y = [3, 4,\n"
+                             "5]\n"
+                             "z = {'a': 5,\n"
+                             "'b':15, 'c':True}\n"
+                             "x = len(y) + 5 - a[\n"
+                             "3] - a[2]\n"
+                             "+ len(z) - z[\n"
+                             "'b']\n")
+
+    def test_backslash_continuation(self):
+        # Backslash means line continuation, except for comments
+        self.check_roundtrip("x=1+\\\n"
+                             "1\n"
+                             "# This is a comment\\\n"
+                             "# This also\n")
+        self.check_roundtrip("# Comment \\\n"
+                             "x = 0")
+
+    def test_string_concatenation(self):
+        # Two string literals on the same line
+        self.check_roundtrip("'' ''")
+
+    def test_random_files(self):
+        # Test roundtrip on random python modules.
+        # pass the '-ucpu' option to process the full directory.
+
+        import glob, random
+        fn = test_support.findfile("tokenize_tests" + os.extsep + "txt")
+        tempdir = os.path.dirname(fn) or os.curdir
+        testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
+
+        if not test_support.is_resource_enabled("cpu"):
+            testfiles = random.sample(testfiles, 10)
+
+        for testfile in testfiles:
+            try:
+                with open(testfile, 'rb') as f:
+                    self.check_roundtrip(f)
+            except:
+                print "Roundtrip failed for file %s" % testfile
+                raise
+
+
     def roundtrip(self, code):
         if isinstance(code, str):
             code = code.encode('utf-8')
@@ -667,13 +713,11 @@ class TestRoundtrip(TestCase):
         self.assertEqual(codelines[1], codelines[2])
 
 
-__test__ = {"doctests" : doctests, 'decistmt': decistmt}
-
 def test_main():
-    from test import test_tokenize
-    test_support.run_doctest(test_tokenize, True)
+    test_support.run_unittest(TokenizeTest)
     test_support.run_unittest(UntokenizeTest)
     test_support.run_unittest(TestRoundtrip)
+    test_support.run_unittest(TestMisc)
 
 if __name__ == "__main__":
     test_main()