Move test_tokenize to doctest.

Done as GHOP 238 by Josip Dzolonga.

4 files changed, 501 insertions, 845 deletions

diff --git a/Lib/test/output/test_tokenize b/Lib/test/output/test_tokenize
deleted file mode 100644
index 1d1c6a8..0000000
--- a/Lib/test/output/test_tokenize
+++ /dev/null
@@ -1,684 +0,0 @@
-test_tokenize
-1,0-1,34:	COMMENT	"# Tests for the 'tokenize' module."
-1,34-1,35:	NL	'\n'
-2,0-2,42:	COMMENT	'# Large bits stolen from test_grammar.py. '
-2,42-2,43:	NL	'\n'
-3,0-3,1:	NL	'\n'
-4,0-4,10:	COMMENT	'# Comments'
-4,10-4,11:	NL	'\n'
-5,0-5,3:	STRING	'"#"'
-5,3-5,4:	NEWLINE	'\n'
-6,0-6,2:	COMMENT	"#'"
-6,2-6,3:	NL	'\n'
-7,0-7,2:	COMMENT	'#"'
-7,2-7,3:	NL	'\n'
-8,0-8,2:	COMMENT	'#\\'
-8,2-8,3:	NL	'\n'
-9,7-9,8:	COMMENT	'#'
-9,8-9,9:	NL	'\n'
-10,4-10,9:	COMMENT	'# abc'
-10,9-10,10:	NL	'\n'
-11,0-12,4:	STRING	"'''#\n#'''"
-12,4-12,5:	NEWLINE	'\n'
-13,0-13,1:	NL	'\n'
-14,0-14,1:	NAME	'x'
-14,2-14,3:	OP	'='
-14,4-14,5:	NUMBER	'1'
-14,7-14,8:	COMMENT	'#'
-14,8-14,9:	NEWLINE	'\n'
-15,0-15,1:	NL	'\n'
-16,0-16,24:	COMMENT	'# Balancing continuation'
-16,24-16,25:	NL	'\n'
-17,0-17,1:	NL	'\n'
-18,0-18,1:	NAME	'a'
-18,2-18,3:	OP	'='
-18,4-18,5:	OP	'('
-18,5-18,6:	NUMBER	'3'
-18,6-18,7:	OP	','
-18,8-18,9:	NUMBER	'4'
-18,9-18,10:	OP	','
-18,10-18,11:	NL	'\n'
-19,2-19,3:	NUMBER	'5'
-19,3-19,4:	OP	','
-19,5-19,6:	NUMBER	'6'
-19,6-19,7:	OP	')'
-19,7-19,8:	NEWLINE	'\n'
-20,0-20,1:	NAME	'y'
-20,2-20,3:	OP	'='
-20,4-20,5:	OP	'['
-20,5-20,6:	NUMBER	'3'
-20,6-20,7:	OP	','
-20,8-20,9:	NUMBER	'4'
-20,9-20,10:	OP	','
-20,10-20,11:	NL	'\n'
-21,2-21,3:	NUMBER	'5'
-21,3-21,4:	OP	']'
-21,4-21,5:	NEWLINE	'\n'
-22,0-22,1:	NAME	'z'
-22,2-22,3:	OP	'='
-22,4-22,5:	OP	'{'
-22,5-22,8:	STRING	"'a'"
-22,8-22,9:	OP	':'
-22,9-22,10:	NUMBER	'5'
-22,10-22,11:	OP	','
-22,11-22,12:	NL	'\n'
-23,2-23,5:	STRING	"'b'"
-23,5-23,6:	OP	':'
-23,6-23,7:	NUMBER	'6'
-23,7-23,8:	OP	'}'
-23,8-23,9:	NEWLINE	'\n'
-24,0-24,1:	NAME	'x'
-24,2-24,3:	OP	'='
-24,4-24,5:	OP	'('
-24,5-24,8:	NAME	'len'
-24,8-24,9:	OP	'('
-24,9-24,10:	OP	'`'
-24,10-24,11:	NAME	'y'
-24,11-24,12:	OP	'`'
-24,12-24,13:	OP	')'
-24,14-24,15:	OP	'+'
-24,16-24,17:	NUMBER	'5'
-24,17-24,18:	OP	'*'
-24,18-24,19:	NAME	'x'
-24,20-24,21:	OP	'-'
-24,22-24,23:	NAME	'a'
-24,23-24,24:	OP	'['
-24,24-24,25:	NL	'\n'
-25,3-25,4:	NUMBER	'3'
-25,5-25,6:	OP	']'
-25,6-25,7:	NL	'\n'
-26,3-26,4:	OP	'-'
-26,5-26,6:	NAME	'x'
-26,7-26,8:	OP	'+'
-26,9-26,12:	NAME	'len'
-26,12-26,13:	OP	'('
-26,13-26,14:	OP	'{'
-26,14-26,15:	NL	'\n'
-27,3-27,4:	OP	'}'
-27,4-27,5:	NL	'\n'
-28,4-28,5:	OP	')'
-28,5-28,6:	NL	'\n'
-29,2-29,3:	OP	')'
-29,3-29,4:	NEWLINE	'\n'
-30,0-30,1:	NL	'\n'
-31,0-31,36:	COMMENT	'# Backslash means line continuation:'
-31,36-31,37:	NL	'\n'
-32,0-32,1:	NAME	'x'
-32,2-32,3:	OP	'='
-32,4-32,5:	NUMBER	'1'
-33,0-33,1:	OP	'+'
-33,2-33,3:	NUMBER	'1'
-33,3-33,4:	NEWLINE	'\n'
-34,0-34,1:	NL	'\n'
-35,0-35,54:	COMMENT	'# Backslash does not means continuation in comments :\\'
-35,54-35,55:	NL	'\n'
-36,0-36,1:	NAME	'x'
-36,2-36,3:	OP	'='
-36,4-36,5:	NUMBER	'0'
-36,5-36,6:	NEWLINE	'\n'
-37,0-37,1:	NL	'\n'
-38,0-38,19:	COMMENT	'# Ordinary integers'
-38,19-38,20:	NL	'\n'
-39,0-39,4:	NUMBER	'0xff'
-39,5-39,7:	OP	'<>'
-39,8-39,11:	NUMBER	'255'
-39,11-39,12:	NEWLINE	'\n'
-40,0-40,4:	NUMBER	'0377'
-40,5-40,7:	OP	'<>'
-40,8-40,11:	NUMBER	'255'
-40,11-40,12:	NEWLINE	'\n'
-41,0-41,10:	NUMBER	'2147483647'
-41,13-41,15:	OP	'!='
-41,16-41,28:	NUMBER	'017777777777'
-41,28-41,29:	NEWLINE	'\n'
-42,0-42,1:	OP	'-'
-42,1-42,11:	NUMBER	'2147483647'
-42,11-42,12:	OP	'-'
-42,12-42,13:	NUMBER	'1'
-42,14-42,16:	OP	'!='
-42,17-42,29:	NUMBER	'020000000000'
-42,29-42,30:	NEWLINE	'\n'
-43,0-43,12:	NUMBER	'037777777777'
-43,13-43,15:	OP	'!='
-43,16-43,17:	OP	'-'
-43,17-43,18:	NUMBER	'1'
-43,18-43,19:	NEWLINE	'\n'
-44,0-44,10:	NUMBER	'0xffffffff'
-44,11-44,13:	OP	'!='
-44,14-44,15:	OP	'-'
-44,15-44,16:	NUMBER	'1'
-44,16-44,17:	NEWLINE	'\n'
-45,0-45,1:	NL	'\n'
-46,0-46,15:	COMMENT	'# Long integers'
-46,15-46,16:	NL	'\n'
-47,0-47,1:	NAME	'x'
-47,2-47,3:	OP	'='
-47,4-47,6:	NUMBER	'0L'
-47,6-47,7:	NEWLINE	'\n'
-48,0-48,1:	NAME	'x'
-48,2-48,3:	OP	'='
-48,4-48,6:	NUMBER	'0l'
-48,6-48,7:	NEWLINE	'\n'
-49,0-49,1:	NAME	'x'
-49,2-49,3:	OP	'='
-49,4-49,23:	NUMBER	'0xffffffffffffffffL'
-49,23-49,24:	NEWLINE	'\n'
-50,0-50,1:	NAME	'x'
-50,2-50,3:	OP	'='
-50,4-50,23:	NUMBER	'0xffffffffffffffffl'
-50,23-50,24:	NEWLINE	'\n'
-51,0-51,1:	NAME	'x'
-51,2-51,3:	OP	'='
-51,4-51,23:	NUMBER	'077777777777777777L'
-51,23-51,24:	NEWLINE	'\n'
-52,0-52,1:	NAME	'x'
-52,2-52,3:	OP	'='
-52,4-52,23:	NUMBER	'077777777777777777l'
-52,23-52,24:	NEWLINE	'\n'
-53,0-53,1:	NAME	'x'
-53,2-53,3:	OP	'='
-53,4-53,35:	NUMBER	'123456789012345678901234567890L'
-53,35-53,36:	NEWLINE	'\n'
-54,0-54,1:	NAME	'x'
-54,2-54,3:	OP	'='
-54,4-54,35:	NUMBER	'123456789012345678901234567890l'
-54,35-54,36:	NEWLINE	'\n'
-55,0-55,1:	NL	'\n'
-56,0-56,24:	COMMENT	'# Floating-point numbers'
-56,24-56,25:	NL	'\n'
-57,0-57,1:	NAME	'x'
-57,2-57,3:	OP	'='
-57,4-57,8:	NUMBER	'3.14'
-57,8-57,9:	NEWLINE	'\n'
-58,0-58,1:	NAME	'x'
-58,2-58,3:	OP	'='
-58,4-58,8:	NUMBER	'314.'
-58,8-58,9:	NEWLINE	'\n'
-59,0-59,1:	NAME	'x'
-59,2-59,3:	OP	'='
-59,4-59,9:	NUMBER	'0.314'
-59,9-59,10:	NEWLINE	'\n'
-60,0-60,17:	COMMENT	'# XXX x = 000.314'
-60,17-60,18:	NL	'\n'
-61,0-61,1:	NAME	'x'
-61,2-61,3:	OP	'='
-61,4-61,8:	NUMBER	'.314'
-61,8-61,9:	NEWLINE	'\n'
-62,0-62,1:	NAME	'x'
-62,2-62,3:	OP	'='
-62,4-62,8:	NUMBER	'3e14'
-62,8-62,9:	NEWLINE	'\n'
-63,0-63,1:	NAME	'x'
-63,2-63,3:	OP	'='
-63,4-63,8:	NUMBER	'3E14'
-63,8-63,9:	NEWLINE	'\n'
-64,0-64,1:	NAME	'x'
-64,2-64,3:	OP	'='
-64,4-64,9:	NUMBER	'3e-14'
-64,9-64,10:	NEWLINE	'\n'
-65,0-65,1:	NAME	'x'
-65,2-65,3:	OP	'='
-65,4-65,9:	NUMBER	'3e+14'
-65,9-65,10:	NEWLINE	'\n'
-66,0-66,1:	NAME	'x'
-66,2-66,3:	OP	'='
-66,4-66,9:	NUMBER	'3.e14'
-66,9-66,10:	NEWLINE	'\n'
-67,0-67,1:	NAME	'x'
-67,2-67,3:	OP	'='
-67,4-67,9:	NUMBER	'.3e14'
-67,9-67,10:	NEWLINE	'\n'
-68,0-68,1:	NAME	'x'
-68,2-68,3:	OP	'='
-68,4-68,9:	NUMBER	'3.1e4'
-68,9-68,10:	NEWLINE	'\n'
-69,0-69,1:	NL	'\n'
-70,0-70,17:	COMMENT	'# String literals'
-70,17-70,18:	NL	'\n'
-71,0-71,1:	NAME	'x'
-71,2-71,3:	OP	'='
-71,4-71,6:	STRING	"''"
-71,6-71,7:	OP	';'
-71,8-71,9:	NAME	'y'
-71,10-71,11:	OP	'='
-71,12-71,14:	STRING	'""'
-71,14-71,15:	OP	';'
-71,15-71,16:	NEWLINE	'\n'
-72,0-72,1:	NAME	'x'
-72,2-72,3:	OP	'='
-72,4-72,8:	STRING	"'\\''"
-72,8-72,9:	OP	';'
-72,10-72,11:	NAME	'y'
-72,12-72,13:	OP	'='
-72,14-72,17:	STRING	'"\'"'
-72,17-72,18:	OP	';'
-72,18-72,19:	NEWLINE	'\n'
-73,0-73,1:	NAME	'x'
-73,2-73,3:	OP	'='
-73,4-73,7:	STRING	'\'"\''
-73,7-73,8:	OP	';'
-73,9-73,10:	NAME	'y'
-73,11-73,12:	OP	'='
-73,13-73,17:	STRING	'"\\""'
-73,17-73,18:	OP	';'
-73,18-73,19:	NEWLINE	'\n'
-74,0-74,1:	NAME	'x'
-74,2-74,3:	OP	'='
-74,4-74,32:	STRING	'"doesn\'t \\"shrink\\" does it"'
-74,32-74,33:	NEWLINE	'\n'
-75,0-75,1:	NAME	'y'
-75,2-75,3:	OP	'='
-75,4-75,31:	STRING	'\'doesn\\\'t "shrink" does it\''
-75,31-75,32:	NEWLINE	'\n'
-76,0-76,1:	NAME	'x'
-76,2-76,3:	OP	'='
-76,4-76,32:	STRING	'"does \\"shrink\\" doesn\'t it"'
-76,32-76,33:	NEWLINE	'\n'
-77,0-77,1:	NAME	'y'
-77,2-77,3:	OP	'='
-77,4-77,31:	STRING	'\'does "shrink" doesn\\\'t it\''
-77,31-77,32:	NEWLINE	'\n'
-78,0-78,1:	NAME	'x'
-78,2-78,3:	OP	'='
-78,4-83,3:	STRING	'"""\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n"""'
-83,3-83,4:	NEWLINE	'\n'
-84,0-84,1:	NAME	'y'
-84,2-84,3:	OP	'='
-84,4-84,63:	STRING	'\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
-84,63-84,64:	NEWLINE	'\n'
-85,0-85,1:	NAME	'y'
-85,2-85,3:	OP	'='
-85,4-90,3:	STRING	'\'\'\'\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n\'\'\''
-90,3-90,4:	OP	';'
-90,4-90,5:	NEWLINE	'\n'
-91,0-91,1:	NAME	'y'
-91,2-91,3:	OP	'='
-91,4-96,1:	STRING	'"\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \'lazy\' dog.\\n\\\n"'
-96,1-96,2:	OP	';'
-96,2-96,3:	NEWLINE	'\n'
-97,0-97,1:	NAME	'y'
-97,2-97,3:	OP	'='
-97,4-102,1:	STRING	'\'\\n\\\nThe \\"quick\\"\\n\\\nbrown fox\\n\\\njumps over\\n\\\nthe \\\'lazy\\\' dog.\\n\\\n\''
-102,1-102,2:	OP	';'
-102,2-102,3:	NEWLINE	'\n'
-103,0-103,1:	NAME	'x'
-103,2-103,3:	OP	'='
-103,4-103,9:	STRING	"r'\\\\'"
-103,10-103,11:	OP	'+'
-103,12-103,17:	STRING	"R'\\\\'"
-103,17-103,18:	NEWLINE	'\n'
-104,0-104,1:	NAME	'x'
-104,2-104,3:	OP	'='
-104,4-104,9:	STRING	"r'\\''"
-104,10-104,11:	OP	'+'
-104,12-104,14:	STRING	"''"
-104,14-104,15:	NEWLINE	'\n'
-105,0-105,1:	NAME	'y'
-105,2-105,3:	OP	'='
-105,4-107,6:	STRING	"r'''\nfoo bar \\\\\nbaz'''"
-107,7-107,8:	OP	'+'
-107,9-108,6:	STRING	"R'''\nfoo'''"
-108,6-108,7:	NEWLINE	'\n'
-109,0-109,1:	NAME	'y'
-109,2-109,3:	OP	'='
-109,4-111,3:	STRING	'r"""foo\nbar \\\\ baz\n"""'
-111,4-111,5:	OP	'+'
-111,6-112,3:	STRING	"R'''spam\n'''"
-112,3-112,4:	NEWLINE	'\n'
-113,0-113,1:	NAME	'x'
-113,2-113,3:	OP	'='
-113,4-113,10:	STRING	"u'abc'"
-113,11-113,12:	OP	'+'
-113,13-113,19:	STRING	"U'ABC'"
-113,19-113,20:	NEWLINE	'\n'
-114,0-114,1:	NAME	'y'
-114,2-114,3:	OP	'='
-114,4-114,10:	STRING	'u"abc"'
-114,11-114,12:	OP	'+'
-114,13-114,19:	STRING	'U"ABC"'
-114,19-114,20:	NEWLINE	'\n'
-115,0-115,1:	NAME	'x'
-115,2-115,3:	OP	'='
-115,4-115,11:	STRING	"ur'abc'"
-115,12-115,13:	OP	'+'
-115,14-115,21:	STRING	"Ur'ABC'"
-115,22-115,23:	OP	'+'
-115,24-115,31:	STRING	"uR'ABC'"
-115,32-115,33:	OP	'+'
-115,34-115,41:	STRING	"UR'ABC'"
-115,41-115,42:	NEWLINE	'\n'
-116,0-116,1:	NAME	'y'
-116,2-116,3:	OP	'='
-116,4-116,11:	STRING	'ur"abc"'
-116,12-116,13:	OP	'+'
-116,14-116,21:	STRING	'Ur"ABC"'
-116,22-116,23:	OP	'+'
-116,24-116,31:	STRING	'uR"ABC"'
-116,32-116,33:	OP	'+'
-116,34-116,41:	STRING	'UR"ABC"'
-116,41-116,42:	NEWLINE	'\n'
-117,0-117,1:	NAME	'x'
-117,2-117,3:	OP	'='
-117,4-117,10:	STRING	"ur'\\\\'"
-117,11-117,12:	OP	'+'
-117,13-117,19:	STRING	"UR'\\\\'"
-117,19-117,20:	NEWLINE	'\n'
-118,0-118,1:	NAME	'x'
-118,2-118,3:	OP	'='
-118,4-118,10:	STRING	"ur'\\''"
-118,11-118,12:	OP	'+'
-118,13-118,15:	STRING	"''"
-118,15-118,16:	NEWLINE	'\n'
-119,0-119,1:	NAME	'y'
-119,2-119,3:	OP	'='
-119,4-121,6:	STRING	"ur'''\nfoo bar \\\\\nbaz'''"
-121,7-121,8:	OP	'+'
-121,9-122,6:	STRING	"UR'''\nfoo'''"
-122,6-122,7:	NEWLINE	'\n'
-123,0-123,1:	NAME	'y'
-123,2-123,3:	OP	'='
-123,4-125,3:	STRING	'Ur"""foo\nbar \\\\ baz\n"""'
-125,4-125,5:	OP	'+'
-125,6-126,3:	STRING	"uR'''spam\n'''"
-126,3-126,4:	NEWLINE	'\n'
-127,0-127,1:	NL	'\n'
-128,0-128,13:	COMMENT	'# Indentation'
-128,13-128,14:	NL	'\n'
-129,0-129,2:	NAME	'if'
-129,3-129,4:	NUMBER	'1'
-129,4-129,5:	OP	':'
-129,5-129,6:	NEWLINE	'\n'
-130,0-130,4:	INDENT	'    '
-130,4-130,5:	NAME	'x'
-130,6-130,7:	OP	'='
-130,8-130,9:	NUMBER	'2'
-130,9-130,10:	NEWLINE	'\n'
-131,0-131,0:	DEDENT	''
-131,0-131,2:	NAME	'if'
-131,3-131,4:	NUMBER	'1'
-131,4-131,5:	OP	':'
-131,5-131,6:	NEWLINE	'\n'
-132,0-132,8:	INDENT	'        '
-132,8-132,9:	NAME	'x'
-132,10-132,11:	OP	'='
-132,12-132,13:	NUMBER	'2'
-132,13-132,14:	NEWLINE	'\n'
-133,0-133,0:	DEDENT	''
-133,0-133,2:	NAME	'if'
-133,3-133,4:	NUMBER	'1'
-133,4-133,5:	OP	':'
-133,5-133,6:	NEWLINE	'\n'
-134,0-134,4:	INDENT	'    '
-134,4-134,9:	NAME	'while'
-134,10-134,11:	NUMBER	'0'
-134,11-134,12:	OP	':'
-134,12-134,13:	NEWLINE	'\n'
-135,0-135,5:	INDENT	'     '
-135,5-135,7:	NAME	'if'
-135,8-135,9:	NUMBER	'0'
-135,9-135,10:	OP	':'
-135,10-135,11:	NEWLINE	'\n'
-136,0-136,11:	INDENT	'           '
-136,11-136,12:	NAME	'x'
-136,13-136,14:	OP	'='
-136,15-136,16:	NUMBER	'2'
-136,16-136,17:	NEWLINE	'\n'
-137,5-137,5:	DEDENT	''
-137,5-137,6:	NAME	'x'
-137,7-137,8:	OP	'='
-137,9-137,10:	NUMBER	'2'
-137,10-137,11:	NEWLINE	'\n'
-138,0-138,0:	DEDENT	''
-138,0-138,0:	DEDENT	''
-138,0-138,2:	NAME	'if'
-138,3-138,4:	NUMBER	'0'
-138,4-138,5:	OP	':'
-138,5-138,6:	NEWLINE	'\n'
-139,0-139,2:	INDENT	'  '
-139,2-139,4:	NAME	'if'
-139,5-139,6:	NUMBER	'2'
-139,6-139,7:	OP	':'
-139,7-139,8:	NEWLINE	'\n'
-140,0-140,3:	INDENT	'   '
-140,3-140,8:	NAME	'while'
-140,9-140,10:	NUMBER	'0'
-140,10-140,11:	OP	':'
-140,11-140,12:	NEWLINE	'\n'
-141,0-141,8:	INDENT	'        '
-141,8-141,10:	NAME	'if'
-141,11-141,12:	NUMBER	'1'
-141,12-141,13:	OP	':'
-141,13-141,14:	NEWLINE	'\n'
-142,0-142,10:	INDENT	'          '
-142,10-142,11:	NAME	'x'
-142,12-142,13:	OP	'='
-142,14-142,15:	NUMBER	'2'
-142,15-142,16:	NEWLINE	'\n'
-143,0-143,1:	NL	'\n'
-144,0-144,11:	COMMENT	'# Operators'
-144,11-144,12:	NL	'\n'
-145,0-145,1:	NL	'\n'
-146,0-146,0:	DEDENT	''
-146,0-146,0:	DEDENT	''
-146,0-146,0:	DEDENT	''
-146,0-146,0:	DEDENT	''
-146,0-146,3:	NAME	'def'
-146,4-146,7:	NAME	'd22'
-146,7-146,8:	OP	'('
-146,8-146,9:	NAME	'a'
-146,9-146,10:	OP	','
-146,11-146,12:	NAME	'b'
-146,12-146,13:	OP	','
-146,14-146,15:	NAME	'c'
-146,15-146,16:	OP	'='
-146,16-146,17:	NUMBER	'1'
-146,17-146,18:	OP	','
-146,19-146,20:	NAME	'd'
-146,20-146,21:	OP	'='
-146,21-146,22:	NUMBER	'2'
-146,22-146,23:	OP	')'
-146,23-146,24:	OP	':'
-146,25-146,29:	NAME	'pass'
-146,29-146,30:	NEWLINE	'\n'
-147,0-147,3:	NAME	'def'
-147,4-147,8:	NAME	'd01v'
-147,8-147,9:	OP	'('
-147,9-147,10:	NAME	'a'
-147,10-147,11:	OP	'='
-147,11-147,12:	NUMBER	'1'
-147,12-147,13:	OP	','
-147,14-147,15:	OP	'*'
-147,15-147,20:	NAME	'restt'
-147,20-147,21:	OP	','
-147,22-147,24:	OP	'**'
-147,24-147,29:	NAME	'restd'
-147,29-147,30:	OP	')'
-147,30-147,31:	OP	':'
-147,32-147,36:	NAME	'pass'
-147,36-147,37:	NEWLINE	'\n'
-148,0-148,1:	NL	'\n'
-149,0-149,1:	OP	'('
-149,1-149,2:	NAME	'x'
-149,2-149,3:	OP	','
-149,4-149,5:	NAME	'y'
-149,5-149,6:	OP	')'
-149,7-149,9:	OP	'<>'
-149,10-149,11:	OP	'('
-149,11-149,12:	OP	'{'
-149,12-149,15:	STRING	"'a'"
-149,15-149,16:	OP	':'
-149,16-149,17:	NUMBER	'1'
-149,17-149,18:	OP	'}'
-149,18-149,19:	OP	','
-149,20-149,21:	OP	'{'
-149,21-149,24:	STRING	"'b'"
-149,24-149,25:	OP	':'
-149,25-149,26:	NUMBER	'2'
-149,26-149,27:	OP	'}'
-149,27-149,28:	OP	')'
-149,28-149,29:	NEWLINE	'\n'
-150,0-150,1:	NL	'\n'
-151,0-151,12:	COMMENT	'# comparison'
-151,12-151,13:	NL	'\n'
-152,0-152,2:	NAME	'if'
-152,3-152,4:	NUMBER	'1'
-152,5-152,6:	OP	'<'
-152,7-152,8:	NUMBER	'1'
-152,9-152,10:	OP	'>'
-152,11-152,12:	NUMBER	'1'
-152,13-152,15:	OP	'=='
-152,16-152,17:	NUMBER	'1'
-152,18-152,20:	OP	'>='
-152,21-152,22:	NUMBER	'1'
-152,23-152,25:	OP	'<='
-152,26-152,27:	NUMBER	'1'
-152,28-152,30:	OP	'<>'
-152,31-152,32:	NUMBER	'1'
-152,33-152,35:	OP	'!='
-152,36-152,37:	NUMBER	'1'
-152,38-152,40:	NAME	'in'
-152,41-152,42:	NUMBER	'1'
-152,43-152,46:	NAME	'not'
-152,47-152,49:	NAME	'in'
-152,50-152,51:	NUMBER	'1'
-152,52-152,54:	NAME	'is'
-152,55-152,56:	NUMBER	'1'
-152,57-152,59:	NAME	'is'
-152,60-152,63:	NAME	'not'
-152,64-152,65:	NUMBER	'1'
-152,65-152,66:	OP	':'
-152,67-152,71:	NAME	'pass'
-152,71-152,72:	NEWLINE	'\n'
-153,0-153,1:	NL	'\n'
-154,0-154,8:	COMMENT	'# binary'
-154,8-154,9:	NL	'\n'
-155,0-155,1:	NAME	'x'
-155,2-155,3:	OP	'='
-155,4-155,5:	NUMBER	'1'
-155,6-155,7:	OP	'&'
-155,8-155,9:	NUMBER	'1'
-155,9-155,10:	NEWLINE	'\n'
-156,0-156,1:	NAME	'x'
-156,2-156,3:	OP	'='
-156,4-156,5:	NUMBER	'1'
-156,6-156,7:	OP	'^'
-156,8-156,9:	NUMBER	'1'
-156,9-156,10:	NEWLINE	'\n'
-157,0-157,1:	NAME	'x'
-157,2-157,3:	OP	'='
-157,4-157,5:	NUMBER	'1'
-157,6-157,7:	OP	'|'
-157,8-157,9:	NUMBER	'1'
-157,9-157,10:	NEWLINE	'\n'
-158,0-158,1:	NL	'\n'
-159,0-159,7:	COMMENT	'# shift'
-159,7-159,8:	NL	'\n'
-160,0-160,1:	NAME	'x'
-160,2-160,3:	OP	'='
-160,4-160,5:	NUMBER	'1'
-160,6-160,8:	OP	'<<'
-160,9-160,10:	NUMBER	'1'
-160,11-160,13:	OP	'>>'
-160,14-160,15:	NUMBER	'1'
-160,15-160,16:	NEWLINE	'\n'
-161,0-161,1:	NL	'\n'
-162,0-162,10:	COMMENT	'# additive'
-162,10-162,11:	NL	'\n'
-163,0-163,1:	NAME	'x'
-163,2-163,3:	OP	'='
-163,4-163,5:	NUMBER	'1'
-163,6-163,7:	OP	'-'
-163,8-163,9:	NUMBER	'1'
-163,10-163,11:	OP	'+'
-163,12-163,13:	NUMBER	'1'
-163,14-163,15:	OP	'-'
-163,16-163,17:	NUMBER	'1'
-163,18-163,19:	OP	'+'
-163,20-163,21:	NUMBER	'1'
-163,21-163,22:	NEWLINE	'\n'
-164,0-164,1:	NL	'\n'
-165,0-165,16:	COMMENT	'# multiplicative'
-165,16-165,17:	NL	'\n'
-166,0-166,1:	NAME	'x'
-166,2-166,3:	OP	'='
-166,4-166,5:	NUMBER	'1'
-166,6-166,7:	OP	'/'
-166,8-166,9:	NUMBER	'1'
-166,10-166,11:	OP	'*'
-166,12-166,13:	NUMBER	'1'
-166,14-166,15:	OP	'%'
-166,16-166,17:	NUMBER	'1'
-166,17-166,18:	NEWLINE	'\n'
-167,0-167,1:	NL	'\n'
-168,0-168,7:	COMMENT	'# unary'
-168,7-168,8:	NL	'\n'
-169,0-169,1:	NAME	'x'
-169,2-169,3:	OP	'='
-169,4-169,5:	OP	'~'
-169,5-169,6:	NUMBER	'1'
-169,7-169,8:	OP	'^'
-169,9-169,10:	NUMBER	'1'
-169,11-169,12:	OP	'&'
-169,13-169,14:	NUMBER	'1'
-169,15-169,16:	OP	'|'
-169,17-169,18:	NUMBER	'1'
-169,19-169,20:	OP	'&'
-169,21-169,22:	NUMBER	'1'
-169,23-169,24:	OP	'^'
-169,25-169,26:	OP	'-'
-169,26-169,27:	NUMBER	'1'
-169,27-169,28:	NEWLINE	'\n'
-170,0-170,1:	NAME	'x'
-170,2-170,3:	OP	'='
-170,4-170,5:	OP	'-'
-170,5-170,6:	NUMBER	'1'
-170,6-170,7:	OP	'*'
-170,7-170,8:	NUMBER	'1'
-170,8-170,9:	OP	'/'
-170,9-170,10:	NUMBER	'1'
-170,11-170,12:	OP	'+'
-170,13-170,14:	NUMBER	'1'
-170,14-170,15:	OP	'*'
-170,15-170,16:	NUMBER	'1'
-170,17-170,18:	OP	'-'
-170,19-170,20:	OP	'-'
-170,20-170,21:	OP	'-'
-170,21-170,22:	OP	'-'
-170,22-170,23:	NUMBER	'1'
-170,23-170,24:	OP	'*'
-170,24-170,25:	NUMBER	'1'
-170,25-170,26:	NEWLINE	'\n'
-171,0-171,1:	NL	'\n'
-172,0-172,10:	COMMENT	'# selector'
-172,10-172,11:	NL	'\n'
-173,0-173,6:	NAME	'import'
-173,7-173,10:	NAME	'sys'
-173,10-173,11:	OP	','
-173,12-173,16:	NAME	'time'
-173,16-173,17:	NEWLINE	'\n'
-174,0-174,1:	NAME	'x'
-174,2-174,3:	OP	'='
-174,4-174,7:	NAME	'sys'
-174,7-174,8:	OP	'.'
-174,8-174,15:	NAME	'modules'
-174,15-174,16:	OP	'['
-174,16-174,22:	STRING	"'time'"
-174,22-174,23:	OP	']'
-174,23-174,24:	OP	'.'
-174,24-174,28:	NAME	'time'
-174,28-174,29:	OP	'('
-174,29-174,30:	OP	')'
-174,30-174,31:	NEWLINE	'\n'
-175,0-175,1:	NL	'\n'
-176,0-176,1:	OP	'@'
-176,1-176,13:	NAME	'staticmethod'
-176,13-176,14:	NEWLINE	'\n'
-177,0-177,3:	NAME	'def'
-177,4-177,7:	NAME	'foo'
-177,7-177,8:	OP	'('
-177,8-177,9:	OP	')'
-177,9-177,10:	OP	':'
-177,11-177,15:	NAME	'pass'
-177,15-177,16:	NEWLINE	'\n'
-178,0-178,1:	NL	'\n'
-179,0-179,0:	ENDMARKER	''
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index be6c18d..d6cfb65 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1,112 +1,502 @@
-"""Tests for the tokenize module.
+doctests = """
+Tests for the tokenize module.
 
-The tests were originally written in the old Python style, where the
-test output was compared to a golden file.  This docstring represents
-the first steps towards rewriting the entire test as a doctest.
+    >>> import glob, random, sys
 
-The tests can be really simple.  Given a small fragment of source
-code, print out a table with the tokens.  The ENDMARK is omitted for
+The tests can be really simple. Given a small fragment of source
+code, print out a table with thokens. The ENDMARK is omitted for
 brevity.
 
->>> dump_tokens("1 + 1")
-NUMBER      '1'           (1, 0) (1, 1)
-OP          '+'           (1, 2) (1, 3)
-NUMBER      '1'           (1, 4) (1, 5)
-
-A comment generates a token here, unlike in the parser module.  The
-comment token is followed by an NL or a NEWLINE token, depending on
-whether the line contains the completion of a statement.
-
->>> dump_tokens("if False:\\n"
-...             "    # NL\\n"
-...             "    True = False # NEWLINE\\n")
-NAME        'if'          (1, 0) (1, 2)
-NAME        'False'       (1, 3) (1, 8)
-OP          ':'           (1, 8) (1, 9)
-NEWLINE     '\\n'          (1, 9) (1, 10)
-COMMENT     '# NL'        (2, 4) (2, 8)
-NL          '\\n'          (2, 8) (2, 9)
-INDENT      '    '        (3, 0) (3, 4)
-NAME        'True'        (3, 4) (3, 8)
-OP          '='           (3, 9) (3, 10)
-NAME        'False'       (3, 11) (3, 16)
-COMMENT     '# NEWLINE'   (3, 17) (3, 26)
-NEWLINE     '\\n'          (3, 26) (3, 27)
-DEDENT      ''            (4, 0) (4, 0)
-
-
-There will be a bunch more tests of specific source patterns.
-
-The tokenize module also defines an untokenize function that should
-regenerate the original program text from the tokens.
-
-There are some standard formatting practices that are easy to get right.
-
->>> roundtrip("if x == 1:\\n"
-...           "    print x\\n")
-if x == 1:
-    print x
+    >>> dump_tokens("1 + 1")
+    NUMBER     '1'           (1, 0) (1, 1)
+    OP         '+'           (1, 2) (1, 3)
+    NUMBER     '1'           (1, 4) (1, 5)
+
+    >>> dump_tokens("if False:\\n"
+    ...             "    # NL\\n"
+    ...             "    True = False # NEWLINE\\n")
+    NAME       'if'          (1, 0) (1, 2)
+    NAME       'False'       (1, 3) (1, 8)
+    OP         ':'           (1, 8) (1, 9)
+    NEWLINE    '\\n'          (1, 9) (1, 10)
+    COMMENT    '# NL'        (2, 4) (2, 8)
+    NL         '\\n'          (2, 8) (2, 9)
+    INDENT     '    '        (3, 0) (3, 4)
+    NAME       'True'        (3, 4) (3, 8)
+    OP         '='           (3, 9) (3, 10)
+    NAME       'False'       (3, 11) (3, 16)
+    COMMENT    '# NEWLINE'   (3, 17) (3, 26)
+    NEWLINE    '\\n'          (3, 26) (3, 27)
+    DEDENT     ''            (4, 0) (4, 0)
+
+    >>> indent_error_file = \"""
+    ... def k(x):
+    ...     x += 2
+    ...   x += 5
+    ... \"""
+
+    >>> for tok in generate_tokens(StringIO(indent_error_file).readline): pass
+    Traceback (most recent call last):
+        ...
+    IndentationError: unindent does not match any outer indentation level
+
+Test roundtrip for `untokenize`. `f` is an open file or a string. The source
+code in f is tokenized, converted back to source code via tokenize.untokenize(),
+and tokenized again from the latter. The test fails if the second tokenization
+doesn't match the first.
+
+    >>> def roundtrip(f):
+    ...     if isinstance(f, str): f = StringIO(f)
+    ...     token_list = list(generate_tokens(f.readline))
+    ...     f.close()
+    ...     tokens1 = [tok[:2] for tok in token_list]
+    ...     new_text = untokenize(tokens1)
+    ...     readline = iter(new_text.splitlines(1)).next
+    ...     tokens2 = [tok[:2] for tok in generate_tokens(readline)]
+    ...     return tokens1 == tokens2
+    ...
+
+There are some standard formattig practises that are easy to get right.
+
+    >>> roundtrip("if x == 1:\\n"
+    ...           "    print x\\n")
+    True
+
+    >>> roundtrip("# This is a comment\\n# This also")
+    True
 
 Some people use different formatting conventions, which makes
-untokenize a little trickier.  Note that this test involves trailing
-whitespace after the colon.  Note that we use hex escapes to make the
-two trailing blanks apparent in the expected output.
-
->>> roundtrip("if   x  ==  1  :  \\n"
-...           "  print x\\n")
-if   x  ==  1  :\x20\x20
-  print x
-
-Comments need to go in the right place.
-
->>> roundtrip("if x == 1:\\n"
-...           "    # A comment by itself.\\n"
-...           "    print x  # Comment here, too.\\n"
-...           "    # Another comment.\\n"
-...           "after_if = True\\n")
-if x == 1:
-    # A comment by itself.
-    print x  # Comment here, too.
-    # Another comment.
-after_if = True
-
->>> roundtrip("if (x  # The comments need to go in the right place\\n"
-...           "    == 1):\\n"
-...           "    print 'x == 1'\\n")
-if (x  # The comments need to go in the right place
-    == 1):
-    print 'x == 1'
-
+untokenize a little trickier. Note that this test involves trailing
+whitespace after the colon. Note that we use hex escapes to make the
+two trailing blanks apperant in the expected output.
+
+    >>> roundtrip("if x == 1 : \\n"
+    ...           "  print x\\n")
+    True
+
+    >>> f = test_support.findfile("tokenize_tests" + os.extsep + "txt")
+    >>> roundtrip(open(f))
+    True
+
+    >>> roundtrip("if x == 1:\\n"
+    ...           "    # A comment by itself.\\n"
+    ...           "    print x # Comment here, too.\\n"
+    ...           "    # Another comment.\\n"
+    ...           "after_if = True\\n")
+    True
+
+    >>> roundtrip("if (x # The comments need to go in the right place\\n"
+    ...           "    == 1):\\n"
+    ...           "    print 'x==1'\\n")
+    True
+
+    >>> roundtrip("class Test: # A comment here\\n"
+    ...           "  # A comment with weird indent\\n"
+    ...           "  after_com = 5\\n"
+    ...           "  def x(m): return m*5 # a one liner\\n"
+    ...           "  def y(m): # A whitespace after the colon\\n"
+    ...           "     return y*4 # 3-space indent\\n")
+    True
+
+Some error-handling code
+
+    >>> roundtrip("try: import somemodule\\n"
+    ...           "except ImportError: # comment\\n"
+    ...           "    print 'Can not import' # comment2\\n"
+    ...           "else:   print 'Loaded'\\n")
+    True
+
+Balancing contunuation
+
+    >>> roundtrip("a = (3,4, \\n"
+    ...           "5,6)\\n"
+    ...           "y = [3, 4,\\n"
+    ...           "5]\\n"
+    ...           "z = {'a': 5,\\n"
+    ...           "'b':15, 'c':True}\\n"
+    ...           "x = len(y) + 5 - a[\\n"
+    ...           "3] - a[2]\\n"
+    ...           "+ len(z) - z[\\n"
+    ...           "'b']\\n")
+    True
+
+Ordinary integers and binary operators
+
+    >>> dump_tokens("0xff <= 255")
+    NUMBER     '0xff'        (1, 0) (1, 4)
+    OP         '<='          (1, 5) (1, 7)
+    NUMBER     '255'         (1, 8) (1, 11)
+    >>> dump_tokens("01234567 > ~0x15")
+    NUMBER     '01234567'    (1, 0) (1, 8)
+    OP         '>'           (1, 9) (1, 10)
+    OP         '~'           (1, 11) (1, 12)
+    NUMBER     '0x15'        (1, 12) (1, 16)
+    >>> dump_tokens("2134568 != 01231515")
+    NUMBER     '2134568'     (1, 0) (1, 7)
+    OP         '!='          (1, 8) (1, 10)
+    NUMBER     '01231515'    (1, 11) (1, 19)
+    >>> dump_tokens("(-124561-1) & 0200000000")
+    OP         '('           (1, 0) (1, 1)
+    OP         '-'           (1, 1) (1, 2)
+    NUMBER     '124561'      (1, 2) (1, 8)
+    OP         '-'           (1, 8) (1, 9)
+    NUMBER     '1'           (1, 9) (1, 10)
+    OP         ')'           (1, 10) (1, 11)
+    OP         '&'           (1, 12) (1, 13)
+    NUMBER     '0200000000'  (1, 14) (1, 24)
+    >>> dump_tokens("0xdeadbeef != -1")
+    NUMBER     '0xdeadbeef'  (1, 0) (1, 10)
+    OP         '!='          (1, 11) (1, 13)
+    OP         '-'           (1, 14) (1, 15)
+    NUMBER     '1'           (1, 15) (1, 16)
+    >>> dump_tokens("0xdeadc0de & 012345")
+    NUMBER     '0xdeadc0de'  (1, 0) (1, 10)
+    OP         '&'           (1, 11) (1, 12)
+    NUMBER     '012345'      (1, 13) (1, 19)
+    >>> dump_tokens("0xFF & 0x15 | 1234")
+    NUMBER     '0xFF'        (1, 0) (1, 4)
+    OP         '&'           (1, 5) (1, 6)
+    NUMBER     '0x15'        (1, 7) (1, 11)
+    OP         '|'           (1, 12) (1, 13)
+    NUMBER     '1234'        (1, 14) (1, 18)
+
+Long integers
+
+    >>> dump_tokens("x = 0L")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '0L'          (1, 4) (1, 6)
+    >>> dump_tokens("x = 0xfffffffffff")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '0xffffffffff (1, 4) (1, 17)
+    >>> dump_tokens("x = 123141242151251616110l")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '123141242151 (1, 4) (1, 26)
+    >>> dump_tokens("x = -15921590215012591L")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    OP         '-'           (1, 4) (1, 5)
+    NUMBER     '159215902150 (1, 5) (1, 23)
+
+Floating point numbers
+
+    >>> dump_tokens("x = 3.14159")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '3.14159'     (1, 4) (1, 11)
+    >>> dump_tokens("x = 314159.")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '314159.'     (1, 4) (1, 11)
+    >>> dump_tokens("x = .314159")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '.314159'     (1, 4) (1, 11)
+    >>> dump_tokens("x = 3e14159")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '3e14159'     (1, 4) (1, 11)
+    >>> dump_tokens("x = 3E123")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '3E123'       (1, 4) (1, 9)
+    >>> dump_tokens("x+y = 3e-1230")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '+'           (1, 1) (1, 2)
+    NAME       'y'           (1, 2) (1, 3)
+    OP         '='           (1, 4) (1, 5)
+    NUMBER     '3e-1230'     (1, 6) (1, 13)
+    >>> dump_tokens("x = 3.14e159")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '3.14e159'    (1, 4) (1, 12)
+
+String literals
+
+    >>> dump_tokens("x = ''; y = \\\"\\\"")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    STRING     "''"          (1, 4) (1, 6)
+    OP         ';'           (1, 6) (1, 7)
+    NAME       'y'           (1, 8) (1, 9)
+    OP         '='           (1, 10) (1, 11)
+    STRING     '""'          (1, 12) (1, 14)
+    >>> dump_tokens("x = '\\\"'; y = \\\"'\\\"")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    STRING     '\\'"\\''       (1, 4) (1, 7)
+    OP         ';'           (1, 7) (1, 8)
+    NAME       'y'           (1, 9) (1, 10)
+    OP         '='           (1, 11) (1, 12)
+    STRING     '"\\'"'        (1, 13) (1, 16)
+    >>> dump_tokens("x = \\\"doesn't \\\"shrink\\\", does it\\\"")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    STRING     '"doesn\\'t "' (1, 4) (1, 14)
+    NAME       'shrink'      (1, 14) (1, 20)
+    STRING     '", does it"' (1, 20) (1, 31)
+    >>> dump_tokens("x = u'abc' + U'ABC'")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    STRING     "u'abc'"      (1, 4) (1, 10)
+    OP         '+'           (1, 11) (1, 12)
+    STRING     "U'ABC'"      (1, 13) (1, 19)
+    >>> dump_tokens('y = u"ABC" + U"ABC"')
+    NAME       'y'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    STRING     'u"ABC"'      (1, 4) (1, 10)
+    OP         '+'           (1, 11) (1, 12)
+    STRING     'U"ABC"'      (1, 13) (1, 19)
+    >>> dump_tokens("x = ur'abc' + Ur'ABC' + uR'ABC' + UR'ABC'")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    STRING     "ur'abc'"     (1, 4) (1, 11)
+    OP         '+'           (1, 12) (1, 13)
+    STRING     "Ur'ABC'"     (1, 14) (1, 21)
+    OP         '+'           (1, 22) (1, 23)
+    STRING     "uR'ABC'"     (1, 24) (1, 31)
+    OP         '+'           (1, 32) (1, 33)
+    STRING     "UR'ABC'"     (1, 34) (1, 41)
+    >>> dump_tokens('y = ur"abc" + Ur"ABC" + uR"ABC" + UR"ABC"')
+    NAME       'y'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    STRING     'ur"abc"'     (1, 4) (1, 11)
+    OP         '+'           (1, 12) (1, 13)
+    STRING     'Ur"ABC"'     (1, 14) (1, 21)
+    OP         '+'           (1, 22) (1, 23)
+    STRING     'uR"ABC"'     (1, 24) (1, 31)
+    OP         '+'           (1, 32) (1, 33)
+    STRING     'UR"ABC"'     (1, 34) (1, 41)
+
+Operators
+
+    >>> dump_tokens("def d22(a, b, c=2, d=2, *k): pass")
+    NAME       'def'         (1, 0) (1, 3)
+    NAME       'd22'         (1, 4) (1, 7)
+    OP         '('           (1, 7) (1, 8)
+    NAME       'a'           (1, 8) (1, 9)
+    OP         ','           (1, 9) (1, 10)
+    NAME       'b'           (1, 11) (1, 12)
+    OP         ','           (1, 12) (1, 13)
+    NAME       'c'           (1, 14) (1, 15)
+    OP         '='           (1, 15) (1, 16)
+    NUMBER     '2'           (1, 16) (1, 17)
+    OP         ','           (1, 17) (1, 18)
+    NAME       'd'           (1, 19) (1, 20)
+    OP         '='           (1, 20) (1, 21)
+    NUMBER     '2'           (1, 21) (1, 22)
+    OP         ','           (1, 22) (1, 23)
+    OP         '*'           (1, 24) (1, 25)
+    NAME       'k'           (1, 25) (1, 26)
+    OP         ')'           (1, 26) (1, 27)
+    OP         ':'           (1, 27) (1, 28)
+    NAME       'pass'        (1, 29) (1, 33)
+    >>> dump_tokens("def d01v_(a=1, *k, **w): pass")
+    NAME       'def'         (1, 0) (1, 3)
+    NAME       'd01v_'       (1, 4) (1, 9)
+    OP         '('           (1, 9) (1, 10)
+    NAME       'a'           (1, 10) (1, 11)
+    OP         '='           (1, 11) (1, 12)
+    NUMBER     '1'           (1, 12) (1, 13)
+    OP         ','           (1, 13) (1, 14)
+    OP         '*'           (1, 15) (1, 16)
+    NAME       'k'           (1, 16) (1, 17)
+    OP         ','           (1, 17) (1, 18)
+    OP         '**'          (1, 19) (1, 21)
+    NAME       'w'           (1, 21) (1, 22)
+    OP         ')'           (1, 22) (1, 23)
+    OP         ':'           (1, 23) (1, 24)
+    NAME       'pass'        (1, 25) (1, 29)
+
+Comparison
+
+    >>> dump_tokens("if 1 < 1 > 1 == 1 >= 5 <= 0x15 <= 0x12 != " +
+    ...             "1 and 5 in 1 not in 1 is 1 or 5 is not 1: pass")
+    NAME       'if'          (1, 0) (1, 2)
+    NUMBER     '1'           (1, 3) (1, 4)
+    OP         '<'           (1, 5) (1, 6)
+    NUMBER     '1'           (1, 7) (1, 8)
+    OP         '>'           (1, 9) (1, 10)
+    NUMBER     '1'           (1, 11) (1, 12)
+    OP         '=='          (1, 13) (1, 15)
+    NUMBER     '1'           (1, 16) (1, 17)
+    OP         '>='          (1, 18) (1, 20)
+    NUMBER     '5'           (1, 21) (1, 22)
+    OP         '<='          (1, 23) (1, 25)
+    NUMBER     '0x15'        (1, 26) (1, 30)
+    OP         '<='          (1, 31) (1, 33)
+    NUMBER     '0x12'        (1, 34) (1, 38)
+    OP         '!='          (1, 39) (1, 41)
+    NUMBER     '1'           (1, 42) (1, 43)
+    NAME       'and'         (1, 44) (1, 47)
+    NUMBER     '5'           (1, 48) (1, 49)
+    NAME       'in'          (1, 50) (1, 52)
+    NUMBER     '1'           (1, 53) (1, 54)
+    NAME       'not'         (1, 55) (1, 58)
+    NAME       'in'          (1, 59) (1, 61)
+    NUMBER     '1'           (1, 62) (1, 63)
+    NAME       'is'          (1, 64) (1, 66)
+    NUMBER     '1'           (1, 67) (1, 68)
+    NAME       'or'          (1, 69) (1, 71)
+    NUMBER     '5'           (1, 72) (1, 73)
+    NAME       'is'          (1, 74) (1, 76)
+    NAME       'not'         (1, 77) (1, 80)
+    NUMBER     '1'           (1, 81) (1, 82)
+    OP         ':'           (1, 82) (1, 83)
+    NAME       'pass'        (1, 84) (1, 88)
+
+Shift
+
+    >>> dump_tokens("x = 1 << 1 >> 5")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '1'           (1, 4) (1, 5)
+    OP         '<<'          (1, 6) (1, 8)
+    NUMBER     '1'           (1, 9) (1, 10)
+    OP         '>>'          (1, 11) (1, 13)
+    NUMBER     '5'           (1, 14) (1, 15)
+
+Additive
+
+    >>> dump_tokens("x = 1 - y + 15 - 01 + 0x124 + z + a[5]")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '1'           (1, 4) (1, 5)
+    OP         '-'           (1, 6) (1, 7)
+    NAME       'y'           (1, 8) (1, 9)
+    OP         '+'           (1, 10) (1, 11)
+    NUMBER     '15'          (1, 12) (1, 14)
+    OP         '-'           (1, 15) (1, 16)
+    NUMBER     '01'          (1, 17) (1, 19)
+    OP         '+'           (1, 20) (1, 21)
+    NUMBER     '0x124'       (1, 22) (1, 27)
+    OP         '+'           (1, 28) (1, 29)
+    NAME       'z'           (1, 30) (1, 31)
+    OP         '+'           (1, 32) (1, 33)
+    NAME       'a'           (1, 34) (1, 35)
+    OP         '['           (1, 35) (1, 36)
+    NUMBER     '5'           (1, 36) (1, 37)
+    OP         ']'           (1, 37) (1, 38)
+
+Multiplicative
+
+    >>> dump_tokens("x = 1//1*1/5*12%0x12")
+    NAME       'x'           (1, 0) (1, 1)
+    OP         '='           (1, 2) (1, 3)
+    NUMBER     '1'           (1, 4) (1, 5)
+    OP         '//'          (1, 5) (1, 7)
+    NUMBER     '1'           (1, 7) (1, 8)
+    OP         '*'           (1, 8) (1, 9)
+    NUMBER     '1'           (1, 9) (1, 10)
+    OP         '/'           (1, 10) (1, 11)
+    NUMBER     '5'           (1, 11) (1, 12)
+    OP         '*'           (1, 12) (1, 13)
+    NUMBER     '12'          (1, 13) (1, 15)
+    OP         '%'           (1, 15) (1, 16)
+    NUMBER     '0x12'        (1, 16) (1, 20)
+
+Unary
+
+    >>> dump_tokens("~1 ^ 1 & 1 |1 ^ -1")
+    OP         '~'           (1, 0) (1, 1)
+    NUMBER     '1'           (1, 1) (1, 2)
+    OP         '^'           (1, 3) (1, 4)
+    NUMBER     '1'           (1, 5) (1, 6)
+    OP         '&'           (1, 7) (1, 8)
+    NUMBER     '1'           (1, 9) (1, 10)
+    OP         '|'           (1, 11) (1, 12)
+    NUMBER     '1'           (1, 12) (1, 13)
+    OP         '^'           (1, 14) (1, 15)
+    OP         '-'           (1, 16) (1, 17)
+    NUMBER     '1'           (1, 17) (1, 18)
+    >>> dump_tokens("-1*1/1+1*1//1 - ---1**1")
+    OP         '-'           (1, 0) (1, 1)
+    NUMBER     '1'           (1, 1) (1, 2)
+    OP         '*'           (1, 2) (1, 3)
+    NUMBER     '1'           (1, 3) (1, 4)
+    OP         '/'           (1, 4) (1, 5)
+    NUMBER     '1'           (1, 5) (1, 6)
+    OP         '+'           (1, 6) (1, 7)
+    NUMBER     '1'           (1, 7) (1, 8)
+    OP         '*'           (1, 8) (1, 9)
+    NUMBER     '1'           (1, 9) (1, 10)
+    OP         '//'          (1, 10) (1, 12)
+    NUMBER     '1'           (1, 12) (1, 13)
+    OP         '-'           (1, 14) (1, 15)
+    OP         '-'           (1, 16) (1, 17)
+    OP         '-'           (1, 17) (1, 18)
+    OP         '-'           (1, 18) (1, 19)
+    NUMBER     '1'           (1, 19) (1, 20)
+    OP         '**'          (1, 20) (1, 22)
+    NUMBER     '1'           (1, 22) (1, 23)
+
+Selector
+
+    >>> dump_tokens("import sys, time\\nx = sys.modules['time'].time()")
+    NAME       'import'      (1, 0) (1, 6)
+    NAME       'sys'         (1, 7) (1, 10)
+    OP         ','           (1, 10) (1, 11)
+    NAME       'time'        (1, 12) (1, 16)
+    NEWLINE    '\\n'          (1, 16) (1, 17)
+    NAME       'x'           (2, 0) (2, 1)
+    OP         '='           (2, 2) (2, 3)
+    NAME       'sys'         (2, 4) (2, 7)
+    OP         '.'           (2, 7) (2, 8)
+    NAME       'modules'     (2, 8) (2, 15)
+    OP         '['           (2, 15) (2, 16)
+    STRING     "'time'"      (2, 16) (2, 22)
+    OP         ']'           (2, 22) (2, 23)
+    OP         '.'           (2, 23) (2, 24)
+    NAME       'time'        (2, 24) (2, 28)
+    OP         '('           (2, 28) (2, 29)
+    OP         ')'           (2, 29) (2, 30)
+
+Methods
+
+    >>> dump_tokens("@staticmethod\\ndef foo(x,y): pass")
+    OP         '@'           (1, 0) (1, 1)
+    NAME       'staticmethod (1, 1) (1, 13)
+    NEWLINE    '\\n'          (1, 13) (1, 14)
+    NAME       'def'         (2, 0) (2, 3)
+    NAME       'foo'         (2, 4) (2, 7)
+    OP         '('           (2, 7) (2, 8)
+    NAME       'x'           (2, 8) (2, 9)
+    OP         ','           (2, 9) (2, 10)
+    NAME       'y'           (2, 10) (2, 11)
+    OP         ')'           (2, 11) (2, 12)
+    OP         ':'           (2, 12) (2, 13)
+    NAME       'pass'        (2, 14) (2, 18)
+
+Backslash means line continuation, except for comments
+
+    >>> roundtrip("x=1+\\\\n"
+    ...           "1\\n"
+    ...           "# This is a comment\\\\n"
+    ...           "# This also\\n")
+    True
+    >>> roundtrip("# Comment \\\\nx = 0")
+    True
+
+    >>>
+    >>> tempdir = os.path.dirname(f) or os.curdir
+    >>> testfiles = glob.glob(os.path.join(tempdir, "test*.py"))
+    >>> if not test_support.is_resource_enabled("compiler"):
+    ...     testfiles = random.sample(testfiles, 10)
+    ...
+    >>> for testfile in testfiles:
+    ...     if not roundtrip(open(testfile)): break
+    ... else: True
+    True
 """
 
-import os, glob, random, time, sys
-from cStringIO import StringIO
-from test.test_support import (verbose, findfile, is_resource_enabled,
-                               TestFailed)
-from tokenize import (tokenize, generate_tokens, untokenize, tok_name,
-                      ENDMARKER, NUMBER, NAME, OP, STRING, COMMENT)
-
-# How much time in seconds can pass before we print a 'Still working' message.
-_PRINT_WORKING_MSG_INTERVAL = 5 * 60
-
-# Test roundtrip for `untokenize`.  `f` is a file path.  The source code in f
-# is tokenized, converted back to source code via tokenize.untokenize(),
-# and tokenized again from the latter.  The test fails if the second
-# tokenization doesn't match the first.
-def test_roundtrip(f):
-    ## print 'Testing:', f
-    fobj = open(f)
-    try:
-        fulltok = list(generate_tokens(fobj.readline))
-    finally:
-        fobj.close()
-
-    t1 = [tok[:2] for tok in fulltok]
-    newtext = untokenize(t1)
-    readline = iter(newtext.splitlines(1)).next
-    t2 = [tok[:2] for tok in generate_tokens(readline)]
-    if t1 != t2:
-        raise TestFailed("untokenize() roundtrip failed for %r" % f)
+
+from test import test_support
+from tokenize import (tokenize, untokenize, generate_tokens, NUMBER, NAME, OP,
+                     STRING, ENDMARKER, tok_name)
+from StringIO import StringIO
+import os
 
 def dump_tokens(s):
     """Print out the tokens in s in a table format.
@@ -118,12 +508,7 @@ def dump_tokens(s):
         if type == ENDMARKER:
             break
         type = tok_name[type]
-        print "%(type)-10.10s  %(token)-13.13r %(start)s %(end)s" % locals()
-
-def roundtrip(s):
-    f = StringIO(s)
-    source = untokenize(generate_tokens(f.readline))
-    print source,
+        print("%(type)-10.10s %(token)-13.13r %(start)s %(end)s" % locals())
 
 # This is an example from the docs, set up as a doctest.
 def decistmt(s):
@@ -163,61 +548,13 @@ def decistmt(s):
             result.append((toknum, tokval))
     return untokenize(result)
 
-def test_main():
-    if verbose:
-        print 'starting...'
-
-    next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
-
-    # This displays the tokenization of tokenize_tests.py to stdout, and
-    # regrtest.py checks that this equals the expected output (in the
-    # test/output/ directory).
-    f = open(findfile('tokenize_tests' + os.extsep + 'txt'))
-    tokenize(f.readline)
-    f.close()
-
-    # Now run test_roundtrip() over tokenize_test.py too, and over all
-    # (if the "compiler" resource is enabled) or a small random sample (if
-    # "compiler" is not enabled) of the test*.py files.
-    f = findfile('tokenize_tests' + os.extsep + 'txt')
-    test_roundtrip(f)
-
-    testdir = os.path.dirname(f) or os.curdir
-    testfiles = glob.glob(testdir + os.sep + 'test*.py')
-    if not is_resource_enabled('compiler'):
-        testfiles = random.sample(testfiles, 10)
-
-    for f in testfiles:
-        # Print still working message since this test can be really slow
-        if next_time <= time.time():
-            next_time = time.time() + _PRINT_WORKING_MSG_INTERVAL
-            print >>sys.__stdout__, '  test_main still working, be patient...'
-            sys.__stdout__.flush()
-
-        test_roundtrip(f)
-
-    # Test detecton of IndentationError.
-    sampleBadText = """\
-def foo():
-    bar
-  baz
-"""
 
-    try:
-        for tok in generate_tokens(StringIO(sampleBadText).readline):
-            pass
-    except IndentationError:
-        pass
-    else:
-        raise TestFailed("Did not detect IndentationError:")
-
-    # Run the doctests in this module.
-    from test import test_tokenize  # i.e., this module
-    from test.test_support import run_doctest
-    run_doctest(test_tokenize, verbose)
-
-    if verbose:
-        print 'finished'
+__test__ = {"doctests" : doctests, 'decistmt': decistmt}
+
+
+def test_main():
+    from test import test_tokenize
+    test_support.run_doctest(test_tokenize, True)
 
 if __name__ == "__main__":
     test_main()
diff --git a/Misc/ACKS b/Misc/ACKS
index f7a08d4..43bbb90 100644
--- a/Misc/ACKS
+++ b/Misc/ACKS
@@ -178,6 +178,7 @@ Luke Dunstan
 Andy Dustman
 Gary Duzan
 Eugene Dvurechenski
+Josip Dzolonga 
 Maxim Dzumanenko
 Hans Eckardt
 Grant Edwards
diff --git a/Misc/NEWS b/Misc/NEWS
index e316c13..539b785 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -50,6 +50,8 @@ Library
 Tests
 -----
 
+- GHOP 238: Convert test_tokenize to use doctest.
+
 - GHOP 237: Rewrite test_thread using unittest.
 
 - Patch #2232: os.tmpfile might fail on Windows if the user has no