Tests for tokenize.py (Ka-Ping Yee)

3 files changed, 775 insertions, 0 deletions

diff --git a/Lib/test/output/test_tokenize b/Lib/test/output/test_tokenize
new file mode 100644
index 0000000..42465a0
--- /dev/null
+++ b/Lib/test/output/test_tokenize
@@ -0,0 +1,592 @@
+test_tokenize
+1,0-1,35:	COMMENT	"# Tests for the 'tokenize' module.\012"
+2,0-2,43:	COMMENT	'# Large bits stolen from test_grammar.py. \012'
+3,0-3,1:	NEWLINE	'\012'
+4,0-4,11:	COMMENT	'# Comments\012'
+5,0-5,3:	STRING	'"#"'
+5,3-5,4:	NEWLINE	'\012'
+6,0-6,3:	COMMENT	"#'\012"
+7,0-7,3:	COMMENT	'#"\012'
+8,0-8,3:	COMMENT	'#\\\012'
+9,7-9,9:	COMMENT	'#\012'
+10,4-10,10:	COMMENT	'# abc\012'
+11,0-12,4:	STRING	"'''#\012#'''"
+12,4-12,5:	NEWLINE	'\012'
+13,0-13,1:	NEWLINE	'\012'
+14,0-14,1:	NAME	'x'
+14,2-14,3:	OP	'='
+14,4-14,5:	NUMBER	'1'
+14,7-14,8:	COMMENT	'#'
+14,8-14,9:	NEWLINE	'\012'
+15,0-15,1:	NEWLINE	'\012'
+16,0-16,25:	COMMENT	'# Balancing continuation\012'
+17,0-17,1:	NEWLINE	'\012'
+18,0-18,1:	NAME	'a'
+18,2-18,3:	OP	'='
+18,4-18,5:	OP	'('
+18,5-18,6:	NUMBER	'3'
+18,6-18,7:	OP	','
+18,8-18,9:	NUMBER	'4'
+18,9-18,10:	OP	','
+18,10-18,11:	NEWLINE	'\012'
+19,2-19,3:	NUMBER	'5'
+19,3-19,4:	OP	','
+19,5-19,6:	NUMBER	'6'
+19,6-19,7:	OP	')'
+19,7-19,8:	NEWLINE	'\012'
+20,0-20,1:	NAME	'y'
+20,2-20,3:	OP	'='
+20,4-20,5:	OP	'['
+20,5-20,6:	NUMBER	'3'
+20,6-20,7:	OP	','
+20,8-20,9:	NUMBER	'4'
+20,9-20,10:	OP	','
+20,10-20,11:	NEWLINE	'\012'
+21,2-21,3:	NUMBER	'5'
+21,3-21,4:	OP	']'
+21,4-21,5:	NEWLINE	'\012'
+22,0-22,1:	NAME	'z'
+22,2-22,3:	OP	'='
+22,4-22,5:	OP	'{'
+22,5-22,8:	STRING	"'a'"
+22,8-22,9:	OP	':'
+22,9-22,10:	NUMBER	'5'
+22,10-22,11:	OP	','
+22,11-22,12:	NEWLINE	'\012'
+23,2-23,5:	STRING	"'b'"
+23,5-23,6:	OP	':'
+23,6-23,7:	NUMBER	'6'
+23,7-23,8:	OP	'}'
+23,8-23,9:	NEWLINE	'\012'
+24,0-24,1:	NAME	'x'
+24,2-24,3:	OP	'='
+24,4-24,5:	OP	'('
+24,5-24,8:	NAME	'len'
+24,8-24,9:	OP	'('
+24,9-24,10:	OP	'`'
+24,10-24,11:	NAME	'y'
+24,11-24,12:	OP	'`'
+24,12-24,13:	OP	')'
+24,14-24,15:	OP	'+'
+24,16-24,17:	NUMBER	'5'
+24,17-24,18:	OP	'*'
+24,18-24,19:	NAME	'x'
+24,20-24,21:	OP	'-'
+24,22-24,23:	NAME	'a'
+24,23-24,24:	OP	'['
+24,24-24,25:	NEWLINE	'\012'
+25,3-25,4:	NUMBER	'3'
+25,5-25,6:	OP	']'
+25,6-25,7:	NEWLINE	'\012'
+26,3-26,4:	OP	'-'
+26,5-26,6:	NAME	'x'
+26,7-26,8:	OP	'+'
+26,9-26,12:	NAME	'len'
+26,12-26,13:	OP	'('
+26,13-26,14:	OP	'{'
+26,14-26,15:	NEWLINE	'\012'
+27,3-27,4:	OP	'}'
+27,4-27,5:	NEWLINE	'\012'
+28,4-28,5:	OP	')'
+28,5-28,6:	NEWLINE	'\012'
+29,2-29,3:	OP	')'
+29,3-29,4:	NEWLINE	'\012'
+30,0-30,1:	NEWLINE	'\012'
+31,0-31,37:	COMMENT	'# Backslash means line continuation:\012'
+32,0-32,1:	NAME	'x'
+32,2-32,3:	OP	'='
+32,4-32,5:	NUMBER	'1'
+33,0-33,1:	OP	'+'
+33,2-33,3:	NUMBER	'1'
+33,3-33,4:	NEWLINE	'\012'
+34,0-34,1:	NEWLINE	'\012'
+35,0-35,55:	COMMENT	'# Backslash does not means continuation in comments :\\\012'
+36,0-36,1:	NAME	'x'
+36,2-36,3:	OP	'='
+36,4-36,5:	NUMBER	'0'
+36,5-36,6:	NEWLINE	'\012'
+37,0-37,1:	NEWLINE	'\012'
+38,0-38,20:	COMMENT	'# Ordinary integers\012'
+39,0-39,4:	NUMBER	'0xff'
+39,5-39,7:	OP	'<>'
+39,8-39,11:	NUMBER	'255'
+39,11-39,12:	NEWLINE	'\012'
+40,0-40,4:	NUMBER	'0377'
+40,5-40,7:	OP	'<>'
+40,8-40,11:	NUMBER	'255'
+40,11-40,12:	NEWLINE	'\012'
+41,0-41,10:	NUMBER	'2147483647'
+41,13-41,15:	OP	'!='
+41,16-41,28:	NUMBER	'017777777777'
+41,28-41,29:	NEWLINE	'\012'
+42,0-42,1:	OP	'-'
+42,1-42,11:	NUMBER	'2147483647'
+42,11-42,12:	OP	'-'
+42,12-42,13:	NUMBER	'1'
+42,14-42,16:	OP	'!='
+42,17-42,29:	NUMBER	'020000000000'
+42,29-42,30:	NEWLINE	'\012'
+43,0-43,12:	NUMBER	'037777777777'
+43,13-43,15:	OP	'!='
+43,16-43,17:	OP	'-'
+43,17-43,18:	NUMBER	'1'
+43,18-43,19:	NEWLINE	'\012'
+44,0-44,10:	NUMBER	'0xffffffff'
+44,11-44,13:	OP	'!='
+44,14-44,15:	OP	'-'
+44,15-44,16:	NUMBER	'1'
+44,16-44,17:	NEWLINE	'\012'
+45,0-45,1:	NEWLINE	'\012'
+46,0-46,16:	COMMENT	'# Long integers\012'
+47,0-47,1:	NAME	'x'
+47,2-47,3:	OP	'='
+47,4-47,6:	NUMBER	'0L'
+47,6-47,7:	NEWLINE	'\012'
+48,0-48,1:	NAME	'x'
+48,2-48,3:	OP	'='
+48,4-48,6:	NUMBER	'0l'
+48,6-48,7:	NEWLINE	'\012'
+49,0-49,1:	NAME	'x'
+49,2-49,3:	OP	'='
+49,4-49,23:	NUMBER	'0xffffffffffffffffL'
+49,23-49,24:	NEWLINE	'\012'
+50,0-50,1:	NAME	'x'
+50,2-50,3:	OP	'='
+50,4-50,23:	NUMBER	'0xffffffffffffffffl'
+50,23-50,24:	NEWLINE	'\012'
+51,0-51,1:	NAME	'x'
+51,2-51,3:	OP	'='
+51,4-51,23:	NUMBER	'077777777777777777L'
+51,23-51,24:	NEWLINE	'\012'
+52,0-52,1:	NAME	'x'
+52,2-52,3:	OP	'='
+52,4-52,23:	NUMBER	'077777777777777777l'
+52,23-52,24:	NEWLINE	'\012'
+53,0-53,1:	NAME	'x'
+53,2-53,3:	OP	'='
+53,4-53,35:	NUMBER	'123456789012345678901234567890L'
+53,35-53,36:	NEWLINE	'\012'
+54,0-54,1:	NAME	'x'
+54,2-54,3:	OP	'='
+54,4-54,35:	NUMBER	'123456789012345678901234567890l'
+54,35-54,36:	NEWLINE	'\012'
+55,0-55,1:	NEWLINE	'\012'
+56,0-56,25:	COMMENT	'# Floating-point numbers\012'
+57,0-57,1:	NAME	'x'
+57,2-57,3:	OP	'='
+57,4-57,8:	NUMBER	'3.14'
+57,8-57,9:	NEWLINE	'\012'
+58,0-58,1:	NAME	'x'
+58,2-58,3:	OP	'='
+58,4-58,8:	NUMBER	'314.'
+58,8-58,9:	NEWLINE	'\012'
+59,0-59,1:	NAME	'x'
+59,2-59,3:	OP	'='
+59,4-59,9:	NUMBER	'0.314'
+59,9-59,10:	NEWLINE	'\012'
+60,0-60,18:	COMMENT	'# XXX x = 000.314\012'
+61,0-61,1:	NAME	'x'
+61,2-61,3:	OP	'='
+61,4-61,8:	NUMBER	'.314'
+61,8-61,9:	NEWLINE	'\012'
+62,0-62,1:	NAME	'x'
+62,2-62,3:	OP	'='
+62,4-62,8:	NUMBER	'3e14'
+62,8-62,9:	NEWLINE	'\012'
+63,0-63,1:	NAME	'x'
+63,2-63,3:	OP	'='
+63,4-63,8:	NUMBER	'3E14'
+63,8-63,9:	NEWLINE	'\012'
+64,0-64,1:	NAME	'x'
+64,2-64,3:	OP	'='
+64,4-64,9:	NUMBER	'3e-14'
+64,9-64,10:	NEWLINE	'\012'
+65,0-65,1:	NAME	'x'
+65,2-65,3:	OP	'='
+65,4-65,9:	NUMBER	'3e+14'
+65,9-65,10:	NEWLINE	'\012'
+66,0-66,1:	NAME	'x'
+66,2-66,3:	OP	'='
+66,4-66,9:	NUMBER	'3.e14'
+66,9-66,10:	NEWLINE	'\012'
+67,0-67,1:	NAME	'x'
+67,2-67,3:	OP	'='
+67,4-67,9:	NUMBER	'.3e14'
+67,9-67,10:	NEWLINE	'\012'
+68,0-68,1:	NAME	'x'
+68,2-68,3:	OP	'='
+68,4-68,9:	NUMBER	'3.1e4'
+68,9-68,10:	NEWLINE	'\012'
+69,0-69,1:	NEWLINE	'\012'
+70,0-70,18:	COMMENT	'# String literals\012'
+71,0-71,1:	NAME	'x'
+71,2-71,3:	OP	'='
+71,4-71,6:	STRING	"''"
+71,6-71,7:	OP	';'
+71,8-71,9:	NAME	'y'
+71,10-71,11:	OP	'='
+71,12-71,14:	STRING	'""'
+71,14-71,15:	OP	';'
+71,15-71,16:	NEWLINE	'\012'
+72,0-72,1:	NAME	'x'
+72,2-72,3:	OP	'='
+72,4-72,8:	STRING	"'\\''"
+72,8-72,9:	OP	';'
+72,10-72,11:	NAME	'y'
+72,12-72,13:	OP	'='
+72,14-72,17:	STRING	'"\'"'
+72,17-72,18:	OP	';'
+72,18-72,19:	NEWLINE	'\012'
+73,0-73,1:	NAME	'x'
+73,2-73,3:	OP	'='
+73,4-73,7:	STRING	'\'"\''
+73,7-73,8:	OP	';'
+73,9-73,10:	NAME	'y'
+73,11-73,12:	OP	'='
+73,13-73,17:	STRING	'"\\""'
+73,17-73,18:	OP	';'
+73,18-73,19:	NEWLINE	'\012'
+74,0-74,1:	NAME	'x'
+74,2-74,3:	OP	'='
+74,4-74,32:	STRING	'"doesn\'t \\"shrink\\" does it"'
+74,32-74,33:	NEWLINE	'\012'
+75,0-75,1:	NAME	'y'
+75,2-75,3:	OP	'='
+75,4-75,31:	STRING	'\'doesn\\\'t "shrink" does it\''
+75,31-75,32:	NEWLINE	'\012'
+76,0-76,1:	NAME	'x'
+76,2-76,3:	OP	'='
+76,4-76,32:	STRING	'"does \\"shrink\\" doesn\'t it"'
+76,32-76,33:	NEWLINE	'\012'
+77,0-77,1:	NAME	'y'
+77,2-77,3:	OP	'='
+77,4-77,31:	STRING	'\'does "shrink" doesn\\\'t it\''
+77,31-77,32:	NEWLINE	'\012'
+78,0-78,1:	NAME	'x'
+78,2-78,3:	OP	'='
+78,4-83,3:	STRING	'"""\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012"""'
+83,3-83,4:	NEWLINE	'\012'
+84,0-84,1:	NAME	'y'
+84,2-84,3:	OP	'='
+84,4-84,63:	STRING	'\'\\nThe "quick"\\nbrown fox\\njumps over\\nthe \\\'lazy\\\' dog.\\n\''
+84,63-84,64:	NEWLINE	'\012'
+85,0-85,1:	NAME	'y'
+85,2-85,3:	OP	'='
+85,4-90,3:	STRING	'\'\'\'\012The "quick"\012brown fox\012jumps over\012the \'lazy\' dog.\012\'\'\''
+90,3-90,4:	OP	';'
+90,4-90,5:	NEWLINE	'\012'
+91,0-91,1:	NAME	'y'
+91,2-91,3:	OP	'='
+91,4-96,1:	STRING	'"\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \'lazy\' dog.\\n\\\012"'
+96,1-96,2:	OP	';'
+96,2-96,3:	NEWLINE	'\012'
+97,0-97,1:	NAME	'y'
+97,2-97,3:	OP	'='
+97,4-102,1:	STRING	'\'\\n\\\012The \\"quick\\"\\n\\\012brown fox\\n\\\012jumps over\\n\\\012the \\\'lazy\\\' dog.\\n\\\012\''
+102,1-102,2:	OP	';'
+102,2-102,3:	NEWLINE	'\012'
+103,0-103,1:	NAME	'x'
+103,2-103,3:	OP	'='
+103,4-103,9:	STRING	"r'\\\\'"
+103,10-103,11:	OP	'+'
+103,12-103,17:	STRING	"R'\\\\'"
+103,17-103,18:	NEWLINE	'\012'
+104,0-104,1:	NAME	'x'
+104,2-104,3:	OP	'='
+104,4-104,9:	STRING	"r'\\''"
+104,10-104,11:	OP	'+'
+104,12-104,14:	STRING	"''"
+104,14-104,15:	NEWLINE	'\012'
+105,0-105,1:	NAME	'y'
+105,2-105,3:	OP	'='
+105,4-107,6:	STRING	"r'''\012foo bar \\\\\012baz'''"
+107,7-107,8:	OP	'+'
+107,9-108,6:	STRING	"R'''\012foo'''"
+108,6-108,7:	NEWLINE	'\012'
+109,0-109,1:	NAME	'y'
+109,2-109,3:	OP	'='
+109,4-111,3:	STRING	'r"""foo\012bar \\\\ baz\012"""'
+111,4-111,5:	OP	'+'
+111,6-112,3:	STRING	"R'''spam\012'''"
+112,3-112,4:	NEWLINE	'\012'
+113,0-113,1:	NEWLINE	'\012'
+114,0-114,14:	COMMENT	'# Indentation\012'
+115,0-115,2:	NAME	'if'
+115,3-115,4:	NUMBER	'1'
+115,4-115,5:	OP	':'
+115,5-115,6:	NEWLINE	'\012'
+116,0-116,4:	INDENT	'    '
+116,4-116,5:	NAME	'x'
+116,6-116,7:	OP	'='
+116,8-116,9:	NUMBER	'2'
+116,9-116,10:	NEWLINE	'\012'
+117,0-117,0:	DEDENT	''
+117,0-117,2:	NAME	'if'
+117,3-117,4:	NUMBER	'1'
+117,4-117,5:	OP	':'
+117,5-117,6:	NEWLINE	'\012'
+118,0-118,8:	INDENT	'        '
+118,8-118,9:	NAME	'x'
+118,10-118,11:	OP	'='
+118,12-118,13:	NUMBER	'2'
+118,13-118,14:	NEWLINE	'\012'
+119,0-119,0:	DEDENT	''
+119,0-119,2:	NAME	'if'
+119,3-119,4:	NUMBER	'1'
+119,4-119,5:	OP	':'
+119,5-119,6:	NEWLINE	'\012'
+120,0-120,4:	INDENT	'    '
+120,4-120,9:	NAME	'while'
+120,10-120,11:	NUMBER	'0'
+120,11-120,12:	OP	':'
+120,12-120,13:	NEWLINE	'\012'
+121,0-121,5:	INDENT	'     '
+121,5-121,7:	NAME	'if'
+121,8-121,9:	NUMBER	'0'
+121,9-121,10:	OP	':'
+121,10-121,11:	NEWLINE	'\012'
+122,0-122,11:	INDENT	'           '
+122,11-122,12:	NAME	'x'
+122,13-122,14:	OP	'='
+122,15-122,16:	NUMBER	'2'
+122,16-122,17:	NEWLINE	'\012'
+123,5-123,5:	DEDENT	''
+123,5-123,6:	NAME	'x'
+123,7-123,8:	OP	'='
+123,9-123,10:	NUMBER	'2'
+123,10-123,11:	NEWLINE	'\012'
+124,0-124,0:	DEDENT	''
+124,0-124,0:	DEDENT	''
+124,0-124,2:	NAME	'if'
+124,3-124,4:	NUMBER	'0'
+124,4-124,5:	OP	':'
+124,5-124,6:	NEWLINE	'\012'
+125,0-125,2:	INDENT	'  '
+125,2-125,4:	NAME	'if'
+125,5-125,6:	NUMBER	'2'
+125,6-125,7:	OP	':'
+125,7-125,8:	NEWLINE	'\012'
+126,0-126,3:	INDENT	'   '
+126,3-126,8:	NAME	'while'
+126,9-126,10:	NUMBER	'0'
+126,10-126,11:	OP	':'
+126,11-126,12:	NEWLINE	'\012'
+127,0-127,8:	INDENT	'        '
+127,8-127,10:	NAME	'if'
+127,11-127,12:	NUMBER	'1'
+127,12-127,13:	OP	':'
+127,13-127,14:	NEWLINE	'\012'
+128,0-128,10:	INDENT	'          '
+128,10-128,11:	NAME	'x'
+128,12-128,13:	OP	'='
+128,14-128,15:	NUMBER	'2'
+128,15-128,16:	NEWLINE	'\012'
+129,0-129,1:	NEWLINE	'\012'
+130,0-130,12:	COMMENT	'# Operators\012'
+131,0-131,1:	NEWLINE	'\012'
+132,0-132,0:	DEDENT	''
+132,0-132,0:	DEDENT	''
+132,0-132,0:	DEDENT	''
+132,0-132,0:	DEDENT	''
+132,0-132,3:	NAME	'def'
+132,4-132,7:	NAME	'd22'
+132,7-132,8:	OP	'('
+132,8-132,9:	NAME	'a'
+132,9-132,10:	OP	','
+132,11-132,12:	NAME	'b'
+132,12-132,13:	OP	','
+132,14-132,15:	NAME	'c'
+132,15-132,16:	OP	'='
+132,16-132,17:	NUMBER	'1'
+132,17-132,18:	OP	','
+132,19-132,20:	NAME	'd'
+132,20-132,21:	OP	'='
+132,21-132,22:	NUMBER	'2'
+132,22-132,23:	OP	')'
+132,23-132,24:	OP	':'
+132,25-132,29:	NAME	'pass'
+132,29-132,30:	NEWLINE	'\012'
+133,0-133,3:	NAME	'def'
+133,4-133,8:	NAME	'd01v'
+133,8-133,9:	OP	'('
+133,9-133,10:	NAME	'a'
+133,10-133,11:	OP	'='
+133,11-133,12:	NUMBER	'1'
+133,12-133,13:	OP	','
+133,14-133,15:	OP	'*'
+133,15-133,19:	NAME	'rest'
+133,19-133,20:	OP	','
+133,21-133,23:	OP	'**'
+133,23-133,27:	NAME	'rest'
+133,27-133,28:	OP	')'
+133,28-133,29:	OP	':'
+133,30-133,34:	NAME	'pass'
+133,34-133,35:	NEWLINE	'\012'
+134,0-134,1:	NEWLINE	'\012'
+135,0-135,1:	OP	'('
+135,1-135,2:	NAME	'x'
+135,2-135,3:	OP	','
+135,4-135,5:	NAME	'y'
+135,5-135,6:	OP	')'
+135,7-135,9:	OP	'<>'
+135,10-135,11:	OP	'('
+135,11-135,12:	OP	'{'
+135,12-135,15:	STRING	"'a'"
+135,15-135,16:	OP	':'
+135,16-135,17:	NUMBER	'1'
+135,17-135,18:	OP	'}'
+135,18-135,19:	OP	','
+135,20-135,21:	OP	'{'
+135,21-135,24:	STRING	"'b'"
+135,24-135,25:	OP	':'
+135,25-135,26:	NUMBER	'2'
+135,26-135,27:	OP	'}'
+135,27-135,28:	OP	')'
+135,28-135,29:	NEWLINE	'\012'
+136,0-136,1:	NEWLINE	'\012'
+137,0-137,13:	COMMENT	'# comparison\012'
+138,0-138,2:	NAME	'if'
+138,3-138,4:	NUMBER	'1'
+138,5-138,6:	OP	'<'
+138,7-138,8:	NUMBER	'1'
+138,9-138,10:	OP	'>'
+138,11-138,12:	NUMBER	'1'
+138,13-138,15:	OP	'=='
+138,16-138,17:	NUMBER	'1'
+138,18-138,20:	OP	'>='
+138,21-138,22:	NUMBER	'1'
+138,23-138,25:	OP	'<='
+138,26-138,27:	NUMBER	'1'
+138,28-138,30:	OP	'<>'
+138,31-138,32:	NUMBER	'1'
+138,33-138,35:	OP	'!='
+138,36-138,37:	NUMBER	'1'
+138,38-138,40:	NAME	'in'
+138,41-138,42:	NUMBER	'1'
+138,43-138,46:	NAME	'not'
+138,47-138,49:	NAME	'in'
+138,50-138,51:	NUMBER	'1'
+138,52-138,54:	NAME	'is'
+138,55-138,56:	NUMBER	'1'
+138,57-138,59:	NAME	'is'
+138,60-138,63:	NAME	'not'
+138,64-138,65:	NUMBER	'1'
+138,65-138,66:	OP	':'
+138,67-138,71:	NAME	'pass'
+138,71-138,72:	NEWLINE	'\012'
+139,0-139,1:	NEWLINE	'\012'
+140,0-140,9:	COMMENT	'# binary\012'
+141,0-141,1:	NAME	'x'
+141,2-141,3:	OP	'='
+141,4-141,5:	NUMBER	'1'
+141,6-141,7:	OP	'&'
+141,8-141,9:	NUMBER	'1'
+141,9-141,10:	NEWLINE	'\012'
+142,0-142,1:	NAME	'x'
+142,2-142,3:	OP	'='
+142,4-142,5:	NUMBER	'1'
+142,6-142,7:	OP	'^'
+142,8-142,9:	NUMBER	'1'
+142,9-142,10:	NEWLINE	'\012'
+143,0-143,1:	NAME	'x'
+143,2-143,3:	OP	'='
+143,4-143,5:	NUMBER	'1'
+143,6-143,7:	OP	'|'
+143,8-143,9:	NUMBER	'1'
+143,9-143,10:	NEWLINE	'\012'
+144,0-144,1:	NEWLINE	'\012'
+145,0-145,8:	COMMENT	'# shift\012'
+146,0-146,1:	NAME	'x'
+146,2-146,3:	OP	'='
+146,4-146,5:	NUMBER	'1'
+146,6-146,8:	OP	'<<'
+146,9-146,10:	NUMBER	'1'
+146,11-146,13:	OP	'>>'
+146,14-146,15:	NUMBER	'1'
+146,15-146,16:	NEWLINE	'\012'
+147,0-147,1:	NEWLINE	'\012'
+148,0-148,11:	COMMENT	'# additive\012'
+149,0-149,1:	NAME	'x'
+149,2-149,3:	OP	'='
+149,4-149,5:	NUMBER	'1'
+149,6-149,7:	OP	'-'
+149,8-149,9:	NUMBER	'1'
+149,10-149,11:	OP	'+'
+149,12-149,13:	NUMBER	'1'
+149,14-149,15:	OP	'-'
+149,16-149,17:	NUMBER	'1'
+149,18-149,19:	OP	'+'
+149,20-149,21:	NUMBER	'1'
+149,21-149,22:	NEWLINE	'\012'
+150,0-150,1:	NEWLINE	'\012'
+151,0-151,17:	COMMENT	'# multiplicative\012'
+152,0-152,1:	NAME	'x'
+152,2-152,3:	OP	'='
+152,4-152,5:	NUMBER	'1'
+152,6-152,7:	OP	'/'
+152,8-152,9:	NUMBER	'1'
+152,10-152,11:	OP	'*'
+152,12-152,13:	NUMBER	'1'
+152,14-152,15:	OP	'%'
+152,16-152,17:	NUMBER	'1'
+152,17-152,18:	NEWLINE	'\012'
+153,0-153,1:	NEWLINE	'\012'
+154,0-154,8:	COMMENT	'# unary\012'
+155,0-155,1:	NAME	'x'
+155,2-155,3:	OP	'='
+155,4-155,5:	OP	'~'
+155,5-155,6:	NUMBER	'1'
+155,7-155,8:	OP	'^'
+155,9-155,10:	NUMBER	'1'
+155,11-155,12:	OP	'&'
+155,13-155,14:	NUMBER	'1'
+155,15-155,16:	OP	'|'
+155,17-155,18:	NUMBER	'1'
+155,19-155,20:	OP	'&'
+155,21-155,22:	NUMBER	'1'
+155,23-155,24:	OP	'^'
+155,25-155,26:	OP	'-'
+155,26-155,27:	NUMBER	'1'
+155,27-155,28:	NEWLINE	'\012'
+156,0-156,1:	NAME	'x'
+156,2-156,3:	OP	'='
+156,4-156,5:	OP	'-'
+156,5-156,6:	NUMBER	'1'
+156,6-156,7:	OP	'*'
+156,7-156,8:	NUMBER	'1'
+156,8-156,9:	OP	'/'
+156,9-156,10:	NUMBER	'1'
+156,11-156,12:	OP	'+'
+156,13-156,14:	NUMBER	'1'
+156,14-156,15:	OP	'*'
+156,15-156,16:	NUMBER	'1'
+156,17-156,18:	OP	'-'
+156,19-156,20:	OP	'-'
+156,20-156,21:	OP	'-'
+156,21-156,22:	OP	'-'
+156,22-156,23:	NUMBER	'1'
+156,23-156,24:	OP	'*'
+156,24-156,25:	NUMBER	'1'
+156,25-156,26:	NEWLINE	'\012'
+157,0-157,1:	NEWLINE	'\012'
+158,0-158,11:	COMMENT	'# selector\012'
+159,0-159,6:	NAME	'import'
+159,7-159,10:	NAME	'sys'
+159,10-159,11:	OP	','
+159,12-159,16:	NAME	'time'
+159,16-159,17:	NEWLINE	'\012'
+160,0-160,1:	NAME	'x'
+160,2-160,3:	OP	'='
+160,4-160,7:	NAME	'sys'
+160,7-160,8:	OP	'.'
+160,8-160,15:	NAME	'modules'
+160,15-160,16:	OP	'['
+160,16-160,22:	STRING	"'time'"
+160,22-160,23:	OP	']'
+160,23-160,24:	OP	'.'
+160,24-160,28:	NAME	'time'
+160,28-160,29:	OP	'('
+160,29-160,30:	OP	')'
+160,30-160,31:	NEWLINE	'\012'
+161,0-161,1:	NEWLINE	'\012'
+162,0-162,0:	ENDMARKER	''
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
new file mode 100644
index 0000000..34a7bfb
--- /dev/null
+++ b/Lib/test/test_tokenize.py
@@ -0,0 +1,22 @@
+from test_support import verbose
+import tokenize, os, sys
+
+def findfile(file):
+	if os.path.isabs(file): return file
+	path = sys.path
+	try:
+	    path = [os.path.dirname(__file__)] + path
+	except NameError:
+	    pass
+	for dn in path:
+		fn = os.path.join(dn, file)
+		if os.path.exists(fn): return fn
+	return file
+
+if verbose:
+    print 'starting...'
+file = open(findfile('tokenize_tests.py'))
+tokenize.tokenize(file.readline)
+if verbose:
+    print 'finished'
+
diff --git a/Lib/test/tokenize_tests.py b/Lib/test/tokenize_tests.py
new file mode 100644
index 0000000..4883668
--- /dev/null
+++ b/Lib/test/tokenize_tests.py
@@ -0,0 +1,161 @@
+# Tests for the 'tokenize' module.
+# Large bits stolen from test_grammar.py. 
+
+# Comments
+"#"
+#'
+#"
+#\
+       #
+    # abc
+'''#
+#'''
+
+x = 1  #
+
+# Balancing continuation
+
+a = (3, 4,
+  5, 6)
+y = [3, 4,
+  5]
+z = {'a':5,
+  'b':6}
+x = (len(`y`) + 5*x - a[
+   3 ]
+   - x + len({
+   }
+    )
+  )
+
+# Backslash means line continuation:
+x = 1 \
++ 1
+
+# Backslash does not means continuation in comments :\
+x = 0
+
+# Ordinary integers
+0xff <> 255
+0377 <> 255
+2147483647   != 017777777777
+-2147483647-1 != 020000000000
+037777777777 != -1
+0xffffffff != -1
+
+# Long integers
+x = 0L
+x = 0l
+x = 0xffffffffffffffffL
+x = 0xffffffffffffffffl
+x = 077777777777777777L
+x = 077777777777777777l
+x = 123456789012345678901234567890L
+x = 123456789012345678901234567890l
+
+# Floating-point numbers
+x = 3.14
+x = 314.
+x = 0.314
+# XXX x = 000.314
+x = .314
+x = 3e14
+x = 3E14
+x = 3e-14
+x = 3e+14
+x = 3.e14
+x = .3e14
+x = 3.1e4
+
+# String literals
+x = ''; y = "";
+x = '\''; y = "'";
+x = '"'; y = "\"";
+x = "doesn't \"shrink\" does it"
+y = 'doesn\'t "shrink" does it'
+x = "does \"shrink\" doesn't it"
+y = 'does "shrink" doesn\'t it'
+x = """
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+"""
+y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n'
+y = '''
+The "quick"
+brown fox
+jumps over
+the 'lazy' dog.
+''';
+y = "\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the 'lazy' dog.\n\
+";
+y = '\n\
+The \"quick\"\n\
+brown fox\n\
+jumps over\n\
+the \'lazy\' dog.\n\
+';
+x = r'\\' + R'\\'
+x = r'\'' + ''
+y = r'''
+foo bar \\
+baz''' + R'''
+foo'''
+y = r"""foo
+bar \\ baz
+""" + R'''spam
+'''
+
+# Indentation
+if 1:
+    x = 2
+if 1:
+        x = 2
+if 1:
+    while 0:
+     if 0:
+           x = 2
+     x = 2
+if 0:
+  if 2:
+   while 0:
+        if 1:
+          x = 2
+
+# Operators
+
+def d22(a, b, c=1, d=2): pass
+def d01v(a=1, *rest, **rest): pass
+
+(x, y) <> ({'a':1}, {'b':2})
+
+# comparison
+if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass
+
+# binary
+x = 1 & 1
+x = 1 ^ 1
+x = 1 | 1
+
+# shift
+x = 1 << 1 >> 1
+
+# additive
+x = 1 - 1 + 1 - 1 + 1
+
+# multiplicative
+x = 1 / 1 * 1 % 1
+
+# unary
+x = ~1 ^ 1 & 1 | 1 & 1 ^ -1
+x = -1*1/1 + 1*1 - ---1*1
+
+# selector
+import sys, time
+x = sys.modules['time'].time()
+