1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
|
from test.test_support import verbose, findfile, is_resource_enabled, TestFailed
import os, glob, random
from tokenize import (tokenize, generate_tokens, untokenize,
NUMBER, NAME, OP, STRING)
if verbose:
print 'starting...'
f = file(findfile('tokenize_tests' + os.extsep + 'txt'))
tokenize(f.readline)
f.close()
###### Test roundtrip for untokenize ##########################
def test_roundtrip(f):
## print 'Testing:', f
f = file(f)
try:
fulltok = list(generate_tokens(f.readline))
finally:
f.close()
t1 = [tok[:2] for tok in fulltok]
newtext = untokenize(t1)
readline = iter(newtext.splitlines(1)).next
t2 = [tok[:2] for tok in generate_tokens(readline)]
assert t1 == t2
f = findfile('tokenize_tests' + os.extsep + 'txt')
test_roundtrip(f)
testdir = os.path.dirname(f) or os.curdir
testfiles = glob.glob(testdir + os.sep + 'test*.py')
if not is_resource_enabled('compiler'):
testfiles = random.sample(testfiles, 10)
for f in testfiles:
test_roundtrip(f)
###### Test detecton of IndentationError ######################
from cStringIO import StringIO
sampleBadText = """
def foo():
bar
baz
"""
try:
for tok in generate_tokens(StringIO(sampleBadText).readline):
pass
except IndentationError:
pass
else:
raise TestFailed("Did not detect IndentationError:")
###### Test example in the docs ###############################
from decimal import Decimal
from cStringIO import StringIO
def decistmt(s):
"""Substitute Decimals for floats in a string of statements.
>>> from decimal import Decimal
>>> s = 'print +21.3e-5*-.1234/81.7'
>>> decistmt(s)
"print +Decimal ('21.3e-5')*-Decimal ('.1234')/Decimal ('81.7')"
>>> exec(s)
-3.21716034272e-007
>>> exec(decistmt(s))
-3.217160342717258261933904529E-7
"""
result = []
g = generate_tokens(StringIO(s).readline) # tokenize the string
for toknum, tokval, _, _, _ in g:
if toknum == NUMBER and '.' in tokval: # replace NUMBER tokens
result.extend([
(NAME, 'Decimal'),
(OP, '('),
(STRING, repr(tokval)),
(OP, ')')
])
else:
result.append((toknum, tokval))
return untokenize(result)
import doctest
doctest.testmod()
if verbose:
print 'finished'
|
