summaryrefslogtreecommitdiffstats
path: root/Lib
diff options
context:
space:
mode:
Diffstat (limited to 'Lib')
-rw-r--r--Lib/tokenize.py42
1 files changed, 3 insertions, 39 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index c97ed8a..cd3c93d 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -28,6 +28,7 @@ import re
import sys
from token import *
from codecs import lookup, BOM_UTF8
+import collections
cookie_re = re.compile("coding[:=]\s*([-\w.]+)")
import token
@@ -44,49 +45,12 @@ ENCODING = N_TOKENS + 2
tok_name[ENCODING] = 'ENCODING'
N_TOKENS += 3
-class TokenInfo(tuple):
- 'TokenInfo(type, string, start, end, line)'
-
- __slots__ = ()
-
- _fields = ('type', 'string', 'start', 'end', 'line')
-
- def __new__(cls, type, string, start, end, line):
- return tuple.__new__(cls, (type, string, start, end, line))
-
- @classmethod
- def _make(cls, iterable, new=tuple.__new__, len=len):
- 'Make a new TokenInfo object from a sequence or iterable'
- result = new(cls, iterable)
- if len(result) != 5:
- raise TypeError('Expected 5 arguments, got %d' % len(result))
- return result
-
+class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')):
def __repr__(self):
- typ = self[0]
+ typ = self.type
return 'TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % \
((('%d (%s)' % (typ, tok_name[typ])),) + self[1:])
- def _asdict(self):
- 'Return a new dict which maps field names to their values'
- return dict(zip(self._fields, self))
-
- def _replace(self, **kwds):
- 'Return a new TokenInfo object replacing specified fields with new values'
- result = self._make(map(kwds.pop, ('type', 'string', 'start', 'end', 'line'), self))
- if kwds:
- raise ValueError('Got unexpected field names: %r' % kwds.keys())
- return result
-
- def __getnewargs__(self):
- return tuple(self)
-
- type = property(lambda t: t[0])
- string = property(lambda t: t[1])
- start = property(lambda t: t[2])
- end = property(lambda t: t[3])
- line = property(lambda t: t[4])
-
def group(*choices): return '(' + '|'.join(choices) + ')'
def any(*choices): return group(*choices) + '*'
def maybe(*choices): return group(*choices) + '?'