diff options
author | Raymond Hettinger <python@rcn.com> | 2010-09-09 07:15:18 (GMT) |
---|---|---|
committer | Raymond Hettinger <python@rcn.com> | 2010-09-09 07:15:18 (GMT) |
commit | 3fb79c747b0cd0884f2a6ede9e36673bec8745f2 (patch) | |
tree | 8fd071418a2f6aafb8a212c37fed71fd9cafb18d | |
parent | 3cfe2e3677be1dbc786fbca17092133d2ae4bd4a (diff) | |
download | cpython-3fb79c747b0cd0884f2a6ede9e36673bec8745f2.zip cpython-3fb79c747b0cd0884f2a6ede9e36673bec8745f2.tar.gz cpython-3fb79c747b0cd0884f2a6ede9e36673bec8745f2.tar.bz2 |
Experiment: Let collections.namedtuple() do the work. This should work now that _collections is pre-built. The buildbots will tell us shortly.
-rw-r--r-- | Lib/tokenize.py | 42 |
1 files changed, 3 insertions, 39 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index c97ed8a..cd3c93d 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -28,6 +28,7 @@ import re import sys from token import * from codecs import lookup, BOM_UTF8 +import collections cookie_re = re.compile("coding[:=]\s*([-\w.]+)") import token @@ -44,49 +45,12 @@ ENCODING = N_TOKENS + 2 tok_name[ENCODING] = 'ENCODING' N_TOKENS += 3 -class TokenInfo(tuple): - 'TokenInfo(type, string, start, end, line)' - - __slots__ = () - - _fields = ('type', 'string', 'start', 'end', 'line') - - def __new__(cls, type, string, start, end, line): - return tuple.__new__(cls, (type, string, start, end, line)) - - @classmethod - def _make(cls, iterable, new=tuple.__new__, len=len): - 'Make a new TokenInfo object from a sequence or iterable' - result = new(cls, iterable) - if len(result) != 5: - raise TypeError('Expected 5 arguments, got %d' % len(result)) - return result - +class TokenInfo(collections.namedtuple('TokenInfo', 'type string start end line')): def __repr__(self): - typ = self[0] + typ = self.type return 'TokenInfo(type=%s, string=%r, start=%r, end=%r, line=%r)' % \ ((('%d (%s)' % (typ, tok_name[typ])),) + self[1:]) - def _asdict(self): - 'Return a new dict which maps field names to their values' - return dict(zip(self._fields, self)) - - def _replace(self, **kwds): - 'Return a new TokenInfo object replacing specified fields with new values' - result = self._make(map(kwds.pop, ('type', 'string', 'start', 'end', 'line'), self)) - if kwds: - raise ValueError('Got unexpected field names: %r' % kwds.keys()) - return result - - def __getnewargs__(self): - return tuple(self) - - type = property(lambda t: t[0]) - string = property(lambda t: t[1]) - start = property(lambda t: t[2]) - end = property(lambda t: t[3]) - line = property(lambda t: t[4]) - def group(*choices): return '(' + '|'.join(choices) + ')' def any(*choices): return group(*choices) + '*' def maybe(*choices): return group(*choices) + '?' |