From 0b23348aa944dcb97c84b34d399c857f039e5773 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Wed, 26 Nov 1997 15:44:34 +0000 Subject: The usual --- Lib/dos-8x3/fileinpu.py | 254 ++++++++++++++++++++++++++++++++++++++++++++++++ Lib/dos-8x3/formatte.py | 2 - Lib/dos-8x3/posixpat.py | 12 +-- Lib/dos-8x3/py_compi.py | 10 +- Lib/dos-8x3/queue.py | 246 ++++++++++++++++++++++++---------------------- Lib/dos-8x3/reconver.py | 186 +++++++++++++++++++++++++++++++++++ Lib/dos-8x3/regex_sy.py | 6 ++ Lib/dos-8x3/rlcomple.py | 6 +- Lib/dos-8x3/test_arr.py | 2 +- Lib/dos-8x3/test_sel.py | 2 +- Lib/dos-8x3/test_sig.py | 5 +- Lib/dos-8x3/test_tok.py | 22 +++++ Lib/dos-8x3/test_typ.py | 9 ++ Lib/dos-8x3/tokenize.py | 161 ++++++++++++++++++++++++++++++ Lib/dos-8x3/userdict.py | 5 + 15 files changed, 793 insertions(+), 135 deletions(-) create mode 100644 Lib/dos-8x3/fileinpu.py create mode 100644 Lib/dos-8x3/reconver.py create mode 100644 Lib/dos-8x3/test_tok.py create mode 100644 Lib/dos-8x3/tokenize.py diff --git a/Lib/dos-8x3/fileinpu.py b/Lib/dos-8x3/fileinpu.py new file mode 100644 index 0000000..b332202 --- /dev/null +++ b/Lib/dos-8x3/fileinpu.py @@ -0,0 +1,254 @@ +"""Helper class to quickly write a loop over all standard input files. + +Typical use is: + + import fileinput + for line in fileinput.input(): + process(line) + +This iterates over the lines of all files listed in sys.argv[1:], +defaulting to sys.stdin if the list is empty. If a filename is '-' it +is also replaced by sys.stdin. To specify an alternative list of +filenames, pass it as the argument to input(). A single file name is +also allowed. + +Functions filename(), lineno() return the filename and cumulative line +number of the line that has just been read; filelineno() returns its +line number in the current file; isfirstline() returns true iff the +line just read is the first line of its file; isstdin() returns true +iff the line was read from sys.stdin. Function nextfile() closes the +current file so that the next iteration will read the first line from +the next file (if any); lines not read from the file will not count +towards the cumulative line count; the filename is not changed until +after the first line of the next file has been read. Function close() +closes the sequence. + +Before any lines have been read, filename() returns None and both line +numbers are zero; nextfile() has no effect. After all lines have been +read, filename() and the line number functions return the values +pertaining to the last line read; nextfile() has no effect. + +All files are opened in text mode. If an I/O error occurs during +opening or reading a file, the IOError exception is raised. + +If sys.stdin is used more than once, the second and further use will +return no lines, except perhaps for interactive use, or if it has been +explicitly reset (e.g. using sys.stdin.seek(0)). + +Empty files are opened and immediately closed; the only time their +presence in the list of filenames is noticeable at all is when the +last file opened is empty. + +It is possible that the last line of a file doesn't end in a newline +character; otherwise lines are returned including the trailing +newline. + +Class FileInput is the implementation; its methods filename(), +lineno(), fileline(), isfirstline(), isstdin(), nextfile() and close() +correspond to the functions in the module. In addition it has a +readline() method which returns the next input line, and a +__getitem__() method which implements the sequence behavior. The +sequence must be accessed in strictly sequential order; sequence +access and readline() cannot be mixed. + +Optional in-place filtering: if the keyword argument inplace=1 is +passed to input() or to the FileInput constructor, the file is moved +to a backup file and standard output is directed to the input file. +This makes it possible to write a filter that rewrites its input file +in place. If the keyword argument backup="." is also +given, it specifies the extension for the backup file, and the backup +file remains around; by default, the extension is ".bak" and it is +deleted when the output file is closed. In-place filtering is +disabled when standard input is read. XXX The current implementation +does not work for MS-DOS 8+3 filesystems. + +XXX Possible additions: + +- optional getopt argument processing +- specify open mode ('r' or 'rb') +- specify buffer size +- fileno() +- isatty() +- read(), read(size), even readlines() + +""" + +import sys, os + +_state = None + +def input(files=(), inplace=0, backup=""): + global _state + if _state and _state._file: + raise RuntimeError, "input() already active" + _state = FileInput(files, inplace, backup) + return _state + +def close(): + global _state + state = _state + _state = None + if state: + state.close() + +def nextfile(): + if not _state: + raise RuntimeError, "no active input()" + return _state.nextfile() + +def filename(): + if not _state: + raise RuntimeError, "no active input()" + return _state.filename() + +def lineno(): + if not _state: + raise RuntimeError, "no active input()" + return _state.lineno() + +def filelineno(): + if not _state: + raise RuntimeError, "no active input()" + return _state.filelineno() + +def isfirstline(): + if not _state: + raise RuntimeError, "no active input()" + return _state.isfirstline() + +def isstdin(): + if not _state: + raise RuntimeError, "no active input()" + return _state.isstdin() + +class FileInput: + + def __init__(self, files=(), inplace=0, backup=""): + if type(files) == type(''): + files = (files,) + else: + files = tuple(files) + if not files: + files = tuple(sys.argv[1:]) + if not files: + files = ('-',) + self._files = files + self._inplace = inplace + self._backup = backup + self._savestdout = None + self._output = None + self._filename = None + self._lineno = 0 + self._filelineno = 0 + self._file = None + self._isstdin = 0 + + def __del__(self): + self.close() + + def close(self): + self.nextfile() + self._files = () + + def __getitem__(self, i): + if i != self._lineno: + raise RuntimeError, "accessing lines out of order" + line = self.readline() + if not line: + raise IndexError, "end of input reached" + return line + + def nextfile(self): + savestdout = self._savestdout + self._savestdout = 0 + if savestdout: + sys.stdout = savestdout + + output = self._output + self._output = 0 + if output: + output.close() + + file = self._file + self._file = 0 + if file and not self._isstdin: + file.close() + + backupfilename = self._backupfilename + self._backupfilename = 0 + if backupfilename and not self._backup: + try: os.unlink(backupfilename) + except: pass + + self._isstdin = 0 + + def readline(self): + if not self._file: + if not self._files: + return "" + self._filename = self._files[0] + self._files = self._files[1:] + self._filelineno = 0 + self._file = None + self._isstdin = 0 + self._backupfilename = 0 + if self._filename == '-': + self._filename = '' + self._file = sys.stdin + self._isstdin = 1 + else: + if self._inplace: + self._backupfilename = ( + self._filename + (self._backup or ".bak")) + try: os.unlink(self._backupfilename) + except os.error: pass + # The next three lines may raise IOError + os.rename(self._filename, self._backupfilename) + self._file = open(self._backupfilename, "r") + self._output = open(self._filename, "w") + self._savestdout = sys.stdout + sys.stdout = self._output + else: + # This may raise IOError + self._file = open(self._filename, "r") + line = self._file.readline() + if line: + self._lineno = self._lineno + 1 + self._filelineno = self._filelineno + 1 + return line + self.nextfile() + # Recursive call + return self.readline() + + def filename(self): + return self._filename + + def lineno(self): + return self._lineno + + def filelineno(self): + return self._filelineno + + def isfirstline(self): + return self._filelineno == 1 + + def isstdin(self): + return self._isstdin + +def _test(): + import getopt + inplace = 0 + backup = 0 + opts, args = getopt.getopt(sys.argv[1:], "ib:") + for o, a in opts: + if o == '-i': inplace = 1 + if o == '-b': backup = a + for line in input(args, inplace=inplace, backup=backup): + if line[-1:] == '\n': line = line[:-1] + if line[-1:] == '\r': line = line[:-1] + print "%d: %s[%d]%s %s" % (lineno(), filename(), filelineno(), + isfirstline() and "*" or "", line) + print "%d: %s[%d]" % (lineno(), filename(), filelineno()) + +if __name__ == '__main__': + _test() diff --git a/Lib/dos-8x3/formatte.py b/Lib/dos-8x3/formatte.py index 25dbe73..504807d 100755 --- a/Lib/dos-8x3/formatte.py +++ b/Lib/dos-8x3/formatte.py @@ -1,5 +1,3 @@ -import regex -import regsub import string import sys from types import StringType diff --git a/Lib/dos-8x3/posixpat.py b/Lib/dos-8x3/posixpat.py index 965184b..9960801 100755 --- a/Lib/dos-8x3/posixpat.py +++ b/Lib/dos-8x3/posixpat.py @@ -266,15 +266,15 @@ def expandvars(path): if '$' not in path: return path if not _varprog: - import regex - _varprog = regex.compile('$\([a-zA-Z0-9_]+\|{[^}]*}\)') + import re + _varprog = re.compile(r'\$(\w+|\{[^}]*\})') i = 0 while 1: - i = _varprog.search(path, i) - if i < 0: + m = _varprog.search(path, i) + if not m: break - name = _varprog.group(1) - j = i + len(_varprog.group(0)) + i, j = m.span(0) + name = m.group(1) if name[:1] == '{' and name[-1:] == '}': name = name[1:-1] if os.environ.has_key(name): diff --git a/Lib/dos-8x3/py_compi.py b/Lib/dos-8x3/py_compi.py index e9e90ff..1adc3a2 100755 --- a/Lib/dos-8x3/py_compi.py +++ b/Lib/dos-8x3/py_compi.py @@ -14,16 +14,22 @@ def wr_long(f, x): def compile(file, cfile = None): import os, marshal, __builtin__ f = open(file) + try: + timestamp = os.fstat(file.fileno()) + except AttributeError: + timestamp = long(os.stat(file)[8]) codestring = f.read() f.close() - timestamp = long(os.stat(file)[8]) codeobject = __builtin__.compile(codestring, file, 'exec') if not cfile: cfile = file + (__debug__ and 'c' or 'o') fc = open(cfile, 'wb') - fc.write(MAGIC) + fc.write('\0\0\0\0') wr_long(fc, timestamp) marshal.dump(codeobject, fc) + fc.flush() + fc.seek(0, 0) + fc.write(MAGIC) fc.close() if os.name == 'mac': import macfs diff --git a/Lib/dos-8x3/queue.py b/Lib/dos-8x3/queue.py index 5125fd5..1cec4e3 100755 --- a/Lib/dos-8x3/queue.py +++ b/Lib/dos-8x3/queue.py @@ -1,121 +1,135 @@ # A multi-producer, multi-consumer queue. -Empty = 'Queue.Empty' # Exception raised by get_nowait() +# define this exception to be compatible with Python 1.5's class +# exceptions, but also when -X option is used. +try: + class Empty(Exception): + pass +except TypeError: + # string based exceptions + Empty = 'Queue.Empty' # Exception raised by get_nowait() class Queue: - - # Initialize a queue object with a given maximum size - # (If maxsize is <= 0, the maximum size is infinite) - def __init__(self, maxsize): - import thread - self._init(maxsize) - self.mutex = thread.allocate_lock() - self.esema = thread.allocate_lock() - self.esema.acquire_lock() - self.fsema = thread.allocate_lock() - - # Get an approximation of the queue size (not reliable!) - def qsize(self): - self.mutex.acquire_lock() - n = self._qsize() - self.mutex.release_lock() - return n - - # Check if the queue is empty (not reliable!) - def empty(self): - self.mutex.acquire_lock() - n = self._empty() - self.mutex.release_lock() - return n - - # Check if the queue is full (not reliable!) - def full(self): - self.mutex.acquire_lock() - n = self._full() - self.mutex.release_lock() - return n - - # Put a new item into the queue - def put(self, item): - self.fsema.acquire_lock() - self.mutex.acquire_lock() - was_empty = self._empty() - self._put(item) - if was_empty: - self.esema.release_lock() - if not self._full(): - self.fsema.release_lock() - self.mutex.release_lock() - - # Get an item from the queue, - # blocking if necessary until one is available - def get(self): - self.esema.acquire_lock() - self.mutex.acquire_lock() - was_full = self._full() - item = self._get() - if was_full: - self.fsema.release_lock() - if not self._empty(): - self.esema.release_lock() - self.mutex.release_lock() - return item - - # Get an item from the queue if one is immediately available, - # raise Empty if the queue is empty or temporarily unavailable - def get_nowait(self): - locked = self.esema.acquire_lock(0) - self.mutex.acquire_lock() - if self._empty(): - # The queue is empyt -- we can't have esema - self.mutex.release_lock() - raise Empty - if not locked: - locked = self.esema.acquire_lock(0) - if not locked: - # Somebody else has esema - # but we have mutex -- - # go out of their way - self.mutex.release_lock() - raise Empty - was_full = self._full() - item = self._get() - if was_full: - self.fsema.release_lock() - if not self._empty(): - self.esema.release_lock() + def __init__(self, maxsize): + """Initialize a queue object with a given maximum size. + + If maxsize is <= 0, the queue size is infinite. + """ + import thread + self._init(maxsize) + self.mutex = thread.allocate_lock() + self.esema = thread.allocate_lock() + self.esema.acquire_lock() + self.fsema = thread.allocate_lock() + + def qsize(self): + """Returns the approximate size of the queue (not reliable!).""" + self.mutex.acquire_lock() + n = self._qsize() + self.mutex.release_lock() + return n + + def empty(self): + """Returns 1 if the queue is empty, 0 otherwise (not reliable!).""" + self.mutex.acquire_lock() + n = self._empty() + self.mutex.release_lock() + return n + + def full(self): + """Returns 1 if the queue is full, 0 otherwise (not reliable!).""" + self.mutex.acquire_lock() + n = self._full() + self.mutex.release_lock() + return n + + def put(self, item): + """Put an item into the queue.""" + self.fsema.acquire_lock() + self.mutex.acquire_lock() + was_empty = self._empty() + self._put(item) + if was_empty: + self.esema.release_lock() + if not self._full(): + self.fsema.release_lock() + self.mutex.release_lock() + + def get(self): + """Gets and returns an item from the queue. + This method blocks if necessary until an item is available. + """ + self.esema.acquire_lock() + self.mutex.acquire_lock() + was_full = self._full() + item = self._get() + if was_full: + self.fsema.release_lock() + if not self._empty(): + self.esema.release_lock() + self.mutex.release_lock() + return item + + # Get an item from the queue if one is immediately available, + # raise Empty if the queue is empty or temporarily unavailable + def get_nowait(self): + """Gets and returns an item from the queue. + Only gets an item if one is immediately available, Otherwise + this raises the Empty exception if the queue is empty or + temporarily unavailable. + """ + locked = self.esema.acquire_lock(0) + self.mutex.acquire_lock() + if self._empty(): + # The queue is empty -- we can't have esema + self.mutex.release_lock() + raise Empty + if not locked: + locked = self.esema.acquire_lock(0) + if not locked: + # Somebody else has esema + # but we have mutex -- + # go out of their way self.mutex.release_lock() - return item - - # XXX Need to define put_nowait() as well. - - - # Override these methods to implement other queue organizations - # (e.g. stack or priority queue). - # These will only be called with appropriate locks held - - # Initialize the queue representation - def _init(self, maxsize): - self.maxsize = maxsize - self.queue = [] - - def _qsize(self): - return len(self.queue) - - # Check wheter the queue is empty - def _empty(self): - return not self.queue - - # Check whether the queue is full - def _full(self): - return self.maxsize > 0 and len(self.queue) == self.maxsize - - # Put a new item in the queue - def _put(self, item): - self.queue.append(item) - - # Get an item from the queue - def _get(self): - item = self.queue[0] - del self.queue[0] - return item + raise Empty + was_full = self._full() + item = self._get() + if was_full: + self.fsema.release_lock() + if not self._empty(): + self.esema.release_lock() + self.mutex.release_lock() + return item + + # XXX Need to define put_nowait() as well. + + + # Override these methods to implement other queue organizations + # (e.g. stack or priority queue). + # These will only be called with appropriate locks held + + # Initialize the queue representation + def _init(self, maxsize): + self.maxsize = maxsize + self.queue = [] + + def _qsize(self): + return len(self.queue) + + # Check wheter the queue is empty + def _empty(self): + return not self.queue + + # Check whether the queue is full + def _full(self): + return self.maxsize > 0 and len(self.queue) == self.maxsize + + # Put a new item in the queue + def _put(self, item): + self.queue.append(item) + + # Get an item from the queue + def _get(self): + item = self.queue[0] + del self.queue[0] + return item diff --git a/Lib/dos-8x3/reconver.py b/Lib/dos-8x3/reconver.py new file mode 100644 index 0000000..f0b61fc --- /dev/null +++ b/Lib/dos-8x3/reconver.py @@ -0,0 +1,186 @@ +#! /usr/bin/env python1.5 + +"""Convert old ("regex") regular expressions to new syntax ("re"). + +When imported as a module, there are two functions, with their own +strings: + + convert(s, syntax=None) -- convert a regex regular expression to re syntax + + quote(s) -- return a quoted string literal + +When used as a script, read a Python string literal (or any other +expression evaluating to a string) from stdin, and write the +translated expression to stdout as a string literal. Unless stdout is +a tty, no trailing \n is written to stdout. This is done so that it +can be used with Emacs C-U M-| (shell-command-on-region with argument +which filters the region through the shell command). + +No attempt has been made at coding for performance. + +Translation table... + + \( ( (unless RE_NO_BK_PARENS set) + \) ) (unless RE_NO_BK_PARENS set) + \| | (unless RE_NO_BK_VBAR set) + \< \b (not quite the same, but alla...) + \> \b (not quite the same, but alla...) + \` \A + \' \Z + +Not translated... + + . + ^ + $ + * + + (unless RE_BK_PLUS_QM set, then to \+) + ? (unless RE_BK_PLUS_QM set, then to \?) + \ + \b + \B + \w + \W + \1 ... \9 + +Special cases... + + Non-printable characters are always replaced by their 3-digit + escape code (except \t, \n, \r, which use mnemonic escapes) + + Newline is turned into | when RE_NEWLINE_OR is set + +XXX To be done... + + [...] (different treatment of backslashed items?) + [^...] (different treatment of backslashed items?) + ^ $ * + ? (in some error contexts these are probably treated differently) + \vDD \DD (in the regex docs but only works when RE_ANSI_HEX set) + +""" + + +import regex +from regex_syntax import * # RE_* + +# Default translation table +mastertable = { + r'\<': r'\b', + r'\>': r'\b', + r'\`': r'\A', + r'\'': r'\Z', + r'\(': '(', + r'\)': ')', + r'\|': '|', + '(': r'\(', + ')': r'\)', + '|': r'\|', + '\t': r'\t', + '\n': r'\n', + '\r': r'\r', +} + + +def convert(s, syntax=None): + """Convert a regex regular expression to re syntax. + + The first argument is the regular expression, as a string object, + just like it would be passed to regex.compile(). (I.e., pass the + actual string object -- string quotes must already have been + removed and the standard escape processing has already been done, + e.g. by eval().) + + The optional second argument is the regex syntax variant to be + used. This is an integer mask as passed to regex.set_syntax(); + the flag bits are defined in regex_syntax. When not specified, or + when None is given, the current regex syntax mask (as retrieved by + regex.get_syntax()) is used -- which is 0 by default. + + The return value is a regular expression, as a string object that + could be passed to re.compile(). (I.e., no string quotes have + been added -- use quote() below, or repr().) + + The conversion is not always guaranteed to be correct. More + syntactical analysis should be performed to detect borderline + cases and decide what to do with them. For example, 'x*?' is not + translated correctly. + + """ + table = mastertable.copy() + if syntax is None: + syntax = regex.get_syntax() + if syntax & RE_NO_BK_PARENS: + del table[r'\('], table[r'\)'] + del table['('], table[')'] + if syntax & RE_NO_BK_VBAR: + del table[r'\|'] + del table['|'] + if syntax & RE_BK_PLUS_QM: + table['+'] = r'\+' + table['?'] = r'\?' + table[r'\+'] = '+' + table[r'\?'] = '?' + if syntax & RE_NEWLINE_OR: + table['\n'] = '|' + res = "" + + i = 0 + end = len(s) + while i < end: + c = s[i] + i = i+1 + if c == '\\': + c = s[i] + i = i+1 + key = '\\' + c + key = table.get(key, key) + res = res + key + else: + c = table.get(c, c) + res = res + c + return res + + +def quote(s, quote=None): + """Convert a string object to a quoted string literal. + + This is similar to repr() but will return a "raw" string (r'...' + or r"...") when the string contains backslashes, instead of + doubling all backslashes. The resulting string does *not* always + evaluate to the same string as the original; however it will do + just the right thing when passed into re.compile(). + + The optional second argument forces the string quote; it must be + a single character which is a valid Python string quote. + + """ + if quote is None: + q = "'" + altq = "'" + if q in s and altq not in s: + q = altq + else: + assert quote in ('"', "'") + q = quote + res = q + for c in s: + if c == q: c = '\\' + c + elif c < ' ' or c > '~': c = "\\%03o" % ord(c) + res = res + c + res = res + q + if '\\' in res: + res = 'r' + res + return res + + +def main(): + """Main program -- called when run as a script.""" + import sys + s = eval(sys.stdin.read()) + sys.stdout.write(quote(convert(s))) + if sys.stdout.isatty(): + sys.stdout.write("\n") + + +if __name__ == '__main__': + main() diff --git a/Lib/dos-8x3/regex_sy.py b/Lib/dos-8x3/regex_sy.py index bb80f4e..8631f42 100755 --- a/Lib/dos-8x3/regex_sy.py +++ b/Lib/dos-8x3/regex_sy.py @@ -32,6 +32,12 @@ RE_NEWLINE_OR = 16 # *, +, ? - only special when not after the beginning, (, or | RE_CONTEXT_INDEP_OPS = 32 +# ANSI sequences (\n etc) and \xhh +RE_ANSI_HEX = 64 + +# No GNU extensions +RE_NO_GNU_EXTENSIONS = 128 + # Now define combinations of bits for the standard possibilities. RE_SYNTAX_AWK = (RE_NO_BK_PARENS | RE_NO_BK_VBAR | RE_CONTEXT_INDEP_OPS) RE_SYNTAX_EGREP = (RE_SYNTAX_AWK | RE_NEWLINE_OR) diff --git a/Lib/dos-8x3/rlcomple.py b/Lib/dos-8x3/rlcomple.py index 285faed..4deb0bc 100644 --- a/Lib/dos-8x3/rlcomple.py +++ b/Lib/dos-8x3/rlcomple.py @@ -41,12 +41,8 @@ used, and this module (and the readline module) are silently inactive. """ import readline -import keyword import __builtin__ import __main__ -import string -import re -import traceback class Completer: @@ -71,6 +67,7 @@ class Completer: currently defines in __main__ that match. """ + import keyword matches = [] n = len(text) for list in [keyword.kwlist, @@ -93,6 +90,7 @@ class Completer: with a __getattr__ hook is evaluated. """ + import re m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text) if not m: return diff --git a/Lib/dos-8x3/test_arr.py b/Lib/dos-8x3/test_arr.py index 832f192..e95fac4 100644 --- a/Lib/dos-8x3/test_arr.py +++ b/Lib/dos-8x3/test_arr.py @@ -52,7 +52,7 @@ def testtype(type, example): print 'array of %s converted to a list: ' % a.typecode, a.tolist() if verbose: print 'array of %s converted to a string: ' \ - % a.typecode, a.tostring() + % a.typecode, `a.tostring()` main() diff --git a/Lib/dos-8x3/test_sel.py b/Lib/dos-8x3/test_sel.py index 85bfa41..53185b1 100755 --- a/Lib/dos-8x3/test_sel.py +++ b/Lib/dos-8x3/test_sel.py @@ -35,7 +35,7 @@ else: def test(): import sys - if sys.platform[:3] in ('win', 'mac'): + if sys.platform[:3] in ('win', 'mac', 'os2'): if verbose: print "Can't test select easily on", sys.platform return diff --git a/Lib/dos-8x3/test_sig.py b/Lib/dos-8x3/test_sig.py index 51b120b..2d33635 100755 --- a/Lib/dos-8x3/test_sig.py +++ b/Lib/dos-8x3/test_sig.py @@ -4,9 +4,8 @@ import signal import os import sys -if sys.platform[:3] == 'win': - raise ImportError, "Can't test signal on Windows" - +if sys.platform[:3] in ('win', 'os2'): + raise ImportError, "Can't test signal on %s" % sys.platform[:3] if verbose: x = '-x' diff --git a/Lib/dos-8x3/test_tok.py b/Lib/dos-8x3/test_tok.py new file mode 100644 index 0000000..34a7bfb --- /dev/null +++ b/Lib/dos-8x3/test_tok.py @@ -0,0 +1,22 @@ +from test_support import verbose +import tokenize, os, sys + +def findfile(file): + if os.path.isabs(file): return file + path = sys.path + try: + path = [os.path.dirname(__file__)] + path + except NameError: + pass + for dn in path: + fn = os.path.join(dn, file) + if os.path.exists(fn): return fn + return file + +if verbose: + print 'starting...' +file = open(findfile('tokenize_tests.py')) +tokenize.tokenize(file.readline) +if verbose: + print 'finished' + diff --git a/Lib/dos-8x3/test_typ.py b/Lib/dos-8x3/test_typ.py index eedf65a..7cca131 100755 --- a/Lib/dos-8x3/test_typ.py +++ b/Lib/dos-8x3/test_typ.py @@ -189,3 +189,12 @@ d.update({1:1, 2:2, 3:3}) if d != {1:1, 2:2, 3:3}: raise TestFailed, 'dict update' if d.copy() != {1:1, 2:2, 3:3}: raise TestFailed, 'dict copy' if {}.copy() != {}: raise TestFailed, 'empty dict copy' +# dict.get() +d = {} +if d.get('c') != None: raise TestFailed, 'missing {} get, no 2nd arg' +if d.get('c', 3) != 3: raise TestFailed, 'missing {} get, w/ 2nd arg' +d = {'a' : 1, 'b' : 2} +if d.get('c') != None: raise TestFailed, 'missing dict get, no 2nd arg' +if d.get('c', 3) != 3: raise TestFailed, 'missing dict get, w/ 2nd arg' +if d.get('a') != 1: raise TestFailed, 'present dict get, no 2nd arg' +if d.get('a', 3) != 1: raise TestFailed, 'present dict get, w/ 2nd arg' diff --git a/Lib/dos-8x3/tokenize.py b/Lib/dos-8x3/tokenize.py new file mode 100644 index 0000000..4883668 --- /dev/null +++ b/Lib/dos-8x3/tokenize.py @@ -0,0 +1,161 @@ +# Tests for the 'tokenize' module. +# Large bits stolen from test_grammar.py. + +# Comments +"#" +#' +#" +#\ + # + # abc +'''# +#''' + +x = 1 # + +# Balancing continuation + +a = (3, 4, + 5, 6) +y = [3, 4, + 5] +z = {'a':5, + 'b':6} +x = (len(`y`) + 5*x - a[ + 3 ] + - x + len({ + } + ) + ) + +# Backslash means line continuation: +x = 1 \ ++ 1 + +# Backslash does not means continuation in comments :\ +x = 0 + +# Ordinary integers +0xff <> 255 +0377 <> 255 +2147483647 != 017777777777 +-2147483647-1 != 020000000000 +037777777777 != -1 +0xffffffff != -1 + +# Long integers +x = 0L +x = 0l +x = 0xffffffffffffffffL +x = 0xffffffffffffffffl +x = 077777777777777777L +x = 077777777777777777l +x = 123456789012345678901234567890L +x = 123456789012345678901234567890l + +# Floating-point numbers +x = 3.14 +x = 314. +x = 0.314 +# XXX x = 000.314 +x = .314 +x = 3e14 +x = 3E14 +x = 3e-14 +x = 3e+14 +x = 3.e14 +x = .3e14 +x = 3.1e4 + +# String literals +x = ''; y = ""; +x = '\''; y = "'"; +x = '"'; y = "\""; +x = "doesn't \"shrink\" does it" +y = 'doesn\'t "shrink" does it' +x = "does \"shrink\" doesn't it" +y = 'does "shrink" doesn\'t it' +x = """ +The "quick" +brown fox +jumps over +the 'lazy' dog. +""" +y = '\nThe "quick"\nbrown fox\njumps over\nthe \'lazy\' dog.\n' +y = ''' +The "quick" +brown fox +jumps over +the 'lazy' dog. +'''; +y = "\n\ +The \"quick\"\n\ +brown fox\n\ +jumps over\n\ +the 'lazy' dog.\n\ +"; +y = '\n\ +The \"quick\"\n\ +brown fox\n\ +jumps over\n\ +the \'lazy\' dog.\n\ +'; +x = r'\\' + R'\\' +x = r'\'' + '' +y = r''' +foo bar \\ +baz''' + R''' +foo''' +y = r"""foo +bar \\ baz +""" + R'''spam +''' + +# Indentation +if 1: + x = 2 +if 1: + x = 2 +if 1: + while 0: + if 0: + x = 2 + x = 2 +if 0: + if 2: + while 0: + if 1: + x = 2 + +# Operators + +def d22(a, b, c=1, d=2): pass +def d01v(a=1, *rest, **rest): pass + +(x, y) <> ({'a':1}, {'b':2}) + +# comparison +if 1 < 1 > 1 == 1 >= 1 <= 1 <> 1 != 1 in 1 not in 1 is 1 is not 1: pass + +# binary +x = 1 & 1 +x = 1 ^ 1 +x = 1 | 1 + +# shift +x = 1 << 1 >> 1 + +# additive +x = 1 - 1 + 1 - 1 + 1 + +# multiplicative +x = 1 / 1 * 1 % 1 + +# unary +x = ~1 ^ 1 & 1 | 1 & 1 ^ -1 +x = -1*1/1 + 1*1 - ---1*1 + +# selector +import sys, time +x = sys.modules['time'].time() + diff --git a/Lib/dos-8x3/userdict.py b/Lib/dos-8x3/userdict.py index 2f4f541..3b9b157 100755 --- a/Lib/dos-8x3/userdict.py +++ b/Lib/dos-8x3/userdict.py @@ -26,3 +26,8 @@ class UserDict: else: for k, v in other.items(): self.data[k] = v + def get(self, key, failobj=None): + if self.data.has_key(key): + return self.data[key] + else: + return failobj -- cgit v0.12