diff options
| -rw-r--r-- | Doc/lib/libshlex.tex | 85 | ||||
| -rw-r--r-- | Lib/shlex.py | 30 |
2 files changed, 70 insertions, 45 deletions
diff --git a/Doc/lib/libshlex.tex b/Doc/lib/libshlex.tex index a070908..f1eea3a 100644 --- a/Doc/lib/libshlex.tex +++ b/Doc/lib/libshlex.tex @@ -13,7 +13,7 @@ simple syntaxes resembling that of the \UNIX{} shell. This will often be useful for writing minilanguages, e.g.\ in run control files for Python applications. -\begin{classdesc}{shlex}{\optional{stream}, \optional{file}} +\begin{classdesc}{shlex}{\optional{stream\optional{, file}}} A \class{shlex} instance or subclass instance is a lexical analyzer object. The initialization argument, if present, specifies where to read characters from. It must be a file- or stream-like object with @@ -35,6 +35,7 @@ equal to \code{sys.stdin}, this second argument defauilts to ``stdin''. A \class{shlex} instance has the following methods: + \begin{methoddesc}{get_token}{} Return a token. If tokens have been stacked using \method{push_token()}, pop a token off the stack. Otherwise, read one @@ -52,35 +53,45 @@ requests. (This is not ordinarily a useful entry point, and is documented here only for the sake of completeness.) \end{methoddesc} -\begin{methoddesc}{openhook}{filename} -When shlex detects a source request (see \member{source} below) -this method is given the following token as argument, and expected to -return a tuple consisting of a filename and an opened stream object. - -Normally, this method just strips any quotes off the argument and -treats it as a filename, calling \code{open()} on it. It is exposed so that -you can use it to implement directory search paths, addition of -file extensions, and other namespace hacks. - +\begin{methoddesc}{sourcehook}{filename} +When \class{shlex} detects a source request (see +\member{source} below) this method is given the following token as +argument, and expected to return a tuple consisting of a filename and +an open file-like object. + +Normally, this method first strips any quotes off the argument. If +the result is an absolute pathname, or there was no previous source +request in effect, or the previous source was a stream +(e.g. \code{sys.stdin}), the result is left alone. Otherwise, if the +result is a relative pathname, the directory part of the name of the +file immediately before it on the source inclusion stack is prepended +(this behavior is like the way the C preprocessor handles +\code{\#include "file.h"}). The result of the manipulations is treated +as a filename, and returned as the first component of the tuple, with +\function{open()} called on it to yield the second component. + +This hook is exposed so that you can use it to implement directory +search paths, addition of file extensions, and other namespace hacks. There is no corresponding `close' hook, but a shlex instance will call -the \code{close()} method of the sourced input stream when it returns EOF. +the \method{close()} method of the sourced input stream when it +returns \EOF. \end{methoddesc} -\begin{methoddesc}{error_leader}{\optional{file}, \optional{line}} +\begin{methoddesc}{error_leader}{\optional{file\optional{, line}}} This method generates an error message leader in the format of a -Unix C compiler error label; the format is '"\%s", line \%d: ', -where the \%s is replaced with the name of the current source file and -the \%d with the current input line number (the optional arguments -can be used to override these). - -This convenience is provided to encourage shlex users to generate -error messages in the standard, parseable format understood by Emacs -and other Unix tools. +\UNIX{} C compiler error label; the format is '"\%s", line \%d: ', +where the \samp{\%s} is replaced with the name of the current source +file and the \samp{\%d} with the current input line number (the +optional arguments can be used to override these). + +This convenience is provided to encourage \module{shlex} users to +generate error messages in the standard, parseable format understood +by Emacs and other \UNIX{} tools. \end{methoddesc} Instances of \class{shlex} subclasses have some public instance -variables which either control lexical analysis or can be used -for debugging: +variables which either control lexical analysis or can be used for +debugging: \begin{memberdesc}{commenters} The string of characters that are recognized as comment beginners. @@ -90,7 +101,7 @@ Includes just \character{\#} by default. \begin{memberdesc}{wordchars} The string of characters that will accumulate into multi-character -tokens. By default, includes all \ASCII{} alphanumerics and +tokens. By default, includes all \ASCII{} alphanumerics and underscore. \end{memberdesc} @@ -114,24 +125,26 @@ be useful to examine this when constructing error messages. \end{memberdesc} \begin{memberdesc}{instream} -The input stream from which this shlex instance is reading characters. +The input stream from which this \class{shlex} instance is reading +characters. \end{memberdesc} \begin{memberdesc}{source} -This member is None by default. If you assign a string to it, that -string will be recognized as a lexical-level inclusion request similar -to the `source' keyword in various shells. That is, the immediately -following token will opened as a filename and input taken from that -stream until EOF, at which point the \code{close()} method of that -stream will be called and the input source will again become the -original input stream. Source requests may be stacked any number of -levels deep. +This member is \code{None} by default. If you assign a string to it, +that string will be recognized as a lexical-level inclusion request +similar to the \samp{source} keyword in various shells. That is, the +immediately following token will opened as a filename and input taken +from that stream until \EOF, at which point the \method{close()} +method of that stream will be called and the input source will again +become the original input stream. Source requests may be stacked any +number of levels deep. \end{memberdesc} \begin{memberdesc}{debug} -If this member is numeric and 1 or more, a shlex instance will print -verbose progress output on its behavior. If you need to use this, -you can read the module source code to learn the details. +If this member is numeric and \code{1} or more, a \class{shlex} +instance will print verbose progress output on its behavior. If you +need to use this, you can read the module source code to learn the +details. \end{memberdesc} Note that any character not declared to be a word character, diff --git a/Lib/shlex.py b/Lib/shlex.py index 575fc35..876dcdf 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -3,8 +3,10 @@ # Module and documentation by Eric S. Raymond, 21 Dec 1998 # Input stacking and error message cleanup added by ESR, March 2000 +import os.path import sys + class shlex: "A lexical analyzer class for simple shell-like syntaxes." def __init__(self, instream=None, infile=None): @@ -26,7 +28,8 @@ class shlex: self.filestack = [] self.source = None if self.debug: - print 'shlex: reading from %s, line %d' % (self.instream,self.lineno) + print 'shlex: reading from %s, line %d' \ + % (self.instream, self.lineno) def push_token(self, tok): "Push a token onto the stack popped by the get_token method" @@ -47,7 +50,7 @@ class shlex: # Handle inclusions while raw == self.source: (newfile, newstream) = self.sourcehook(self.read_token()) - self.filestack = [(self.infile,self.instream,self.lineno)] + self.filestack + self.filestack.insert(0, (self.infile, self.instream, self.lineno)) self.infile = newfile self.instream = newstream self.lineno = 1 @@ -63,7 +66,8 @@ class shlex: (self.infile, self.instream, self.lineno) = self.filestack[0] self.filestack = self.filestack[1:] if self.debug: - print 'shlex: popping to %s, line %d' % (self.instream, self.lineno) + print 'shlex: popping to %s, line %d' \ + % (self.instream, self.lineno) self.state = ' ' raw = self.get_token() # Neither inclusion nor EOF @@ -82,7 +86,8 @@ class shlex: if nextchar == '\n': self.lineno = self.lineno + 1 if self.debug >= 3: - print "shlex: in state " + repr(self.state) + " I see character: " + repr(nextchar) + print "shlex: in state", repr(self.state), \ + "I see character:", repr(nextchar) if self.state == None: self.token = ''; # past end of file break @@ -156,6 +161,9 @@ class shlex: "Hook called on a filename to be sourced." if newfile[0] == '"': newfile = newfile[1:-1] + # This implements cpp-like semantics for relative-path inclusion. + if type(self.infile) == type("") and not os.path.isabs(newfile): + newfile = os.path.join(os.path.dirname(self.infile), newfile) return (newfile, open(newfile, "r")) def error_leader(self, infile=None, lineno=None): @@ -166,12 +174,16 @@ class shlex: lineno = self.lineno return "\"%s\", line %d: " % (infile, lineno) -if __name__ == '__main__': - lexer = shlex() +if __name__ == '__main__': + if len(sys.argv) == 1: + lexer = shlex() + else: + file = sys.argv[1] + lexer = shlex(open(file), file) while 1: tt = lexer.get_token() - print "Token: " + repr(tt) - if not tt: + if tt: + print "Token: " + repr(tt) + else: break - |