diff options
| -rw-r--r-- | Doc/lib/libshlex.tex | 136 | ||||
| -rw-r--r-- | Lib/shlex.py | 132 | ||||
| -rw-r--r-- | Lib/test/test_shlex.py | 191 | ||||
| -rw-r--r-- | Misc/NEWS | 4 |
4 files changed, 416 insertions, 47 deletions
diff --git a/Doc/lib/libshlex.tex b/Doc/lib/libshlex.tex index eecded7..c0c4e56 100644 --- a/Doc/lib/libshlex.tex +++ b/Doc/lib/libshlex.tex @@ -4,26 +4,16 @@ \declaremodule{standard}{shlex} \modulesynopsis{Simple lexical analysis for \UNIX\ shell-like languages.} \moduleauthor{Eric S. Raymond}{esr@snark.thyrsus.com} +\moduleauthor{Gustavo Niemeyer}{niemeyer@conectiva.com} \sectionauthor{Eric S. Raymond}{esr@snark.thyrsus.com} +\sectionauthor{Gustavo Niemeyer}{niemeyer@conectiva.com} \versionadded{1.5.2} The \class{shlex} class makes it easy to write lexical analyzers for simple syntaxes resembling that of the \UNIX{} shell. This will often -be useful for writing minilanguages, e.g.\ in run control files for -Python applications. - -\begin{classdesc}{shlex}{\optional{stream\optional{, file}}} -A \class{shlex} instance or subclass instance is a lexical analyzer -object. The initialization argument, if present, specifies where to -read characters from. It must be a file- or stream-like object with -\method{read()} and \method{readline()} methods. If no argument is given, -input will be taken from \code{sys.stdin}. The second optional -argument is a filename string, which sets the initial value of the -\member{infile} member. If the stream argument is omitted or -equal to \code{sys.stdin}, this second argument defaults to ``stdin''. -\end{classdesc} - +be useful for writing minilanguages, (e.g. in run control files for +Python applications) or for parsing quoted strings. \begin{seealso} \seemodule{ConfigParser}{Parser for configuration files similar to the @@ -31,16 +21,50 @@ equal to \code{sys.stdin}, this second argument defaults to ``stdin''. \end{seealso} +\subsection{Module Contents} + +The \module{shlex} module defines the following functions: + +\begin{funcdesc}{split}{s\optional{, posix=\code{True}\optional{, + spaces=\code{True}}}} +Split the string \var{s} using shell-like syntax. If \code{posix} is +\code{True}, operate in posix mode. If \code{spaces} is \code{True}, it +will only split words in whitespaces (setting the +\member{whitespace_split} member of the \class{shlex} instance). +\versionadded{2.3} +\end{funcdesc} + +The \module{shlex} module defines the following classes: + +\begin{classdesc}{shlex}{\optional{instream=\code{sys.stdin}\optional{, + infile=\code{None}\optional{, + posix=\code{False}}}}} +A \class{shlex} instance or subclass instance is a lexical analyzer +object. The initialization argument, if present, specifies where to +read characters from. It must be a file-/stream-like object with +\method{read()} and \method{readline()} methods, or a string (strings +are accepted since Python 2.3). If no argument is given, input will be +taken from \code{sys.stdin}. The second optional argument is a filename +string, which sets the initial value of the \member{infile} member. If +the \var{instream} argument is omitted or equal to \code{sys.stdin}, +this second argument defaults to ``stdin''. The \var{posix} argument +was introduced in Python 2.3, and defines the operational mode. When +\var{posix} is not true (default), the \class{shlex} instance will +operate in compatibility mode. When operating in posix mode, +\class{shlex} will try to be as close as possible to the posix shell +parsing rules. See~\ref{shlex-objects}. +\end{classdesc} + \subsection{shlex Objects \label{shlex-objects}} A \class{shlex} instance has the following methods: - \begin{methoddesc}{get_token}{} Return a token. If tokens have been stacked using \method{push_token()}, pop a token off the stack. Otherwise, read one from the input stream. If reading encounters an immediate -end-of-file, an empty string is returned. +end-of-file, \member{self.eof} is returned (the empty string (\code{""}) +in non-posix mode, and \code{None} in posix mode). \end{methoddesc} \begin{methoddesc}{push_token}{str} @@ -132,6 +156,12 @@ bounds tokens. By default, includes space, tab, linefeed and carriage-return. \end{memberdesc} +\begin{memberdesc}{escape} +Characters that will be considered as escape. This will be only used +in posix mode, and includes just \character{\textbackslash} by default. +\versionadded{2.3} +\end{memberdesc} + \begin{memberdesc}{quotes} Characters that will be considered string quotes. The token accumulates until the same quote is encountered again (thus, different @@ -139,6 +169,20 @@ quote types protect each other as in the shell.) By default, includes \ASCII{} single and double quotes. \end{memberdesc} +\begin{memberdesc}{escapedquotes} +Characters in \member{quotes} that will interpret escape characters +defined in \member{escape}. This is only used in posix mode, and includes +just \character{"} by default. +\versionadded{2.3} +\end{memberdesc} + +\begin{memberdesc}{whitespace_split} +If true, tokens will only be split in whitespaces. This is useful, for +example, for parsing command lines with \class{shlex}, getting tokens +in a similar way to shell arguments. +\versionadded{2.3} +\end{memberdesc} + \begin{memberdesc}{infile} The name of the current input file, as initially set at class instantiation time or stacked by later source requests. It may @@ -168,13 +212,6 @@ need to use this, you can read the module source code to learn the details. \end{memberdesc} -Note that any character not declared to be a word character, -whitespace, or a quote will be returned as a single-character token. - -Quote and comment characters are not recognized within words. Thus, -the bare words \samp{ain't} and \samp{ain\#t} would be returned as single -tokens by the default parser. - \begin{memberdesc}{lineno} Source line number (count of newlines seen so far plus one). \end{memberdesc} @@ -183,3 +220,56 @@ Source line number (count of newlines seen so far plus one). The token buffer. It may be useful to examine this when catching exceptions. \end{memberdesc} + +\begin{memberdesc}{eof} +Token used to determine end of file. This will be set to the empty +string (\code{""}), in non-posix mode, and to \code{None} in posix +mode. +\versionadded{2.3} +\end{memberdesc} + +\subsection{Parsing Rules\label{shlex-parsing-rules}} + +When operating in non-posix mode, \class{shlex} with try to obey to the +following rules. + +\begin{itemize} +\item Quote characters are not recognized within words + (\code{Do"Not"Separate} is parsed as the single word + \code{Do"Not"Separate}); +\item Escape characters are not recognized; +\item Enclosing characters in quotes preserve the literal value of + all characters within the quotes; +\item Closing quotes separate words (\code{"Do"Separate} is parsed + as \code{"Do"} and \code{Separate}); +\item If \member{whitespace_split} is \code{False}, any character not + declared to be a word character, whitespace, or a quote will be + returned as a single-character token. If it is \code{True}, + \class{shlex} will only split words in whitespaces; +\item EOF is signaled with an empty string (\code{""}); +\item It's not possible to parse empty strings, even if quoted. +\end{itemize} + +When operating in posix mode, \class{shlex} will try to obey to the +following parsing rules. + +\begin{itemize} +\item Quotes are stripped out, and do not separate words + (\code{"Do"Not"Separate"} is parsed as the single word + \code{DoNotSeparate}); +\item Non-quoted escape characters (e.g. \character{\textbackslash}) + preserve the literal value of the next character that follows; +\item Enclosing characters in quotes which are not part of + \member{escapedquotes} (e.g. \character{'}) preserve the literal + value of all characters within the quotes; +\item Enclosing characters in quotes which are part of + \member{escapedquotes} (e.g. \character{"}) preserves the literal + value of all characters within the quotes, with the exception of + the characters mentioned in \member{escape}. The escape characters + retain its special meaning only when followed by the quote in use, + or the escape character itself. Otherwise the escape character + will be considered a normal character. +\item EOF is signaled with a \code{None} value; +\item Quoted empty strings (\code{""}) are allowed; +\end{itemize} + diff --git a/Lib/shlex.py b/Lib/shlex.py index 7ffa79c..863e404 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -1,28 +1,51 @@ +# -*- coding: iso-8859-1 -*- """A lexical analyzer class for simple shell-like syntaxes.""" # Module and documentation by Eric S. Raymond, 21 Dec 1998 # Input stacking and error message cleanup added by ESR, March 2000 # push_source() and pop_source() made explicit by ESR, January 2001. +# Posix compliance, split(), string arguments, and +# iterator interface by Gustavo Niemeyer, April 2003. import os.path import sys -__all__ = ["shlex"] +from types import StringTypes + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + +__all__ = ["shlex", "split"] class shlex: "A lexical analyzer class for simple shell-like syntaxes." - def __init__(self, instream=None, infile=None): + def __init__(self, instream=None, infile=None, posix=0): + if type(instream) in StringTypes: + instream = StringIO(instream) if instream is not None: self.instream = instream self.infile = infile else: self.instream = sys.stdin self.infile = None + self.posix = posix + if posix: + self.eof = None + else: + self.eof = '' self.commenters = '#' self.wordchars = ('abcdfeghijklmnopqrstuvwxyz' 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_') + if self.posix: + self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ' + 'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ') self.whitespace = ' \t\r\n' + self.whitespace_split = 0 self.quotes = '\'"' + self.escape = '\\' + self.escapedquotes = '"' self.state = ' ' self.pushback = [] self.lineno = 1 @@ -42,6 +65,8 @@ class shlex: def push_source(self, newstream, newfile=None): "Push an input source onto the lexer's input source stack." + if type(newstream) in StringTypes: + newstream = StringIO(newstream) self.filestack.insert(0, (self.infile, self.instream, self.lineno)) self.infile = newfile self.instream = newstream @@ -73,29 +98,31 @@ class shlex: # No pushback. Get a token. raw = self.read_token() # Handle inclusions - while raw == self.source: - spec = self.sourcehook(self.read_token()) - if spec: - (newfile, newstream) = spec - self.push_source(newstream, newfile) - raw = self.get_token() + if self.source is not None: + while raw == self.source: + spec = self.sourcehook(self.read_token()) + if spec: + (newfile, newstream) = spec + self.push_source(newstream, newfile) + raw = self.get_token() # Maybe we got EOF instead? - while raw == "": + while raw == self.eof: if len(self.filestack) == 0: - return "" + return self.eof else: self.pop_source() raw = self.get_token() - # Neither inclusion nor EOF + # Neither inclusion nor EOF if self.debug >= 1: - if raw: + if raw != self.eof: print "shlex: token=" + `raw` else: print "shlex: token=EOF" return raw def read_token(self): - "Read a token from the input stream (no pushback or inclusions)" + quoted = 0 + escapedstate = ' ' while 1: nextchar = self.instream.read(1) if nextchar == '\n': @@ -113,35 +140,65 @@ class shlex: elif nextchar in self.whitespace: if self.debug >= 2: print "shlex: I see whitespace in whitespace state" - if self.token: + if self.token or (self.posix and quoted): break # emit current token else: continue elif nextchar in self.commenters: self.instream.readline() self.lineno = self.lineno + 1 + elif self.posix and nextchar in self.escape: + escapedstate = 'a' + self.state = nextchar elif nextchar in self.wordchars: self.token = nextchar self.state = 'a' elif nextchar in self.quotes: - self.token = nextchar + if not self.posix: + self.token = nextchar self.state = nextchar + elif self.whitespace_split: + self.token = nextchar + self.state = 'a' else: self.token = nextchar - if self.token: + if self.token or (self.posix and quoted): break # emit current token else: continue elif self.state in self.quotes: - self.token = self.token + nextchar - if nextchar == self.state: - self.state = ' ' - break - elif not nextchar: # end of file + quoted = 1 + if not nextchar: # end of file if self.debug >= 2: print "shlex: I see EOF in quotes state" # XXX what error should be raised here? raise ValueError, "No closing quotation" + if nextchar == self.state: + if not self.posix: + self.token = self.token + nextchar + self.state = ' ' + break + else: + self.state = 'a' + elif self.posix and nextchar in self.escape and \ + self.state in self.escapedquotes: + escapedstate = self.state + self.state = nextchar + else: + self.token = self.token + nextchar + elif self.state in self.escape: + if not nextchar: # end of file + if self.debug >= 2: + print "shlex: I see EOF in escape state" + # XXX what error should be raised here? + raise ValueError, "No escaped character" + # In posix shells, only the quote itself or the escape + # character may be escaped within quotes. + if escapedstate in self.quotes and \ + nextchar != self.state and nextchar != escapedstate: + self.token = self.token + self.state + self.token = self.token + nextchar + self.state = escapedstate elif self.state == 'a': if not nextchar: self.state = None # end of file @@ -150,14 +207,26 @@ class shlex: if self.debug >= 2: print "shlex: I see whitespace in word state" self.state = ' ' - if self.token: + if self.token or (self.posix and quoted): break # emit current token else: continue elif nextchar in self.commenters: self.instream.readline() self.lineno = self.lineno + 1 - elif nextchar in self.wordchars or nextchar in self.quotes: + if self.posix: + self.state = ' ' + if self.token or (self.posix and quoted): + break # emit current token + else: + continue + elif self.posix and nextchar in self.quotes: + self.state = nextchar + elif self.posix and nextchar in self.escape: + escapedstate = 'a' + self.state = nextchar + elif nextchar in self.wordchars or nextchar in self.quotes \ + or self.whitespace_split: self.token = self.token + nextchar else: self.pushback = [nextchar] + self.pushback @@ -170,6 +239,8 @@ class shlex: continue result = self.token self.token = '' + if self.posix and not quoted and result == '': + result = None if self.debug > 1: if result: print "shlex: raw token=" + `result` @@ -182,7 +253,7 @@ class shlex: if newfile[0] == '"': newfile = newfile[1:-1] # This implements cpp-like semantics for relative-path inclusion. - if type(self.infile) == type("") and not os.path.isabs(newfile): + if type(self.infile) in StringTypes and not os.path.isabs(newfile): newfile = os.path.join(os.path.dirname(self.infile), newfile) return (newfile, open(newfile, "r")) @@ -194,6 +265,19 @@ class shlex: lineno = self.lineno return "\"%s\", line %d: " % (infile, lineno) + def __iter__(self): + return self + + def next(self): + token = self.get_token() + if token == self.eof: + raise StopIteration + return token + +def split(s, posix=1, spaces=1): + lex = shlex(s, posix=posix) + lex.whitespace_split = spaces + return list(lex) if __name__ == '__main__': if len(sys.argv) == 1: diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py new file mode 100644 index 0000000..f8ef88c --- /dev/null +++ b/Lib/test/test_shlex.py @@ -0,0 +1,191 @@ +# -*- coding: iso-8859-1 -*- +import unittest +import os, sys +import shlex + +try: + from cStringIO import StringIO +except ImportError: + from StringIO import StringIO + + +# The original test data set was from shellwords, by Hartmut Goebel. + +data = r"""x|x| +foo bar|foo|bar| + foo bar|foo|bar| + foo bar |foo|bar| +foo bar bla fasel|foo|bar|bla|fasel| +x y z xxxx|x|y|z|xxxx| +\x bar|\|x|bar| +\ x bar|\|x|bar| +\ bar|\|bar| +foo \x bar|foo|\|x|bar| +foo \ x bar|foo|\|x|bar| +foo \ bar|foo|\|bar| +foo "bar" bla|foo|"bar"|bla| +"foo" "bar" "bla"|"foo"|"bar"|"bla"| +"foo" bar "bla"|"foo"|bar|"bla"| +"foo" bar bla|"foo"|bar|bla| +foo 'bar' bla|foo|'bar'|bla| +'foo' 'bar' 'bla'|'foo'|'bar'|'bla'| +'foo' bar 'bla'|'foo'|bar|'bla'| +'foo' bar bla|'foo'|bar|bla| +blurb foo"bar"bar"fasel" baz|blurb|foo"bar"bar"fasel"|baz| +blurb foo'bar'bar'fasel' baz|blurb|foo'bar'bar'fasel'|baz| +""|""| +''|''| +foo "" bar|foo|""|bar| +foo '' bar|foo|''|bar| +foo "" "" "" bar|foo|""|""|""|bar| +foo '' '' '' bar|foo|''|''|''|bar| +\""|\|""| +"\"|"\"| +"foo\ bar"|"foo\ bar"| +"foo\\ bar"|"foo\\ bar"| +"foo\\ bar\"|"foo\\ bar\"| +"foo\\" bar\""|"foo\\"|bar|\|""| +"foo\\ bar\" dfadf"|"foo\\ bar\"|dfadf"| +"foo\\\ bar\" dfadf"|"foo\\\ bar\"|dfadf"| +"foo\\\x bar\" dfadf"|"foo\\\x bar\"|dfadf"| +"foo\x bar\" dfadf"|"foo\x bar\"|dfadf"| +\''|\|''| +'foo\ bar'|'foo\ bar'| +'foo\\ bar'|'foo\\ bar'| +"foo\\\x bar\" df'a\ 'df'|"foo\\\x bar\"|df'a|\|'df'| +\"foo"|\|"foo"| +\"foo"\x|\|"foo"|\|x| +"foo\x"|"foo\x"| +"foo\ "|"foo\ "| +foo\ xx|foo|\|xx| +foo\ x\x|foo|\|x|\|x| +foo\ x\x\""|foo|\|x|\|x|\|""| +"foo\ x\x"|"foo\ x\x"| +"foo\ x\x\\"|"foo\ x\x\\"| +"foo\ x\x\\""foobar"|"foo\ x\x\\"|"foobar"| +"foo\ x\x\\"\''"foobar"|"foo\ x\x\\"|\|''|"foobar"| +"foo\ x\x\\"\'"fo'obar"|"foo\ x\x\\"|\|'"fo'|obar"| +"foo\ x\x\\"\'"fo'obar" 'don'\''t'|"foo\ x\x\\"|\|'"fo'|obar"|'don'|\|''|t'| +'foo\ bar'|'foo\ bar'| +'foo\\ bar'|'foo\\ bar'| +foo\ bar|foo|\|bar| +foo#bar\nbaz|foobaz| +:-) ;-)|:|-|)|;|-|)| +áéíóú|á|é|í|ó|ú| +""" + +posix_data = r"""x|x| +foo bar|foo|bar| + foo bar|foo|bar| + foo bar |foo|bar| +foo bar bla fasel|foo|bar|bla|fasel| +x y z xxxx|x|y|z|xxxx| +\x bar|x|bar| +\ x bar| x|bar| +\ bar| bar| +foo \x bar|foo|x|bar| +foo \ x bar|foo| x|bar| +foo \ bar|foo| bar| +foo "bar" bla|foo|bar|bla| +"foo" "bar" "bla"|foo|bar|bla| +"foo" bar "bla"|foo|bar|bla| +"foo" bar bla|foo|bar|bla| +foo 'bar' bla|foo|bar|bla| +'foo' 'bar' 'bla'|foo|bar|bla| +'foo' bar 'bla'|foo|bar|bla| +'foo' bar bla|foo|bar|bla| +blurb foo"bar"bar"fasel" baz|blurb|foobarbarfasel|baz| +blurb foo'bar'bar'fasel' baz|blurb|foobarbarfasel|baz| +""|| +''|| +foo "" bar|foo||bar| +foo '' bar|foo||bar| +foo "" "" "" bar|foo||||bar| +foo '' '' '' bar|foo||||bar| +\"|"| +"\""|"| +"foo\ bar"|foo\ bar| +"foo\\ bar"|foo\ bar| +"foo\\ bar\""|foo\ bar"| +"foo\\" bar\"|foo\|bar"| +"foo\\ bar\" dfadf"|foo\ bar" dfadf| +"foo\\\ bar\" dfadf"|foo\\ bar" dfadf| +"foo\\\x bar\" dfadf"|foo\\x bar" dfadf| +"foo\x bar\" dfadf"|foo\x bar" dfadf| +\'|'| +'foo\ bar'|foo\ bar| +'foo\\ bar'|foo\\ bar| +"foo\\\x bar\" df'a\ 'df"|foo\\x bar" df'a\ 'df| +\"foo|"foo| +\"foo\x|"foox| +"foo\x"|foo\x| +"foo\ "|foo\ | +foo\ xx|foo xx| +foo\ x\x|foo xx| +foo\ x\x\"|foo xx"| +"foo\ x\x"|foo\ x\x| +"foo\ x\x\\"|foo\ x\x\| +"foo\ x\x\\""foobar"|foo\ x\x\foobar| +"foo\ x\x\\"\'"foobar"|foo\ x\x\'foobar| +"foo\ x\x\\"\'"fo'obar"|foo\ x\x\'fo'obar| +"foo\ x\x\\"\'"fo'obar" 'don'\''t'|foo\ x\x\'fo'obar|don't| +"foo\ x\x\\"\'"fo'obar" 'don'\''t' \\|foo\ x\x\'fo'obar|don't|\| +'foo\ bar'|foo\ bar| +'foo\\ bar'|foo\\ bar| +foo\ bar|foo bar| +foo#bar\nbaz|foo|baz| +:-) ;-)|:-)|;-)| +áéíóú|áéíóú| +""" + +class ShlexTest(unittest.TestCase): + def setUp(self): + self.data = [x.split("|")[:-1] + for x in data.splitlines()] + self.posix_data = [x.split("|")[:-1] + for x in posix_data.splitlines()] + for item in self.data: + item[0] = item[0].replace(r"\n", "\n") + for item in self.posix_data: + item[0] = item[0].replace(r"\n", "\n") + + def splitTest(self, data, posix, spaces): + for i in range(len(data)): + l = shlex.split(data[i][0], posix=posix, spaces=spaces) + self.assertEqual(l, data[i][1:], + "%s: %s != %s" % + (data[i][0], l, data[i][1:])) + + def oldSplit(self, s): + ret = [] + lex = shlex.shlex(StringIO(s)) + tok = lex.get_token() + while tok: + ret.append(tok) + tok = lex.get_token() + return ret + + def testSplit(self): + """Test data splitting with non-posix parser""" + self.splitTest(self.data, posix=0, spaces=0) + + def testSplitPosix(self): + """Test data splitting with posix parser""" + self.splitTest(self.posix_data, posix=1, spaces=1) + + def testCompat(self): + """Test compatibility interface""" + for i in range(len(self.data)): + l = self.oldSplit(self.data[i][0]) + self.assertEqual(l, self.data[i][1:], + "%s: %s != %s" % + (self.data[i][0], l, self.data[i][1:])) + +# Allow this test to be used with old shlex.py +if not getattr(shlex, "split", None): + for methname in dir(ShlexTest): + if methname.startswith("test") and methname != "testCompat": + delattr(ShlexTest, methname) + +if __name__ == "__main__": + unittest.main() @@ -155,6 +155,10 @@ Library - New csv package makes it easy to read/write CSV files. +- Module shlex has been extended to allow posix-like shell parsings, + including a split() function for easy spliting of quoted strings and + commands. An iterator interface was also implemented. + Tools/Demos ----------- |