Implemented posix-mode parsing support in shlex.py, as dicussed in

mailing list, and in patch #722686.

4 files changed, 416 insertions, 47 deletions

diff --git a/Doc/lib/libshlex.tex b/Doc/lib/libshlex.tex
index eecded7..c0c4e56 100644
--- a/Doc/lib/libshlex.tex
+++ b/Doc/lib/libshlex.tex
@@ -4,26 +4,16 @@
 \declaremodule{standard}{shlex}
 \modulesynopsis{Simple lexical analysis for \UNIX\ shell-like languages.}
 \moduleauthor{Eric S. Raymond}{esr@snark.thyrsus.com}
+\moduleauthor{Gustavo Niemeyer}{niemeyer@conectiva.com}
 \sectionauthor{Eric S. Raymond}{esr@snark.thyrsus.com}
+\sectionauthor{Gustavo Niemeyer}{niemeyer@conectiva.com}
 
 \versionadded{1.5.2}
 
 The \class{shlex} class makes it easy to write lexical analyzers for
 simple syntaxes resembling that of the \UNIX{} shell.  This will often
-be useful for writing minilanguages, e.g.\ in run control files for
-Python applications.
-
-\begin{classdesc}{shlex}{\optional{stream\optional{, file}}}
-A \class{shlex} instance or subclass instance is a lexical analyzer
-object.  The initialization argument, if present, specifies where to
-read characters from. It must be a file- or stream-like object with
-\method{read()} and \method{readline()} methods.  If no argument is given,
-input will be taken from \code{sys.stdin}.  The second optional 
-argument is a filename string, which sets the initial value of the
-\member{infile} member.  If the stream argument is omitted or
-equal to \code{sys.stdin}, this second argument defaults to ``stdin''.
-\end{classdesc}
-
+be useful for writing minilanguages, (e.g. in run control files for
+Python applications) or for parsing quoted strings.
 
 \begin{seealso}
   \seemodule{ConfigParser}{Parser for configuration files similar to the
@@ -31,16 +21,50 @@ equal to \code{sys.stdin}, this second argument defaults to ``stdin''.
 \end{seealso}
 
 
+\subsection{Module Contents}
+
+The \module{shlex} module defines the following functions:
+
+\begin{funcdesc}{split}{s\optional{, posix=\code{True}\optional{,
+			spaces=\code{True}}}}
+Split the string \var{s} using shell-like syntax. If \code{posix} is
+\code{True}, operate in posix mode. If \code{spaces} is \code{True}, it
+will only split words in whitespaces (setting the
+\member{whitespace_split} member of the \class{shlex} instance).
+\versionadded{2.3}
+\end{funcdesc}
+
+The \module{shlex} module defines the following classes:
+
+\begin{classdesc}{shlex}{\optional{instream=\code{sys.stdin}\optional{,
+			 infile=\code{None}\optional{,
+			 posix=\code{False}}}}}
+A \class{shlex} instance or subclass instance is a lexical analyzer
+object.  The initialization argument, if present, specifies where to
+read characters from. It must be a file-/stream-like object with
+\method{read()} and \method{readline()} methods, or a string (strings
+are accepted since Python 2.3). If no argument is given, input will be
+taken from \code{sys.stdin}.  The second optional argument is a filename
+string, which sets the initial value of the \member{infile} member.  If
+the \var{instream} argument is omitted or equal to \code{sys.stdin},
+this second argument defaults to ``stdin''.  The \var{posix} argument
+was introduced in Python 2.3, and defines the operational mode. When
+\var{posix} is not true (default), the \class{shlex} instance will
+operate in compatibility mode. When operating in posix mode,
+\class{shlex} will try to be as close as possible to the posix shell
+parsing rules. See~\ref{shlex-objects}.
+\end{classdesc}
+
 \subsection{shlex Objects \label{shlex-objects}}
 
 A \class{shlex} instance has the following methods:
 
-
 \begin{methoddesc}{get_token}{}
 Return a token.  If tokens have been stacked using
 \method{push_token()}, pop a token off the stack.  Otherwise, read one
 from the input stream.  If reading encounters an immediate
-end-of-file, an empty string is returned. 
+end-of-file, \member{self.eof} is returned (the empty string (\code{""})
+in non-posix mode, and \code{None} in posix mode).
 \end{methoddesc}
 
 \begin{methoddesc}{push_token}{str}
@@ -132,6 +156,12 @@ bounds tokens.  By default, includes space, tab, linefeed and
 carriage-return.
 \end{memberdesc}
 
+\begin{memberdesc}{escape}
+Characters that will be considered as escape. This will be only used
+in posix mode, and includes just \character{\textbackslash} by default.
+\versionadded{2.3}
+\end{memberdesc}
+
 \begin{memberdesc}{quotes}
 Characters that will be considered string quotes.  The token
 accumulates until the same quote is encountered again (thus, different
@@ -139,6 +169,20 @@ quote types protect each other as in the shell.)  By default, includes
 \ASCII{} single and double quotes.
 \end{memberdesc}
 
+\begin{memberdesc}{escapedquotes}
+Characters in \member{quotes} that will interpret escape characters
+defined in \member{escape}. This is only used in posix mode, and includes
+just \character{"} by default.
+\versionadded{2.3}
+\end{memberdesc}
+
+\begin{memberdesc}{whitespace_split}
+If true, tokens will only be split in whitespaces. This is useful, for
+example, for parsing command lines with \class{shlex}, getting tokens
+in a similar way to shell arguments.
+\versionadded{2.3}
+\end{memberdesc}
+
 \begin{memberdesc}{infile}
 The name of the current input file, as initially set at class
 instantiation time or stacked by later source requests.  It may
@@ -168,13 +212,6 @@ need to use this, you can read the module source code to learn the
 details.
 \end{memberdesc}
 
-Note that any character not declared to be a word character,
-whitespace, or a quote will be returned as a single-character token.
-
-Quote and comment characters are not recognized within words.  Thus,
-the bare words \samp{ain't} and \samp{ain\#t} would be returned as single
-tokens by the default parser.
-
 \begin{memberdesc}{lineno}
 Source line number (count of newlines seen so far plus one).
 \end{memberdesc}
@@ -183,3 +220,56 @@ Source line number (count of newlines seen so far plus one).
 The token buffer.  It may be useful to examine this when catching
 exceptions.
 \end{memberdesc}
+
+\begin{memberdesc}{eof}
+Token used to determine end of file. This will be set to the empty
+string (\code{""}), in non-posix mode, and to \code{None} in posix
+mode.
+\versionadded{2.3}
+\end{memberdesc}
+
+\subsection{Parsing Rules\label{shlex-parsing-rules}}
+
+When operating in non-posix mode, \class{shlex} with try to obey to the
+following rules.
+
+\begin{itemize}
+\item Quote characters are not recognized within words
+      (\code{Do"Not"Separate} is parsed as the single word
+      \code{Do"Not"Separate});
+\item Escape characters are not recognized;
+\item Enclosing characters in quotes preserve the literal value of
+      all characters within the quotes;
+\item Closing quotes separate words (\code{"Do"Separate} is parsed
+      as \code{"Do"} and \code{Separate});
+\item If \member{whitespace_split} is \code{False}, any character not
+      declared to be a word character, whitespace, or a quote will be
+      returned as a single-character token. If it is \code{True},
+      \class{shlex} will only split words in whitespaces;
+\item EOF is signaled with an empty string (\code{""});
+\item It's not possible to parse empty strings, even if quoted.
+\end{itemize}
+
+When operating in posix mode, \class{shlex} will try to obey to the
+following parsing rules.
+
+\begin{itemize}
+\item Quotes are stripped out, and do not separate words
+      (\code{"Do"Not"Separate"} is parsed as the single word
+      \code{DoNotSeparate});
+\item Non-quoted escape characters (e.g. \character{\textbackslash})
+      preserve the literal value of the next character that follows;
+\item Enclosing characters in quotes which are not part of
+      \member{escapedquotes} (e.g. \character{'}) preserve the literal
+      value of all characters within the quotes;
+\item Enclosing characters in quotes which are part of
+      \member{escapedquotes} (e.g. \character{"}) preserves the literal
+      value of all characters within the quotes, with the exception of
+      the characters mentioned in \member{escape}. The escape characters
+      retain its special meaning only when followed by the quote in use,
+      or the escape character itself. Otherwise the escape character
+      will be considered a normal character.
+\item EOF is signaled with a \code{None} value;
+\item Quoted empty strings (\code{""}) are allowed;
+\end{itemize}
+
diff --git a/Lib/shlex.py b/Lib/shlex.py
index 7ffa79c..863e404 100644
--- a/Lib/shlex.py
+++ b/Lib/shlex.py
@@ -1,28 +1,51 @@
+# -*- coding: iso-8859-1 -*-
 """A lexical analyzer class for simple shell-like syntaxes."""
 
 # Module and documentation by Eric S. Raymond, 21 Dec 1998
 # Input stacking and error message cleanup added by ESR, March 2000
 # push_source() and pop_source() made explicit by ESR, January 2001.
+# Posix compliance, split(), string arguments, and
+# iterator interface by Gustavo Niemeyer, April 2003.
 
 import os.path
 import sys
 
-__all__ = ["shlex"]
+from types import StringTypes
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+__all__ = ["shlex", "split"]
 
 class shlex:
     "A lexical analyzer class for simple shell-like syntaxes."
-    def __init__(self, instream=None, infile=None):
+    def __init__(self, instream=None, infile=None, posix=0):
+        if type(instream) in StringTypes:
+            instream = StringIO(instream)
         if instream is not None:
             self.instream = instream
             self.infile = infile
         else:
             self.instream = sys.stdin
             self.infile = None
+        self.posix = posix
+        if posix:
+            self.eof = None
+        else:
+            self.eof = ''
         self.commenters = '#'
         self.wordchars = ('abcdfeghijklmnopqrstuvwxyz'
                           'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_')
+        if self.posix:
+            self.wordchars += ('ßàáâãäåæçèéêëìíîïðñòóôõöøùúûüýþÿ'
+                               'ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖØÙÚÛÜÝÞ')
         self.whitespace = ' \t\r\n'
+        self.whitespace_split = 0
         self.quotes = '\'"'
+        self.escape = '\\'
+        self.escapedquotes = '"'
         self.state = ' '
         self.pushback = []
         self.lineno = 1
@@ -42,6 +65,8 @@ class shlex:
 
     def push_source(self, newstream, newfile=None):
         "Push an input source onto the lexer's input source stack."
+        if type(newstream) in StringTypes:
+            newstream = StringIO(newstream)
         self.filestack.insert(0, (self.infile, self.instream, self.lineno))
         self.infile = newfile
         self.instream = newstream
@@ -73,29 +98,31 @@ class shlex:
         # No pushback.  Get a token.
         raw = self.read_token()
         # Handle inclusions
-        while raw == self.source:
-            spec = self.sourcehook(self.read_token())
-            if spec:
-                (newfile, newstream) = spec
-                self.push_source(newstream, newfile)
-            raw = self.get_token()
+        if self.source is not None:
+            while raw == self.source:
+                spec = self.sourcehook(self.read_token())
+                if spec:
+                    (newfile, newstream) = spec
+                    self.push_source(newstream, newfile)
+                raw = self.get_token()
         # Maybe we got EOF instead?
-        while raw == "":
+        while raw == self.eof:
             if len(self.filestack) == 0:
-                return ""
+                return self.eof
             else:
                 self.pop_source()
                 raw = self.get_token()
-         # Neither inclusion nor EOF
+        # Neither inclusion nor EOF
         if self.debug >= 1:
-            if raw:
+            if raw != self.eof:
                 print "shlex: token=" + `raw`
             else:
                 print "shlex: token=EOF"
         return raw
 
     def read_token(self):
-        "Read a token from the input stream (no pushback or inclusions)"
+        quoted = 0
+        escapedstate = ' '
         while 1:
             nextchar = self.instream.read(1)
             if nextchar == '\n':
@@ -113,35 +140,65 @@ class shlex:
                 elif nextchar in self.whitespace:
                     if self.debug >= 2:
                         print "shlex: I see whitespace in whitespace state"
-                    if self.token:
+                    if self.token or (self.posix and quoted):
                         break   # emit current token
                     else:
                         continue
                 elif nextchar in self.commenters:
                     self.instream.readline()
                     self.lineno = self.lineno + 1
+                elif self.posix and nextchar in self.escape:
+                    escapedstate = 'a'
+                    self.state = nextchar
                 elif nextchar in self.wordchars:
                     self.token = nextchar
                     self.state = 'a'
                 elif nextchar in self.quotes:
-                    self.token = nextchar
+                    if not self.posix:
+                        self.token = nextchar
                     self.state = nextchar
+                elif self.whitespace_split:
+                    self.token = nextchar
+                    self.state = 'a'
                 else:
                     self.token = nextchar
-                    if self.token:
+                    if self.token or (self.posix and quoted):
                         break   # emit current token
                     else:
                         continue
             elif self.state in self.quotes:
-                self.token = self.token + nextchar
-                if nextchar == self.state:
-                    self.state = ' '
-                    break
-                elif not nextchar:      # end of file
+                quoted = 1
+                if not nextchar:      # end of file
                     if self.debug >= 2:
                         print "shlex: I see EOF in quotes state"
                     # XXX what error should be raised here?
                     raise ValueError, "No closing quotation"
+                if nextchar == self.state:
+                    if not self.posix:
+                        self.token = self.token + nextchar
+                        self.state = ' '
+                        break
+                    else:
+                        self.state = 'a'
+                elif self.posix and nextchar in self.escape and \
+                     self.state in self.escapedquotes:
+                    escapedstate = self.state
+                    self.state = nextchar
+                else:
+                    self.token = self.token + nextchar
+            elif self.state in self.escape:
+                if not nextchar:      # end of file
+                    if self.debug >= 2:
+                        print "shlex: I see EOF in escape state"
+                    # XXX what error should be raised here?
+                    raise ValueError, "No escaped character"
+                # In posix shells, only the quote itself or the escape
+                # character may be escaped within quotes.
+                if escapedstate in self.quotes and \
+                   nextchar != self.state and nextchar != escapedstate:
+                    self.token = self.token + self.state
+                self.token = self.token + nextchar
+                self.state = escapedstate
             elif self.state == 'a':
                 if not nextchar:
                     self.state = None   # end of file
@@ -150,14 +207,26 @@ class shlex:
                     if self.debug >= 2:
                         print "shlex: I see whitespace in word state"
                     self.state = ' '
-                    if self.token:
+                    if self.token or (self.posix and quoted):
                         break   # emit current token
                     else:
                         continue
                 elif nextchar in self.commenters:
                     self.instream.readline()
                     self.lineno = self.lineno + 1
-                elif nextchar in self.wordchars or nextchar in self.quotes:
+                    if self.posix:
+                        self.state = ' '
+                        if self.token or (self.posix and quoted):
+                            break   # emit current token
+                        else:
+                            continue
+                elif self.posix and nextchar in self.quotes:
+                    self.state = nextchar
+                elif self.posix and nextchar in self.escape:
+                    escapedstate = 'a'
+                    self.state = nextchar
+                elif nextchar in self.wordchars or nextchar in self.quotes \
+                    or self.whitespace_split:
                     self.token = self.token + nextchar
                 else:
                     self.pushback = [nextchar] + self.pushback
@@ -170,6 +239,8 @@ class shlex:
                         continue
         result = self.token
         self.token = ''
+        if self.posix and not quoted and result == '':
+            result = None
         if self.debug > 1:
             if result:
                 print "shlex: raw token=" + `result`
@@ -182,7 +253,7 @@ class shlex:
         if newfile[0] == '"':
             newfile = newfile[1:-1]
         # This implements cpp-like semantics for relative-path inclusion.
-        if type(self.infile) == type("") and not os.path.isabs(newfile):
+        if type(self.infile) in StringTypes and not os.path.isabs(newfile):
             newfile = os.path.join(os.path.dirname(self.infile), newfile)
         return (newfile, open(newfile, "r"))
 
@@ -194,6 +265,19 @@ class shlex:
             lineno = self.lineno
         return "\"%s\", line %d: " % (infile, lineno)
 
+    def __iter__(self):
+        return self
+
+    def next(self):
+        token = self.get_token()
+        if token == self.eof:
+            raise StopIteration
+        return token
+
+def split(s, posix=1, spaces=1):
+    lex = shlex(s, posix=posix)
+    lex.whitespace_split = spaces
+    return list(lex)
 
 if __name__ == '__main__':
     if len(sys.argv) == 1:
diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py
new file mode 100644
index 0000000..f8ef88c
--- /dev/null
+++ b/Lib/test/test_shlex.py
@@ -0,0 +1,191 @@
+# -*- coding: iso-8859-1 -*-
+import unittest
+import os, sys
+import shlex
+
+try:
+    from cStringIO import StringIO
+except ImportError:
+    from StringIO import StringIO
+
+
+# The original test data set was from shellwords, by Hartmut Goebel.
+
+data = r"""x|x|
+foo bar|foo|bar|
+ foo bar|foo|bar|
+ foo bar |foo|bar|
+foo   bar    bla     fasel|foo|bar|bla|fasel|
+x y  z              xxxx|x|y|z|xxxx|
+\x bar|\|x|bar|
+\ x bar|\|x|bar|
+\ bar|\|bar|
+foo \x bar|foo|\|x|bar|
+foo \ x bar|foo|\|x|bar|
+foo \ bar|foo|\|bar|
+foo "bar" bla|foo|"bar"|bla|
+"foo" "bar" "bla"|"foo"|"bar"|"bla"|
+"foo" bar "bla"|"foo"|bar|"bla"|
+"foo" bar bla|"foo"|bar|bla|
+foo 'bar' bla|foo|'bar'|bla|
+'foo' 'bar' 'bla'|'foo'|'bar'|'bla'|
+'foo' bar 'bla'|'foo'|bar|'bla'|
+'foo' bar bla|'foo'|bar|bla|
+blurb foo"bar"bar"fasel" baz|blurb|foo"bar"bar"fasel"|baz|
+blurb foo'bar'bar'fasel' baz|blurb|foo'bar'bar'fasel'|baz|
+""|""|
+''|''|
+foo "" bar|foo|""|bar|
+foo '' bar|foo|''|bar|
+foo "" "" "" bar|foo|""|""|""|bar|
+foo '' '' '' bar|foo|''|''|''|bar|
+\""|\|""|
+"\"|"\"|
+"foo\ bar"|"foo\ bar"|
+"foo\\ bar"|"foo\\ bar"|
+"foo\\ bar\"|"foo\\ bar\"|
+"foo\\" bar\""|"foo\\"|bar|\|""|
+"foo\\ bar\" dfadf"|"foo\\ bar\"|dfadf"|
+"foo\\\ bar\" dfadf"|"foo\\\ bar\"|dfadf"|
+"foo\\\x bar\" dfadf"|"foo\\\x bar\"|dfadf"|
+"foo\x bar\" dfadf"|"foo\x bar\"|dfadf"|
+\''|\|''|
+'foo\ bar'|'foo\ bar'|
+'foo\\ bar'|'foo\\ bar'|
+"foo\\\x bar\" df'a\ 'df'|"foo\\\x bar\"|df'a|\|'df'|
+\"foo"|\|"foo"|
+\"foo"\x|\|"foo"|\|x|
+"foo\x"|"foo\x"|
+"foo\ "|"foo\ "|
+foo\ xx|foo|\|xx|
+foo\ x\x|foo|\|x|\|x|
+foo\ x\x\""|foo|\|x|\|x|\|""|
+"foo\ x\x"|"foo\ x\x"|
+"foo\ x\x\\"|"foo\ x\x\\"|
+"foo\ x\x\\""foobar"|"foo\ x\x\\"|"foobar"|
+"foo\ x\x\\"\''"foobar"|"foo\ x\x\\"|\|''|"foobar"|
+"foo\ x\x\\"\'"fo'obar"|"foo\ x\x\\"|\|'"fo'|obar"|
+"foo\ x\x\\"\'"fo'obar" 'don'\''t'|"foo\ x\x\\"|\|'"fo'|obar"|'don'|\|''|t'|
+'foo\ bar'|'foo\ bar'|
+'foo\\ bar'|'foo\\ bar'|
+foo\ bar|foo|\|bar|
+foo#bar\nbaz|foobaz|
+:-) ;-)|:|-|)|;|-|)|
+áéíóú|á|é|í|ó|ú|
+"""
+
+posix_data = r"""x|x|
+foo bar|foo|bar|
+ foo bar|foo|bar|
+ foo bar |foo|bar|
+foo   bar    bla     fasel|foo|bar|bla|fasel|
+x y  z              xxxx|x|y|z|xxxx|
+\x bar|x|bar|
+\ x bar| x|bar|
+\ bar| bar|
+foo \x bar|foo|x|bar|
+foo \ x bar|foo| x|bar|
+foo \ bar|foo| bar|
+foo "bar" bla|foo|bar|bla|
+"foo" "bar" "bla"|foo|bar|bla|
+"foo" bar "bla"|foo|bar|bla|
+"foo" bar bla|foo|bar|bla|
+foo 'bar' bla|foo|bar|bla|
+'foo' 'bar' 'bla'|foo|bar|bla|
+'foo' bar 'bla'|foo|bar|bla|
+'foo' bar bla|foo|bar|bla|
+blurb foo"bar"bar"fasel" baz|blurb|foobarbarfasel|baz|
+blurb foo'bar'bar'fasel' baz|blurb|foobarbarfasel|baz|
+""||
+''||
+foo "" bar|foo||bar|
+foo '' bar|foo||bar|
+foo "" "" "" bar|foo||||bar|
+foo '' '' '' bar|foo||||bar|
+\"|"|
+"\""|"|
+"foo\ bar"|foo\ bar|
+"foo\\ bar"|foo\ bar|
+"foo\\ bar\""|foo\ bar"|
+"foo\\" bar\"|foo\|bar"|
+"foo\\ bar\" dfadf"|foo\ bar" dfadf|
+"foo\\\ bar\" dfadf"|foo\\ bar" dfadf|
+"foo\\\x bar\" dfadf"|foo\\x bar" dfadf|
+"foo\x bar\" dfadf"|foo\x bar" dfadf|
+\'|'|
+'foo\ bar'|foo\ bar|
+'foo\\ bar'|foo\\ bar|
+"foo\\\x bar\" df'a\ 'df"|foo\\x bar" df'a\ 'df|
+\"foo|"foo|
+\"foo\x|"foox|
+"foo\x"|foo\x|
+"foo\ "|foo\ |
+foo\ xx|foo xx|
+foo\ x\x|foo xx|
+foo\ x\x\"|foo xx"|
+"foo\ x\x"|foo\ x\x|
+"foo\ x\x\\"|foo\ x\x\|
+"foo\ x\x\\""foobar"|foo\ x\x\foobar|
+"foo\ x\x\\"\'"foobar"|foo\ x\x\'foobar|
+"foo\ x\x\\"\'"fo'obar"|foo\ x\x\'fo'obar|
+"foo\ x\x\\"\'"fo'obar" 'don'\''t'|foo\ x\x\'fo'obar|don't|
+"foo\ x\x\\"\'"fo'obar" 'don'\''t' \\|foo\ x\x\'fo'obar|don't|\|
+'foo\ bar'|foo\ bar|
+'foo\\ bar'|foo\\ bar|
+foo\ bar|foo bar|
+foo#bar\nbaz|foo|baz|
+:-) ;-)|:-)|;-)|
+áéíóú|áéíóú|
+"""
+
+class ShlexTest(unittest.TestCase):
+    def setUp(self):
+        self.data = [x.split("|")[:-1]
+                     for x in data.splitlines()]
+        self.posix_data = [x.split("|")[:-1]
+                           for x in posix_data.splitlines()]
+        for item in self.data:
+            item[0] = item[0].replace(r"\n", "\n")
+        for item in self.posix_data:
+            item[0] = item[0].replace(r"\n", "\n")
+
+    def splitTest(self, data, posix, spaces):
+        for i in range(len(data)):
+            l = shlex.split(data[i][0], posix=posix, spaces=spaces)
+            self.assertEqual(l, data[i][1:],
+                             "%s: %s != %s" %
+                             (data[i][0], l, data[i][1:]))
+
+    def oldSplit(self, s):
+        ret = []
+        lex = shlex.shlex(StringIO(s))
+        tok = lex.get_token()
+        while tok:
+            ret.append(tok)
+            tok = lex.get_token()
+        return ret
+    
+    def testSplit(self):
+        """Test data splitting with non-posix parser"""
+        self.splitTest(self.data, posix=0, spaces=0) 
+
+    def testSplitPosix(self):
+        """Test data splitting with posix parser"""
+        self.splitTest(self.posix_data, posix=1, spaces=1) 
+
+    def testCompat(self):
+        """Test compatibility interface"""
+        for i in range(len(self.data)):
+            l = self.oldSplit(self.data[i][0])
+            self.assertEqual(l, self.data[i][1:],
+                             "%s: %s != %s" %
+                             (self.data[i][0], l, self.data[i][1:]))
+
+# Allow this test to be used with old shlex.py
+if not getattr(shlex, "split", None):
+    for methname in dir(ShlexTest):
+        if methname.startswith("test") and methname != "testCompat":
+            delattr(ShlexTest, methname)
+
+if __name__ == "__main__":
+    unittest.main()
diff --git a/Misc/NEWS b/Misc/NEWS
index 7efea39..7a40c52 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -155,6 +155,10 @@ Library
 
 - New csv package makes it easy to read/write CSV files.
 
+- Module shlex has been extended to allow posix-like shell parsings,
+  including a split() function for easy spliting of quoted strings and
+  commands. An iterator interface was also implemented.
+
 Tools/Demos
 -----------