Revised TclFindElement() interface.

The final argument had been bracePtr, the address of a boolean var, where
the caller can be told whether or not the parsed list element was enclosed
in braces.  In practice, no callers really care about that.  What the callers
really want to know is whether the list element value exists as a literal
substring of the string being parsed, or whether a call to
TclCopyAndCollpase() is needed to produce the list element value.  Now the
final argument is changed to do what callers actually need.  This is a
better fit for the calls in tclParse.c, where now a good deal of
post-processing checking for "naked backslashes" is no longer necessary.
***POTENTIAL INCOMPATIBILITY***
For any callers calling in via the internal stubs table who really do use
the final argument explicitly to check for the enclosing brace scenario.
Simply looking for the braces where they must be is the revision available
to those callers, and it will backport cleanly.
Tests for expanded literals quoting detection.

7 files changed, 88 insertions, 44 deletions

diff --git a/ChangeLog b/ChangeLog
index f7e8a0d..831fb5c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,5 +1,25 @@
 2011-05-02  Don Porter  <dgp@users.sourceforge.net>
 
+	* generic/tclCmdMZ.c:	Revised TclFindElement() interface.  The
+	* generic/tclDictObj.c:	final argument had been bracePtr, the address
+	* generic/tclListObj.c:	of a boolean var, where the caller can be told
+	* generic/tclParse.c:	whether or not the parsed list element was
+	* generic/tclUtil.c:	enclosed in braces.  In practice, no callers
+	really care about that.  What the callers really want to know is 
+	whether the list element value exists as a literal substring of the
+	string being parsed, or whether a call to TclCopyAndCollpase() is
+	needed to produce the list element value.  Now the final argument
+	is changed to do what callers actually need.  This is a better fit
+	for the calls in tclParse.c, where now a good deal of post-processing
+	checking for "naked backslashes" is no longer necessary.
+	***POTENTIAL INCOMPATIBILITY***
+	For any callers calling in via the internal stubs table who really
+	do use the final argument explicitly to check for the enclosing brace
+	scenario.  Simply looking for the braces where they must be is the
+	revision available to those callers, and it will backport cleanly.
+
+	* tests/parse.test:	Tests for expanded literals quoting detection.
+
 	* generic/tclInt.h:	Replace TclCountSpaceRuns() with
 	* generic/tclListObj.c:	TclMaxListLength() which is the function we
 	* generic/tclUtil.c:	actually want.
diff --git a/generic/tclCmdMZ.c b/generic/tclCmdMZ.c
index c34ce6c..e7c7152 100644
--- a/generic/tclCmdMZ.c
+++ b/generic/tclCmdMZ.c
@@ -1643,7 +1643,7 @@ StringIsCmd(
 	     */
 
 	    const char *elemStart, *nextElem;
-	    int lenRemain, elemSize, hasBrace;
+	    int lenRemain, elemSize;
 	    register const char *p;
 
 	    string1 = TclGetStringFromObj(objPtr, &length1);
@@ -1652,7 +1652,7 @@ StringIsCmd(
 	    for (p=string1, lenRemain=length1; lenRemain > 0;
 		    p=nextElem, lenRemain=end-nextElem) {
 		if (TCL_ERROR == TclFindElement(NULL, p, lenRemain,
-			&elemStart, &nextElem, &elemSize, &hasBrace)) {
+			&elemStart, &nextElem, &elemSize, NULL)) {
 		    Tcl_Obj *tmpStr;
 
 		    /*
diff --git a/generic/tclDictObj.c b/generic/tclDictObj.c
index 508c2af..b33bb31 100644
--- a/generic/tclDictObj.c
+++ b/generic/tclDictObj.c
@@ -567,7 +567,7 @@ SetDictFromAny(
     const char *string;
     char *s;
     const char *elemStart, *nextElem;
-    int lenRemain, length, elemSize, hasBrace, result, isNew;
+    int lenRemain, length, elemSize, result, isNew;
     const char *limit;	/* Points just after string's last byte. */
     register const char *p;
     register Tcl_Obj *keyPtr, *valuePtr;
@@ -649,8 +649,10 @@ SetDictFromAny(
     for (p = string, lenRemain = length;
 	    lenRemain > 0;
 	    p = nextElem, lenRemain = (limit - nextElem)) {
+	int literal;
+
 	result = TclFindElement(interp, p, lenRemain,
-		&elemStart, &nextElem, &elemSize, &hasBrace);
+		&elemStart, &nextElem, &elemSize, &literal);
 	if (result != TCL_OK) {
 	    if (interp != NULL) {
 		Tcl_SetErrorCode(interp, "TCL", "VALUE", "DICTIONARY", NULL);
@@ -667,7 +669,7 @@ SetDictFromAny(
 	 */
 
 	s = ckalloc(elemSize + 1);
-	if (hasBrace) {
+	if (literal) {
 	    memcpy(s, elemStart, (size_t) elemSize);
 	    s[elemSize] = 0;
 	} else {
@@ -685,7 +687,7 @@ SetDictFromAny(
 	}
 
 	result = TclFindElement(interp, p, lenRemain,
-		&elemStart, &nextElem, &elemSize, &hasBrace);
+		&elemStart, &nextElem, &elemSize, &literal);
 	if (result != TCL_OK) {
 	    if (interp != NULL) {
 		Tcl_SetErrorCode(interp, "TCL", "VALUE", "DICTIONARY", NULL);
@@ -703,7 +705,7 @@ SetDictFromAny(
 	 */
 
 	s = ckalloc(elemSize + 1);
-	if (hasBrace) {
+	if (literal) {
 	    memcpy(s, elemStart, (size_t) elemSize);
 	    s[elemSize] = 0;
 	} else {
diff --git a/generic/tclListObj.c b/generic/tclListObj.c
index 03843b8..f330937 100644
--- a/generic/tclListObj.c
+++ b/generic/tclListObj.c
@@ -1717,7 +1717,7 @@ SetListFromAny(
     const char *string;
     char *s;
     const char *elemStart, *nextElem;
-    int lenRemain, length, estCount, elemSize, hasBrace, i, j, result;
+    int lenRemain, length, estCount, elemSize, i, j, result;
     const char *limit;		/* Points just after string's last byte. */
     register const char *p;
     register Tcl_Obj **elemPtrs;
@@ -1801,8 +1801,10 @@ SetListFromAny(
     for (p=string, lenRemain=length, i=0;
 	    lenRemain > 0;
 	    p=nextElem, lenRemain=limit-nextElem, i++) {
+	int literal;
+
 	result = TclFindElement(interp, p, lenRemain, &elemStart, &nextElem,
-		&elemSize, &hasBrace);
+		&elemSize, &literal);
 	if (result != TCL_OK) {
 	    for (j = 0;  j < i;  j++) {
 		elemPtr = elemPtrs[j];
@@ -1827,7 +1829,7 @@ SetListFromAny(
 	 */
 
 	s = ckalloc(elemSize + 1);
-	if (hasBrace) {
+	if (literal) {
 	    memcpy(s, elemStart, (size_t) elemSize);
 	    s[elemSize] = 0;
 	} else {
diff --git a/generic/tclParse.c b/generic/tclParse.c
index 429d33a..4afb219 100644
--- a/generic/tclParse.c
+++ b/generic/tclParse.c
@@ -433,7 +433,7 @@ Tcl_ParseCommand(
 	    }
 
 	    if (isLiteral) {
-		int elemCount = 0, code = TCL_OK, nakedbs = 0;
+		int elemCount = 0, code = TCL_OK, literal = 1;
 		const char *nextElem, *listEnd, *elemStart;
 
 		/*
@@ -455,35 +455,26 @@ Tcl_ParseCommand(
 		 */
 
 		while (nextElem < listEnd) {
-		    int size, brace;
+		    int size;
 
 		    code = TclFindElement(NULL, nextElem, listEnd - nextElem,
-			    &elemStart, &nextElem, &size, &brace);
-		    if (code != TCL_OK) {
+			    &elemStart, &nextElem, &size, &literal);
+		    if ((code != TCL_OK) || !literal) {
 			break;
 		    }
-		    if (!brace) {
-			const char *s;
-
-			for(s=elemStart;size>0;s++,size--) {
-			    if ((*s)=='\\') {
-				nakedbs = 1;
-				break;
-			    }
-			}
-		    }
 		    if (elemStart < listEnd) {
 			elemCount++;
 		    }
 		}
 
-		if ((code != TCL_OK) || nakedbs) {
+		if ((code != TCL_OK) || !literal) {
 		    /*
-		     * Some list element could not be parsed, or contained
-		     * naked backslashes. This means the literal string was
-		     * not in fact a valid nor canonical list. Defer the
-		     * handling of this to compile/eval time, where code is
-		     * already in place to report the "attempt to expand a
+		     * Some list element could not be parsed, or is not
+		     * present as a literal substring of the script.  The
+		     * compiler cannot handle list elements that get generated
+		     * by a call to TclCopyAndCollapse(). Defer  the
+		     * handling of  this to  compile/eval time, where  code is
+		     * already  in place to  report the  "attempt to  expand a
 		     * non-list" error or expand lists that require
 		     * substitution.
 		     */
@@ -505,6 +496,7 @@ Tcl_ParseCommand(
 		     * tokens representing the expanded list.
 		     */
 
+		    CONST char *listStart;
 		    int growthNeeded = wordIndex + 2*elemCount
 			    - parsePtr->numTokens;
 
@@ -524,9 +516,9 @@ Tcl_ParseCommand(
 		     * word value.
 		     */
 
-		    nextElem = tokenPtr[1].start;
+		    listStart = nextElem = tokenPtr[1].start;
 		    while (nextElem < listEnd) {
-			int quoted, brace;
+			int quoted;
 	
 			tokenPtr->type = TCL_TOKEN_SIMPLE_WORD;
 			tokenPtr->numComponents = 1;
@@ -536,9 +528,11 @@ Tcl_ParseCommand(
 			tokenPtr->numComponents = 0;
 			TclFindElement(NULL, nextElem, listEnd - nextElem,
 				&(tokenPtr->start), &nextElem,
-				&(tokenPtr->size), &brace);
+				&(tokenPtr->size), NULL);
 
-			quoted = brace || tokenPtr->start[-1] == '"';
+			quoted = (tokenPtr->start[-1] == '{'
+				|| tokenPtr->start[-1] == '"')
+				&& tokenPtr->start > listStart;
 			tokenPtr[-1].start = tokenPtr->start - quoted;
 			tokenPtr[-1].size = tokenPtr->start + tokenPtr->size
 				- tokenPtr[-1].start + quoted;
diff --git a/generic/tclUtil.c b/generic/tclUtil.c
index d4630af..7baadff 100644
--- a/generic/tclUtil.c
+++ b/generic/tclUtil.c
@@ -180,8 +180,13 @@ TclMaxListLength(
  *	after the opening brace and *sizePtr will not include either of the
  *	braces. If there isn't an element in the list, *sizePtr will be zero,
  *	and both *elementPtr and *termPtr will point just after the last
- *	character in the list. Note: this function does NOT collapse backslash
- *	sequences.
+ *	character in the list. If literalPtr is non-NULL, *literalPtr is set
+ *	to a boolean value indicating whether the substring returned as
+ *	the values of **elementPtr and *sizePtr is the literal value of
+ *	a list element.  If not, a call to TclCopyAndCollapse() is needed
+ *	to produce the actual value of the list element.  Note: this function
+ *	does NOT collapse backslash sequences, but uses *literalPtr to tell
+ * 	callers when it is required for them to do so.
  *
  * Side effects:
  *	None.
@@ -205,8 +210,12 @@ TclFindElement(
 				 * argument (next arg or end of list). */
     int *sizePtr,		/* If non-zero, fill in with size of
 				 * element. */
-    int *bracePtr)		/* If non-zero, fill in with non-zero/zero to
-				 * indicate that arg was/wasn't in braces. */
+    int *literalPtr)		/* If non-zero, fill in with non-zero/zero to
+				 * indicate that the substring of *sizePtr
+				 * bytes starting at **elementPtr is/is not
+				 * the literal list element and therefore
+				 * does not/does require a call to 
+				 * TclCopyAndCollapse() by the caller. */
 {
     const char *p = list;
     const char *elemStart;	/* Points to first byte of first element. */
@@ -215,6 +224,7 @@ TclFindElement(
     int inQuotes = 0;
     int size = 0;		/* lint. */
     int numChars;
+    int literal = 1;
     const char *p2;
 
     /*
@@ -240,9 +250,6 @@ TclFindElement(
 	p++;
     }
     elemStart = p;
-    if (bracePtr != 0) {
-	*bracePtr = openBraces;
-    }
 
     /*
      * Find element's end (a space, close brace, or the end of the string).
@@ -302,6 +309,15 @@ TclFindElement(
 	     */
 
 	case '\\':
+	    if (openBraces == 0) {
+		/*
+		 * A backslash sequence not within a brace quoted element
+		 * means the value of the element is different from the
+		 * substring we are parsing.  A call to TclCopyAndCollapse()
+		 * is needed to produce the element value.  Inform the caller.
+		 */
+		literal = 0;
+	    }
 	    TclParseBackslash(p, limit - p, &numChars, NULL);
 	    p += (numChars - 1);
 	    break;
@@ -392,6 +408,9 @@ TclFindElement(
     if (sizePtr != 0) {
 	*sizePtr = size;
     }
+    if (literalPtr != 0) {
+	*literalPtr = literal;
+    }
     return TCL_OK;
 }
 
@@ -484,7 +503,7 @@ Tcl_SplitList(
 {
     const char **argv, *end, *element;
     char *p;
-    int length, size, i, result, elSize, brace;
+    int length, size, i, result, elSize;
 
     /*
      * Allocate enough space to work in. A (CONST char *) for each
@@ -502,9 +521,10 @@ Tcl_SplitList(
     for (i = 0, p = ((char *) argv) + size*sizeof(char *);
 	    *list != 0;  i++) {
 	const char *prevList = list;
+	int literal;
 
 	result = TclFindElement(interp, list, length, &element, &list,
-		&elSize, &brace);
+		&elSize, &literal);
 	length -= (list - prevList);
 	if (result != TCL_OK) {
 	    ckfree(argv);
@@ -524,7 +544,7 @@ Tcl_SplitList(
 	    return TCL_ERROR;
 	}
 	argv[i] = p;
-	if (brace) {
+	if (literal) {
 	    memcpy(p, element, (size_t) elSize);
 	    p += elSize;
 	    *p = 0;
diff --git a/tests/parse.test b/tests/parse.test
index 21d8779..3523975 100644
--- a/tests/parse.test
+++ b/tests/parse.test
@@ -233,6 +233,12 @@ test parse-5.28 {Tcl_ParseCommand: {*} parsing, expanded literals} testparser {
 test parse-5.29 {Tcl_ParseCommand: {*} parsing, expanded literals, naked backslashes} testparser {
     testparser {{*}{a \n b}} 0
 } {- {{*}{a \n b}} 1 expand {{*}{a \n b}} 1 text {a \n b} 0 {}}
+test parse-5.30 {Tcl_ParseCommand: {*} parsing, expanded literals} testparser {
+    testparser {{*}"a b"} 0
+} {- {{*}"a b"} 2 simple a 1 text a 0 simple b 1 text b 0 {}}
+test parse-5.31 {Tcl_ParseCommand: {*} parsing, expanded literals, naked backslashes} testparser {
+    testparser {{*}"a \n b"} 0
+} {- {{*}"a \n b"} 1 expand {{*}"a \n b"} 3 text {a } 0 backslash {\n} 0 text { b} 0 {}}
 
 test parse-6.1 {ParseTokens procedure, empty word} testparser {
     testparser {""} 0