* generic/tclBasic.c (Tcl_CreateInterp, Tcl_EvalTokensStandard,

	EvalTokensStandard, Tcl_EvalEx, EvalEx, TclAdvanceContinuations,
	TclEvalObjEx):
	* generic/tclCmdMZ.c (Tcl_SwitchObjCmd, ListLines):
	* generic/tclCompCmds.c (*):
	* generic/tclCompile.c (TclSetByteCodeFromAny, TclInitCompileEnv,
	TclFreeCompileEnv, TclCompileScript):
	* generic/tclCompile.h (CompileEnv):
	* generic/tclInt.h (ContLineLoc, Interp):
	* generic/tclObj.c (ThreadSpecificData, ContLineLocFree,
	TclThreadFinalizeObjects, TclInitObjSubsystem,
	TclContinuationsEnter, TclContinuationsEnterDerived,
	TclContinuationsCopy, TclContinuationsGet, TclFreeObj):
	* generic/tclProc.c (TclCreateProc):
	* generic/tclVar.c (TclPtrSetVar):
	* tests/info.test (info-30.0-22):

	Extended parser, compiler, and execution with code and attendant
	data structures tracking the positions of continuation lines which
	are not visible in script's, to properly account for them while
	counting lines for #280, during direct and compiled execution.

1 files changed, 170 insertions, 13 deletions

diff --git a/generic/tclCompile.c b/generic/tclCompile.c
index b6d486e..aa25f26 100644
--- a/generic/tclCompile.c
+++ b/generic/tclCompile.c
@@ -11,7 +11,7 @@
  * See the file "license.terms" for information on usage and redistribution
  * of this file, and for a DISCLAIMER OF ALL WARRANTIES.
  *
- * RCS: @(#) $Id: tclCompile.c,v 1.43.2.15 2009/07/14 16:31:49 andreas_kupries Exp $
+ * RCS: @(#) $Id: tclCompile.c,v 1.43.2.16 2009/08/25 20:59:10 andreas_kupries Exp $
  */
 
 #include "tclInt.h"
@@ -307,7 +307,7 @@ static int		SetByteCodeFromAny _ANSI_ARGS_((Tcl_Interp *interp,
 static void		EnterCmdWordData _ANSI_ARGS_((
     			    ExtCmdLoc *eclPtr, int srcOffset, Tcl_Token* tokenPtr,
 			    CONST char* cmd, int len, int numWords, int line,
-			    int** lines));
+			    int* clNext, int** lines, CompileEnv* envPtr));
 #endif
 
 
@@ -367,7 +367,9 @@ TclSetByteCodeFromAny(interp, objPtr, hookProc, clientData)
     register int i;
     int length, nested, result;
     char *string;
-
+#ifdef TCL_TIP280
+    ContLineLoc* clLocPtr;
+#endif
 #ifdef TCL_COMPILE_DEBUG
     if (!traceInitialized) {
         if (Tcl_LinkVar(interp, "tcl_traceCompile",
@@ -396,6 +398,24 @@ TclSetByteCodeFromAny(interp, objPtr, hookProc, clientData)
 
     TclInitCompileEnv(interp, &compEnv, string, length,
 		      iPtr->invokeCmdFramePtr, iPtr->invokeWord);
+    /*
+     * Now we check if we have data about invisible continuation lines for the
+     * script, and make it available to the compile environment, if so.
+     *
+     * It is not clear if the script Tcl_Obj* can be free'd while the compiler
+     * is using it, leading to the release of the associated ContLineLoc
+     * structure as well. To ensure that the latter doesn't happen we set a
+     * lock on it. We release this lock in the function TclFreeCompileEnv (),
+     * found in this file. The "lineCLPtr" hashtable is managed in the file
+     * "tclObj.c".
+     */
+
+    clLocPtr = TclContinuationsGet (objPtr);
+    if (clLocPtr) {
+	compEnv.clLoc  = clLocPtr;
+	compEnv.clNext = &compEnv.clLoc->loc[0];
+	Tcl_Preserve (compEnv.clLoc);
+    }
 #endif
     result = TclCompileScript(interp, string, length, nested, &compEnv);
 
@@ -872,6 +892,15 @@ TclInitCompileEnv(interp, envPtr, string, numBytes, invoker, word)
 
 	/* ctx going out of scope */
     }
+
+    /*
+     * Initialize the data about invisible continuation lines as empty,
+     * i.e. not used. The caller (TclSetByteCodeFromAny) will set this up, if
+     * such data is available.
+     */
+
+    envPtr->clLoc  = NULL;
+    envPtr->clNext = NULL;
 #endif
 
     envPtr->auxDataArrayPtr = envPtr->staticAuxDataArraySpace;
@@ -921,6 +950,17 @@ TclFreeCompileEnv(envPtr)
     if (envPtr->mallocedAuxDataArray) {
 	ckfree((char *) envPtr->auxDataArrayPtr);
     }
+#ifdef TCL_TIP280
+    /*
+     * If we used data about invisible continuation lines, then now is the
+     * time to release on our hold on it. The lock was set in function
+     * TclSetByteCodeFromAny(), found in this file.
+     */
+
+    if (envPtr->clLoc) {
+	Tcl_Release (envPtr->clLoc);
+    }
+#endif
 }
 
 #ifdef TCL_TIP280
@@ -1030,6 +1070,7 @@ TclCompileScript(interp, script, numBytes, nested, envPtr)
     ExtCmdLoc* eclPtr = envPtr->extCmdMapPtr;
     int* wlines;
     int  wlineat, cmdLine;
+    int* clNext;
 #endif
 
     Tcl_DStringInit(&ds);
@@ -1050,6 +1091,7 @@ TclCompileScript(interp, script, numBytes, nested, envPtr)
     gotParse = 0;
 #ifdef TCL_TIP280
     cmdLine = envPtr->line;
+    clNext  = envPtr->clNext;
 #endif
 
     do {
@@ -1169,10 +1211,13 @@ TclCompileScript(interp, script, numBytes, nested, envPtr)
 	     * 'wlines'.
 	     */
 
-	    TclAdvanceLines (&cmdLine, p, parse.commandStart);
+	    TclAdvanceLines         (&cmdLine, p, parse.commandStart);
+	    TclAdvanceContinuations (&cmdLine, &clNext,
+				     parse.commandStart - envPtr->source);
 	    EnterCmdWordData (eclPtr, (parse.commandStart - envPtr->source),
-			      parse.tokenPtr, parse.commandStart, parse.commandSize,
-			      parse.numWords, cmdLine, &wlines);
+			      parse.tokenPtr, parse.commandStart,
+			      parse.commandSize, parse.numWords,
+			      cmdLine, clNext, &wlines, envPtr);
 	    wlineat = eclPtr->nuloc - 1;
 #endif
 
@@ -1180,7 +1225,8 @@ TclCompileScript(interp, script, numBytes, nested, envPtr)
 		    wordIdx < parse.numWords;
 		    wordIdx++, tokenPtr += (tokenPtr->numComponents + 1)) {
 #ifdef TCL_TIP280
-	        envPtr->line = eclPtr->loc [wlineat].line [wordIdx];
+	        envPtr->line   = eclPtr->loc [wlineat].line [wordIdx];
+		envPtr->clNext = eclPtr->loc [wlineat].next [wordIdx];
 #endif
 		if (tokenPtr->type == TCL_TOKEN_SIMPLE_WORD) {
 		    /*
@@ -1268,6 +1314,13 @@ TclCompileScript(interp, script, numBytes, nested, envPtr)
 
 			objIndex = TclRegisterNewLiteral(envPtr,
 				tokenPtr[1].start, tokenPtr[1].size);
+#ifdef TCL_TIP280
+			if (envPtr->clNext) {
+			    TclContinuationsEnterDerived (envPtr->literalArrayPtr[objIndex].objPtr,
+							  tokenPtr[1].start - envPtr->source,
+							  eclPtr->loc [wlineat].next [wordIdx]);
+			}
+#endif
 		    }
 		    TclEmitPush(objIndex, envPtr);
 		} else {
@@ -1320,7 +1373,9 @@ TclCompileScript(interp, script, numBytes, nested, envPtr)
 	     * the reduced form now
 	     */
 	    ckfree ((char*) eclPtr->loc [wlineat].line);
-	    eclPtr->loc [wlineat].line    = wlines;
+	    ckfree ((char*) eclPtr->loc [wlineat].next);
+	    eclPtr->loc [wlineat].line = wlines;
+	    eclPtr->loc [wlineat].next = NULL;
 #endif
 	} /* end if parse.numWords > 0 */
 
@@ -1333,7 +1388,8 @@ TclCompileScript(interp, script, numBytes, nested, envPtr)
 	p = next;
 #ifdef TCL_TIP280
 	/* TIP #280 : Track lines in the just compiled command */
-	TclAdvanceLines (&cmdLine, parse.commandStart, p);
+	TclAdvanceLines         (&cmdLine, parse.commandStart, p);
+	TclAdvanceContinuations (&cmdLine, &clNext, p - envPtr->source);
 #endif
 	Tcl_FreeParse(&parse);
 	gotParse = 0;
@@ -1440,6 +1496,43 @@ TclCompileTokens(interp, tokenPtr, count, envPtr)
     int numObjsToConcat, nameBytes, localVarName, localVar;
     int length, i, code;
     unsigned char *entryCodeNext = envPtr->codeNext;
+#ifdef TCL_TIP280
+#define NUM_STATIC_POS 20
+    int isLiteral, maxNumCL, numCL;
+    int* clPosition;
+
+    /*
+     * For the handling of continuation lines in literals we first check if
+     * this is actually a literal. For if not we can forego the additional
+     * processing. Otherwise we pre-allocate a small table to store the
+     * locations of all continuation lines we find in this literal, if
+     * any. The table is extended if needed.
+     *
+     * Note: Different to the equivalent code in function
+     * 'EvalTokensStandard()' (see file "tclBasic.c") we do not seem to need
+     * the 'adjust' variable. We also do not seem to need code which merges
+     * continuation line information of multiple words which concat'd at
+     * runtime. Either that or I have not managed to find a test case for
+     * these two possibilities yet. It might be a difference between compile-
+     * versus runtime processing.
+     */
+
+    numCL     = 0;
+    maxNumCL  = 0;
+    isLiteral = 1;
+    for (i=0 ; i < count; i++) {
+	if ((tokenPtr[i].type != TCL_TOKEN_TEXT) &&
+	    (tokenPtr[i].type != TCL_TOKEN_BS)) {
+	    isLiteral = 0;
+	    break;
+	}
+    }
+
+    if (isLiteral) {
+	maxNumCL   = NUM_STATIC_POS;
+	clPosition = (int*) ckalloc (maxNumCL*sizeof(int));
+    }
+#endif
 
     Tcl_DStringInit(&textBuffer);
     numObjsToConcat = 0;
@@ -1454,6 +1547,38 @@ TclCompileTokens(interp, tokenPtr, count, envPtr)
 		length = Tcl_UtfBackslash(tokenPtr->start, (int *) NULL,
 			buffer);
 		Tcl_DStringAppend(&textBuffer, buffer, length);
+
+#ifdef TCL_TIP280
+		/*
+		 * If the backslash sequence we found is in a literal, and
+		 * represented a continuation line, we compute and store its
+		 * location (as char offset to the beginning of the _result_
+		 * script). We may have to extend the table of locations.
+		 *
+		 * Note that the continuation line information is relevant
+		 * even if the word we are processing is not a literal, as it
+		 * can affect nested commands. See the branch for
+		 * TCL_TOKEN_COMMAND below, where the adjustment we are
+		 * tracking here is taken into account. The good thing is that
+		 * we do not need a table of everything, just the number of
+		 * lines we have to add as correction.
+		 */
+
+		if ((length == 1) && (buffer[0] == ' ') &&
+		    (tokenPtr->start[1] == '\n')) {
+		    if (isLiteral) {
+			int clPos = Tcl_DStringLength (&textBuffer);
+
+			if (numCL >= maxNumCL) {
+			    maxNumCL *= 2;
+			    clPosition = (int*) ckrealloc ((char*)clPosition,
+							   maxNumCL*sizeof(int));
+			}
+			clPosition[numCL] = clPos;
+			numCL ++;
+		    }
+		}
+#endif
 		break;
 
 	    case TCL_TOKEN_COMMAND:
@@ -1470,6 +1595,13 @@ TclCompileTokens(interp, tokenPtr, count, envPtr)
 		    TclEmitPush(literal, envPtr);
 		    numObjsToConcat++;
 		    Tcl_DStringFree(&textBuffer);
+#ifdef TCL_TIP280
+		    if (numCL) {
+			TclContinuationsEnter(envPtr->literalArrayPtr[literal].objPtr,
+					      numCL, clPosition);
+		    }
+		    numCL = 0;
+#endif
 		}
 		
 		code = TclCompileScript(interp, tokenPtr->start+1,
@@ -1594,6 +1726,14 @@ TclCompileTokens(interp, tokenPtr, count, envPtr)
 	        Tcl_DStringLength(&textBuffer), /*onHeap*/ 0);
 	TclEmitPush(literal, envPtr);
 	numObjsToConcat++;
+
+#ifdef TCL_TIP280
+	if (numCL) {
+	    TclContinuationsEnter(envPtr->literalArrayPtr[literal].objPtr,
+				  numCL, clPosition);
+	}
+	numCL = 0;
+#endif
     }
 
     /*
@@ -1616,11 +1756,20 @@ TclCompileTokens(interp, tokenPtr, count, envPtr)
 	TclEmitPush(TclRegisterLiteral(envPtr, "", 0, /*onHeap*/ 0),
 	        envPtr);
     }
-    Tcl_DStringFree(&textBuffer);
-    return TCL_OK;
+    code = TCL_OK;
 
     error:
     Tcl_DStringFree(&textBuffer);
+#ifdef TCL_TIP280
+    /*
+     * Release the temp table we used to collect the locations of
+     * continuation lines, if any.
+     */
+
+    if (maxNumCL) {
+	ckfree ((char*) clPosition);
+    }
+#endif
     return code;
 }
 
@@ -2426,7 +2575,7 @@ EnterCmdExtentData(envPtr, cmdIndex, numSrcBytes, numCodeBytes)
  */
 
 static void
-EnterCmdWordData(eclPtr, srcOffset, tokenPtr, cmd, len, numWords, line, wlines)
+EnterCmdWordData(eclPtr, srcOffset, tokenPtr, cmd, len, numWords, line, clNext, wlines, envPtr)
     ExtCmdLoc *eclPtr;		/* Points to the map environment
 				 * structure in which to enter command
 				 * location information. */
@@ -2436,12 +2585,15 @@ EnterCmdWordData(eclPtr, srcOffset, tokenPtr, cmd, len, numWords, line, wlines)
     int         len;
     int numWords;
     int line;
+    int* clNext;
     int** wlines;
+    CompileEnv* envPtr;
 {    
     ECL*        ePtr;
     int         wordIdx;
     CONST char* last;
     int         wordLine;
+    int*        wordNext;
     int*        wwlines;
 
     if (eclPtr->nuloc >= eclPtr->nloc) {
@@ -2475,19 +2627,24 @@ EnterCmdWordData(eclPtr, srcOffset, tokenPtr, cmd, len, numWords, line, wlines)
     ePtr            = &eclPtr->loc [eclPtr->nuloc];
     ePtr->srcOffset = srcOffset;
     ePtr->line      = (int*) ckalloc (numWords * sizeof (int));
+    ePtr->next      = (int**) ckalloc (numWords * sizeof (int*));
     ePtr->nline     = numWords;
     wwlines         = (int*) ckalloc (numWords * sizeof (int));
 
     last     = cmd;
     wordLine = line;
+    wordNext = clNext;
     for (wordIdx = 0;
 	 wordIdx < numWords;
 	 wordIdx++, tokenPtr += (tokenPtr->numComponents + 1)) {
-        TclAdvanceLines (&wordLine, last, tokenPtr->start);
+        TclAdvanceLines         (&wordLine, last, tokenPtr->start);
+	TclAdvanceContinuations (&wordLine, &wordNext,
+				 tokenPtr->start - envPtr->source);
 	wwlines    [wordIdx] = (TclWordKnownAtCompileTime (tokenPtr)
 				? wordLine
 				: -1);
 	ePtr->line [wordIdx] = wordLine;
+	ePtr->next [wordIdx] = wordNext;
 	last = tokenPtr->start;
     }