diff options
| -rw-r--r-- | doc/Encoding.3 | 8 | ||||
| -rw-r--r-- | doc/chan.n | 9 | ||||
| -rw-r--r-- | doc/encoding.n | 15 | ||||
| -rw-r--r-- | generic/tcl.h | 9 | ||||
| -rw-r--r-- | generic/tclCmdAH.c | 2 | ||||
| -rw-r--r-- | generic/tclEncoding.c | 75 | ||||
| -rw-r--r-- | generic/tclIO.c | 22 | ||||
| -rw-r--r-- | generic/tclInt.h | 1 | ||||
| -rw-r--r-- | library/http/http.tcl | 12 | ||||
| -rw-r--r-- | tests/encodingVectors.tcl | 2 | ||||
| -rw-r--r-- | tests/ioCmd.test | 8 |
11 files changed, 35 insertions, 128 deletions
diff --git a/doc/Encoding.3 b/doc/Encoding.3 index 0d70658..aaf7313 100644 --- a/doc/Encoding.3 +++ b/doc/Encoding.3 @@ -105,9 +105,9 @@ byte is converted and then to reset to an initial state. The \fBTCL_PROFILE_*\fR bits defined in the \fBPROFILES\fR section below control the encoding profile to be used for dealing with invalid data or other errors in the encoding transform. -\fBTCL_ENCODING_STOPONERROR\fR is present for backward compatibility with -Tcl 8.6 and forces the encoding profile to \fBstrict\fR. - +The flag \fBTCL_ENCODING_STOPONERROR\fR has no effect, +it only has meaning in Tcl 8.x. +.PP Some flags bits may not be usable with some functions as noted in the function descriptions below. .AP Tcl_EncodingState *statePtr in/out @@ -590,7 +590,7 @@ with at most one of \fBTCL_ENCODING_PROFILE_TCL8\fR, \fBTCL_ENCODING_PROFILE_STRICT\fR or \fBTCL_ENCODING_PROFILE_REPLACE\fR. These correspond to the \fBtcl8\fR, \fBstrict\fR and \fBreplace\fR profiles respectively. If none are specified, a version-dependent default profile is used. -For Tcl 9.0, the default profile is \fBtcl8\fR. +For Tcl 9.0, the default profile is \fBstrict\fR. .PP For details about profiles, see the \fBPROFILES\fR section in the documentation of the \fBencoding\fR command. @@ -200,16 +200,15 @@ platforms it is \fBcrlf\fR for both input and output. \fBbinary\fR . Like \fBlf\fR, no end-of-line translation is performed, but in addition, sets -\fB\-eofchar\fR to the empty string to disable it, sets \fB\-encoding\fR to -\fBiso8859-1\fR, and sets \fB-profile\fR to \fBstrict\fR so the the channel is -fully configured for binary input and output: Each byte read from the channel +\fB\-eofchar\fR to the empty string to disable it, and sets \fB\-encoding\fR +to \fBiso8859-1\fR. With this one setting, a channel is fully configured +for binary input and output: Each byte read from the channel becomes the Unicode character having the same value as that byte, and each character written to the channel becomes a single byte in the output. This makes it possible to work seamlessly with binary data as long as each character in the data remains in the range of 0 to 255 so that there is no distinction between binary data and text. For example, A JPEG image can be read from a such a channel, manipulated, and then written back to such a channel. - .TP \fBcr\fR . @@ -245,7 +244,7 @@ files to slow destinations like network sockets. .PP \fB\-size\fR limits the number of characters copied. .PP -If \fB\-command\fR is gviven, \fBchan copy\fR returns immediately, works in the +If \fB\-command\fR is given, \fBchan copy\fR returns immediately, works in the background, and calls \fIcallback\fR when the copy completes, providing as an additional argument the number of characters written to \fIoutputChan\fR. If an error occurres during the background copy, another argument provides message diff --git a/doc/encoding.n b/doc/encoding.n index 793348f..255e070 100644 --- a/doc/encoding.n +++ b/doc/encoding.n @@ -111,9 +111,14 @@ encoding. The following profiles are currently implemented. .VS "TCL8.7 TIP656" .TP +\fBstrict\fR +. +The default profile. The operation fails when invalid data for the encoding +are encountered. +.TP \fBtcl8\fR . -The default profile. Provides for behaviour identical to that of Tcl 8.6: When +Provides for behaviour identical to that of Tcl 8.6: When decoding, for encodings \fBother than utf-8\fR, each invalid byte is interpreted as the Unicode value given by that one byte. For example, the byte 0x80, which is invalid in the ASCII encoding would be mapped to the Unicode value U+0080. @@ -127,10 +132,6 @@ an additional special case, the sequence 0xC0 0x80 is mapped to U+0000. When encoding, each character that cannot be represented in the encoding is replaced by an encoding-dependent character, usually the question mark \fB?\fR. .TP -\fBstrict\fR -. -The operation fails when invalid data for the encoding are encountered. -.TP \fBreplace\fR . When decoding, invalid bytes are replaced by U+FFFD, the Unicode REPLACEMENT @@ -180,7 +181,7 @@ unexpected byte sequence starting at index 1: '\ex80' Example 3: Get partial data and the error location: .PP .CS -% codepoints [encoding convertfrom -profile strict -failindex idx ascii AB\ex80] +% codepoints [encoding convertfrom -failindex idx ascii AB\ex80] U+000041 U+000042 % set idx 2 @@ -193,7 +194,7 @@ Example 4: Encode a character that is not representable in ISO8859-1: A? % encoding convertto -profile strict iso8859-1 A\eu0141 unexpected character at index 1: 'U+000141' -% encoding convertto -profile strict -failindex idx iso8859-1 A\eu0141 +% encoding convertto -failindex idx iso8859-1 A\eu0141 A % set idx 1 diff --git a/generic/tcl.h b/generic/tcl.h index b43fcec..45e949b 100644 --- a/generic/tcl.h +++ b/generic/tcl.h @@ -1972,14 +1972,9 @@ typedef struct Tcl_EncodingType { * changes, ensure ENCODING_PROFILE_* macros in tclInt.h are modified if * necessary. */ +#define TCL_ENCODING_PROFILE_STRICT TCL_ENCODING_STOPONERROR #define TCL_ENCODING_PROFILE_TCL8 0x01000000 -#define TCL_ENCODING_PROFILE_STRICT 0x02000000 -#define TCL_ENCODING_PROFILE_REPLACE 0x03000000 -#if TCL_MAJOR_VERSION < 9 -#define TCL_ENCODING_PROFILE_DEFAULT TCL_ENCODING_PROFILE_TCL8 -#else -#define TCL_ENCODING_PROFILE_DEFAULT TCL_ENCODING_PROFILE_TCL8 -#endif +#define TCL_ENCODING_PROFILE_REPLACE 0x02000000 /* * The following definitions are the error codes returned by the conversion diff --git a/generic/tclCmdAH.c b/generic/tclCmdAH.c index bc8bf30..7009110 100644 --- a/generic/tclCmdAH.c +++ b/generic/tclCmdAH.c @@ -435,7 +435,7 @@ EncodingConvertParseOptions ( Tcl_Encoding encoding; Tcl_Obj *dataObj; Tcl_Obj *failVarObj; - int profile = TCL_ENCODING_PROFILE_TCL8; + int profile = TCL_ENCODING_PROFILE_STRICT; /* * Possible combinations: diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index cb252b3..d7f9980 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -199,17 +199,13 @@ static struct TclEncodingProfiles { {"tcl8", TCL_ENCODING_PROFILE_TCL8}, }; #define PROFILE_TCL8(flags_) \ - ((ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_TCL8) \ - || (ENCODING_PROFILE_GET(flags_) == 0 \ - && TCL_ENCODING_PROFILE_DEFAULT == TCL_ENCODING_PROFILE_TCL8)) + (ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_TCL8) + #define PROFILE_STRICT(flags_) \ - ((ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_STRICT) \ - || (ENCODING_PROFILE_GET(flags_) == 0 \ - && TCL_ENCODING_PROFILE_DEFAULT == TCL_ENCODING_PROFILE_STRICT)) + (!PROFILE_TCL8(flags_) && !PROFILE_REPLACE(flags_)) + #define PROFILE_REPLACE(flags_) \ - ((ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_REPLACE) \ - || (ENCODING_PROFILE_GET(flags_) == 0 \ - && TCL_ENCODING_PROFILE_DEFAULT == TCL_ENCODING_PROFILE_REPLACE)) + (ENCODING_PROFILE_GET(flags_) == TCL_ENCODING_PROFILE_REPLACE) #define UNICODE_REPLACE_CHAR ((Tcl_UniChar)0xFFFD) #define SURROGATE(c_) (((c_) & ~0x7FF) == 0xD800) @@ -1170,10 +1166,6 @@ Tcl_ExternalToUtfDString( * Possible flags values: * target encoding. It should be composed by OR-ing the following: * - *At most one* of TCL_ENCODING_PROFILE{DEFAULT,TCL8,STRICT} - * - TCL_ENCODING_STOPONERROR: Backward compatibility. Sets the profile - * to TCL_ENCODING_PROFILE_STRICT overriding any specified profile flags - * Any other flag bits will cause an error to be returned (for future - * compatibility) * * Results: * The return value is one of @@ -1500,8 +1492,6 @@ Tcl_UtfToExternalDString( * the source buffer are invalid or cannot be represented in the * target encoding. It should be composed by OR-ing the following: * - *At most one* of TCL_ENCODING_PROFILE{DEFAULT,TCL8,STRICT} - * - TCL_ENCODING_STOPONERROR: Backward compatibility. Sets the profile - * to TCL_ENCODING_PROFILE_STRICT overriding any specified profile flags * * Results: * The return value is one of @@ -2451,7 +2441,6 @@ BinaryProc( if (dstLen < 0) { dstLen = 0; } - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_CHAR_LIMIT) && srcLen > *dstCharsPtr) { srcLen = *dstCharsPtr; } @@ -2519,7 +2508,6 @@ UtfToUtfProc( srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= 6; } @@ -2748,7 +2736,6 @@ Utf32ToUtfProc( int result, numChars, charLimit = INT_MAX; int ch = 0, bytesLeft = srcLen % 4; - flags = TclEncodingSetProfileFlags(flags); flags |= PTR2INT(clientData); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; @@ -2928,7 +2915,6 @@ UtfToUtf32Proc( srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } @@ -3026,7 +3012,6 @@ Utf16ToUtfProc( int result, numChars, charLimit = INT_MAX; unsigned short ch = 0; - flags = TclEncodingSetProfileFlags(flags); flags |= PTR2INT(clientData); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; @@ -3216,7 +3201,6 @@ UtfToUtf16Proc( srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } @@ -3322,7 +3306,6 @@ UtfToUcs2Proc( int result, numChars, len; Tcl_UniChar ch = 0; - flags = TclEncodingSetProfileFlags(flags); flags |= PTR2INT(clientData); srcStart = src; srcEnd = src + srcLen; @@ -3445,7 +3428,6 @@ TableToUtfProc( const unsigned short *pageZero; TableEncodingData *dataPtr = (TableEncodingData *)clientData; - flags = TclEncodingSetProfileFlags(flags); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; } @@ -3589,7 +3571,6 @@ TableFromUtfProc( srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } @@ -3696,7 +3677,6 @@ Iso88591ToUtfProc( const char *dstEnd, *dstStart; int result, numChars, charLimit = INT_MAX; - flags = TclEncodingSetProfileFlags(flags); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; } @@ -3790,7 +3770,6 @@ Iso88591FromUtfProc( srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } @@ -3938,7 +3917,6 @@ EscapeToUtfProc( int state, result, numChars, charLimit = INT_MAX; const char *dstStart, *dstEnd; - flags = TclEncodingSetProfileFlags(flags); if (flags & TCL_ENCODING_CHAR_LIMIT) { charLimit = *dstCharsPtr; } @@ -4168,7 +4146,6 @@ EscapeFromUtfProc( srcStart = src; srcEnd = src + srcLen; srcClose = srcEnd; - flags = TclEncodingSetProfileFlags(flags); if ((flags & TCL_ENCODING_END) == 0) { srcClose -= TCL_UTF_MAX; } @@ -4617,48 +4594,6 @@ TclEncodingProfileIdToName( /* *------------------------------------------------------------------------ * - * TclEncodingSetProfileFlags -- - * - * Maps the flags supported in the encoding C API's to internal flags. - * - * For backward compatibility reasons, TCL_ENCODING_STOPONERROR is - * is mapped to the TCL_ENCODING_PROFILE_STRICT overwriting any profile - * specified. - * - * If no profile or an invalid profile is specified, it is set to - * the default. - * - * Results: - * Internal encoding flag mask. - * - * Side effects: - * None. - * - *------------------------------------------------------------------------ - */ -int TclEncodingSetProfileFlags(int flags) -{ - if (flags & TCL_ENCODING_STOPONERROR) { - ENCODING_PROFILE_SET(flags, TCL_ENCODING_PROFILE_STRICT); - } else { - int profile = ENCODING_PROFILE_GET(flags); - switch (profile) { - case TCL_ENCODING_PROFILE_TCL8: - case TCL_ENCODING_PROFILE_STRICT: - case TCL_ENCODING_PROFILE_REPLACE: - break; - case 0: /* Unspecified by caller */ - default: - ENCODING_PROFILE_SET(flags, TCL_ENCODING_PROFILE_DEFAULT); - break; - } - } - return flags; -} - -/* - *------------------------------------------------------------------------ - * * TclGetEncodingProfiles -- * * Get the list of supported encoding profiles. diff --git a/generic/tclIO.c b/generic/tclIO.c index 0c91428..6b74af4 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -1680,12 +1680,8 @@ Tcl_CreateChannel( statePtr->encoding = Tcl_GetEncoding(NULL, name); statePtr->inputEncodingState = NULL; statePtr->inputEncodingFlags = TCL_ENCODING_START; - ENCODING_PROFILE_SET(statePtr->inputEncodingFlags, - TCL_ENCODING_PROFILE_DEFAULT); statePtr->outputEncodingState = NULL; statePtr->outputEncodingFlags = TCL_ENCODING_START; - ENCODING_PROFILE_SET(statePtr->outputEncodingFlags, - TCL_ENCODING_PROFILE_DEFAULT); /* * Set the channel up initially in AUTO input translation mode to accept @@ -8167,12 +8163,6 @@ Tcl_SetChannelOption( if ((newValue[0] == '\0') || (strcmp(newValue, "binary") == 0)) { encoding = Tcl_GetEncoding(NULL, "iso8859-1"); - ENCODING_PROFILE_SET(statePtr->inputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->inputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); - ENCODING_PROFILE_SET(statePtr->outputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->outputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); } else { encoding = Tcl_GetEncoding(interp, newValue); if (encoding == NULL) { @@ -8281,12 +8271,6 @@ Tcl_SetChannelOption( statePtr->inEofChar = 0; Tcl_FreeEncoding(statePtr->encoding); statePtr->encoding = Tcl_GetEncoding(NULL, "iso8859-1"); - ENCODING_PROFILE_SET(statePtr->inputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->inputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); - ENCODING_PROFILE_SET(statePtr->outputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->outputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); } else if (strcmp(readMode, "lf") == 0) { translation = TCL_TRANSLATE_LF; } else if (strcmp(readMode, "cr") == 0) { @@ -8336,12 +8320,6 @@ Tcl_SetChannelOption( statePtr->outputTranslation = TCL_TRANSLATE_LF; Tcl_FreeEncoding(statePtr->encoding); statePtr->encoding = Tcl_GetEncoding(NULL, "iso8859-1"); - ENCODING_PROFILE_SET(statePtr->inputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->inputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); - ENCODING_PROFILE_SET(statePtr->outputEncodingFlags - ,ENCODING_PROFILE_GET(statePtr->outputEncodingFlags) - |TCL_ENCODING_PROFILE_STRICT); } else if (strcmp(writeMode, "lf") == 0) { statePtr->outputTranslation = TCL_TRANSLATE_LF; } else if (strcmp(writeMode, "cr") == 0) { diff --git a/generic/tclInt.h b/generic/tclInt.h index e52eaf1..b35f8aa 100644 --- a/generic/tclInt.h +++ b/generic/tclInt.h @@ -2954,7 +2954,6 @@ TclEncodingProfileNameToId(Tcl_Interp *interp, int *profilePtr); MODULE_SCOPE const char *TclEncodingProfileIdToName(Tcl_Interp *interp, int profileId); -MODULE_SCOPE int TclEncodingSetProfileFlags(int flags); MODULE_SCOPE void TclGetEncodingProfiles(Tcl_Interp *interp); /* diff --git a/library/http/http.tcl b/library/http/http.tcl index 6c3c068..c159cb5 100644 --- a/library/http/http.tcl +++ b/library/http/http.tcl @@ -1747,7 +1747,7 @@ proc http::OpenSocket {token DoLater} { fconfigure $sock -translation {auto crlf} \ -buffersize $state(-blocksize) if {[package vsatisfies [package provide Tcl] 9.0-]} { - fconfigure $sock -profile tcl8 + fconfigure $sock -profile replace } ##Log socket opened, DONE fconfigure - token $token } @@ -2168,7 +2168,7 @@ proc http::Connected {token proto phost srvurl} { fconfigure $sock -translation [list $trRead crlf] \ -buffersize $state(-blocksize) if {[package vsatisfies [package provide Tcl] 9.0-]} { - fconfigure $sock -profile tcl8 + fconfigure $sock -profile replace } # The following is disallowed in safe interpreters, but the socket is @@ -2561,7 +2561,7 @@ proc http::ReceiveResponse {token} { fconfigure $sock -translation [list auto $trWrite] \ -buffersize $state(-blocksize) if {[package vsatisfies [package provide Tcl] 9.0-]} { - fconfigure $sock -profile tcl8 + fconfigure $sock -profile replace } Log ^D$tk begin receiving response - token $token @@ -4555,7 +4555,7 @@ proc http::Eot {token {reason {}}} { set enc [CharsetToEncoding $state(charset)] if {$enc ne "binary"} { if {[package vsatisfies [package provide Tcl] 9.0-]} { - set state(body) [encoding convertfrom -profile tcl8 $enc $state(body)] + set state(body) [encoding convertfrom -profile replace $enc $state(body)] } else { set state(body) [encoding convertfrom $enc $state(body)] } @@ -4642,7 +4642,7 @@ proc http::GuessType {token} { return 0 } if {[package vsatisfies [package provide Tcl] 9.0-]} { - set state(body) [encoding convertfrom -profile tcl8 $enc $state(body)] + set state(body) [encoding convertfrom -profile replace $enc $state(body)] } else { set state(body) [encoding convertfrom $enc $state(body)] } @@ -4727,7 +4727,7 @@ proc http::quoteString {string} { # than [regsub]/[subst]). [Bug 1020491] if {[package vsatisfies [package provide Tcl] 9.0-]} { - set string [encoding convertto -profile tcl8 $http(-urlencoding) $string] + set string [encoding convertto -profile replace $http(-urlencoding) $string] } else { set string [encoding convertto $http(-urlencoding) $string] } diff --git a/tests/encodingVectors.tcl b/tests/encodingVectors.tcl index 1b569a1..2f97d1f 100644 --- a/tests/encodingVectors.tcl +++ b/tests/encodingVectors.tcl @@ -10,7 +10,7 @@ # List of defined encoding profiles set encProfiles {tcl8 strict replace} -set encDefaultProfile tcl8; # Should reflect the default from implementation +set encDefaultProfile strict; # Should reflect the default from implementation # encValidStrings - Table of valid strings. # diff --git a/tests/ioCmd.test b/tests/ioCmd.test index e603731..e347e86 100644 --- a/tests/ioCmd.test +++ b/tests/ioCmd.test @@ -240,11 +240,11 @@ test iocmd-8.7 {fconfigure command} -setup { file delete $path(test1) } -body { set f1 [open $path(test1) w] - fconfigure $f1 -translation lf -eofchar {} -encoding utf-16 -profile tcl8 + fconfigure $f1 -translation lf -eofchar {} -encoding utf-16 fconfigure $f1 } -cleanup { catch {close $f1} -} -result {-blocking 1 -buffering full -buffersize 4096 -encoding utf-16 -eofchar {} -profile tcl8 -translation lf} +} -result {-blocking 1 -buffering full -buffersize 4096 -encoding utf-16 -eofchar {} -profile strict -translation lf} test iocmd-8.8 {fconfigure command} -setup { file delete $path(test1) set x {} @@ -262,11 +262,11 @@ test iocmd-8.9 {fconfigure command} -setup { } -body { set f1 [open $path(test1) w] fconfigure $f1 -translation binary -buffering none -buffersize 4040 \ - -eofchar {} -encoding binary -profile tcl8 + -eofchar {} -encoding binary fconfigure $f1 } -cleanup { catch {close $f1} -} -result {-blocking 1 -buffering none -buffersize 4040 -encoding iso8859-1 -eofchar {} -profile tcl8 -translation lf} +} -result {-blocking 1 -buffering none -buffersize 4040 -encoding iso8859-1 -eofchar {} -profile strict -translation lf} test iocmd-8.10 {fconfigure command} -returnCodes error -body { fconfigure a b } -result {can not find channel named "a"} |