From 56a57cbad4a080e6df254ee672e523bb43601e0e Mon Sep 17 00:00:00 2001 From: hobbs Date: Sat, 2 Mar 2002 04:55:31 +0000 Subject: * tests/encoding.test: added encoding-23.* tests * generic/tclIO.c (FilterInputBytes): reset the TCL_ENCODING_START flags in the ChannelState when using 'gets'. [Bug #523988] Also reduced the value of ENCODING_LINESIZE from 30 to 20 as this seems to improve the performance of 'gets' according to tclbench. --- ChangeLog | 17 +++++++++- generic/tclIO.c | 15 ++++++--- tests/encoding.test | 92 ++++++++++++++++++++++++++++++++++++++++++----------- 3 files changed, 100 insertions(+), 24 deletions(-) diff --git a/ChangeLog b/ChangeLog index b357462..be0ce13 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,10 +1,25 @@ +2002-03-01 Jeff Hobbs + + * library/encoding/iso2022-jp.enc: + * library/encoding/iso2022.enc: + * tools/encoding/iso2022-jp.esc: + * tools/encoding/iso2022.esc: gave $B precedence over $@, + based on comments (point 1) in [Bug #219283] (rfc 1468) + + * tests/encoding.test: added encoding-23.* tests + * generic/tclIO.c (FilterInputBytes): reset the TCL_ENCODING_START + flags in the ChannelState when using 'gets'. [Bug #523988] + Also reduced the value of ENCODING_LINESIZE from 30 to 20 as this + seems to improve the performance of 'gets' according to tclbench. + 2002-02-28 Jeff Hobbs * generic/tclCmdMZ.c (TraceCommandProc): ensure that TraceCommandInfo structure was also deleted when a command was deleted to prevent a mem leak. - * generic/tclBasic.c (Tcl_CreateObjTrace): set tracePtr->flags correctly. + * generic/tclBasic.c (Tcl_CreateObjTrace): set tracePtr->flags + correctly. * generic/tclTimer.c (TimerExitProc): remove remaining events in tls on thread exit. diff --git a/generic/tclIO.c b/generic/tclIO.c index df8a8c2..cefab6c 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -10,7 +10,7 @@ * See the file "license.terms" for information on usage and redistribution * of this file, and for a DISCLAIMER OF ALL WARRANTIES. * - * RCS: @(#) $Id: tclIO.c,v 1.52 2002/02/15 14:28:49 dkf Exp $ + * RCS: @(#) $Id: tclIO.c,v 1.53 2002/03/02 04:55:31 hobbs Exp $ */ #include "tclInt.h" @@ -3777,7 +3777,7 @@ FilterInputBytes(chanPtr, gsPtr) char *dst; int offset, toRead, dstNeeded, spaceLeft, result, rawLen, length; Tcl_Obj *objPtr; -#define ENCODING_LINESIZE 30 /* Lower bound on how many bytes to convert +#define ENCODING_LINESIZE 20 /* Lower bound on how many bytes to convert * at a time. Since we don't know a priori * how many bytes of storage this many source * bytes will use, we actually need at least @@ -3806,7 +3806,7 @@ FilterInputBytes(chanPtr, gsPtr) * seen EOL. Need to read more bytes from the channel device. * Side effect is to allocate another channel buffer. */ - + read: if (statePtr->flags & CHANNEL_BLOCKED) { if (statePtr->flags & CHANNEL_NONBLOCKING) { @@ -3859,7 +3859,14 @@ FilterInputBytes(chanPtr, gsPtr) result = Tcl_ExternalToUtf(NULL, gsPtr->encoding, raw, rawLen, statePtr->inputEncodingFlags, &statePtr->inputEncodingState, dst, spaceLeft, &gsPtr->rawRead, &gsPtr->bytesWrote, - &gsPtr->charsWrote); + &gsPtr->charsWrote); + + /* + * Make sure that if we go through 'gets', that we reset the + * TCL_ENCODING_START flag still. [Bug #523988] + */ + statePtr->inputEncodingFlags &= ~TCL_ENCODING_START; + if (result == TCL_CONVERT_MULTIBYTE) { /* * The last few bytes in this channel buffer were the start of a diff --git a/tests/encoding.test b/tests/encoding.test index 21c53b0..bfe7e3c 100644 --- a/tests/encoding.test +++ b/tests/encoding.test @@ -8,7 +8,7 @@ # See the file "license.terms" for information on usage and redistribution # of this file, and for a DISCLAIMER OF ALL WARRANTIES. # -# RCS: @(#) $Id: encoding.test,v 1.8 2000/04/10 17:18:58 ericm Exp $ +# RCS: @(#) $Id: encoding.test,v 1.9 2002/03/02 04:55:31 hobbs Exp $ if {[lsearch [namespace children] ::tcltest] == -1} { package require tcltest @@ -196,6 +196,18 @@ test encoding-10.1 {Tcl_UtfToExternal} { set x } "ab\x8c\xc1g" +proc viewable {str} { + set res "" + foreach c [split $str {}] { + if {[string is print $c]} { + append res $c + } else { + append res "\\u[format %4.4x [scan $c %c]]" + } + } + return "$str ($res)" +} + test encoding-11.1 {LoadEncodingFile: unknown encoding} {testencoding} { set system [encoding system] set path [testencoding path] @@ -216,8 +228,11 @@ test encoding-11.4 {LoadEncodingFile: multi-byte} { encoding convertfrom shiftjis \x8c\xc1 } "\u4e4e" test encoding-11.5 {LoadEncodingFile: escape file} { - encoding convertto iso2022 \u4e4e -} "\x1b(B\x1b$@8C" + viewable [encoding convertto iso2022 \u4e4e] +} [viewable "\x1b(B\x1b\$B8C"] +test encoding-11.5.1 {LoadEncodingFile: escape file} { + viewable [encoding convertto iso2022-jp \u4e4e] +} [viewable "\x1b(B\x1b\$B8C"] test encoding-11.6 {LoadEncodingFile: invalid file} {testencoding} { set system [encoding system] set path [testencoding path] @@ -262,8 +277,8 @@ test encoding-12.5 {LoadTableEncoding: symbol encoding} { } "\x67\x67\u3b3" test encoding-13.1 {LoadEscapeTable} { - set x [encoding convertto iso2022 ab\u4e4e\u68d9g] -} "\x1b(Bab\x1b$@8C\x1b$\(DD%\x1b(Bg" + viewable [set x [encoding convertto iso2022 ab\u4e4e\u68d9g]] +} [viewable "\x1b(Bab\x1b\$B8C\x1b\$\(DD%\x1b(Bg"] test encoding-14.1 {BinaryProc} { encoding convertto identity \x12\x34\x56\xff\x69 @@ -295,23 +310,62 @@ test encoding-21.1 {EscapeToUtfProc} { test encoding-22.1 {EscapeFromUtfProc} { } {} +set ::iso2022encData "\u001b\$B;d\$I\$b\$G\$O!\"%A%C%W\$49XF~;~\$K\$4EPO?\$\$\$?\$@\$\$\$?\$4=;=j\$r%-%c%C%7%e%\"%&%H\$N:]\$N\u001b(B +\u001b\$B>.@Z= 0} { + if {$count} { + incr count 1 ; # account for newline + append out \n + } + append out $line + incr count $num + } + close $fid + if {[string compare $::iso2022uniData $out]} { + return -code error "iso2022-jp read in doesn't match original" + } + list $count $out +} [list [string length $::iso2022uniData] $::iso2022uniData] +test encoding-23.3 {iso2022-jp escape encoding test} { + # read $fis reads size in chars, not raw bytes. + set fid [open iso2022.txt r] + fconfigure $fid -encoding iso2022-jp + set data [read $fid 50] + close $fid + set data +} [string range $::iso2022uniData 0 49] ; # 0 .. 49 inclusive == 50 + +::tcltest::removeFile iso2022.txt + # EscapeFreeProc, GetTableEncoding, unilen # are fully tested by the rest of this file # cleanup ::tcltest::cleanupTests return - - - - - - - - - - - - - - -- cgit v0.12