From 4655526dd72cdeab8558be6bf5bbddc7f98e51dc Mon Sep 17 00:00:00 2001 From: pooryorick Date: Sun, 16 Apr 2023 12:58:24 +0000 Subject: Fix mistake in test io-75.14 and make some spelling corrections. --- tests/io.test | 27 +++++++++++++-------------- tests/ioCmd.test | 2 +- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/tests/io.test b/tests/io.test index d98317a..f1966c7 100644 --- a/tests/io.test +++ b/tests/io.test @@ -9273,7 +9273,6 @@ test io-75.6 {invalid utf-8 encoding, gets is not ignored (-profile strict)} -se fconfigure $f -encoding utf-8 -buffering none -eofchar {} \ -translation lf -profile strict } -body { -after 1 gets $f } -cleanup { close $f @@ -9450,23 +9449,23 @@ test io-75.14 { invalid utf-8 encoding [gets] continues in non-strict mode after error } -setup { - set chan [file tempfile] - fconfigure $f -encoding binary + set chan [file tempfile] + fconfigure $chan -encoding binary # \xc0\n is an invalid utf-8 sequence - puts -nonewline $f a\nb\nc\xc0\nd\n - flush $f - seek $f 0 - fconfigure $f -encoding utf-8 -buffering none -eofchar {} \ + puts -nonewline $chan a\nb\nc\xc0\nd\n + flush $chan + seek $chan 0 + fconfigure $chan -encoding utf-8 -buffering none -eofchar {} \ -translation lf -profile strict } -body { - lappend res [gets $f] - lappend res [gets $f] - set status [catch {gets $f} cres copts] + lappend res [gets $chan] + lappend res [gets $chan] + set status [catch {gets $chan} cres copts] lappend res $status $cres - chan configure $f -profile tcl8 - lappend res [gets $f] - lappend res [gets $f] - close $f + chan configure $chan -profile tcl8 + lappend res [gets $chan] + lappend res [gets $chan] + close $chan return $res } -match glob -result {a b 1 {error reading "*":\ invalid or incomplete multibyte or wide character} cÀ d} diff --git a/tests/ioCmd.test b/tests/ioCmd.test index 4163b1b..5d731b9 100644 --- a/tests/ioCmd.test +++ b/tests/ioCmd.test @@ -1058,7 +1058,7 @@ test iocmd-23.1 {chan read, regular data return} -match glob -body { rename foo {} set res } -result {{read rc* 4096} {read rc* 4096} snarfsnarf} -test iocmd-23.2 {chan read, bad data return, to much} -match glob -body { +test iocmd-23.2 {chan read, bad data return, too much} -match glob -body { set res {} proc foo {args} { oninit; onfinal; track -- cgit v0.12 From 898f3544928665515cb9a05c7063b4828a8376af Mon Sep 17 00:00:00 2001 From: pooryorick Date: Sun, 16 Apr 2023 14:16:51 +0000 Subject: Remove unneeded ENCODING_UTF flags. --- generic/tclEncoding.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index 6feb686..9469703 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -647,13 +647,13 @@ TclInitEncodingSubsystem(void) type.freeProc = NULL; type.nullSize = 2; type.encodingName = "utf-16le"; - type.clientData = INT2PTR(TCL_ENCODING_LE|ENCODING_UTF); + type.clientData = INT2PTR(TCL_ENCODING_LE); Tcl_CreateEncoding(&type); type.encodingName = "utf-16be"; - type.clientData = INT2PTR(ENCODING_UTF); + type.clientData = NULL; Tcl_CreateEncoding(&type); type.encodingName = "utf-16"; - type.clientData = INT2PTR(leFlags|ENCODING_UTF); + type.clientData = INT2PTR(leFlags); Tcl_CreateEncoding(&type); #ifndef TCL_NO_DEPRECATED -- cgit v0.12 From 249b57ef37ebce777fcd204a11d93b0e612689ce Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sun, 16 Apr 2023 17:58:44 +0000 Subject: some spelling corrections --- tests/ioCmd.test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ioCmd.test b/tests/ioCmd.test index 2a1f616..e132b9a 100644 --- a/tests/ioCmd.test +++ b/tests/ioCmd.test @@ -1072,7 +1072,7 @@ test iocmd-23.1 {chan read, regular data return} -match glob -body { rename foo {} set res } -result {{read rc* 4096} {read rc* 4096} snarfsnarf} -test iocmd-23.2 {chan read, bad data return, to much} -match glob -body { +test iocmd-23.2 {chan read, bad data return, too much} -match glob -body { set res {} proc foo {args} { oninit; onfinal; track @@ -2383,7 +2383,7 @@ test iocmd.tf-23.1 {chan read, regular data return} -match glob -body { rename foo {} set res } -constraints {testchannel thread} -result {{read rc* 4096} {read rc* 4096} snarfsnarf} -test iocmd.tf-23.2 {chan read, bad data return, to much} -match glob -body { +test iocmd.tf-23.2 {chan read, bad data return, too much} -match glob -body { set res {} proc foo {args} { oninit; onfinal; track -- cgit v0.12 From 61722ce13d58865dabec38cc1ec9f52152f191af Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sun, 16 Apr 2023 18:07:22 +0000 Subject: Remove unneeded ENCODING_UTF flags --- generic/tclEncoding.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index db98d54..774485d 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -602,7 +602,7 @@ TclInitEncodingSubsystem(void) type.nullSize = 1; type.clientData = INT2PTR(ENCODING_UTF); Tcl_CreateEncoding(&type); - type.clientData = INT2PTR(0); + type.clientData = NULL; type.encodingName = "cesu-8"; Tcl_CreateEncoding(&type); @@ -614,7 +614,7 @@ TclInitEncodingSubsystem(void) type.clientData = INT2PTR(TCL_ENCODING_LE); Tcl_CreateEncoding(&type); type.encodingName = "ucs-2be"; - type.clientData = INT2PTR(0); + type.clientData = NULL; Tcl_CreateEncoding(&type); type.encodingName = "ucs-2"; type.clientData = INT2PTR(leFlags); @@ -628,7 +628,7 @@ TclInitEncodingSubsystem(void) type.clientData = INT2PTR(TCL_ENCODING_LE); Tcl_CreateEncoding(&type); type.encodingName = "utf-32be"; - type.clientData = INT2PTR(0); + type.clientData = NULL; Tcl_CreateEncoding(&type); type.encodingName = "utf-32"; type.clientData = INT2PTR(leFlags); @@ -639,13 +639,13 @@ TclInitEncodingSubsystem(void) type.freeProc = NULL; type.nullSize = 2; type.encodingName = "utf-16le"; - type.clientData = INT2PTR(TCL_ENCODING_LE|ENCODING_UTF); + type.clientData = INT2PTR(TCL_ENCODING_LE); Tcl_CreateEncoding(&type); type.encodingName = "utf-16be"; - type.clientData = INT2PTR(ENCODING_UTF); + type.clientData = NULL; Tcl_CreateEncoding(&type); type.encodingName = "utf-16"; - type.clientData = INT2PTR(leFlags|ENCODING_UTF); + type.clientData = INT2PTR(leFlags); Tcl_CreateEncoding(&type); #ifndef TCL_NO_DEPRECATED -- cgit v0.12 From 87df69042cc59415cf4e792eabeed95e11478566 Mon Sep 17 00:00:00 2001 From: pooryorick Date: Sun, 16 Apr 2023 23:10:16 +0000 Subject: Fix for [eabcbd08274f2d22], [read] error persists on channel with strict encoding after encoding is changed to binary. --- generic/tclIO.c | 4 +++- tests/io.test | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/generic/tclIO.c b/generic/tclIO.c index 4a88f52..7e475cc 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -4616,7 +4616,7 @@ Tcl_GetsObj( if (GotFlag(statePtr, CHANNEL_ENCODING_ERROR)) { UpdateInterest(chanPtr); - ResetFlag(statePtr, CHANNEL_ENCODING_ERROR); + ResetFlag(statePtr, CHANNEL_EOF|CHANNEL_BLOCKED); Tcl_SetErrno(EILSEQ); return TCL_INDEX_NONE; } @@ -5941,6 +5941,7 @@ DoReadChars( int factor = UTF_EXPANSION_FACTOR; if (GotFlag(statePtr, CHANNEL_ENCODING_ERROR)) { + ResetFlag(statePtr, CHANNEL_EOF|CHANNEL_STICKY_EOF|CHANNEL_BLOCKED); /* TODO: We don't need this call? */ UpdateInterest(chanPtr); Tcl_SetErrno(EILSEQ); @@ -6117,6 +6118,7 @@ finish: * succesfully red before the error. Return an error so that callers * like [read] can also return an error. */ + ResetFlag(statePtr, CHANNEL_EOF|CHANNEL_ENCODING_ERROR); Tcl_SetErrno(EILSEQ); copied = -1; } diff --git a/tests/io.test b/tests/io.test index f1966c7..764c776 100644 --- a/tests/io.test +++ b/tests/io.test @@ -9322,6 +9322,30 @@ test io-75.8 {invalid utf-8 encoding eof handling (-profile strict)} -setup { removeFile io-75.8 } -result {41 1 {}} +test io-75.8.eoflater {invalid utf-8 encoding eof handling (-profile strict)} -setup { + set res {} + set fn [makeFile {} io-75.8] + set f [open $fn w+] + fconfigure $f -encoding binary + # \x81 is invalid in utf-8. -eofchar is not detected, because it comes later. + puts -nonewline $f A\x81\x1A + flush $f + seek $f 0 + fconfigure $f -encoding utf-8 -buffering none -eofchar \x1A \ + -translation lf -profile strict +} -body { + after 1 + set status [catch {read $f} cres copts] + lappend res $status + lappend res [eof $f] + chan configure $f -encoding iso8859-1 + lappend res [read $f] + close $f + set res +} -cleanup { + removeFile io-75.8 +} -result "1 0 \x81" + test io-75.9 {unrepresentable character write passes and is replaced by ?} -setup { set fn [makeFile {} io-75.9] set f [open $fn w+] -- cgit v0.12 From e08e6f2733a5715dfd8ff6062b5787f5e3ceb728 Mon Sep 17 00:00:00 2001 From: pooryorick Date: Mon, 17 Apr 2023 12:12:42 +0000 Subject: Correct some test descriptions in encoding.test. --- tests/encoding.test | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/encoding.test b/tests/encoding.test index 35340a6..e73d3b8 100644 --- a/tests/encoding.test +++ b/tests/encoding.test @@ -568,7 +568,7 @@ test encoding-16.19 {Utf16ToUtfProc, bug [d19fe0a5b]} -body { test encoding-16.20 {Utf16ToUtfProc, bug [d19fe0a5b]} -constraints deprecated -body { encoding convertfrom utf-16 "\xD8\xD8" } -result \uD8D8 -test encoding-16.21 {Utf16ToUtfProc, bug [d19fe0a5b]} -body { +test encoding-16.21 {Utf32ToUtfProc, bug [d19fe0a5b]} -body { encoding convertfrom utf-32 "\x00\x00\x00\x00\x41\x41" } -result \x00\uFFFD test encoding-16.22 {Utf16ToUtfProc, strict, bug [db7a085bd9]} -body { @@ -596,10 +596,10 @@ test encoding-17.3 {UtfToUtf16Proc} -body { test encoding-17.4 {UtfToUtf16Proc} -body { encoding convertto -profile tcl8 utf-16le "\uD8D8" } -result "\xD8\xD8" -test encoding-17.5 {UtfToUtf16Proc} -body { +test encoding-17.5 {UtfToUtf32Proc} -body { encoding convertto utf-32le "\U460DC" } -result "\xDC\x60\x04\x00" -test encoding-17.6 {UtfToUtf16Proc} -body { +test encoding-17.6 {UtfToUtf32Proc} -body { encoding convertto utf-32be "\U460DC" } -result "\x00\x04\x60\xDC" test encoding-17.7 {UtfToUtf16Proc} -body { -- cgit v0.12 From 75f7e5328ad3f05b450b89ff6f480264055b8eff Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 17 Apr 2023 20:03:38 +0000 Subject: Plug memory leak caused by [46e7091a77b71ed7|this] commit --- generic/tclIOCmd.c | 1 + 1 file changed, 1 insertion(+) diff --git a/generic/tclIOCmd.c b/generic/tclIOCmd.c index 5a766c6..7411855 100644 --- a/generic/tclIOCmd.c +++ b/generic/tclIOCmd.c @@ -433,6 +433,7 @@ Tcl_ReadObjCmd( TclChannelPreserve(chan); charactersRead = Tcl_ReadChars(chan, resultPtr, toRead, 0); if (charactersRead == TCL_IO_FAILURE) { + Tcl_DecrRefCount(resultPtr); /* * TIP #219. * Capture error messages put by the driver into the bypass area and -- cgit v0.12 From 6f003899fecfb7886609e740a4539109cc92144c Mon Sep 17 00:00:00 2001 From: pooryorick Date: Mon, 17 Apr 2023 22:01:17 +0000 Subject: Make encoding profile explicit in some tests. --- tests/encoding.test | 42 +++++++++++++++++++++--------------------- tests/io.test | 2 +- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/tests/encoding.test b/tests/encoding.test index e73d3b8..09f3e42 100644 --- a/tests/encoding.test +++ b/tests/encoding.test @@ -465,7 +465,7 @@ test encoding-15.25 {UtfToUtfProc CESU-8} { encoding convertfrom cesu-8 \x00 } \x00 test encoding-15.26 {UtfToUtfProc CESU-8} { - encoding convertfrom cesu-8 \xC0\x80 + encoding convertfrom -profile tcl8 cesu-8 \xC0\x80 } \x00 test encoding-15.27 {UtfToUtfProc -profile strict CESU-8} { encoding convertfrom -profile strict cesu-8 \x00 @@ -492,7 +492,7 @@ test encoding-16.2 {Utf16ToUtfProc} -body { list $val [format %x [scan $val %c]] } -result "\U460DC 460dc" test encoding-16.3 {Utf16ToUtfProc} -body { - set val [encoding convertfrom utf-16 "\xDC\xDC"] + set val [encoding convertfrom -profile tcl8 utf-16 "\xDC\xDC"] list $val [format %x [scan $val %c]] } -result "\uDCDC dcdc" test encoding-16.4 {Ucs2ToUtfProc} -body { @@ -504,11 +504,11 @@ test encoding-16.5 {Ucs2ToUtfProc} -body { list $val [format %x [scan $val %c]] } -result "\U460DC 460dc" test encoding-16.6 {Utf32ToUtfProc} -body { - set val [encoding convertfrom utf-32le NN\0\0] + set val [encoding convertfrom -profile strict utf-32le NN\0\0] list $val [format %x [scan $val %c]] } -result "乎 4e4e" test encoding-16.7 {Utf32ToUtfProc} -body { - set val [encoding convertfrom utf-32be \0\0NN] + set val [encoding convertfrom -profile strict utf-32be \0\0NN] list $val [format %x [scan $val %c]] } -result "乎 4e4e" test encoding-16.8 {Utf32ToUtfProc} -body { @@ -516,28 +516,28 @@ test encoding-16.8 {Utf32ToUtfProc} -body { list $val [format %x [scan $val %c]] } -result "\uFFFD fffd" test encoding-16.9 {Utf32ToUtfProc} -constraints utf32 -body { - encoding convertfrom utf-32le \x00\xD8\x00\x00 + encoding convertfrom -profile tcl8 utf-32le \x00\xD8\x00\x00 } -result \uD800 test encoding-16.10 {Utf32ToUtfProc} -body { - encoding convertfrom utf-32le \x00\xDC\x00\x00 + encoding convertfrom -profile tcl8 utf-32le \x00\xDC\x00\x00 } -result \uDC00 test encoding-16.11 {Utf32ToUtfProc} -body { - encoding convertfrom utf-32le \x00\xD8\x00\x00\x00\xDC\x00\x00 + encoding convertfrom -profile tcl8 utf-32le \x00\xD8\x00\x00\x00\xDC\x00\x00 } -result \uD800\uDC00 test encoding-16.12 {Utf32ToUtfProc} -constraints utf32 -body { - encoding convertfrom utf-32le \x00\xDC\x00\x00\x00\xD8\x00\x00 + encoding convertfrom -profile tcl8 utf-32le \x00\xDC\x00\x00\x00\xD8\x00\x00 } -result \uDC00\uD800 test encoding-16.13 {Utf16ToUtfProc} -body { - encoding convertfrom utf-16le \x00\xD8 + encoding convertfrom -profile tcl8 utf-16le \x00\xD8 } -result \uD800 test encoding-16.14 {Utf16ToUtfProc} -body { - encoding convertfrom utf-16le \x00\xDC + encoding convertfrom -profile tcl8 utf-16le \x00\xDC } -result \uDC00 test encoding-16.15 {Utf16ToUtfProc} -body { encoding convertfrom utf-16le \x00\xD8\x00\xDC } -result \U010000 test encoding-16.16 {Utf16ToUtfProc} -body { - encoding convertfrom utf-16le \x00\xDC\x00\xD8 + encoding convertfrom -profile tcl8 utf-16le \x00\xDC\x00\xD8 } -result \uDC00\uD800 test encoding-16.17 {Utf32ToUtfProc} -body { list [encoding convertfrom -profile strict -failindex idx utf-32le \x41\x00\x00\x00\x00\xD8\x00\x00\x42\x00\x00\x00] [set idx] @@ -563,13 +563,13 @@ test encoding-16.18 { } [namespace current]] } -result done test encoding-16.19 {Utf16ToUtfProc, bug [d19fe0a5b]} -body { - encoding convertfrom utf-16 "\x41\x41\x41" + encoding convertfrom -profile tcl8 utf-16 "\x41\x41\x41" } -result \u4141\uFFFD test encoding-16.20 {Utf16ToUtfProc, bug [d19fe0a5b]} -constraints deprecated -body { encoding convertfrom utf-16 "\xD8\xD8" } -result \uD8D8 test encoding-16.21 {Utf32ToUtfProc, bug [d19fe0a5b]} -body { - encoding convertfrom utf-32 "\x00\x00\x00\x00\x41\x41" + encoding convertfrom -profile tcl8 utf-32 "\x00\x00\x00\x00\x41\x41" } -result \x00\uFFFD test encoding-16.22 {Utf16ToUtfProc, strict, bug [db7a085bd9]} -body { encoding convertfrom -profile strict utf-16le \x00\xD8 @@ -578,10 +578,10 @@ test encoding-16.23 {Utf16ToUtfProc, strict, bug [db7a085bd9]} -body { encoding convertfrom -profile strict utf-16le \x00\xDC } -returnCodes 1 -result {unexpected byte sequence starting at index 0: '\x00'} test encoding-16.24 {Utf32ToUtfProc} -body { - encoding convertfrom utf-32 "\xFF\xFF\xFF\xFF" + encoding convertfrom -profile tcl8 utf-32 "\xFF\xFF\xFF\xFF" } -result \uFFFD test encoding-16.25 {Utf32ToUtfProc} -body { - encoding convertfrom utf-32 "\x01\x00\x00\x01" + encoding convertfrom -profile tcl8 utf-32 "\x01\x00\x00\x01" } -result \uFFFD test encoding-17.1 {UtfToUtf16Proc} -body { @@ -622,7 +622,7 @@ test encoding-17.12 {Utf32ToUtfProc} -body { } -returnCodes error -result {unexpected byte sequence starting at index 0: '\x00'} test encoding-18.1 {TableToUtfProc on invalid input} -body { - list [catch {encoding convertto jis0208 \\} res] $res + list [catch {encoding convertto -profile tcl8 jis0208 \\} res] $res } -result {0 !)} test encoding-18.2 {TableToUtfProc on invalid input with -profile strict} -body { list [catch {encoding convertto -profile strict jis0208 \\} res] $res @@ -634,14 +634,14 @@ test encoding-18.4 {TableToUtfProc on invalid input with -failindex -profile str list [catch {encoding convertto -failindex pos -profile strict jis0208 \\} res] $res $pos } -result {0 {} 0} test encoding-18.5 {TableToUtfProc on invalid input with -failindex} -body { - list [catch {encoding convertto -failindex pos jis0208 \\} res] $res $pos + list [catch {encoding convertto -profile tcl8 -failindex pos jis0208 \\} res] $res $pos } -result {0 !) -1} test encoding-18.6 {TableToUtfProc on invalid input with -profile tcl8} -body { list [catch {encoding convertto -profile tcl8 jis0208 \\} res] $res } -result {0 !)} test encoding-19.1 {TableFromUtfProc} -body { - encoding convertfrom ascii AÁ + encoding convertfrom -profile tcl8 ascii AÁ } -result AÁ test encoding-19.2 {TableFromUtfProc} -body { encoding convertfrom -profile tcl8 ascii AÁ @@ -650,7 +650,7 @@ test encoding-19.3 {TableFromUtfProc} -body { encoding convertfrom -profile strict ascii AÁ } -returnCodes 1 -result {unexpected byte sequence starting at index 1: '\xC1'} test encoding-19.4 {TableFromUtfProc} -body { - list [encoding convertfrom -failindex idx ascii AÁ] [set idx] + list [encoding convertfrom -profile tcl8 -failindex idx ascii AÁ] [set idx] } -result [list A\xC1 -1] test encoding-19.5 {TableFromUtfProc} -body { list [encoding convertfrom -failindex idx -profile strict ascii A\xC1] [set idx] @@ -799,7 +799,7 @@ test encoding-24.14 {Parse valid or invalid utf-8} { string length [encoding convertfrom utf-8 "\xC2\x80"] } 1 test encoding-24.15 {Parse valid or invalid utf-8} -body { - encoding convertfrom utf-8 "Z\xE0\x80" + encoding convertfrom -profile tcl8 utf-8 "Z\xE0\x80" } -result Z\xE0\u20AC test encoding-24.16 {Parse valid or invalid utf-8} -constraints testbytestring -body { encoding convertto utf-8 [testbytestring "Z\u4343\x80"] @@ -862,7 +862,7 @@ test encoding-24.34 {Try to generate invalid utf-8 with -profile tcl8} -body { encoding convertto -profile tcl8 utf-8 \uFFFF } -result \xEF\xBF\xBF test encoding-24.35 {Parse invalid utf-8} -constraints utf32 -body { - encoding convertfrom utf-8 \xED\xA0\x80 + encoding convertfrom -profile tcl8 utf-8 \xED\xA0\x80 } -result \uD800 test encoding-24.36 {Parse invalid utf-8 with -profile strict} -body { encoding convertfrom -profile strict utf-8 \xED\xA0\x80 diff --git a/tests/io.test b/tests/io.test index 764c776..6ded150 100644 --- a/tests/io.test +++ b/tests/io.test @@ -1620,7 +1620,7 @@ test io-12.9 {ReadChars: multibyte chars split} -body { puts -nonewline $f [string repeat a 9]\xC2 close $f set f [open $path(test1)] - fconfigure $f -encoding utf-8 -buffersize 10 + fconfigure $f -encoding utf-8 -profile tcl8 -buffersize 10 set in [read $f] close $f scan [string index $in end] %c -- cgit v0.12