From 33f4149ee57d7c60a267c0f72bec5dabba389613 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sun, 8 Jan 2023 22:47:45 +0000 Subject: Fix for [https://core.tcl-lang.org/tk/tktview?name=370b1ff03e|370b1ff03e]. Not complete/correct yet, since this backouts the fix for [4dbfa46caa] --- generic/tclEncoding.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index d10d9ca..cfad548 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -2409,7 +2409,7 @@ UtfToUtfProc( */ if (flags & TCL_ENCODING_MODIFIED) { - if ((STOPONERROR) && (flags & TCL_ENCODING_CHAR_LIMIT)) { + if (STOPONERROR) { result = TCL_CONVERT_MULTIBYTE; break; } @@ -3199,7 +3199,7 @@ TableFromUtfProc( word = fromUnicode[(ch >> 8)][ch & 0xFF]; if ((word == 0) && (ch != 0)) { - if ((STOPONERROR) && (flags & TCL_ENCODING_CHAR_LIMIT)) { + if (STOPONERROR) { result = TCL_CONVERT_UNKNOWN; break; } -- cgit v0.12 From 5021d1c11f0e3287ce96351e20e79552a92e7177 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Mon, 16 Jan 2023 13:10:07 +0000 Subject: New flag TCL_ENCODING_HACK_FLAG to control the behaviour. (This is NOT the way to do it, but it's only meant for experimenting) --- generic/tcl.h | 1 + generic/tclEncoding.c | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/generic/tcl.h b/generic/tcl.h index f373382..36e1a35 100644 --- a/generic/tcl.h +++ b/generic/tcl.h @@ -2145,6 +2145,7 @@ typedef struct Tcl_EncodingType { #define TCL_ENCODING_MODIFIED 0x20 #define TCL_ENCODING_NOCOMPLAIN 0x40 #define TCL_ENCODING_STRICT 0x44 +#define TCL_ENCODING_HACK_FLAG (1<<20) /* * The following definitions are the error codes returned by the conversion diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index cfad548..2c4382d 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -2409,7 +2409,7 @@ UtfToUtfProc( */ if (flags & TCL_ENCODING_MODIFIED) { - if (STOPONERROR) { + if ((STOPONERROR) && ((flags & TCL_ENCODING_CHAR_LIMIT) || (flags & TCL_ENCODING_HACK_FLAG))) { result = TCL_CONVERT_MULTIBYTE; break; } @@ -3199,7 +3199,7 @@ TableFromUtfProc( word = fromUnicode[(ch >> 8)][ch & 0xFF]; if ((word == 0) && (ch != 0)) { - if (STOPONERROR) { + if ((STOPONERROR) && (flags & TCL_ENCODING_CHAR_LIMIT)) { result = TCL_CONVERT_UNKNOWN; break; } -- cgit v0.12 From 925f00c5cad128feb5c4e49b7dd31edc205b4746 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Tue, 17 Jan 2023 14:15:39 +0000 Subject: Use TCL_ENCODING_HACK_FLAG in TableFromUtfProc too --- generic/tclEncoding.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index 2c4382d..3b9ab3e 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -3199,7 +3199,7 @@ TableFromUtfProc( word = fromUnicode[(ch >> 8)][ch & 0xFF]; if ((word == 0) && (ch != 0)) { - if ((STOPONERROR) && (flags & TCL_ENCODING_CHAR_LIMIT)) { + if ((STOPONERROR) && ((flags & TCL_ENCODING_CHAR_LIMIT) || (flags & TCL_ENCODING_HACK_FLAG))) { result = TCL_CONVERT_UNKNOWN; break; } -- cgit v0.12 From 5d50502a2952145d5a6eaa4482ccd79628c1e16f Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Wed, 18 Jan 2023 13:26:15 +0000 Subject: Forget about TCL_ENCODING_HACK_FLAG, this should be the fix. Testing ... --- generic/tcl.h | 1 - generic/tclEncoding.c | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/generic/tcl.h b/generic/tcl.h index 36e1a35..f373382 100644 --- a/generic/tcl.h +++ b/generic/tcl.h @@ -2145,7 +2145,6 @@ typedef struct Tcl_EncodingType { #define TCL_ENCODING_MODIFIED 0x20 #define TCL_ENCODING_NOCOMPLAIN 0x40 #define TCL_ENCODING_STRICT 0x44 -#define TCL_ENCODING_HACK_FLAG (1<<20) /* * The following definitions are the error codes returned by the conversion diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index 3b9ab3e..ca96057 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -2409,7 +2409,7 @@ UtfToUtfProc( */ if (flags & TCL_ENCODING_MODIFIED) { - if ((STOPONERROR) && ((flags & TCL_ENCODING_CHAR_LIMIT) || (flags & TCL_ENCODING_HACK_FLAG))) { + if ((STOPONERROR) && (flags & TCL_ENCODING_CHAR_LIMIT)) { result = TCL_CONVERT_MULTIBYTE; break; } @@ -3199,7 +3199,7 @@ TableFromUtfProc( word = fromUnicode[(ch >> 8)][ch & 0xFF]; if ((word == 0) && (ch != 0)) { - if ((STOPONERROR) && ((flags & TCL_ENCODING_CHAR_LIMIT) || (flags & TCL_ENCODING_HACK_FLAG))) { + if (STOPONERROR) { result = TCL_CONVERT_UNKNOWN; break; } -- cgit v0.12 From 31226696c3e4dd02735044bc1fbf316c0955e65a Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Wed, 18 Jan 2023 14:10:09 +0000 Subject: Some test-cases need -nocomplainencoding 1, because they use the fallback behavior. --- tests/chanio.test | 6 +++--- tests/io.test | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/chanio.test b/tests/chanio.test index 2189cc4..6b45da9 100644 --- a/tests/chanio.test +++ b/tests/chanio.test @@ -252,7 +252,7 @@ test chan-io-3.3 {WriteChars: compatibility with WriteBytes: flush on line} -bod test chan-io-3.4 {WriteChars: loop over stage buffer} -body { # stage buffer maps to more than can be queued at once. set f [open $path(test1) w] - chan configure $f -encoding jis0208 -buffersize 16 + chan configure $f -encoding jis0208 -buffersize 16 -nocomplainencoding 1 chan puts -nonewline $f "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\" set x [list [contents $path(test1)]] chan close $f @@ -265,7 +265,7 @@ test chan-io-3.5 {WriteChars: saved != 0} -body { # be moved to beginning of next channel buffer to preserve requested # buffersize. set f [open $path(test1) w] - chan configure $f -encoding jis0208 -buffersize 17 + chan configure $f -encoding jis0208 -buffersize 17 -nocomplainencoding 1 chan puts -nonewline $f "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\" set x [list [contents $path(test1)]] chan close $f @@ -298,7 +298,7 @@ test chan-io-3.7 {WriteChars: (bufPtr->nextAdded > bufPtr->length)} -body { # on flush. The truncated bytes are moved to the beginning of the next # channel buffer. set f [open $path(test1) w] - chan configure $f -encoding jis0208 -buffersize 17 + chan configure $f -encoding jis0208 -buffersize 17 -nocomplainencoding 1 chan puts -nonewline $f "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\" set x [list [contents $path(test1)]] chan close $f diff --git a/tests/io.test b/tests/io.test index d10e1e4..d2e687d 100644 --- a/tests/io.test +++ b/tests/io.test @@ -272,7 +272,7 @@ test io-3.4 {WriteChars: loop over stage buffer} -body { # stage buffer maps to more than can be queued at once. set f [open $path(test1) w] - fconfigure $f -encoding jis0208 -buffersize 16 + fconfigure $f -encoding jis0208 -buffersize 16 -nocomplainencoding 1 puts -nonewline $f "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\" set x [list [contents $path(test1)]] close $f @@ -286,7 +286,7 @@ test io-3.5 {WriteChars: saved != 0} -body { # requested buffersize. set f [open $path(test1) w] - fconfigure $f -encoding jis0208 -buffersize 17 + fconfigure $f -encoding jis0208 -buffersize 17 -nocomplainencoding 1 puts -nonewline $f "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\" set x [list [contents $path(test1)]] close $f @@ -319,7 +319,7 @@ test io-3.7 {WriteChars: (bufPtr->nextAdded > bufPtr->length)} -body { # of the next channel buffer. set f [open $path(test1) w] - fconfigure $f -encoding jis0208 -buffersize 17 + fconfigure $f -encoding jis0208 -buffersize 17 -nocomplainencoding 1 puts -nonewline $f "\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\" set x [list [contents $path(test1)]] close $f -- cgit v0.12