From 9a8559d4cb683fe7f03d28704ec74cbc08835fb2 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Thu, 30 Apr 2020 14:54:59 +0000 Subject: First, prove that bug [ed29806baf] is present in 8.7 too. Let's see what test-cases fail when we no longer check the validity of the 3th trail byte. --- generic/tclUtf.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/generic/tclUtf.c b/generic/tclUtf.c index e1cde17..693651d 100644 --- a/generic/tclUtf.c +++ b/generic/tclUtf.c @@ -597,16 +597,16 @@ Tcl_UtfToChar16( if (((src[1] & 0xC0) == 0x80) && ((src[2] & 0xC0) == 0x80)) { /* * Four-byte-character lead byte followed by at least two trail bytes. - * (validity of 3th trail byte will be tested later) + * We don't test the validity of 3th trail byte, see [ed29806ba] */ Tcl_UniChar high = (((byte & 0x07) << 8) | ((src[1] & 0x3F) << 2) | ((src[2] & 0x3F) >> 4)) - 0x40; - if ((high < 0x400) && ((src[3] & 0xC0) == 0x80)) { + if (high < 0x400) { /* produce high surrogate, advance source pointer */ *chPtr = 0xD800 + high; return 1; } - /* out of range, < 0x10000 or > 0x10FFFF or invalid 3th byte */ + /* out of range, < 0x10000 or > 0x10FFFF */ } /* -- cgit v0.12 From 62c00ac54a6f93ad1324d7e7aa5ef43623ca2415 Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Fri, 1 May 2020 08:51:09 +0000 Subject: Fix [ed29806baf] by introducing TclUCS4Complete(). All other calls of Tcl_UtfToUniChar() are suspicious, because those cannot handle 4-byte UTF-8 sequences reliable. So, there's more work to do, but this part can already be backported to Tcl 8.6 and see where we get. --- generic/tclDecls.h | 2 +- generic/tclEncoding.c | 14 +++++++------- generic/tclInt.h | 5 ++++- 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/generic/tclDecls.h b/generic/tclDecls.h index 4531be3..c713469 100644 --- a/generic/tclDecls.h +++ b/generic/tclDecls.h @@ -4181,7 +4181,7 @@ extern const TclStubs *tclStubsPtr; #if defined(USE_TCL_STUBS) && (TCL_UTF_MAX > 3) # undef Tcl_UtfCharComplete # define Tcl_UtfCharComplete(src, length) (((unsigned)((unsigned char)*(src) - 0xF0) < 5) \ - ? 4 : tclStubsPtr->tcl_UtfCharComplete((src), (length))) + ? ((length) >= 4) : tclStubsPtr->tcl_UtfCharComplete((src), (length))) #endif #endif /* _TCLDECLS */ diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index 4789b7f..422627b 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -2300,7 +2300,7 @@ UtfToUtfProc( const char *srcStart, *srcEnd, *srcClose; const char *dstStart, *dstEnd; int result, numChars, charLimit = INT_MAX; - Tcl_UniChar *chPtr = (Tcl_UniChar *) statePtr; + int *chPtr = (int *) statePtr; if (flags & TCL_ENCODING_START) { *statePtr = 0; @@ -2321,7 +2321,7 @@ UtfToUtfProc( dstEnd = dst + dstLen - TCL_UTF_MAX; for (numChars = 0; src < srcEnd && numChars <= charLimit; numChars++) { - if ((src > srcClose) && (!Tcl_UtfCharComplete(src, srcEnd - src))) { + if ((src > srcClose) && (!TclUCS4Complete(src, srcEnd - src))) { /* * If there is more string to follow, this will ensure that the * last UTF-8 character in the source buffer hasn't been cut off. @@ -2349,9 +2349,9 @@ UtfToUtfProc( *dst++ = 0; src += 2; - } else if (!Tcl_UtfCharComplete(src, srcEnd - src)) { + } else if (!TclUCS4Complete(src, srcEnd - src)) { /* - * Always check before using TclUtfToUniChar. Not doing can so + * Always check before using TclUtfToUCS4. Not doing can so * cause it run beyond the end of the buffer! If we happen such an * incomplete char its bytes are made to represent themselves. */ @@ -2360,11 +2360,11 @@ UtfToUtfProc( src += 1; dst += Tcl_UniCharToUtf(*chPtr, dst); } else { - src += TclUtfToUniChar(src, chPtr); + src += TclUtfToUCS4(src, chPtr); if ((*chPtr | 0x7FF) == 0xDFFF) { /* A surrogate character is detected, handle especially */ - Tcl_UniChar low = *chPtr; - size_t len = (src <= srcEnd-3) ? Tcl_UtfToUniChar(src, &low) : 0; + int low = *chPtr; + size_t len = (src <= srcEnd-3) ? TclUtfToUCS4(src, &low) : 0; if (((low | 0x3FF) != 0xDFFF) || (*chPtr & 0x400)) { *dst++ = (char) (((*chPtr >> 12) | 0xE0) & 0xEF); *dst++ = (char) (((*chPtr >> 6) | 0x80) & 0xBF); diff --git a/generic/tclInt.h b/generic/tclInt.h index 2ff644e..5f660e3 100644 --- a/generic/tclInt.h +++ b/generic/tclInt.h @@ -3252,8 +3252,11 @@ MODULE_SCOPE int TclUtfCasecmp(const char *cs, const char *ct); MODULE_SCOPE int TclUtfCount(int ch); #if TCL_UTF_MAX > 3 # define TclUtfToUCS4 Tcl_UtfToUniChar +# define TclUCS4Complete Tcl_UtfCharComplete #else - MODULE_SCOPE int TclUtfToUCS4(const char *src, int *ucs4Ptr); + MODULE_SCOPE int TclUtfToUCS4(const char *src, int *ucs4Ptr); +# define TclUCS4Complete(src, length) (((unsigned)((unsigned char)*(src) - 0xF0) < 5) \ + ? ((length) >= 4) : Tcl_UtfCharComplete((src), (length))) #endif MODULE_SCOPE Tcl_Obj * TclpNativeToNormalized(ClientData clientData); MODULE_SCOPE Tcl_Obj * TclpFilesystemPathType(Tcl_Obj *pathPtr); -- cgit v0.12