From 6cd305a23929b3f7c41c64fbbb552c9c579dd39d Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sat, 10 Sep 2022 23:25:22 +0000 Subject: Branch meant for investigation of [6978c01b65]. Ongoing --- .github/workflows/linux-build.yml | 5 -- .github/workflows/mac-build.yml | 66 ----------------- .github/workflows/onefiledist.yml | 146 -------------------------------------- .github/workflows/win-build.yml | 11 --- tests/io.test | 8 +-- 5 files changed, 4 insertions(+), 232 deletions(-) delete mode 100644 .github/workflows/mac-build.yml delete mode 100644 .github/workflows/onefiledist.yml diff --git a/.github/workflows/linux-build.yml b/.github/workflows/linux-build.yml index 7ba9e89..0bcb51f 100644 --- a/.github/workflows/linux-build.yml +++ b/.github/workflows/linux-build.yml @@ -9,12 +9,7 @@ jobs: matrix: cfgopt: - "" - - "CFLAGS=-DTCL_UTF_MAX=3" - "CFLAGS=-DTCL_NO_DEPRECATED=1" - - "--disable-shared" - - "--enable-symbols" - - "--enable-symbols=mem" - - "--enable-symbols=all" defaults: run: shell: bash diff --git a/.github/workflows/mac-build.yml b/.github/workflows/mac-build.yml deleted file mode 100644 index 1ec784a..0000000 --- a/.github/workflows/mac-build.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: macOS -on: [push] -permissions: - contents: read -jobs: - xcode: - runs-on: macos-11 - defaults: - run: - shell: bash - working-directory: macosx - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Prepare - run: | - touch tclStubInit.c tclOOStubInit.c tclOOScript.h - working-directory: generic - - name: Build - run: make all - env: - CFLAGS: -arch x86_64 -arch arm64e - - name: Run Tests - run: make test styles=develop - env: - ERROR_ON_FAILURES: 1 - MAC_CI: 1 - clang: - runs-on: macos-11 - strategy: - matrix: - cfgopt: - - "" - - "--disable-shared" - - "--enable-symbols" - - "--enable-symbols=mem" - - "--enable-symbols=all" - defaults: - run: - shell: bash - working-directory: unix - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Prepare - run: | - touch tclStubInit.c tclOOStubInit.c tclOOScript.h - mkdir "$HOME/install dir" - working-directory: generic - - name: Configure ${{ matrix.cfgopt }} - # Note that macOS is always a 64 bit platform - run: ./configure --enable-dtrace --enable-framework ${CFGOPT} "--prefix=$HOME/install" || (cat config.log && exit 1) - env: - CFLAGS: -arch x86_64 -arch arm64e - CFGOPT: ${{ matrix.cfgopt }} - - name: Build - run: | - make all tcltest - env: - CFLAGS: -arch x86_64 -arch arm64e - - name: Run Tests - run: | - make test - env: - ERROR_ON_FAILURES: 1 - MAC_CI: 1 diff --git a/.github/workflows/onefiledist.yml b/.github/workflows/onefiledist.yml deleted file mode 100644 index 45ce720..0000000 --- a/.github/workflows/onefiledist.yml +++ /dev/null @@ -1,146 +0,0 @@ -name: Build Binaries -on: [push] -permissions: - contents: read -jobs: - linux: - name: Linux - runs-on: ubuntu-20.04 - defaults: - run: - shell: bash - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Prepare - run: | - touch generic/tclStubInit.c generic/tclOOStubInit.c - mkdir 1dist - echo "VER_PATH=$(cd tools; pwd)/addVerToFile.tcl" >> $GITHUB_ENV - working-directory: . - - name: Configure - run: ./configure --disable-symbols --disable-shared --enable-zipfs - working-directory: unix - - name: Build - run: | - make tclsh - make shell SCRIPT="$VER_PATH $GITHUB_ENV" - echo "TCL_ZIP=`pwd`/`echo libtcl*.zip`" >> $GITHUB_ENV - working-directory: unix - - name: Package - run: | - cp ../unix/tclsh tclsh${TCL_PATCHLEVEL}_snapshot - chmod +x tclsh${TCL_PATCHLEVEL}_snapshot - tar -cf tclsh${TCL_PATCHLEVEL}_snapshot.tar tclsh${TCL_PATCHLEVEL}_snapshot - working-directory: 1dist - - name: Upload - uses: actions/upload-artifact@v3 - with: - name: Tclsh ${{ env.TCL_PATCHLEVEL }} Linux single-file build (snapshot) - path: 1dist/*.tar - macos: - name: macOS - runs-on: macos-11 - defaults: - run: - shell: bash - steps: - - name: Checkout - uses: actions/checkout@v3 - - name: Checkout create-dmg - uses: actions/checkout@v3 - with: - repository: create-dmg/create-dmg - ref: v1.0.8 - path: create-dmg - - name: Prepare - run: | - mkdir 1dist - touch generic/tclStubInit.c generic/tclOOStubInit.c || true - wget https://github.com/culler/macher/releases/download/v1.3/macher - sudo cp macher /usr/local/bin - sudo chmod a+x /usr/local/bin/macher - echo "VER_PATH=$(cd tools; pwd)/addVerToFile.tcl" >> $GITHUB_ENV - echo "CREATE_DMG=$(cd create-dmg;pwd)/create-dmg" >> $GITHUB_ENV - echo "CFLAGS=-arch x86_64 -arch arm64e" >> $GITHUB_ENV - - name: Configure - run: ./configure --disable-symbols --disable-shared --enable-zipfs - working-directory: unix - - name: Build - run: | - make tclsh - make shell SCRIPT="$VER_PATH $GITHUB_ENV" - echo "TCL_BIN=`pwd`/tclsh" >> $GITHUB_ENV - echo "TCL_ZIP=`pwd`/`echo libtcl*.zip`" >> $GITHUB_ENV - working-directory: unix - - name: Package - run: | - mkdir contents - cp $TCL_BIN contents/tclsh${TCL_PATCHLEVEL}_snapshot - chmod +x contents/tclsh${TCL_PATCHLEVEL}_snapshot - cat > contents/README.txt <> $GITHUB_ENV - mkdir 1dist - working-directory: . - - name: Configure - run: ./configure $CFGOPT - working-directory: win - - name: Build - run: | - make binaries libraries - echo "TCL_ZIP=`pwd`/`echo libtcl*.zip`" >> $GITHUB_ENV - working-directory: win - - name: Get Exact Version - run: | - ./tclsh*.exe $VER_PATH $GITHUB_ENV - working-directory: win - - name: Set Executable Name - run: | - cp ../win/tclsh*.exe tclsh${TCL_PATCHLEVEL}_snapshot.exe - working-directory: 1dist - - name: Upload - uses: actions/upload-artifact@v3 - with: - name: Tclsh ${{ env.TCL_PATCHLEVEL }} Windows single-file build (snapshot) - path: '1dist/*_snapshot.exe' diff --git a/.github/workflows/win-build.yml b/.github/workflows/win-build.yml index ba4e5ba..6fe21d5 100644 --- a/.github/workflows/win-build.yml +++ b/.github/workflows/win-build.yml @@ -14,12 +14,7 @@ jobs: strategy: matrix: cfgopt: - - "" - - "OPTS=utf16" - "CHECKS=nodep" - - "OPTS=static" - - "OPTS=symbols" - - "OPTS=symbols STATS=compdbg,memdbg" # Using powershell means we need to explicitly stop on failure steps: - name: Checkout @@ -53,13 +48,7 @@ jobs: strategy: matrix: cfgopt: - - "" - - "CFLAGS=-DTCL_UTF_MAX=3" - "CFLAGS=-DTCL_NO_DEPRECATED=1" - - "--disable-shared" - - "--enable-symbols" - - "--enable-symbols=mem" - - "--enable-symbols=all" # Using powershell means we need to explicitly stop on failure steps: - name: Install MSYS2 diff --git a/tests/io.test b/tests/io.test index 5c45918..49cec51 100644 --- a/tests/io.test +++ b/tests/io.test @@ -8954,7 +8954,7 @@ test io-74.1 {[104f2885bb] improper cache validity check} -setup { # Note: the following tests 75.1 to 75.3 are in preparation for TCL 9.0, where # those should result in an error result -test io-75.1 {multibyte encoding error read results in raw bytes} -constraints deprecated -setup { +test io-75.1 {multibyte encoding error read results in raw bytes} -setup { set fn [makeFile {} io-75.1] set f [open $fn w+] fconfigure $f -encoding binary @@ -8969,10 +8969,10 @@ test io-75.1 {multibyte encoding error read results in raw bytes} -constraints d } -cleanup { close $f removeFile io-75.1 -} -returnCodes ok -result "A\xC0\x40" +} -returnCodes ok -result "A" # for TCL 9.0, the result is error -test io-75.2 {unrepresentable character write passes and is replaced by ?} -constraints deprecated -setup { +test io-75.2 {unrepresentable character write passes and is replaced by ?} -setup { set fn [makeFile {} io-75.2] set f [open $fn w+] fconfigure $f -encoding iso8859-1 @@ -8985,7 +8985,7 @@ test io-75.2 {unrepresentable character write passes and is replaced by ?} -cons } -cleanup { close $f removeFile io-75.2 -} -returnCodes ok -result "A?" +} -returnCodes ok -result "A" # Incomplete sequence test. # This error may IMHO only be detected with the close. -- cgit v0.12 From 0f674c548252976d41f9ac1c0cf4668cb8eaaabe Mon Sep 17 00:00:00 2001 From: oehhar Date: Sun, 11 Sep 2022 08:39:20 +0000 Subject: Tests io-75.x tolerant encoding: correct indention and comments --- tests/io.test | 58 +++++++++++++++++++++++++++++----------------------------- 1 file changed, 29 insertions(+), 29 deletions(-) diff --git a/tests/io.test b/tests/io.test index 764e9f0..e338749 100644 --- a/tests/io.test +++ b/tests/io.test @@ -8814,58 +8814,58 @@ test io-74.1 {[104f2885bb] improper cache validity check} -setup { removeFile io-74.1 } -returnCodes error -match glob -result {can not find channel named "*"} -# Note: the following tests 75.1 to 75.3 are in preparation for TCL 9.0, where -# those should result in an error result +# The following tests 75.1 to 75.3 exercice strict or tolerant channel +# encoding. +# TCL 8.6 only offers tolerant channel encoding, what is tested here. test io-75.1 {multibyte encoding error read results in raw bytes} -setup { - set fn [makeFile {} io-75.1] + set fn [makeFile {} io-75.1] set f [open $fn w+] fconfigure $f -encoding binary - # In UTF-8, a byte 0xCx starts a multibyte sequence and must be followed - # by a byte > 0x7F. This is violated to get an invalid sequence. - puts -nonewline $f "A\xC0\x40" - flush $f - seek $f 0 - fconfigure $f -encoding utf-8 -buffering none + # In UTF-8, a byte 0xCx starts a multibyte sequence and must be followed + # by a byte > 0x7F. This is violated to get an invalid sequence. + puts -nonewline $f "A\xC0\x40" + flush $f + seek $f 0 + fconfigure $f -encoding utf-8 -buffering none } -body { - read $f + read $f } -cleanup { - close $f - removeFile io-75.1 + close $f + removeFile io-75.1 } -returnCodes ok -result "A\xC0\x40" # for TCL 9.0, the result is error test io-75.2 {unrepresentable character write passes and is replaced by ?} -setup { - set fn [makeFile {} io-75.2] + set fn [makeFile {} io-75.2] set f [open $fn w+] fconfigure $f -encoding iso8859-1 } -body { - # the following command gets in result error in TCL 9.0 - puts -nonewline $f "A\u2022" - flush $f - seek $f 0 - read $f + puts -nonewline $f "A\u2022" + flush $f + seek $f 0 + read $f } -cleanup { - close $f - removeFile io-75.2 + close $f + removeFile io-75.2 } -returnCodes ok -result "A?" # Incomplete sequence test. # This error may IMHO only be detected with the close. # But the read already returns the incomplete sequence. test io-75.3 {incomplete multibyte encoding read is ignored} -setup { - set fn [makeFile {} io-75.3] + set fn [makeFile {} io-75.3] set f [open $fn w+] fconfigure $f -encoding binary - puts -nonewline $f "A\xC0" - flush $f - seek $f 0 - fconfigure $f -encoding utf-8 -buffering none + puts -nonewline $f "A\xC0" + flush $f + seek $f 0 + fconfigure $f -encoding utf-8 -buffering none } -body { - set d [read $f] - close $f - set d + set d [read $f] + close $f + set d } -cleanup { - removeFile io-75.3 + removeFile io-75.3 } -returnCodes ok -result "A\xC0" # ### ### ### ######### ######### ######### -- cgit v0.12 From 32439d945eea3cc4754f2779090075c16256f18a Mon Sep 17 00:00:00 2001 From: oehhar Date: Sun, 11 Sep 2022 13:45:04 +0000 Subject: Ticket [6978c01b65]: write not encodable character->report to script level Test io-75.5 now ok. --- generic/tclIO.c | 13 +++++++++++++ tests/io.test | 4 ++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/generic/tclIO.c b/generic/tclIO.c index 5317e30..732e103 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -4356,6 +4356,19 @@ Write( statePtr->outputEncodingFlags &= ~TCL_ENCODING_START; + /* + * See io-75.2, TCL bug 6978c01b65. + * Check, if an encoding error occured and should be reported to the + * script level. + * This happens, if a written character may not be represented by the + * current output encoding and strict encoding is active.hao_ + */ + + if (result == TCL_CONVERT_UNKNOWN) { + Tcl_SetErrno(EILSEQ); + return -1; + } + if ((result != TCL_OK) && (srcRead + dstWrote == 0)) { /* * We're reading from invalid/incomplete UTF-8. diff --git a/tests/io.test b/tests/io.test index 8b93317..9204208 100644 --- a/tests/io.test +++ b/tests/io.test @@ -8977,7 +8977,7 @@ test io-75.5 {unrepresentable character write passes and is replaced by ?} -setu set fn [makeFile {} io-75.5] set f [open $fn w+] fconfigure $f -encoding iso8859-1 -} -constraints knownBug -body { +} -body { puts -nonewline $f "A\u2022" } -body { puts -nonewline $f "A\u2022" @@ -8987,7 +8987,7 @@ test io-75.5 {unrepresentable character write passes and is replaced by ?} -setu } -cleanup { close $f removeFile io-75.5 -} -returnCodes error +} -returnCodes error -match glob -result {error writing "*": illegal byte sequence} # Incomplete sequence test. # This error may IMHO only be detected with the close. -- cgit v0.12 From 8f0f3b11657ba48eee382942394b7741af6b02cf Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sun, 11 Sep 2022 20:48:49 +0000 Subject: Change io-75.5 to test for both written output and which exception is thrown. This shows the bug is not fixed yet .... :-( --- tests/io.test | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/io.test b/tests/io.test index 9204208..3c7811a 100644 --- a/tests/io.test +++ b/tests/io.test @@ -8977,17 +8977,17 @@ test io-75.5 {unrepresentable character write passes and is replaced by ?} -setu set fn [makeFile {} io-75.5] set f [open $fn w+] fconfigure $f -encoding iso8859-1 -} -body { +} -constraints knownBug -body { puts -nonewline $f "A\u2022" } -body { - puts -nonewline $f "A\u2022" + catch {puts -nonewline $f "A\u2022"} msg flush $f seek $f 0 - read $f + list [read $f] $msg } -cleanup { close $f removeFile io-75.5 -} -returnCodes error -match glob -result {error writing "*": illegal byte sequence} +} -match glob -result [list {A} {error writing "*": illegal byte sequence}] # Incomplete sequence test. # This error may IMHO only be detected with the close. -- cgit v0.12 From fcfbc3a606dd31e90fcc3a6bb847042e19e1722b Mon Sep 17 00:00:00 2001 From: "jan.nijtmans" Date: Sun, 11 Sep 2022 21:37:34 +0000 Subject: complete the fix --- generic/tclIO.c | 20 +++++++++++++++++++- tests/io.test | 8 ++++---- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/generic/tclIO.c b/generic/tclIO.c index 5313eed..c6402f1 100644 --- a/generic/tclIO.c +++ b/generic/tclIO.c @@ -4335,6 +4335,7 @@ Write( char *nextNewLine = NULL; int endEncoding, saved = 0, total = 0, flushed = 0, needNlFlush = 0; char safe[BUFFER_PADDING]; + int encodingError = 0; if (srcLen) { WillWrite(chanPtr); @@ -4351,7 +4352,7 @@ Write( nextNewLine = (char *)memchr(src, '\n', srcLen); } - while (srcLen + saved + endEncoding > 0) { + while (srcLen + saved + endEncoding > 0 && !encodingError) { ChannelBuffer *bufPtr; char *dst; int result, srcRead, dstLen, dstWrote, srcLimit = srcLen; @@ -4390,6 +4391,19 @@ Write( statePtr->outputEncodingFlags &= ~TCL_ENCODING_START; + /* + * See io-75.2, TCL bug 6978c01b65. + * Check, if an encoding error occured and should be reported to the + * script level. + * This happens, if a written character may not be represented by the + * current output encoding and strict encoding is active. + */ + + if (result == TCL_CONVERT_UNKNOWN) { + encodingError = 1; + result = TCL_OK; + } + if ((result != TCL_OK) && (srcRead + dstWrote == 0)) { /* * We're reading from invalid/incomplete UTF-8. @@ -4497,6 +4511,10 @@ Write( UpdateInterest(chanPtr); + if (encodingError) { + Tcl_SetErrno(EILSEQ); + return -1; + } return total; } diff --git a/tests/io.test b/tests/io.test index 3d6b0da..cd04b115 100644 --- a/tests/io.test +++ b/tests/io.test @@ -8977,14 +8977,14 @@ test io-75.2 {unrepresentable character write passes and is replaced by ?} -setu set f [open $fn w+] fconfigure $f -encoding iso8859-1 } -body { - puts -nonewline $f "A\u2022" + catch {puts -nonewline $f "A\u2022"} msg flush $f seek $f 0 - read $f + list [read $f] $msg } -cleanup { close $f - removeFile io-75.2 -} -returnCodes ok -result "A" + removeFile io-75.5 +} -match glob -result [list {A} {error writing "*": illegal byte sequence}] # Incomplete sequence test. # This error may IMHO only be detected with the close. -- cgit v0.12