merge trunk

5 files changed, 312 insertions, 23 deletions

diff --git a/tests/dstring.test b/tests/dstring.test
index 06121a3..5feb355 100644
--- a/tests/dstring.test
+++ b/tests/dstring.test
@@ -180,16 +180,37 @@ test dstring-2.12 {appending list elements} -constraints testdstring -setup {
 } -cleanup {
     testdstring free
 } -result {x #}
-test dstring-2.13 {appending list elements} -constraints testdstring -body {
-    # This test shows lack of sophistication in Tcl_DStringAppendElement's
-    # decision about whether #-quoting can be disabled.
+test dstring-2.13 {appending list elements} -constraints testdstring -setup {
     testdstring free
+} -body {
+    # This test checks the sophistication in Tcl_DStringAppendElement's
+    # decision about whether #-quoting can be disabled.
     testdstring append "x " -1
     testdstring element #
     testdstring get
 } -cleanup {
     testdstring free
-} -result {x {#}}
+} -result {x #}
+test dstring-2.14 {appending list elements} -constraints testdstring -setup {
+    testdstring free
+} -body {
+    testdstring append "  " -1
+    testdstring element #
+    testdstring get
+} -cleanup {
+    testdstring free
+} -result {  {#}}
+test dstring-2.15 {appending list elements} -constraints testdstring -setup {
+    testdstring free
+} -body {
+    # This test checks the sophistication in Tcl_DStringAppendElement's
+    # decision about whether #-quoting can be disabled.
+    testdstring append "x  " -1
+    testdstring element #
+    testdstring get
+} -cleanup {
+    testdstring free
+} -result {x  #}
 
 test dstring-3.1 {nested sublists} -constraints testdstring -setup {
     testdstring free
@@ -306,10 +327,11 @@ test dstring-3.9 {appending list elements} -constraints testdstring -setup {
 } -cleanup {
     testdstring free
 } -result {x {x #}}
-test dstring-3.10 {appending list elements} -constraints testdstring -body {
-    # This test shows lack of sophistication in Tcl_DStringAppendElement's
-    # decision about whether #-quoting can be disabled.
+test dstring-3.10 {appending list elements} -constraints testdstring -setup {
     testdstring free
+} -body {
+    # This test checks the sophistication in Tcl_DStringAppendElement's
+    # decision about whether #-quoting can be disabled.
     testdstring append x -1
     testdstring start
     testdstring append "x " -1
@@ -318,7 +340,33 @@ test dstring-3.10 {appending list elements} -constraints testdstring -body {
     testdstring get
 } -cleanup {
     testdstring free
-} -result {x {x {#}}}
+} -result {x {x #}}
+test dstring-3.11 {appending list elements} -constraints testdstring -setup {
+    testdstring free
+} -body {
+    testdstring append x -1
+    testdstring start
+    testdstring append "  " -1
+    testdstring element #
+    testdstring end
+    testdstring get
+} -cleanup {
+    testdstring free
+} -result {x {  {#}}}
+test dstring-3.12 {appending list elements} -constraints testdstring -setup {
+    testdstring free
+} -body {
+    # This test checks the sophistication in Tcl_DStringAppendElement's
+    # decision about whether #-quoting can be disabled.
+    testdstring append x -1
+    testdstring start
+    testdstring append "x  " -1
+    testdstring element #
+    testdstring end
+    testdstring get
+} -cleanup {
+    testdstring free
+} -result {x {x  #}}
 
 test dstring-4.1 {truncation} -constraints testdstring -setup {
     testdstring free
diff --git a/tests/encoding.test b/tests/encoding.test
index be504fb..f21fd0e 100644
--- a/tests/encoding.test
+++ b/tests/encoding.test
@@ -33,6 +33,7 @@ proc runtests {} {
     variable x
 
 # Some tests require the testencoding command
+testConstraint testbytestring [llength [info commands testbytestring]]
 testConstraint testencoding [llength [info commands testencoding]]
 testConstraint testbytestring [llength [info commands testbytestring]]
 testConstraint teststringbytes [llength [info commands teststringbytes]]
@@ -316,11 +317,15 @@ test encoding-15.2 {UtfToUtfProc null character output} testbytestring {
     binary scan [testbytestring [encoding convertto utf-8 \u0000]] H* z
     set z
 } 00
-test encoding-15.3 {UtfToUtfProc null character input} teststringbytes {
+test encoding-15.3.a {UtfToUtfProc null character input} testbytestring {
+    binary scan [testbytestring \xc0\x80] H* z
+    set z
+} 00
+test encoding-15.3.b {UtfToUtfProc null character input} testbytestring {
     set y [encoding convertfrom utf-8 [encoding convertto utf-8 \u0000]]
-    binary scan [teststringbytes $y] H* z
+    binary scan [testbytestring $y] H* z
     set z
-} c080
+} 00
 test encoding-15.4 {UtfToUtfProc emoji character input} -body {
     set x \xED\xA0\xBD\xED\xB8\x82
     set y [encoding convertfrom utf-8 \xED\xA0\xBD\xED\xB8\x82]
diff --git a/tests/string.test b/tests/string.test
index b8f01a5..82c8771 100644
--- a/tests/string.test
+++ b/tests/string.test
@@ -32,6 +32,7 @@ testConstraint testobj [expr {[info commands testobj] ne {}}]
 testConstraint testindexobj [expr {[info commands testindexobj] ne {}}]
 testConstraint testevalex [expr {[info commands testevalex] ne {}}]
 testConstraint fullutf [expr {[string length \U010000] == 1}]
+testConstraint testbytestring   [llength [info commands testbytestring]]
 
 # Used for constraining memory leak tests
 testConstraint memory [llength [info commands memory]]
@@ -1813,6 +1814,34 @@ test string-20.5.$noComp {string trimright} {
 test string-20.6.$noComp {string trimright, unicode default} {
     run {string trimright ABC\u1361\x85\x00\xA0\u1680\u180E\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u200B\u2028\u2029\u202F\u205F\u3000}
 } ABC\u1361
+test string-20.7.$noComp {string trim on not valid utf-8 sequence (consider NTS as continuation char), bug [c61818e4c9]} {testbytestring knownBug} {
+    set result {}
+    set a [testbytestring \xC0\x80\x88]
+    set b foo$a
+    set m [list \x00 U \x88 V [testbytestring \x88] W]
+    lappend result [string map $m $b]
+    lappend result [string map $m [run {string trimright $b x}]]
+    lappend result [string map $m [run {string trimright $b \x00}]]
+    lappend result [string map $m [run {string trimleft $b fox}]]
+    lappend result [string map $m [run {string trimleft $b fo\x00}]]
+    lappend result [string map $m [run {string trim $b fox}]]
+    lappend result [string map $m [run {string trim $b fo\x00}]]
+} [list {*}[lrepeat 3 fooUV] {*}[lrepeat 2 UV V]]
+test string-20.8 {[c61818e4c9] [string trimright] fails when UtfPrev is ok} {testbytestring knownBug} {
+    set result {}
+    set a [testbytestring \xE8\x80]
+    set b foo$a
+    set m [list \xE8 U \x80 V [testbytestring \xE8] W [testbytestring \x80] X]]
+    lappend result [string map $m $b]
+    lappend result [string map $m [run {string trimright $b x}]]
+    lappend result [string map $m [run {string trimright $b \xE8}]]
+    lappend result [string map $m [run {string trimright $b [testbytestring \xE8]}]]
+    lappend result [string map $m [run {string trimright $b \x80}]]
+    lappend result [string map $m [run {string trimright $b [testbytestring \x80]}]]
+    lappend result [string map $m [run {string trimright $b \xE8\x80}]]
+    lappend result [string map $m [run {string trimright $b [testbytestring \xE8\x80]}]]
+    lappend result [string map $m [run {string trimright $b \u0000}]]
+} [list {*}[lrepeat 4 fooUV] {*}[lrepeat 2 fooU] {*}[lrepeat 2 foo] fooUV]
 
 test string-21.1.$noComp {string wordend} -body {
     list [catch {run {string wordend a}} msg] $msg
@@ -1902,10 +1931,15 @@ test string-22.12.$noComp {string wordstart, unicode} -body {
 test string-22.13.$noComp {string wordstart, unicode} -body {
     run {string wordstart "\uC700\uC700 abc" 8}
 } -result 3
-test string-22.14.$noComp {string wordstart, unicode} -body {
+test string-22.14.$noComp {string wordstart, invalid UTF-8} -constraints testbytestring -body {
+    # See Bug c61818e4c9
+    set demo [testbytestring "abc def\xE0\xA9ghi"]
+    run {string index $demo [string wordstart $demo 10]}
+} -result g
+test string-22.15.$noComp {string wordstart, unicode} -body {
     run {string wordstart "\U1D7CA\U1D7CA abc" 0}
 } -result 0
-test string-22.15.$noComp {string wordstart, unicode} -constraints fullutf -body {
+test string-22.16.$noComp {string wordstart, unicode} -constraints fullutf -body {
     run {string wordstart "\U1D7CA\U1D7CA abc" 10}
 } -result 3
 
diff --git a/tests/utf.test b/tests/utf.test
index 8f1a249..dc117d6 100644
--- a/tests/utf.test
+++ b/tests/utf.test
@@ -155,8 +155,167 @@ test utf-5.2 {Tcl_UtfFindLast} {testfindlast testbytestring} {
 test utf-6.1 {Tcl_UtfNext} {
 } {}
 
-test utf-7.1 {Tcl_UtfPrev} {
-} {}
+testConstraint testutfprev [llength [info commands testutfprev]]
+
+test utf-7.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev {}
+} 0
+test utf-7.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A
+} 0
+test utf-7.3 {Tcl_UtfPrev} testutfprev {
+    testutfprev AA
+} 1
+test utf-7.4 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8
+} 1
+test utf-7.4.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8\xA0\xA0\xA0 2
+} 1
+test utf-7.4.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8\xF8\xA0\xA0 2
+} 1
+test utf-7.5 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4
+} 1
+test utf-7.5.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4\xA0\xA0\xA0 2
+} 1
+test utf-7.5.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4\xF8\xA0\xA0 2
+} 1
+test utf-7.6 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8
+} 1
+test utf-7.6.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8\xA0\xA0\xA0 2
+} 1
+test utf-7.6.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8\xF8\xA0\xA0 2
+} 1
+test utf-7.7 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0
+} 1
+test utf-7.7.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0\xA0\xA0\xA0 2
+} 1
+test utf-7.7.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0\xF8\xA0\xA0 2
+} 1
+test utf-7.8 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0
+} 1
+test utf-7.8.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0\xA0\xA0\xA0 2
+} 1
+test utf-7.8.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0\xF8\xA0\xA0 2
+} 1
+test utf-7.9 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8\xA0
+} 2
+test utf-7.9.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8\xA0\xA0\xA0 3
+} 2
+test utf-7.9.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8\xA0\xF8\xA0 3
+} 2
+test utf-7.10 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4\xA0
+} 1
+test utf-7.10.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4\xA0\xA0\xA0 3
+} 1
+test utf-7.10.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4\xA0\xF8\xA0 3
+} 1
+test utf-7.11 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8\xA0
+} 1
+test utf-7.11.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8\xA0\xA0\xA0 3
+} 1
+test utf-7.11.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8\xA0\xF8\xA0 3
+} 1
+test utf-7.12 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0\xA0
+} 1
+test utf-7.12.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0\xA0\xA0\xA0 3
+} 1
+test utf-7.12.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0\xA0\xF8\xA0 3
+} 1
+test utf-7.13 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0\xA0
+} 2
+test utf-7.13.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0\xA0\xA0\xA0 3
+} 2
+test utf-7.13.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0\xA0\xF8\xA0 3
+} 2
+test utf-7.14 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8\xA0\xA0
+} 3
+test utf-7.14.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8\xA0\xA0\xA0 4
+} 3
+test utf-7.14.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8\xA0\xA0\xF8 4
+} 3
+test utf-7.15 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4\xA0\xA0
+} 1
+test utf-7.15.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4\xA0\xA0\xA0 4
+} 1
+test utf-7.15.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4\xA0\xA0\xF8 4
+} 1
+test utf-7.16 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8\xA0\xA0
+} 1
+test utf-7.16.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8\xA0\xA0\xA0 4
+} 1
+test utf-7.16.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8\xA0\xA0\xF8 4
+} 1
+test utf-7.17 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0\xA0\xA0
+} 3
+test utf-7.17.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0\xA0\xA0\xA0 4
+} 3
+test utf-7.17.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0\xA0\xA0\xF8 4
+} 3
+test utf-7.18 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0\xA0\xA0
+} 3
+test utf-7.18.1 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0\xA0\xA0\xA0 4
+} 3
+test utf-7.18.2 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0\xA0\xA0\xF8 4
+} 3
+test utf-7.19 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF8\xA0\xA0\xA0
+} 4
+test utf-7.20 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xF4\xA0\xA0\xA0
+} 1
+test utf-7.21 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xE8\xA0\xA0\xA0
+} 4
+test utf-7.22 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xD0\xA0\xA0\xA0
+} 4
+test utf-7.23 {Tcl_UtfPrev} testutfprev {
+    testutfprev A\xA0\xA0\xA0\xA0
+} 4
 
 test utf-8.1 {Tcl_UniCharAtIndex: index = 0} {
     string index abcd 0
diff --git a/tests/util.test b/tests/util.test
index 49fbebf..6a91a69 100644
--- a/tests/util.test
+++ b/tests/util.test
@@ -384,6 +384,10 @@ test util-5.50 {Tcl_StringMatch} {
 test util-5.51 {Tcl_StringMatch} {
     Wrapper_Tcl_StringMatch "" ""
 } 1
+test util-5.52 {Tcl_StringMatch} {
+    Wrapper_Tcl_StringMatch \[a\u0000 a\x80
+} 0
+
 
 test util-6.5 {Tcl_PrintDouble - make sure there's a decimal point} {
     concat x[expr 2.0]
@@ -433,25 +437,64 @@ test util-8.4 {TclNeedSpace - correct UTF8 handling} testdstring {
     llength [testdstring get]
 } 2
 test util-8.5 {TclNeedSpace - correct UTF8 handling} testdstring {
-    # Note that in this test TclNeedSpace actually gets it wrong,
-    # claiming we need a space when we really do not.  Extra space
-    # between list elements is harmless though, and better to have
-    # extra space in really weird string reps of lists, than to
-    # invest the effort required to make TclNeedSpace foolproof.
     testdstring free
     testdstring append {\\ } -1
     testdstring element foo
     list [llength [testdstring get]] [string length [testdstring get]]
-} {2 7}
+} {2 6}
 test util-8.6 {TclNeedSpace - correct UTF8 handling} testdstring {
-    # Another example of TclNeedSpace harmlessly getting it wrong.
     testdstring free
     testdstring append {\\ } -1
     testdstring append \{ -1
     testdstring element foo
     testdstring append \} -1
     list [llength [testdstring get]] [string length [testdstring get]]
-} {2 9}
+} {2 8}
+test util-8.7 {TclNeedSpace - watch out for escaped space} {
+    testdstring free
+    testdstring append {\ } -1
+    testdstring start
+    testdstring end
+
+    # Should make {\  {}}
+    list [llength [testdstring get]] [string index [testdstring get] 3]
+} {2 \{}
+test util-8.8 {TclNeedSpace - watch out for escaped space} {
+    testdstring free
+    testdstring append {\\ } -1
+    testdstring start
+    testdstring end
+
+    # Should make {\\ {}}
+    list [llength [testdstring get]] [string index [testdstring get] 3]
+} {2 \{}
+test util-8.9 {TclNeedSpace - watch out for escaped space} {
+    testdstring free
+    testdstring append {\\\ } -1
+    testdstring start
+    testdstring end
+
+    # Should make {\\\  {}}
+    list [llength [testdstring get]] [string index [testdstring get] 5]
+} {2 \{}
+test util-8.10 {TclNeedSpace - watch out for escaped space} {
+    testdstring free
+    testdstring append {\\\\\\\ } -1
+    testdstring start
+    testdstring end
+
+    # Should make {\\\\\\\  {}}
+    list [llength [testdstring get]] [string index [testdstring get] 9]
+} {2 \{}
+test util-8.11 {TclNeedSpace - watch out for escaped space} {
+    testdstring free
+    testdstring append {\\\\\\\\ } -1
+    testdstring start
+    testdstring end
+
+    # Should make {\\\\\\\\ {}}
+    list [llength [testdstring get]] [string index [testdstring get] 9]
+} {2 \{}
 
 test util-9.0.0 {Tcl_GetIntForIndex} {
     string index abcd 0