Improved "testutfnext" command

2 files changed, 123 insertions, 129 deletions

diff --git a/generic/tclTest.c b/generic/tclTest.c
index 8c3f2cf..008fdb7 100644
--- a/generic/tclTest.c
+++ b/generic/tclTest.c
@@ -6811,7 +6811,7 @@ SimpleListVolumes(void)
 /*
  * Used to check operations of Tcl_UtfNext.
  *
- * Usage: testutfnext $bytes $offset
+ * Usage: testutfnext -bytestring $bytes
  */
 
 static int
@@ -6821,40 +6821,46 @@ TestUtfNextCmd(
     int objc,
     Tcl_Obj *const objv[])
 {
-    int numBytes, offset = 0;
+    int numBytes;
     char *bytes;
-    const char *result;
-    Tcl_Obj *copy;
-
-    if (objc < 2 || objc > 3) {
-	Tcl_WrongNumArgs(interp, 1, objv, "bytes ?offset?");
-	return TCL_ERROR;
+    const char *result, *first;
+    char buffer[32];
+    static const char tobetested[] = "\xFF\xFE\xF4\xF2\xF0\xEF\xE8\xE3\xE2\xE1\xE0\xC2\xC1\xC0\x82";
+    const char *p = tobetested;
+
+    if (objc != 3 || strcmp(Tcl_GetString(objv[1]), "-bytestring")) {
+	if (objc != 2) {
+	    Tcl_WrongNumArgs(interp, 1, objv, "?-bytestring? bytes");
+	    return TCL_ERROR;
+	}
+	bytes = Tcl_GetStringFromObj(objv[1], &numBytes);
+    } else {
+	bytes = (char *) TclGetBytesFromObj(interp, objv[2], &numBytes);
+	if (bytes == NULL) {
+	    return TCL_ERROR;
+	}
     }
 
-    bytes = (char *) TclGetBytesFromObj(interp, objv[1], &numBytes);
-    if (bytes == NULL) {
+    if (numBytes > (int)sizeof(buffer)-2) {
+	Tcl_AppendResult(interp, "\"testutfnext\" can only handle 30 bytes", NULL);
 	return TCL_ERROR;
     }
 
-    if (objc == 3) {
-	if (TCL_OK != Tcl_GetIntForIndex(interp, objv[2], numBytes, &offset)) {
+    memcpy(buffer + 1, bytes, numBytes);
+    buffer[0] = buffer[numBytes + 1] = '\x00';
+
+    first = Tcl_UtfNext(buffer + 1);
+    while ((buffer[0] = *p++) != '\0') {
+	/* Run Tcl_UtfNext with many more possible bytes at src[-1], all should give the same result */
+	result = Tcl_UtfNext(buffer + 1);
+	if (first != result) {
+	    Tcl_AppendResult(interp, "Tcl_UtfNext is not supposed to read src[-1]", NULL);
 	    return TCL_ERROR;
 	}
-	if (offset < 0) {
-	    offset = 0;
-	}
-	if (offset > numBytes) {
-	    offset = numBytes;
-	}
     }
-    copy = Tcl_DuplicateObj(objv[1]);
-    bytes = (char *) Tcl_SetByteArrayLength(copy, numBytes+1);
-    bytes[numBytes] = '\0';
 
-    result = Tcl_UtfNext(bytes + offset);
-    Tcl_SetObjResult(interp, Tcl_NewIntObj(result - bytes));
+    Tcl_SetObjResult(interp, Tcl_NewIntObj(first - buffer - 1));
 
-    Tcl_DecrRefCount(copy);
     return TCL_OK;
 }
 /*
diff --git a/tests/utf.test b/tests/utf.test
index d4f45e9..af1e553 100644
--- a/tests/utf.test
+++ b/tests/utf.test
@@ -156,296 +156,284 @@ test utf-5.2 {Tcl_UtfFindLast} {testfindlast testbytestring} {
 test utf-6.1 {Tcl_UtfNext} testutfnext {
     # This takes the pointer one past the terminating NUL.
     # This is really an invalid call.
-    testutfnext {}
+    testutfnext -bytestring {}
 } 1
 test utf-6.2 {Tcl_UtfNext} testutfnext {
-    testutfnext A
+    testutfnext -bytestring A
 } 1
 test utf-6.3 {Tcl_UtfNext} testutfnext {
-    testutfnext AA
+    testutfnext -bytestring AA
 } 1
 test utf-6.4 {Tcl_UtfNext} testutfnext {
-    testutfnext A\xA0
+    testutfnext -bytestring A\xA0
 } 1
 test utf-6.5 {Tcl_UtfNext} testutfnext {
-    testutfnext A\xD0
+    testutfnext -bytestring A\xD0
 } 1
 test utf-6.6 {Tcl_UtfNext} testutfnext {
-    testutfnext A\xE8
+    testutfnext -bytestring A\xE8
 } 1
 test utf-6.7 {Tcl_UtfNext} testutfnext {
-    testutfnext A\xF2
+    testutfnext -bytestring A\xF2
 } 1
 test utf-6.8 {Tcl_UtfNext} testutfnext {
-    testutfnext A\xF8
+    testutfnext -bytestring A\xF8
 } 1
 test utf-6.9 {Tcl_UtfNext} testutfnext {
-    testutfnext \xA0
+    testutfnext -bytestring \xA0
 } 1
 test utf-6.10 {Tcl_UtfNext} testutfnext {
-    testutfnext \xA0G
+    testutfnext -bytestring \xA0G
 } 1
 test utf-6.11 {Tcl_UtfNext} testutfnext {
-    testutfnext \xA0\xA0
+    testutfnext -bytestring \xA0\xA0
 } 1
 test utf-6.12 {Tcl_UtfNext} testutfnext {
-    testutfnext \xA0\xD0
+    testutfnext -bytestring \xA0\xD0
 } 1
 test utf-6.13 {Tcl_UtfNext} testutfnext {
-    testutfnext \xA0\xE8
+    testutfnext -bytestring \xA0\xE8
 } 1
 test utf-6.14 {Tcl_UtfNext} testutfnext {
-    testutfnext \xA0\xF2
+    testutfnext -bytestring \xA0\xF2
 } 1
 test utf-6.15 {Tcl_UtfNext} testutfnext {
-    testutfnext \xA0\xF8
+    testutfnext -bytestring \xA0\xF8
 } 1
 test utf-6.16 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0
+    testutfnext -bytestring \xD0
 } 1
 test utf-6.17 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0G
+    testutfnext -bytestring \xD0G
 } 1
 test utf-6.18 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xA0
+    testutfnext -bytestring \xD0\xA0
 } 2
 test utf-6.19 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xD0
+    testutfnext -bytestring \xD0\xD0
 } 1
 test utf-6.20 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xE8
+    testutfnext -bytestring \xD0\xE8
 } 1
 test utf-6.21 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xF2
+    testutfnext -bytestring \xD0\xF2
 } 1
 test utf-6.22 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xF8
+    testutfnext -bytestring \xD0\xF8
 } 1
 test utf-6.23 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8
+    testutfnext -bytestring \xE8
 } 1
 test utf-6.24 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8G
+    testutfnext -bytestring \xE8G
 } 1
 test utf-6.25 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0
+    testutfnext -bytestring \xE8\xA0
 } 1
 test utf-6.26 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xD0
+    testutfnext -bytestring \xE8\xD0
 } 1
 test utf-6.27 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xE8
+    testutfnext -bytestring \xE8\xE8
 } 1
 test utf-6.28 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xF2
+    testutfnext -bytestring \xE8\xF2
 } 1
 test utf-6.29 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xF8
+    testutfnext -bytestring \xE8\xF8
 } 1
 test utf-6.30 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2
+    testutfnext -bytestring \xF2
 } 1
 test utf-6.31 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2G
+    testutfnext -bytestring \xF2G
 } 1
 test utf-6.32 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0
+    testutfnext -bytestring \xF2\xA0
 } 1
 test utf-6.33 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xD0
+    testutfnext -bytestring \xF2\xD0
 } 1
 test utf-6.34 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xE8
+    testutfnext -bytestring \xF2\xE8
 } 1
 test utf-6.35 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xF2
+    testutfnext -bytestring \xF2\xF2
 } 1
 test utf-6.36 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xF8
+    testutfnext -bytestring \xF2\xF8
 } 1
 test utf-6.37 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF8
+    testutfnext -bytestring \xF8
 } 1
 test utf-6.38 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF8G
+    testutfnext -bytestring \xF8G
 } 1
 test utf-6.39 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF8\xA0
+    testutfnext -bytestring \xF8\xA0
 } 1
 test utf-6.40 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF8\xD0
+    testutfnext -bytestring \xF8\xD0
 } 1
 test utf-6.41 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF8\xE8
+    testutfnext -bytestring \xF8\xE8
 } 1
 test utf-6.42 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF8\xF2
+    testutfnext -bytestring \xF8\xF2
 } 1
 test utf-6.43 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF8\xF8
+    testutfnext -bytestring \xF8\xF8
 } 1
 test utf-6.44 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xA0G
+    testutfnext -bytestring \xD0\xA0G
 } 2
 test utf-6.45 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xA0\xA0
+    testutfnext -bytestring \xD0\xA0\xA0
 } 2
 test utf-6.46 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xA0\xD0
+    testutfnext -bytestring \xD0\xA0\xD0
 } 2
 test utf-6.47 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xA0\xE8
+    testutfnext -bytestring \xD0\xA0\xE8
 } 2
 test utf-6.48 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xA0\xF2
+    testutfnext -bytestring \xD0\xA0\xF2
 } 2
 test utf-6.49 {Tcl_UtfNext} testutfnext {
-    testutfnext \xD0\xA0\xF8
+    testutfnext -bytestring \xD0\xA0\xF8
 } 2
 test utf-6.50 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0G
+    testutfnext -bytestring \xE8\xA0G
 } 1
 test utf-6.51 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xA0
+    testutfnext -bytestring \xE8\xA0\xA0
 } 3
 test utf-6.52 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xD0
+    testutfnext -bytestring \xE8\xA0\xD0
 } 1
 test utf-6.53 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xE8
+    testutfnext -bytestring \xE8\xA0\xE8
 } 1
 test utf-6.54 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xF2
+    testutfnext -bytestring \xE8\xA0\xF2
 } 1
 test utf-6.55 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xF8
+    testutfnext -bytestring \xE8\xA0\xF8
 } 1
 test utf-6.56 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0G
+    testutfnext -bytestring \xF2\xA0G
 } 1
 test utf-6.57 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0
+    testutfnext -bytestring \xF2\xA0\xA0
 } 1
 test utf-6.58 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xD0
+    testutfnext -bytestring \xF2\xA0\xD0
 } 1
 test utf-6.59 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xE8
+    testutfnext -bytestring \xF2\xA0\xE8
 } 1
 test utf-6.60 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xF2
+    testutfnext -bytestring \xF2\xA0\xF2
 } 1
 test utf-6.61 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xF8
+    testutfnext -bytestring \xF2\xA0\xF8
 } 1
 test utf-6.62 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xA0G
+    testutfnext -bytestring \xE8\xA0\xA0G
 } 3
 test utf-6.63 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xA0\xA0
+    testutfnext -bytestring \xE8\xA0\xA0\xA0
 } 3
 test utf-6.64 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xA0\xD0
+    testutfnext -bytestring \xE8\xA0\xA0\xD0
 } 3
 test utf-6.65 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xA0\xE8
+    testutfnext -bytestring \xE8\xA0\xA0\xE8
 } 3
 test utf-6.66 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xA0\xF2
+    testutfnext -bytestring \xE8\xA0\xA0\xF2
 } 3
 test utf-6.67 {Tcl_UtfNext} testutfnext {
-    testutfnext \xE8\xA0\xA0\xF8
+    testutfnext -bytestring \xE8\xA0\xA0\xF8
 } 3
 test utf-6.68 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0G
+    testutfnext -bytestring \xF2\xA0\xA0G
 } 1
 test utf-6.69 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xA0
+    testutfnext -bytestring \xF2\xA0\xA0\xA0
 } 4
 test utf-6.70 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xD0
+    testutfnext -bytestring \xF2\xA0\xA0\xD0
 } 1
 test utf-6.71 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xE8
+    testutfnext -bytestring \xF2\xA0\xA0\xE8
 } 1
 test utf-6.71 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xF2
+    testutfnext -bytestring \xF2\xA0\xA0\xF2
 } 1
 test utf-6.73 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xF8
+    testutfnext -bytestring \xF2\xA0\xA0\xF8
 } 1
 test utf-6.74 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xA0G
+    testutfnext -bytestring \xF2\xA0\xA0\xA0G
 } 4
 test utf-6.75 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xA0\xA0
+    testutfnext -bytestring \xF2\xA0\xA0\xA0\xA0
 } 4
 test utf-6.76 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xA0\xD0
+    testutfnext -bytestring \xF2\xA0\xA0\xA0\xD0
 } 4
 test utf-6.77 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xA0\xE8
+    testutfnext -bytestring \xF2\xA0\xA0\xA0\xE8
 } 4
 test utf-6.78 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xA0\xF2
+    testutfnext -bytestring \xF2\xA0\xA0\xA0\xF2
 } 4
 test utf-6.79 {Tcl_UtfNext} testutfnext {
-    testutfnext \xF2\xA0\xA0\xA0G\xF8
+    testutfnext -bytestring \xF2\xA0\xA0\xA0G\xF8
 } 4
 test utf-6.80 {Tcl_UtfNext - overlong sequences} testutfnext {
-    testutfnext \xC0\x80
+    testutfnext -bytestring \xC0\x80
 } 2
 test utf-6.81 {Tcl_UtfNext - overlong sequences} testutfnext {
-    testutfnext \xC0\x81
+    testutfnext -bytestring \xC0\x81
 } 1
 test utf-6.82 {Tcl_UtfNext - overlong sequences} testutfnext {
-    testutfnext \xC1\x80
+    testutfnext -bytestring \xC1\x80
 } 1
 test utf-6.83 {Tcl_UtfNext - overlong sequences} testutfnext {
-    testutfnext \xC2\x80
+    testutfnext -bytestring \xC2\x80
 } 2
 test utf-6.84 {Tcl_UtfNext - overlong sequences} testutfnext {
-    testutfnext \xE0\x80\x80
+    testutfnext -bytestring \xE0\x80\x80
 } 1
 test utf-6.85 {Tcl_UtfNext - overlong sequences} testutfnext {
-    testutfnext \xE0\xA0\x80
+    testutfnext -bytestring \xE0\xA0\x80
 } 3
 test utf-6.86 {Tcl_UtfNext - overlong sequences} testutfnext {
-    testutfnext \xF0\x80\x80\x80
+    testutfnext -bytestring \xF0\x80\x80\x80
 } 1
 test utf-6.87 {Tcl_UtfNext - overlong sequences} testutfnext {
-    testutfnext \xF0\x90\x80\x80
+    testutfnext -bytestring \xF0\x90\x80\x80
 } 4
 test utf-6.88 {Tcl_UtfNext, pointing to 2th byte of 3-byte valid sequence} testutfnext {
-    testutfnext \xA0\xA0
+    testutfnext -bytestring \xA0\xA0
 } 1
-test utf-6.88.1 {Tcl_UtfNext, pointing to 2th byte of 3-byte valid sequence} testutfnext {
-    testutfnext \xE8\xA0\xA0 1
-} 2
 test utf-6.89 {Tcl_UtfNext, pointing to 2th byte of 3-byte invalid sequence} testutfnext {
-    testutfnext \x80\x80
+    testutfnext -bytestring \x80\x80
 } 1
-test utf-6.89.1 {Tcl_UtfNext, pointing to 2th byte of 3-byte invalid sequence} testutfnext {
-    testutfnext \xF0\x80\x80 1
-} 2
 test utf-6.90 {Tcl_UtfNext, validity check [493dccc2de]} testutfnext {
-    testutfnext \xF4\x8F\xBF\xBF
+    testutfnext -bytestring \xF4\x8F\xBF\xBF
 } 4
 test utf-6.91 {Tcl_UtfNext, validity check [493dccc2de]} testutfnext {
-    testutfnext \xF4\x90\x80\x80
+    testutfnext -bytestring \xF4\x90\x80\x80
 } 1
 test utf-6.92 {Tcl_UtfNext, pointing to 2th byte of 4-byte valid sequence} testutfnext {
-    testutfnext \xA0\xA0\xA0
+    testutfnext -bytestring \xA0\xA0\xA0
 } 1
-test utf-6.92.1 {Tcl_UtfNext, pointing to 2th byte of 4-byte valid sequence} testutfnext {
-    testutfnext \xF2\xA0\xA0\xA0 1
-} 2
 test utf-6.93 {Tcl_UtfNext, pointing to 2th byte of 4-byte invalid sequence} testutfnext {
-    testutfnext \x80\x80\x80
+    testutfnext -bytestring \x80\x80\x80
 } 1
-test utf-6.93.1 {Tcl_UtfNext, pointing to 2th byte of 4-byte invalid sequence} testutfnext {
-    testutfnext \xF0\x80\x80\x80 1
-} 2
 
 test utf-7.1 {Tcl_UtfPrev} testutfprev {
     testutfprev {}