diff options
34 files changed, 144 insertions, 113 deletions
diff --git a/generic/tclEncoding.c b/generic/tclEncoding.c index e79cefd..ce5626f 100644 --- a/generic/tclEncoding.c +++ b/generic/tclEncoding.c @@ -36,7 +36,7 @@ typedef struct { * encoding is deleted. */ void *clientData; /* Arbitrary value associated with encoding * type. Passed to conversion functions. */ - size_t nullSize; /* Number of 0x00 bytes that signify + Tcl_Size nullSize; /* Number of 0x00 bytes that signify * end-of-string in this encoding. This number * is used to determine the source string * length when the srcLen argument is @@ -375,7 +375,7 @@ int Tcl_SetEncodingSearchPath( Tcl_Obj *searchPath) { - size_t dummy; + Tcl_Size dummy; if (TCL_ERROR == TclListObjLengthM(NULL, searchPath, &dummy)) { return TCL_ERROR; @@ -422,7 +422,7 @@ void TclSetLibraryPath( Tcl_Obj *path) { - size_t dummy; + Tcl_Size dummy; if (TCL_ERROR == TclListObjLengthM(NULL, path, &dummy)) { return; @@ -458,7 +458,7 @@ TclSetLibraryPath( static void FillEncodingFileMap(void) { - size_t i, numDirs = 0; + Tcl_Size i, numDirs = 0; Tcl_Obj *map, *searchPath; searchPath = Tcl_GetEncodingSearchPath(); @@ -473,7 +473,7 @@ FillEncodingFileMap(void) * entries found, we favor files earlier on the search path. */ - size_t j, numFiles; + Tcl_Size j, numFiles; Tcl_Obj *directory, *matchFileList; Tcl_Obj **filev; Tcl_GlobTypeData readableFiles = { @@ -942,7 +942,7 @@ Tcl_GetEncodingNames( * *--------------------------------------------------------------------------- */ -size_t +Tcl_Size Tcl_GetEncodingNulLength( Tcl_Encoding encoding) { @@ -1109,7 +1109,7 @@ Tcl_ExternalToUtfDString( Tcl_Encoding encoding, /* The encoding for the source string, or NULL * for the default system encoding. */ const char *src, /* Source string in specified encoding. */ - size_t srcLen, /* Source string length in bytes, or -1 for + Tcl_Size srcLen, /* Source string length in bytes, or < 0 for * encoding-specific string length. */ Tcl_DString *dstPtr) /* Uninitialized or free DString in which the * converted string is stored. */ @@ -1146,12 +1146,12 @@ Tcl_ExternalToUtfDString( *------------------------------------------------------------------------- */ -size_t +Tcl_Size Tcl_ExternalToUtfDStringEx( Tcl_Encoding encoding, /* The encoding for the source string, or NULL * for the default system encoding. */ const char *src, /* Source string in specified encoding. */ - size_t srcLen, /* Source string length in bytes, or TCL_INDEX_NONE for + Tcl_Size srcLen, /* Source string length in bytes, or < 0 for * encoding-specific string length. */ int flags, /* Conversion control flags. */ Tcl_DString *dstPtr) /* Uninitialized or free DString in which the @@ -1161,7 +1161,7 @@ Tcl_ExternalToUtfDStringEx( Tcl_EncodingState state; const Encoding *encodingPtr; int result, soFar, srcRead, dstWrote, dstChars; - size_t dstLen; + Tcl_Size dstLen; const char *srcStart = src; Tcl_DStringInit(dstPtr); @@ -1192,7 +1192,7 @@ Tcl_ExternalToUtfDStringEx( src += srcRead; if (result != TCL_CONVERT_NOSPACE) { Tcl_DStringSetLength(dstPtr, soFar); - return (result == TCL_OK) ? TCL_INDEX_NONE : (size_t)(src - srcStart); + return (result == TCL_OK) ? TCL_INDEX_NONE : (Tcl_Size)(src - srcStart); } flags &= ~TCL_ENCODING_START; srcLen -= srcRead; @@ -1229,8 +1229,8 @@ Tcl_ExternalToUtf( Tcl_Encoding encoding, /* The encoding for the source string, or NULL * for the default system encoding. */ const char *src, /* Source string in specified encoding. */ - size_t srcLen, /* Source string length in bytes, or -1 - * for encoding-specific string length. */ + Tcl_Size srcLen, /* Source string length in bytes, or < 0 for + * encoding-specific string length. */ int flags, /* Conversion control flags. */ Tcl_EncodingState *statePtr,/* Place for conversion routine to store state * information used during a piecewise @@ -1239,7 +1239,7 @@ Tcl_ExternalToUtf( * routine under control of flags argument. */ char *dst, /* Output buffer in which converted string is * stored. */ - size_t dstLen, /* The maximum length of output buffer in + Tcl_Size dstLen, /* The maximum length of output buffer in * bytes. */ int *srcReadPtr, /* Filled with the number of bytes from the * source string that were converted. This may @@ -1346,7 +1346,7 @@ Tcl_UtfToExternalDString( Tcl_Encoding encoding, /* The encoding for the converted string, or * NULL for the default system encoding. */ const char *src, /* Source string in UTF-8. */ - size_t srcLen, /* Source string length in bytes, or -1 for + Tcl_Size srcLen, /* Source string length in bytes, or < 0 for * strlen(). */ Tcl_DString *dstPtr) /* Uninitialized or free DString in which the * converted string is stored. */ @@ -1384,12 +1384,12 @@ Tcl_UtfToExternalDString( *------------------------------------------------------------------------- */ -size_t +Tcl_Size Tcl_UtfToExternalDStringEx( Tcl_Encoding encoding, /* The encoding for the converted string, or * NULL for the default system encoding. */ const char *src, /* Source string in UTF-8. */ - size_t srcLen, /* Source string length in bytes, or < 0 for + Tcl_Size srcLen, /* Source string length in bytes, or < 0 for * strlen(). */ int flags, /* Conversion control flags. */ Tcl_DString *dstPtr) /* Uninitialized or free DString in which the @@ -1400,7 +1400,7 @@ Tcl_UtfToExternalDStringEx( const Encoding *encodingPtr; int result, soFar, srcRead, dstWrote, dstChars; const char *srcStart = src; - size_t dstLen; + Tcl_Size dstLen; Tcl_DStringInit(dstPtr); dst = Tcl_DStringValue(dstPtr); @@ -1429,7 +1429,7 @@ Tcl_UtfToExternalDStringEx( while (i >= soFar) { Tcl_DStringSetLength(dstPtr, i--); } - return (result == TCL_OK) ? TCL_INDEX_NONE : (size_t)(src - srcStart); + return (result == TCL_OK) ? TCL_INDEX_NONE : (Tcl_Size)(src - srcStart); } flags &= ~TCL_ENCODING_START; @@ -1467,8 +1467,8 @@ Tcl_UtfToExternal( Tcl_Encoding encoding, /* The encoding for the converted string, or * NULL for the default system encoding. */ const char *src, /* Source string in UTF-8. */ - size_t srcLen, /* Source string length in bytes, or -1 - * for strlen(). */ + Tcl_Size srcLen, /* Source string length in bytes, or < 0 for + * strlen(). */ int flags, /* Conversion control flags. */ Tcl_EncodingState *statePtr,/* Place for conversion routine to store state * information used during a piecewise @@ -1477,7 +1477,7 @@ Tcl_UtfToExternal( * routine under control of flags argument. */ char *dst, /* Output buffer in which converted string * is stored. */ - size_t dstLen, /* The maximum length of output buffer in + Tcl_Size dstLen, /* The maximum length of output buffer in * bytes. */ int *srcReadPtr, /* Filled with the number of bytes from the * source string that were converted. This may @@ -1588,7 +1588,7 @@ OpenEncodingFileChannel( Tcl_Obj *map = TclGetProcessGlobalValue(&encodingFileMap); Tcl_Obj **dir, *path, *directory = NULL; Tcl_Channel chan = NULL; - size_t i, numDirs; + Tcl_Size i, numDirs; TclListObjGetElementsM(NULL, searchPath, &numDirs, &dir); Tcl_IncrRefCount(nameObj); @@ -1853,7 +1853,7 @@ LoadTableEncoding( for (i = 0; i < numPages; i++) { int ch; const char *p; - size_t expected = 3 + 16 * (16 * 4 + 1); + Tcl_Size expected = 3 + 16 * (16 * 4 + 1); if (Tcl_ReadChars(chan, objPtr, expected, 0) != expected) { return NULL; @@ -2089,7 +2089,7 @@ LoadEscapeEncoding( Tcl_DStringInit(&escapeData); while (1) { - size_t argc; + Tcl_Size argc; const char **argv; char *line; Tcl_DString lineString; @@ -2368,10 +2368,10 @@ UtfToUtfProc( if (flags & ENCODING_INPUT) { if ((len < 2) && (ch != 0) && (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX))) { - result = TCL_CONVERT_SYNTAX; - break; + goto utf8Syntax; } else if ((ch > 0xFFFF) && !(flags & ENCODING_UTF) && (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) || (flags & ENCODING_FAILINDEX))) { + utf8Syntax: result = TCL_CONVERT_SYNTAX; break; } @@ -2483,7 +2483,7 @@ Utf32ToUtfProc( const char *srcStart, *srcEnd; const char *dstEnd, *dstStart; int result, numChars, charLimit = INT_MAX; - int ch; + int ch, bytesLeft = srcLen % 4; flags |= PTR2INT(clientData); if (flags & TCL_ENCODING_CHAR_LIMIT) { @@ -2495,9 +2495,9 @@ Utf32ToUtfProc( * Check alignment with utf-32 (4 == sizeof(UTF-32)) */ - if ((srcLen % 4) != 0) { + if (bytesLeft != 0) { result = TCL_CONVERT_MULTIBYTE; - srcLen &= -4; + srcLen -= bytesLeft; } srcStart = src; @@ -2544,6 +2544,22 @@ Utf32ToUtfProc( src += sizeof(unsigned int); } + if ((flags & TCL_ENCODING_END) && (result == TCL_CONVERT_MULTIBYTE)) { + /* We have a single byte left-over at the end */ + if (dst > dstEnd) { + result = TCL_CONVERT_NOSPACE; + } else { + /* destination is not full, so we really are at the end now */ + if ((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT) { + result = TCL_CONVERT_SYNTAX; + } else { + result = TCL_OK; + dst += Tcl_UniCharToUtf(0xFFFD, dst); + numChars++; + src += bytesLeft; + } + } + } *srcReadPtr = src - srcStart; *dstWrotePtr = dst - dstStart; *dstCharsPtr = numChars; @@ -2756,6 +2772,22 @@ Utf16ToUtfProc( /* Bug [10c2c17c32]. If Hi surrogate, finish 3-byte UTF-8 */ dst += Tcl_UniCharToUtf(-1, dst); } + if ((flags & TCL_ENCODING_END) && (result == TCL_CONVERT_MULTIBYTE)) { + /* We have a single byte left-over at the end */ + if (dst > dstEnd) { + result = TCL_CONVERT_NOSPACE; + } else { + /* destination is not full, so we really are at the end now */ + if (((flags & TCL_ENCODING_STRICT) == TCL_ENCODING_STRICT)) { + result = TCL_CONVERT_SYNTAX; + } else { + result = TCL_OK; + dst += Tcl_UniCharToUtf(0xFFFD, dst); + numChars++; + src++; + } + } + } *srcReadPtr = src - srcStart; *dstWrotePtr = dst - dstStart; *dstCharsPtr = numChars; @@ -2940,8 +2972,8 @@ UtfToUcs2Proc( len = TclUtfToUniChar(src, &ch); if ((ch >= 0xD800) && (len < 3)) { if (STOPONERROR) { - result = TCL_CONVERT_UNKNOWN; - break; + result = TCL_CONVERT_UNKNOWN; + break; } src += len; src += TclUtfToUniChar(src, &ch); @@ -2951,8 +2983,8 @@ UtfToUcs2Proc( len = TclUtfToUniChar(src, &ch); if (ch > 0xFFFF) { if (STOPONERROR) { - result = TCL_CONVERT_UNKNOWN; - break; + result = TCL_CONVERT_UNKNOWN; + break; } ch = 0xFFFD; } @@ -4022,11 +4054,11 @@ unilen4( static void InitializeEncodingSearchPath( char **valuePtr, - size_t *lengthPtr, + TCL_HASH_TYPE *lengthPtr, Tcl_Encoding *encodingPtr) { const char *bytes; - size_t i, numDirs, numBytes; + Tcl_Size i, numDirs, numBytes; Tcl_Obj *libPathObj, *encodingObj, *searchPathObj; TclNewLiteralStringObj(encodingObj, "encoding"); diff --git a/library/encoding/ascii.enc b/library/encoding/ascii.enc index e0320b8..284a9f5 100644 --- a/library/encoding/ascii.enc +++ b/library/encoding/ascii.enc @@ -9,7 +9,7 @@ S 0040004100420043004400450046004700480049004A004B004C004D004E004F 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F -0070007100720073007400750076007700780079007A007B007C007D007E0000 +0070007100720073007400750076007700780079007A007B007C007D007E007F 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 diff --git a/library/encoding/big5.enc b/library/encoding/big5.enc index 26179f4..d6ff760 100644 --- a/library/encoding/big5.enc +++ b/library/encoding/big5.enc @@ -10,8 +10,8 @@ M 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080008100820083008400850086008700880089008A008B008C008D008E008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +0000000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 diff --git a/library/encoding/cp1250.enc b/library/encoding/cp1250.enc index 070ad90..f40b485 100644 --- a/library/encoding/cp1250.enc +++ b/library/encoding/cp1250.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -20AC0081201A0083201E2026202020210088203001602039015A0164017D0179 -009020182019201C201D202220132014009821220161203A015B0165017E017A +20AC0000201A0000201E2026202020210000203001602039015A0164017D0179 +000020182019201C201D202220132014000021220161203A015B0165017E017A 00A002C702D8014100A4010400A600A700A800A9015E00AB00AC00AD00AE017B 00B000B102DB014200B400B500B600B700B80105015F00BB013D02DD013E017C 015400C100C2010200C40139010600C7010C00C9011800CB011A00CD00CE010E diff --git a/library/encoding/cp1251.enc b/library/encoding/cp1251.enc index 376b1b4..f9513c2 100644 --- a/library/encoding/cp1251.enc +++ b/library/encoding/cp1251.enc @@ -11,7 +11,7 @@ S 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F 04020403201A0453201E20262020202120AC203004092039040A040C040B040F -045220182019201C201D202220132014009821220459203A045A045C045B045F +045220182019201C201D202220132014000021220459203A045A045C045B045F 00A0040E045E040800A4049000A600A7040100A9040400AB00AC00AD00AE0407 00B000B104060456049100B500B600B704512116045400BB0458040504550457 0410041104120413041404150416041704180419041A041B041C041D041E041F diff --git a/library/encoding/cp1252.enc b/library/encoding/cp1252.enc index dd525ea..b45a7f8 100644 --- a/library/encoding/cp1252.enc +++ b/library/encoding/cp1252.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -20AC0081201A0192201E20262020202102C62030016020390152008D017D008F -009020182019201C201D20222013201402DC21220161203A0153009D017E0178 +20AC0000201A0192201E20262020202102C620300160203901520000017D0000 +000020182019201C201D20222013201402DC21220161203A01530000017E0178 00A000A100A200A300A400A500A600A700A800A900AA00AB00AC00AD00AE00AF 00B000B100B200B300B400B500B600B700B800B900BA00BB00BC00BD00BE00BF 00C000C100C200C300C400C500C600C700C800C900CA00CB00CC00CD00CE00CF diff --git a/library/encoding/cp1253.enc b/library/encoding/cp1253.enc index a8754c3..dcc8084 100644 --- a/library/encoding/cp1253.enc +++ b/library/encoding/cp1253.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -20AC0081201A0192201E20262020202100882030008A2039008C008D008E008F -009020182019201C201D20222013201400982122009A203A009C009D009E009F +20AC0000201A0192201E20262020202100002030000020390000000000000000 +000020182019201C201D202220132014000021220000203A0000000000000000 00A00385038600A300A400A500A600A700A800A9000000AB00AC00AD00AE2015 00B000B100B200B3038400B500B600B703880389038A00BB038C00BD038E038F 0390039103920393039403950396039703980399039A039B039C039D039E039F diff --git a/library/encoding/cp1254.enc b/library/encoding/cp1254.enc index b9e3b3c..4922f3c 100644 --- a/library/encoding/cp1254.enc +++ b/library/encoding/cp1254.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -20AC0081201A0192201E20262020202102C62030016020390152008D008E008F -009020182019201C201D20222013201402DC21220161203A0153009D009E0178 +20AC0000201A0192201E20262020202102C62030016020390152000000000000 +000020182019201C201D20222013201402DC21220161203A0153000000000178 00A000A100A200A300A400A500A600A700A800A900AA00AB00AC00AD00AE00AF 00B000B100B200B300B400B500B600B700B800B900BA00BB00BC00BD00BE00BF 00C000C100C200C300C400C500C600C700C800C900CA00CB00CC00CD00CE00CF diff --git a/library/encoding/cp1255.enc b/library/encoding/cp1255.enc index 6e78b95..74ef0c1 100644 --- a/library/encoding/cp1255.enc +++ b/library/encoding/cp1255.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -20AC0081201A0192201E20262020202102C62030008A2039008C008D008E008F -009020182019201C201D20222013201402DC2122009A203A009C009D009E009F +20AC0000201A0192201E20262020202102C62030000020390000000000000000 +000020182019201C201D20222013201402DC21220000203A0000000000000000 00A000A100A200A320AA00A500A600A700A800A900D700AB00AC00AD00AE00AF 00B000B100B200B300B400B500B600B700B800B900F700BB00BC00BD00BE00BF 05B005B105B205B305B405B505B605B705B805B9000005BB05BC05BD05BE05BF diff --git a/library/encoding/cp1257.enc b/library/encoding/cp1257.enc index 4aa135d..42c6905 100644 --- a/library/encoding/cp1257.enc +++ b/library/encoding/cp1257.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -20AC0081201A0083201E20262020202100882030008A2039008C00A802C700B8 -009020182019201C201D20222013201400982122009A203A009C00AF02DB009F +20AC0000201A0000201E2026202020210000203000002039000000A802C700B8 +000020182019201C201D202220132014000021220000203A000000AF02DB0000 00A0000000A200A300A4000000A600A700D800A9015600AB00AC00AD00AE00C6 00B000B100B200B300B400B500B600B700F800B9015700BB00BC00BD00BE00E6 0104012E0100010600C400C501180112010C00C90179011601220136012A013B diff --git a/library/encoding/cp1258.enc b/library/encoding/cp1258.enc index 95fdef8..bbe2b12 100644 --- a/library/encoding/cp1258.enc +++ b/library/encoding/cp1258.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -20AC0081201A0192201E20262020202102C62030008A20390152008D008E008F -009020182019201C201D20222013201402DC2122009A203A0153009D009E0178 +20AC0000201A0192201E20262020202102C62030000020390152000000000000 +000020182019201C201D20222013201402DC21220000203A0153000000000178 00A000A100A200A300A400A500A600A700A800A900AA00AB00AC00AD00AE00AF 00B000B100B200B300B400B500B600B700B800B900BA00BB00BC00BD00BE00BF 00C000C100C2010200C400C500C600C700C800C900CA00CB030000CD00CE00CF diff --git a/library/encoding/cp864.enc b/library/encoding/cp864.enc index 71f9e62..dad7c20 100644 --- a/library/encoding/cp864.enc +++ b/library/encoding/cp864.enc @@ -11,7 +11,7 @@ S 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F 00B000B72219221A259225002502253C2524252C251C25342510250C25142518 -03B2221E03C600B100BD00BC224800AB00BBFEF7FEF8009B009CFEFBFEFC009F +03B2221E03C600B100BD00BC224800AB00BBFEF7FEF800000000FEFBFEFC0000 00A000ADFE8200A300A4FE8400000000FE8EFE8FFE95FE99060CFE9DFEA1FEA5 0660066106620663066406650666066706680669FED1061BFEB1FEB5FEB9061F 00A2FE80FE81FE83FE85FECAFE8BFE8DFE91FE93FE97FE9BFE9FFEA3FEA7FEA9 diff --git a/library/encoding/cp869.enc b/library/encoding/cp869.enc index 9fd2929..4670826 100644 --- a/library/encoding/cp869.enc +++ b/library/encoding/cp869.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080008100820083008400850386008700B700AC00A620182019038820150389 -038A03AA038C00930094038E03AB00A9038F00B200B303AC00A303AD03AE03AF +0000000000000000000000000386000000B700AC00A620182019038820150389 +038A03AA038C00000000038E03AB00A9038F00B200B303AC00A303AD03AE03AF 03CA039003CC03CD039103920393039403950396039700BD0398039900AB00BB 25912592259325022524039A039B039C039D256325512557255D039E039F2510 25142534252C251C2500253C03A003A1255A25542569256625602550256C03A3 diff --git a/library/encoding/cp874.enc b/library/encoding/cp874.enc index 0487b97..e2e8433 100644 --- a/library/encoding/cp874.enc +++ b/library/encoding/cp874.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -20AC008100820083008420260086008700880089008A008B008C008D008E008F -009020182019201C201D20222013201400980099009A009B009C009D009E009F +20AC000000000000000020260000000000000000000000000000000000000000 +000020182019201C201D20222013201400000000000000000000000000000000 00A00E010E020E030E040E050E060E070E080E090E0A0E0B0E0C0E0D0E0E0E0F 0E100E110E120E130E140E150E160E170E180E190E1A0E1B0E1C0E1D0E1E0E1F 0E200E210E220E230E240E250E260E270E280E290E2A0E2B0E2C0E2D0E2E0E2F diff --git a/library/encoding/cp932.enc b/library/encoding/cp932.enc index 8da8cd6..0699000 100644 --- a/library/encoding/cp932.enc +++ b/library/encoding/cp932.enc @@ -10,7 +10,7 @@ M 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080000000000000000000850086000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F diff --git a/library/encoding/cp949.enc b/library/encoding/cp949.enc index 2f3ec39..459dbd9 100644 --- a/library/encoding/cp949.enc +++ b/library/encoding/cp949.enc @@ -10,7 +10,7 @@ M 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 diff --git a/library/encoding/cp950.enc b/library/encoding/cp950.enc index f33d785..f582bd9 100644 --- a/library/encoding/cp950.enc +++ b/library/encoding/cp950.enc @@ -10,8 +10,8 @@ M 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080008100820083008400850086008700880089008A008B008C008D008E008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +0000000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 diff --git a/library/encoding/dingbats.enc b/library/encoding/dingbats.enc index 9729487..bd466b2 100644 --- a/library/encoding/dingbats.enc +++ b/library/encoding/dingbats.enc @@ -10,8 +10,8 @@ S 2730273127322733273427352736273727382739273A273B273C273D273E273F 2740274127422743274427452746274727482749274A274B25CF274D25A0274F 27502751275225B225BC25C6275625D727582759275A275B275C275D275E007F -0080008100820083008400850086008700880089008A008B008C008D008E008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +0000000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000276127622763276427652766276726632666266526602460246124622463 2464246524662467246824692776277727782779277A277B277C277D277E277F 2780278127822783278427852786278727882789278A278B278C278D278E278F diff --git a/library/encoding/ebcdic.enc b/library/encoding/ebcdic.enc index f451de5..f83ce7d 100644 --- a/library/encoding/ebcdic.enc +++ b/library/encoding/ebcdic.enc @@ -1,3 +1,4 @@ +# Encoding file: ebcdic, single-byte S 006F 0 1 00 diff --git a/library/encoding/euc-cn.enc b/library/encoding/euc-cn.enc index 4b2f8c7..ff0f984 100644 --- a/library/encoding/euc-cn.enc +++ b/library/encoding/euc-cn.enc @@ -10,8 +10,8 @@ M 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080008100820083008400850086008700880089008A008B008C008D008E008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +0000000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 diff --git a/library/encoding/euc-jp.enc b/library/encoding/euc-jp.enc index db56c88..d4337d9 100644 --- a/library/encoding/euc-jp.enc +++ b/library/encoding/euc-jp.enc @@ -10,8 +10,8 @@ M 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080008100820083008400850086008700880089008A008B008C008D0000008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +0000000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 diff --git a/library/encoding/euc-kr.enc b/library/encoding/euc-kr.enc index 5e9bb93..0433260 100644 --- a/library/encoding/euc-kr.enc +++ b/library/encoding/euc-kr.enc @@ -10,8 +10,8 @@ M 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080008100820083008400850086008700880089008A008B008C008D008E008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +0000000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 diff --git a/library/encoding/gb1988.enc b/library/encoding/gb1988.enc index 298732c..8254684 100644 --- a/library/encoding/gb1988.enc +++ b/library/encoding/gb1988.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D203E007F -0080008100820083008400850086008700880089008A008B008C008D008E008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +0000000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F FF80FF81FF82FF83FF84FF85FF86FF87FF88FF89FF8AFF8BFF8CFF8DFF8EFF8F diff --git a/library/encoding/jis0201.enc b/library/encoding/jis0201.enc index 64f423f..70e099d 100644 --- a/library/encoding/jis0201.enc +++ b/library/encoding/jis0201.enc @@ -10,8 +10,8 @@ S 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D203E007F -0080008100820083008400850086008700880089008A008B008C008D008E008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +0000000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F FF80FF81FF82FF83FF84FF85FF86FF87FF88FF89FF8AFF8BFF8CFF8DFF8EFF8F diff --git a/library/encoding/macDingbats.enc b/library/encoding/macDingbats.enc index 28449cd..9fa47b5 100644 --- a/library/encoding/macDingbats.enc +++ b/library/encoding/macDingbats.enc @@ -10,8 +10,8 @@ S 2730273127322733273427352736273727382739273A273B273C273D273E273F 2740274127422743274427452746274727482749274A274B25CF274D25A0274F 27502751275225B225BC25C6275625D727582759275A275B275C275D275E007F -F8D7F8D8F8D9F8DAF8DBF8DCF8DDF8DEF8DFF8E0F8E1F8E2F8E3F8E4008E008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +F8D7F8D8F8D9F8DAF8DBF8DCF8DDF8DEF8DFF8E0F8E1F8E2F8E3F8E400000000 +0000000000000000000000000000000000000000000000000000000000000000 0000276127622763276427652766276726632666266526602460246124622463 2464246524662467246824692776277727782779277A277B277C277D277E277F 2780278127822783278427852786278727882789278A278B278C278D278E278F diff --git a/library/encoding/macJapan.enc b/library/encoding/macJapan.enc index dba24bd..9f3f03b 100644 --- a/library/encoding/macJapan.enc +++ b/library/encoding/macJapan.enc @@ -10,7 +10,7 @@ M 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 00A0FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F diff --git a/library/encoding/shiftjis.enc b/library/encoding/shiftjis.enc index 140aec4..3ba972e 100644 --- a/library/encoding/shiftjis.enc +++ b/library/encoding/shiftjis.enc @@ -10,7 +10,7 @@ M 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F 0070007100720073007400750076007700780079007A007B007C007D007E007F -0080000000000000000000850086008700000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 0000FF61FF62FF63FF64FF65FF66FF67FF68FF69FF6AFF6BFF6CFF6DFF6EFF6F FF70FF71FF72FF73FF74FF75FF76FF77FF78FF79FF7AFF7BFF7CFF7DFF7EFF7F diff --git a/library/encoding/symbol.enc b/library/encoding/symbol.enc index ffda9e3..ebd2f49 100644 --- a/library/encoding/symbol.enc +++ b/library/encoding/symbol.enc @@ -10,8 +10,8 @@ S 03A0039803A103A303A403A503C203A9039E03A80396005B2234005D22A5005F F8E503B103B203C703B403B503C603B303B703B903D503BA03BB03BC03BD03BF 03C003B803C103C303C403C503D603C903BE03C803B6007B007C007D223C007F -0080008100820083008400850086008700880089008A008B008C008D008E008F -0090009100920093009400950096009700980099009A009B009C009D009E009F +0000000000000000000000000000000000000000000000000000000000000000 +0000000000000000000000000000000000000000000000000000000000000000 000003D2203222642044221E0192266326662665266021942190219121922193 00B000B12033226500D7221D2202202200F72260226122482026F8E6F8E721B5 21352111211C21182297229522052229222A2283228722842282228622082209 diff --git a/library/encoding/tis-620.enc b/library/encoding/tis-620.enc index 2e9142a..af77326 100644 --- a/library/encoding/tis-620.enc +++ b/library/encoding/tis-620.enc @@ -9,7 +9,7 @@ S 0040004100420043004400450046004700480049004A004B004C004D004E004F 0050005100520053005400550056005700580059005A005B005C005D005E005F 0060006100620063006400650066006700680069006A006B006C006D006E006F -0070007100720073007400750076007700780079007A007B007C007D007E0000 +0070007100720073007400750076007700780079007A007B007C007D007E007F 0000000000000000000000000000000000000000000000000000000000000000 0000000000000000000000000000000000000000000000000000000000000000 00000E010E020E030E040E050E060E070E080E090E0A0E0B0E0C0E0D0E0E0E0F diff --git a/tests/chanio.test b/tests/chanio.test index f02711e..ec0dbbd 100644 --- a/tests/chanio.test +++ b/tests/chanio.test @@ -6731,8 +6731,8 @@ test chan-io-52.5 {TclCopyChannel, all} -constraints {fcopy} -setup { } -body { set f1 [open $thisScript] set f2 [open $path(test1) w] - chan configure $f1 -translation lf -blocking 0 - chan configure $f2 -translation lf -blocking 0 + chan configure $f1 -translation lf -encoding iso8859-1 -blocking 0 + chan configure $f2 -translation lf -encoding iso8859-1 -blocking 0 chan copy $f1 $f2 -size -1 ;# -1 means 'copy all', same as if no -size specified. set result [list [chan configure $f1 -blocking] [chan configure $f2 -blocking]] chan close $f1 @@ -6747,8 +6747,8 @@ test chan-io-52.5a {TclCopyChannel, all, other negative value} -setup { } -constraints {fcopy} -body { set f1 [open $thisScript] set f2 [open $path(test1) w] - chan configure $f1 -translation lf -blocking 0 - chan configure $f2 -translation lf -blocking 0 + chan configure $f1 -translation lf -encoding iso8859-1 -blocking 0 + chan configure $f2 -translation lf -encoding iso8859-1 -blocking 0 chan copy $f1 $f2 -size -2 ;# < 0 behaves like -1, copy all set result [list [chan configure $f1 -blocking] [chan configure $f2 -blocking]] chan close $f1 @@ -6763,8 +6763,8 @@ test chan-io-52.5b {TclCopyChannel, all, wrap to negative value} -setup { } -constraints {fcopy} -body { set f1 [open $thisScript] set f2 [open $path(test1) w] - chan configure $f1 -translation lf -blocking 0 - chan configure $f2 -translation lf -blocking 0 + chan configure $f1 -translation lf -encoding iso8859-1 -blocking 0 + chan configure $f2 -translation lf -encoding iso8859-1 -blocking 0 chan copy $f1 $f2 -size 3221176172 ;# Wrapped to < 0, behaves like -1, copy all set result [list [chan configure $f1 -blocking] [chan configure $f2 -blocking]] chan close $f1 @@ -6797,8 +6797,8 @@ test chan-io-52.7 {TclCopyChannel} -constraints {fcopy} -setup { } -body { set f1 [open $thisScript] set f2 [open $path(test1) w] - chan configure $f1 -translation lf -blocking 0 - chan configure $f2 -translation lf -blocking 0 + chan configure $f1 -translation lf -encoding iso8859-1 -blocking 0 + chan configure $f2 -translation lf -encoding iso8859-1 -blocking 0 chan copy $f1 $f2 set result [list [chan configure $f1 -blocking] [chan configure $f2 -blocking]] if {[file size $thisScript] == [file size $path(test1)]} { diff --git a/tests/encoding.test b/tests/encoding.test index f59c6d8..e0e1598 100644 --- a/tests/encoding.test +++ b/tests/encoding.test @@ -535,7 +535,7 @@ test encoding-16.17 {Utf32ToUtfProc} -body { list [encoding convertfrom -strict -failindex idx utf-32le \x41\x00\x00\x00\x00\xD8\x00\x00\x42\x00\x00\x00] [set idx] } -result {A 4} -test encoding-16.9 { +test encoding-16.18 { Utf16ToUtfProc, Tcl_UniCharToUtf, surrogate pairs in utf-16 } -body { apply [list {} { @@ -554,10 +554,15 @@ test encoding-16.9 { return done } [namespace current]] } -result done - - - - +test encoding-16.19 {UnicodeToUtfProc, bug [d19fe0a5b]} -body { + encoding convertfrom utf-16 "\x41\x41\x41" +} -result \u4141\uFFFD +test encoding-16.20 {UnicodeToUtfProc, bug [d19fe0a5b]} -constraints deprecated -body { + encoding convertfrom utf-16 "\xD8\xD8" +} -result \uD8D8 +test encoding-16.21 {UnicodeToUtfProc, bug [d19fe0a5b]} -body { + encoding convertfrom utf-32 "\x00\x00\x00\x00\x41\x41" +} -result \x00\uFFFD test encoding-17.1 {UtfToUtf16Proc} -body { encoding convertto utf-16 "\U460DC" diff --git a/tests/io.test b/tests/io.test index fdc9247..cb1c691 100644 --- a/tests/io.test +++ b/tests/io.test @@ -7280,8 +7280,8 @@ test io-52.5 {TclCopyChannel, all} {fcopy} { file delete $path(test1) set f1 [open $thisScript] set f2 [open $path(test1) w] - fconfigure $f1 -translation lf -blocking 0 - fconfigure $f2 -translation lf -blocking 0 + fconfigure $f1 -translation lf -encoding iso8859-1 -blocking 0 + fconfigure $f2 -translation lf -encoding iso8859-1 -blocking 0 fcopy $f1 $f2 -size -1 ;# -1 means 'copy all', same as if no -size specified. set result [list [fconfigure $f1 -blocking] [fconfigure $f2 -blocking]] close $f1 @@ -7297,8 +7297,8 @@ test io-52.5a {TclCopyChannel, all, other negative value} {fcopy} { file delete $path(test1) set f1 [open $thisScript] set f2 [open $path(test1) w] - fconfigure $f1 -translation lf -blocking 0 - fconfigure $f2 -translation lf -blocking 0 + fconfigure $f1 -translation lf -encoding iso8859-1 -blocking 0 + fconfigure $f2 -translation lf -encoding iso8859-1 -blocking 0 fcopy $f1 $f2 -size -2 ;# < 0 behaves like -1, copy all set result [list [fconfigure $f1 -blocking] [fconfigure $f2 -blocking]] close $f1 @@ -7314,8 +7314,8 @@ test io-52.5b {TclCopyChannel, all, wrap to negative value} {fcopy} { file delete $path(test1) set f1 [open $thisScript] set f2 [open $path(test1) w] - fconfigure $f1 -translation lf -blocking 0 - fconfigure $f2 -translation lf -blocking 0 + fconfigure $f1 -translation lf -encoding iso8859-1 -blocking 0 + fconfigure $f2 -translation lf -encoding iso8859-1 -blocking 0 fcopy $f1 $f2 -size 3221176172 ;# Wrapped to < 0, behaves like -1, copy all set result [list [fconfigure $f1 -blocking] [fconfigure $f2 -blocking]] close $f1 @@ -7348,8 +7348,8 @@ test io-52.7 {TclCopyChannel} {fcopy} { file delete $path(test1) set f1 [open $thisScript] set f2 [open $path(test1) w] - fconfigure $f1 -translation lf -blocking 0 - fconfigure $f2 -translation lf -blocking 0 + fconfigure $f1 -translation lf -encoding iso8859-1 -blocking 0 + fconfigure $f2 -translation lf -encoding iso8859-1 -blocking 0 fcopy $f1 $f2 set result [list [fconfigure $f1 -blocking] [fconfigure $f2 -blocking]] set s1 [file size $thisScript] diff --git a/tools/encoding/Makefile b/tools/encoding/Makefile index 361239e..ff19492 100644 --- a/tools/encoding/Makefile +++ b/tools/encoding/Makefile @@ -71,11 +71,6 @@ encodings: clean txt2enc $(EUC_ENCODINGS) done @echo @echo Compiling special versions of encoding files. - @for p in ascii.txt; do \ - enc=`echo $$p | sed 's/\..*$$/\.enc/'`; \ - echo $$enc; \ - ./txt2enc -m $$p > $$enc; \ - done @for p in jis0208.txt; do \ enc=`echo $$p | sed 's/\..*$$/\.enc/'`; \ echo $$enc; \ diff --git a/tools/encoding/txt2enc.c b/tools/encoding/txt2enc.c index 7ee797b..32c7344 100644 --- a/tools/encoding/txt2enc.c +++ b/tools/encoding/txt2enc.c @@ -106,7 +106,7 @@ main(int argc, char **argv) fputs(" -f\tfallback character (default: QUESTION MARK)\n", stderr); fputs(" -t\toverride implicit type with single, double, or multi\n", stderr); fputs(" -s\tsymbol+ascii encoding\n", stderr); - fputs(" -m\tdon't implicitly include range 0080 to 00FF\n", stderr); + fputs(" -m\tdon't implicitly include 007F\n", stderr); return 1; } @@ -208,10 +208,8 @@ main(int argc, char **argv) toUnicode[0][i] = i; } if (fixmissing) { - for (i = 0x7F; i < 0xA0; i++) { - if (toUnicode[i] == NULL && toUnicode[0][i] == 0) { - toUnicode[0][i] = i; - } + if (toUnicode[0x7F] == NULL && toUnicode[0][0x7F] == 0) { + toUnicode[0][0x7F] = 0x7F; } } } |