From 8b30b72948d5f16d59f8799f3889f563e2c6e24d Mon Sep 17 00:00:00 2001 From: Thiago Macieira Date: Fri, 5 Mar 2010 14:10:57 +0100 Subject: Make the UTF-8 encoder/decoder not accept Unicode non-characters Reviewed-By: Denis Dzyubenko --- src/corelib/codecs/qutfcodec.cpp | 28 +++++++++++-- tests/auto/utf8/tst_utf8.cpp | 85 ++++++++++++++++++++++++++++++++++------ 2 files changed, 97 insertions(+), 16 deletions(-) diff --git a/src/corelib/codecs/qutfcodec.cpp b/src/corelib/codecs/qutfcodec.cpp index 742b2e7..233bd8f 100644 --- a/src/corelib/codecs/qutfcodec.cpp +++ b/src/corelib/codecs/qutfcodec.cpp @@ -48,6 +48,19 @@ QT_BEGIN_NAMESPACE enum { Endian = 0, Data = 1 }; +static inline bool isUnicodeNonCharacter(uint ucs4) +{ + // Unicode has a couple of "non-characters" that one can use internally, + // but are not allowed to be used for text interchange. + // + // Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF, + // U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and + // U+FDEF (inclusive) + + return (ucs4 & 0xfffe) == 0xfffe + || (ucs4 - 0xfdd0U) < 16; +} + QByteArray QUtf8::convertFromUnicode(const QChar *uc, int len, QTextCodec::ConverterState *state) { uchar replacement = '?'; @@ -106,6 +119,14 @@ QByteArray QUtf8::convertFromUnicode(const QChar *uc, int len, QTextCodec::Conve if (u < 0x0800) { *cursor++ = 0xc0 | ((uchar) (u >> 6)); } else { + // is it one of the Unicode non-characters? + if (isUnicodeNonCharacter(u)) { + *cursor = replacement; + ++ch; + ++invalid; + continue; + } + if (u > 0xffff) { *cursor++ = 0xf0 | ((uchar) (u >> 18)); *cursor++ = 0x80 | (((uchar) (u >> 12)) & 0x3f); @@ -172,15 +193,16 @@ QString QUtf8::convertToUnicode(const char *chars, int len, QTextCodec::Converte --need; if (!need) { // utf-8 bom composes into 0xfeff code point + bool nonCharacter; if (!headerdone && uc == 0xfeff) { // dont do anything, just skip the BOM - } else if (uc > 0xffff && uc < 0x110000) { + } else if (!(nonCharacter = isUnicodeNonCharacter(uc)) && uc > 0xffff && uc < 0x110000) { // surrogate pair Q_ASSERT((qch - (ushort*)result.unicode()) + 2 < result.length()); *qch++ = QChar::highSurrogate(uc); *qch++ = QChar::lowSurrogate(uc); - } else if ((uc < min_uc) || (uc >= 0xd800 && uc <= 0xdfff) || (uc >= 0xfffe)) { - // error: overlong sequence, UTF16 surrogate or BOM + } else if ((uc < min_uc) || (uc >= 0xd800 && uc <= 0xdfff) || nonCharacter || uc >= 0x110000) { + // error: overlong sequence, UTF16 surrogate or non-character *qch++ = replacement; ++invalid; } else { diff --git a/tests/auto/utf8/tst_utf8.cpp b/tests/auto/utf8/tst_utf8.cpp index 1183b81..7bbbfab 100644 --- a/tests/auto/utf8/tst_utf8.cpp +++ b/tests/auto/utf8/tst_utf8.cpp @@ -73,6 +73,9 @@ private slots: void invalidUtf8_data(); void invalidUtf8(); + + void nonCharacters_data(); + void nonCharacters(); }; void tst_Utf8::initTestCase() @@ -134,8 +137,8 @@ void tst_Utf8::roundTrip_data() static const uint utf32_5[] = { 0x010203 }; QTest::newRow("utf8_5") << QByteArray(utf8_5) << QString::fromUcs4(utf32_5, 1); - static const char utf8_6[] = "\364\217\277\277"; // U+10FFFF - static const uint utf32_6[] = { 0x10FFFF }; + static const char utf8_6[] = "\364\217\277\275"; // U+10FFFD + static const uint utf32_6[] = { 0x10FFFD }; QTest::newRow("utf8_6") << QByteArray(utf8_6) << QString::fromUcs4(utf32_6, 1); static const char utf8_7[] = "abc\302\240\303\241\303\251\307\275 \342\202\254def"; @@ -144,10 +147,10 @@ void tst_Utf8::roundTrip_data() ' ', 0x20AC, 'd', 'e', 'f', 0 }; QTest::newRow("utf8_7") << QByteArray(utf8_7) << QString::fromUtf16(utf16_7); - static const char utf8_8[] = "abc\302\240\303\241\303\251\307\275 \364\217\277\277 \342\202\254def"; + static const char utf8_8[] = "abc\302\240\303\241\303\251\307\275 \364\217\277\275 \342\202\254def"; static const uint utf32_8[] = { 'a', 'b', 'c', 0x00A0, 0x00E1, 0x00E9, 0x01FD, - ' ', 0x10FFFF, ' ', + ' ', 0x10FFFD, ' ', 0x20AC, 'd', 'e', 'f', 0 }; QTest::newRow("utf8_8") << QByteArray(utf8_8) << QString::fromUcs4(utf32_8); } @@ -214,14 +217,6 @@ void tst_Utf8::invalidUtf8_data() QTest::newRow("4chars-2") << QByteArray("\xF0\x90\xC0\x80"); QTest::newRow("4chars-3") << QByteArray("\xF0\xC0\x80\x80"); - // U+FFFE and U+FFFF are non-characters and must not be present - // U+FFFE: 1111 11 1111 11 1110 - // encoding: xxxz:1111 xz11:1111 xz11:1110 - QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE"); - // U+FFFF: 1111 11 1111 11 1111 - // encoding: xxxz:1111 xz11:1111 xz11:1111 - QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF"); - // Surrogate pairs must now be present either // U+D800: 1101 10 0000 00 0000 // encoding: xxxz:1101 xz10:0000 xz00:0000 @@ -302,7 +297,7 @@ void tst_Utf8::invalidUtf8() QFETCH_GLOBAL(bool, useLocale); QSharedPointer decoder = QSharedPointer(codec->makeDecoder()); - QString decoded = decoder->toUnicode(utf8); + decoder->toUnicode(utf8); // Only enforce correctness on our UTF-8 decoder // The system's UTF-8 codec is sometimes buggy @@ -314,5 +309,69 @@ void tst_Utf8::invalidUtf8() qWarning("System codec does not report failure when it should. Should report bug upstream."); } +void tst_Utf8::nonCharacters_data() +{ + QTest::addColumn("utf8"); + QTest::addColumn("utf16"); + + // Unicode has a couple of "non-characters" that one can use internally, + // but are not allowed to be used for text interchange. + // + // Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF, + // U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and + // U+FDEF (inclusive) + + // U+FDD0 through U+FDEF + for (int i = 0; i < 16; ++i) { + char utf8[] = { 0357, 0267, 0220 + i, 0 }; + QString utf16 = QChar(0xfdd0 + i); + QTest::newRow(qPrintable(QString::number(0xfdd0 + i, 16))) << QByteArray(utf8) << utf16; + } + + // the last two in Planes 1 through 16 + for (uint plane = 1; plane <= 16; ++plane) { + for (uint lower = 0xfffe; lower < 0x10000; ++lower) { + uint ucs4 = (plane << 16) | lower; + char utf8[] = { 0xf0 | uchar(ucs4 >> 18), + 0x80 | (uchar(ucs4 >> 12) & 0x3f), + 0x80 | (uchar(ucs4 >> 6) & 0x3f), + 0x80 | (uchar(ucs4) & 0x3f), + 0 }; + ushort utf16[] = { QChar::highSurrogate(ucs4), QChar::lowSurrogate(ucs4), 0 }; + + QTest::newRow(qPrintable(QString::number(ucs4, 16))) << QByteArray(utf8) << QString::fromUtf16(utf16); + } + } + + QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE") << QString(QChar(0xfffe)); + QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF") << QString(QChar(0xffff)); +} + +void tst_Utf8::nonCharacters() +{ + QFETCH(QByteArray, utf8); + QFETCH(QString, utf16); + QFETCH_GLOBAL(bool, useLocale); + + QSharedPointer decoder = QSharedPointer(codec->makeDecoder()); + decoder->toUnicode(utf8); + + // Only enforce correctness on our UTF-8 decoder + // The system's UTF-8 codec is sometimes buggy + // GNU libc's iconv is known to accept U+FFFF and U+FFFE encoded as UTF-8 + // OS X's iconv is known to accept those, plus surrogates and codepoints above U+10FFFF + if (!useLocale) + QVERIFY(decoder->hasFailure()); + else if (!decoder->hasFailure()) + qWarning("System codec does not report failure when it should. Should report bug upstream."); + + QSharedPointer encoder(codec->makeEncoder()); + encoder->fromUnicode(utf16); + if (!useLocale) + QVERIFY(encoder->hasFailure()); + else if (!encoder->hasFailure()) + qWarning("System codec does not report failure when it should. Should report bug upstream."); +} + QTEST_MAIN(tst_Utf8) #include "tst_utf8.moc" -- cgit v0.12