diff options
Diffstat (limited to 'tests/auto/utf8')
| -rw-r--r-- | tests/auto/utf8/tst_utf8.cpp | 85 |
1 files changed, 72 insertions, 13 deletions
diff --git a/tests/auto/utf8/tst_utf8.cpp b/tests/auto/utf8/tst_utf8.cpp index 1183b81..7bbbfab 100644 --- a/tests/auto/utf8/tst_utf8.cpp +++ b/tests/auto/utf8/tst_utf8.cpp @@ -73,6 +73,9 @@ private slots: void invalidUtf8_data(); void invalidUtf8(); + + void nonCharacters_data(); + void nonCharacters(); }; void tst_Utf8::initTestCase() @@ -134,8 +137,8 @@ void tst_Utf8::roundTrip_data() static const uint utf32_5[] = { 0x010203 }; QTest::newRow("utf8_5") << QByteArray(utf8_5) << QString::fromUcs4(utf32_5, 1); - static const char utf8_6[] = "\364\217\277\277"; // U+10FFFF - static const uint utf32_6[] = { 0x10FFFF }; + static const char utf8_6[] = "\364\217\277\275"; // U+10FFFD + static const uint utf32_6[] = { 0x10FFFD }; QTest::newRow("utf8_6") << QByteArray(utf8_6) << QString::fromUcs4(utf32_6, 1); static const char utf8_7[] = "abc\302\240\303\241\303\251\307\275 \342\202\254def"; @@ -144,10 +147,10 @@ void tst_Utf8::roundTrip_data() ' ', 0x20AC, 'd', 'e', 'f', 0 }; QTest::newRow("utf8_7") << QByteArray(utf8_7) << QString::fromUtf16(utf16_7); - static const char utf8_8[] = "abc\302\240\303\241\303\251\307\275 \364\217\277\277 \342\202\254def"; + static const char utf8_8[] = "abc\302\240\303\241\303\251\307\275 \364\217\277\275 \342\202\254def"; static const uint utf32_8[] = { 'a', 'b', 'c', 0x00A0, 0x00E1, 0x00E9, 0x01FD, - ' ', 0x10FFFF, ' ', + ' ', 0x10FFFD, ' ', 0x20AC, 'd', 'e', 'f', 0 }; QTest::newRow("utf8_8") << QByteArray(utf8_8) << QString::fromUcs4(utf32_8); } @@ -214,14 +217,6 @@ void tst_Utf8::invalidUtf8_data() QTest::newRow("4chars-2") << QByteArray("\xF0\x90\xC0\x80"); QTest::newRow("4chars-3") << QByteArray("\xF0\xC0\x80\x80"); - // U+FFFE and U+FFFF are non-characters and must not be present - // U+FFFE: 1111 11 1111 11 1110 - // encoding: xxxz:1111 xz11:1111 xz11:1110 - QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE"); - // U+FFFF: 1111 11 1111 11 1111 - // encoding: xxxz:1111 xz11:1111 xz11:1111 - QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF"); - // Surrogate pairs must now be present either // U+D800: 1101 10 0000 00 0000 // encoding: xxxz:1101 xz10:0000 xz00:0000 @@ -302,7 +297,7 @@ void tst_Utf8::invalidUtf8() QFETCH_GLOBAL(bool, useLocale); QSharedPointer<QTextDecoder> decoder = QSharedPointer<QTextDecoder>(codec->makeDecoder()); - QString decoded = decoder->toUnicode(utf8); + decoder->toUnicode(utf8); // Only enforce correctness on our UTF-8 decoder // The system's UTF-8 codec is sometimes buggy @@ -314,5 +309,69 @@ void tst_Utf8::invalidUtf8() qWarning("System codec does not report failure when it should. Should report bug upstream."); } +void tst_Utf8::nonCharacters_data() +{ + QTest::addColumn<QByteArray>("utf8"); + QTest::addColumn<QString>("utf16"); + + // Unicode has a couple of "non-characters" that one can use internally, + // but are not allowed to be used for text interchange. + // + // Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF, + // U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and + // U+FDEF (inclusive) + + // U+FDD0 through U+FDEF + for (int i = 0; i < 16; ++i) { + char utf8[] = { 0357, 0267, 0220 + i, 0 }; + QString utf16 = QChar(0xfdd0 + i); + QTest::newRow(qPrintable(QString::number(0xfdd0 + i, 16))) << QByteArray(utf8) << utf16; + } + + // the last two in Planes 1 through 16 + for (uint plane = 1; plane <= 16; ++plane) { + for (uint lower = 0xfffe; lower < 0x10000; ++lower) { + uint ucs4 = (plane << 16) | lower; + char utf8[] = { 0xf0 | uchar(ucs4 >> 18), + 0x80 | (uchar(ucs4 >> 12) & 0x3f), + 0x80 | (uchar(ucs4 >> 6) & 0x3f), + 0x80 | (uchar(ucs4) & 0x3f), + 0 }; + ushort utf16[] = { QChar::highSurrogate(ucs4), QChar::lowSurrogate(ucs4), 0 }; + + QTest::newRow(qPrintable(QString::number(ucs4, 16))) << QByteArray(utf8) << QString::fromUtf16(utf16); + } + } + + QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE") << QString(QChar(0xfffe)); + QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF") << QString(QChar(0xffff)); +} + +void tst_Utf8::nonCharacters() +{ + QFETCH(QByteArray, utf8); + QFETCH(QString, utf16); + QFETCH_GLOBAL(bool, useLocale); + + QSharedPointer<QTextDecoder> decoder = QSharedPointer<QTextDecoder>(codec->makeDecoder()); + decoder->toUnicode(utf8); + + // Only enforce correctness on our UTF-8 decoder + // The system's UTF-8 codec is sometimes buggy + // GNU libc's iconv is known to accept U+FFFF and U+FFFE encoded as UTF-8 + // OS X's iconv is known to accept those, plus surrogates and codepoints above U+10FFFF + if (!useLocale) + QVERIFY(decoder->hasFailure()); + else if (!decoder->hasFailure()) + qWarning("System codec does not report failure when it should. Should report bug upstream."); + + QSharedPointer<QTextEncoder> encoder(codec->makeEncoder()); + encoder->fromUnicode(utf16); + if (!useLocale) + QVERIFY(encoder->hasFailure()); + else if (!encoder->hasFailure()) + qWarning("System codec does not report failure when it should. Should report bug upstream."); +} + QTEST_MAIN(tst_Utf8) #include "tst_utf8.moc" |