summaryrefslogtreecommitdiffstats
path: root/tests/auto/utf8
diff options
context:
space:
mode:
authorThiago Macieira <thiago.macieira@nokia.com>2010-03-05 13:10:57 (GMT)
committerThiago Macieira <thiago.macieira@nokia.com>2010-03-05 15:57:27 (GMT)
commit8b30b72948d5f16d59f8799f3889f563e2c6e24d (patch)
tree6279bd2f3faeb0be880225dc787858b98d3a935f /tests/auto/utf8
parent4f02ca5c9458299185d447da87058d6e8ca1b260 (diff)
downloadQt-8b30b72948d5f16d59f8799f3889f563e2c6e24d.zip
Qt-8b30b72948d5f16d59f8799f3889f563e2c6e24d.tar.gz
Qt-8b30b72948d5f16d59f8799f3889f563e2c6e24d.tar.bz2
Make the UTF-8 encoder/decoder not accept Unicode non-characters
Reviewed-By: Denis Dzyubenko
Diffstat (limited to 'tests/auto/utf8')
-rw-r--r--tests/auto/utf8/tst_utf8.cpp85
1 files changed, 72 insertions, 13 deletions
diff --git a/tests/auto/utf8/tst_utf8.cpp b/tests/auto/utf8/tst_utf8.cpp
index 1183b81..7bbbfab 100644
--- a/tests/auto/utf8/tst_utf8.cpp
+++ b/tests/auto/utf8/tst_utf8.cpp
@@ -73,6 +73,9 @@ private slots:
void invalidUtf8_data();
void invalidUtf8();
+
+ void nonCharacters_data();
+ void nonCharacters();
};
void tst_Utf8::initTestCase()
@@ -134,8 +137,8 @@ void tst_Utf8::roundTrip_data()
static const uint utf32_5[] = { 0x010203 };
QTest::newRow("utf8_5") << QByteArray(utf8_5) << QString::fromUcs4(utf32_5, 1);
- static const char utf8_6[] = "\364\217\277\277"; // U+10FFFF
- static const uint utf32_6[] = { 0x10FFFF };
+ static const char utf8_6[] = "\364\217\277\275"; // U+10FFFD
+ static const uint utf32_6[] = { 0x10FFFD };
QTest::newRow("utf8_6") << QByteArray(utf8_6) << QString::fromUcs4(utf32_6, 1);
static const char utf8_7[] = "abc\302\240\303\241\303\251\307\275 \342\202\254def";
@@ -144,10 +147,10 @@ void tst_Utf8::roundTrip_data()
' ', 0x20AC, 'd', 'e', 'f', 0 };
QTest::newRow("utf8_7") << QByteArray(utf8_7) << QString::fromUtf16(utf16_7);
- static const char utf8_8[] = "abc\302\240\303\241\303\251\307\275 \364\217\277\277 \342\202\254def";
+ static const char utf8_8[] = "abc\302\240\303\241\303\251\307\275 \364\217\277\275 \342\202\254def";
static const uint utf32_8[] = { 'a', 'b', 'c', 0x00A0,
0x00E1, 0x00E9, 0x01FD,
- ' ', 0x10FFFF, ' ',
+ ' ', 0x10FFFD, ' ',
0x20AC, 'd', 'e', 'f', 0 };
QTest::newRow("utf8_8") << QByteArray(utf8_8) << QString::fromUcs4(utf32_8);
}
@@ -214,14 +217,6 @@ void tst_Utf8::invalidUtf8_data()
QTest::newRow("4chars-2") << QByteArray("\xF0\x90\xC0\x80");
QTest::newRow("4chars-3") << QByteArray("\xF0\xC0\x80\x80");
- // U+FFFE and U+FFFF are non-characters and must not be present
- // U+FFFE: 1111 11 1111 11 1110
- // encoding: xxxz:1111 xz11:1111 xz11:1110
- QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE");
- // U+FFFF: 1111 11 1111 11 1111
- // encoding: xxxz:1111 xz11:1111 xz11:1111
- QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF");
-
// Surrogate pairs must now be present either
// U+D800: 1101 10 0000 00 0000
// encoding: xxxz:1101 xz10:0000 xz00:0000
@@ -302,7 +297,7 @@ void tst_Utf8::invalidUtf8()
QFETCH_GLOBAL(bool, useLocale);
QSharedPointer<QTextDecoder> decoder = QSharedPointer<QTextDecoder>(codec->makeDecoder());
- QString decoded = decoder->toUnicode(utf8);
+ decoder->toUnicode(utf8);
// Only enforce correctness on our UTF-8 decoder
// The system's UTF-8 codec is sometimes buggy
@@ -314,5 +309,69 @@ void tst_Utf8::invalidUtf8()
qWarning("System codec does not report failure when it should. Should report bug upstream.");
}
+void tst_Utf8::nonCharacters_data()
+{
+ QTest::addColumn<QByteArray>("utf8");
+ QTest::addColumn<QString>("utf16");
+
+ // Unicode has a couple of "non-characters" that one can use internally,
+ // but are not allowed to be used for text interchange.
+ //
+ // Those are the last two entries each Unicode Plane (U+FFFE, U+FFFF,
+ // U+1FFFE, U+1FFFF, etc.) as well as the entries between U+FDD0 and
+ // U+FDEF (inclusive)
+
+ // U+FDD0 through U+FDEF
+ for (int i = 0; i < 16; ++i) {
+ char utf8[] = { 0357, 0267, 0220 + i, 0 };
+ QString utf16 = QChar(0xfdd0 + i);
+ QTest::newRow(qPrintable(QString::number(0xfdd0 + i, 16))) << QByteArray(utf8) << utf16;
+ }
+
+ // the last two in Planes 1 through 16
+ for (uint plane = 1; plane <= 16; ++plane) {
+ for (uint lower = 0xfffe; lower < 0x10000; ++lower) {
+ uint ucs4 = (plane << 16) | lower;
+ char utf8[] = { 0xf0 | uchar(ucs4 >> 18),
+ 0x80 | (uchar(ucs4 >> 12) & 0x3f),
+ 0x80 | (uchar(ucs4 >> 6) & 0x3f),
+ 0x80 | (uchar(ucs4) & 0x3f),
+ 0 };
+ ushort utf16[] = { QChar::highSurrogate(ucs4), QChar::lowSurrogate(ucs4), 0 };
+
+ QTest::newRow(qPrintable(QString::number(ucs4, 16))) << QByteArray(utf8) << QString::fromUtf16(utf16);
+ }
+ }
+
+ QTest::newRow("fffe") << QByteArray("\xEF\xBF\xBE") << QString(QChar(0xfffe));
+ QTest::newRow("ffff") << QByteArray("\xEF\xBF\xBF") << QString(QChar(0xffff));
+}
+
+void tst_Utf8::nonCharacters()
+{
+ QFETCH(QByteArray, utf8);
+ QFETCH(QString, utf16);
+ QFETCH_GLOBAL(bool, useLocale);
+
+ QSharedPointer<QTextDecoder> decoder = QSharedPointer<QTextDecoder>(codec->makeDecoder());
+ decoder->toUnicode(utf8);
+
+ // Only enforce correctness on our UTF-8 decoder
+ // The system's UTF-8 codec is sometimes buggy
+ // GNU libc's iconv is known to accept U+FFFF and U+FFFE encoded as UTF-8
+ // OS X's iconv is known to accept those, plus surrogates and codepoints above U+10FFFF
+ if (!useLocale)
+ QVERIFY(decoder->hasFailure());
+ else if (!decoder->hasFailure())
+ qWarning("System codec does not report failure when it should. Should report bug upstream.");
+
+ QSharedPointer<QTextEncoder> encoder(codec->makeEncoder());
+ encoder->fromUnicode(utf16);
+ if (!useLocale)
+ QVERIFY(encoder->hasFailure());
+ else if (!encoder->hasFailure())
+ qWarning("System codec does not report failure when it should. Should report bug upstream.");
+}
+
QTEST_MAIN(tst_Utf8)
#include "tst_utf8.moc"