From 27e77b7ceeeffe9227b5b96fdd63e3433e0eb63e Mon Sep 17 00:00:00 2001 From: Kent Hansen Date: Mon, 8 Nov 2010 13:25:30 +0100 Subject: Make qsTr() work with Unicode (non-Latin-1) strings Converting the source string/context/comment to Latin-1 is very broken, since JS strings are UTF-16. The strings should be converted to UTF-8, which should also be the default encoding for qsTranslate(). Effectively, this bug meant that only Latin-1 characters could be used in source strings; the translations themselves could have non-Latin-1 characters. But there was data loss in the case where you passed a source string for which no translation was found (since the Latin-1-ized string would be returned). Task-number: QTBUG-14989 Reviewed-by: Jedrzej Nowacki --- src/script/api/qscriptengine.cpp | 18 ++-- tests/auto/qscriptengine/idtranslatable-unicode.js | 5 ++ tests/auto/qscriptengine/qscriptengine.qrc | 2 + tests/auto/qscriptengine/translatable-unicode.js | 9 ++ .../translations/idtranslatable-unicode.qm | Bin 0 -> 209 bytes .../translations/idtranslatable-unicode.ts | 26 ++++++ .../translations/translatable-unicode.qm | Bin 0 -> 322 bytes .../translations/translatable-unicode.ts | 37 +++++++++ tests/auto/qscriptengine/tst_qscriptengine.cpp | 91 +++++++++++++++++++++ 9 files changed, 179 insertions(+), 9 deletions(-) create mode 100644 tests/auto/qscriptengine/idtranslatable-unicode.js create mode 100644 tests/auto/qscriptengine/translatable-unicode.js create mode 100644 tests/auto/qscriptengine/translations/idtranslatable-unicode.qm create mode 100644 tests/auto/qscriptengine/translations/idtranslatable-unicode.ts create mode 100644 tests/auto/qscriptengine/translations/translatable-unicode.qm create mode 100644 tests/auto/qscriptengine/translations/translatable-unicode.ts diff --git a/src/script/api/qscriptengine.cpp b/src/script/api/qscriptengine.cpp index 2d5e5f4..69abcad 100644 --- a/src/script/api/qscriptengine.cpp +++ b/src/script/api/qscriptengine.cpp @@ -808,7 +808,7 @@ JSC::JSValue JSC_HOST_CALL functionQsTranslate(JSC::ExecState *exec, JSC::JSObje JSC::UString comment; if (args.size() > 2) comment = args.at(2).toString(exec); - QCoreApplication::Encoding encoding = QCoreApplication::CodecForTr; + QCoreApplication::Encoding encoding = QCoreApplication::UnicodeUTF8; if (args.size() > 3) { JSC::UString encStr = args.at(3).toString(exec); if (encStr == "CodecForTr") @@ -824,9 +824,9 @@ JSC::JSValue JSC_HOST_CALL functionQsTranslate(JSC::ExecState *exec, JSC::JSObje #endif JSC::UString result; #ifndef QT_NO_QOBJECT - result = QCoreApplication::translate(QScript::convertToLatin1(context).constData(), - QScript::convertToLatin1(text).constData(), - QScript::convertToLatin1(comment).constData(), + result = QCoreApplication::translate(context.UTF8String().c_str(), + text.UTF8String().c_str(), + comment.UTF8String().c_str(), encoding, n); #else result = text; @@ -878,10 +878,10 @@ JSC::JSValue JSC_HOST_CALL functionQsTr(JSC::ExecState *exec, JSC::JSObject*, JS #endif JSC::UString result; #ifndef QT_NO_QOBJECT - result = QCoreApplication::translate(QScript::convertToLatin1(context).constData(), - QScript::convertToLatin1(text).constData(), - QScript::convertToLatin1(comment).constData(), - QCoreApplication::CodecForTr, n); + result = QCoreApplication::translate(context.UTF8String().c_str(), + text.UTF8String().c_str(), + comment.UTF8String().c_str(), + QCoreApplication::UnicodeUTF8, n); #else result = text; #endif @@ -907,7 +907,7 @@ JSC::JSValue JSC_HOST_CALL functionQsTrId(JSC::ExecState *exec, JSC::JSObject*, int n = -1; if (args.size() > 1) n = args.at(1).toInt32(exec); - return JSC::jsString(exec, qtTrId(QScript::convertToLatin1(id).constData(), n)); + return JSC::jsString(exec, qtTrId(id.UTF8String().c_str(), n)); } JSC::JSValue JSC_HOST_CALL functionQsTrIdNoOp(JSC::ExecState *, JSC::JSObject*, JSC::JSValue, const JSC::ArgList &args) diff --git a/tests/auto/qscriptengine/idtranslatable-unicode.js b/tests/auto/qscriptengine/idtranslatable-unicode.js new file mode 100644 index 0000000..e17d617 --- /dev/null +++ b/tests/auto/qscriptengine/idtranslatable-unicode.js @@ -0,0 +1,5 @@ +qsTrId('\u01F8\u01D2\u0199\u01D0\u01E1'); + +QT_TRID_NOOP("\u0191\u01CE\u0211\u0229\u019C\u018E\u019A\u01D0"); + +qsTrId("\u0181\u01A1\u0213\u018F\u018C", 10); diff --git a/tests/auto/qscriptengine/qscriptengine.qrc b/tests/auto/qscriptengine/qscriptengine.qrc index fa55a5b..d05d115 100644 --- a/tests/auto/qscriptengine/qscriptengine.qrc +++ b/tests/auto/qscriptengine/qscriptengine.qrc @@ -2,5 +2,7 @@ translations/translatable_la.qm translations/idtranslatable_la.qm + translations/translatable-unicode.qm + translations/idtranslatable-unicode.qm diff --git a/tests/auto/qscriptengine/translatable-unicode.js b/tests/auto/qscriptengine/translatable-unicode.js new file mode 100644 index 0000000..afe2aff --- /dev/null +++ b/tests/auto/qscriptengine/translatable-unicode.js @@ -0,0 +1,9 @@ +qsTr("H\u2082O"); +qsTranslate("\u010C\u0101\u011F\u0115", "CO\u2082"); + +var unicode_strings = [ + QT_TR_NOOP("\u0391\u0392\u0393"), + QT_TRANSLATE_NOOP("\u010C\u0101\u011F\u0115", "\u0414\u0415\u0416") +]; + +qsTr("H\u2082O", "not the same H\u2082O"); diff --git a/tests/auto/qscriptengine/translations/idtranslatable-unicode.qm b/tests/auto/qscriptengine/translations/idtranslatable-unicode.qm new file mode 100644 index 0000000..8c5fb91 Binary files /dev/null and b/tests/auto/qscriptengine/translations/idtranslatable-unicode.qm differ diff --git a/tests/auto/qscriptengine/translations/idtranslatable-unicode.ts b/tests/auto/qscriptengine/translations/idtranslatable-unicode.ts new file mode 100644 index 0000000..74ebf43 --- /dev/null +++ b/tests/auto/qscriptengine/translations/idtranslatable-unicode.ts @@ -0,0 +1,26 @@ + + + +UTF-8 + + + + + + Ƨưƈȼȝȿș + + + + + Ǡȡȋȅȕ + + + + + + Ƒưǹ + %n ƒơǒ(ș) + + + + diff --git a/tests/auto/qscriptengine/translations/translatable-unicode.qm b/tests/auto/qscriptengine/translations/translatable-unicode.qm new file mode 100644 index 0000000..aa75ce6 Binary files /dev/null and b/tests/auto/qscriptengine/translations/translatable-unicode.qm differ diff --git a/tests/auto/qscriptengine/translations/translatable-unicode.ts b/tests/auto/qscriptengine/translations/translatable-unicode.ts new file mode 100644 index 0000000..1b8b4d2 --- /dev/null +++ b/tests/auto/qscriptengine/translations/translatable-unicode.ts @@ -0,0 +1,37 @@ + + + +UTF-8 + + translatable-unicode + + + H₂O + ͻͼͽ + + + + ΑΒΓ + ӜҴѼ + + + + H₂O + not the same H₂O + ԶՊՒ + + + + Čāğĕ + + + CO₂ + בךע + + + + ДЕЖ + خسس + + + diff --git a/tests/auto/qscriptengine/tst_qscriptengine.cpp b/tests/auto/qscriptengine/tst_qscriptengine.cpp index 5f38c22..d529b8b 100644 --- a/tests/auto/qscriptengine/tst_qscriptengine.cpp +++ b/tests/auto/qscriptengine/tst_qscriptengine.cpp @@ -186,6 +186,10 @@ private slots: void translationContext_data(); void translationContext(); void translateScriptIdBased(); + void translateScriptUnicode_data(); + void translateScriptUnicode(); + void translateScriptUnicodeIdBased_data(); + void translateScriptUnicodeIdBased(); void functionScopes(); void nativeFunctionScopes(); void evaluateProgram(); @@ -4950,6 +4954,93 @@ void tst_QScriptEngine::translateScriptIdBased() QString::fromLatin1("qtn_foo_bar")); // Doesn't have plural } +// How to add a new test row: +// - Find a nice list of Unicode characters to choose from +// - Write source string/context/comment in .js using Unicode escape sequences (\uABCD) +// - Update corresponding .ts file (e.g. lupdate foo.js -ts foo.ts -codecfortr UTF-8) +// - Enter translation in Linguist +// - Update corresponding .qm file (e.g. lrelease foo.ts) +// - Evaluate script that performs translation; make sure the correct result is returned +// (e.g. by setting the resulting string as the text of a QLabel and visually verifying +// that it looks the same as what you entered in Linguist :-) ) +// - Generate the expectedTranslation column data using toUtf8().toHex() +void tst_QScriptEngine::translateScriptUnicode_data() +{ + QTest::addColumn("expression"); + QTest::addColumn("fileName"); + QTest::addColumn("expectedTranslation"); + + QString fileName = QString::fromLatin1("translatable-unicode.js"); + QTest::newRow("qsTr('H\\u2082O')@translatable-unicode.js") + << QString::fromLatin1("qsTr('H\\u2082O')") << fileName << QString::fromUtf8("\xcd\xbb\xcd\xbc\xcd\xbd"); + QTest::newRow("qsTranslate('\\u010C\\u0101\\u011F\\u0115', 'CO\\u2082')@translatable-unicode.js") + << QString::fromLatin1("qsTranslate('\\u010C\\u0101\\u011F\\u0115', 'CO\\u2082')") << fileName << QString::fromUtf8("\xd7\x91\xd7\x9a\xd7\xa2"); + QTest::newRow("qsTr('\\u0391\\u0392\\u0393')@translatable-unicode.js") + << QString::fromLatin1("qsTr('\\u0391\\u0392\\u0393')") << fileName << QString::fromUtf8("\xd3\x9c\xd2\xb4\xd1\xbc"); + QTest::newRow("qsTranslate('\\u010C\\u0101\\u011F\\u0115', '\\u0414\\u0415\\u0416')@translatable-unicode.js") + << QString::fromLatin1("qsTranslate('\\u010C\\u0101\\u011F\\u0115', '\\u0414\\u0415\\u0416')") << fileName << QString::fromUtf8("\xd8\xae\xd8\xb3\xd8\xb3"); + QTest::newRow("qsTr('H\\u2082O', 'not the same H\\u2082O')@translatable-unicode.js") + << QString::fromLatin1("qsTr('H\\u2082O', 'not the same H\\u2082O')") << fileName << QString::fromUtf8("\xd4\xb6\xd5\x8a\xd5\x92"); + QTest::newRow("qsTr('H\\u2082O')") + << QString::fromLatin1("qsTr('H\\u2082O')") << QString() << QString::fromUtf8("\x48\xe2\x82\x82\x4f"); + QTest::newRow("qsTranslate('\\u010C\\u0101\\u011F\\u0115', 'CO\\u2082')") + << QString::fromLatin1("qsTranslate('\\u010C\\u0101\\u011F\\u0115', 'CO\\u2082')") << QString() << QString::fromUtf8("\xd7\x91\xd7\x9a\xd7\xa2"); +} + +void tst_QScriptEngine::translateScriptUnicode() +{ + QFETCH(QString, expression); + QFETCH(QString, fileName); + QFETCH(QString, expectedTranslation); + + QScriptEngine engine; + + QTranslator translator; + QVERIFY(translator.load(":/translations/translatable-unicode")); + QCoreApplication::instance()->installTranslator(&translator); + engine.installTranslatorFunctions(); + + QCOMPARE(engine.evaluate(expression, fileName).toString(), expectedTranslation); + QVERIFY(!engine.hasUncaughtException()); + + QCoreApplication::instance()->removeTranslator(&translator); +} + +void tst_QScriptEngine::translateScriptUnicodeIdBased_data() +{ + QTest::addColumn("expression"); + QTest::addColumn("expectedTranslation"); + + QTest::newRow("qsTrId('\\u01F8\\u01D2\\u0199\\u01D0\\u01E1'')") + << QString::fromLatin1("qsTrId('\\u01F8\\u01D2\\u0199\\u01D0\\u01E1')") << QString::fromUtf8("\xc6\xa7\xc6\xb0\xc6\x88\xc8\xbc\xc8\x9d\xc8\xbf\xc8\x99"); + QTest::newRow("qsTrId('\\u0191\\u01CE\\u0211\\u0229\\u019C\\u018E\\u019A\\u01D0')") + << QString::fromLatin1("qsTrId('\\u0191\\u01CE\\u0211\\u0229\\u019C\\u018E\\u019A\\u01D0')") << QString::fromUtf8("\xc7\xa0\xc8\xa1\xc8\x8b\xc8\x85\xc8\x95"); + QTest::newRow("qsTrId('\\u0181\\u01A1\\u0213\\u018F\\u018C', 10)") + << QString::fromLatin1("qsTrId('\\u0181\\u01A1\\u0213\\u018F\\u018C', 10)") << QString::fromUtf8("\x31\x30\x20\xc6\x92\xc6\xa1\xc7\x92\x28\xc8\x99\x29"); + QTest::newRow("qsTrId('\\u0181\\u01A1\\u0213\\u018F\\u018C')") + << QString::fromLatin1("qsTrId('\\u0181\\u01A1\\u0213\\u018F\\u018C')") << QString::fromUtf8("\xc6\x91\xc6\xb0\xc7\xb9"); + QTest::newRow("qsTrId('\\u01CD\\u0180\\u01A8\\u0190\\u019E\\u01AB')") + << QString::fromLatin1("qsTrId('\\u01CD\\u0180\\u01A8\\u0190\\u019E\\u01AB')") << QString::fromUtf8("\xc7\x8d\xc6\x80\xc6\xa8\xc6\x90\xc6\x9e\xc6\xab"); +} + +void tst_QScriptEngine::translateScriptUnicodeIdBased() +{ + QFETCH(QString, expression); + QFETCH(QString, expectedTranslation); + + QScriptEngine engine; + + QTranslator translator; + QVERIFY(translator.load(":/translations/idtranslatable-unicode")); + QCoreApplication::instance()->installTranslator(&translator); + engine.installTranslatorFunctions(); + + QCOMPARE(engine.evaluate(expression).toString(), expectedTranslation); + QVERIFY(!engine.hasUncaughtException()); + + QCoreApplication::instance()->removeTranslator(&translator); +} + void tst_QScriptEngine::functionScopes() { QScriptEngine eng; -- cgit v0.12