diff options
Diffstat (limited to 'tests/auto/selftests')
| -rw-r--r-- | tests/auto/selftests/badxml/badxml.pro | 11 | ||||
| -rw-r--r-- | tests/auto/selftests/badxml/tst_badxml.cpp | 206 | ||||
| -rw-r--r-- | tests/auto/selftests/expected_skip.txt | 8 | ||||
| -rw-r--r-- | tests/auto/selftests/expected_xunit.txt | 40 | ||||
| -rw-r--r-- | tests/auto/selftests/selftests.pro | 2 | ||||
| -rw-r--r-- | tests/auto/selftests/selftests.qrc | 1 | ||||
| -rw-r--r-- | tests/auto/selftests/tst_selftests.cpp | 99 | ||||
| -rwxr-xr-x | tests/auto/selftests/xunit/tst_xunit | bin | 0 -> 11624 bytes | |||
| -rw-r--r-- | tests/auto/selftests/xunit/tst_xunit.cpp | 115 | ||||
| -rw-r--r-- | tests/auto/selftests/xunit/xunit.pro | 15 |
10 files changed, 480 insertions, 17 deletions
diff --git a/tests/auto/selftests/badxml/badxml.pro b/tests/auto/selftests/badxml/badxml.pro new file mode 100644 index 0000000..323791a --- /dev/null +++ b/tests/auto/selftests/badxml/badxml.pro @@ -0,0 +1,11 @@ +load(qttest_p4) + +SOURCES += tst_badxml.cpp +QT = core + +mac:CONFIG -= app_bundle +CONFIG -= debug_and_release_target +!win32:CONFIG += debug + +TARGET = badxml + diff --git a/tests/auto/selftests/badxml/tst_badxml.cpp b/tests/auto/selftests/badxml/tst_badxml.cpp new file mode 100644 index 0000000..fa5b717 --- /dev/null +++ b/tests/auto/selftests/badxml/tst_badxml.cpp @@ -0,0 +1,206 @@ +/**************************************************************************** +** +** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies). +** Contact: Qt Software Information (qt-info@nokia.com) +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the either Technology Preview License Agreement or the +** Beta Release License Agreement. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain +** additional rights. These rights are described in the Nokia Qt LGPL +** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this +** package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** If you are unsure which license is appropriate for your use, please +** contact the sales department at qt-sales@nokia.com. +** $QT_END_LICENSE$ +** +****************************************************************************/ + + +#include <QtCore> +#include <QtTest/QtTest> + +/* + This test makes a testlog containing lots of characters which have a special meaning in + XML, with the purpose of exposing bugs in testlib's XML output code. +*/ +class tst_BadXml : public QObject +{ + Q_OBJECT + +private slots: + void badDataTag() const; + void badDataTag_data() const; + + void badMessage() const; + void badMessage_data() const; + + void failWithNoFile() const; + +public: + static QList<QByteArray> const& badStrings(); +}; + +/* + Custom metaobject to make it possible to change class name at runtime. +*/ +class EmptyClass : public tst_BadXml +{ Q_OBJECT }; + +class tst_BadXmlSub : public tst_BadXml +{ +public: + const QMetaObject* metaObject() const; + + static char const* className; +}; +char const* tst_BadXmlSub::className = "tst_BadXml"; + +const QMetaObject* tst_BadXmlSub::metaObject() const +{ + const QMetaObject& empty = EmptyClass::staticMetaObject; + static QMetaObject mo = { + { empty.d.superdata, empty.d.stringdata, empty.d.data, empty.d.extradata } + }; + static char currentClassName[1024]; + qstrcpy(currentClassName, className); + int len = qstrlen(className); + currentClassName[len] = 0; + currentClassName[len+1] = 0; + + mo.d.stringdata = currentClassName; + + return &mo; +} + +/* + Outputs incidents and benchmark results with the current data tag set to a bad string. +*/ +void tst_BadXml::badDataTag() const +{ + qDebug("a message"); + + QBENCHMARK { + } + + QFAIL("a failure"); +} + +void tst_BadXml::badDataTag_data() const +{ + QTest::addColumn<int>("dummy"); + + foreach (char const* str, badStrings()) { + QTest::newRow(str) << 0; + } +} + +void tst_BadXml::failWithNoFile() const +{ + QTest::qFail("failure message", 0, 0); +} + +/* + Outputs a message containing a bad string. +*/ +void tst_BadXml::badMessage() const +{ + QFETCH(QByteArray, message); + qDebug("%s", message.constData()); +} + +void tst_BadXml::badMessage_data() const +{ + QTest::addColumn<QByteArray>("message"); + + int i = 0; + foreach (QByteArray const& str, badStrings()) { + QTest::newRow(qPrintable(QString::fromLatin1("string %1").arg(i++))) << str; + } +} + +/* + Returns a list of strings likely to expose bugs in XML output code. +*/ +QList<QByteArray> const& tst_BadXml::badStrings() +{ + static QList<QByteArray> out; + if (out.isEmpty()) { + out << "end cdata ]]> text ]]> more text"; + out << "quotes \" text\" more text"; + out << "xml close > open < tags < text"; + out << "all > \" mixed ]]> up > \" in < the ]]> hopes < of triggering \"< ]]> bugs"; + } + return out; +} + +int main(int argc, char** argv) +{ + QCoreApplication app(argc, argv); + + /* + tst_selftests can't handle multiple XML documents in a single testrun, so we'll + decide before we begin which of our "bad strings" we want to use for our testcase + name. + */ + int badstring = -1; + QVector<char const*> args; + for (int i = 0; i < argc; ++i) { + if (!strcmp(argv[i], "-badstring")) { + bool ok = false; + if (i < argc-1) { + badstring = QByteArray(argv[i+1]).toInt(&ok); + ++i; + } + if (!ok) { + qFatal("Bad `-badstring' option"); + } + } + else { + args << argv[i]; + } + } + /* + We just want testlib to output a benchmark result, we don't actually care about the value, + so just do one iteration to save time. + */ + args << "-iterations" << "1"; + + if (badstring == -1) { + tst_BadXml test; + return QTest::qExec(&test, args.count(), const_cast<char**>(args.data())); + } + + QList<QByteArray> badstrings = tst_BadXml::badStrings(); + if (badstring >= badstrings.count()) + qFatal("`-badstring %d' is out of range", badstring); + + tst_BadXmlSub test; + test.className = badstrings[badstring].constData(); + return QTest::qExec(&test, args.count(), const_cast<char**>(args.data())); +} + +#include "tst_badxml.moc" diff --git a/tests/auto/selftests/expected_skip.txt b/tests/auto/selftests/expected_skip.txt index f3be5b5..88c0426 100644 --- a/tests/auto/selftests/expected_skip.txt +++ b/tests/auto/selftests/expected_skip.txt @@ -1,12 +1,12 @@ ********* Start testing of tst_Skip ********* -Config: Using QTest library 4.3.0, Qt 4.3.0 +Config: Using QTest library 4.6.0, Qt 4.6.0 PASS : tst_Skip::initTestCase() SKIP : tst_Skip::test() skipping all - Loc: [/home/fenglich/dev/qt-4.3/tests/auto/selftests/skip/tst_skip.cpp(35)] + Loc: [/home/rmcgover/depot/qt-git/mainline/tests/auto/selftests/skip/tst_skip.cpp(68)] SKIP : tst_Skip::emptytest() skipping all - Loc: [/home/fenglich/dev/qt-4.3/tests/auto/selftests/skip/tst_skip.cpp(45)] + Loc: [/home/rmcgover/depot/qt-git/mainline/tests/auto/selftests/skip/tst_skip.cpp(78)] SKIP : tst_Skip::singleSkip(local 1) skipping one - Loc: [/home/fenglich/dev/qt-4.3/tests/auto/selftests/skip/tst_skip.cpp(64)] + Loc: [/home/rmcgover/depot/qt-git/mainline/tests/auto/selftests/skip/tst_skip.cpp(97)] QDEBUG : tst_Skip::singleSkip(local 2) this line should only be reached once (true) PASS : tst_Skip::singleSkip() PASS : tst_Skip::cleanupTestCase() diff --git a/tests/auto/selftests/expected_xunit.txt b/tests/auto/selftests/expected_xunit.txt new file mode 100644 index 0000000..cb74491 --- /dev/null +++ b/tests/auto/selftests/expected_xunit.txt @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8" ?> +<testsuite errors="5" failures="3" tests="9" name="tst_Xunit"> + <properties> + <property value="4.6.0" name="QTestVersion"/> + <property value="4.6.0" name="QtVersion"/> + </properties> + <testcase result="pass" name="initTestCase"/> + <testcase result="pass" name="testFunc1"> + <!-- message="just a QWARN() !" type="warn" --> + </testcase> + <testcase result="fail" name="testFunc2"> + <!-- message="a qDebug() call with comment-ending stuff -->" type="qdebug" --> + <failure message="Compared values are not the same + Actual (2): 2 + Expected (3): 3" result="fail"/> + </testcase> + <testcase name="testFunc3"> + <!-- message="skipping this function!" type="skip" --> + </testcase> + <testcase result="fail" name="testFunc4"> + <failure message="a forced failure!" result="fail"/> + </testcase> + <testcase result="xfail" name="testFunc5"> + <!-- message="this failure is expected" type="info" --> + </testcase> + <testcase result="xfail" name="testFunc6"> + <!-- message="this failure is also expected" type="info" --> + </testcase> + <testcase result="xpass" name="testFunc7"> + <failure message="'true' returned FALSE. ()" result="xpass"/> + </testcase> + <testcase result="pass" name="cleanupTestCase"/> + <system-err> +<![CDATA[just a QWARN() !]]> +<![CDATA[a qDebug() call with comment-ending stuff -->]]> +<![CDATA[skipping this function!]]> +<![CDATA[this failure is expected]]> +<![CDATA[this failure is also expected]]> + </system-err> +</testsuite> diff --git a/tests/auto/selftests/selftests.pro b/tests/auto/selftests/selftests.pro index 2fa1d50..ca69afa 100644 --- a/tests/auto/selftests/selftests.pro +++ b/tests/auto/selftests/selftests.pro @@ -5,7 +5,7 @@ SUBDIRS = subtest test warnings maxwarnings cmptest globaldata skipglobal skip \ skipinit skipinitdata datetime singleskip assert waitwithoutgui differentexec \ exception qexecstringlist datatable commandlinedata\ benchlibwalltime benchlibcallgrind benchlibeventcounter benchlibtickcounter \ - benchliboptions + benchliboptions xunit badxml INSTALLS = diff --git a/tests/auto/selftests/selftests.qrc b/tests/auto/selftests/selftests.qrc index 125619e..d57ff29 100644 --- a/tests/auto/selftests/selftests.qrc +++ b/tests/auto/selftests/selftests.qrc @@ -34,5 +34,6 @@ <file>expected_benchlibeventcounter.txt</file> <file>expected_benchliboptions.txt</file> <file>expected_benchlibtickcounter.txt</file> + <file>expected_xunit.txt</file> </qresource> </RCC> diff --git a/tests/auto/selftests/tst_selftests.cpp b/tests/auto/selftests/tst_selftests.cpp index 6776b12..eca9aa6 100644 --- a/tests/auto/selftests/tst_selftests.cpp +++ b/tests/auto/selftests/tst_selftests.cpp @@ -54,9 +54,13 @@ private slots: void runSubTest(); void checkXML() const; void checkXML_data(); + void checkXunitxml() const; + void checkXunitxml_data(); private: QStringList m_checkXMLBlacklist; + QStringList m_checkXunitBlacklist; + void doRunSubTest(QString &subdir, QStringList &arguments ); }; struct BenchmarkResult @@ -97,7 +101,7 @@ inline bool qCompare variance = 0.001; } else if (r1.unit == "ticks") { - variance = 0.0001; + variance = 0.001; } if (variance == 0.) { /* No variance allowed - compare whole string */ @@ -186,17 +190,16 @@ void tst_Selftests::runSubTest_data() QTest::newRow("benchlibtickcounter") << "benchlibtickcounter" << QStringList("-tickcounter"); #endif + QTest::newRow("xunit") << "xunit" << QStringList("-xunitxml"); + } -void tst_Selftests::runSubTest() +void tst_Selftests::doRunSubTest(QString &subdir, QStringList &arguments ) { - QFETCH(QString, subdir); - QFETCH(QStringList, arguments); - QProcess proc; proc.setEnvironment(QStringList("")); proc.start(subdir + "/" + subdir, arguments); - QVERIFY(proc.waitForFinished()); + QVERIFY2(proc.waitForFinished(), qPrintable(proc.errorString())); const QByteArray out(proc.readAllStandardOutput()); const QByteArray err(proc.readAllStandardError()); @@ -206,8 +209,8 @@ void tst_Selftests::runSubTest() #if defined(Q_OS_WIN) if(subdir != QLatin1String("exception") && subdir != QLatin1String("fetchbogus")) #endif - QVERIFY2(err.isEmpty(), err.constData()); - + if(subdir != QLatin1String("xunit")) + QVERIFY2(err.isEmpty(), err.constData()); QList<QByteArray> res = splitLines(out); QList<QByteArray> exp = expectedResult(subdir); @@ -255,8 +258,8 @@ void tst_Selftests::runSubTest() { if(output != expected && qstrcmp(QTest::currentDataTag(), "subtest") == 0) { - /* The floating point formatting differs between platforms, so let's just skip it. */ - continue; + /* The floating point formatting differs between platforms, so let's just skip it. */ + continue; } else { /* @@ -284,6 +287,14 @@ void tst_Selftests::runSubTest() } } +void tst_Selftests::runSubTest() +{ + QFETCH(QString, subdir); + QFETCH(QStringList, arguments); + + doRunSubTest(subdir, arguments); +} + void tst_Selftests::initTestCase() { m_checkXMLBlacklist.append("crashes"); // This test crashes @@ -300,6 +311,8 @@ void tst_Selftests::initTestCase() m_checkXMLBlacklist.append("subtest"); m_checkXMLBlacklist.append("globaldata"); m_checkXMLBlacklist.append("warnings"); + + m_checkXunitBlacklist = m_checkXMLBlacklist; } void tst_Selftests::checkXML() const @@ -310,7 +323,53 @@ void tst_Selftests::checkXML() const if(m_checkXMLBlacklist.contains(subdir)) return; - arguments.prepend("-xml"); + QStringList args; + /* Test both old (-flush) and new XML logger implementation */ + for (int i = 0; i < 2; ++i) { + bool flush = i; + args = arguments; + args.prepend("-xml"); + if (flush) args.prepend("-flush"); + + QProcess proc; + proc.setEnvironment(QStringList("")); + proc.start(subdir + "/" + subdir, args); + QVERIFY(proc.waitForFinished()); + + QByteArray out(proc.readAllStandardOutput()); + QByteArray err(proc.readAllStandardError()); + + /* Some platforms decides to output a message for uncaught exceptions. For instance, + * this is what windows platforms says: + * "This application has requested the Runtime to terminate it in an unusual way. + * Please contact the application's support team for more information." */ + if(subdir != QLatin1String("exception") && subdir != QLatin1String("fetchbogus")) + QVERIFY2(err.isEmpty(), err.constData()); + + QXmlStreamReader reader(out); + + while(!reader.atEnd()) + reader.readNext(); + + QVERIFY2(!reader.error(), qPrintable(QString("(flush %0) line %1, col %2: %3") + .arg(flush) + .arg(reader.lineNumber()) + .arg(reader.columnNumber()) + .arg(reader.errorString()) + )); + } +} + +void tst_Selftests::checkXunitxml() const +{ + QFETCH(QString, subdir); + QFETCH(QStringList, arguments); + + if(m_checkXunitBlacklist.contains(subdir)) + return; + + arguments.prepend("-xunitxml"); + arguments.prepend("-flush"); QProcess proc; proc.setEnvironment(QStringList("")); @@ -320,6 +379,8 @@ void tst_Selftests::checkXML() const QByteArray out(proc.readAllStandardOutput()); QByteArray err(proc.readAllStandardError()); +// qDebug()<<out; + /* Some platforms decides to output a message for uncaught exceptions. For instance, * this is what windows platforms says: * "This application has requested the Runtime to terminate it in an unusual way. @@ -332,12 +393,26 @@ void tst_Selftests::checkXML() const while(!reader.atEnd()) reader.readNext(); - QVERIFY(!reader.error()); + QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3") + .arg(reader.lineNumber()) + .arg(reader.columnNumber()) + .arg(reader.errorString()) + )); +} + +void tst_Selftests::checkXunitxml_data() +{ + checkXML_data(); } void tst_Selftests::checkXML_data() { runSubTest_data(); + QTest::newRow("badxml 1") << "badxml" << QStringList(); + QTest::newRow("badxml 2") << "badxml" << (QStringList() << "-badstring" << "0"); + QTest::newRow("badxml 3") << "badxml" << (QStringList() << "-badstring" << "1"); + QTest::newRow("badxml 4") << "badxml" << (QStringList() << "-badstring" << "2"); + QTest::newRow("badxml 5") << "badxml" << (QStringList() << "-badstring" << "3"); } /* Parse line into the BenchmarkResult it represents. */ diff --git a/tests/auto/selftests/xunit/tst_xunit b/tests/auto/selftests/xunit/tst_xunit Binary files differnew file mode 100755 index 0000000..31d03a8 --- /dev/null +++ b/tests/auto/selftests/xunit/tst_xunit diff --git a/tests/auto/selftests/xunit/tst_xunit.cpp b/tests/auto/selftests/xunit/tst_xunit.cpp new file mode 100644 index 0000000..c3c90ab --- /dev/null +++ b/tests/auto/selftests/xunit/tst_xunit.cpp @@ -0,0 +1,115 @@ +/**************************************************************************** +** +** Copyright (C) 2008 Nokia Corporation and/or its subsidiary(-ies). +** Contact: Qt Software Information (qt-info@nokia.com) +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the either Technology Preview License Agreement or the +** Beta Release License Agreement. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain +** additional rights. These rights are described in the Nokia Qt LGPL +** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this +** package. +** +** GNU General Public License Usage +** Alternatively, this file may be used under the terms of the GNU +** General Public License version 3.0 as published by the Free Software +** Foundation and appearing in the file LICENSE.GPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU General Public License version 3.0 requirements will be +** met: http://www.gnu.org/copyleft/gpl.html. +** +** If you are unsure which license is appropriate for your use, please +** contact the sales department at qt-sales@nokia.com. +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include <QtTest/QtTest> + +class tst_Xunit : public QObject +{ + Q_OBJECT + +public: + tst_Xunit(); + +private slots: + void testFunc1(); + void testFunc2(); + void testFunc3(); + void testFunc4(); + void testFunc5(); + void testFunc6(); + void testFunc7(); +}; + +tst_Xunit::tst_Xunit() +{ +} + +void tst_Xunit::testFunc1() +{ + QWARN("just a QWARN() !"); + QCOMPARE(1,1); +} + +void tst_Xunit::testFunc2() +{ + qDebug("a qDebug() call with comment-ending stuff -->"); + QCOMPARE(2, 3); +} + +void tst_Xunit::testFunc3() +{ + QSKIP("skipping this function!", SkipAll); +} + +void tst_Xunit::testFunc4() +{ + QFAIL("a forced failure!"); +} + +/* + Note there are two testfunctions which give expected failures. + This is so we can test that expected failures don't add to failure + counts and unexpected passes do. If we had one xfail and one xpass + testfunction, we couldn't test which one of them adds to the failure + count. +*/ + +void tst_Xunit::testFunc5() +{ + QEXPECT_FAIL("", "this failure is expected", Abort); + QVERIFY(false); +} + +void tst_Xunit::testFunc6() +{ + QEXPECT_FAIL("", "this failure is also expected", Abort); + QVERIFY(false); +} + +void tst_Xunit::testFunc7() +{ + QEXPECT_FAIL("", "this pass is unexpected", Abort); + QVERIFY(true); +} + + +QTEST_APPLESS_MAIN(tst_Xunit) +#include "tst_xunit.moc" diff --git a/tests/auto/selftests/xunit/xunit.pro b/tests/auto/selftests/xunit/xunit.pro new file mode 100644 index 0000000..81ca157 --- /dev/null +++ b/tests/auto/selftests/xunit/xunit.pro @@ -0,0 +1,15 @@ +load(qttest_p4) +SOURCES += tst_xunit.cpp + +wince*: { + addImages.sources = images/* + addImages.path = images + DEPLOYMENT += addImages + DEFINES += SRCDIR=\\\".\\\" +} else { + contains(QT_CONFIG, qt3support): QT += qt3support + DEFINES += SRCDIR=\\\"$$PWD\\\" +} + +TARGET = xunit + |