summaryrefslogtreecommitdiffstats
path: root/tests/auto/selftests
diff options
context:
space:
mode:
Diffstat (limited to 'tests/auto/selftests')
-rw-r--r--tests/auto/selftests/badxml/badxml.pro11
-rw-r--r--tests/auto/selftests/badxml/tst_badxml.cpp206
-rw-r--r--tests/auto/selftests/expected_skip.txt8
-rw-r--r--tests/auto/selftests/expected_xunit.txt40
-rw-r--r--tests/auto/selftests/selftests.pro2
-rw-r--r--tests/auto/selftests/selftests.qrc1
-rw-r--r--tests/auto/selftests/tst_selftests.cpp99
-rwxr-xr-xtests/auto/selftests/xunit/tst_xunitbin0 -> 11624 bytes
-rw-r--r--tests/auto/selftests/xunit/tst_xunit.cpp115
-rw-r--r--tests/auto/selftests/xunit/xunit.pro15
10 files changed, 480 insertions, 17 deletions
diff --git a/tests/auto/selftests/badxml/badxml.pro b/tests/auto/selftests/badxml/badxml.pro
new file mode 100644
index 0000000..323791a
--- /dev/null
+++ b/tests/auto/selftests/badxml/badxml.pro
@@ -0,0 +1,11 @@
+load(qttest_p4)
+
+SOURCES += tst_badxml.cpp
+QT = core
+
+mac:CONFIG -= app_bundle
+CONFIG -= debug_and_release_target
+!win32:CONFIG += debug
+
+TARGET = badxml
+
diff --git a/tests/auto/selftests/badxml/tst_badxml.cpp b/tests/auto/selftests/badxml/tst_badxml.cpp
new file mode 100644
index 0000000..fa5b717
--- /dev/null
+++ b/tests/auto/selftests/badxml/tst_badxml.cpp
@@ -0,0 +1,206 @@
+/****************************************************************************
+**
+** Copyright (C) 2009 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: Qt Software Information (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the either Technology Preview License Agreement or the
+** Beta Release License Agreement.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain
+** additional rights. These rights are described in the Nokia Qt LGPL
+** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
+** package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+** If you are unsure which license is appropriate for your use, please
+** contact the sales department at qt-sales@nokia.com.
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+
+#include <QtCore>
+#include <QtTest/QtTest>
+
+/*
+ This test makes a testlog containing lots of characters which have a special meaning in
+ XML, with the purpose of exposing bugs in testlib's XML output code.
+*/
+class tst_BadXml : public QObject
+{
+ Q_OBJECT
+
+private slots:
+ void badDataTag() const;
+ void badDataTag_data() const;
+
+ void badMessage() const;
+ void badMessage_data() const;
+
+ void failWithNoFile() const;
+
+public:
+ static QList<QByteArray> const& badStrings();
+};
+
+/*
+ Custom metaobject to make it possible to change class name at runtime.
+*/
+class EmptyClass : public tst_BadXml
+{ Q_OBJECT };
+
+class tst_BadXmlSub : public tst_BadXml
+{
+public:
+ const QMetaObject* metaObject() const;
+
+ static char const* className;
+};
+char const* tst_BadXmlSub::className = "tst_BadXml";
+
+const QMetaObject* tst_BadXmlSub::metaObject() const
+{
+ const QMetaObject& empty = EmptyClass::staticMetaObject;
+ static QMetaObject mo = {
+ { empty.d.superdata, empty.d.stringdata, empty.d.data, empty.d.extradata }
+ };
+ static char currentClassName[1024];
+ qstrcpy(currentClassName, className);
+ int len = qstrlen(className);
+ currentClassName[len] = 0;
+ currentClassName[len+1] = 0;
+
+ mo.d.stringdata = currentClassName;
+
+ return &mo;
+}
+
+/*
+ Outputs incidents and benchmark results with the current data tag set to a bad string.
+*/
+void tst_BadXml::badDataTag() const
+{
+ qDebug("a message");
+
+ QBENCHMARK {
+ }
+
+ QFAIL("a failure");
+}
+
+void tst_BadXml::badDataTag_data() const
+{
+ QTest::addColumn<int>("dummy");
+
+ foreach (char const* str, badStrings()) {
+ QTest::newRow(str) << 0;
+ }
+}
+
+void tst_BadXml::failWithNoFile() const
+{
+ QTest::qFail("failure message", 0, 0);
+}
+
+/*
+ Outputs a message containing a bad string.
+*/
+void tst_BadXml::badMessage() const
+{
+ QFETCH(QByteArray, message);
+ qDebug("%s", message.constData());
+}
+
+void tst_BadXml::badMessage_data() const
+{
+ QTest::addColumn<QByteArray>("message");
+
+ int i = 0;
+ foreach (QByteArray const& str, badStrings()) {
+ QTest::newRow(qPrintable(QString::fromLatin1("string %1").arg(i++))) << str;
+ }
+}
+
+/*
+ Returns a list of strings likely to expose bugs in XML output code.
+*/
+QList<QByteArray> const& tst_BadXml::badStrings()
+{
+ static QList<QByteArray> out;
+ if (out.isEmpty()) {
+ out << "end cdata ]]> text ]]> more text";
+ out << "quotes \" text\" more text";
+ out << "xml close > open < tags < text";
+ out << "all > \" mixed ]]> up > \" in < the ]]> hopes < of triggering \"< ]]> bugs";
+ }
+ return out;
+}
+
+int main(int argc, char** argv)
+{
+ QCoreApplication app(argc, argv);
+
+ /*
+ tst_selftests can't handle multiple XML documents in a single testrun, so we'll
+ decide before we begin which of our "bad strings" we want to use for our testcase
+ name.
+ */
+ int badstring = -1;
+ QVector<char const*> args;
+ for (int i = 0; i < argc; ++i) {
+ if (!strcmp(argv[i], "-badstring")) {
+ bool ok = false;
+ if (i < argc-1) {
+ badstring = QByteArray(argv[i+1]).toInt(&ok);
+ ++i;
+ }
+ if (!ok) {
+ qFatal("Bad `-badstring' option");
+ }
+ }
+ else {
+ args << argv[i];
+ }
+ }
+ /*
+ We just want testlib to output a benchmark result, we don't actually care about the value,
+ so just do one iteration to save time.
+ */
+ args << "-iterations" << "1";
+
+ if (badstring == -1) {
+ tst_BadXml test;
+ return QTest::qExec(&test, args.count(), const_cast<char**>(args.data()));
+ }
+
+ QList<QByteArray> badstrings = tst_BadXml::badStrings();
+ if (badstring >= badstrings.count())
+ qFatal("`-badstring %d' is out of range", badstring);
+
+ tst_BadXmlSub test;
+ test.className = badstrings[badstring].constData();
+ return QTest::qExec(&test, args.count(), const_cast<char**>(args.data()));
+}
+
+#include "tst_badxml.moc"
diff --git a/tests/auto/selftests/expected_skip.txt b/tests/auto/selftests/expected_skip.txt
index f3be5b5..88c0426 100644
--- a/tests/auto/selftests/expected_skip.txt
+++ b/tests/auto/selftests/expected_skip.txt
@@ -1,12 +1,12 @@
********* Start testing of tst_Skip *********
-Config: Using QTest library 4.3.0, Qt 4.3.0
+Config: Using QTest library 4.6.0, Qt 4.6.0
PASS : tst_Skip::initTestCase()
SKIP : tst_Skip::test() skipping all
- Loc: [/home/fenglich/dev/qt-4.3/tests/auto/selftests/skip/tst_skip.cpp(35)]
+ Loc: [/home/rmcgover/depot/qt-git/mainline/tests/auto/selftests/skip/tst_skip.cpp(68)]
SKIP : tst_Skip::emptytest() skipping all
- Loc: [/home/fenglich/dev/qt-4.3/tests/auto/selftests/skip/tst_skip.cpp(45)]
+ Loc: [/home/rmcgover/depot/qt-git/mainline/tests/auto/selftests/skip/tst_skip.cpp(78)]
SKIP : tst_Skip::singleSkip(local 1) skipping one
- Loc: [/home/fenglich/dev/qt-4.3/tests/auto/selftests/skip/tst_skip.cpp(64)]
+ Loc: [/home/rmcgover/depot/qt-git/mainline/tests/auto/selftests/skip/tst_skip.cpp(97)]
QDEBUG : tst_Skip::singleSkip(local 2) this line should only be reached once (true)
PASS : tst_Skip::singleSkip()
PASS : tst_Skip::cleanupTestCase()
diff --git a/tests/auto/selftests/expected_xunit.txt b/tests/auto/selftests/expected_xunit.txt
new file mode 100644
index 0000000..cb74491
--- /dev/null
+++ b/tests/auto/selftests/expected_xunit.txt
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuite errors="5" failures="3" tests="9" name="tst_Xunit">
+ <properties>
+ <property value="4.6.0" name="QTestVersion"/>
+ <property value="4.6.0" name="QtVersion"/>
+ </properties>
+ <testcase result="pass" name="initTestCase"/>
+ <testcase result="pass" name="testFunc1">
+ <!-- message="just a QWARN() !" type="warn" -->
+ </testcase>
+ <testcase result="fail" name="testFunc2">
+ <!-- message="a qDebug() call with comment&#x002D;ending stuff &#x002D;&#x002D;&gt;" type="qdebug" -->
+ <failure message="Compared values are not the same
+ Actual (2): 2
+ Expected (3): 3" result="fail"/>
+ </testcase>
+ <testcase name="testFunc3">
+ <!-- message="skipping this function!" type="skip" -->
+ </testcase>
+ <testcase result="fail" name="testFunc4">
+ <failure message="a forced failure!" result="fail"/>
+ </testcase>
+ <testcase result="xfail" name="testFunc5">
+ <!-- message="this failure is expected" type="info" -->
+ </testcase>
+ <testcase result="xfail" name="testFunc6">
+ <!-- message="this failure is also expected" type="info" -->
+ </testcase>
+ <testcase result="xpass" name="testFunc7">
+ <failure message="&apos;true&apos; returned FALSE. ()" result="xpass"/>
+ </testcase>
+ <testcase result="pass" name="cleanupTestCase"/>
+ <system-err>
+<![CDATA[just a QWARN() !]]>
+<![CDATA[a qDebug() call with comment-ending stuff -->]]>
+<![CDATA[skipping this function!]]>
+<![CDATA[this failure is expected]]>
+<![CDATA[this failure is also expected]]>
+ </system-err>
+</testsuite>
diff --git a/tests/auto/selftests/selftests.pro b/tests/auto/selftests/selftests.pro
index 2fa1d50..ca69afa 100644
--- a/tests/auto/selftests/selftests.pro
+++ b/tests/auto/selftests/selftests.pro
@@ -5,7 +5,7 @@ SUBDIRS = subtest test warnings maxwarnings cmptest globaldata skipglobal skip \
skipinit skipinitdata datetime singleskip assert waitwithoutgui differentexec \
exception qexecstringlist datatable commandlinedata\
benchlibwalltime benchlibcallgrind benchlibeventcounter benchlibtickcounter \
- benchliboptions
+ benchliboptions xunit badxml
INSTALLS =
diff --git a/tests/auto/selftests/selftests.qrc b/tests/auto/selftests/selftests.qrc
index 125619e..d57ff29 100644
--- a/tests/auto/selftests/selftests.qrc
+++ b/tests/auto/selftests/selftests.qrc
@@ -34,5 +34,6 @@
<file>expected_benchlibeventcounter.txt</file>
<file>expected_benchliboptions.txt</file>
<file>expected_benchlibtickcounter.txt</file>
+ <file>expected_xunit.txt</file>
</qresource>
</RCC>
diff --git a/tests/auto/selftests/tst_selftests.cpp b/tests/auto/selftests/tst_selftests.cpp
index 6776b12..eca9aa6 100644
--- a/tests/auto/selftests/tst_selftests.cpp
+++ b/tests/auto/selftests/tst_selftests.cpp
@@ -54,9 +54,13 @@ private slots:
void runSubTest();
void checkXML() const;
void checkXML_data();
+ void checkXunitxml() const;
+ void checkXunitxml_data();
private:
QStringList m_checkXMLBlacklist;
+ QStringList m_checkXunitBlacklist;
+ void doRunSubTest(QString &subdir, QStringList &arguments );
};
struct BenchmarkResult
@@ -97,7 +101,7 @@ inline bool qCompare
variance = 0.001;
}
else if (r1.unit == "ticks") {
- variance = 0.0001;
+ variance = 0.001;
}
if (variance == 0.) {
/* No variance allowed - compare whole string */
@@ -186,17 +190,16 @@ void tst_Selftests::runSubTest_data()
QTest::newRow("benchlibtickcounter") << "benchlibtickcounter" << QStringList("-tickcounter");
#endif
+ QTest::newRow("xunit") << "xunit" << QStringList("-xunitxml");
+
}
-void tst_Selftests::runSubTest()
+void tst_Selftests::doRunSubTest(QString &subdir, QStringList &arguments )
{
- QFETCH(QString, subdir);
- QFETCH(QStringList, arguments);
-
QProcess proc;
proc.setEnvironment(QStringList(""));
proc.start(subdir + "/" + subdir, arguments);
- QVERIFY(proc.waitForFinished());
+ QVERIFY2(proc.waitForFinished(), qPrintable(proc.errorString()));
const QByteArray out(proc.readAllStandardOutput());
const QByteArray err(proc.readAllStandardError());
@@ -206,8 +209,8 @@ void tst_Selftests::runSubTest()
#if defined(Q_OS_WIN)
if(subdir != QLatin1String("exception") && subdir != QLatin1String("fetchbogus"))
#endif
- QVERIFY2(err.isEmpty(), err.constData());
-
+ if(subdir != QLatin1String("xunit"))
+ QVERIFY2(err.isEmpty(), err.constData());
QList<QByteArray> res = splitLines(out);
QList<QByteArray> exp = expectedResult(subdir);
@@ -255,8 +258,8 @@ void tst_Selftests::runSubTest()
{
if(output != expected && qstrcmp(QTest::currentDataTag(), "subtest") == 0)
{
- /* The floating point formatting differs between platforms, so let's just skip it. */
- continue;
+ /* The floating point formatting differs between platforms, so let's just skip it. */
+ continue;
}
else {
/*
@@ -284,6 +287,14 @@ void tst_Selftests::runSubTest()
}
}
+void tst_Selftests::runSubTest()
+{
+ QFETCH(QString, subdir);
+ QFETCH(QStringList, arguments);
+
+ doRunSubTest(subdir, arguments);
+}
+
void tst_Selftests::initTestCase()
{
m_checkXMLBlacklist.append("crashes"); // This test crashes
@@ -300,6 +311,8 @@ void tst_Selftests::initTestCase()
m_checkXMLBlacklist.append("subtest");
m_checkXMLBlacklist.append("globaldata");
m_checkXMLBlacklist.append("warnings");
+
+ m_checkXunitBlacklist = m_checkXMLBlacklist;
}
void tst_Selftests::checkXML() const
@@ -310,7 +323,53 @@ void tst_Selftests::checkXML() const
if(m_checkXMLBlacklist.contains(subdir))
return;
- arguments.prepend("-xml");
+ QStringList args;
+ /* Test both old (-flush) and new XML logger implementation */
+ for (int i = 0; i < 2; ++i) {
+ bool flush = i;
+ args = arguments;
+ args.prepend("-xml");
+ if (flush) args.prepend("-flush");
+
+ QProcess proc;
+ proc.setEnvironment(QStringList(""));
+ proc.start(subdir + "/" + subdir, args);
+ QVERIFY(proc.waitForFinished());
+
+ QByteArray out(proc.readAllStandardOutput());
+ QByteArray err(proc.readAllStandardError());
+
+ /* Some platforms decides to output a message for uncaught exceptions. For instance,
+ * this is what windows platforms says:
+ * "This application has requested the Runtime to terminate it in an unusual way.
+ * Please contact the application's support team for more information." */
+ if(subdir != QLatin1String("exception") && subdir != QLatin1String("fetchbogus"))
+ QVERIFY2(err.isEmpty(), err.constData());
+
+ QXmlStreamReader reader(out);
+
+ while(!reader.atEnd())
+ reader.readNext();
+
+ QVERIFY2(!reader.error(), qPrintable(QString("(flush %0) line %1, col %2: %3")
+ .arg(flush)
+ .arg(reader.lineNumber())
+ .arg(reader.columnNumber())
+ .arg(reader.errorString())
+ ));
+ }
+}
+
+void tst_Selftests::checkXunitxml() const
+{
+ QFETCH(QString, subdir);
+ QFETCH(QStringList, arguments);
+
+ if(m_checkXunitBlacklist.contains(subdir))
+ return;
+
+ arguments.prepend("-xunitxml");
+ arguments.prepend("-flush");
QProcess proc;
proc.setEnvironment(QStringList(""));
@@ -320,6 +379,8 @@ void tst_Selftests::checkXML() const
QByteArray out(proc.readAllStandardOutput());
QByteArray err(proc.readAllStandardError());
+// qDebug()<<out;
+
/* Some platforms decides to output a message for uncaught exceptions. For instance,
* this is what windows platforms says:
* "This application has requested the Runtime to terminate it in an unusual way.
@@ -332,12 +393,26 @@ void tst_Selftests::checkXML() const
while(!reader.atEnd())
reader.readNext();
- QVERIFY(!reader.error());
+ QVERIFY2(!reader.error(), qPrintable(QString("line %1, col %2: %3")
+ .arg(reader.lineNumber())
+ .arg(reader.columnNumber())
+ .arg(reader.errorString())
+ ));
+}
+
+void tst_Selftests::checkXunitxml_data()
+{
+ checkXML_data();
}
void tst_Selftests::checkXML_data()
{
runSubTest_data();
+ QTest::newRow("badxml 1") << "badxml" << QStringList();
+ QTest::newRow("badxml 2") << "badxml" << (QStringList() << "-badstring" << "0");
+ QTest::newRow("badxml 3") << "badxml" << (QStringList() << "-badstring" << "1");
+ QTest::newRow("badxml 4") << "badxml" << (QStringList() << "-badstring" << "2");
+ QTest::newRow("badxml 5") << "badxml" << (QStringList() << "-badstring" << "3");
}
/* Parse line into the BenchmarkResult it represents. */
diff --git a/tests/auto/selftests/xunit/tst_xunit b/tests/auto/selftests/xunit/tst_xunit
new file mode 100755
index 0000000..31d03a8
--- /dev/null
+++ b/tests/auto/selftests/xunit/tst_xunit
Binary files differ
diff --git a/tests/auto/selftests/xunit/tst_xunit.cpp b/tests/auto/selftests/xunit/tst_xunit.cpp
new file mode 100644
index 0000000..c3c90ab
--- /dev/null
+++ b/tests/auto/selftests/xunit/tst_xunit.cpp
@@ -0,0 +1,115 @@
+/****************************************************************************
+**
+** Copyright (C) 2008 Nokia Corporation and/or its subsidiary(-ies).
+** Contact: Qt Software Information (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the either Technology Preview License Agreement or the
+** Beta Release License Agreement.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain
+** additional rights. These rights are described in the Nokia Qt LGPL
+** Exception version 1.0, included in the file LGPL_EXCEPTION.txt in this
+** package.
+**
+** GNU General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU
+** General Public License version 3.0 as published by the Free Software
+** Foundation and appearing in the file LICENSE.GPL included in the
+** packaging of this file. Please review the following information to
+** ensure the GNU General Public License version 3.0 requirements will be
+** met: http://www.gnu.org/copyleft/gpl.html.
+**
+** If you are unsure which license is appropriate for your use, please
+** contact the sales department at qt-sales@nokia.com.
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QtTest/QtTest>
+
+class tst_Xunit : public QObject
+{
+ Q_OBJECT
+
+public:
+ tst_Xunit();
+
+private slots:
+ void testFunc1();
+ void testFunc2();
+ void testFunc3();
+ void testFunc4();
+ void testFunc5();
+ void testFunc6();
+ void testFunc7();
+};
+
+tst_Xunit::tst_Xunit()
+{
+}
+
+void tst_Xunit::testFunc1()
+{
+ QWARN("just a QWARN() !");
+ QCOMPARE(1,1);
+}
+
+void tst_Xunit::testFunc2()
+{
+ qDebug("a qDebug() call with comment-ending stuff -->");
+ QCOMPARE(2, 3);
+}
+
+void tst_Xunit::testFunc3()
+{
+ QSKIP("skipping this function!", SkipAll);
+}
+
+void tst_Xunit::testFunc4()
+{
+ QFAIL("a forced failure!");
+}
+
+/*
+ Note there are two testfunctions which give expected failures.
+ This is so we can test that expected failures don't add to failure
+ counts and unexpected passes do. If we had one xfail and one xpass
+ testfunction, we couldn't test which one of them adds to the failure
+ count.
+*/
+
+void tst_Xunit::testFunc5()
+{
+ QEXPECT_FAIL("", "this failure is expected", Abort);
+ QVERIFY(false);
+}
+
+void tst_Xunit::testFunc6()
+{
+ QEXPECT_FAIL("", "this failure is also expected", Abort);
+ QVERIFY(false);
+}
+
+void tst_Xunit::testFunc7()
+{
+ QEXPECT_FAIL("", "this pass is unexpected", Abort);
+ QVERIFY(true);
+}
+
+
+QTEST_APPLESS_MAIN(tst_Xunit)
+#include "tst_xunit.moc"
diff --git a/tests/auto/selftests/xunit/xunit.pro b/tests/auto/selftests/xunit/xunit.pro
new file mode 100644
index 0000000..81ca157
--- /dev/null
+++ b/tests/auto/selftests/xunit/xunit.pro
@@ -0,0 +1,15 @@
+load(qttest_p4)
+SOURCES += tst_xunit.cpp
+
+wince*: {
+ addImages.sources = images/*
+ addImages.path = images
+ DEPLOYMENT += addImages
+ DEFINES += SRCDIR=\\\".\\\"
+} else {
+ contains(QT_CONFIG, qt3support): QT += qt3support
+ DEFINES += SRCDIR=\\\"$$PWD\\\"
+}
+
+TARGET = xunit
+