From d29e1dda8fbc24087b9cf0bf08d929bef74f5c96 Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Sat, 22 May 2010 17:54:33 +1000 Subject: Move `check' target for autotests into testcase.prf testcase.prf is installed, thus allowing `make check' to work outside of the Qt source tree. --- mkspecs/features/qttest_p4.prf | 36 +---------------------------- mkspecs/features/testcase.prf | 51 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 35 deletions(-) create mode 100644 mkspecs/features/testcase.prf diff --git a/mkspecs/features/qttest_p4.prf b/mkspecs/features/qttest_p4.prf index d2011d0..d1c7c2b 100644 --- a/mkspecs/features/qttest_p4.prf +++ b/mkspecs/features/qttest_p4.prf @@ -1,5 +1,5 @@ isEmpty(TEMPLATE):TEMPLATE=app -CONFIG += qt warn_on console depend_includepath +CONFIG += qt warn_on console depend_includepath testcase qtAddLibrary(QtTest) @@ -13,39 +13,5 @@ symbian:{ # prefix test binary with tst_ !contains(TARGET, ^tst_.*):TARGET = $$join(TARGET,,"tst_") - -check.files = -check.path = . -!isEmpty(DESTDIR): check.commands = cd ./$(DESTDIR) && -macx: check.commands += ./$(QMAKE_TARGET).app/Contents/MacOS/$(QMAKE_TARGET) -else:unix: check.commands += ./$(QMAKE_TARGET) -else:win32: { - CONFIG(debug, debug|release):check.commands += $(DESTDIR_TARGET) - else:check.commands += $(DESTDIR_TARGET) -} -embedded: check.commands += -qws -QMAKE_EXTRA_TARGETS += check - -!debug_and_release|build_pass { - check.depends = first -} else { - check.CONFIG = recursive - # In debug and release mode, only run the test once. - # Run debug if available, release otherwise. - debug_and_release { - check.target = dummy_check - check.recurse_target = check - debug { - real_check.depends = debug-check - real_check.target = check - QMAKE_EXTRA_TARGETS += real_check - } else { - real_check.depends = release-check - real_check.target = check - QMAKE_EXTRA_TARGETS += real_check - } - } -} - target.path += $$[QT_INSTALL_PREFIX]/tests/qt4 INSTALLS += target diff --git a/mkspecs/features/testcase.prf b/mkspecs/features/testcase.prf new file mode 100644 index 0000000..db4f673 --- /dev/null +++ b/mkspecs/features/testcase.prf @@ -0,0 +1,51 @@ +!contains(TEMPLATE,subdirs) { + +check.files = +check.path = . + +# If the test ends up in a different directory, we should cd to that directory. +# Note that qmake modifies DESTDIR after this file is processed, +# therefore, testing DESTDIR for emptiness is not sufficient. +!isEmpty(DESTDIR):!contains(DESTDIR,^\./?): check.commands = cd $(DESTDIR) && +contains(TARGET,.*/.*): check.commands = cd $(DESTDIR) && + +# Allow for a custom test runner script +check.commands += $(TESTRUNNER) + +macx { + app_bundle: check.commands += ./$(QMAKE_TARGET).app/Contents/MacOS/$(QMAKE_TARGET) + else: check.commands += ./$(QMAKE_TARGET) +} +else:unix: check.commands += ./$(QMAKE_TARGET) +else: check.commands += $(DESTDIR_TARGET) + +# For Qt/Embedded, run every test app as a QWS server +embedded: check.commands += -qws + +# Allow for custom arguments to tests +check.commands += $(TESTARGS) +QMAKE_EXTRA_TARGETS *= check + +!debug_and_release|build_pass { + check.depends = first +} else { + check.CONFIG = recursive + # In debug and release mode, only run the test once. + # Run debug if available, release otherwise. + debug_and_release { + check.target = dummy_check + check.recurse_target = check + debug { + real_check.depends = debug-check + real_check.target = check + QMAKE_EXTRA_TARGETS += real_check + } else { + real_check.depends = release-check + real_check.target = check + QMAKE_EXTRA_TARGETS += real_check + } + } +} + +} + -- cgit v0.12 From 012a777ecec0ca74cf95683f069cb79774b7093a Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Mon, 31 May 2010 10:22:24 +1000 Subject: Fix `make check' for debug-and-release on Windows. --- mkspecs/features/testcase.prf | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/mkspecs/features/testcase.prf b/mkspecs/features/testcase.prf index db4f673..2a56d7d 100644 --- a/mkspecs/features/testcase.prf +++ b/mkspecs/features/testcase.prf @@ -6,8 +6,22 @@ check.path = . # If the test ends up in a different directory, we should cd to that directory. # Note that qmake modifies DESTDIR after this file is processed, # therefore, testing DESTDIR for emptiness is not sufficient. -!isEmpty(DESTDIR):!contains(DESTDIR,^\./?): check.commands = cd $(DESTDIR) && -contains(TARGET,.*/.*): check.commands = cd $(DESTDIR) && +# Also note that in debug-and-release mode we don't want to cd into the debug/release +# directory (e.g. if the test goes to foo/release/tst_thing.exe, we want to do +# cd foo && release/tst_thing.exe ). +MUNGED_DESTDIR=$$DESTDIR +MUNGED_TARGET=$$TARGET +win32:debug_and_release { + contains(DESTDIR,^release$)|contains(DESTDIR,^debug$):MUNGED_DESTDIR= + + # In debug-and-release mode, the first ../ in TARGET breaks out of the debug/release + # subdirectory. However, since make's working directory is already outside of the + # debug/release subdirectory, this first ../ should be ignored when deciding if + # we have to change directory before running the test. + MUNGED_TARGET=$$replace(MUNGED_TARGET,^\.\./,) +} +!isEmpty(MUNGED_DESTDIR):!contains(MUNGED_DESTDIR,^\./?):check.commands = cd $(DESTDIR) && +contains(MUNGED_TARGET,.*/.*):check.commands = cd $(DESTDIR) && # Allow for a custom test runner script check.commands += $(TESTRUNNER) -- cgit v0.12 From d7da9ec2a9e3752b1480007aa1fb213765aa8c90 Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Mon, 31 May 2010 11:12:02 +1000 Subject: Add a test for the `make check' feature. --- .../auto/maketestselftest/checktest/checktest.pro | 7 ++ tests/auto/maketestselftest/checktest/main.cpp | 88 ++++++++++++++++++++++ tests/auto/maketestselftest/maketestselftest.pro | 9 +-- tests/auto/maketestselftest/test/test.pro | 10 +++ .../auto/maketestselftest/tst_maketestselftest.cpp | 71 +++++++++++++++++ 5 files changed, 179 insertions(+), 6 deletions(-) create mode 100644 tests/auto/maketestselftest/checktest/checktest.pro create mode 100644 tests/auto/maketestselftest/checktest/main.cpp create mode 100644 tests/auto/maketestselftest/test/test.pro diff --git a/tests/auto/maketestselftest/checktest/checktest.pro b/tests/auto/maketestselftest/checktest/checktest.pro new file mode 100644 index 0000000..79c5ca5 --- /dev/null +++ b/tests/auto/maketestselftest/checktest/checktest.pro @@ -0,0 +1,7 @@ +TEMPLATE = app +TARGET = checktest +CONFIG += console +CONFIG -= app_bundle +DESTDIR = ./ +QT = core +SOURCES += main.cpp diff --git a/tests/auto/maketestselftest/checktest/main.cpp b/tests/auto/maketestselftest/checktest/main.cpp new file mode 100644 index 0000000..e775665 --- /dev/null +++ b/tests/auto/maketestselftest/checktest/main.cpp @@ -0,0 +1,88 @@ +/**************************************************************************** +** +** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies). +** All rights reserved. +** Contact: Nokia Corporation (qt-info@nokia.com) +** +** This file is part of the test suite of the Qt Toolkit. +** +** $QT_BEGIN_LICENSE:LGPL$ +** No Commercial Usage +** This file contains pre-release code and may not be distributed. +** You may use this file in accordance with the terms and conditions +** contained in the Technology Preview License Agreement accompanying +** this package. +** +** GNU Lesser General Public License Usage +** Alternatively, this file may be used under the terms of the GNU Lesser +** General Public License version 2.1 as published by the Free Software +** Foundation and appearing in the file LICENSE.LGPL included in the +** packaging of this file. Please review the following information to +** ensure the GNU Lesser General Public License version 2.1 requirements +** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html. +** +** In addition, as a special exception, Nokia gives you certain additional +** rights. These rights are described in the Nokia Qt LGPL Exception +** version 1.1, included in the file LGPL_EXCEPTION.txt in this package. +** +** If you have questions regarding the use of this file, please contact +** Nokia at qt-info@nokia.com. +** +** +** +** +** +** +** +** +** $QT_END_LICENSE$ +** +****************************************************************************/ + +#include +#include +#include +#include +#include + +#include +#include + +void fail(QString const& message) +{ + printf("CHECKTEST FAIL: %s\n", qPrintable(message)); + exit(0); +} + +void pass(QString const& message) +{ + printf("CHECKTEST PASS: %s\n", qPrintable(message)); + exit(0); +} + +int main(int argc, char** argv) +{ + QCoreApplication app(argc, argv); + + QStringList args = app.arguments(); + args.removeFirst(); // ourself + + QString args_quoted = QString("'%1'").arg(args.join("','")); + if (args.count() != 1) { + fail(QString("Expected exactly one argument, got: %1").arg(args_quoted)); + } + + QString test = args.at(0); + + QFileInfo testfile(test); + if (!testfile.exists()) { + fail(QString("File %1 does not exist (my working directory is: %2, my args are: %3)") + .arg(test) + .arg(QDir::currentPath()) + .arg(args_quoted) + ); + } + + pass(args_quoted); +} + diff --git a/tests/auto/maketestselftest/maketestselftest.pro b/tests/auto/maketestselftest/maketestselftest.pro index 6cc1744..a27d5d7 100644 --- a/tests/auto/maketestselftest/maketestselftest.pro +++ b/tests/auto/maketestselftest/maketestselftest.pro @@ -1,9 +1,6 @@ -load(qttest_p4) - -SOURCES += tst_maketestselftest.cpp -QT = core - -DEFINES += SRCDIR=\\\"$$PWD/\\\" +TEMPLATE = subdirs +SUBDIRS = checktest test +test.depends = checktest requires(!cross_compile) diff --git a/tests/auto/maketestselftest/test/test.pro b/tests/auto/maketestselftest/test/test.pro new file mode 100644 index 0000000..2fae25f --- /dev/null +++ b/tests/auto/maketestselftest/test/test.pro @@ -0,0 +1,10 @@ +load(qttest_p4) + +TARGET = ../tst_maketestselftest +SOURCES += ../tst_maketestselftest.cpp +QT = core + +DEFINES += SRCDIR=\\\"$$PWD/..\\\" + +requires(!cross_compile) + diff --git a/tests/auto/maketestselftest/tst_maketestselftest.cpp b/tests/auto/maketestselftest/tst_maketestselftest.cpp index 437e143..9f8e2c5 100644 --- a/tests/auto/maketestselftest/tst_maketestselftest.cpp +++ b/tests/auto/maketestselftest/tst_maketestselftest.cpp @@ -66,6 +66,8 @@ private slots: void naming_convention(); void naming_convention_data(); + void make_check(); + private: QStringList find_subdirs(QString const&, FindSubdirsMode, QString const& = QString()); @@ -446,6 +448,75 @@ QStringList tst_MakeTestSelfTest::find_subdirs(QString const& pro_file, FindSubd return out; } +void tst_MakeTestSelfTest::make_check() +{ + /* + Run `make check' over the whole tests tree with a custom TESTRUNNER, + to verify that the TESTRUNNER mechanism works right. + */ + QString testsDir(SRCDIR "/.."); + QString checktest(SRCDIR "/checktest/checktest"); + +#ifdef Q_OS_WIN32 + checktest.replace("/", "\\"); + checktest += ".exe"; +#endif + + QProcess make; + make.setWorkingDirectory(testsDir); + + QStringList arguments; + arguments << "-k"; + arguments << "check"; + arguments << QString("TESTRUNNER=%1").arg(checktest); + + // find the right make; from externaltests.cpp + static const char makes[] = + "nmake.exe\0" + "mingw32-make.exe\0" + "gmake\0" + "make\0" + ; + + bool ok = false; + for (const char *p = makes; *p; p += strlen(p) + 1) { + make.start(p, arguments); + if (make.waitForStarted()) { + ok = true; + break; + } + } + + if (!ok) { + QFAIL("Could not find the right make tool in PATH"); + } + + QVERIFY(make.waitForFinished(1000 * 60 * 10)); + QCOMPARE(make.exitStatus(), QProcess::NormalExit); + + int fail = 0; + int pass = 0; + QList out = make.readAllStandardOutput().split('\n'); + foreach (QByteArray line, out) { + while (line.endsWith("\r")) { + line.chop(1); + } + if (line.startsWith("CHECKTEST FAIL")) { + QWARN(line.constData()); + ++fail; + } + if (line.startsWith("CHECKTEST PASS")) { + ++pass; + } + } + + // We can't check that the exit code of make is 0, because some tests + // may have failed to compile, but that doesn't mean `make check' is broken. + // We do assume there are at least this many unbroken tests, though. + QCOMPARE(fail, 0); + QVERIFY(pass > 50); +} + QStringList find_test_class(QString const& filename) { QStringList out; -- cgit v0.12 From c0932de90e141d6f3eef3388c3caae22f3982798 Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Thu, 3 Jun 2010 13:44:07 +1000 Subject: Fixed tst_maketestselftest on QWS and Windows. --- tests/auto/maketestselftest/checktest/main.cpp | 13 ++++++++++++- tests/auto/maketestselftest/test/test.pro | 8 ++++++++ tests/auto/maketestselftest/tst_maketestselftest.cpp | 10 ++++++---- 3 files changed, 26 insertions(+), 5 deletions(-) diff --git a/tests/auto/maketestselftest/checktest/main.cpp b/tests/auto/maketestselftest/checktest/main.cpp index e775665..af98953 100644 --- a/tests/auto/maketestselftest/checktest/main.cpp +++ b/tests/auto/maketestselftest/checktest/main.cpp @@ -68,8 +68,19 @@ int main(int argc, char** argv) args.removeFirst(); // ourself QString args_quoted = QString("'%1'").arg(args.join("','")); + +#ifdef Q_WS_QWS + { + // for QWS we expect tests to be run as the QWS server + QString qws = args.takeLast(); + if (qws != "-qws") { + fail(QString("Expected test to be run with `-qws', but it wasn't; args: %1").arg(args_quoted)); + } + } +#endif + if (args.count() != 1) { - fail(QString("Expected exactly one argument, got: %1").arg(args_quoted)); + fail(QString("These arguments are not what I expected: %1").arg(args_quoted)); } QString test = args.at(0); diff --git a/tests/auto/maketestselftest/test/test.pro b/tests/auto/maketestselftest/test/test.pro index 2fae25f..d9de51e 100644 --- a/tests/auto/maketestselftest/test/test.pro +++ b/tests/auto/maketestselftest/test/test.pro @@ -8,3 +8,11 @@ DEFINES += SRCDIR=\\\"$$PWD/..\\\" requires(!cross_compile) +win32 { + CONFIG(debug, debug|release) { + TARGET = ../../debug/tst_maketestselftest +} else { + TARGET = ../../release/tst_maketestselftest + } +} + diff --git a/tests/auto/maketestselftest/tst_maketestselftest.cpp b/tests/auto/maketestselftest/tst_maketestselftest.cpp index 9f8e2c5..c3cbf76 100644 --- a/tests/auto/maketestselftest/tst_maketestselftest.cpp +++ b/tests/auto/maketestselftest/tst_maketestselftest.cpp @@ -494,16 +494,15 @@ void tst_MakeTestSelfTest::make_check() QVERIFY(make.waitForFinished(1000 * 60 * 10)); QCOMPARE(make.exitStatus(), QProcess::NormalExit); - int fail = 0; int pass = 0; QList out = make.readAllStandardOutput().split('\n'); + QStringList fails; foreach (QByteArray line, out) { while (line.endsWith("\r")) { line.chop(1); } if (line.startsWith("CHECKTEST FAIL")) { - QWARN(line.constData()); - ++fail; + fails << QString::fromLocal8Bit(line); } if (line.startsWith("CHECKTEST PASS")) { ++pass; @@ -513,7 +512,10 @@ void tst_MakeTestSelfTest::make_check() // We can't check that the exit code of make is 0, because some tests // may have failed to compile, but that doesn't mean `make check' is broken. // We do assume there are at least this many unbroken tests, though. - QCOMPARE(fail, 0); + QVERIFY2(fails.count() == 0, + qPrintable(QString("`make check' doesn't work for %1 tests:\n%2") + .arg(fails.count()).arg(fails.join("\n"))) + ); QVERIFY(pass > 50); } -- cgit v0.12 From 21bd54d5cf754ca5a5928cbd3c2a0ab17e64f633 Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Fri, 4 Jun 2010 11:58:56 +1000 Subject: Skip tst_maketestselftest::make_check by default on Windows nmake and checktest are too slow on Windows for this test to be enabled by default. Set RUN_SLOW_TESTS=1 to run the test manually. --- tests/auto/maketestselftest/tst_maketestselftest.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/auto/maketestselftest/tst_maketestselftest.cpp b/tests/auto/maketestselftest/tst_maketestselftest.cpp index c3cbf76..3d1bbed 100644 --- a/tests/auto/maketestselftest/tst_maketestselftest.cpp +++ b/tests/auto/maketestselftest/tst_maketestselftest.cpp @@ -458,6 +458,9 @@ void tst_MakeTestSelfTest::make_check() QString checktest(SRCDIR "/checktest/checktest"); #ifdef Q_OS_WIN32 + if (qgetenv("RUN_SLOW_TESTS").isEmpty()) { + QSKIP("This test is too slow to run by default on Windows. Set RUN_SLOW_TESTS=1 to run it.", SkipAll); + } checktest.replace("/", "\\"); checktest += ".exe"; #endif -- cgit v0.12 From c5beba29f1ac44ce225d73da67503f23f8aeffdc Mon Sep 17 00:00:00 2001 From: Rohan McGovern Date: Fri, 4 Jun 2010 17:08:37 +1000 Subject: Fixed unescaped backslashes in testcase.prf --- mkspecs/features/testcase.prf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mkspecs/features/testcase.prf b/mkspecs/features/testcase.prf index 2a56d7d..7a7c9e3 100644 --- a/mkspecs/features/testcase.prf +++ b/mkspecs/features/testcase.prf @@ -18,9 +18,9 @@ win32:debug_and_release { # subdirectory. However, since make's working directory is already outside of the # debug/release subdirectory, this first ../ should be ignored when deciding if # we have to change directory before running the test. - MUNGED_TARGET=$$replace(MUNGED_TARGET,^\.\./,) + MUNGED_TARGET=$$replace(MUNGED_TARGET,^\\.\\./,) } -!isEmpty(MUNGED_DESTDIR):!contains(MUNGED_DESTDIR,^\./?):check.commands = cd $(DESTDIR) && +!isEmpty(MUNGED_DESTDIR):!contains(MUNGED_DESTDIR,^\\./?):check.commands = cd $(DESTDIR) && contains(MUNGED_TARGET,.*/.*):check.commands = cd $(DESTDIR) && # Allow for a custom test runner script -- cgit v0.12