Add a test for the `make check' feature.

5 files changed, 179 insertions, 6 deletions

diff --git a/tests/auto/maketestselftest/checktest/checktest.pro b/tests/auto/maketestselftest/checktest/checktest.pro
new file mode 100644
index 0000000..79c5ca5
--- /dev/null
+++ b/tests/auto/maketestselftest/checktest/checktest.pro
@@ -0,0 +1,7 @@
+TEMPLATE = app
+TARGET = checktest
+CONFIG += console
+CONFIG -= app_bundle
+DESTDIR = ./
+QT = core
+SOURCES += main.cpp
diff --git a/tests/auto/maketestselftest/checktest/main.cpp b/tests/auto/maketestselftest/checktest/main.cpp
new file mode 100644
index 0000000..e775665
--- /dev/null
+++ b/tests/auto/maketestselftest/checktest/main.cpp
@@ -0,0 +1,88 @@
+/****************************************************************************
+**
+** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include <QCoreApplication>
+#include <QDir>
+#include <QFile>
+#include <QFileInfo>
+#include <QStringList>
+
+#include <stdio.h>
+#include <stdlib.h>
+
+void fail(QString const& message)
+{
+    printf("CHECKTEST FAIL: %s\n", qPrintable(message));
+    exit(0);
+}
+
+void pass(QString const& message)
+{
+    printf("CHECKTEST PASS: %s\n", qPrintable(message));
+    exit(0);
+}
+
+int main(int argc, char** argv)
+{
+    QCoreApplication app(argc, argv);
+
+    QStringList args = app.arguments();
+    args.removeFirst(); // ourself
+
+    QString args_quoted = QString("'%1'").arg(args.join("','"));
+    if (args.count() != 1) {
+        fail(QString("Expected exactly one argument, got: %1").arg(args_quoted));
+    }
+
+    QString test = args.at(0);
+
+    QFileInfo testfile(test);
+    if (!testfile.exists()) {
+        fail(QString("File %1 does not exist (my working directory is: %2, my args are: %3)")
+            .arg(test)
+            .arg(QDir::currentPath())
+            .arg(args_quoted)
+        );
+    }
+
+    pass(args_quoted);
+}
+
diff --git a/tests/auto/maketestselftest/maketestselftest.pro b/tests/auto/maketestselftest/maketestselftest.pro
index 6cc1744..a27d5d7 100644
--- a/tests/auto/maketestselftest/maketestselftest.pro
+++ b/tests/auto/maketestselftest/maketestselftest.pro
@@ -1,9 +1,6 @@
-load(qttest_p4)
-
-SOURCES += tst_maketestselftest.cpp
-QT = core
-
-DEFINES += SRCDIR=\\\"$$PWD/\\\"
+TEMPLATE = subdirs
+SUBDIRS = checktest test
+test.depends = checktest
 
 requires(!cross_compile)
 
diff --git a/tests/auto/maketestselftest/test/test.pro b/tests/auto/maketestselftest/test/test.pro
new file mode 100644
index 0000000..2fae25f
--- /dev/null
+++ b/tests/auto/maketestselftest/test/test.pro
@@ -0,0 +1,10 @@
+load(qttest_p4)
+
+TARGET = ../tst_maketestselftest
+SOURCES += ../tst_maketestselftest.cpp
+QT = core
+
+DEFINES += SRCDIR=\\\"$$PWD/..\\\"
+
+requires(!cross_compile)
+
diff --git a/tests/auto/maketestselftest/tst_maketestselftest.cpp b/tests/auto/maketestselftest/tst_maketestselftest.cpp
index 437e143..9f8e2c5 100644
--- a/tests/auto/maketestselftest/tst_maketestselftest.cpp
+++ b/tests/auto/maketestselftest/tst_maketestselftest.cpp
@@ -66,6 +66,8 @@ private slots:
     void naming_convention();
     void naming_convention_data();
 
+    void make_check();
+
 private:
     QStringList find_subdirs(QString const&, FindSubdirsMode, QString const& = QString());
 
@@ -446,6 +448,75 @@ QStringList tst_MakeTestSelfTest::find_subdirs(QString const& pro_file, FindSubd
     return out;
 }
 
+void tst_MakeTestSelfTest::make_check()
+{
+    /*
+        Run `make check' over the whole tests tree with a custom TESTRUNNER,
+        to verify that the TESTRUNNER mechanism works right.
+    */
+    QString testsDir(SRCDIR "/..");
+    QString checktest(SRCDIR "/checktest/checktest");
+
+#ifdef Q_OS_WIN32
+    checktest.replace("/", "\\");
+    checktest += ".exe";
+#endif
+
+    QProcess make;
+    make.setWorkingDirectory(testsDir);
+
+    QStringList arguments;
+    arguments << "-k";
+    arguments << "check";
+    arguments << QString("TESTRUNNER=%1").arg(checktest);
+
+    // find the right make; from externaltests.cpp
+    static const char makes[] =
+        "nmake.exe\0"
+        "mingw32-make.exe\0"
+        "gmake\0"
+        "make\0"
+    ;
+
+    bool ok = false;
+    for (const char *p = makes; *p; p += strlen(p) + 1) {
+        make.start(p, arguments);
+        if (make.waitForStarted()) {
+            ok = true;
+            break;
+        }
+    }
+
+    if (!ok) {
+        QFAIL("Could not find the right make tool in PATH");
+    }
+
+    QVERIFY(make.waitForFinished(1000 * 60 * 10));
+    QCOMPARE(make.exitStatus(), QProcess::NormalExit);
+
+    int fail = 0;
+    int pass = 0;
+    QList<QByteArray> out = make.readAllStandardOutput().split('\n');
+    foreach (QByteArray line, out) {
+        while (line.endsWith("\r")) {
+            line.chop(1);
+        }
+        if (line.startsWith("CHECKTEST FAIL")) {
+            QWARN(line.constData());
+            ++fail;
+        }
+        if (line.startsWith("CHECKTEST PASS")) {
+            ++pass;
+        }
+    }
+
+    // We can't check that the exit code of make is 0, because some tests
+    // may have failed to compile, but that doesn't mean `make check' is broken.
+    // We do assume there are at least this many unbroken tests, though.
+    QCOMPARE(fail, 0);
+    QVERIFY(pass > 50);
+}
+
 QStringList find_test_class(QString const& filename)
 {
     QStringList out;