Make the lancelot/baseline test system generically usable

To facilitate usage outside of tst_lancelot:
- Added new QBaseLineTest simple API
- Added "tst_baselineexample" autotest to show usage
- Improved reporting
- Results from all testfunctions collected in same report

5 files changed, 216 insertions, 2 deletions

diff --git a/tests/arthur/common/baselineprotocol.cpp b/tests/arthur/common/baselineprotocol.cpp
index c7e9e72..e4445bd 100644
--- a/tests/arthur/common/baselineprotocol.cpp
+++ b/tests/arthur/common/baselineprotocol.cpp
@@ -48,6 +48,7 @@
 #include <QFileInfo>
 #include <QDir>
 #include <QTime>
+#include <QPointer>
 
 #ifndef QMAKESPEC
 #define QMAKESPEC "Unknown"
@@ -264,6 +265,10 @@ QDataStream & operator>> (QDataStream &stream, ImageItem &ii)
     return stream;
 }
 
+BaselineProtocol::BaselineProtocol()
+{
+}
+
 BaselineProtocol::~BaselineProtocol()
 {
     socket.close();
diff --git a/tests/arthur/common/baselineprotocol.h b/tests/arthur/common/baselineprotocol.h
index 1f4d593..2315bc3 100644
--- a/tests/arthur/common/baselineprotocol.h
+++ b/tests/arthur/common/baselineprotocol.h
@@ -38,6 +38,7 @@
 ** $QT_END_LICENSE$
 **
 ****************************************************************************/
+
 #ifndef BASELINEPROTOCOL_H
 #define BASELINEPROTOCOL_H
 
@@ -46,6 +47,7 @@
 #include <QImage>
 #include <QVector>
 #include <QMap>
+#include <QPointer>
 
 #define QLS QLatin1String
 #define QLC QLatin1Char
@@ -87,7 +89,8 @@ public:
     enum ItemStatus {
         Ok = 0,
         BaselineNotFound = 1,
-        IgnoreItem = 2
+        IgnoreItem = 2,
+        Mismatch = 3
     };
 
     QString testFunction;
@@ -107,12 +110,15 @@ Q_DECLARE_METATYPE(ImageItem);
 
 typedef QVector<ImageItem> ImageItemList;
 
+
 class BaselineProtocol
 {
 public:
-    BaselineProtocol() {}
+    BaselineProtocol();
     ~BaselineProtocol();
 
+    static BaselineProtocol *instance(QObject *parent = 0);
+
     // ****************************************************
     // Important constants here
     // ****************************************************
@@ -136,6 +142,8 @@ public:
     };
 
     // For client:
+
+    // For advanced client:
     bool connect(const QString &testCase, bool *dryrun = 0);
     bool requestBaselineChecksums(const QString &testFunction, ImageItemList *itemList);
     bool submitNewBaseline(const ImageItem &item, QByteArray *serverMsg);
diff --git a/tests/arthur/common/qbaselinetest.cpp b/tests/arthur/common/qbaselinetest.cpp
new file mode 100644
index 0000000..e95b510
--- /dev/null
+++ b/tests/arthur/common/qbaselinetest.cpp
@@ -0,0 +1,124 @@
+/****************************************************************************
+**
+** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#include "qbaselinetest.h"
+#include "baselineprotocol.h"
+
+namespace QBaselineTest {
+
+bool connected = false;
+bool triedConnecting = false;
+BaselineProtocol proto;
+
+
+bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArray *msg, bool *error)
+{
+    QByteArray itemName;
+    bool hasName = qstrlen(name);
+    const char *tag = QTest::currentDataTag();
+    if (qstrlen(tag)) {
+        itemName = tag;
+        if (hasName)
+            itemName.append('_').append(name);
+    } else {
+        itemName = hasName ? name : "default_name";
+    }
+
+    *msg = "Baseline check of image '" + itemName + "': ";
+
+    if (!triedConnecting) {
+        triedConnecting = true;
+        if (!proto.connect(QTest::testObject()->metaObject()->className())) {
+            *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1();
+            *error = true;
+            return true;
+        }
+        connected = true;
+    }
+    if (!connected) {
+        *msg = "Not connected to baseline server.";
+        *error = true;
+        return true;
+    }
+
+    ImageItem item;
+    item.itemName = QLatin1String(itemName);
+    item.itemChecksum = checksum;
+    item.testFunction = QLatin1String(QTest::currentTestFunction());
+    ImageItemList list;
+    list.append(item);
+    if (!proto.requestBaselineChecksums(QLatin1String(QTest::currentTestFunction()), &list) || list.isEmpty()) {
+        *msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1();
+        *error = true;
+        return true;
+    }
+    item.image = img;
+    item.imageChecksums.prepend(ImageItem::computeChecksum(img));
+    QByteArray srvMsg;
+    switch (list.at(0).status) {
+    case ImageItem::IgnoreItem :
+        qDebug() << msg->constData() << "Ignored, blacklisted on server.";
+        return true;
+        break;
+    case ImageItem::BaselineNotFound:
+        if (proto.submitNewBaseline(item, &srvMsg))
+            qDebug() << msg->constData() << "Baseline not found on server. New baseline uploaded.";
+        else
+            qDebug() << msg->constData() << "Baseline not found on server. Uploading of new baseline failed:" << srvMsg;
+        return true;
+        break;
+    case ImageItem::Ok:
+        break;
+    default:
+        qWarning() << "Unexpected reply from baseline server.";
+        return true;
+        break;
+    }
+    *error = false;
+    // The actual comparison of the given image with the baseline:
+    if (list.at(0).imageChecksums.contains(item.imageChecksums.at(0)))
+        return true;
+    proto.submitMismatch(item, &srvMsg);
+    *msg += "Mismatch. See report:\n   " + srvMsg;
+    return false;
+}
+
+}
diff --git a/tests/arthur/common/qbaselinetest.h b/tests/arthur/common/qbaselinetest.h
new file mode 100644
index 0000000..3445c72
--- /dev/null
+++ b/tests/arthur/common/qbaselinetest.h
@@ -0,0 +1,64 @@
+/****************************************************************************
+**
+** Copyright (C) 2010 Nokia Corporation and/or its subsidiary(-ies).
+** All rights reserved.
+** Contact: Nokia Corporation (qt-info@nokia.com)
+**
+** This file is part of the test suite of the Qt Toolkit.
+**
+** $QT_BEGIN_LICENSE:LGPL$
+** No Commercial Usage
+** This file contains pre-release code and may not be distributed.
+** You may use this file in accordance with the terms and conditions
+** contained in the Technology Preview License Agreement accompanying
+** this package.
+**
+** GNU Lesser General Public License Usage
+** Alternatively, this file may be used under the terms of the GNU Lesser
+** General Public License version 2.1 as published by the Free Software
+** Foundation and appearing in the file LICENSE.LGPL included in the
+** packaging of this file.  Please review the following information to
+** ensure the GNU Lesser General Public License version 2.1 requirements
+** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.
+**
+** In addition, as a special exception, Nokia gives you certain additional
+** rights.  These rights are described in the Nokia Qt LGPL Exception
+** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.
+**
+** If you have questions regarding the use of this file, please contact
+** Nokia at qt-info@nokia.com.
+**
+**
+**
+**
+**
+**
+**
+**
+** $QT_END_LICENSE$
+**
+****************************************************************************/
+
+#ifndef BASELINETEST_H
+#define BASELINETEST_H
+
+#include <QTest>
+
+namespace QBaselineTest {
+bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArray *msg, bool *error);
+}
+
+#define QBASELINE_CHECK_SUM(image, name, checksum)\
+do {\
+    QByteArray _msg;\
+    bool _err = false;\
+    if (!QBaselineTest::checkImage((image), (name), (checksum), &_msg, &_err)) {\
+        QFAIL(_msg.constData());\
+    } else if (_err) {\
+        QSKIP(_msg.constData(), SkipSingle);\
+    }\
+} while (0)
+
+#define QBASELINE_CHECK(image, name) QBASELINE_CHECK_SUM(image, name, 0)
+
+#endif // BASELINETEST_H
diff --git a/tests/arthur/common/qbaselinetest.pri b/tests/arthur/common/qbaselinetest.pri
new file mode 100644
index 0000000..5420c6e
--- /dev/null
+++ b/tests/arthur/common/qbaselinetest.pri
@@ -0,0 +1,13 @@
+QT *= testlib
+
+SOURCES += \
+        $$PWD/qbaselinetest.cpp
+
+HEADERS += \
+        $$PWD/qbaselinetest.h
+
+win32|symbian*:MKSPEC=$$replace(QMAKESPEC, \\\\, /)
+else:MKSPEC=$$QMAKESPEC
+DEFINES += QMAKESPEC=\\\"$$MKSPEC\\\"
+
+include($$PWD/baselineprotocol.pri)