Added api for efficient data driven baseline testing

3 files changed, 133 insertions, 63 deletions

diff --git a/tests/arthur/common/qbaselinetest.cpp b/tests/arthur/common/qbaselinetest.cpp
index e95b510..79241d5 100644
--- a/tests/arthur/common/qbaselinetest.cpp
+++ b/tests/arthur/common/qbaselinetest.cpp
@@ -44,56 +44,45 @@
 
 namespace QBaselineTest {
 
+BaselineProtocol proto;
 bool connected = false;
 bool triedConnecting = false;
-BaselineProtocol proto;
-
 
-bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArray *msg, bool *error)
-{
-    QByteArray itemName;
-    bool hasName = qstrlen(name);
-    const char *tag = QTest::currentDataTag();
-    if (qstrlen(tag)) {
-        itemName = tag;
-        if (hasName)
-            itemName.append('_').append(name);
-    } else {
-        itemName = hasName ? name : "default_name";
-    }
+QByteArray curFunction;
+ImageItemList itemList;
+bool gotBaselines;
 
-    *msg = "Baseline check of image '" + itemName + "': ";
 
+bool connect(QByteArray *msg, bool *error)
+{
     if (!triedConnecting) {
         triedConnecting = true;
         if (!proto.connect(QTest::testObject()->metaObject()->className())) {
             *msg += "Failed to connect to baseline server: " + proto.errorMessage().toLatin1();
             *error = true;
-            return true;
+            return false;
         }
         connected = true;
     }
     if (!connected) {
         *msg = "Not connected to baseline server.";
         *error = true;
-        return true;
+        return false;
     }
+    return true;
+}
 
-    ImageItem item;
-    item.itemName = QLatin1String(itemName);
-    item.itemChecksum = checksum;
-    item.testFunction = QLatin1String(QTest::currentTestFunction());
-    ImageItemList list;
-    list.append(item);
-    if (!proto.requestBaselineChecksums(QLatin1String(QTest::currentTestFunction()), &list) || list.isEmpty()) {
-        *msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1();
-        *error = true;
-        return true;
-    }
+
+bool compareItem(const ImageItem &baseline, const QImage &img, QByteArray *msg, bool *error)
+{
+    ImageItem item = baseline;
     item.image = img;
+    item.imageChecksums.clear();
     item.imageChecksums.prepend(ImageItem::computeChecksum(img));
     QByteArray srvMsg;
-    switch (list.at(0).status) {
+    switch (baseline.status) {
+    case ImageItem::Ok:
+        break;
     case ImageItem::IgnoreItem :
         qDebug() << msg->constData() << "Ignored, blacklisted on server.";
         return true;
@@ -105,8 +94,6 @@ bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArra
             qDebug() << msg->constData() << "Baseline not found on server. Uploading of new baseline failed:" << srvMsg;
         return true;
         break;
-    case ImageItem::Ok:
-        break;
     default:
         qWarning() << "Unexpected reply from baseline server.";
         return true;
@@ -114,11 +101,93 @@ bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArra
     }
     *error = false;
     // The actual comparison of the given image with the baseline:
-    if (list.at(0).imageChecksums.contains(item.imageChecksums.at(0)))
+    if (baseline.imageChecksums.contains(item.imageChecksums.at(0)))
         return true;
     proto.submitMismatch(item, &srvMsg);
     *msg += "Mismatch. See report:\n   " + srvMsg;
     return false;
 }
 
+bool checkImage(const QImage &img, const char *name, quint16 checksum, QByteArray *msg, bool *error)
+{
+    if (!connected && !connect(msg, error))
+        return true;
+
+    QByteArray itemName;
+    bool hasName = qstrlen(name);
+    const char *tag = QTest::currentDataTag();
+    if (qstrlen(tag)) {
+        itemName = tag;
+        if (hasName)
+            itemName.append('_').append(name);
+    } else {
+        itemName = hasName ? name : "default_name";
+    }
+
+    *msg = "Baseline check of image '" + itemName + "': ";
+
+
+    ImageItem item;
+    item.itemName = QString::fromLatin1(itemName);
+    item.itemChecksum = checksum;
+    item.testFunction = QString::fromLatin1(QTest::currentTestFunction());
+    ImageItemList list;
+    list.append(item);
+    if (!proto.requestBaselineChecksums(QLatin1String(QTest::currentTestFunction()), &list) || list.isEmpty()) {
+        *msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1();
+        *error = true;
+        return true;
+    }
+
+    return compareItem(list.at(0), img, msg, error);
+}
+
+
+QTestData &newRow(const char *dataTag, quint16 checksum)
+{
+    if (QTest::currentTestFunction() != curFunction) {
+        curFunction = QTest::currentTestFunction();
+        itemList.clear();
+        gotBaselines = false;
+    }
+    ImageItem item;
+    item.itemName = QString::fromLatin1(dataTag);
+    item.itemChecksum = checksum;
+    item.testFunction = QString::fromLatin1(QTest::currentTestFunction());
+    itemList.append(item);
+
+    return QTest::newRow(dataTag);
+}
+
+
+bool testImage(const QImage& img, QByteArray *msg, bool *error)
+{
+    if (!connected && !connect(msg, error))
+        return true;
+
+    if (QTest::currentTestFunction() != curFunction || itemList.isEmpty()) {
+        qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow()";
+        return true;
+    }
+
+    if (!gotBaselines) {
+        if (!proto.requestBaselineChecksums(QString::fromLatin1(QTest::currentTestFunction()), &itemList) || itemList.isEmpty()) {
+            *msg = "Communication with baseline server failed: " + proto.errorMessage().toLatin1();
+            *error = true;
+            return true;
+        }
+        gotBaselines = true;
+    }
+
+    QString curTag = QString::fromLatin1(QTest::currentDataTag());
+    ImageItemList::const_iterator it = itemList.constBegin();
+    while (it != itemList.constEnd() && it->itemName != curTag)
+        ++it;
+    if (it == itemList.constEnd()) {
+        qWarning() << "Usage error: QBASELINE_TEST used without corresponding QBaselineTest::newRow() for row" << curTag;
+        return true;
+    }
+    return compareItem(*it, img, msg, error);
+}
+
 }
diff --git a/tests/arthur/common/qbaselinetest.h b/tests/arthur/common/qbaselinetest.h
index 3445c72..e27cda2 100644
--- a/tests/arthur/common/qbaselinetest.h
+++ b/tests/arthur/common/qbaselinetest.h
@@ -46,6 +46,8 @@
 
 namespace QBaselineTest {
 bool checkImage(const QImage& img, const char *name, quint16 checksum, QByteArray *msg, bool *error);
+bool testImage(const QImage& img, QByteArray *msg, bool *error);
+QTestData &newRow(const char *dataTag, quint16 checksum = 0);
 }
 
 #define QBASELINE_CHECK_SUM(image, name, checksum)\
@@ -61,4 +63,15 @@ do {\
 
 #define QBASELINE_CHECK(image, name) QBASELINE_CHECK_SUM(image, name, 0)
 
+#define QBASELINE_TEST(image)\
+do {\
+    QByteArray _msg;\
+    bool _err = false;\
+    if (!QBaselineTest::testImage((image), &_msg, &_err)) {\
+        QFAIL(_msg.constData());\
+    } else if (_err) {\
+        QSKIP(_msg.constData(), SkipSingle);\
+    }\
+} while (0)
+
 #endif // BASELINETEST_H
diff --git a/tests/auto/baselineexample/tst_baselineexample.cpp b/tests/auto/baselineexample/tst_baselineexample.cpp
index 28cbec5..b97cc63 100644
--- a/tests/auto/baselineexample/tst_baselineexample.cpp
+++ b/tests/auto/baselineexample/tst_baselineexample.cpp
@@ -54,10 +54,8 @@ private Q_SLOTS:
     void testMultipleImages();
     void testDataDriven_data();
     void testDataDriven();
-    void testDataDrivenMultiple_data();
-    void testDataDrivenMultiple();
-    void testChecksum_data();
-    void testChecksum();
+    void testDataDrivenChecksum_data();
+    void testDataDrivenChecksum();
 };
 
 
@@ -98,9 +96,11 @@ void tst_BaselineExample::testMultipleImages()
 void tst_BaselineExample::testDataDriven_data()
 {
     QTest::addColumn<QString>("label");
-    QTest::newRow("short") << "Ok!";
-    QTest::newRow("long") << "A really long button text that just does not seem to end";
-    QTest::newRow("empty") << "";
+    QBaselineTest::newRow("short") << "Ok!";
+    QBaselineTest::newRow("long") << "A really long button text that just does not seem to end";
+    QBaselineTest::newRow("empty") << "";
+    QBaselineTest::newRow("signs") << "!@#$%^&*()_";
+    QBaselineTest::newRow("html") << "<b>BOLD</b>";
 }
 
 
@@ -111,45 +111,33 @@ void tst_BaselineExample::testDataDriven()
     b.resize(100, 50);
     b.show();
     QTest::qWaitForWindowShown(&b);
-    QBASELINE_CHECK(QPixmap::grabWidget(&b).toImage(), 0);
+    QBASELINE_TEST(QPixmap::grabWidget(&b).toImage());
 }
 
 
-void tst_BaselineExample::testDataDrivenMultiple_data()
+void tst_BaselineExample::testDataDrivenChecksum_data()
 {
-    testDataDriven_data();
-}
-
-
-void tst_BaselineExample::testDataDrivenMultiple()
-{
-    QFETCH(QString, label);
-    QPushButton b(label);
-    b.resize(100, 50);
-    b.show();
-    QTest::qWaitForWindowShown(&b);
-    QBASELINE_CHECK(QPixmap::grabWidget(&b).toImage(), "normal");
-
-    b.setText(label.prepend('&'));
-    QTest::qWait(50);
-    QBASELINE_CHECK(QPixmap::grabWidget(&b).toImage(), "shortcut");
-}
+    QTest::addColumn<QString>("label");
 
+    const int numItems = 5;
+    const char *tags[numItems] = {"short", "long", "empty", "signs", "html"};
+    const char *labels[numItems] = {"Ok!", "A really long button text that just does not seem to end", "", "!@#$%^&*()_", "<b>BOLD</b>"};
 
-void tst_BaselineExample::testChecksum_data()
-{
-    testDataDriven_data();
+    for (int i = 0; i<numItems; i++) {
+        quint16 checksum = qChecksum(labels[i], qstrlen(labels[i]));
+        QBaselineTest::newRow(tags[i], checksum) << labels[i];
+    }
 }
 
 
-void tst_BaselineExample::testChecksum()
+void tst_BaselineExample::testDataDrivenChecksum()
 {
     QFETCH(QString, label);
     QPushButton b(label);
     b.resize(100, 50);
     b.show();
     QTest::qWaitForWindowShown(&b);
-    QBASELINE_CHECK_SUM(QPixmap::grabWidget(&b).toImage(), 0, quint16(qHash(label)));
+    QBASELINE_TEST(QPixmap::grabWidget(&b).toImage());
 }