Add support for blacklisting/whitelisting tests on the report page.

Also changed how the GL tests are run: they now render into a
multisampled FBO, which seems to be more stable than to a plain
GL widget.

4 files changed, 145 insertions, 27 deletions

diff --git a/tests/arthur/baselineserver/src/baselineserver.cpp b/tests/arthur/baselineserver/src/baselineserver.cpp
index 41e48eb..072e216 100644
--- a/tests/arthur/baselineserver/src/baselineserver.cpp
+++ b/tests/arthur/baselineserver/src/baselineserver.cpp
@@ -143,11 +143,6 @@ void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock)
             << itemList.at(0).engineAsString() << "pixel format" << itemList.at(0).formatAsString();
 
     for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) {
-        if (i->scriptName.startsWith(QLS("porter_duff"))) {
-            // Example of blacklisting on server.
-            i->status = ImageItem::IgnoreItem;
-            continue;
-        }
         i->imageChecksums.clear();
         QString prefix = pathForItem(*i, true);
         QFile file(prefix + QLS("metadata"));
@@ -161,11 +156,29 @@ void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock)
             i->status = ImageItem::BaselineNotFound;
     }
 
+    // Find and mark blacklisted items
+    if (itemList.count() > 0) {
+        QString prefix = pathForItem(itemList.at(0), true).section(QLC('/'), 0, -2);
+        QFile file(prefix + QLS("/.blacklist"));
+        if (file.open(QIODevice::ReadOnly)) {
+            QTextStream in(&file);
+            do {
+                QString scriptName = in.readLine();
+                if (!scriptName.isNull()) {
+                    for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) {
+                        if (i->scriptName == scriptName)
+                            i->status = ImageItem::IgnoreItem;
+                    }
+                }
+            } while (!in.atEnd());
+        }
+    }
+
     QByteArray block;
     QDataStream ods(&block, QIODevice::WriteOnly);
     ods << itemList;
     proto.sendBlock(BaselineProtocol::Ack, block);
-    report.start(BaselineServer::storagePath(), runId, plat);
+    report.start(BaselineServer::storagePath(), runId, plat, itemList);
 }
 
 
@@ -216,6 +229,14 @@ void BaselineHandler::receiveDisconnect()
 }
 
 
+QString BaselineHandler::itemSubPath(const QString &engine, const QString &format, bool isBaseline)
+{
+    if (isBaseline)
+        return QString(QLS("baselines_%1_%2/")).arg(engine, format);
+    else
+        return QString(QLS("mismatches_%1_%2/")).arg(engine, format);
+}
+
 QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, bool absolute)
 {
     if (pathForRun.isNull()) {
@@ -233,9 +254,9 @@ QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, boo
 
     QString storePath = pathForRun;
     if (isBaseline)
-        storePath += QString(QLS("baselines_%1_%2/")).arg(item.engineAsString(), item.formatAsString());
+        storePath += itemSubPath(item.engineAsString(), item.formatAsString(), isBaseline);
     else
-        storePath += QString(QLS("mismatches_%1_%2/")).arg(item.engineAsString(), item.formatAsString()) + runId + QLC('/');
+        storePath += itemSubPath(item.engineAsString(), item.formatAsString(), isBaseline) + runId + QLC('/');
 
     QString itemName = item.scriptName;
     if (itemName.contains(QLC('.')))
@@ -253,11 +274,8 @@ QString BaselineHandler::updateAllBaselines(const QString &host, const QString &
                                             const QString &engine, const QString &format)
 {
     QString basePath(BaselineServer::storagePath());
-    QString srcDir(basePath + host + QLC('/')
-                   + QString(QLS("mismatches_%1_%2/")).arg(engine, format)
-                   + id);
-    QString dstDir(basePath + host + QLC('/')
-                   + QString(QLS("baselines_%1_%2/")).arg(engine, format));
+    QString srcDir(basePath + host + QLC('/') + itemSubPath(engine, format, false) + id);
+    QString dstDir(basePath + host + QLC('/') + itemSubPath(engine, format));
 
     QDir dir(srcDir);
     QStringList nameFilter;
@@ -307,6 +325,54 @@ QString BaselineHandler::updateSingleBaseline(const QString &oldBaseline, const
     return res;
 }
 
+QString BaselineHandler::blacklistTest(const QString &scriptName, const QString &host, const QString &engine,
+                                       const QString &format)
+{
+    QString configFile(BaselineServer::storagePath() + host + QLC('/')
+                       + itemSubPath(engine, format) + QLS(".blacklist"));
+    QFile file(configFile);
+    if (file.open(QIODevice::Append)) {
+        QTextStream out(&file);
+        out << scriptName << endl;
+        return QLS("Blacklisted ") + scriptName;
+    } else {
+        return QLS("Unable to update blacklisted tests.");
+    }
+}
+
+QString BaselineHandler::whitelistTest(const QString &scriptName, const QString &host, const QString &engine,
+                                       const QString &format)
+{
+    QString configFile(BaselineServer::storagePath() + host + QLC('/')
+                       + itemSubPath(engine, format) + QLS(".blacklist"));
+    QFile file(configFile);
+    QStringList tests;
+    if (file.open(QIODevice::ReadOnly)) {
+        QTextStream in(&file);
+        do {
+            tests << in.readLine();
+        } while (!in.atEnd());
+        if (tests.count() != 0) {
+            QMutableStringListIterator it(tests);
+            while (it.hasNext()) {
+                it.next();
+                if (it.value() == scriptName)
+                    it.remove();
+            }
+        }
+        file.close();
+        if (file.open(QIODevice::WriteOnly | QIODevice::Truncate)) {
+            QTextStream out(&file);
+            for (int i=0; i<tests.count(); ++i)
+                out << tests.at(i);
+            return QLS("Whitelisted ") + scriptName;
+        } else {
+            QLS("Unable to whitelist ") + scriptName + QLS(". Unable to truncate blacklist file.");
+        }
+    }
+    return QLS("Unable to whitelist ") + scriptName + QLS(". Unable to open blacklist file.");
+}
+
 void BaselineHandler::testPathMapping()
 {
     qDebug() << "Storage prefix:" << BaselineServer::storagePath();
diff --git a/tests/arthur/baselineserver/src/baselineserver.h b/tests/arthur/baselineserver/src/baselineserver.h
index b77bc09..ba71ce5 100644
--- a/tests/arthur/baselineserver/src/baselineserver.h
+++ b/tests/arthur/baselineserver/src/baselineserver.h
@@ -63,6 +63,10 @@ public:
     static QString updateAllBaselines(const QString &host, const QString &id,
                                       const QString &engine, const QString &format);
     static QString updateSingleBaseline(const QString &oldBaseline, const QString &newBaseline);
+    static QString blacklistTest(const QString &scriptName, const QString &host,
+                                 const QString &engine, const QString &format);
+    static QString whitelistTest(const QString &scriptName, const QString &host,
+                                 const QString &engine, const QString &format);
 
 private slots:
     void receiveRequest();
@@ -76,6 +80,8 @@ private:
     QString computeMismatchScore(const QImage& baseline, const QImage& rendered);
     QString engineForItem(const ImageItem &item);
 
+    static QString itemSubPath(const QString &engine, const QString &format, bool isBaseline = true);
+
     BaselineProtocol proto;
     PlatformInfo plat;
     bool connectionEstablished;
diff --git a/tests/arthur/baselineserver/src/htmlpage.cpp b/tests/arthur/baselineserver/src/htmlpage.cpp
index 4f53c67..7759c8e 100644
--- a/tests/arthur/baselineserver/src/htmlpage.cpp
+++ b/tests/arthur/baselineserver/src/htmlpage.cpp
@@ -20,13 +20,14 @@ QString HTMLPage::filePath()
     return path;
 }
 
-void HTMLPage::start(const QString &storagepath, const QString &runId, const PlatformInfo pinfo)
+void HTMLPage::start(const QString &storagepath, const QString &runId, const PlatformInfo pinfo, const ImageItemList &itemList)
 {
     end();
 
     id = runId;
     plat = pinfo;
     root = storagepath;
+    imageItems = itemList;
     QString dir = root + QLS("reports/");
     QDir cwd;
     if (!cwd.exists(dir))
@@ -52,20 +53,18 @@ void HTMLPage::writeHeader(const ImageItem &item)
     out << "<h3>Build key: " << plat.buildKey << "</h3>\n";
     out << "<h3>Engine: " << item.engineAsString() << "</h3>\n";
     out << "<h3>Format: " << item.formatAsString() << "</h3>\n\n";
-    out << "<h3><a href=\"/cgi-bin/server.cgi?update=all&id="<< id << "&host=" << plat.hostname
+    out << "<h3><a href=\"/cgi-bin/server.cgi?cmd=updateAllBaselines&id="<< id << "&host=" << plat.hostname
         << "&engine=" << item.engineAsString() << "&format=" << item.formatAsString()
         << "&url=" << pageUrl
-        << "\">Update all baselines</a></h3>\n\n";
+        << "\">Update all baselines</a><br>";
     out << "<table border=\"2\">\n"
            "<tr>\n"
            "<td><b>Script</b></td>\n"
            "<td><b>Baseline</b></td>\n"
            "<td><b>Rendered</b></td>\n"
            "<td><b>Comparison</b></td>\n"
-           "<td><b>Fuzzy Comparison</b></td>\n"
-           "<td><b>Score</b></td>\n"
-           "<td><b>Update</b></td>\n"
-           "</b></tr>\n\n";
+           "<td><b>Info/Action</b></td>\n"
+           "</b></tr><br>";
 }
 
 
@@ -82,23 +81,53 @@ void HTMLPage::addItem(const QString &baseline, const QString &rendered, const I
         headerWritten = true;
     }
     QString compared = generateCompared(baseline, rendered);
-    QString fuzzy = generateCompared(baseline, rendered, true);
     QString pageUrl = BaselineServer::baseUrl() + path;
 
     out << "<tr>\n";
     out << "<td>" << item.scriptName << "</td>\n";
-    QStringList images = QStringList() << baseline << rendered << compared << fuzzy;
+    QStringList images = QStringList() << baseline << rendered << compared;
     foreach(const QString& img, images)
         out << "<td><a href=\"/" << img << "\"><img src=\"/" << img << "\" width=240 height=240></a></td>\n";
-    out << "<td></td><td><a href=\"/cgi-bin/server.cgi?update=single&oldBaseline=" << baseline
-        << "&newBaseline=" << rendered << "&url=" << pageUrl << "\">Update baseline</a></td>\n";
+    out << "<td><a href=\"/cgi-bin/server.cgi?cmd=updateSingleBaseline&oldBaseline=" << baseline
+        << "&newBaseline=" << rendered << "&url=" << pageUrl << "\">Update baseline</a><br>"
+           "<a href=\"/cgi-bin/server.cgi?cmd=blacklist&scriptName=" << item.scriptName
+           << "&host=" << plat.hostname << "&engine=" << item.engineAsString()
+           << "&format=" << item.formatAsString()
+           << "&url=" << pageUrl << "\">Blacklist test</a></td>\n";
     out << "<tr>\n\n";
+
+    QMutableVectorIterator<ImageItem> it(imageItems);
+    while (it.hasNext()) {
+        it.next();
+        if (it.value().scriptName == item.scriptName) {
+            it.remove();
+            break;
+        }
+    }
 }
 
 
 void HTMLPage::end()
 {
     if (file.isOpen()) {
+        // Add the names of the scripts that passed the test, or were blacklisted
+        QString pageUrl = BaselineServer::baseUrl() + path;
+        for (int i=0; i<imageItems.count(); ++i) {
+            out << "<tr><td>" << imageItems.at(i).scriptName << "</td><td>N/A</td><td>N/A</td><td>N/A</td><td>";
+            if (imageItems.at(i).status == ImageItem::IgnoreItem) {
+                out << "<span style=\"background-color:yellow\">Blacklisted</span><br>"
+                       "<a href=\"/cgi-bin/server.cgi?cmd=whitelist&scriptName="
+                    << imageItems.at(i).scriptName << "&host=" << plat.hostname
+                    << "&engine=" << imageItems.at(i).engineAsString()
+                    << "&format=" << imageItems.at(i).formatAsString()
+                    << "&url=" << pageUrl
+                    << "\">Whitelist test</a>";
+            } else {
+                out << "<span style=\"color:green\">Test passed</span>";
+            }
+            out << "</td><tr>";
+        }
+
         writeFooter();
         out.flush();
         file.close();
@@ -132,14 +161,30 @@ void HTMLPage::handleCGIQuery(const QString &query)
 //      << "Contents of QUERY_STRING:<br>"
 //      << "Full string = " << query << "<br>";
 
-    if (cgiUrl.queryItemValue(QLS("update")) == QLS("single")) {
+    QString command(cgiUrl.queryItemValue("cmd"));
+
+    if (command == QLS("updateSingleBaseline")) {
         s << BaselineHandler::updateSingleBaseline(cgiUrl.queryItemValue(QLS("oldBaseline")),
                                                    cgiUrl.queryItemValue(QLS("newBaseline")));
-    } else {
+    } else if (command == QLS("updateAllBaselines")) {
         s << BaselineHandler::updateAllBaselines(cgiUrl.queryItemValue(QLS("host")),
                                                  cgiUrl.queryItemValue(QLS("id")),
                                                  cgiUrl.queryItemValue(QLS("engine")),
                                                  cgiUrl.queryItemValue(QLS("format")));
+    } else if (command == QLS("blacklist")) {
+        // blacklist a test
+        s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("scriptName")),
+                                                  cgiUrl.queryItemValue(QLS("host")),
+                                                  cgiUrl.queryItemValue(QLS("engine")),
+                                                  cgiUrl.queryItemValue(QLS("format")));
+    } else if (command == QLS("whitelist")) {
+        // whitelist a test
+        s << BaselineHandler::whitelistTest(cgiUrl.queryItemValue(QLS("scriptName")),
+                                                    cgiUrl.queryItemValue(QLS("host")),
+                                                    cgiUrl.queryItemValue(QLS("engine")),
+                                                    cgiUrl.queryItemValue(QLS("format")));
+    } else {
+        s << "Unknown query:<br>" << query << "<br>";
     }
     s << "<p><a href=\"" << cgiUrl.queryItemValue(QLS("url")) << "\">Back to report</a>";
     s << "</HTML>";
diff --git a/tests/arthur/baselineserver/src/htmlpage.h b/tests/arthur/baselineserver/src/htmlpage.h
index 29b4ff5..a937052 100644
--- a/tests/arthur/baselineserver/src/htmlpage.h
+++ b/tests/arthur/baselineserver/src/htmlpage.h
@@ -11,7 +11,7 @@ public:
     HTMLPage();
     ~HTMLPage();
 
-    void start(const QString &storagePath, const QString &runId, const PlatformInfo pinfo);
+    void start(const QString &storagePath, const QString &runId, const PlatformInfo pinfo, const ImageItemList &itemList);
     void addItem(const QString &baseline, const QString &rendered, const ImageItem &item);
     void end();
     QString filePath();
@@ -29,6 +29,7 @@ private:
     QTextStream out;
     QString id;
     PlatformInfo plat;
+    ImageItemList imageItems;
     bool headerWritten;
 };