From 52111401594472503ce21096fc720807a9253b8d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Trond=20Kjern=C3=A5sen?= Date: Wed, 6 Oct 2010 15:04:31 +0200 Subject: Add support for blacklisting/whitelisting tests on the report page. Also changed how the GL tests are run: they now render into a multisampled FBO, which seems to be more stable than to a plain GL widget. --- tests/arthur/baselineserver/src/baselineserver.cpp | 92 +++++++++++++++++++--- tests/arthur/baselineserver/src/baselineserver.h | 6 ++ tests/arthur/baselineserver/src/htmlpage.cpp | 71 ++++++++++++++--- tests/arthur/baselineserver/src/htmlpage.h | 3 +- tests/auto/lancelot/tst_lancelot.cpp | 20 ++--- 5 files changed, 156 insertions(+), 36 deletions(-) diff --git a/tests/arthur/baselineserver/src/baselineserver.cpp b/tests/arthur/baselineserver/src/baselineserver.cpp index 41e48eb..072e216 100644 --- a/tests/arthur/baselineserver/src/baselineserver.cpp +++ b/tests/arthur/baselineserver/src/baselineserver.cpp @@ -143,11 +143,6 @@ void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock) << itemList.at(0).engineAsString() << "pixel format" << itemList.at(0).formatAsString(); for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) { - if (i->scriptName.startsWith(QLS("porter_duff"))) { - // Example of blacklisting on server. - i->status = ImageItem::IgnoreItem; - continue; - } i->imageChecksums.clear(); QString prefix = pathForItem(*i, true); QFile file(prefix + QLS("metadata")); @@ -161,11 +156,29 @@ void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock) i->status = ImageItem::BaselineNotFound; } + // Find and mark blacklisted items + if (itemList.count() > 0) { + QString prefix = pathForItem(itemList.at(0), true).section(QLC('/'), 0, -2); + QFile file(prefix + QLS("/.blacklist")); + if (file.open(QIODevice::ReadOnly)) { + QTextStream in(&file); + do { + QString scriptName = in.readLine(); + if (!scriptName.isNull()) { + for (ImageItemList::iterator i = itemList.begin(); i != itemList.end(); ++i) { + if (i->scriptName == scriptName) + i->status = ImageItem::IgnoreItem; + } + } + } while (!in.atEnd()); + } + } + QByteArray block; QDataStream ods(&block, QIODevice::WriteOnly); ods << itemList; proto.sendBlock(BaselineProtocol::Ack, block); - report.start(BaselineServer::storagePath(), runId, plat); + report.start(BaselineServer::storagePath(), runId, plat, itemList); } @@ -216,6 +229,14 @@ void BaselineHandler::receiveDisconnect() } +QString BaselineHandler::itemSubPath(const QString &engine, const QString &format, bool isBaseline) +{ + if (isBaseline) + return QString(QLS("baselines_%1_%2/")).arg(engine, format); + else + return QString(QLS("mismatches_%1_%2/")).arg(engine, format); +} + QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, bool absolute) { if (pathForRun.isNull()) { @@ -233,9 +254,9 @@ QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, boo QString storePath = pathForRun; if (isBaseline) - storePath += QString(QLS("baselines_%1_%2/")).arg(item.engineAsString(), item.formatAsString()); + storePath += itemSubPath(item.engineAsString(), item.formatAsString(), isBaseline); else - storePath += QString(QLS("mismatches_%1_%2/")).arg(item.engineAsString(), item.formatAsString()) + runId + QLC('/'); + storePath += itemSubPath(item.engineAsString(), item.formatAsString(), isBaseline) + runId + QLC('/'); QString itemName = item.scriptName; if (itemName.contains(QLC('.'))) @@ -253,11 +274,8 @@ QString BaselineHandler::updateAllBaselines(const QString &host, const QString & const QString &engine, const QString &format) { QString basePath(BaselineServer::storagePath()); - QString srcDir(basePath + host + QLC('/') - + QString(QLS("mismatches_%1_%2/")).arg(engine, format) - + id); - QString dstDir(basePath + host + QLC('/') - + QString(QLS("baselines_%1_%2/")).arg(engine, format)); + QString srcDir(basePath + host + QLC('/') + itemSubPath(engine, format, false) + id); + QString dstDir(basePath + host + QLC('/') + itemSubPath(engine, format)); QDir dir(srcDir); QStringList nameFilter; @@ -307,6 +325,54 @@ QString BaselineHandler::updateSingleBaseline(const QString &oldBaseline, const return res; } +QString BaselineHandler::blacklistTest(const QString &scriptName, const QString &host, const QString &engine, + const QString &format) +{ + QString configFile(BaselineServer::storagePath() + host + QLC('/') + + itemSubPath(engine, format) + QLS(".blacklist")); + QFile file(configFile); + if (file.open(QIODevice::Append)) { + QTextStream out(&file); + out << scriptName << endl; + return QLS("Blacklisted ") + scriptName; + } else { + return QLS("Unable to update blacklisted tests."); + } +} + +QString BaselineHandler::whitelistTest(const QString &scriptName, const QString &host, const QString &engine, + const QString &format) +{ + QString configFile(BaselineServer::storagePath() + host + QLC('/') + + itemSubPath(engine, format) + QLS(".blacklist")); + QFile file(configFile); + QStringList tests; + if (file.open(QIODevice::ReadOnly)) { + QTextStream in(&file); + do { + tests << in.readLine(); + } while (!in.atEnd()); + if (tests.count() != 0) { + QMutableStringListIterator it(tests); + while (it.hasNext()) { + it.next(); + if (it.value() == scriptName) + it.remove(); + } + } + file.close(); + if (file.open(QIODevice::WriteOnly | QIODevice::Truncate)) { + QTextStream out(&file); + for (int i=0; iBuild key: " << plat.buildKey << "\n"; out << "

Engine: " << item.engineAsString() << "

\n"; out << "

Format: " << item.formatAsString() << "

\n\n"; - out << "

Update all baselines

\n\n"; + << "\">Update all baselines
"; out << "\n" "\n" "\n" "\n" "\n" "\n" - "\n" - "\n" - "\n" - "\n\n"; + "\n" + "
"; } @@ -82,23 +81,53 @@ void HTMLPage::addItem(const QString &baseline, const QString &rendered, const I headerWritten = true; } QString compared = generateCompared(baseline, rendered); - QString fuzzy = generateCompared(baseline, rendered, true); QString pageUrl = BaselineServer::baseUrl() + path; out << "\n"; out << "\n"; - QStringList images = QStringList() << baseline << rendered << compared << fuzzy; + QStringList images = QStringList() << baseline << rendered << compared; foreach(const QString& img, images) out << "\n"; - out << "\n"; + out << "\n"; out << "\n\n"; + + QMutableVectorIterator it(imageItems); + while (it.hasNext()) { + it.next(); + if (it.value().scriptName == item.scriptName) { + it.remove(); + break; + } + } } void HTMLPage::end() { if (file.isOpen()) { + // Add the names of the scripts that passed the test, or were blacklisted + QString pageUrl = BaselineServer::baseUrl() + path; + for (int i=0; i"; + } + writeFooter(); out.flush(); file.close(); @@ -132,14 +161,30 @@ void HTMLPage::handleCGIQuery(const QString &query) // << "Contents of QUERY_STRING:
" // << "Full string = " << query << "
"; - if (cgiUrl.queryItemValue(QLS("update")) == QLS("single")) { + QString command(cgiUrl.queryItemValue("cmd")); + + if (command == QLS("updateSingleBaseline")) { s << BaselineHandler::updateSingleBaseline(cgiUrl.queryItemValue(QLS("oldBaseline")), cgiUrl.queryItemValue(QLS("newBaseline"))); - } else { + } else if (command == QLS("updateAllBaselines")) { s << BaselineHandler::updateAllBaselines(cgiUrl.queryItemValue(QLS("host")), cgiUrl.queryItemValue(QLS("id")), cgiUrl.queryItemValue(QLS("engine")), cgiUrl.queryItemValue(QLS("format"))); + } else if (command == QLS("blacklist")) { + // blacklist a test + s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("scriptName")), + cgiUrl.queryItemValue(QLS("host")), + cgiUrl.queryItemValue(QLS("engine")), + cgiUrl.queryItemValue(QLS("format"))); + } else if (command == QLS("whitelist")) { + // whitelist a test + s << BaselineHandler::whitelistTest(cgiUrl.queryItemValue(QLS("scriptName")), + cgiUrl.queryItemValue(QLS("host")), + cgiUrl.queryItemValue(QLS("engine")), + cgiUrl.queryItemValue(QLS("format"))); + } else { + s << "Unknown query:
" << query << "
"; } s << "

Back to report"; s << ""; diff --git a/tests/arthur/baselineserver/src/htmlpage.h b/tests/arthur/baselineserver/src/htmlpage.h index 29b4ff5..a937052 100644 --- a/tests/arthur/baselineserver/src/htmlpage.h +++ b/tests/arthur/baselineserver/src/htmlpage.h @@ -11,7 +11,7 @@ public: HTMLPage(); ~HTMLPage(); - void start(const QString &storagePath, const QString &runId, const PlatformInfo pinfo); + void start(const QString &storagePath, const QString &runId, const PlatformInfo pinfo, const ImageItemList &itemList); void addItem(const QString &baseline, const QString &rendered, const ImageItem &item); void end(); QString filePath(); @@ -29,6 +29,7 @@ private: QTextStream out; QString id; PlatformInfo plat; + ImageItemList imageItems; bool headerWritten; }; diff --git a/tests/auto/lancelot/tst_lancelot.cpp b/tests/auto/lancelot/tst_lancelot.cpp index d7a675d..21044d4 100644 --- a/tests/auto/lancelot/tst_lancelot.cpp +++ b/tests/auto/lancelot/tst_lancelot.cpp @@ -189,10 +189,11 @@ void tst_Lancelot::testOpenGL_data() void tst_Lancelot::testOpenGL() { bool ok = false; - QGLWidget glWidget; + QGLWidget glWidget; if (glWidget.isValid() && glWidget.format().directRendering() && ((QGLFormat::openGLVersionFlags() & QGLFormat::OpenGL_Version_2_0) - || (QGLFormat::openGLVersionFlags() & QGLFormat::OpenGL_ES_Version_2_0))) + || (QGLFormat::openGLVersionFlags() & QGLFormat::OpenGL_ES_Version_2_0)) + && QGLFramebufferObject::hasOpenGLFramebufferObjects()) { glWidget.makeCurrent(); if (!QByteArray((const char *)glGetString(GL_VERSION)).contains("Mesa")) @@ -272,13 +273,13 @@ ImageItem tst_Lancelot::render(const ImageItem &item) } else if (item.engine == ImageItem::OpenGL) { QGLWidget glWidget; if (glWidget.isValid()) { - glWidget.resize(800, 800); - glWidget.show(); -#ifdef Q_WS_X11 - qt_x11_wait_for_window_manager(&glWidget); -#endif - paint(&glWidget, script, QFileInfo(filePath).absoluteFilePath()); // eh yuck (filePath stuff) - res.image = glWidget.grabFrameBuffer().convertToFormat(item.renderFormat); + glWidget.makeCurrent(); + QGLFramebufferObjectFormat fboFormat; + fboFormat.setSamples(16); + fboFormat.setAttachment(QGLFramebufferObject::CombinedDepthStencil); + QGLFramebufferObject fbo(800, 800, fboFormat); + paint(&fbo, script, QFileInfo(filePath).absoluteFilePath()); // eh yuck (filePath stuff) + res.image = fbo.toImage().convertToFormat(item.renderFormat); res.imageChecksums.append(ImageItem::computeChecksum(res.image)); } } @@ -290,6 +291,7 @@ void tst_Lancelot::paint(QPaintDevice *device, const QStringList &script, const { QPainter p(device); PaintCommands pcmd(script, 800, 800); + pcmd.setType(ImageType); pcmd.setPainter(&p); pcmd.setFilePath(filePath); pcmd.runCommands(); -- cgit v0.12

ScriptBaselineRenderedComparisonFuzzy ComparisonScoreUpdate
Info/Action
" << item.scriptName << "Update baselineUpdate baseline
" + "Blacklist test
" << imageItems.at(i).scriptName << "N/AN/AN/A"; + if (imageItems.at(i).status == ImageItem::IgnoreItem) { + out << "Blacklisted
" + "Whitelist test"; + } else { + out << "Test passed"; + } + out << "