From 9546ec3b4d56436bb96709dc012fdb33beed45dc Mon Sep 17 00:00:00 2001 From: aavit Date: Wed, 10 Nov 2010 11:29:06 +0100 Subject: New and much improved mapping of items to fs storage --- tests/arthur/baselineserver/src/baselineserver.cpp | 99 ++++++++++++++-------- tests/arthur/baselineserver/src/baselineserver.h | 6 +- tests/arthur/baselineserver/src/htmlpage.cpp | 2 +- 3 files changed, 68 insertions(+), 39 deletions(-) diff --git a/tests/arthur/baselineserver/src/baselineserver.cpp b/tests/arthur/baselineserver/src/baselineserver.cpp index 03d6499..2a2fc08 100644 --- a/tests/arthur/baselineserver/src/baselineserver.cpp +++ b/tests/arthur/baselineserver/src/baselineserver.cpp @@ -38,6 +38,10 @@ ** $QT_END_LICENSE$ ** ****************************************************************************/ + +#define QT_USE_FAST_CONCATENATION +#define QT_USE_FAST_OPERATOR_PLUS + #include "baselineserver.h" #include #include @@ -63,11 +67,9 @@ BaselineServer::BaselineServer(QObject *parent) QString BaselineServer::storagePath() { if (storage.isEmpty()) { - QByteArray envDir = qgetenv("QT_LANCELOT_DIR"); - if (!envDir.isEmpty()) - storage = QLS(envDir.append('/')); - else - storage = QLS("/var/www/"); + storage = QLS(qgetenv("QT_LANCELOT_DIR")); + if (storage.isEmpty()) + storage = QLS("/var/www"); } return storage; } @@ -278,34 +280,37 @@ void BaselineHandler::receiveDisconnect() } -QString BaselineHandler::itemSubPath(const QString &engine, const QString &format, bool isBaseline) +void BaselineHandler::mapPlatformInfo() { - if (isBaseline) - return QString(QLS("baselines_%1_%2/")).arg(engine, format); - else - return QString(QLS("mismatches_%1_%2/")).arg(engine, format); + mapped = plat; + + // Map hostname + QString host = plat.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any + if (host.isEmpty() || host == QLS("localhost")) { + host = plat.value(PI_HostAddress); + } else { + // remove index postfix typical of vm hostnames + host.remove(QRegExp(QLS("\\d+$"))); + if (host.endsWith(QLC('-'))) + host.chop(1); + } + if (host.isEmpty()) + host = QLS("unknownhost"); + mapped.insert(PI_HostName, host); + + // Map qmakespec + QString mkspec = plat.value(PI_QMakeSpec); + mapped.insert(PI_QMakeSpec, mkspec.replace(QLC('/'), QLC('_'))); + + // Map Qt version + QString ver = plat.value(PI_QtVersion); + mapped.insert(PI_QtVersion, ver.prepend(QLS("Qt-"))); //### TBD: remove patch version } QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, bool absolute) { - if (pathForRun.isNull()) { - QString host = plat.value(PI_HostName).section(QLC('.'), 0, 0); // Filter away domain, if any - if (host.isEmpty() || host == QLS("localhost")) { - host = proto.socket.peerAddress().toString(); - if (host.isEmpty()) - host = QLS("Unknown"); - } else { - host.replace(QRegExp(QLS("^(bq|oslo?)-(.*)$")), QLS("\\2")); - host.replace(QRegExp(QLS("^(.*)-\\d+$")), QLS("vm-\\1")); - } - pathForRun = host + QLC('/'); - } - - QString storePath = pathForRun; - if (isBaseline) - storePath += itemSubPath(item.engineAsString(), item.formatAsString(), isBaseline); - else - storePath += itemSubPath(item.engineAsString(), item.formatAsString(), isBaseline) + runId + QLC('/'); + if (mapped.isEmpty()) + mapPlatformInfo(); QString itemName = item.scriptName; if (itemName.contains(QLC('.'))) @@ -313,15 +318,26 @@ QString BaselineHandler::pathForItem(const ImageItem &item, bool isBaseline, boo itemName.append(QLC('_')); itemName.append(QString::number(item.scriptChecksum, 16).rightJustified(4, QLC('0'))); + QStringList path; if (absolute) - storePath.prepend(BaselineServer::storagePath()); - return storePath + itemName + QLC('.'); + path += BaselineServer::storagePath(); + path += QLS(isBaseline ? "baselines" : "mismatches"); + path += item.engineAsString() + QLC('_') + item.formatAsString(); + path += mapped.value(PI_QtVersion); + path += mapped.value(PI_QMakeSpec); + path += mapped.value(PI_HostName); + if (!isBaseline) + path += runId; + path += itemName + QLC('.'); + + return path.join(QLS("/")); } QString BaselineHandler::updateAllBaselines(const QString &host, const QString &id, const QString &engine, const QString &format) { +#if 0 QString basePath(BaselineServer::storagePath()); QString srcDir(basePath + host + QLC('/') + itemSubPath(engine, format, false) + id); QString dstDir(basePath + host + QLC('/') + itemSubPath(engine, format)); @@ -352,12 +368,15 @@ QString BaselineHandler::updateAllBaselines(const QString &host, const QString & "Command output: 
%2
").arg(proc.errorString(), proc.readAll().constData()); return res; +#else + return QString(); +#endif } QString BaselineHandler::updateSingleBaseline(const QString &oldBaseline, const QString &newBaseline) { QString res; - QString basePath(BaselineServer::storagePath()); + QString basePath(BaselineServer::storagePath() + QLC('/')); QString srcBase(basePath + newBaseline.left(newBaseline.length() - 3)); QString dstDir(basePath + oldBaseline.left(oldBaseline.lastIndexOf(QLC('/')))); @@ -377,6 +396,7 @@ QString BaselineHandler::updateSingleBaseline(const QString &oldBaseline, const QString BaselineHandler::blacklistTest(const QString &scriptName, const QString &host, const QString &engine, const QString &format) { +#if 0 QString configFile(BaselineServer::storagePath() + host + QLC('/') + itemSubPath(engine, format) + QLS(".blacklist")); QFile file(configFile); @@ -387,11 +407,15 @@ QString BaselineHandler::blacklistTest(const QString &scriptName, const QString } else { return QLS("Unable to update blacklisted tests."); } +#else + return QString(); +#endif } QString BaselineHandler::whitelistTest(const QString &scriptName, const QString &host, const QString &engine, const QString &format) { +#if 0 QString configFile(BaselineServer::storagePath() + host + QLC('/') + itemSubPath(engine, format) + QLS(".blacklist")); QFile file(configFile); @@ -420,6 +444,9 @@ QString BaselineHandler::whitelistTest(const QString &scriptName, const QString } } return QLS("Unable to whitelist ") + scriptName + QLS(". Unable to open blacklist file."); +#else + return QString(); +#endif } void BaselineHandler::testPathMapping() @@ -433,6 +460,9 @@ void BaselineHandler::testPathMapping() << QLS("osl-mac-master-6.test.qt.nokia.com") << QLS("sv-xp-vs-010") << QLS("sv-xp-vs-011") + << QLS("sv-solaris-sparc-008") + << QLS("macbuilder-02.test.troll.no") + << QLS("bqvm1164") << QLS("chimera") << QLS("localhost"); @@ -445,11 +475,12 @@ void BaselineHandler::testPathMapping() plat.insert(PI_QtVersion, QLS("4.8.0")); plat.insert(PI_BuildKey, QLS("(nobuildkey)")); + plat.insert(PI_QMakeSpec, "linux-g++"); foreach(const QString& host, hosts) { - pathForRun = QString(); + mapped.clear(); plat.insert(PI_HostName, host); - qDebug() << "Baseline from" << host << "->" << pathForItem(item, true).remove(BaselineServer::storagePath()); - qDebug() << "Mismatch from" << host << "->" << pathForItem(item, false).remove(BaselineServer::storagePath()); + qDebug() << "Baseline from" << host << "->" << pathForItem(item, true); + qDebug() << "Mismatch from" << host << "->" << pathForItem(item, false); } } diff --git a/tests/arthur/baselineserver/src/baselineserver.h b/tests/arthur/baselineserver/src/baselineserver.h index e311527..be65a9d 100644 --- a/tests/arthur/baselineserver/src/baselineserver.h +++ b/tests/arthur/baselineserver/src/baselineserver.h @@ -115,18 +115,16 @@ private slots: private: void provideBaselineChecksums(const QByteArray &itemListBlock); void storeImage(const QByteArray &itemBlock, bool isBaseline); + void mapPlatformInfo(); QString pathForItem(const ImageItem &item, bool isBaseline = true, bool absolute = true); const char *logtime(); QString computeMismatchScore(const QImage& baseline, const QImage& rendered); - QString engineForItem(const ImageItem &item); - - static QString itemSubPath(const QString &engine, const QString &format, bool isBaseline = true); BaselineProtocol proto; PlatformInfo plat; + PlatformInfo mapped; bool connectionEstablished; QString runId; - QString pathForRun; HTMLPage report; }; diff --git a/tests/arthur/baselineserver/src/htmlpage.cpp b/tests/arthur/baselineserver/src/htmlpage.cpp index ad7f8b7..29ce262 100644 --- a/tests/arthur/baselineserver/src/htmlpage.cpp +++ b/tests/arthur/baselineserver/src/htmlpage.cpp @@ -67,7 +67,7 @@ void HTMLPage::start(const QString &storagepath, const QString &runId, const Pla id = runId; plat = pinfo; address = hostAddress; - root = storagepath; + root = storagepath + QLC('/'); imageItems = itemList; QString dir = root + QLS("reports/"); QDir cwd; -- cgit v0.12 From b9395570df56ae597514d294473ce0c6ba3bec06 Mon Sep 17 00:00:00 2001 From: aavit Date: Fri, 12 Nov 2010 15:29:16 +0100 Subject: Filter runs on git branch and repo --- tests/arthur/baselineserver/src/baselineserver.cpp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/arthur/baselineserver/src/baselineserver.cpp b/tests/arthur/baselineserver/src/baselineserver.cpp index 2a2fc08..eb756fc 100644 --- a/tests/arthur/baselineserver/src/baselineserver.cpp +++ b/tests/arthur/baselineserver/src/baselineserver.cpp @@ -149,7 +149,6 @@ void BaselineHandler::receiveRequest() proto.socket.disconnectFromHost(); return; } - connectionEstablished = true; QString logMsg; foreach (QString key, plat.keys()) { if (key != PI_HostName && key != PI_HostAddress) @@ -157,6 +156,20 @@ void BaselineHandler::receiveRequest() } qDebug() << runId << logtime() << "Connection established with" << plat.value(PI_HostName) << "[" << qPrintable(plat.value(PI_HostAddress)) << "]" << logMsg; + + // Filter on branch + QString branch = plat.value(PI_PulseGitBranch); + if (branch.isEmpty()) { + // Not run by Pulse, i.e. ad hoc run: Ok. + } + else if (branch != QLS("master-integration") || !plat.value(PI_GitCommit).contains(QLS("Merge branch 'master' of scm.dev.nokia.troll.no:qt/oslo-staging-2 into master-integration"))) { + qDebug() << runId << logtime() << "Did not pass branch/staging repo filter, disconnecting."; + // TBD: Cleaner termination + proto.socket.disconnectFromHost(); + return; + } + + connectionEstablished = true; return; } -- cgit v0.12 From eedfeabfba1bbbc26792966238463ca3d22665a9 Mon Sep 17 00:00:00 2001 From: aavit Date: Thu, 18 Nov 2010 12:44:27 +0100 Subject: Fix black- and whitelisting of individual items --- tests/arthur/baselineserver/src/baselineserver.cpp | 75 +++++++--------------- tests/arthur/baselineserver/src/baselineserver.h | 5 +- tests/arthur/baselineserver/src/htmlpage.cpp | 42 +++++------- tests/arthur/baselineserver/src/htmlpage.h | 4 +- 4 files changed, 41 insertions(+), 85 deletions(-) diff --git a/tests/arthur/baselineserver/src/baselineserver.cpp b/tests/arthur/baselineserver/src/baselineserver.cpp index eb756fc..3f4ae95 100644 --- a/tests/arthur/baselineserver/src/baselineserver.cpp +++ b/tests/arthur/baselineserver/src/baselineserver.cpp @@ -54,6 +54,7 @@ QString BaselineServer::storage; + BaselineServer::BaselineServer(QObject *parent) : QTcpServer(parent) { @@ -221,9 +222,9 @@ void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock) } // Find and mark blacklisted items + QString context = pathForItem(itemList.at(0), true, false).section(QLC('/'), 0, -2); if (itemList.count() > 0) { - QString prefix = pathForItem(itemList.at(0), true).section(QLC('/'), 0, -2); - QFile file(prefix + QLS("/.blacklist")); + QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST")); if (file.open(QIODevice::ReadOnly)) { QTextStream in(&file); do { @@ -242,7 +243,7 @@ void BaselineHandler::provideBaselineChecksums(const QByteArray &itemListBlock) QDataStream ods(&block, QIODevice::WriteOnly); ods << itemList; proto.sendBlock(BaselineProtocol::Ack, block); - report.start(BaselineServer::storagePath(), runId, plat, proto.socket.peerAddress().toString(), itemList); + report.start(BaselineServer::storagePath(), runId, plat, context, itemList); } @@ -406,61 +407,29 @@ QString BaselineHandler::updateSingleBaseline(const QString &oldBaseline, const return res; } -QString BaselineHandler::blacklistTest(const QString &scriptName, const QString &host, const QString &engine, - const QString &format) +QString BaselineHandler::blacklistTest(const QString &context, const QString &itemId, bool removeFromBlacklist) { -#if 0 - QString configFile(BaselineServer::storagePath() + host + QLC('/') - + itemSubPath(engine, format) + QLS(".blacklist")); - QFile file(configFile); - if (file.open(QIODevice::Append)) { - QTextStream out(&file); - out << scriptName << endl; - return QLS("Blacklisted ") + scriptName; + QFile file(BaselineServer::storagePath() + QLC('/') + context + QLS("/BLACKLIST")); + QStringList blackList; + if (file.open(QIODevice::ReadWrite)) { + while (!file.atEnd()) + blackList.append(file.readLine().trimmed()); + + if (removeFromBlacklist) + blackList.removeAll(itemId); + else if (!blackList.contains(itemId)) + blackList.append(itemId); + + file.resize(0); + foreach (QString id, blackList) + file.write(id.toLatin1() + '\n'); + file.close(); + return QLS(removeFromBlacklist ? "Whitelisted " : "Blacklisted ") + itemId + QLS(" in context ") + context; } else { - return QLS("Unable to update blacklisted tests."); + return QLS("Unable to update blacklisted tests, failed to open ") + file.fileName(); } -#else - return QString(); -#endif } -QString BaselineHandler::whitelistTest(const QString &scriptName, const QString &host, const QString &engine, - const QString &format) -{ -#if 0 - QString configFile(BaselineServer::storagePath() + host + QLC('/') - + itemSubPath(engine, format) + QLS(".blacklist")); - QFile file(configFile); - QStringList tests; - if (file.open(QIODevice::ReadOnly)) { - QTextStream in(&file); - do { - tests << in.readLine(); - } while (!in.atEnd()); - if (tests.count() != 0) { - QMutableStringListIterator it(tests); - while (it.hasNext()) { - it.next(); - if (it.value() == scriptName) - it.remove(); - } - } - file.close(); - if (file.open(QIODevice::WriteOnly | QIODevice::Truncate)) { - QTextStream out(&file); - for (int i=0; i\n"; - out << "Update baseline
" -#if 0 - "Blacklist test" -#endif + + out << "

Replace baseline with rendered

" + << "

Blacklist this item

" << "\n"; - out << "\n\n"; + out << "\n\n"; QMutableVectorIterator it(imageItems); while (it.hasNext()) { @@ -162,13 +159,10 @@ void HTMLPage::end() for (int i=0; i" << imageItems.at(i).scriptName << "N/AN/AN/A"; if (imageItems.at(i).status == ImageItem::IgnoreItem) { - out << "Blacklisted
" - "Whitelist test"; + out << "Blacklisted " + << "Whitelist item"; } else { out << "Test passed"; } @@ -220,16 +214,12 @@ void HTMLPage::handleCGIQuery(const QString &query) cgiUrl.queryItemValue(QLS("format"))); } else if (command == QLS("blacklist")) { // blacklist a test - s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("scriptName")), - cgiUrl.queryItemValue(QLS("host")), - cgiUrl.queryItemValue(QLS("engine")), - cgiUrl.queryItemValue(QLS("format"))); + s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), + cgiUrl.queryItemValue(QLS("itemId"))); } else if (command == QLS("whitelist")) { // whitelist a test - s << BaselineHandler::whitelistTest(cgiUrl.queryItemValue(QLS("scriptName")), - cgiUrl.queryItemValue(QLS("host")), - cgiUrl.queryItemValue(QLS("engine")), - cgiUrl.queryItemValue(QLS("format"))); + s << BaselineHandler::blacklistTest(cgiUrl.queryItemValue(QLS("context")), + cgiUrl.queryItemValue(QLS("itemId")), true); } else { s << "Unknown query:
" << query << "
"; } diff --git a/tests/arthur/baselineserver/src/htmlpage.h b/tests/arthur/baselineserver/src/htmlpage.h index bef6b55..fa4d1ed 100644 --- a/tests/arthur/baselineserver/src/htmlpage.h +++ b/tests/arthur/baselineserver/src/htmlpage.h @@ -51,7 +51,7 @@ public: HTMLPage(); ~HTMLPage(); - void start(const QString &storagePath, const QString &runId, const PlatformInfo pinfo, const QString &hostAddress, const ImageItemList &itemList); + void start(const QString &storagePath, const QString &runId, const PlatformInfo pinfo, const QString &context, const ImageItemList &itemList); void addItem(const QString &baseline, const QString &rendered, const ImageItem &item); void end(); QString filePath(); @@ -69,7 +69,7 @@ private: QTextStream out; QString id; PlatformInfo plat; - QString address; + QString ctx; ImageItemList imageItems; bool headerWritten; }; -- cgit v0.12 From e614223831f2b3cc9051ae88586370a7d5b63db2 Mon Sep 17 00:00:00 2001 From: aavit Date: Fri, 19 Nov 2010 13:22:02 +0100 Subject: Added clean abort, dryrun mode. And stop ignoring the unused byte in RGB32 --- tests/arthur/baselineserver/src/baselineserver.cpp | 3 +- tests/arthur/common/baselineprotocol.cpp | 51 ++++++++++++++++------ tests/arthur/common/baselineprotocol.h | 8 +++- tests/auto/lancelot/tst_lancelot.cpp | 15 +++---- 4 files changed, 53 insertions(+), 24 deletions(-) diff --git a/tests/arthur/baselineserver/src/baselineserver.cpp b/tests/arthur/baselineserver/src/baselineserver.cpp index 3f4ae95..7679f13 100644 --- a/tests/arthur/baselineserver/src/baselineserver.cpp +++ b/tests/arthur/baselineserver/src/baselineserver.cpp @@ -165,11 +165,12 @@ void BaselineHandler::receiveRequest() } else if (branch != QLS("master-integration") || !plat.value(PI_GitCommit).contains(QLS("Merge branch 'master' of scm.dev.nokia.troll.no:qt/oslo-staging-2 into master-integration"))) { qDebug() << runId << logtime() << "Did not pass branch/staging repo filter, disconnecting."; - // TBD: Cleaner termination + proto.sendBlock(BaselineProtocol::Abort, QByteArray("This branch/staging repo is not assigned to be tested.")); proto.socket.disconnectFromHost(); return; } + proto.sendBlock(BaselineProtocol::Ack, QByteArray()); connectionEstablished = true; return; } diff --git a/tests/arthur/common/baselineprotocol.cpp b/tests/arthur/common/baselineprotocol.cpp index 2a41104..6d26e9a 100644 --- a/tests/arthur/common/baselineprotocol.cpp +++ b/tests/arthur/common/baselineprotocol.cpp @@ -47,11 +47,28 @@ #include #include #include +#include #ifndef QMAKESPEC #define QMAKESPEC "Unknown" #endif +#if defined(Q_OS_WIN) +#include +#endif +#if defined(Q_OS_UNIX) +#include +#endif +void BaselineProtocol::sysSleep(int ms) +{ +#if defined(Q_OS_WIN) + Sleep(DWORD(ms)); +#else + struct timespec ts = { ms / 1000, (ms % 1000) * 1000 * 1000 }; + nanosleep(&ts, NULL); +#endif +} + PlatformInfo::PlatformInfo(bool useLocal) : QMap() { @@ -141,12 +158,6 @@ quint64 ImageItem::computeChecksum(const QImage &image) p += bpl; } } - if (img.format() == QImage::Format_RGB32) { // Thank you, Haavard - quint32 *p = (quint32 *)img.bits(); - const quint32 *end = p + (img.byteCount()/4); - while (pinsert(PI_HostAddress, socket.peerAddress().toString()); } - if (!sendBlock(Ack, QByteArray())) - return false; return true; } diff --git a/tests/arthur/common/baselineprotocol.h b/tests/arthur/common/baselineprotocol.h index 162a19f..9f59454 100644 --- a/tests/arthur/common/baselineprotocol.h +++ b/tests/arthur/common/baselineprotocol.h @@ -123,7 +123,7 @@ public: enum Constant { ProtocolVersion = 2, ServerPort = 54129, - Timeout = 10000 + Timeout = 5000 }; enum Command { @@ -135,10 +135,12 @@ public: AcceptMismatch = 5, // Responses Ack = 128, + Abort = 129, + DoDryRun = 130 }; // For client: - bool connect(); + bool connect(bool *dryrun = 0); bool requestBaselineChecksums(ImageItemList *itemList); bool submitNewBaseline(const ImageItem &item, QByteArray *serverMsg); bool submitMismatch(const ImageItem &item, QByteArray *serverMsg); @@ -153,6 +155,8 @@ private: bool sendBlock(Command cmd, const QByteArray &block); bool receiveBlock(Command *cmd, QByteArray *block); + void sysSleep(int ms); + QString errMsg; QTcpSocket socket; diff --git a/tests/auto/lancelot/tst_lancelot.cpp b/tests/auto/lancelot/tst_lancelot.cpp index a06a251..8467672 100644 --- a/tests/auto/lancelot/tst_lancelot.cpp +++ b/tests/auto/lancelot/tst_lancelot.cpp @@ -72,6 +72,7 @@ private: BaselineProtocol proto; ImageItemList baseList; QHash scripts; + bool dryRunMode; private slots: void initTestCase(); @@ -103,13 +104,8 @@ void tst_Lancelot::initTestCase() #if defined(Q_OS_SOMEPLATFORM) QSKIP("This test is not supported on this platform.", SkipAll); #endif - if (!proto.connect()) { - QTest::qSleep(3000); // Wait a bit and try again, the server might just be restarting - if (!proto.connect()) { - QWARN(qPrintable(proto.errorMessage())); - QSKIP("Communication with baseline image server failed.", SkipAll); - } - } + if (!proto.connect(&dryRunMode)) + QSKIP(qPrintable(proto.errorMessage()), SkipAll); QDir qpsDir(scriptsDir); QStringList files = qpsDir.entryList(QStringList() << QLatin1String("*.qps"), QDir::Files | QDir::Readable); @@ -250,7 +246,10 @@ void tst_Lancelot::runTestSuite() QByteArray serverMsg; if (!proto.submitMismatch(rendered, &serverMsg)) serverMsg = "Failed to submit mismatching image to server."; - QFAIL("Rendered image differs from baseline.\n" + serverMsg); + if (dryRunMode) + qDebug() << "Dryrun mode, ignoring detected mismatch." << serverMsg; + else + QFAIL("Rendered image differs from baseline.\n" + serverMsg); } } -- cgit v0.12