Make the examples test for QtDeclarative pass.

- Fix check to indicate immediate errors, skip the loading
  state (using QTRY_VERIFY from shared/utils.h)
  and check for errors after loading again.
- Exclude all broken examples.
- Exclude shaders if import path is missing or OpenGL is
  not present.
- Exclude Mac .app folders
- Fix the DeclarativeViewer to check for the presence
  of the ImageMagick and ffmpeg executables only once,
  reducing test time.
- Do not check for ImageMagick by running its command
  line tool 'convert' on Windows, since Windows
  has a tool of the same name that converts file
  systems (!).
- Fix doc snippets to load correctly.

Task-number: QTQAINFRA-428
Change-Id: Icc0a983bc42857b41ab1d9e93336f8265bfbec36
Reviewed-by: Janne Anttila <janne.anttila@digia.com>
Reviewed-by: Caroline Chao <caroline.chao@digia.com>
Reviewed-by: Christopher Adams <chris.adams@qinetic.com.au>

2 files changed, 55 insertions, 33 deletions

diff --git a/tests/auto/declarative/examples/examples.pro b/tests/auto/declarative/examples/examples.pro
index 1f99b9e..fd4be3d 100644
--- a/tests/auto/declarative/examples/examples.pro
+++ b/tests/auto/declarative/examples/examples.pro
@@ -18,5 +18,3 @@ wince*|symbian: {
 }
 
 CONFIG += parallel_test
-
-CONFIG+=insignificant_test # QTQAINFRA-428
diff --git a/tests/auto/declarative/examples/tst_examples.cpp b/tests/auto/declarative/examples/tst_examples.cpp
index 32a8af7..97450e3 100644
--- a/tests/auto/declarative/examples/tst_examples.cpp
+++ b/tests/auto/declarative/examples/tst_examples.cpp
@@ -40,9 +40,11 @@
 ****************************************************************************/
 #include <qtest.h>
 #include <QLibraryInfo>
+#include <QFileInfo>
 #include <QDir>
 #include <QProcess>
 #include <QDebug>
+#include "../../../shared/util.h"
 #include "qmlruntime.h"
 #include <QDeclarativeView>
 #include <QDeclarativeError>
@@ -64,7 +66,6 @@ private slots:
 
     void namingConvention();
 private:
-    QString qmlruntime;
     QStringList excludedDirs;
 
     void namingConvention(const QDir &);
@@ -73,32 +74,42 @@ private:
 
 tst_examples::tst_examples()
 {
-    QString binaries = QLibraryInfo::location(QLibraryInfo::BinariesPath);
-
-#if defined(Q_WS_MAC)
-    qmlruntime = QDir(binaries).absoluteFilePath("qml.app/Contents/MacOS/qml");
-#elif defined(Q_WS_WIN)
-    qmlruntime = QDir(binaries).absoluteFilePath("qml.exe");
-#else
-    qmlruntime = QDir(binaries).absoluteFilePath("qml");
-#endif
-
-
     // Add directories you want excluded here
-    excludedDirs << "doc/src/snippets/declarative/visualdatamodel_rootindex";
-    excludedDirs << "doc/src/snippets/declarative/qtbinding";
+    excludedDirs << "doc/src/snippets/declarative/visualdatamodel_rootindex"
+                 << "doc/src/snippets/declarative/qtbinding";
+    // Known to violate naming conventions, QTQAINFRA-428
+    excludedDirs << "demos/mobile/qtbubblelevel/qml"
+                 << "demos/mobile/quickhit";
+    // Layouts do not install, QTQAINFRA-428
+    excludedDirs << "examples/declarative/cppextensions/qgraphicslayouts/qgraphicsgridlayout/qml/qgraphicsgridlayout"
+                 << "examples/declarative/cppextensions/qgraphicslayouts/qgraphicslinearlayout/qml/qgraphicslinearlayout";
+    // Various QML errors, QTQAINFRA-428
+    excludedDirs << "doc/src/snippets/declarative/imports";
+
+    // Check shaders which are not present for configurations without OpenGL or when not built.
+    const QString shaderExample = QLatin1String("examples/declarative/shadereffects");
+#ifdef QT_NO_OPENGL
+    excludedDirs << shaderExample;
+#else
+    const QString importPaths = QLibraryInfo::location(QLibraryInfo::ImportsPath);
+    if (!QFileInfo(importPaths + QLatin1String("/Qt/labs/shaders")).isDir())
+        excludedDirs << shaderExample;
+#endif // QT_NO_OPENGL
 
 #ifdef QT_NO_WEBKIT
-    excludedDirs << "examples/declarative/modelviews/webview";
-    excludedDirs << "demos/declarative/webbrowser";
-#endif
+    excludedDirs << "examples/declarative/modelviews/webview"
+                 << "demos/declarative/webbrowser"
+                 << "doc/src/snippets/declarative/webview";
+#endif // QT_NO_WEBKIT
 
 #ifdef QT_NO_XMLPATTERNS
-    excludedDirs << "examples/declarative/xml/xmldata";
-    excludedDirs << "demos/declarative/twitter";
-    excludedDirs << "demos/declarative/flickr";
-    excludedDirs << "demos/declarative/photoviewer";
-#endif
+    excludedDirs << "examples/declarative/xml/xmldata"
+                 << "demos/declarative/twitter"
+                 << "demos/declarative/flickr"
+                 << "demos/declarative/photoviewer"
+                 << "demos/declarative/rssnews/qml/rssnews"
+                 << "doc/src/snippets/declarative";
+#endif // QT_NO_XMLPATTERNS
 }
 
 /*
@@ -148,11 +159,14 @@ void tst_examples::namingConvention()
 
 QStringList tst_examples::findQmlFiles(const QDir &d)
 {
-    for (int ii = 0; ii < excludedDirs.count(); ++ii) {
-        QString s = excludedDirs.at(ii);
-        if (d.absolutePath().endsWith(s))
+    const QString absolutePath = d.absolutePath();
+#ifdef Q_OS_MAC // Mac: Do not recurse into bundle folders of built examples.
+    if (absolutePath.endsWith(QLatin1String(".app")))
+        return QStringList();
+#endif
+    foreach (const QString &excludedDir, excludedDirs)
+        if (absolutePath.endsWith(excludedDir))
             return QStringList();
-    }
 
     QStringList rv;
 
@@ -206,23 +220,33 @@ static void silentErrorsMsgHandler(QtMsgType, const char *)
 {
 }
 
+static inline QByteArray msgViewerErrors(const QList<QDeclarativeError> &l)
+{
+    QString errors;
+    QDebug(&errors) << '\n' << l;
+    return errors.toLocal8Bit();
+}
+
 void tst_examples::examples()
 {
     QFETCH(QString, file);
+    QVERIFY2(QFileInfo(file).exists(),
+             qPrintable(QString::fromLatin1("'%1' does not exist.").arg(QDir::toNativeSeparators(file))));
 
     QDeclarativeViewer viewer;
 
     QtMsgHandler old = qInstallMsgHandler(silentErrorsMsgHandler);
     QVERIFY(viewer.open(file));
     qInstallMsgHandler(old);
+    QVERIFY2(viewer.view()->status() != QDeclarativeView::Error,
+             msgViewerErrors(viewer.view()->errors()).constData());
+    QTRY_VERIFY(viewer.view()->status() != QDeclarativeView::Loading);
+    QVERIFY2(viewer.view()->status() == QDeclarativeView::Ready,
+             msgViewerErrors(viewer.view()->errors()).constData());
 
-    if (viewer.view()->status() == QDeclarativeView::Error)
-        qWarning() << viewer.view()->errors();
-
-    QCOMPARE(viewer.view()->status(), QDeclarativeView::Ready);
     viewer.show();
 
-    QTest::qWaitForWindowShown(&viewer);
+    QVERIFY(QTest::qWaitForWindowShown(&viewer));
 }
 
 QTEST_MAIN(tst_examples)