QNetworkAccessManager: enable synchronous HTTP calls

To enable synchronous calls, an attribute in the QNetworkRequest has
to be set. If set, when QNetworkAccessManager::get() (and post(),
put()) returns, the reply is finished and all data has been read in
case of success.
This feature is semi-public for now (usable, but not documented).
To enable this, an attribute in the QNetworkRequest must be set. If
this attribute is set, we open a new connection to the server with
only one channel and call the channels' sockets' waitFor* methods.

Reviewed-by: Markus Goetz

15 files changed, 209 insertions, 40 deletions

diff --git a/src/network/access/qhttpnetworkconnection.cpp b/src/network/access/qhttpnetworkconnection.cpp
index 4d27531..0531595 100644
--- a/src/network/access/qhttpnetworkconnection.cpp
+++ b/src/network/access/qhttpnetworkconnection.cpp
@@ -327,8 +327,6 @@ bool QHttpNetworkConnectionPrivate::handleAuthenticateChallenge(QAbstractSocket
     Q_ASSERT(socket);
     Q_ASSERT(reply);
 
-    Q_Q(QHttpNetworkConnection);
-
     resend = false;
     //create the response header to be used with QAuthenticatorPrivate.
     QList<QPair<QByteArray, QByteArray> > fields = reply->header();
@@ -854,12 +852,17 @@ QHttpNetworkReply* QHttpNetworkConnection::sendRequest(const QHttpNetworkRequest
     return d->queueRequest(request);
 }
 
-bool QHttpNetworkConnection::isEncrypted() const
+bool QHttpNetworkConnection::isSsl() const
 {
     Q_D(const QHttpNetworkConnection);
     return d->encrypt;
 }
 
+QHttpNetworkConnectionChannel *QHttpNetworkConnection::channels() const
+{
+    return d_func()->channels;
+}
+
 #ifndef QT_NO_NETWORKPROXY
 void QHttpNetworkConnection::setCacheProxy(const QNetworkProxy &networkProxy)
 {
diff --git a/src/network/access/qhttpnetworkconnection_p.h b/src/network/access/qhttpnetworkconnection_p.h
index 8461426c..9f23cbf 100644
--- a/src/network/access/qhttpnetworkconnection_p.h
+++ b/src/network/access/qhttpnetworkconnection_p.h
@@ -108,7 +108,9 @@ public:
     QNetworkProxy transparentProxy() const;
 #endif
 
-    bool isEncrypted() const;
+    bool isSsl() const;
+
+    QHttpNetworkConnectionChannel *channels() const;
 
 #ifndef QT_NO_OPENSSL
     void setSslConfiguration(const QSslConfiguration &config);
diff --git a/src/network/access/qhttpnetworkconnectionchannel.cpp b/src/network/access/qhttpnetworkconnectionchannel.cpp
index 02daa50..c8caad4 100644
--- a/src/network/access/qhttpnetworkconnectionchannel.cpp
+++ b/src/network/access/qhttpnetworkconnectionchannel.cpp
@@ -180,7 +180,6 @@ bool QHttpNetworkConnectionChannel::sendRequest()
         reply->d_func()->autoDecompress = request.d->autoDecompress;
         reply->d_func()->pipeliningUsed = false;
 
-        pendingEncrypt = false;
         // if the url contains authentication parameters, use the new ones
         // both channels will use the new authentication parameters
         if (!request.url().userInfo().isEmpty() && request.withCredentials()) {
@@ -1019,6 +1018,7 @@ void QHttpNetworkConnectionChannel::_q_encrypted()
     if (!socket)
         return; // ### error
     state = QHttpNetworkConnectionChannel::IdleState;
+    pendingEncrypt = false;
     sendRequest();
 }
 
diff --git a/src/network/access/qhttpnetworkconnectionchannel_p.h b/src/network/access/qhttpnetworkconnectionchannel_p.h
index 442086a..fd18042 100644
--- a/src/network/access/qhttpnetworkconnectionchannel_p.h
+++ b/src/network/access/qhttpnetworkconnectionchannel_p.h
@@ -158,6 +158,8 @@ public:
     bool isSocketWaiting() const;
     bool isSocketReading() const;
 
+    friend class QNetworkAccessHttpBackend;
+
     protected slots:
     void _q_receiveReply();
     void _q_bytesWritten(qint64 bytes); // proceed sending
diff --git a/src/network/access/qhttpnetworkreply.cpp b/src/network/access/qhttpnetworkreply.cpp
index e4eb7c4..21bc427 100644
--- a/src/network/access/qhttpnetworkreply.cpp
+++ b/src/network/access/qhttpnetworkreply.cpp
@@ -188,6 +188,12 @@ QByteArray QHttpNetworkReply::readAny()
     return d->responseData.read();
 }
 
+QByteArray QHttpNetworkReply::readAll()
+{
+    Q_D(QHttpNetworkReply);
+    return d->responseData.readAll();
+}
+
 void QHttpNetworkReply::setDownstreamLimited(bool dsl)
 {
     Q_D(QHttpNetworkReply);
diff --git a/src/network/access/qhttpnetworkreply_p.h b/src/network/access/qhttpnetworkreply_p.h
index 3f79d81..9cf805c 100644
--- a/src/network/access/qhttpnetworkreply_p.h
+++ b/src/network/access/qhttpnetworkreply_p.h
@@ -126,6 +126,7 @@ public:
     qint64 bytesAvailable() const;
     qint64 bytesAvailableNextBlock() const;
     QByteArray readAny();
+    QByteArray readAll();
     void setDownstreamLimited(bool t);
 
     bool isFinished() const;
diff --git a/src/network/access/qnetworkaccessbackend.cpp b/src/network/access/qnetworkaccessbackend.cpp
index 05eb6cb..12b6400 100644
--- a/src/network/access/qnetworkaccessbackend.cpp
+++ b/src/network/access/qnetworkaccessbackend.cpp
@@ -142,6 +142,7 @@ void QNetworkAccessBackend::emitReplyUploadProgress(qint64 bytesSent, qint64 byt
 QNetworkAccessBackend::QNetworkAccessBackend()
     : manager(0)
     , reply(0)
+    , synchronous(false)
 {
 }
 
diff --git a/src/network/access/qnetworkaccessbackend_p.h b/src/network/access/qnetworkaccessbackend_p.h
index 7faa5cb..c9ec37e 100644
--- a/src/network/access/qnetworkaccessbackend_p.h
+++ b/src/network/access/qnetworkaccessbackend_p.h
@@ -157,6 +157,9 @@ public:
     QVariant attribute(QNetworkRequest::Attribute code) const;
     void setAttribute(QNetworkRequest::Attribute code, const QVariant &value);
 
+    bool isSynchronous() { return synchronous; }
+    void setSynchronous(bool sync) { synchronous = sync; }
+
     // return true if the QNonContiguousByteDevice of the upload
     // data needs to support reset(). Currently needed for HTTP.
     // This will possibly enable buffering of the upload data.
@@ -166,6 +169,8 @@ public:
     virtual bool canResume() const { return false; }
     virtual void setResumeOffset(quint64 offset) { Q_UNUSED(offset); }
 
+    virtual bool processRequestSynchronously() { return false; }
+
 protected:
     // Create the device used for reading the upload data
     QNonContiguousByteDevice* createUploadByteDevice();
@@ -200,6 +205,7 @@ private:
     friend class QNetworkReplyImplPrivate;
     QNetworkAccessManagerPrivate *manager;
     QNetworkReplyImplPrivate *reply;
+    bool synchronous;
 };
 
 class QNetworkAccessBackendFactory
diff --git a/src/network/access/qnetworkaccessdatabackend.cpp b/src/network/access/qnetworkaccessdatabackend.cpp
index efb6e3e..74aebdb 100644
--- a/src/network/access/qnetworkaccessdatabackend.cpp
+++ b/src/network/access/qnetworkaccessdatabackend.cpp
@@ -122,4 +122,10 @@ bool QNetworkAccessDataBackend::waitForUpstreamBytesWritten(int)
     return false;
 }
 
+bool QNetworkAccessDataBackend::processRequestSynchronously()
+{
+    start();
+    return true;
+}
+
 QT_END_NAMESPACE
diff --git a/src/network/access/qnetworkaccessdatabackend_p.h b/src/network/access/qnetworkaccessdatabackend_p.h
index a7c63d5..0e5a494 100644
--- a/src/network/access/qnetworkaccessdatabackend_p.h
+++ b/src/network/access/qnetworkaccessdatabackend_p.h
@@ -68,6 +68,8 @@ public:
     virtual void closeUpstreamChannel();
     virtual bool waitForDownstreamReadyRead(int msecs);
     virtual bool waitForUpstreamBytesWritten(int msecs);
+
+    virtual bool processRequestSynchronously();
 };
 
 class QNetworkAccessDataBackendFactory: public QNetworkAccessBackendFactory
diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp
index 80b05a4..070111d 100644
--- a/src/network/access/qnetworkaccesshttpbackend.cpp
+++ b/src/network/access/qnetworkaccesshttpbackend.cpp
@@ -50,6 +50,7 @@
 #include "qnetworkrequest_p.h"
 #include "qnetworkcookie_p.h"
 #include "QtCore/qdatetime.h"
+#include "QtCore/qelapsedtimer.h"
 #include "QtNetwork/qsslconfiguration.h"
 
 #ifndef QT_NO_HTTP
@@ -318,7 +319,10 @@ void QNetworkAccessHttpBackend::disconnectFromHttp()
 
         // Get the object cache that stores our QHttpNetworkConnection objects
         QNetworkAccessCache *cache = QNetworkAccessManagerPrivate::getObjectCache(this);
-        cache->releaseEntry(cacheKey);
+
+        // synchronous calls are not put into the cache, so for them the key is empty
+        if (!cacheKey.isEmpty())
+            cache->releaseEntry(cacheKey);
     }
 
     // This is abut disconnecting signals, not about disconnecting TCP connections
@@ -639,34 +643,49 @@ void QNetworkAccessHttpBackend::open()
     if (transparentProxy.type() == QNetworkProxy::DefaultProxy &&
         cacheProxy.type() == QNetworkProxy::DefaultProxy) {
         // unsuitable proxies
-        QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
-                                  Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProxyNotFoundError),
-                                  Q_ARG(QString, tr("No suitable proxy found")));
-        QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection);
-        return;
+        if (isSynchronous()) {
+            error(QNetworkReply::ProxyNotFoundError, tr("No suitable proxy found"));
+            finished();
+        } else {
+            QMetaObject::invokeMethod(this, "error", Qt::QueuedConnection,
+                                      Q_ARG(QNetworkReply::NetworkError, QNetworkReply::ProxyNotFoundError),
+                                      Q_ARG(QString, tr("No suitable proxy found")));
+            QMetaObject::invokeMethod(this, "finished", Qt::QueuedConnection);
+        }
+            return;
     }
 #endif
 
-    // check if we have an open connection to this host
-    cacheKey = makeCacheKey(this, theProxy);
-    QNetworkAccessCache *cache = QNetworkAccessManagerPrivate::getObjectCache(this);
-    // the http object is actually a QHttpNetworkConnection
-    http = static_cast<QNetworkAccessCachedHttpConnection *>(cache->requestEntryNow(cacheKey));
-    if (http == 0) {
-        // no entry in cache; create an object
-        // the http object is actually a QHttpNetworkConnection
-        http = new QNetworkAccessCachedHttpConnection(url.host(), url.port(), encrypt);
-
+    if (isSynchronous()) {
+        // for synchronous requests, we just create a new connection
+        http = new QHttpNetworkConnection(1, url.host(), url.port(), encrypt, this);
 #ifndef QT_NO_NETWORKPROXY
         http->setTransparentProxy(transparentProxy);
         http->setCacheProxy(cacheProxy);
 #endif
+        postRequest();
+        processRequestSynchronously();
+    } else {
+        // check if we have an open connection to this host
+        cacheKey = makeCacheKey(this, theProxy);
+        QNetworkAccessCache *cache = QNetworkAccessManagerPrivate::getObjectCache(this);
+        // the http object is actually a QHttpNetworkConnection
+        http = static_cast<QNetworkAccessCachedHttpConnection *>(cache->requestEntryNow(cacheKey));
+        if (http == 0) {
+            // no entry in cache; create an object
+            // the http object is actually a QHttpNetworkConnection
+            http = new QNetworkAccessCachedHttpConnection(url.host(), url.port(), encrypt);
 
-        // cache the QHttpNetworkConnection corresponding to this cache key
-        cache->addEntry(cacheKey, http);
-    }
+#ifndef QT_NO_NETWORKPROXY
+            http->setTransparentProxy(transparentProxy);
+            http->setCacheProxy(cacheProxy);
+#endif
 
-    postRequest();
+            // cache the QHttpNetworkConnection corresponding to this cache key
+            cache->addEntry(cacheKey, static_cast<QNetworkAccessCachedHttpConnection *>(http.data()));
+        }
+        postRequest();
+    }
 }
 
 void QNetworkAccessHttpBackend::closeDownstreamChannel()
@@ -1125,6 +1144,87 @@ void QNetworkAccessHttpBackend::setResumeOffset(quint64 offset)
     resumeOffset = offset;
 }
 
+bool QNetworkAccessHttpBackend::processRequestSynchronously()
+{
+    QHttpNetworkConnectionChannel *channel = &http->channels()[0];
+
+    // Disconnect all socket signals. They will only confuse us when using waitFor*
+    QObject::disconnect(channel->socket, 0, 0, 0);
+
+    qint64 timeout = 20*1000; // 20 sec
+    QElapsedTimer timeoutTimer;
+
+    bool waitResult = channel->socket->waitForConnected(timeout);
+    timeoutTimer.start();
+
+    if (!waitResult || channel->socket->state() != QAbstractSocket::Connected) {
+        error(QNetworkReply::UnknownNetworkError, QLatin1String("could not connect"));
+        return false;
+    }
+    channel->_q_connected(); // this will send the request (via sendRequest())
+
+#ifndef QT_NO_OPENSSL
+    if (http->isSsl()) {
+        qint64 remainingTimeEncrypted = timeout - timeoutTimer.elapsed();
+        if (!static_cast<QSslSocket *>(channel->socket)->waitForEncrypted(remainingTimeEncrypted)) {
+            error(QNetworkReply::SslHandshakeFailedError,
+                  QLatin1String("could not encrypt or timeout while encrypting"));
+            return false;
+        }
+        channel->_q_encrypted();
+    }
+#endif
+
+    // if we get a 401 or 407, we might need to send the request twice, see below
+    bool authenticating = false;
+
+    do {
+        channel->sendRequest();
+
+        qint64 remainingTimeBytesWritten;
+        while(channel->socket->bytesToWrite() > 0 ||
+              channel->state == QHttpNetworkConnectionChannel::WritingState) {
+            remainingTimeBytesWritten = timeout - timeoutTimer.elapsed();
+            channel->sendRequest(); // triggers channel->socket->write()
+            if (!channel->socket->waitForBytesWritten(remainingTimeBytesWritten)) {
+                error(QNetworkReply::TimeoutError,
+                      QLatin1String("could not write bytes to socket or timeout while writing"));
+                return false;
+            }
+        }
+
+        qint64 remainingTimeBytesRead = timeout - timeoutTimer.elapsed();
+        // Loop for at most remainingTime until either the socket disconnects
+        // or the reply is finished
+        do {
+            waitResult = channel->socket->waitForReadyRead(remainingTimeBytesRead);
+            remainingTimeBytesRead = timeout - timeoutTimer.elapsed();
+            if (!waitResult || remainingTimeBytesRead <= 0
+                || channel->socket->state() != QAbstractSocket::ConnectedState) {
+                error(QNetworkReply::TimeoutError,
+                      QLatin1String("could not read from socket or timeout while reading"));
+                return false;
+            }
+
+            if (channel->socket->bytesAvailable())
+                channel->_q_readyRead();
+
+            if (!httpReply)
+                return false; // we got a 401 or 407 and cannot handle it (it might happen that
+                              // disconnectFromHttp() was called, in that case the reply is zero)
+            // ### I am quite sure this does not work for NTLM
+            // ### how about uploading to an auth / proxyAuth site?
+
+            authenticating = (httpReply->statusCode() == 401 || httpReply->statusCode() == 407);
+
+            if (httpReply->isFinished())
+                break;
+        } while (remainingTimeBytesRead > 0);
+    } while (authenticating);
+
+    return true;
+}
+
 QT_END_NAMESPACE
 
 #endif // QT_NO_HTTP
diff --git a/src/network/access/qnetworkaccesshttpbackend_p.h b/src/network/access/qnetworkaccesshttpbackend_p.h
index 5de3429..cc2f9ac 100644
--- a/src/network/access/qnetworkaccesshttpbackend_p.h
+++ b/src/network/access/qnetworkaccesshttpbackend_p.h
@@ -99,6 +99,8 @@ public:
     bool canResume() const;
     void setResumeOffset(quint64 offset);
 
+    virtual bool processRequestSynchronously();
+
 private slots:
     void replyReadyRead();
     void replyFinished();
@@ -111,7 +113,7 @@ private slots:
 
 private:
     QHttpNetworkReply *httpReply;
-    QPointer<QNetworkAccessCachedHttpConnection> http;
+    QPointer<QHttpNetworkConnection> http;
     QByteArray cacheKey;
     QNetworkAccessBackendUploadIODevice *uploadDevice;
 
diff --git a/src/network/access/qnetworkaccessmanager.cpp b/src/network/access/qnetworkaccessmanager.cpp
index effd79e..4bc036e 100644
--- a/src/network/access/qnetworkaccessmanager.cpp
+++ b/src/network/access/qnetworkaccessmanager.cpp
@@ -1060,12 +1060,14 @@ QNetworkReply *QNetworkAccessManager::createRequest(QNetworkAccessManager::Opera
         priv->backend->setParent(reply);
         priv->backend->reply = priv;
     }
-    // fourth step: setup the reply
-    priv->setup(op, request, outgoingData);
 
 #ifndef QT_NO_OPENSSL
     reply->setSslConfiguration(request.sslConfiguration());
 #endif
+
+    // fourth step: setup the reply
+    priv->setup(op, request, outgoingData);
+
     return reply;
 }
 
@@ -1141,6 +1143,11 @@ void QNetworkAccessManagerPrivate::authenticationRequired(QNetworkAccessBackend
         }
     }
 
+    // if we emit a signal here in synchronous mode, the user might spin
+    // an event loop, which might recurse and lead to problems
+    if (backend->isSynchronous())
+        return;
+
     backend->reply->urlForLastAuthentication = url;
     emit q->authenticationRequired(backend->reply->q_func(), authenticator);
     cacheCredentials(url, authenticator);
@@ -1168,6 +1175,11 @@ void QNetworkAccessManagerPrivate::proxyAuthenticationRequired(QNetworkAccessBac
         }
     }
 
+    // if we emit a signal here in synchronous mode, the user might spin
+    // an event loop, which might recurse and lead to problems
+    if (backend->isSynchronous())
+        return;
+
     backend->reply->lastProxyAuthentication = proxy;
     emit q->proxyAuthenticationRequired(proxy, authenticator);
     cacheProxyCredentials(proxy, authenticator);
diff --git a/src/network/access/qnetworkreplyimpl.cpp b/src/network/access/qnetworkreplyimpl.cpp
index 5850494..cf6e674 100644
--- a/src/network/access/qnetworkreplyimpl.cpp
+++ b/src/network/access/qnetworkreplyimpl.cpp
@@ -85,7 +85,7 @@ void QNetworkReplyImplPrivate::_q_startOperation()
     }
 
 #ifndef QT_NO_BEARERMANAGEMENT
-    if (!backend->start()) {
+    if (!backend->start()) { // ### we should call that method even if bearer is not used
         // backend failed to start because the session state is not Connected.
         // QNetworkAccessManager will call reply->backend->start() again for us when the session
         // state changes.
@@ -109,11 +109,15 @@ void QNetworkReplyImplPrivate::_q_startOperation()
     }
 #endif
 
-    if (state != Finished) {
-        if (operation == QNetworkAccessManager::GetOperation)
-            pendingNotifications.append(NotifyDownstreamReadyWrite);
+    if (backend->isSynchronous()) {
+        state = Finished;
+    } else {
+        if (state != Finished) {
+            if (operation == QNetworkAccessManager::GetOperation)
+                pendingNotifications.append(NotifyDownstreamReadyWrite);
 
-        handleNotifications();
+            handleNotifications();
+        }
     }
 }
 
@@ -287,7 +291,25 @@ void QNetworkReplyImplPrivate::setup(QNetworkAccessManager::Operation op, const
     url = request.url();
     operation = op;
 
-    if (outgoingData && backend) {
+    q->QIODevice::open(QIODevice::ReadOnly);
+    // Internal code that does a HTTP reply for the synchronous Ajax
+    // in QtWebKit.
+    QVariant synchronousHttpAttribute = req.attribute(
+            static_cast<QNetworkRequest::Attribute>(QNetworkRequest::DownloadBufferAttribute + 1));
+    if (synchronousHttpAttribute.toBool()) {
+        backend->setSynchronous(true);
+        if (outgoingData && outgoingData->isSequential()) {
+            outgoingDataBuffer = new QRingBuffer();
+            QByteArray data;
+            do {
+                data = outgoingData->readAll();
+                if (data.isEmpty())
+                    break;
+                outgoingDataBuffer->append(data);
+            } while (1);
+        }
+    }
+    if (outgoingData && backend && !backend->isSynchronous()) {
         // there is data to be uploaded, e.g. HTTP POST.
 
         if (!backend->needsResetableUploadData() || !outgoingData->isSequential()) {
@@ -298,7 +320,7 @@ void QNetworkReplyImplPrivate::setup(QNetworkAccessManager::Operation op, const
         } else {
             bool bufferingDisallowed =
                     req.attribute(QNetworkRequest::DoNotBufferUploadDataAttribute,
-                                             false).toBool();
+                                  false).toBool();
 
             if (bufferingDisallowed) {
                 // if a valid content-length header for the request was supplied, we can disable buffering
@@ -323,17 +345,18 @@ void QNetworkReplyImplPrivate::setup(QNetworkAccessManager::Operation op, const
         // for HTTP, we want to send out the request as fast as possible to the network, without
         // invoking methods in a QueuedConnection
 #ifndef QT_NO_HTTP
-        if (qobject_cast<QNetworkAccessHttpBackend *>(backend)) {
+        if (qobject_cast<QNetworkAccessHttpBackend *>(backend) || (backend && backend->isSynchronous())) {
             _q_startOperation();
         } else {
             QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection);
         }
 #else
-        QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection);
+        if (backend->isSynchronous())
+            _q_startOperation();
+        else
+            QMetaObject::invokeMethod(q, "_q_startOperation", Qt::QueuedConnection);
 #endif // QT_NO_HTTP
-    }
-
-    q->QIODevice::open(QIODevice::ReadOnly);
+        }
 }
 
 void QNetworkReplyImplPrivate::backendNotify(InternalNotifications notification)
diff --git a/src/network/access/qnetworkrequest.h b/src/network/access/qnetworkrequest.h
index cdadf0f..9bcc900 100644
--- a/src/network/access/qnetworkrequest.h
+++ b/src/network/access/qnetworkrequest.h
@@ -85,6 +85,9 @@ public:
         MaximumDownloadBufferSizeAttribute, // internal
         DownloadBufferAttribute, // internal
 
+        // (DownloadBufferAttribute + 1) is reserved internal for QSynchronousHttpNetworkReply
+        // add the enum in 4.8
+
         User = 1000,
         UserMax = 32767
     };