Merge branch 'master' of scm.dev.troll.no:qt/qt-earth-team into symbian-socket-engine

Conflicts:
	src/s60installs/bwins/QtCoreu.def
	src/s60installs/bwins/QtGuiu.def
	src/s60installs/bwins/QtNetworku.def
	src/s60installs/eabi/QtCoreu.def
	src/s60installs/eabi/QtGuiu.def
	src/s60installs/eabi/QtNetworku.def
	src/s60installs/eabi/QtOpenVGu.def
	tests/auto/qabstractnetworkcache/tst_qabstractnetworkcache.cpp

6 files changed, 96 insertions, 47 deletions

diff --git a/src/network/access/qftp.cpp b/src/network/access/qftp.cpp
index 45fc11f..4ff45ba 100644
--- a/src/network/access/qftp.cpp
+++ b/src/network/access/qftp.cpp
@@ -371,9 +371,9 @@ qint64 QFtpDTP::read(char *data, qint64 maxlen)
     if (socket && socket->state() == QTcpSocket::ConnectedState) {
         read = socket->read(data, maxlen);
     } else {
-        read = bytesFromSocket.size();
+        read = qMin(maxlen, qint64(bytesFromSocket.size()));
         memcpy(data, bytesFromSocket.data(), read);
-        bytesFromSocket.clear();
+        bytesFromSocket.remove(0, read);
     }
 
     bytesDone += read;
diff --git a/src/network/access/qhttpthreaddelegate.cpp b/src/network/access/qhttpthreaddelegate.cpp
index 73277e3..7a17a7f 100644
--- a/src/network/access/qhttpthreaddelegate.cpp
+++ b/src/network/access/qhttpthreaddelegate.cpp
@@ -79,6 +79,11 @@ static QNetworkReply::NetworkError statusCodeFromHttp(int httpStatusCode, const
         code = QNetworkReply::ProxyAuthenticationRequiredError;
         break;
 
+    case 418:               // I'm a teapot
+        code = QNetworkReply::ProtocolInvalidOperationError;
+        break;
+
+
     default:
         if (httpStatusCode > 500) {
             // some kind of server error
diff --git a/src/network/access/qnetworkaccesshttpbackend.cpp b/src/network/access/qnetworkaccesshttpbackend.cpp
index 8016ef4..b0ca9e0 100644
--- a/src/network/access/qnetworkaccesshttpbackend.cpp
+++ b/src/network/access/qnetworkaccesshttpbackend.cpp
@@ -220,7 +220,7 @@ QNetworkAccessHttpBackend::~QNetworkAccessHttpBackend()
     2) If we have a cache entry for this url populate headers so the server can return 304
     3) Calculate if response_is_fresh and if so send the cache and set loadedFromCache to true
  */
-void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache)
+bool QNetworkAccessHttpBackend::loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest)
 {
     QNetworkRequest::CacheLoadControl CacheLoadControlAttribute =
         (QNetworkRequest::CacheLoadControl)request().attribute(QNetworkRequest::CacheLoadControlAttribute, QNetworkRequest::PreferNetwork).toInt();
@@ -231,24 +231,24 @@ void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest,
             httpRequest.setHeaderField("Cache-Control", "no-cache");
             httpRequest.setHeaderField("Pragma", "no-cache");
         }
-        return;
+        return false;
     }
 
     // The disk cache API does not currently support partial content retrieval.
     // That is why we don't use the disk cache for any such requests.
     if (request().hasRawHeader("Range"))
-        return;
+        return false;
 
     QAbstractNetworkCache *nc = networkCache();
     if (!nc)
-        return;                 // no local cache
+        return false;                 // no local cache
 
     QNetworkCacheMetaData metaData = nc->metaData(url());
     if (!metaData.isValid())
-        return;                 // not in cache
+        return false;                 // not in cache
 
     if (!metaData.saveToDisk())
-        return;
+        return false;
 
     QNetworkHeadersPrivate cacheHeaders;
     QNetworkHeadersPrivate::RawHeadersList::ConstIterator it;
@@ -263,11 +263,16 @@ void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest,
         httpRequest.setHeaderField("If-Modified-Since", QNetworkHeadersPrivate::toHttpDate(lastModified));
 
     if (CacheLoadControlAttribute == QNetworkRequest::PreferNetwork) {
+        // PreferNetwork == send request with "If-None-Match" and "If-Modified-Since" header,
+        // which will return a 304 Not Modifed if resource has not been changed.
+        // We might read from cache later, if receiving a 304.
+        return false;
+    } else if (CacheLoadControlAttribute == QNetworkRequest::PreferCache) {
         it = cacheHeaders.findRawHeader("Cache-Control");
         if (it != cacheHeaders.rawHeaders.constEnd()) {
             QHash<QByteArray, QByteArray> cacheControl = parseHttpOptionHeader(it->second);
             if (cacheControl.contains("must-revalidate"))
-                return;
+                return false;
         }
     }
 
@@ -339,14 +344,12 @@ void QNetworkAccessHttpBackend::validateCache(QHttpNetworkRequest &httpRequest,
 #endif
 
     if (!response_is_fresh)
-        return;
+        return false;
 
-    loadedFromCache = true;
 #if defined(QNETWORKACCESSHTTPBACKEND_DEBUG)
     qDebug() << "response_is_fresh" << CacheLoadControlAttribute;
 #endif
-    if (!sendCacheContents(metaData))
-        loadedFromCache = false;
+    return sendCacheContents(metaData);
 }
 
 static QHttpNetworkRequest::Priority convert(const QNetworkRequest::Priority& prio)
@@ -443,12 +446,12 @@ void QNetworkAccessHttpBackend::postRequest()
     switch (operation()) {
     case QNetworkAccessManager::GetOperation:
         httpRequest.setOperation(QHttpNetworkRequest::Get);
-        validateCache(httpRequest, loadedFromCache);
+        loadedFromCache = loadFromCacheIfAllowed(httpRequest);
         break;
 
     case QNetworkAccessManager::HeadOperation:
         httpRequest.setOperation(QHttpNetworkRequest::Head);
-        validateCache(httpRequest, loadedFromCache);
+        loadedFromCache = loadFromCacheIfAllowed(httpRequest);
         break;
 
     case QNetworkAccessManager::PostOperation:
@@ -480,6 +483,13 @@ void QNetworkAccessHttpBackend::postRequest()
         break;                  // can't happen
     }
 
+    if (loadedFromCache) {
+        // commented this out since it will be called later anyway
+        // by copyFinished()
+        //QNetworkAccessBackend::finished();
+        return;    // no need to send the request! :)
+    }
+
     QList<QByteArray> headers = request().rawHeaderList();
     if (resumeOffset != 0) {
         if (headers.contains("Range")) {
@@ -507,13 +517,6 @@ void QNetworkAccessHttpBackend::postRequest()
     foreach (const QByteArray &header, headers)
         httpRequest.setHeaderField(header, request().rawHeader(header));
 
-    if (loadedFromCache) {
-        // commented this out since it will be called later anyway
-        // by copyFinished()
-        //QNetworkAccessBackend::finished();
-        return;    // no need to send the request! :)
-    }
-
     if (request().attribute(QNetworkRequest::HttpPipeliningAllowedAttribute).toBool() == true)
         httpRequest.setPipeliningAllowed(true);
 
@@ -1153,7 +1156,7 @@ QNetworkCacheMetaData QNetworkAccessHttpBackend::fetchCacheMetaData(const QNetwo
         attributes.insert(QNetworkRequest::HttpStatusCodeAttribute, statusCode);
         attributes.insert(QNetworkRequest::HttpReasonPhraseAttribute, reasonPhrase);
     } else {
-        // this is a redirection, keep the attributes intact
+        // this is the server telling us the resource has not changed, keep the attributes intact
         attributes = oldMetaData.attributes();
     }
     metaData.setAttributes(attributes);
diff --git a/src/network/access/qnetworkaccesshttpbackend_p.h b/src/network/access/qnetworkaccesshttpbackend_p.h
index 712dd2f..4778bd0 100644
--- a/src/network/access/qnetworkaccesshttpbackend_p.h
+++ b/src/network/access/qnetworkaccesshttpbackend_p.h
@@ -148,7 +148,7 @@ private:
 
     quint64 resumeOffset;
 
-    void validateCache(QHttpNetworkRequest &httpRequest, bool &loadedFromCache);
+    bool loadFromCacheIfAllowed(QHttpNetworkRequest &httpRequest);
     void invalidateCache();
     void postRequest();
     void readFromHttp();
diff --git a/src/network/access/qnetworkdiskcache.cpp b/src/network/access/qnetworkdiskcache.cpp
index 2040b01..271494d 100644
--- a/src/network/access/qnetworkdiskcache.cpp
+++ b/src/network/access/qnetworkdiskcache.cpp
@@ -50,13 +50,15 @@
 #include <qdir.h>
 #include <qdatetime.h>
 #include <qdiriterator.h>
-#include <qcryptographichash.h>
 #include <qurl.h>
-
+#include <qcryptographichash.h>
 #include <qdebug.h>
 
-#define CACHE_PREFIX  QLatin1String("cache_")
-#define CACHE_POSTFIX QLatin1String(".cache")
+#define CACHE_POSTFIX QLatin1String(".d")
+#define PREPARED_SLASH QLatin1String("prepared/")
+#define CACHE_VERSION 7
+#define DATA_DIR QLatin1String("data")
+
 #define MAX_COMPRESSION_SIZE (1024 * 1024 * 3)
 
 #ifndef QT_NO_NETWORKDISKCACHE
@@ -153,6 +155,9 @@ void QNetworkDiskCache::setCacheDirectory(const QString &cacheDir)
     d->cacheDirectory = dir.absolutePath();
     if (!d->cacheDirectory.endsWith(QLatin1Char('/')))
         d->cacheDirectory += QLatin1Char('/');
+
+    d->dataDirectory = d->cacheDirectory + DATA_DIR + QString::number(CACHE_VERSION) + QLatin1Char('/');
+    d->prepareLayout();
 }
 
 /*!
@@ -244,6 +249,26 @@ void QNetworkDiskCache::insert(QIODevice *device)
     d->inserting.erase(it);
 }
 
+
+/*!
+    Create subdirectories and other housekeeping on the filesystem.
+    Prevents too many files from being present in any single directory.
+*/
+void QNetworkDiskCachePrivate::prepareLayout()
+{
+    QDir helper;
+    helper.mkpath(cacheDirectory + PREPARED_SLASH);
+
+    //Create directory and subdirectories 0-F
+    helper.mkpath(dataDirectory);
+    for (uint i = 0; i < 16 ; i++) {
+        QString str = QString::number(i, 16);
+        QString subdir = dataDirectory + str;
+        helper.mkdir(subdir);
+    }
+}
+
+
 void QNetworkDiskCachePrivate::storeItem(QCacheItem *cacheItem)
 {
     Q_Q(QNetworkDiskCache);
@@ -324,7 +349,7 @@ bool QNetworkDiskCachePrivate::removeFile(const QString &file)
         return false;
     QFileInfo info(file);
     QString fileName = info.fileName();
-    if (!fileName.endsWith(CACHE_POSTFIX) || !fileName.startsWith(CACHE_PREFIX))
+    if (!fileName.endsWith(CACHE_POSTFIX))
         return false;
     qint64 size = info.size();
     if (QFile::remove(file)) {
@@ -508,6 +533,9 @@ qint64 QNetworkDiskCache::expire()
         return 0;
     }
 
+    // close file handle to prevent "in use" error when QFile::remove() is called
+    d->lastItem.reset();
+
     QDir::Filters filters = QDir::AllDirs | QDir:: Files | QDir::NoDotAndDotDot;
     QDirIterator it(cacheDirectory(), filters, QDirIterator::Subdirectories);
 
@@ -517,7 +545,7 @@ qint64 QNetworkDiskCache::expire()
         QString path = it.next();
         QFileInfo info = it.fileInfo();
         QString fileName = info.fileName();
-        if (fileName.endsWith(CACHE_POSTFIX) && fileName.startsWith(CACHE_PREFIX)) {
+        if (fileName.endsWith(CACHE_POSTFIX)) {
             cacheItems.insert(info.created(), path);
             totalSize += info.size();
         }
@@ -544,8 +572,6 @@ qint64 QNetworkDiskCache::expire()
                 << "Kept:" << cacheItems.count() - removedFiles;
     }
 #endif
-    if (removedFiles > 0)
-        d->lastItem.reset();
     return totalSize;
 }
 
@@ -564,7 +590,10 @@ void QNetworkDiskCache::clear()
     d->maximumCacheSize = size;
 }
 
-QByteArray QNetworkDiskCachePrivate::generateId(const QUrl &url) const
+/*!
+    Given a URL, generates a unique enough filename (and subdirectory)
+ */
+QString QNetworkDiskCachePrivate::uniqueFileName(const QUrl &url)
 {
     QUrl cleanUrl = url;
     cleanUrl.setPassword(QString());
@@ -572,29 +601,32 @@ QByteArray QNetworkDiskCachePrivate::generateId(const QUrl &url) const
 
     QCryptographicHash hash(QCryptographicHash::Sha1);
     hash.addData(cleanUrl.toEncoded());
-    return hash.result().toHex();
+    // convert sha1 to base36 form and return first 8 bytes for use as string
+    QByteArray id =  QByteArray::number(*(qlonglong*)hash.result().data(), 36).left(8);
+    // generates <one-char subdir>/<8-char filname.d>
+    uint code = (uint)id.at(id.length()-1) % 16;
+    QString pathFragment = QString::number(code, 16) + QLatin1Char('/')
+                             + QLatin1String(id) + CACHE_POSTFIX;
+
+    return pathFragment;
 }
 
 QString QNetworkDiskCachePrivate::tmpCacheFileName() const
 {
-    QDir dir;
-    dir.mkpath(cacheDirectory + QLatin1String("prepared/"));
-    return cacheDirectory + QLatin1String("prepared/") + CACHE_PREFIX + QLatin1String("XXXXXX") + CACHE_POSTFIX;
+    //The subdirectory is presumed to be already read for use.
+    return cacheDirectory + PREPARED_SLASH + QLatin1String("XXXXXX") + CACHE_POSTFIX;
 }
 
+/*!
+    Generates fully qualified path of cached resource from a URL.
+ */
 QString QNetworkDiskCachePrivate::cacheFileName(const QUrl &url) const
 {
     if (!url.isValid())
         return QString();
-    QString directory = cacheDirectory + url.scheme() + QLatin1Char('/');
-    if (!QFile::exists(directory)) {
-        // ### make a static QDir function for this...
-        QDir dir;
-        dir.mkpath(directory);
-    }
 
-    QString fileName = CACHE_PREFIX + QLatin1String(generateId(url)) + CACHE_POSTFIX;
-    return  directory + fileName;
+    QString fullpath = dataDirectory + uniqueFileName(url);
+    return  fullpath;
 }
 
 /*!
@@ -631,7 +663,7 @@ bool QCacheItem::canCompress() const
 enum
 {
     CacheMagic = 0xe8,
-    CurrentCacheVersion = 7
+    CurrentCacheVersion = CACHE_VERSION
 };
 
 void QCacheItem::writeHeader(QFile *device) const
@@ -682,6 +714,12 @@ bool QCacheItem::read(QFile *device, bool readData)
         data.setData(qUncompress(dataBA));
         data.open(QBuffer::ReadOnly);
     }
+
+    // quick and dirty check if metadata's URL field and the file's name are in synch
+    QString expectedFilename = QNetworkDiskCachePrivate::uniqueFileName(metaData.url());
+    if (!device->fileName().endsWith(expectedFilename))
+        return false;
+
     return metaData.isValid();
 }
 
diff --git a/src/network/access/qnetworkdiskcache_p.h b/src/network/access/qnetworkdiskcache_p.h
index 8659066..13db04f 100644
--- a/src/network/access/qnetworkdiskcache_p.h
+++ b/src/network/access/qnetworkdiskcache_p.h
@@ -104,14 +104,17 @@ public:
         , currentCacheSize(-1)
         {}
 
-    QByteArray generateId(const QUrl &url) const;
+    static QString uniqueFileName(const QUrl &url);
     QString cacheFileName(const QUrl &url) const;
     QString tmpCacheFileName() const;
     bool removeFile(const QString &file);
     void storeItem(QCacheItem *item);
+    void prepareLayout();
+    static quint32 crc32(const char *data, uint len);
 
     mutable QCacheItem lastItem;
     QString cacheDirectory;
+    QString dataDirectory;
     qint64 maximumCacheSize;
     qint64 currentCacheSize;