QNetworkDiskCache: change file organization

Faster disk cache for mobile devices. Reduce file-system touching
operations and work around FAT performance issues. Features new
on-disk layout. Cached objects are now saved in multiple
subdirectories and filenames are shorter in length.

Merge-Request: 2505
Reviewed-by: Peter Hartmann

4 files changed, 78 insertions, 44 deletions

diff --git a/src/network/access/qnetworkdiskcache.cpp b/src/network/access/qnetworkdiskcache.cpp
index 2040b01..a5c0b3d 100644
--- a/src/network/access/qnetworkdiskcache.cpp
+++ b/src/network/access/qnetworkdiskcache.cpp
@@ -50,13 +50,15 @@
 #include <qdir.h>
 #include <qdatetime.h>
 #include <qdiriterator.h>
-#include <qcryptographichash.h>
 #include <qurl.h>
-
+#include <qcryptographichash.h>
 #include <qdebug.h>
 
-#define CACHE_PREFIX  QLatin1String("cache_")
-#define CACHE_POSTFIX QLatin1String(".cache")
+#define CACHE_POSTFIX QLatin1String(".d")
+#define PREPARED_SLASH QLatin1String("prepared/")
+#define CACHE_VERSION 7
+#define DATA_DIR QLatin1String("data")
+
 #define MAX_COMPRESSION_SIZE (1024 * 1024 * 3)
 
 #ifndef QT_NO_NETWORKDISKCACHE
@@ -153,6 +155,9 @@ void QNetworkDiskCache::setCacheDirectory(const QString &cacheDir)
     d->cacheDirectory = dir.absolutePath();
     if (!d->cacheDirectory.endsWith(QLatin1Char('/')))
         d->cacheDirectory += QLatin1Char('/');
+
+    d->dataDirectory = d->cacheDirectory + DATA_DIR + QString::number(CACHE_VERSION) + QLatin1Char('/');
+    d->prepareLayout();
 }
 
 /*!
@@ -244,6 +249,27 @@ void QNetworkDiskCache::insert(QIODevice *device)
     d->inserting.erase(it);
 }
 
+
+/*!
+    Create subdirectories and other housekeeping on the filesystem.
+    Prevents too many files from being present in any single directory.
+*/
+void QNetworkDiskCachePrivate::prepareLayout()
+{
+    QDir helper;
+    helper.mkpath(cacheDirectory + PREPARED_SLASH);
+
+    //Create directory and subdirectories 0-F
+    helper.mkpath(dataDirectory);
+    for ( uint i = 0; i < 16 ; i++ ) {
+        QString str = QString::number(i, 16);
+        QString subdir = dataDirectory + str;
+        helper.mkdir(subdir);
+    }
+
+}
+
+
 void QNetworkDiskCachePrivate::storeItem(QCacheItem *cacheItem)
 {
     Q_Q(QNetworkDiskCache);
@@ -324,7 +350,7 @@ bool QNetworkDiskCachePrivate::removeFile(const QString &file)
         return false;
     QFileInfo info(file);
     QString fileName = info.fileName();
-    if (!fileName.endsWith(CACHE_POSTFIX) || !fileName.startsWith(CACHE_PREFIX))
+    if (!fileName.endsWith(CACHE_POSTFIX))
         return false;
     qint64 size = info.size();
     if (QFile::remove(file)) {
@@ -409,7 +435,6 @@ QIODevice *QNetworkDiskCache::data(const QUrl &url)
 #endif
             if (p) {
                 buffer->setData((const char *)p, size);
-                file.take()->setParent(buffer.data());
             } else {
                 buffer->setData(file->readAll());
             }
@@ -508,6 +533,9 @@ qint64 QNetworkDiskCache::expire()
         return 0;
     }
 
+    // close file handle to prevent "in use" error when QFile::remove() is called
+    d->lastItem.reset();
+
     QDir::Filters filters = QDir::AllDirs | QDir:: Files | QDir::NoDotAndDotDot;
     QDirIterator it(cacheDirectory(), filters, QDirIterator::Subdirectories);
 
@@ -517,7 +545,7 @@ qint64 QNetworkDiskCache::expire()
         QString path = it.next();
         QFileInfo info = it.fileInfo();
         QString fileName = info.fileName();
-        if (fileName.endsWith(CACHE_POSTFIX) && fileName.startsWith(CACHE_PREFIX)) {
+        if (fileName.endsWith(CACHE_POSTFIX)) {
             cacheItems.insert(info.created(), path);
             totalSize += info.size();
         }
@@ -544,8 +572,6 @@ qint64 QNetworkDiskCache::expire()
                 << "Kept:" << cacheItems.count() - removedFiles;
     }
 #endif
-    if (removedFiles > 0)
-        d->lastItem.reset();
     return totalSize;
 }
 
@@ -564,7 +590,10 @@ void QNetworkDiskCache::clear()
     d->maximumCacheSize = size;
 }
 
-QByteArray QNetworkDiskCachePrivate::generateId(const QUrl &url) const
+/*!
+    Given a URL, generates a unique enough filename (and subdirectory)
+ */
+QString QNetworkDiskCachePrivate::uniqueFileName(const QUrl &url)
 {
     QUrl cleanUrl = url;
     cleanUrl.setPassword(QString());
@@ -572,29 +601,32 @@ QByteArray QNetworkDiskCachePrivate::generateId(const QUrl &url) const
 
     QCryptographicHash hash(QCryptographicHash::Sha1);
     hash.addData(cleanUrl.toEncoded());
-    return hash.result().toHex();
+    // convert sha1 to base36 form and return first 8 bytes for use as string
+    QByteArray id =  QByteArray::number(*(qlonglong*)hash.result().data(), 36).left(8);
+    // generates <one-char subdir>/<8-char filname.d>
+    uint code = (uint)id.at(id.length()-1) % 16;
+    QString pathFragment = QString::number(code, 16) + QLatin1String("/")
+                             + QLatin1String(id) + CACHE_POSTFIX;
+
+    return pathFragment;
 }
 
 QString QNetworkDiskCachePrivate::tmpCacheFileName() const
 {
-    QDir dir;
-    dir.mkpath(cacheDirectory + QLatin1String("prepared/"));
-    return cacheDirectory + QLatin1String("prepared/") + CACHE_PREFIX + QLatin1String("XXXXXX") + CACHE_POSTFIX;
+    //The subdirectory is presumed to be already read for use.
+    return cacheDirectory + PREPARED_SLASH + QLatin1String("XXXXXX") + CACHE_POSTFIX;
 }
 
+/*!
+    Genrates fully qualified path of cached resource from a URL.
+ */
 QString QNetworkDiskCachePrivate::cacheFileName(const QUrl &url) const
 {
     if (!url.isValid())
         return QString();
-    QString directory = cacheDirectory + url.scheme() + QLatin1Char('/');
-    if (!QFile::exists(directory)) {
-        // ### make a static QDir function for this...
-        QDir dir;
-        dir.mkpath(directory);
-    }
 
-    QString fileName = CACHE_PREFIX + QLatin1String(generateId(url)) + CACHE_POSTFIX;
-    return  directory + fileName;
+    QString fullpath = dataDirectory + uniqueFileName(url);
+    return  fullpath;
 }
 
 /*!
@@ -631,7 +663,7 @@ bool QCacheItem::canCompress() const
 enum
 {
     CacheMagic = 0xe8,
-    CurrentCacheVersion = 7
+    CurrentCacheVersion = CACHE_VERSION
 };
 
 void QCacheItem::writeHeader(QFile *device) const
@@ -682,6 +714,12 @@ bool QCacheItem::read(QFile *device, bool readData)
         data.setData(qUncompress(dataBA));
         data.open(QBuffer::ReadOnly);
     }
+
+    // quick and dirty check if metadata's URL field and the file's name are in synch
+    QString expectedFilename = QNetworkDiskCachePrivate::uniqueFileName(metaData.url());
+    if (!device->fileName().endsWith(expectedFilename))
+        return false;
+
     return metaData.isValid();
 }
 
diff --git a/src/network/access/qnetworkdiskcache_p.h b/src/network/access/qnetworkdiskcache_p.h
index 8659066..13db04f 100644
--- a/src/network/access/qnetworkdiskcache_p.h
+++ b/src/network/access/qnetworkdiskcache_p.h
@@ -104,14 +104,17 @@ public:
         , currentCacheSize(-1)
         {}
 
-    QByteArray generateId(const QUrl &url) const;
+    static QString uniqueFileName(const QUrl &url);
     QString cacheFileName(const QUrl &url) const;
     QString tmpCacheFileName() const;
     bool removeFile(const QString &file);
     void storeItem(QCacheItem *item);
+    void prepareLayout();
+    static quint32 crc32(const char *data, uint len);
 
     mutable QCacheItem lastItem;
     QString cacheDirectory;
+    QString dataDirectory;
     qint64 maximumCacheSize;
     qint64 currentCacheSize;
 
diff --git a/tests/auto/declarative/qdeclarativedom/data/importlib/sublib/qmldir b/tests/auto/declarative/qdeclarativedom/data/importlib/sublib/qmldir
index 98d6b74..e69de29 100644
--- a/tests/auto/declarative/qdeclarativedom/data/importlib/sublib/qmldir
+++ b/tests/auto/declarative/qdeclarativedom/data/importlib/sublib/qmldir
@@ -1,2 +0,0 @@
-Foo 1.1 Foo.qml
-Foo 1.0 Foo.qml
diff --git a/tests/auto/qnetworkdiskcache/tst_qnetworkdiskcache.cpp b/tests/auto/qnetworkdiskcache/tst_qnetworkdiskcache.cpp
index e974fcc..91b0164 100644
--- a/tests/auto/qnetworkdiskcache/tst_qnetworkdiskcache.cpp
+++ b/tests/auto/qnetworkdiskcache/tst_qnetworkdiskcache.cpp
@@ -46,6 +46,8 @@
 #include "../../shared/util.h"
 
 #define EXAMPLE_URL "http://user:pass@www.example.com/#foo"
+//cached objects are organized into these many subdirs
+#define NUM_SUBDIRECTORIES 16
 
 class tst_QNetworkDiskCache : public QObject
 {
@@ -176,8 +178,7 @@ void tst_QNetworkDiskCache::initTestCase()
     cache.clear();
     QString s = QDir::tempPath() + "/diskCache/";
     QDir dir;
-    dir.rmdir(s + "http");
-    dir.rmdir(s + "https");
+    dir.rmdir(s + "data7"); // the number is the internal cache version
     dir.rmdir(s + "prepared");
     dir.rmdir(s);
 }
@@ -277,16 +278,16 @@ void tst_QNetworkDiskCache::clear()
     QVERIFY(cache.cacheSize() > qint64(0));
 
     QString cacheDirectory = cache.cacheDirectory();
-    QCOMPARE(countFiles(cacheDirectory).count(), 3);
+    QCOMPARE(countFiles(cacheDirectory).count(), NUM_SUBDIRECTORIES + 3);
     cache.clear();
-    QCOMPARE(countFiles(cacheDirectory).count(), 2);
+    QCOMPARE(countFiles(cacheDirectory).count(), NUM_SUBDIRECTORIES + 2);
 
     // don't delete files that it didn't create
-    QTemporaryFile file(cacheDirectory + "/cache_XXXXXX");
+    QTemporaryFile file(cacheDirectory + "/XXXXXX");
     if (file.open()) {
-        QCOMPARE(countFiles(cacheDirectory).count(), 3);
+        QCOMPARE(countFiles(cacheDirectory).count(), NUM_SUBDIRECTORIES + 3);
         cache.clear();
-        QCOMPARE(countFiles(cacheDirectory).count(), 3);
+        QCOMPARE(countFiles(cacheDirectory).count(), NUM_SUBDIRECTORIES + 3);
     }
 }
 
@@ -317,12 +318,6 @@ void tst_QNetworkDiskCache::data()
     QUrl url(EXAMPLE_URL);
     cache.setupWithOne(url, data);
 
-    // flush the cache
-    QTemporaryFile file(cache.cacheDirectory() + "/cache_XXXXXX.cache");
-    if (file.open()) {
-        cache.call_fileMetaData(file.fileName());
-    }
-
     for (int i = 0; i < 3; ++i) {
         QIODevice *d = cache.data(url);
         QVERIFY(d);
@@ -362,9 +357,9 @@ void tst_QNetworkDiskCache::remove()
     QUrl url(EXAMPLE_URL);
     cache.setupWithOne(url);
     QString cacheDirectory = cache.cacheDirectory();
-    QCOMPARE(countFiles(cacheDirectory).count(), 3);
+    QCOMPARE(countFiles(cacheDirectory).count(), NUM_SUBDIRECTORIES + 3);
     cache.remove(url);
-    QCOMPARE(countFiles(cacheDirectory).count(), 2);
+    QCOMPARE(countFiles(cacheDirectory).count(), NUM_SUBDIRECTORIES + 2);
 }
 
 void tst_QNetworkDiskCache::setCacheDirectory_data()
@@ -414,7 +409,7 @@ void tst_QNetworkDiskCache::fileMetaData()
 
     QString cacheDirectory = cache.cacheDirectory();
     QStringList list = countFiles(cacheDirectory);
-    QCOMPARE(list.count(), 3);
+    QCOMPARE(list.count(), NUM_SUBDIRECTORIES + 3);
     foreach(QString fileName, list) {
         QFileInfo info(fileName);
         if (info.isFile()) {
@@ -491,7 +486,7 @@ void tst_QNetworkDiskCache::oldCacheVersionFile()
     if (pass == 0) {
         QString name;
         {
-        QTemporaryFile file(cache.cacheDirectory() + "/cache_XXXXXX.cache");
+        QTemporaryFile file(cache.cacheDirectory() + "/XXXXXX.d");
         file.setAutoRemove(false);
         QVERIFY(file.open());
         QDataStream out(&file);
@@ -507,7 +502,7 @@ void tst_QNetworkDiskCache::oldCacheVersionFile()
         QVERIFY(!QFile::exists(name));
     } else {
         QStringList files = countFiles(cache.cacheDirectory());
-        QCOMPARE(files.count(), 3);
+        QCOMPARE(files.count(), NUM_SUBDIRECTORIES + 3);
         // find the file
         QString cacheFile;
         foreach (QString file, files) {