Out of memory fix for qUncompress

qUncompress shouldn't crash when running out of memory, since it might deal
with buffers which are not under user control (same behavior as Qt 4.5).
It will however throw a std::bad_alloc exception if Qt is compiled with
exception handling.

Reviewed-by: Harald Fernengel
Reviewed-by: Ralf Engels
Reviewed-by: Lars Knoll

3 files changed, 48 insertions, 23 deletions

diff --git a/src/corelib/tools/qbytearray.cpp b/src/corelib/tools/qbytearray.cpp
index 6d55ef9..d8557e5 100644
--- a/src/corelib/tools/qbytearray.cpp
+++ b/src/corelib/tools/qbytearray.cpp
@@ -47,6 +47,7 @@
 #include "qlocale.h"
 #include "qlocale_p.h"
 #include "qunicodetables_p.h"
+#include "qscopedpointer.h"
 #ifndef QT_NO_DATASTREAM
 #include <qdatastream.h>
 #endif
@@ -535,38 +536,49 @@ QByteArray qUncompress(const uchar* data, int nbytes)
     ulong expectedSize = (data[0] << 24) | (data[1] << 16) |
                        (data[2] <<  8) | (data[3]      );
     ulong len = qMax(expectedSize, 1ul);
-    QByteArray baunzip;
-    int res;
-    do {
-        QT_TRY {
-            baunzip.resize(len);
-            res = ::uncompress((uchar*)baunzip.data(), &len,
-                                (uchar*)data+4, nbytes-4);
-        } QT_CATCH (const std::bad_alloc &) {
-            res = Z_MEM_ERROR;
+    QScopedPointer<QByteArray::Data, QScopedPointerPodDeleter> d;
+
+    forever {
+        ulong alloc = len;
+        d.reset(q_check_ptr(static_cast<QByteArray::Data *>(qRealloc(d.data(), sizeof(QByteArray::Data) + alloc))));
+        if (!d) {
+            // we are not allowed to crash here when compiling with QT_NO_EXCEPTIONS
+            qWarning("qUncompress: could not allocate enough memory to uncompress data");
+            return QByteArray();
         }
 
+        int res = ::uncompress((uchar*)d->array, &len,
+                               (uchar*)data+4, nbytes-4);
+
         switch (res) {
         case Z_OK:
-            if ((int)len != baunzip.size())
-                baunzip.resize(len);
-            break;
+            if (len != alloc) {
+                d.reset(q_check_ptr(static_cast<QByteArray::Data *>(qRealloc(d.data(), sizeof(QByteArray::Data) + len))));
+                if (!d) {
+                    // we are not allowed to crash here when compiling with QT_NO_EXCEPTIONS
+                    qWarning("qUncompress: could not allocate enough memory to uncompress data");
+                    return QByteArray();
+                }
+            }
+            d->ref = 1;
+            d->alloc = d->size = len;
+            d->data = d->array;
+
+            return QByteArray(d.take(), 0, 0);
+
         case Z_MEM_ERROR:
             qWarning("qUncompress: Z_MEM_ERROR: Not enough memory");
-            break;
+            return QByteArray();
+
         case Z_BUF_ERROR:
             len *= 2;
-            break;
+            continue;
+
         case Z_DATA_ERROR:
             qWarning("qUncompress: Z_DATA_ERROR: Input data is corrupted");
-            break;
+            return QByteArray();
         }
-    } while (res == Z_BUF_ERROR);
-
-    if (res != Z_OK)
-        baunzip = QByteArray();
-
-    return baunzip;
+    }
 }
 #endif
 
diff --git a/src/corelib/tools/qbytearray.h b/src/corelib/tools/qbytearray.h
index 191f6a2..9621dbf 100644
--- a/src/corelib/tools/qbytearray.h
+++ b/src/corelib/tools/qbytearray.h
@@ -372,6 +372,7 @@ private:
 
     friend class QByteRef;
     friend class QString;
+    friend QByteArray qUncompress(const uchar *data, int nbytes);
 public:
     typedef Data * DataPtr;
     inline DataPtr &data_ptr() { return d; }
diff --git a/tests/auto/qbytearray/tst_qbytearray.cpp b/tests/auto/qbytearray/tst_qbytearray.cpp
index 44cfe4a..b0bf3df 100644
--- a/tests/auto/qbytearray/tst_qbytearray.cpp
+++ b/tests/auto/qbytearray/tst_qbytearray.cpp
@@ -236,6 +236,7 @@ void tst_QByteArray::qUncompress_data()
 void tst_QByteArray::qUncompress()
 {
     QFETCH(QByteArray, in);
+    QFETCH(QByteArray, out);
 
 #if defined Q_OS_HPUX && !defined __ia64 && defined Q_CC_GNU
     QSKIP("Corrupt data causes this tests to lock up on HP-UX / PA-RISC with gcc", SkipAll);
@@ -245,9 +246,20 @@ void tst_QByteArray::qUncompress()
     QSKIP("Corrupt data causes this test to lock up on QNX", SkipAll);
 #endif
 
-    QTEST(::qUncompress(in), "out");
+    QByteArray res;
+    QT_TRY {
+        res = ::qUncompress(in);
+    } QT_CATCH(const std::bad_alloc &) {
+        res = QByteArray();
+    }
+    QCOMPARE(res, out);
 
-    QTEST(::qUncompress(in + "blah"), "out");
+    QT_TRY {
+        res = ::qUncompress(in + "blah");
+    } QT_CATCH(const std::bad_alloc &) {
+        res = QByteArray();
+    }
+    QCOMPARE(res, out);
 }
 #endif