QThreadPool: fix race at time of thread expiry.

The current synchronization mechanism was racy: decrementing waitingThreads
and then hoping that the wakeOne will wake a thread before its expiry
timeout happens. In other words, on timeout, a just-assigned task would
never run. And then no other task would run, if maxThreadCount is reached.

Fixed by using a queue of waiting threads (rather than just a count), and by
moving the wait condition into the thread itself, so we know precisely
which one we're waking up, and we can remove it from the set of waiting threads
before waking it up, and therefore it can determine on wakeup whether it
has work to do (caller removed it from the queue) or it expired (it's still
in the queue). This is reliable, whereas the return value from QWaitCondition::wait
isn't reliable, when the main thread has already decided that this thread
has work to do.

Task-number: QTBUG-3786
Backport from qtbase/a9b6a78e54670a70b96c122b10ad7bd64d166514

Change-Id: Ic766ff67dea7a8bb8f1bc893943060ee5428d782
Reviewed-by: Thiago Macieira <thiago.macieira@intel.com>

2 files changed, 13 insertions, 18 deletions

diff --git a/src/corelib/concurrent/qthreadpool.cpp b/src/corelib/concurrent/qthreadpool.cpp
index 26a5337..fb90f4e 100644
--- a/src/corelib/concurrent/qthreadpool.cpp
+++ b/src/corelib/concurrent/qthreadpool.cpp
@@ -68,6 +68,7 @@ public:
     void run();
     void registerThreadInactive();
 
+    QWaitCondition runnableReady;
     QThreadPoolPrivate *manager;
     QRunnable *runnable;
 };
@@ -135,14 +136,13 @@ void QThreadPoolThread::run()
         // if too many threads are active, expire this thread
         bool expired = manager->tooManyThreadsActive();
         if (!expired) {
-            ++manager->waitingThreads;
+            manager->waitingThreads.enqueue(this);
             registerThreadInactive();
             // wait for work, exiting after the expiry timeout is reached
-            expired = !manager->runnableReady.wait(locker.mutex(), manager->expiryTimeout);
+            runnableReady.wait(locker.mutex(), manager->expiryTimeout);
             ++manager->activeThreads;
-    
-            if (expired)
-                --manager->waitingThreads;
+            if (manager->waitingThreads.removeOne(this))
+                expired = true;
         }
         if (expired) {
             manager->expiredThreads.enqueue(this);
@@ -167,7 +167,6 @@ QThreadPoolPrivate:: QThreadPoolPrivate()
       expiryTimeout(30000),
       maxThreadCount(qAbs(QThread::idealThreadCount())),
       reservedThreads(0),
-      waitingThreads(0),
       activeThreads(0)
 { }
 
@@ -183,11 +182,10 @@ bool QThreadPoolPrivate::tryStart(QRunnable *task)
     if (activeThreadCount() >= maxThreadCount)
         return false;
 
-    if (waitingThreads > 0) {
+    if (waitingThreads.count() > 0) {
         // recycle an available thread
-        --waitingThreads;
         enqueueTask(task);
-        runnableReady.wakeOne();
+        waitingThreads.takeFirst()->runnableReady.wakeOne();
         return true;
     }
 
@@ -225,7 +223,7 @@ int QThreadPoolPrivate::activeThreadCount() const
 {
     return (allThreads.count()
             - expiredThreads.count()
-            - waitingThreads
+            - waitingThreads.count()
             + reservedThreads);
 }
 
@@ -266,7 +264,6 @@ void QThreadPoolPrivate::reset()
 {
     QMutexLocker locker(&mutex);
     isExiting = true;
-    runnableReady.wakeAll();
 
     do {
         // make a copy of the set so that we can iterate without the lock
@@ -275,6 +272,7 @@ void QThreadPoolPrivate::reset()
         locker.unlock();
 
         foreach (QThreadPoolThread *thread, allThreadsCopy) {
+            thread->runnableReady.wakeAll();
             thread->wait();
             delete thread;
         }
@@ -283,7 +281,7 @@ void QThreadPoolPrivate::reset()
         // repeat until all newly arrived threads have also completed
     } while (!allThreads.isEmpty());
 
-    waitingThreads = 0;
+    waitingThreads.clear();
     expiredThreads.clear();
 
     isExiting = false;
@@ -474,10 +472,8 @@ void QThreadPool::start(QRunnable *runnable, int priority)
     if (!d->tryStart(runnable)) {
         d->enqueueTask(runnable, priority);
 
-        if (d->waitingThreads > 0) {
-            --d->waitingThreads;
-            d->runnableReady.wakeOne();
-        }
+        if (!d->waitingThreads.isEmpty())
+            d->waitingThreads.takeFirst()->runnableReady.wakeOne();
     }
 }
 
diff --git a/src/corelib/concurrent/qthreadpool_p.h b/src/corelib/concurrent/qthreadpool_p.h
index 67d3606..3846333 100644
--- a/src/corelib/concurrent/qthreadpool_p.h
+++ b/src/corelib/concurrent/qthreadpool_p.h
@@ -87,8 +87,8 @@ public:
     void stealRunnable(QRunnable *);
 
     mutable QMutex mutex;
-    QWaitCondition runnableReady;
     QSet<QThreadPoolThread *> allThreads;
+    QQueue<QThreadPoolThread *> waitingThreads;
     QQueue<QThreadPoolThread *> expiredThreads;
     QList<QPair<QRunnable *, int> > queue;
     QWaitCondition noActiveThreads;
@@ -97,7 +97,6 @@ public:
     int expiryTimeout;
     int maxThreadCount;
     int reservedThreads;
-    int waitingThreads;
     int activeThreads;
 };