os.urandom() now blocks on Linux

Issue #27776: The os.urandom() function does now block on Linux 3.17 and newer
until the system urandom entropy pool is initialized to increase the security.

This change is part of the PEP 524.

8 files changed, 132 insertions, 61 deletions

diff --git a/Doc/library/os.rst b/Doc/library/os.rst
index 69c559c..e2b6e64 100644
--- a/Doc/library/os.rst
+++ b/Doc/library/os.rst
@@ -3968,14 +3968,27 @@ Random numbers
    returned data should be unpredictable enough for cryptographic applications,
    though its exact quality depends on the OS implementation.
 
-   On Linux, the ``getrandom()`` syscall is used if available and the urandom
-   entropy pool is initialized (``getrandom()`` does not block).
-   On a Unix-like system this will query ``/dev/urandom``. On Windows, it
-   will use ``CryptGenRandom()``.  If a randomness source is not found,
-   :exc:`NotImplementedError` will be raised.
-
-   For an easy-to-use interface to the random number generator
-   provided by your platform, please see :class:`random.SystemRandom`.
+   On Linux, if the ``getrandom()`` syscall is available, it is used in
+   blocking mode: block until the system urandom entropy pool is initialized
+   (128 bits of entropy are collected by the kernel). See the :pep:`524` for
+   the rationale. On Linux, the :func:`getrandom` function can be used to get
+   random bytes in non-blocking mode (using the :data:`GRND_NONBLOCK` flag) or
+   to poll until the system urandom entropy pool is initialized.
+
+   On a Unix-like system, random bytes are read from the ``/dev/urandom``
+   device. If the ``/dev/urandom`` device is not available or not readable, the
+   :exc:`NotImplementedError` exception is raised.
+
+   On Windows, it will use ``CryptGenRandom()``.
+
+   .. seealso::
+      The :mod:`secrets` module provides higher level functions. For an
+      easy-to-use interface to the random number generator provided by your
+      platform, please see :class:`random.SystemRandom`.
+
+   .. versionchanged:: 3.6.0
+      On Linux, ``getrandom()`` is now used in blocking mode to increase the
+      security.
 
    .. versionchanged:: 3.5.2
       On Linux, if the ``getrandom()`` syscall blocks (the urandom entropy pool
diff --git a/Doc/whatsnew/3.6.rst b/Doc/whatsnew/3.6.rst
index 683ea82..f40a3c0 100644
--- a/Doc/whatsnew/3.6.rst
+++ b/Doc/whatsnew/3.6.rst
@@ -70,6 +70,12 @@ Standard library improvements:
 
 * PEP 519: :ref:`Adding a file system path protocol <pep-519>`
 
+Security improvements:
+
+* On Linux, :func:`os.urandom` now blocks until the system urandom entropy pool
+  is initialized to increase the security. See the :pep:`524` for the
+  rationale.
+
 Windows improvements:
 
 * The ``py.exe`` launcher, when used interactively, no longer prefers
@@ -345,6 +351,9 @@ New Modules
 Improved Modules
 ================
 
+On Linux, :func:`os.urandom` now blocks until the system urandom entropy pool
+is initialized to increase the security. See the :pep:`524` for the rationale.
+
 
 asyncio
 -------
@@ -913,6 +922,9 @@ Changes in 'python' Command Behavior
 Changes in the Python API
 -------------------------
 
+* On Linux, :func:`os.urandom` now blocks until the system urandom entropy pool
+  is initialized to increase the security.
+
 * When :meth:`importlib.abc.Loader.exec_module` is defined,
   :meth:`importlib.abc.Loader.create_module` must also be defined.
 
diff --git a/Include/pylifecycle.h b/Include/pylifecycle.h
index e96eb70..cf149b2 100644
--- a/Include/pylifecycle.h
+++ b/Include/pylifecycle.h
@@ -117,7 +117,8 @@ PyAPI_FUNC(PyOS_sighandler_t) PyOS_getsig(int);
 PyAPI_FUNC(PyOS_sighandler_t) PyOS_setsig(int, PyOS_sighandler_t);
 
 /* Random */
-PyAPI_FUNC(int) _PyOS_URandom (void *buffer, Py_ssize_t size);
+PyAPI_FUNC(int) _PyOS_URandom(void *buffer, Py_ssize_t size);
+PyAPI_FUNC(int) _PyOS_URandomNonblock(void *buffer, Py_ssize_t size);
 
 #ifdef __cplusplus
 }
diff --git a/Lib/random.py b/Lib/random.py
index 5abcdbc..82f6013 100644
--- a/Lib/random.py
+++ b/Lib/random.py
@@ -105,15 +105,6 @@ class Random(_random.Random):
 
         """
 
-        if a is None:
-            try:
-                # Seed with enough bytes to span the 19937 bit
-                # state space for the Mersenne Twister
-                a = int.from_bytes(_urandom(2500), 'big')
-            except NotImplementedError:
-                import time
-                a = int(time.time() * 256) # use fractional seconds
-
         if version == 1 and isinstance(a, (str, bytes)):
             x = ord(a[0]) << 7 if a else 0
             for c in a:
diff --git a/Misc/NEWS b/Misc/NEWS
index c0c75c8..0438f07 100644
--- a/Misc/NEWS
+++ b/Misc/NEWS
@@ -89,6 +89,10 @@ Core and Builtins
 Library
 -------
 
+- Issue #27776: The :func:`os.urandom` function does now block on Linux 3.17
+  and newer until the system urandom entropy pool is initialized to increase
+  the security. This change is part of the :pep:`524`.
+
 - Issue #27778: Expose the Linux ``getrandom()`` syscall as a new
   :func:`os.getrandom` function. This change is part of the :pep:`524`.
 
diff --git a/Modules/_randommodule.c b/Modules/_randommodule.c
index 63a060c..1cf57ae 100644
--- a/Modules/_randommodule.c
+++ b/Modules/_randommodule.c
@@ -165,7 +165,7 @@ init_genrand(RandomObject *self, uint32_t s)
 /* initialize by an array with array-length */
 /* init_key is the array for initializing keys */
 /* key_length is its length */
-static PyObject *
+static void
 init_by_array(RandomObject *self, uint32_t init_key[], size_t key_length)
 {
     size_t i, j, k;       /* was signed in the original code. RDH 12/16/2002 */
@@ -190,8 +190,6 @@ init_by_array(RandomObject *self, uint32_t init_key[], size_t key_length)
     }
 
     mt[0] = 0x80000000U; /* MSB is 1; assuring non-zero initial array */
-    Py_INCREF(Py_None);
-    return Py_None;
 }
 
 /*
@@ -199,6 +197,37 @@ init_by_array(RandomObject *self, uint32_t init_key[], size_t key_length)
  * Twister download.
  */
 
+static int
+random_seed_urandom(RandomObject *self)
+{
+    PY_UINT32_T key[N];
+
+    if (_PyOS_URandomNonblock(key, sizeof(key)) < 0) {
+        return -1;
+    }
+    init_by_array(self, key, Py_ARRAY_LENGTH(key));
+    return 0;
+}
+
+static void
+random_seed_time_pid(RandomObject *self)
+{
+    _PyTime_t now;
+    uint32_t key[5];
+
+    now = _PyTime_GetSystemClock();
+    key[0] = (PY_UINT32_T)(now & 0xffffffffU);
+    key[1] = (PY_UINT32_T)(now >> 32);
+
+    key[2] = (PY_UINT32_T)getpid();
+
+    now = _PyTime_GetMonotonicClock();
+    key[3] = (PY_UINT32_T)(now & 0xffffffffU);
+    key[4] = (PY_UINT32_T)(now >> 32);
+
+    init_by_array(self, key, Py_ARRAY_LENGTH(key));
+}
+
 static PyObject *
 random_seed(RandomObject *self, PyObject *args)
 {
@@ -212,14 +241,17 @@ random_seed(RandomObject *self, PyObject *args)
     if (!PyArg_UnpackTuple(args, "seed", 0, 1, &arg))
         return NULL;
 
-    if (arg == NULL || arg == Py_None) {
-        time_t now;
+     if (arg == NULL || arg == Py_None) {
+        if (random_seed_urandom(self) >= 0) {
+            PyErr_Clear();
 
-        time(&now);
-        init_genrand(self, (uint32_t)now);
-        Py_INCREF(Py_None);
-        return Py_None;
+            /* Reading system entropy failed, fall back on the worst entropy:
+               use the current time and process identifier. */
+            random_seed_time_pid(self);
+        }
+        Py_RETURN_NONE;
     }
+
     /* This algorithm relies on the number being unsigned.
      * So: if the arg is a PyLong, use its absolute value.
      * Otherwise use its hash value, cast to unsigned.
@@ -269,7 +301,11 @@ random_seed(RandomObject *self, PyObject *args)
         }
     }
 #endif
-    result = init_by_array(self, key, keyused);
+    init_by_array(self, key, keyused);
+
+    Py_INCREF(Py_None);
+    result = Py_None;
+
 Done:
     Py_XDECREF(n);
     PyMem_Free(key);
diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c
index 2dc6d7b..0b9b3f6 100644
--- a/Modules/posixmodule.c
+++ b/Modules/posixmodule.c
@@ -11168,8 +11168,7 @@ os_urandom_impl(PyObject *module, Py_ssize_t size)
     if (bytes == NULL)
         return NULL;
 
-    result = _PyOS_URandom(PyBytes_AS_STRING(bytes),
-                        PyBytes_GET_SIZE(bytes));
+    result = _PyOS_URandom(PyBytes_AS_STRING(bytes), PyBytes_GET_SIZE(bytes));
     if (result == -1) {
         Py_DECREF(bytes);
         return NULL;
diff --git a/Python/random.c b/Python/random.c
index 6fdce64..ea506ee 100644
--- a/Python/random.c
+++ b/Python/random.c
@@ -77,7 +77,7 @@ win32_urandom(unsigned char *buffer, Py_ssize_t size, int raise)
 }
 
 /* Issue #25003: Don't use getentropy() on Solaris (available since
-   Solaris 11.3), it is blocking whereas os.urandom() should not block. */
+ * Solaris 11.3), it is blocking whereas os.urandom() should not block. */
 #elif defined(HAVE_GETENTROPY) && !defined(sun)
 #define PY_GETENTROPY 1
 
@@ -121,24 +121,20 @@ py_getentropy(char *buffer, Py_ssize_t size, int raise)
 
 /* Call getrandom()
    - Return 1 on success
-   - Return 0 if getrandom() syscall is not available (fails with ENOSYS).
+   - Return 0 if getrandom() syscall is not available (fails with ENOSYS)
+     or if getrandom(GRND_NONBLOCK) fails with EAGAIN (blocking=0 and system
+     urandom not initialized yet) and raise=0.
    - Raise an exception (if raise is non-zero) and return -1 on error:
      getrandom() failed with EINTR and the Python signal handler raised an
      exception, or getrandom() failed with a different error. */
 static int
-py_getrandom(void *buffer, Py_ssize_t size, int raise)
+py_getrandom(void *buffer, Py_ssize_t size, int blocking, int raise)
 {
-    /* Is getrandom() supported by the running kernel?
-       Need Linux kernel 3.17 or newer, or Solaris 11.3 or newer */
+    /* Is getrandom() supported by the running kernel? Set to 0 if getrandom()
+       fails with ENOSYS. Need Linux kernel 3.17 or newer, or Solaris 11.3
+       or newer */
     static int getrandom_works = 1;
-
-    /* getrandom() on Linux will block if called before the kernel has
-       initialized the urandom entropy pool. This will cause Python
-       to hang on startup if called very early in the boot process -
-       see https://bugs.python.org/issue26839. To avoid this, use the
-       GRND_NONBLOCK flag. */
-    const int flags = GRND_NONBLOCK;
-
+    int flags;
     char *dest;
     long n;
 
@@ -146,6 +142,7 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise)
         return 0;
     }
 
+    flags = blocking ? 0 : GRND_NONBLOCK;
     dest = buffer;
     while (0 < size) {
 #ifdef sun
@@ -185,15 +182,12 @@ py_getrandom(void *buffer, Py_ssize_t size, int raise)
                 getrandom_works = 0;
                 return 0;
             }
-            if (errno == EAGAIN) {
-                /* If we failed with EAGAIN, the entropy pool was
-                   uninitialized. In this case, we return failure to fall
-                   back to reading from /dev/urandom.
-
-                   Note: In this case the data read will not be random so
-                   should not be used for cryptographic purposes. Retaining
-                   the existing semantics for practical purposes. */
-                getrandom_works = 0;
+
+            /* getrandom(GRND_NONBLOCK) fails with EAGAIN if the system urandom
+               is not initialiazed yet. For _PyRandom_Init(), we ignore their
+               error and fall back on reading /dev/urandom which never blocks,
+               even if the system urandom is not initialized yet. */
+            if (errno == EAGAIN && !raise && !blocking) {
                 return 0;
             }
 
@@ -228,13 +222,13 @@ static struct {
 } urandom_cache = { -1 };
 
 
-/* Read 'size' random bytes from getrandom(). Fall back on reading from
+/* Read 'size' random bytes from py_getrandom(). Fall back on reading from
    /dev/urandom if getrandom() is not available.
 
    Return 0 on success. Raise an exception (if raise is non-zero) and return -1
    on error. */
 static int
-dev_urandom(char *buffer, Py_ssize_t size, int raise)
+dev_urandom(char *buffer, Py_ssize_t size, int blocking, int raise)
 {
     int fd;
     Py_ssize_t n;
@@ -245,7 +239,7 @@ dev_urandom(char *buffer, Py_ssize_t size, int raise)
     assert(size > 0);
 
 #ifdef PY_GETRANDOM
-    res = py_getrandom(buffer, size, raise);
+    res = py_getrandom(buffer, size, blocking, raise);
     if (res < 0) {
         return -1;
     }
@@ -381,7 +375,7 @@ lcg_urandom(unsigned int x0, unsigned char *buffer, size_t size)
      syscall)
    - Don't release the GIL to call syscalls. */
 static int
-pyurandom(void *buffer, Py_ssize_t size, int raise)
+pyurandom(void *buffer, Py_ssize_t size, int blocking, int raise)
 {
     if (size < 0) {
         if (raise) {
@@ -400,7 +394,7 @@ pyurandom(void *buffer, Py_ssize_t size, int raise)
 #elif defined(PY_GETENTROPY)
     return py_getentropy(buffer, size, raise);
 #else
-    return dev_urandom(buffer, size, raise);
+    return dev_urandom(buffer, size, blocking, raise);
 #endif
 }
 
@@ -408,11 +402,29 @@ pyurandom(void *buffer, Py_ssize_t size, int raise)
    number generator (RNG). It is suitable for most cryptographic purposes
    except long living private keys for asymmetric encryption.
 
-   Return 0 on success, raise an exception and return -1 on error. */
+   On Linux 3.17 and newer, the getrandom() syscall is used in blocking mode:
+   block until the system urandom entropy pool is initialized (128 bits are
+   collected by the kernel).
+
+   Return 0 on success. Raise an exception and return -1 on error. */
 int
 _PyOS_URandom(void *buffer, Py_ssize_t size)
 {
-    return pyurandom(buffer, size, 1);
+    return pyurandom(buffer, size, 1, 1);
+}
+
+/* Fill buffer with size pseudo-random bytes from the operating system random
+   number generator (RNG). It is not suitable for cryptographic purpose.
+
+   On Linux 3.17 and newer (when getrandom() syscall is used), if the system
+   urandom is not initialized yet, the function returns "weak" entropy read
+   from /dev/urandom.
+
+   Return 0 on success. Raise an exception and return -1 on error. */
+int
+_PyOS_URandomNonblock(void *buffer, Py_ssize_t size)
+{
+    return pyurandom(buffer, size, 0, 1);
 }
 
 void
@@ -456,8 +468,11 @@ _PyRandom_Init(void)
         int res;
 
         /* _PyRandom_Init() is called very early in the Python initialization
-           and so exceptions cannot be used (use raise=0). */
-        res = pyurandom(secret, secret_size, 0);
+           and so exceptions cannot be used (use raise=0).
+
+           _PyRandom_Init() must not block Python initialization: call
+           pyurandom() is non-blocking mode (blocking=0): see the PEP 524. */
+        res = pyurandom(secret, secret_size, 0, 0);
         if (res < 0) {
             Py_FatalError("failed to get random numbers to initialize Python");
         }