00146-hashlib-fips.patch

00146 #
Support OpenSSL FIPS mode (e.g. when OPENSSL_FORCE_FIPS_MODE=1 is set)
- handle failures from OpenSSL (e.g. on attempts to use MD5 in a
  FIPS-enforcing environment)
- add a new "usedforsecurity" keyword argument to the various digest
  algorithms in hashlib so that you can whitelist a callsite with
  "usedforsecurity=False"
(sent upstream for python 3 as http://bugs.python.org/issue9216; this is a
backport to python 2.7; see RHEL6 patch 119)
- enforce usage of the _hashlib implementation: don't fall back to the _md5
  and _sha* modules (leading to clearer error messages if fips selftests
  fail)
- don't build the _md5 and _sha* modules; rely on the _hashlib implementation
  of hashlib (for example, md5.py will use _hashlib's implementation of MD5,
  if permitted by the FIPS setting)
(rhbz#563986)

2 files changed, 182 insertions, 53 deletions

diff --git a/Modules/Setup.dist b/Modules/Setup.dist
index 2513d71..9a8d630 100644
--- a/Modules/Setup.dist
+++ b/Modules/Setup.dist
@@ -252,14 +252,14 @@ imageop imageop.c	# Operations on images
 # Message-Digest Algorithm, described in RFC 1321.  The necessary files
 # md5.c and md5.h are included here.
 
-_md5 md5module.c md5.c
+#_md5 md5module.c md5.c
 
 
 # The _sha module implements the SHA checksum algorithms.
 # (NIST's Secure Hash Algorithms.)
-_sha shamodule.c
-_sha256 sha256module.c
-_sha512 sha512module.c
+#_sha shamodule.c
+#_sha256 sha256module.c
+#_sha512 sha512module.c
 
 
 # SGI IRIX specific modules -- off by default.
diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 5df08e5..b80b4ec 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -41,6 +41,7 @@
 /* We use the object interface to discover what hashes OpenSSL supports. */
 #include <openssl/objects.h>
 #include "openssl/err.h"
+#include <openssl/ssl.h>
 
 #define MUNCH_SIZE INT_MAX
 
@@ -80,10 +81,19 @@ typedef struct {
 
 static PyTypeObject EVPtype;
 
+/* Struct to hold all the cached information we need on a specific algorithm.
+   We have one of these per algorithm */
+typedef struct {
+    PyObject *name_obj;
+    EVP_MD_CTX ctxs[2];
+    /* ctx_ptrs will point to ctxs unless an error occurred, when it will
+       be NULL: */
+    EVP_MD_CTX *ctx_ptrs[2];
+    PyObject *error_msgs[2];
+} EVPCachedInfo;
 
-#define DEFINE_CONSTS_FOR_NEW(Name)  \
-    static PyObject *CONST_ ## Name ## _name_obj = NULL; \
-    static EVP_MD_CTX *CONST_new_ ## Name ## _ctx_p = NULL;
+#define DEFINE_CONSTS_FOR_NEW(Name) \
+    static EVPCachedInfo cached_info_ ##Name;
 
 DEFINE_CONSTS_FOR_NEW(md5)
 DEFINE_CONSTS_FOR_NEW(sha1)
@@ -167,6 +177,48 @@ EVP_hash(EVPobject *self, const void *vp, Py_ssize_t len)
     }
 }
 
+static void
+mc_ctx_init(EVP_MD_CTX *ctx, int usedforsecurity)
+{
+    EVP_MD_CTX_init(ctx);
+
+    /*
+      If the user has declared that this digest is being used in a
+      non-security role (e.g. indexing into a data structure), set
+      the exception flag for openssl to allow it
+    */
+    if (!usedforsecurity) {
+#ifdef EVP_MD_CTX_FLAG_NON_FIPS_ALLOW
+        EVP_MD_CTX_set_flags(ctx,
+                             EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+#endif
+    }
+}
+
+/* Get an error msg for the last error as a PyObject */
+static PyObject *
+error_msg_for_last_error(void)
+{
+    char *errstr;
+
+    errstr = ERR_error_string(ERR_peek_last_error(), NULL);
+    ERR_clear_error();
+
+    return PyString_FromString(errstr); /* Can be NULL */
+}
+
+static void
+set_evp_exception(void)
+{
+    char *errstr;
+
+    errstr = ERR_error_string(ERR_peek_last_error(), NULL);
+    ERR_clear_error();
+
+    PyErr_SetString(PyExc_ValueError, errstr);
+}
+
+
 /* Internal methods for a hash object */
 
 static void
@@ -388,14 +440,15 @@ EVP_repr(PyObject *self)
 static int
 EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds)
 {
-    static char *kwlist[] = {"name", "string", NULL};
+    static char *kwlist[] = {"name", "string", "usedforsecurity", NULL};
     PyObject *name_obj = NULL;
+    int usedforsecurity = 1;
     Py_buffer view = { 0 };
     char *nameStr;
     const EVP_MD *digest;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s*:HASH", kwlist,
-                                     &name_obj, &view)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwds, "O|s*i:HASH", kwlist,
+                                     &name_obj, &view, &usedforsecurity)) {
         return -1;
     }
 
@@ -411,7 +464,12 @@ EVP_tp_init(EVPobject *self, PyObject *args, PyObject *kwds)
         PyBuffer_Release(&view);
         return -1;
     }
-    EVP_DigestInit(self->ctx, digest);
+    mc_ctx_init(&self->ctx, usedforsecurity);
+    if (!EVP_DigestInit_ex(&self->ctx, digest, NULL)) {
+        set_evp_exception();
+        PyBuffer_Release(&view);
+        return -1;
+    }
 
     self->name = name_obj;
     Py_INCREF(self->name);
@@ -495,7 +553,8 @@ static PyTypeObject EVPtype = {
 static PyObject *
 EVPnew(PyObject *name_obj,
        const EVP_MD *digest, const EVP_MD_CTX *initial_ctx,
-       const unsigned char *cp, Py_ssize_t len)
+       const unsigned char *cp, Py_ssize_t len,
+       int usedforsecurity)
 {
     EVPobject *self;
 
@@ -510,7 +569,12 @@ EVPnew(PyObject *name_obj,
     if (initial_ctx) {
         EVP_MD_CTX_copy(self->ctx, initial_ctx);
     } else {
-        EVP_DigestInit(self->ctx, digest);
+        mc_ctx_init(&self->ctx, usedforsecurity);
+        if (!EVP_DigestInit_ex(&self->ctx, digest, NULL)) {
+            set_evp_exception();
+            Py_DECREF(self);
+            return NULL;
+        }
     }
 
     if (cp && len) {
@@ -534,20 +598,28 @@ PyDoc_STRVAR(EVP_new__doc__,
 An optional string argument may be provided and will be\n\
 automatically hashed.\n\
 \n\
-The MD5 and SHA1 algorithms are always supported.\n");
+The MD5 and SHA1 algorithms are always supported.\n\
+\n\
+An optional \"usedforsecurity=True\" keyword argument is provided for use in\n\
+environments that enforce FIPS-based restrictions.  Some implementations of\n\
+OpenSSL can be configured to prevent the usage of non-secure algorithms (such\n\
+as MD5).  If you have a non-security use for these algorithms (e.g. a hash\n\
+table), you can override this argument by marking the callsite as\n\
+\"usedforsecurity=False\".");
 
 static PyObject *
 EVP_new(PyObject *self, PyObject *args, PyObject *kwdict)
 {
-    static char *kwlist[] = {"name", "string", NULL};
+    static char *kwlist[] = {"name", "string", "usedforsecurity", NULL};
     PyObject *name_obj = NULL;
     Py_buffer view = { 0 };
     PyObject *ret_obj;
     char *name;
     const EVP_MD *digest;
+    int usedforsecurity = 1;
 
-    if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|s*:new", kwlist,
-                                     &name_obj, &view)) {
+    if (!PyArg_ParseTupleAndKeywords(args, kwdict, "O|s*i:new", kwlist,
+                                     &name_obj, &view, &usedforsecurity)) {
         return NULL;
     }
 
@@ -560,7 +632,7 @@ EVP_new(PyObject *self, PyObject *args, PyObject *kwdict)
     digest = EVP_get_digestbyname(name);
 
     ret_obj = EVPnew(name_obj, digest, NULL, (unsigned char*)view.buf,
-                     view.len);
+                     view.len, usedforsecurity);
     PyBuffer_Release(&view);
 
     return ret_obj;
@@ -820,51 +892,111 @@ generate_hash_name_list(void)
 
 
 /*
- *  This macro generates constructor function definitions for specific
- *  hash algorithms.  These constructors are much faster than calling
- *  the generic one passing it a python string and are noticeably
- *  faster than calling a python new() wrapper.  Thats important for
+ *  This macro and function generates a family of constructor function
+ *  definitions for specific hash algorithms.  These constructors are much
+ *  faster than calling the generic one passing it a python string and are
+ *  noticably faster than calling a python new() wrapper.  That's important for
  *  code that wants to make hashes of a bunch of small strings.
  */
 #define GEN_CONSTRUCTOR(NAME)  \
     static PyObject * \
-    EVP_new_ ## NAME (PyObject *self, PyObject *args) \
+    EVP_new_ ## NAME (PyObject *self, PyObject *args, PyObject *kwdict) \
     { \
-        Py_buffer view = { 0 }; \
-        PyObject *ret_obj; \
-     \
-        if (!PyArg_ParseTuple(args, "|s*:" #NAME , &view)) { \
-            return NULL; \
-        } \
-     \
-        ret_obj = EVPnew( \
-                    CONST_ ## NAME ## _name_obj, \
-                    NULL, \
-                    CONST_new_ ## NAME ## _ctx_p, \
-                    (unsigned char*)view.buf, view.len); \
-        PyBuffer_Release(&view); \
-        return ret_obj; \
+        return implement_specific_EVP_new(self, args, kwdict,       \
+                                          "|s*i:" #NAME,            \
+                                          &cached_info_ ## NAME );  \
     }
 
+static PyObject *
+implement_specific_EVP_new(PyObject *self, PyObject *args, PyObject *kwdict,
+                           const char *format,
+                           EVPCachedInfo *cached_info)
+{
+    static char *kwlist[] = {"string", "usedforsecurity", NULL};
+    Py_buffer view = { 0 };
+    int usedforsecurity = 1;
+    int idx;
+    PyObject *ret_obj = NULL;
+
+    assert(cached_info);
+
+    if (!PyArg_ParseTupleAndKeywords(args, kwdict, format, kwlist,
+                                     &view, &usedforsecurity)) {
+        return NULL;
+    }
+
+    idx = usedforsecurity ? 1 : 0;
+
+    /*
+     * If an error occurred during creation of the global content, the ctx_ptr
+     * will be NULL, and the error_msg will hopefully be non-NULL:
+     */
+    if (cached_info->ctx_ptrs[idx]) {
+        /* We successfully initialized this context; copy it: */
+        ret_obj = EVPnew(cached_info->name_obj,
+                         NULL,
+                         cached_info->ctx_ptrs[idx],
+                         (unsigned char*)view.buf, view.len,
+                         usedforsecurity);
+    } else {
+        /* Some kind of error happened initializing the global context for
+           this (digest, usedforsecurity) pair.
+           Raise an exception with the saved error message: */
+        if (cached_info->error_msgs[idx]) {
+            PyErr_SetObject(PyExc_ValueError, cached_info->error_msgs[idx]);
+        } else {
+            PyErr_SetString(PyExc_ValueError, "Error initializing hash");
+        }
+    }
+
+    PyBuffer_Release(&view);
+
+    return ret_obj;
+}
+
 /* a PyMethodDef structure for the constructor */
 #define CONSTRUCTOR_METH_DEF(NAME)  \
-    {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, METH_VARARGS, \
+    {"openssl_" #NAME, (PyCFunction)EVP_new_ ## NAME, \
+        METH_VARARGS |METH_KEYWORDS, \
         PyDoc_STR("Returns a " #NAME \
                   " hash object; optionally initialized with a string") \
     }
 
-/* used in the init function to setup a constructor: initialize OpenSSL
-   constructor constants if they haven't been initialized already.  */
-#define INIT_CONSTRUCTOR_CONSTANTS(NAME)  do { \
-    if (CONST_ ## NAME ## _name_obj == NULL) { \
-    CONST_ ## NAME ## _name_obj = PyString_FromString(#NAME); \
-        if (EVP_get_digestbyname(#NAME)) { \
-            CONST_new_ ## NAME ## _ctx_p = EVP_MD_CTX_new(); \
-            EVP_DigestInit(CONST_new_ ## NAME ## _ctx_p, EVP_get_digestbyname(#NAME)); \
-        } \
-    } \
+/*
+  Macro/function pair to set up the constructors.
+
+  Try to initialize a context for each hash twice, once with
+  EVP_MD_CTX_FLAG_NON_FIPS_ALLOW and once without.
+
+  Any that have errors during initialization will end up wit a NULL ctx_ptrs
+  entry, and err_msgs will be set (unless we're very low on memory)
+*/
+#define INIT_CONSTRUCTOR_CONSTANTS(NAME)  do {    \
+    init_constructor_constant(&cached_info_ ## NAME, #NAME); \
 } while (0);
 
+static void
+init_constructor_constant(EVPCachedInfo *cached_info, const char *name)
+{
+    assert(cached_info);
+    cached_info->name_obj = PyString_FromString(name);
+    if (EVP_get_digestbyname(name)) {
+        int i;
+        for (i=0; i<2; i++) {
+            mc_ctx_init(&cached_info->ctxs[i], i);
+            if (EVP_DigestInit_ex(&cached_info->ctxs[i],
+                                  EVP_get_digestbyname(name), NULL)) {
+                /* Success: */
+                cached_info->ctx_ptrs[i] = &cached_info->ctxs[i];
+            } else {
+                /* Failure: */
+                cached_info->ctx_ptrs[i] = NULL;
+                cached_info->error_msgs[i] = error_msg_for_last_error();
+            }
+        }
+    }
+}
+
 GEN_CONSTRUCTOR(md5)
 GEN_CONSTRUCTOR(sha1)
 #ifdef _OPENSSL_SUPPORTS_SHA2
@@ -901,17 +1033,14 @@ init_hashlib(void)
 {
     PyObject *m, *openssl_md_meth_names;
 
+    SSL_load_error_strings();
+    SSL_library_init();
 #if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
     /* Load all digest algorithms and initialize cpuid */
     OPENSSL_add_all_algorithms_noconf();
     ERR_load_crypto_strings();
 #endif
 
-    /* TODO build EVP_functions openssl_* entries dynamically based
-     * on what hashes are supported rather than listing many
-     * but having some be unsupported.  Only init appropriate
-     * constants. */
-
     Py_TYPE(&EVPtype) = &PyType_Type;
     if (PyType_Ready(&EVPtype) < 0)
         return;