bpo-40479: Fix hashlib issue with OpenSSL 3.0.0 (GH-20107)

OpenSSL 3.0.0-alpha2 was released today. The FIPS_mode() function has
been deprecated and removed. It no longer makes sense with the new
provider and context system in OpenSSL 3.0.0.

EVP_default_properties_is_fips_enabled() is good enough for our needs in
unit tests. It's an internal API, too.

Signed-off-by: Christian Heimes <christian@python.org>

2 files changed, 17 insertions, 8 deletions

diff --git a/Modules/_hashopenssl.c b/Modules/_hashopenssl.c
index 91834e5..b7661b4 100644
--- a/Modules/_hashopenssl.c
+++ b/Modules/_hashopenssl.c
@@ -1109,19 +1109,25 @@ _hashlib.get_fips_mode -> int
 
 Determine the OpenSSL FIPS mode of operation.
 
+For OpenSSL 3.0.0 and newer it returns the state of the default provider
+in the default OSSL context. It's not quite the same as FIPS_mode() but good
+enough for unittests.
+
 Effectively any non-zero return value indicates FIPS mode;
 values other than 1 may have additional significance.
-
-See OpenSSL documentation for the FIPS_mode() function for details.
 [clinic start generated code]*/
 
 static int
 _hashlib_get_fips_mode_impl(PyObject *module)
-/*[clinic end generated code: output=87eece1bab4d3fa9 input=c2799c3132a36d6c]*/
+/*[clinic end generated code: output=87eece1bab4d3fa9 input=2db61538c41c6fef]*/
 
 {
+    int result;
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
+    result = EVP_default_properties_is_fips_enabled(NULL);
+#else
     ERR_clear_error();
-    int result = FIPS_mode();
+    result = FIPS_mode();
     if (result == 0) {
         // "If the library was built without support of the FIPS Object Module,
         // then the function will return 0 with an error code of
@@ -1134,6 +1140,7 @@ _hashlib_get_fips_mode_impl(PyObject *module)
         }
     }
     return result;
+#endif
 }
 #endif  // !LIBRESSL_VERSION_NUMBER
 
diff --git a/Modules/clinic/_hashopenssl.c.h b/Modules/clinic/_hashopenssl.c.h
index 275784d..1b0c6d0 100644
--- a/Modules/clinic/_hashopenssl.c.h
+++ b/Modules/clinic/_hashopenssl.c.h
@@ -733,10 +733,12 @@ PyDoc_STRVAR(_hashlib_get_fips_mode__doc__,
 "\n"
 "Determine the OpenSSL FIPS mode of operation.\n"
 "\n"
-"Effectively any non-zero return value indicates FIPS mode;\n"
-"values other than 1 may have additional significance.\n"
+"For OpenSSL 3.0.0 and newer it returns the state of the default provider\n"
+"in the default OSSL context. It\'s not quite the same as FIPS_mode() but good\n"
+"enough for unittests.\n"
 "\n"
-"See OpenSSL documentation for the FIPS_mode() function for details.");
+"Effectively any non-zero return value indicates FIPS mode;\n"
+"values other than 1 may have additional significance.");
 
 #define _HASHLIB_GET_FIPS_MODE_METHODDEF    \
     {"get_fips_mode", (PyCFunction)_hashlib_get_fips_mode, METH_NOARGS, _hashlib_get_fips_mode__doc__},
@@ -769,4 +771,4 @@ exit:
 #ifndef _HASHLIB_GET_FIPS_MODE_METHODDEF
     #define _HASHLIB_GET_FIPS_MODE_METHODDEF
 #endif /* !defined(_HASHLIB_GET_FIPS_MODE_METHODDEF) */
-/*[clinic end generated code: output=b0703dd5a043394d input=a9049054013a1b77]*/
+/*[clinic end generated code: output=4babbd88389a196b input=a9049054013a1b77]*/