Fix the CRT argument error handling for VisualStudio .NET 2005. Install a CRT error handler and disable the assertion for debug builds. This causes CRT to set errno to EINVAL.

This update fixes crash cases in the test suite where the default CRT error handler would cause process exit.

2 files changed, 38 insertions, 1 deletions

diff --git a/Objects/exceptions.c b/Objects/exceptions.c
index 369365b..38d7350 100644
--- a/Objects/exceptions.c
+++ b/Objects/exceptions.c
@@ -1967,6 +1967,29 @@ static PyMethodDef functions[] = {
     if (PyDict_SetItemString(bdict, # TYPE, PyExc_ ## TYPE)) \
         Py_FatalError("Module dictionary insertion problem.");
 
+#if defined _MSC_VER && _MSC_VER >= 1400
+/* crt variable checking in VisualStudio .NET 2005 */
+#include <crtdbg.h>
+
+static int	prevCrtReportMode;
+static _invalid_parameter_handler	prevCrtHandler;
+
+/* Invalid parameter handler.  Sets a ValueError exception */
+static void
+InvalidParameterHandler(
+    const wchar_t * expression,
+    const wchar_t * function,
+    const wchar_t * file,
+    unsigned int line,
+    uintptr_t pReserved)
+{
+    /* Do nothing, allow execution to continue.  Usually this
+     * means that the CRT will set errno to EINVAL
+     */
+}
+#endif
+
+
 PyMODINIT_FUNC
 _PyExc_Init(void)
 {
@@ -2096,6 +2119,13 @@ _PyExc_Init(void)
         Py_FatalError("Cannot pre-allocate MemoryError instance\n");
 
     Py_DECREF(bltinmod);
+
+#if defined _MSC_VER && _MSC_VER >= 1400
+    /* Set CRT argument error handler */
+    prevCrtHandler = _set_invalid_parameter_handler(InvalidParameterHandler);
+    /* turn off assertions in debug mode */
+    prevCrtReportMode = _CrtSetReportMode(_CRT_ASSERT, 0);
+#endif
 }
 
 void
@@ -2103,4 +2133,9 @@ _PyExc_Fini(void)
 {
     Py_XDECREF(PyExc_MemoryErrorInst);
     PyExc_MemoryErrorInst = NULL;
+#if defined _MSC_VER && _MSC_VER >= 1400
+    /* reset CRT error handling */
+    _set_invalid_parameter_handler(prevCrtHandler);
+    _CrtSetReportMode(_CRT_ASSERT, prevCrtReportMode);
+#endif
 }
diff --git a/Objects/fileobject.c b/Objects/fileobject.c
index 2b37e74..050146a 100644
--- a/Objects/fileobject.c
+++ b/Objects/fileobject.c
@@ -241,13 +241,15 @@ open_the_file(PyFileObject *f, char *name, char *mode)
 	}
 
 	if (f->f_fp == NULL) {
-#ifdef _MSC_VER
+#if defined  _MSC_VER && _MSC_VER < 1400
 		/* MSVC 6 (Microsoft) leaves errno at 0 for bad mode strings,
 		 * across all Windows flavors.  When it sets EINVAL varies
 		 * across Windows flavors, the exact conditions aren't
 		 * documented, and the answer lies in the OS's implementation
 		 * of Win32's CreateFile function (whose source is secret).
 		 * Seems the best we can do is map EINVAL to ENOENT.
+		 * Starting with Visual Studio .NET 2005, EINVAL is correctly
+		 * set by our CRT error handler (set in exceptions.c.)
 		 */
 		if (errno == 0)	/* bad mode string */
 			errno = EINVAL;