gh-120010: Fix invalid (nan+nanj) results in _Py_c_prod() (GH-120287)

In some cases, previously computed as (nan+nanj), we could recover
meaningful component values in the result, see e.g. the C11, Annex
G.5.1, routine _Cmultd():

>>> z = 1e300+1j
>>> z*(nan+infj)  # was (nan+nanj)
(-inf+infj)

That also fix some complex powers for small integer exponents, computed
with optimized algorithm (by squaring):

>>> z**5  # was (nan+nanj)
Traceback (most recent call last):
  File "<python-input-1>", line 1, in <module>
    z**5
    ~^^~
OverflowError: complex exponentiation

3 files changed, 75 insertions, 4 deletions

diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py
index 179556f..fd002fb 100644
--- a/Lib/test/test_complex.py
+++ b/Lib/test/test_complex.py
@@ -299,6 +299,22 @@ class ComplexTest(ComplexesAreIdenticalMixin, unittest.TestCase):
         self.assertRaises(TypeError, operator.mul, 1j, None)
         self.assertRaises(TypeError, operator.mul, None, 1j)
 
+        for z, w, r in [(1e300+1j, complex(INF, INF), complex(NAN, INF)),
+                        (1e300+1j, complex(NAN, INF), complex(-INF, INF)),
+                        (1e300+1j, complex(INF, NAN), complex(INF, INF)),
+                        (complex(INF, 1), complex(NAN, INF), complex(NAN, INF)),
+                        (complex(INF, 1), complex(INF, NAN), complex(INF, NAN)),
+                        (complex(NAN, 1), complex(1, INF), complex(-INF, NAN)),
+                        (complex(1, NAN), complex(1, INF), complex(NAN, INF)),
+                        (complex(1e200, NAN), complex(1e200, NAN), complex(INF, NAN)),
+                        (complex(1e200, NAN), complex(NAN, 1e200), complex(NAN, INF)),
+                        (complex(NAN, 1e200), complex(1e200, NAN), complex(NAN, INF)),
+                        (complex(NAN, 1e200), complex(NAN, 1e200), complex(-INF, NAN)),
+                        (complex(NAN, NAN), complex(NAN, NAN), complex(NAN, NAN))]:
+            with self.subTest(z=z, w=w, r=r):
+                self.assertComplexesAreIdentical(z * w, r)
+                self.assertComplexesAreIdentical(w * z, r)
+
     def test_mod(self):
         # % is no longer supported on complex numbers
         with self.assertRaises(TypeError):
@@ -340,6 +356,7 @@ class ComplexTest(ComplexesAreIdenticalMixin, unittest.TestCase):
         self.assertAlmostEqual(pow(1j, 200), 1)
         self.assertRaises(ValueError, pow, 1+1j, 1+1j, 1+1j)
         self.assertRaises(OverflowError, pow, 1e200+1j, 1e200+1j)
+        self.assertRaises(OverflowError, pow, 1e200+1j, 5)
         self.assertRaises(TypeError, pow, 1j, None)
         self.assertRaises(TypeError, pow, None, 1j)
         self.assertAlmostEqual(pow(1j, 0.5), 0.7071067811865476+0.7071067811865475j)
diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst
new file mode 100644
index 0000000..7954c7f
--- /dev/null
+++ b/Misc/NEWS.d/next/Core_and_Builtins/2024-06-04-08-26-25.gh-issue-120010._z-AWz.rst
@@ -0,0 +1,2 @@
+Correct invalid corner cases which resulted in ``(nan+nanj)`` output in complex
+multiplication, e.g., ``(1e300+1j)*(nan+infj)``.  Patch by Sergey B Kirpichev.
diff --git a/Objects/complexobject.c b/Objects/complexobject.c
index 8fbca3c..bf6187e 100644
--- a/Objects/complexobject.c
+++ b/Objects/complexobject.c
@@ -85,11 +85,63 @@ _Py_c_neg(Py_complex a)
 }
 
 Py_complex
-_Py_c_prod(Py_complex a, Py_complex b)
+_Py_c_prod(Py_complex z, Py_complex w)
 {
-    Py_complex r;
-    r.real = a.real*b.real - a.imag*b.imag;
-    r.imag = a.real*b.imag + a.imag*b.real;
+    double a = z.real, b = z.imag, c = w.real, d = w.imag;
+    double ac = a*c, bd = b*d, ad = a*d, bc = b*c;
+    Py_complex r = {ac - bd, ad + bc};
+
+    /* Recover infinities that computed as nan+nanj.  See e.g. the C11,
+       Annex G.5.1, routine _Cmultd(). */
+    if (isnan(r.real) && isnan(r.imag)) {
+        int recalc = 0;
+
+        if (isinf(a) || isinf(b)) {  /* z is infinite */
+            /* "Box" the infinity and change nans in the other factor to 0 */
+            a = copysign(isinf(a) ? 1.0 : 0.0, a);
+            b = copysign(isinf(b) ? 1.0 : 0.0, b);
+            if (isnan(c)) {
+                c = copysign(0.0, c);
+            }
+            if (isnan(d)) {
+                d = copysign(0.0, d);
+            }
+            recalc = 1;
+        }
+        if (isinf(c) || isinf(d)) {  /* w is infinite */
+            /* "Box" the infinity and change nans in the other factor to 0 */
+            c = copysign(isinf(c) ? 1.0 : 0.0, c);
+            d = copysign(isinf(d) ? 1.0 : 0.0, d);
+            if (isnan(a)) {
+                a = copysign(0.0, a);
+            }
+            if (isnan(b)) {
+                b = copysign(0.0, b);
+            }
+            recalc = 1;
+        }
+        if (!recalc && (isinf(ac) || isinf(bd) || isinf(ad) || isinf(bc))) {
+            /* Recover infinities from overflow by changing nans to 0 */
+            if (isnan(a)) {
+                a = copysign(0.0, a);
+            }
+            if (isnan(b)) {
+                b = copysign(0.0, b);
+            }
+            if (isnan(c)) {
+                c = copysign(0.0, c);
+            }
+            if (isnan(d)) {
+                d = copysign(0.0, d);
+            }
+            recalc = 1;
+        }
+        if (recalc) {
+            r.real = Py_INFINITY*(a*c - b*d);
+            r.imag = Py_INFINITY*(a*d + b*c);
+        }
+    }
+
     return r;
 }