[3.12] gh-105497: [Enum] Fix flag mask inversion when unnamed flags exist (GH-106468) (#106620)

gh-105497: [Enum] Fix flag mask inversion when unnamed flags exist (GH-106468)

For example:

    class Flag(enum.Flag):
        A = 0x01
        B = 0x02
        MASK = 0xff

    ~Flag.MASK is Flag(0)
(cherry picked from commit 95b7426f45edb570869a5513c142f29ed9f851a1)

Co-authored-by: Ethan Furman <ethan@stoneleaf.us>

3 files changed, 86 insertions, 61 deletions

diff --git a/Lib/enum.py b/Lib/enum.py
index 92a9632..56ebfbd 100644
--- a/Lib/enum.py
+++ b/Lib/enum.py
@@ -1539,14 +1539,10 @@ class Flag(Enum, boundary=STRICT):
 
     def __invert__(self):
         if self._inverted_ is None:
-            if self._boundary_ is KEEP:
-                # use all bits
+            if self._boundary_ in (EJECT, KEEP):
                 self._inverted_ = self.__class__(~self._value_)
             else:
-                # use canonical bits (i.e. calculate flags not in this member)
-                self._inverted_ = self.__class__(self._singles_mask_ ^ self._value_)
-            if isinstance(self._inverted_, self.__class__):
-                self._inverted_._inverted_ = self
+                self._inverted_ = self.__class__(self._singles_mask_ & ~self._value_)
         return self._inverted_
 
     __rand__ = __and__
diff --git a/Lib/test/test_enum.py b/Lib/test/test_enum.py
index 291213a..592deb8 100644
--- a/Lib/test/test_enum.py
+++ b/Lib/test/test_enum.py
@@ -818,6 +818,89 @@ class _FlagTests:
             self.MainEnum('RED')
         self.assertIs(ctx.exception.__context__, None)
 
+    def test_closed_invert_expectations(self):
+        class ClosedAB(self.enum_type):
+            A = 1
+            B = 2
+            MASK = 3
+        A, B = ClosedAB
+        AB_MASK = ClosedAB.MASK
+        #
+        self.assertIs(~A, B)
+        self.assertIs(~B, A)
+        self.assertIs(~(A|B), ClosedAB(0))
+        self.assertIs(~AB_MASK, ClosedAB(0))
+        self.assertIs(~ClosedAB(0), (A|B))
+        #
+        class ClosedXYZ(self.enum_type):
+            X = 4
+            Y = 2
+            Z = 1
+            MASK = 7
+        X, Y, Z = ClosedXYZ
+        XYZ_MASK = ClosedXYZ.MASK
+        #
+        self.assertIs(~X, Y|Z)
+        self.assertIs(~Y, X|Z)
+        self.assertIs(~Z, X|Y)
+        self.assertIs(~(X|Y), Z)
+        self.assertIs(~(X|Z), Y)
+        self.assertIs(~(Y|Z), X)
+        self.assertIs(~(X|Y|Z), ClosedXYZ(0))
+        self.assertIs(~XYZ_MASK, ClosedXYZ(0))
+        self.assertIs(~ClosedXYZ(0), (X|Y|Z))
+
+    def test_open_invert_expectations(self):
+        class OpenAB(self.enum_type):
+            A = 1
+            B = 2
+            MASK = 255
+        A, B = OpenAB
+        AB_MASK = OpenAB.MASK
+        #
+        if OpenAB._boundary_ in (EJECT, KEEP):
+            self.assertIs(~A, OpenAB(254))
+            self.assertIs(~B, OpenAB(253))
+            self.assertIs(~(A|B), OpenAB(252))
+            self.assertIs(~AB_MASK, OpenAB(0))
+            self.assertIs(~OpenAB(0), AB_MASK)
+        else:
+            self.assertIs(~A, B)
+            self.assertIs(~B, A)
+            self.assertIs(~(A|B), OpenAB(0))
+            self.assertIs(~AB_MASK, OpenAB(0))
+            self.assertIs(~OpenAB(0), (A|B))
+        #
+        class OpenXYZ(self.enum_type):
+            X = 4
+            Y = 2
+            Z = 1
+            MASK = 31
+        X, Y, Z = OpenXYZ
+        XYZ_MASK = OpenXYZ.MASK
+        #
+        if OpenXYZ._boundary_ in (EJECT, KEEP):
+            self.assertIs(~X, OpenXYZ(27))
+            self.assertIs(~Y, OpenXYZ(29))
+            self.assertIs(~Z, OpenXYZ(30))
+            self.assertIs(~(X|Y), OpenXYZ(25))
+            self.assertIs(~(X|Z), OpenXYZ(26))
+            self.assertIs(~(Y|Z), OpenXYZ(28))
+            self.assertIs(~(X|Y|Z), OpenXYZ(24))
+            self.assertIs(~XYZ_MASK, OpenXYZ(0))
+            self.assertTrue(~OpenXYZ(0), XYZ_MASK)
+        else:
+            self.assertIs(~X, Y|Z)
+            self.assertIs(~Y, X|Z)
+            self.assertIs(~Z, X|Y)
+            self.assertIs(~(X|Y), Z)
+            self.assertIs(~(X|Z), Y)
+            self.assertIs(~(Y|Z), X)
+            self.assertIs(~(X|Y|Z), OpenXYZ(0))
+            self.assertIs(~XYZ_MASK, OpenXYZ(0))
+            self.assertTrue(~OpenXYZ(0), (X|Y|Z))
+
+
 class TestPlainEnum(_EnumTests, _PlainOutputTests, unittest.TestCase):
     enum_type = Enum
 
@@ -3045,33 +3128,6 @@ class OldTestFlag(unittest.TestCase):
         WHITE = RED|GREEN|BLUE
         BLANCO = RED|GREEN|BLUE
 
-    class Complete(Flag):
-        A = 0x01
-        B = 0x02
-
-    class Partial(Flag):
-        A = 0x01
-        B = 0x02
-        MASK = 0xff
-
-    class CompleteInt(IntFlag):
-        A = 0x01
-        B = 0x02
-
-    class PartialInt(IntFlag):
-        A = 0x01
-        B = 0x02
-        MASK = 0xff
-
-    class CompleteIntStrict(IntFlag, boundary=STRICT):
-        A = 0x01
-        B = 0x02
-
-    class PartialIntStrict(IntFlag, boundary=STRICT):
-        A = 0x01
-        B = 0x02
-        MASK = 0xff
-
     def test_or(self):
         Perm = self.Perm
         for i in Perm:
@@ -3115,34 +3171,6 @@ class OldTestFlag(unittest.TestCase):
         self.assertIs(Open.RO ^ Open.CE, Open.CE)
         self.assertIs(Open.CE ^ Open.CE, Open.RO)
 
-    def test_invert(self):
-        Perm = self.Perm
-        RW = Perm.R | Perm.W
-        RX = Perm.R | Perm.X
-        WX = Perm.W | Perm.X
-        RWX = Perm.R | Perm.W | Perm.X
-        values = list(Perm) + [RW, RX, WX, RWX, Perm(0)]
-        for i in values:
-            self.assertIs(type(~i), Perm)
-            self.assertEqual(~~i, i)
-        for i in Perm:
-            self.assertIs(~~i, i)
-        Open = self.Open
-        self.assertIs(Open.WO & ~Open.WO, Open.RO)
-        self.assertIs((Open.WO|Open.CE) & ~Open.WO, Open.CE)
-        Complete = self.Complete
-        self.assertIs(~Complete.A, Complete.B)
-        Partial = self.Partial
-        self.assertIs(~Partial.A, Partial.B)
-        CompleteInt = self.CompleteInt
-        self.assertIs(~CompleteInt.A, CompleteInt.B)
-        PartialInt = self.PartialInt
-        self.assertIs(~PartialInt.A, PartialInt(254))
-        CompleteIntStrict = self.CompleteIntStrict
-        self.assertIs(~CompleteIntStrict.A, CompleteIntStrict.B)
-        PartialIntStrict = self.PartialIntStrict
-        self.assertIs(~PartialIntStrict.A, PartialIntStrict.B)
-
     def test_bool(self):
         Perm = self.Perm
         for f in Perm:
diff --git a/Misc/NEWS.d/next/Library/2023-07-05-14-34-10.gh-issue-105497.HU5u89.rst b/Misc/NEWS.d/next/Library/2023-07-05-14-34-10.gh-issue-105497.HU5u89.rst
new file mode 100644
index 0000000..f4f2db0
--- /dev/null
+++ b/Misc/NEWS.d/next/Library/2023-07-05-14-34-10.gh-issue-105497.HU5u89.rst
@@ -0,0 +1 @@
+Fix flag mask inversion when unnamed flags exist.