bpo-42382: In importlib.metadata, `EntryPoint` objects now expose `dist` (#23758)

* bpo-42382: In importlib.metadata, `EntryPoint` objects now expose a `.dist` object referencing the `Distribution` when constructed from a `Distribution`.

Also, sync importlib_metadata 3.3:

- Add support for package discovery under package normalization rules.
- The object returned by `metadata()` now has a formally-defined protocol called `PackageMetadata` with declared support for the `.get_all()` method.

* Add blurb

* Remove latent footnote.

4 files changed, 158 insertions, 80 deletions

diff --git a/Lib/test/test_importlib/fixtures.py b/Lib/test/test_importlib/fixtures.py
index 8fa9290..429313e 100644
--- a/Lib/test/test_importlib/fixtures.py
+++ b/Lib/test/test_importlib/fixtures.py
@@ -7,6 +7,7 @@ import textwrap
 import contextlib
 
 from test.support.os_helper import FS_NONASCII
+from typing import Dict, Union
 
 
 @contextlib.contextmanager
@@ -71,8 +72,13 @@ class OnSysPath(Fixtures):
         self.fixtures.enter_context(self.add_sys_path(self.site_dir))
 
 
+# Except for python/mypy#731, prefer to define
+# FilesDef = Dict[str, Union['FilesDef', str]]
+FilesDef = Dict[str, Union[Dict[str, Union[Dict[str, str], str]], str]]
+
+
 class DistInfoPkg(OnSysPath, SiteDir):
-    files = {
+    files: FilesDef = {
         "distinfo_pkg-1.0.0.dist-info": {
             "METADATA": """
                 Name: distinfo-pkg
@@ -86,19 +92,55 @@ class DistInfoPkg(OnSysPath, SiteDir):
                 [entries]
                 main = mod:main
                 ns:sub = mod:main
-            """
-            },
+            """,
+        },
         "mod.py": """
             def main():
                 print("hello world")
             """,
-        }
+    }
 
     def setUp(self):
         super(DistInfoPkg, self).setUp()
         build_files(DistInfoPkg.files, self.site_dir)
 
 
+class DistInfoPkgWithDot(OnSysPath, SiteDir):
+    files: FilesDef = {
+        "pkg_dot-1.0.0.dist-info": {
+            "METADATA": """
+                Name: pkg.dot
+                Version: 1.0.0
+                """,
+        },
+    }
+
+    def setUp(self):
+        super(DistInfoPkgWithDot, self).setUp()
+        build_files(DistInfoPkgWithDot.files, self.site_dir)
+
+
+class DistInfoPkgWithDotLegacy(OnSysPath, SiteDir):
+    files: FilesDef = {
+        "pkg.dot-1.0.0.dist-info": {
+            "METADATA": """
+                Name: pkg.dot
+                Version: 1.0.0
+                """,
+        },
+        "pkg.lot.egg-info": {
+            "METADATA": """
+                Name: pkg.lot
+                Version: 1.0.0
+                """,
+        },
+    }
+
+    def setUp(self):
+        super(DistInfoPkgWithDotLegacy, self).setUp()
+        build_files(DistInfoPkgWithDotLegacy.files, self.site_dir)
+
+
 class DistInfoPkgOffPath(SiteDir):
     def setUp(self):
         super(DistInfoPkgOffPath, self).setUp()
@@ -106,7 +148,7 @@ class DistInfoPkgOffPath(SiteDir):
 
 
 class EggInfoPkg(OnSysPath, SiteDir):
-    files = {
+    files: FilesDef = {
         "egginfo_pkg.egg-info": {
             "PKG-INFO": """
                 Name: egginfo-pkg
@@ -129,13 +171,13 @@ class EggInfoPkg(OnSysPath, SiteDir):
                 [test]
                 pytest
             """,
-            "top_level.txt": "mod\n"
-            },
+            "top_level.txt": "mod\n",
+        },
         "mod.py": """
             def main():
                 print("hello world")
             """,
-        }
+    }
 
     def setUp(self):
         super(EggInfoPkg, self).setUp()
@@ -143,7 +185,7 @@ class EggInfoPkg(OnSysPath, SiteDir):
 
 
 class EggInfoFile(OnSysPath, SiteDir):
-    files = {
+    files: FilesDef = {
         "egginfo_file.egg-info": """
             Metadata-Version: 1.0
             Name: egginfo_file
@@ -156,7 +198,7 @@ class EggInfoFile(OnSysPath, SiteDir):
             Description: UNKNOWN
             Platform: UNKNOWN
             """,
-        }
+    }
 
     def setUp(self):
         super(EggInfoFile, self).setUp()
@@ -164,12 +206,12 @@ class EggInfoFile(OnSysPath, SiteDir):
 
 
 class LocalPackage:
-    files = {
+    files: FilesDef = {
         "setup.py": """
             import setuptools
             setuptools.setup(name="local-pkg", version="2.0.1")
             """,
-        }
+    }
 
     def setUp(self):
         self.fixtures = contextlib.ExitStack()
@@ -214,8 +256,7 @@ def build_files(file_defs, prefix=pathlib.Path()):
 
 class FileBuilder:
     def unicode_filename(self):
-        return FS_NONASCII or \
-            self.skip("File system does not support non-ascii.")
+        return FS_NONASCII or self.skip("File system does not support non-ascii.")
 
 
 def DALS(str):
diff --git a/Lib/test/test_importlib/test_main.py b/Lib/test/test_importlib/test_main.py
index a26bab6..c937361 100644
--- a/Lib/test/test_importlib/test_main.py
+++ b/Lib/test/test_importlib/test_main.py
@@ -1,5 +1,3 @@
-# coding: utf-8
-
 import re
 import json
 import pickle
@@ -14,10 +12,14 @@ except ImportError:
 
 from . import fixtures
 from importlib.metadata import (
-    Distribution, EntryPoint,
-    PackageNotFoundError, distributions,
-    entry_points, metadata, version,
-    )
+    Distribution,
+    EntryPoint,
+    PackageNotFoundError,
+    distributions,
+    entry_points,
+    metadata,
+    version,
+)
 
 
 class BasicTests(fixtures.DistInfoPkg, unittest.TestCase):
@@ -70,12 +72,11 @@ class ImportTests(fixtures.DistInfoPkg, unittest.TestCase):
             name='ep',
             value='importlib.metadata',
             group='grp',
-            )
+        )
         assert ep.load() is importlib.metadata
 
 
-class NameNormalizationTests(
-        fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
+class NameNormalizationTests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
     @staticmethod
     def pkg_with_dashes(site_dir):
         """
@@ -144,11 +145,15 @@ class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
         metadata_dir.mkdir()
         metadata = metadata_dir / 'METADATA'
         with metadata.open('w', encoding='utf-8') as fp:
-            fp.write(textwrap.dedent("""
+            fp.write(
+                textwrap.dedent(
+                    """
                 Name: portend
 
                 pôrˈtend
-                """).lstrip())
+                """
+                ).lstrip()
+            )
         return 'portend'
 
     def test_metadata_loads(self):
@@ -162,24 +167,12 @@ class NonASCIITests(fixtures.OnSysPath, fixtures.SiteDir, unittest.TestCase):
         assert meta.get_payload() == 'pôrˈtend\n'
 
 
-class DiscoveryTests(fixtures.EggInfoPkg,
-                     fixtures.DistInfoPkg,
-                     unittest.TestCase):
-
+class DiscoveryTests(fixtures.EggInfoPkg, fixtures.DistInfoPkg, unittest.TestCase):
     def test_package_discovery(self):
         dists = list(distributions())
-        assert all(
-            isinstance(dist, Distribution)
-            for dist in dists
-            )
-        assert any(
-            dist.metadata['Name'] == 'egginfo-pkg'
-            for dist in dists
-            )
-        assert any(
-            dist.metadata['Name'] == 'distinfo-pkg'
-            for dist in dists
-            )
+        assert all(isinstance(dist, Distribution) for dist in dists)
+        assert any(dist.metadata['Name'] == 'egginfo-pkg' for dist in dists)
+        assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists)
 
     def test_invalid_usage(self):
         with self.assertRaises(ValueError):
@@ -265,10 +258,21 @@ class TestEntryPoints(unittest.TestCase):
     def test_attr(self):
         assert self.ep.attr is None
 
+    def test_sortable(self):
+        """
+        EntryPoint objects are sortable, but result is undefined.
+        """
+        sorted(
+            [
+                EntryPoint('b', 'val', 'group'),
+                EntryPoint('a', 'val', 'group'),
+            ]
+        )
+
 
 class FileSystem(
-        fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder,
-        unittest.TestCase):
+    fixtures.OnSysPath, fixtures.SiteDir, fixtures.FileBuilder, unittest.TestCase
+):
     def test_unicode_dir_on_sys_path(self):
         """
         Ensure a Unicode subdirectory of a directory on sys.path
@@ -277,5 +281,5 @@ class FileSystem(
         fixtures.build_files(
             {self.unicode_filename(): {}},
             prefix=self.site_dir,
-            )
+        )
         list(distributions())
diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py
index 1d7b29a..df00ae9 100644
--- a/Lib/test/test_importlib/test_metadata_api.py
+++ b/Lib/test/test_importlib/test_metadata_api.py
@@ -2,20 +2,26 @@ import re
 import textwrap
 import unittest
 
-from collections.abc import Iterator
-
 from . import fixtures
 from importlib.metadata import (
-    Distribution, PackageNotFoundError, distribution,
-    entry_points, files, metadata, requires, version,
-    )
+    Distribution,
+    PackageNotFoundError,
+    distribution,
+    entry_points,
+    files,
+    metadata,
+    requires,
+    version,
+)
 
 
 class APITests(
-        fixtures.EggInfoPkg,
-        fixtures.DistInfoPkg,
-        fixtures.EggInfoFile,
-        unittest.TestCase):
+    fixtures.EggInfoPkg,
+    fixtures.DistInfoPkg,
+    fixtures.DistInfoPkgWithDot,
+    fixtures.EggInfoFile,
+    unittest.TestCase,
+):
 
     version_pattern = r'\d+\.\d+(\.\d)?'
 
@@ -33,16 +39,28 @@ class APITests(
         with self.assertRaises(PackageNotFoundError):
             distribution('does-not-exist')
 
+    def test_name_normalization(self):
+        names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot'
+        for name in names:
+            with self.subTest(name):
+                assert distribution(name).metadata['Name'] == 'pkg.dot'
+
+    def test_prefix_not_matched(self):
+        prefixes = 'p', 'pkg', 'pkg.'
+        for prefix in prefixes:
+            with self.subTest(prefix):
+                with self.assertRaises(PackageNotFoundError):
+                    distribution(prefix)
+
     def test_for_top_level(self):
         self.assertEqual(
-            distribution('egginfo-pkg').read_text('top_level.txt').strip(),
-            'mod')
+            distribution('egginfo-pkg').read_text('top_level.txt').strip(), 'mod'
+        )
 
     def test_read_text(self):
         top_level = [
-            path for path in files('egginfo-pkg')
-            if path.name == 'top_level.txt'
-            ][0]
+            path for path in files('egginfo-pkg') if path.name == 'top_level.txt'
+        ][0]
         self.assertEqual(top_level.read_text(), 'mod\n')
 
     def test_entry_points(self):
@@ -51,6 +69,13 @@ class APITests(
         self.assertEqual(ep.value, 'mod:main')
         self.assertEqual(ep.extras, [])
 
+    def test_entry_points_distribution(self):
+        entries = dict(entry_points()['entries'])
+        for entry in ("main", "ns:sub"):
+            ep = entries[entry]
+            self.assertIn(ep.dist.name, ('distinfo-pkg', 'egginfo-pkg'))
+            self.assertEqual(ep.dist.version, "1.0.0")
+
     def test_metadata_for_this_package(self):
         md = metadata('egginfo-pkg')
         assert md['author'] == 'Steven Ma'
@@ -75,13 +100,8 @@ class APITests(
     def test_file_hash_repr(self):
         assertRegex = self.assertRegex
 
-        util = [
-            p for p in files('distinfo-pkg')
-            if p.name == 'mod.py'
-            ][0]
-        assertRegex(
-            repr(util.hash),
-            '<FileHash mode: sha256 value: .*>')
+        util = [p for p in files('distinfo-pkg') if p.name == 'mod.py'][0]
+        assertRegex(repr(util.hash), '<FileHash mode: sha256 value: .*>')
 
     def test_files_dist_info(self):
         self._test_files(files('distinfo-pkg'))
@@ -99,10 +119,7 @@ class APITests(
     def test_requires_egg_info(self):
         deps = requires('egginfo-pkg')
         assert len(deps) == 2
-        assert any(
-            dep == 'wheel >= 1.0; python_version >= "2.7"'
-            for dep in deps
-            )
+        assert any(dep == 'wheel >= 1.0; python_version >= "2.7"' for dep in deps)
 
     def test_requires_dist_info(self):
         deps = requires('distinfo-pkg')
@@ -112,7 +129,8 @@ class APITests(
         assert "pytest; extra == 'test'" in deps
 
     def test_more_complex_deps_requires_text(self):
-        requires = textwrap.dedent("""
+        requires = textwrap.dedent(
+            """
             dep1
             dep2
 
@@ -124,7 +142,8 @@ class APITests(
 
             [extra2:python_version < "3"]
             dep5
-            """)
+            """
+        )
         deps = sorted(Distribution._deps_from_requires_text(requires))
         expected = [
             'dep1',
@@ -132,7 +151,7 @@ class APITests(
             'dep3; python_version < "3"',
             'dep4; extra == "extra1"',
             'dep5; (python_version < "3") and extra == "extra2"',
-            ]
+        ]
         # It's important that the environment marker expression be
         # wrapped in parentheses to avoid the following 'and' binding more
         # tightly than some other part of the environment expression.
@@ -140,17 +159,27 @@ class APITests(
         assert deps == expected
 
 
+class LegacyDots(fixtures.DistInfoPkgWithDotLegacy, unittest.TestCase):
+    def test_name_normalization(self):
+        names = 'pkg.dot', 'pkg_dot', 'pkg-dot', 'pkg..dot', 'Pkg.Dot'
+        for name in names:
+            with self.subTest(name):
+                assert distribution(name).metadata['Name'] == 'pkg.dot'
+
+    def test_name_normalization_versionless_egg_info(self):
+        names = 'pkg.lot', 'pkg_lot', 'pkg-lot', 'pkg..lot', 'Pkg.Lot'
+        for name in names:
+            with self.subTest(name):
+                assert distribution(name).metadata['Name'] == 'pkg.lot'
+
+
 class OffSysPathTests(fixtures.DistInfoPkgOffPath, unittest.TestCase):
     def test_find_distributions_specified_path(self):
         dists = Distribution.discover(path=[str(self.site_dir)])
-        assert any(
-            dist.metadata['Name'] == 'distinfo-pkg'
-            for dist in dists
-            )
+        assert any(dist.metadata['Name'] == 'distinfo-pkg' for dist in dists)
 
     def test_distribution_at_pathlib(self):
-        """Demonstrate how to load metadata direct from a directory.
-        """
+        """Demonstrate how to load metadata direct from a directory."""
         dist_info_path = self.site_dir / 'distinfo_pkg-1.0.0.dist-info'
         dist = Distribution.at(dist_info_path)
         assert dist.version == '1.0.0'
diff --git a/Lib/test/test_importlib/test_zip.py b/Lib/test/test_importlib/test_zip.py
index a5399c1..74783fc 100644
--- a/Lib/test/test_importlib/test_zip.py
+++ b/Lib/test/test_importlib/test_zip.py
@@ -3,8 +3,12 @@ import unittest
 
 from contextlib import ExitStack
 from importlib.metadata import (
-    distribution, entry_points, files, PackageNotFoundError,
-    version, distributions,
+    PackageNotFoundError,
+    distribution,
+    distributions,
+    entry_points,
+    files,
+    version,
 )
 from importlib import resources