Issue #23731: Implement PEP 488.

The concept of .pyo files no longer exists. Now .pyc files have an
optional `opt-` tag which specifies if any extra optimizations beyond
the peepholer were applied.

1 files changed, 96 insertions, 18 deletions

diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py
index 5d35d14..69466b2 100644
--- a/Lib/test/test_importlib/test_util.py
+++ b/Lib/test/test_importlib/test_util.py
@@ -5,6 +5,7 @@ machinery = util.import_importlib('importlib.machinery')
 importlib_util = util.import_importlib('importlib.util')
 
 import os
+import string
 import sys
 from test import support
 import types
@@ -562,7 +563,8 @@ class PEP3147Tests:
         path = os.path.join('foo', 'bar', 'baz', 'qux.py')
         expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
                               'qux.{}.pyc'.format(self.tag))
-        self.assertEqual(self.util.cache_from_source(path, True), expect)
+        self.assertEqual(self.util.cache_from_source(path, optimization=''),
+                         expect)
 
     def test_cache_from_source_no_cache_tag(self):
         # No cache tag means NotImplementedError.
@@ -575,43 +577,103 @@ class PEP3147Tests:
         path = os.path.join('foo.bar', 'file')
         expect = os.path.join('foo.bar', '__pycache__',
                               'file{}.pyc'.format(self.tag))
-        self.assertEqual(self.util.cache_from_source(path, True), expect)
+        self.assertEqual(self.util.cache_from_source(path, optimization=''),
+                         expect)
 
-    def test_cache_from_source_optimized(self):
-        # Given the path to a .py file, return the path to its PEP 3147
-        # defined .pyo file (i.e. under __pycache__).
+    def test_cache_from_source_debug_override(self):
+        # Given the path to a .py file, return the path to its PEP 3147/PEP 488
+        # defined .pyc file (i.e. under __pycache__).
         path = os.path.join('foo', 'bar', 'baz', 'qux.py')
-        expect = os.path.join('foo', 'bar', 'baz', '__pycache__',
-                              'qux.{}.pyo'.format(self.tag))
-        self.assertEqual(self.util.cache_from_source(path, False), expect)
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            self.assertEqual(self.util.cache_from_source(path, False),
+                             self.util.cache_from_source(path, optimization=1))
+            self.assertEqual(self.util.cache_from_source(path, True),
+                             self.util.cache_from_source(path, optimization=''))
+        with warnings.catch_warnings():
+            warnings.simplefilter('error')
+            with self.assertRaises(DeprecationWarning):
+                self.util.cache_from_source(path, False)
+            with self.assertRaises(DeprecationWarning):
+                self.util.cache_from_source(path, True)
 
     def test_cache_from_source_cwd(self):
         path = 'foo.py'
         expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag))
-        self.assertEqual(self.util.cache_from_source(path, True), expect)
+        self.assertEqual(self.util.cache_from_source(path, optimization=''),
+                         expect)
 
     def test_cache_from_source_override(self):
         # When debug_override is not None, it can be any true-ish or false-ish
         # value.
         path = os.path.join('foo', 'bar', 'baz.py')
-        partial_expect = os.path.join('foo', 'bar', '__pycache__',
-                                      'baz.{}.py'.format(self.tag))
-        self.assertEqual(self.util.cache_from_source(path, []), partial_expect + 'o')
-        self.assertEqual(self.util.cache_from_source(path, [17]),
-                         partial_expect + 'c')
         # However if the bool-ishness can't be determined, the exception
         # propagates.
         class Bearish:
             def __bool__(self): raise RuntimeError
-        with self.assertRaises(RuntimeError):
-            self.util.cache_from_source('/foo/bar/baz.py', Bearish())
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            self.assertEqual(self.util.cache_from_source(path, []),
+                             self.util.cache_from_source(path, optimization=1))
+            self.assertEqual(self.util.cache_from_source(path, [17]),
+                             self.util.cache_from_source(path, optimization=''))
+            with self.assertRaises(RuntimeError):
+                self.util.cache_from_source('/foo/bar/baz.py', Bearish())
+
+
+    def test_cache_from_source_optimization_empty_string(self):
+        # Setting 'optimization' to '' leads to no optimization tag (PEP 488).
+        path = 'foo.py'
+        expect = os.path.join('__pycache__', 'foo.{}.pyc'.format(self.tag))
+        self.assertEqual(self.util.cache_from_source(path, optimization=''),
+                         expect)
+
+    def test_cache_from_source_optimization_None(self):
+        # Setting 'optimization' to None uses the interpreter's optimization.
+        # (PEP 488)
+        path = 'foo.py'
+        optimization_level = sys.flags.optimize
+        almost_expect = os.path.join('__pycache__', 'foo.{}'.format(self.tag))
+        if optimization_level == 0:
+            expect = almost_expect + '.pyc'
+        elif optimization_level <= 2:
+            expect = almost_expect + '.opt-{}.pyc'.format(optimization_level)
+        else:
+            msg = '{!r} is a non-standard optimization level'.format(optimization_level)
+            self.skipTest(msg)
+        self.assertEqual(self.util.cache_from_source(path, optimization=None),
+                         expect)
+
+    def test_cache_from_source_optimization_set(self):
+        # The 'optimization' parameter accepts anything that has a string repr
+        # that passes str.alnum().
+        path = 'foo.py'
+        valid_characters = string.ascii_letters + string.digits
+        almost_expect = os.path.join('__pycache__', 'foo.{}'.format(self.tag))
+        got = self.util.cache_from_source(path, optimization=valid_characters)
+        # Test all valid characters are accepted.
+        self.assertEqual(got,
+                         almost_expect + '.opt-{}.pyc'.format(valid_characters))
+        # str() should be called on argument.
+        self.assertEqual(self.util.cache_from_source(path, optimization=42),
+                         almost_expect + '.opt-42.pyc')
+        # Invalid characters raise ValueError.
+        with self.assertRaises(ValueError):
+            self.util.cache_from_source(path, optimization='path/is/bad')
+
+    def test_cache_from_source_debug_override_optimization_both_set(self):
+        # Can only set one of the optimization-related parameters.
+        with warnings.catch_warnings():
+            warnings.simplefilter('ignore')
+            with self.assertRaises(TypeError):
+                self.util.cache_from_source('foo.py', False, optimization='')
 
     @unittest.skipUnless(os.sep == '\\' and os.altsep == '/',
                      'test meaningful only where os.altsep is defined')
     def test_sep_altsep_and_sep_cache_from_source(self):
         # Windows path and PEP 3147 where sep is right of altsep.
         self.assertEqual(
-            self.util.cache_from_source('\\foo\\bar\\baz/qux.py', True),
+            self.util.cache_from_source('\\foo\\bar\\baz/qux.py', optimization=''),
             '\\foo\\bar\\baz\\__pycache__\\qux.{}.pyc'.format(self.tag))
 
     @unittest.skipUnless(sys.implementation.cache_tag is not None,
@@ -649,7 +711,12 @@ class PEP3147Tests:
             ValueError, self.util.source_from_cache, '__pycache__/foo.pyc')
 
     def test_source_from_cache_too_many_dots(self):
-        # Too many dots in final path component -> ValueError
+        with self.assertRaises(ValueError):
+            self.util.source_from_cache(
+                    '__pycache__/foo.cpython-32.opt-1.foo.pyc')
+
+    def test_source_from_cache_not_opt(self):
+        # Non-`opt-` path component -> ValueError
         self.assertRaises(
             ValueError, self.util.source_from_cache,
             '__pycache__/foo.cpython-32.foo.pyc')
@@ -660,6 +727,17 @@ class PEP3147Tests:
             ValueError, self.util.source_from_cache,
             '/foo/bar/foo.cpython-32.foo.pyc')
 
+    def test_source_from_cache_optimized_bytecode(self):
+        # Optimized bytecode is not an issue.
+        path = os.path.join('__pycache__', 'foo.{}.opt-1.pyc'.format(self.tag))
+        self.assertEqual(self.util.source_from_cache(path), 'foo.py')
+
+    def test_source_from_cache_missing_optimization(self):
+        # An empty optimization level is a no-no.
+        path = os.path.join('__pycache__', 'foo.{}.opt-.pyc'.format(self.tag))
+        with self.assertRaises(ValueError):
+            self.util.source_from_cache(path)
+
 
 (Frozen_PEP3147Tests,
  Source_PEP3147Tests