diff options
| -rw-r--r-- | Doc/library/2to3.rst | 32 | ||||
| -rw-r--r-- | Lib/lib2to3/main.py | 93 | ||||
| -rw-r--r-- | Lib/lib2to3/refactor.py | 20 | ||||
| -rw-r--r-- | Lib/lib2to3/tests/test_main.py | 98 | ||||
| -rw-r--r-- | Lib/lib2to3/tests/test_refactor.py | 36 | ||||
| -rw-r--r-- | Misc/NEWS | 5 |
6 files changed, 270 insertions, 14 deletions
diff --git a/Doc/library/2to3.rst b/Doc/library/2to3.rst index fe1c2be..a13fb65 100644 --- a/Doc/library/2to3.rst +++ b/Doc/library/2to3.rst @@ -94,6 +94,38 @@ change can also be enabled manually with the :option:`-p` flag. Use :option:`-p` to run fixers on code that already has had its print statements converted. +The :option:`-o` or :option:`--output-dir` option allows specification of an +alternate directory for processed output files to be written to. The +:option:`-n` flag is required when using this as backup files do not make sense +when not overwriting the input files. + +.. versionadded:: 3.2.3 + The :option:`-o` option was added. + +The :option:`-W` or :option:`--write-unchanged-files` flag tells 2to3 to always +write output files even if no changes were required to the file. This is most +useful with :option:`-o` so that an entire Python source tree is copied with +translation from one directory to another. +This option implies the :option:`-w` flag as it would not make sense otherwise. + +.. versionadded:: 3.2.3 + The :option:`-W` flag was added. + +The :option:`--add-suffix` option specifies a string to append to all output +filenames. The :option:`-n` flag is required when specifying this as backups +are not necessary when writing to different filenames. Example:: + + $ 2to3 -n -W --add-suffix=3 example.py + +Will cause a converted file named ``example.py3`` to be written. + +.. versionadded:: 3.2.3 + The :option:`--add-suffix` option was added. + +To translate an entire project from one directory tree to another use:: + + $ 2to3 --output-dir=python3-version/mycode -W -n python2-version/mycode + .. _2to3-fixers: diff --git a/Lib/lib2to3/main.py b/Lib/lib2to3/main.py index 20ee434..ddcf002 100644 --- a/Lib/lib2to3/main.py +++ b/Lib/lib2to3/main.py @@ -25,12 +25,41 @@ def diff_texts(a, b, filename): class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool): """ + A refactoring tool that can avoid overwriting its input files. Prints output to stdout. + + Output files can optionally be written to a different directory and or + have an extra file suffix appended to their name for use in situations + where you do not want to replace the input files. """ - def __init__(self, fixers, options, explicit, nobackups, show_diffs): + def __init__(self, fixers, options, explicit, nobackups, show_diffs, + input_base_dir='', output_dir='', append_suffix=''): + """ + Args: + fixers: A list of fixers to import. + options: A dict with RefactoringTool configuration. + explicit: A list of fixers to run even if they are explicit. + nobackups: If true no backup '.bak' files will be created for those + files that are being refactored. + show_diffs: Should diffs of the refactoring be printed to stdout? + input_base_dir: The base directory for all input files. This class + will strip this path prefix off of filenames before substituting + it with output_dir. Only meaningful if output_dir is supplied. + All files processed by refactor() must start with this path. + output_dir: If supplied, all converted files will be written into + this directory tree instead of input_base_dir. + append_suffix: If supplied, all files output by this tool will have + this appended to their filename. Useful for changing .py to + .py3 for example by passing append_suffix='3'. + """ self.nobackups = nobackups self.show_diffs = show_diffs + if input_base_dir and not input_base_dir.endswith(os.sep): + input_base_dir += os.sep + self._input_base_dir = input_base_dir + self._output_dir = output_dir + self._append_suffix = append_suffix super(StdoutRefactoringTool, self).__init__(fixers, options, explicit) def log_error(self, msg, *args, **kwargs): @@ -38,6 +67,23 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool): self.logger.error(msg, *args, **kwargs) def write_file(self, new_text, filename, old_text, encoding): + orig_filename = filename + if self._output_dir: + if filename.startswith(self._input_base_dir): + filename = os.path.join(self._output_dir, + filename[len(self._input_base_dir):]) + else: + raise ValueError('filename %s does not start with the ' + 'input_base_dir %s' % ( + filename, self._input_base_dir)) + if self._append_suffix: + filename += self._append_suffix + if orig_filename != filename: + output_dir = os.path.dirname(filename) + if not os.path.isdir(output_dir): + os.makedirs(output_dir) + self.log_message('Writing converted %s to %s.', orig_filename, + filename) if not self.nobackups: # Make backup backup = filename + ".bak" @@ -55,6 +101,9 @@ class StdoutRefactoringTool(refactor.MultiprocessRefactoringTool): write(new_text, filename, old_text, encoding) if not self.nobackups: shutil.copymode(backup, filename) + if orig_filename != filename: + # Preserve the file mode in the new output directory. + shutil.copymode(orig_filename, filename) def print_output(self, old, new, filename, equal): if equal: @@ -113,11 +162,33 @@ def main(fixer_pkg, args=None): help="Write back modified files") parser.add_option("-n", "--nobackups", action="store_true", default=False, help="Don't write backups for modified files") + parser.add_option("-o", "--output-dir", action="store", type="str", + default="", help="Put output files in this directory " + "instead of overwriting the input files. Requires -n.") + parser.add_option("-W", "--write-unchanged-files", action="store_true", + help="Also write files even if no changes were required" + " (useful with --output-dir); implies -w.") + parser.add_option("--add-suffix", action="store", type="str", default="", + help="Append this string to all output filenames." + " Requires -n if non-empty. " + "ex: --add-suffix='3' will generate .py3 files.") # Parse command line arguments refactor_stdin = False flags = {} options, args = parser.parse_args(args) + if options.write_unchanged_files: + flags["write_unchanged_files"] = True + if not options.write: + warn("--write-unchanged-files/-W implies -w.") + options.write = True + # If we allowed these, the original files would be renamed to backup names + # but not replaced. + if options.output_dir and not options.nobackups: + parser.error("Can't use --output-dir/-o without -n.") + if options.add_suffix and not options.nobackups: + parser.error("Can't use --add-suffix without -n.") + if not options.write and options.no_diffs: warn("not writing files and not printing diffs; that's not very useful") if not options.write and options.nobackups: @@ -143,6 +214,7 @@ def main(fixer_pkg, args=None): # Set up logging handler level = logging.DEBUG if options.verbose else logging.INFO logging.basicConfig(format='%(name)s: %(message)s', level=level) + logger = logging.getLogger('lib2to3.main') # Initialize the refactoring tool avail_fixes = set(refactor.get_fixers_from_package(fixer_pkg)) @@ -159,8 +231,23 @@ def main(fixer_pkg, args=None): else: requested = avail_fixes.union(explicit) fixer_names = requested.difference(unwanted_fixes) - rt = StdoutRefactoringTool(sorted(fixer_names), flags, sorted(explicit), - options.nobackups, not options.no_diffs) + input_base_dir = os.path.commonprefix(args) + if (input_base_dir and not input_base_dir.endswith(os.sep) + and not os.path.isdir(input_base_dir)): + # One or more similar names were passed, their directory is the base. + # os.path.commonprefix() is ignorant of path elements, this corrects + # for that weird API. + input_base_dir = os.path.dirname(input_base_dir) + if options.output_dir: + input_base_dir = input_base_dir.rstrip(os.sep) + logger.info('Output in %r will mirror the input directory %r layout.', + options.output_dir, input_base_dir) + rt = StdoutRefactoringTool( + sorted(fixer_names), flags, sorted(explicit), + options.nobackups, not options.no_diffs, + input_base_dir=input_base_dir, + output_dir=options.output_dir, + append_suffix=options.add_suffix) # Refactor all files and directories passed as arguments if not rt.errors: diff --git a/Lib/lib2to3/refactor.py b/Lib/lib2to3/refactor.py index ae5e40f..7cd034a 100644 --- a/Lib/lib2to3/refactor.py +++ b/Lib/lib2to3/refactor.py @@ -173,7 +173,8 @@ class FixerError(Exception): class RefactoringTool(object): - _default_options = {"print_function" : False} + _default_options = {"print_function" : False, + "write_unchanged_files" : False} CLASS_PREFIX = "Fix" # The prefix for fixer classes FILE_PREFIX = "fix_" # The prefix for modules with a fixer within @@ -195,6 +196,10 @@ class RefactoringTool(object): self.grammar = pygram.python_grammar_no_print_statement else: self.grammar = pygram.python_grammar + # When this is True, the refactor*() methods will call write_file() for + # files processed even if they were not changed during refactoring. If + # and only if the refactor method's write parameter was True. + self.write_unchanged_files = self.options.get("write_unchanged_files") self.errors = [] self.logger = logging.getLogger("RefactoringTool") self.fixer_log = [] @@ -341,13 +346,13 @@ class RefactoringTool(object): if doctests_only: self.log_debug("Refactoring doctests in %s", filename) output = self.refactor_docstring(input, filename) - if output != input: + if self.write_unchanged_files or output != input: self.processed_file(output, filename, input, write, encoding) else: self.log_debug("No doctest changes in %s", filename) else: tree = self.refactor_string(input, filename) - if tree and tree.was_changed: + if self.write_unchanged_files or (tree and tree.was_changed): # The [:-1] is to take off the \n we added earlier self.processed_file(str(tree)[:-1], filename, write=write, encoding=encoding) @@ -386,13 +391,13 @@ class RefactoringTool(object): if doctests_only: self.log_debug("Refactoring doctests in stdin") output = self.refactor_docstring(input, "<stdin>") - if output != input: + if self.write_unchanged_files or output != input: self.processed_file(output, "<stdin>", input) else: self.log_debug("No doctest changes in stdin") else: tree = self.refactor_string(input, "<stdin>") - if tree and tree.was_changed: + if self.write_unchanged_files or (tree and tree.was_changed): self.processed_file(str(tree), "<stdin>", input) else: self.log_debug("No changes in stdin") @@ -502,7 +507,7 @@ class RefactoringTool(object): def processed_file(self, new_text, filename, old_text=None, write=False, encoding=None): """ - Called when a file has been refactored, and there are changes. + Called when a file has been refactored and there may be changes. """ self.files.append(filename) if old_text is None: @@ -513,7 +518,8 @@ class RefactoringTool(object): self.print_output(old_text, new_text, filename, equal) if equal: self.log_debug("No changes to %s", filename) - return + if not self.write_unchanged_files: + return if write: self.write_file(new_text, filename, old_text, encoding) else: diff --git a/Lib/lib2to3/tests/test_main.py b/Lib/lib2to3/tests/test_main.py index 754d5dd..407afa1 100644 --- a/Lib/lib2to3/tests/test_main.py +++ b/Lib/lib2to3/tests/test_main.py @@ -1,18 +1,30 @@ # -*- coding: utf-8 -*- -import sys import codecs -import logging import io +import logging +import os +import shutil +import sys +import tempfile import unittest from lib2to3 import main +TEST_DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +PY2_TEST_MODULE = os.path.join(TEST_DATA_DIR, "py2_test_grammar.py") + + class TestMain(unittest.TestCase): + def setUp(self): + self.temp_dir = None # tearDown() will rmtree this directory if set. + def tearDown(self): # Clean up logging configuration down by main. del logging.root.handlers[:] + if self.temp_dir: + shutil.rmtree(self.temp_dir) def run_2to3_capture(self, args, in_capture, out_capture, err_capture): save_stdin = sys.stdin @@ -39,3 +51,85 @@ class TestMain(unittest.TestCase): self.assertTrue("-print 'nothing'" in output) self.assertTrue("WARNING: couldn't encode <stdin>'s diff for " "your terminal" in err.getvalue()) + + def setup_test_source_trees(self): + """Setup a test source tree and output destination tree.""" + self.temp_dir = tempfile.mkdtemp() # tearDown() cleans this up. + self.py2_src_dir = os.path.join(self.temp_dir, "python2_project") + self.py3_dest_dir = os.path.join(self.temp_dir, "python3_project") + os.mkdir(self.py2_src_dir) + os.mkdir(self.py3_dest_dir) + # Turn it into a package with a few files. + self.setup_files = [] + open(os.path.join(self.py2_src_dir, "__init__.py"), "w").close() + self.setup_files.append("__init__.py") + shutil.copy(PY2_TEST_MODULE, self.py2_src_dir) + self.setup_files.append(os.path.basename(PY2_TEST_MODULE)) + self.trivial_py2_file = os.path.join(self.py2_src_dir, "trivial.py") + self.init_py2_file = os.path.join(self.py2_src_dir, "__init__.py") + with open(self.trivial_py2_file, "w") as trivial: + trivial.write("print 'I need a simple conversion.'") + self.setup_files.append("trivial.py") + + def test_filename_changing_on_output_single_dir(self): + """2to3 a single directory with a new output dir and suffix.""" + self.setup_test_source_trees() + out = io.StringIO() + err = io.StringIO() + suffix = "TEST" + ret = self.run_2to3_capture( + ["-n", "--add-suffix", suffix, "--write-unchanged-files", + "--no-diffs", "--output-dir", + self.py3_dest_dir, self.py2_src_dir], + io.StringIO(""), out, err) + self.assertEqual(ret, 0) + stderr = err.getvalue() + self.assertIn(" implies -w.", stderr) + self.assertIn( + "Output in %r will mirror the input directory %r layout" % ( + self.py3_dest_dir, self.py2_src_dir), stderr) + self.assertEqual(set(name+suffix for name in self.setup_files), + set(os.listdir(self.py3_dest_dir))) + for name in self.setup_files: + self.assertIn("Writing converted %s to %s" % ( + os.path.join(self.py2_src_dir, name), + os.path.join(self.py3_dest_dir, name+suffix)), stderr) + self.assertRegexpMatches(stderr, r"No changes to .*/__init__\.py") + self.assertNotRegex(stderr, r"No changes to .*/trivial\.py") + + def test_filename_changing_on_output_two_files(self): + """2to3 two files in one directory with a new output dir.""" + self.setup_test_source_trees() + err = io.StringIO() + py2_files = [self.trivial_py2_file, self.init_py2_file] + expected_files = set(os.path.basename(name) for name in py2_files) + ret = self.run_2to3_capture( + ["-n", "-w", "--write-unchanged-files", + "--no-diffs", "--output-dir", self.py3_dest_dir] + py2_files, + io.StringIO(""), io.StringIO(), err) + self.assertEqual(ret, 0) + stderr = err.getvalue() + self.assertIn( + "Output in %r will mirror the input directory %r layout" % ( + self.py3_dest_dir, self.py2_src_dir), stderr) + self.assertEqual(expected_files, set(os.listdir(self.py3_dest_dir))) + + def test_filename_changing_on_output_single_file(self): + """2to3 a single file with a new output dir.""" + self.setup_test_source_trees() + err = io.StringIO() + ret = self.run_2to3_capture( + ["-n", "-w", "--no-diffs", "--output-dir", self.py3_dest_dir, + self.trivial_py2_file], + io.StringIO(""), io.StringIO(), err) + self.assertEqual(ret, 0) + stderr = err.getvalue() + self.assertIn( + "Output in %r will mirror the input directory %r layout" % ( + self.py3_dest_dir, self.py2_src_dir), stderr) + self.assertEqual(set([os.path.basename(self.trivial_py2_file)]), + set(os.listdir(self.py3_dest_dir))) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/lib2to3/tests/test_refactor.py b/Lib/lib2to3/tests/test_refactor.py index 54edeb4..4b87ed6 100644 --- a/Lib/lib2to3/tests/test_refactor.py +++ b/Lib/lib2to3/tests/test_refactor.py @@ -53,6 +53,12 @@ class TestRefactoringTool(unittest.TestCase): self.assertTrue(rt.driver.grammar is pygram.python_grammar_no_print_statement) + def test_write_unchanged_files_option(self): + rt = self.rt() + self.assertFalse(rt.write_unchanged_files) + rt = self.rt({"write_unchanged_files" : True}) + self.assertTrue(rt.write_unchanged_files) + def test_fixer_loading_helpers(self): contents = ["explicit", "first", "last", "parrot", "preorder"] non_prefixed = refactor.get_all_fix_names("myfixes") @@ -176,7 +182,9 @@ from __future__ import print_function""" "<stdin>", False] self.assertEqual(results, expected) - def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS): + def check_file_refactoring(self, test_file, fixers=_2TO3_FIXERS, + options=None, mock_log_debug=None, + actually_write=True): tmpdir = tempfile.mkdtemp(prefix="2to3-test_refactor") self.addCleanup(shutil.rmtree, tmpdir) # make a copy of the tested file that we can write to @@ -189,11 +197,15 @@ from __future__ import print_function""" return fp.read() old_contents = read_file() - rt = self.rt(fixers=fixers) + rt = self.rt(fixers=fixers, options=options) + if mock_log_debug: + rt.log_debug = mock_log_debug rt.refactor_file(test_file) self.assertEqual(old_contents, read_file()) + if not actually_write: + return rt.refactor_file(test_file, True) new_contents = read_file() self.assertNotEqual(old_contents, new_contents) @@ -203,6 +215,26 @@ from __future__ import print_function""" test_file = os.path.join(FIXER_DIR, "parrot_example.py") self.check_file_refactoring(test_file, _DEFAULT_FIXERS) + def test_refactor_file_write_unchanged_file(self): + test_file = os.path.join(FIXER_DIR, "parrot_example.py") + debug_messages = [] + def recording_log_debug(msg, *args): + debug_messages.append(msg % args) + self.check_file_refactoring(test_file, fixers=(), + options={"write_unchanged_files": True}, + mock_log_debug=recording_log_debug, + actually_write=False) + # Testing that it logged this message when write=False was passed is + # sufficient to see that it did not bail early after "No changes". + message_regex = r"Not writing changes to .*%s%s" % ( + os.sep, os.path.basename(test_file)) + for message in debug_messages: + if "Not writing changes" in message: + self.assertRegexpMatches(message, message_regex) + break + else: + self.fail("%r not matched in %r" % (message_regex, debug_messages)) + def test_refactor_dir(self): def check(structure, expected): def mock_refactor_file(self, f, *args): @@ -443,6 +443,11 @@ Build Tools/Demos ----------- +- Issue #13930: 2to3 is now able to write its converted output files to another + directory tree as well as copying unchanged files and altering the file + suffix. See its new -o, -W and --add-suffix options. This makes it more + useful in many automated code translation workflows. + - Issue #13628: python-gdb.py is now able to retrieve more frames in the Python traceback if Python is optimized. |