diff options
Diffstat (limited to 'Tools/scripts')
| -rw-r--r-- | Tools/scripts/README | 4 | ||||
| -rwxr-xr-x | Tools/scripts/diff.py | 16 | ||||
| -rwxr-xr-x | Tools/scripts/findnocoding.py | 4 | ||||
| -rwxr-xr-x | Tools/scripts/highlight.py | 181 | ||||
| -rwxr-xr-x | Tools/scripts/import_diagnostics.py | 37 | ||||
| -rwxr-xr-x | Tools/scripts/patchcheck.py | 53 | ||||
| -rwxr-xr-x | Tools/scripts/pysource.py | 2 | ||||
| -rwxr-xr-x | Tools/scripts/pyvenv | 11 | ||||
| -rwxr-xr-x | Tools/scripts/reindent.py | 18 | ||||
| -rwxr-xr-x | Tools/scripts/run_tests.py | 51 |
10 files changed, 339 insertions, 38 deletions
diff --git a/Tools/scripts/README b/Tools/scripts/README index 8c02529..d65d1fd 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -15,7 +15,7 @@ db2pickle.py Dump a database file to a pickle diff.py Print file diffs in context, unified, or ndiff formats dutree.py Format du(1) output as a tree sorted by size eptags.py Create Emacs TAGS file for Python modules -find_recursionlimit.py Find the maximum recursion limit on this machine +find_recursionlimit.py Find the maximum recursion limit on this machine finddiv.py A grep-like tool that looks for division operators findlinksto.py Recursively find symbolic links to a given path prefix findnocoding.py Find source files which need an encoding declaration @@ -28,6 +28,7 @@ ftpmirror.py FTP mirror script google.py Open a webbrowser with Google gprof2html.py Transform gprof(1) output into useful HTML h2py.py Translate #define's into Python assignments +highlight.py Python syntax highlighting with HTML output idle3 Main program to start IDLE ifdef.py Remove #if(n)def groups from C sources lfcr.py Change LF line endings to CRLF (Unix to Windows) @@ -53,6 +54,7 @@ redemo.py Basic regular expression demonstration facility reindent.py Change .py files to use 4-space indents reindent-rst.py Fix-up reStructuredText file whitespace rgrep.py Reverse grep through a file (useful for big logfiles) +run_tests.py Run the test suite with more sensible default options serve.py Small wsgiref-based web server, used in make serve in Doc suff.py Sort a list of files by suffix svneol.py Set svn:eol-style on all files in directory diff --git a/Tools/scripts/diff.py b/Tools/scripts/diff.py index 9efb078..f9b14bf 100755 --- a/Tools/scripts/diff.py +++ b/Tools/scripts/diff.py @@ -9,6 +9,12 @@ """ import sys, os, time, difflib, optparse +from datetime import datetime, timezone + +def file_mtime(path): + t = datetime.fromtimestamp(os.stat(path).st_mtime, + timezone.utc) + return t.astimezone().isoformat() def main(): @@ -30,10 +36,12 @@ def main(): n = options.lines fromfile, tofile = args - fromdate = time.ctime(os.stat(fromfile).st_mtime) - todate = time.ctime(os.stat(tofile).st_mtime) - fromlines = open(fromfile, 'U').readlines() - tolines = open(tofile, 'U').readlines() + fromdate = file_mtime(fromfile) + todate = file_mtime(tofile) + with open(fromfile, 'U') as ff: + fromlines = ff.readlines() + with open(tofile, 'U') as tf: + tolines = tf.readlines() if options.u: diff = difflib.unified_diff(fromlines, tolines, fromfile, tofile, fromdate, todate, n=n) diff --git a/Tools/scripts/findnocoding.py b/Tools/scripts/findnocoding.py index a494a48..5aa1feb 100755 --- a/Tools/scripts/findnocoding.py +++ b/Tools/scripts/findnocoding.py @@ -2,7 +2,7 @@ """List all those Python files that require a coding directive -Usage: nocoding.py dir1 [dir2...] +Usage: findnocoding.py dir1 [dir2...] """ __author__ = "Oleg Broytmann, Georg Brandl" @@ -50,7 +50,7 @@ def has_correct_encoding(text, codec): def needs_declaration(fullpath): try: - infile = open(fullpath, 'rU') + infile = open(fullpath) except IOError: # Oops, the file was removed - ignore it return None diff --git a/Tools/scripts/highlight.py b/Tools/scripts/highlight.py new file mode 100755 index 0000000..5112523 --- /dev/null +++ b/Tools/scripts/highlight.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +'''Add syntax highlighting to Python source code + +Example command-line calls: + + # Show syntax highlighted code in the terminal window + $ ./highlight.py myfile.py + + # Colorize myfile.py and display in a browser + $ ./highlight.py -b myfile.py + + # Create an HTML section that can be embedded in an existing webpage + ./highlight.py -s myfile.py + + # Create a complete HTML file + $ ./highlight.py -c myfile.py > myfile.html + +''' + +__all__ = ['colorize_html', 'build_page', 'default_css', 'default_html', + 'colorize_ansi', 'default_ansi'] +__author__ = 'Raymond Hettinger' + +import keyword, tokenize, cgi, functools + +def is_builtin(s): + 'Return True if s is the name of a builtin' + return s in vars(__builtins__) + +def combine_range(lines, start, end): + 'Join content from a range of lines between start and end' + (srow, scol), (erow, ecol) = start, end + if srow == erow: + rows = [lines[srow-1][scol:ecol]] + else: + rows = [lines[srow-1][scol:]] + lines[srow: erow-1] + [lines[erow-1][:ecol]] + return ''.join(rows), end + +def isolate_tokens(source): + 'Generate chunks of source and identify chunks to be highlighted' + lines = source.splitlines(True) + lines.append('') + readline = functools.partial(next, iter(lines), '') + kind = tok_str = '' + tok_type = tokenize.COMMENT + written = (1, 0) + for tok in tokenize.generate_tokens(readline): + prev_tok_type, prev_tok_str = tok_type, tok_str + tok_type, tok_str, (srow, scol), (erow, ecol), logical_lineno = tok + kind = '' + if tok_type == tokenize.COMMENT: + kind = 'comment' + elif tok_type == tokenize.OP and tok_str[:1] not in '{}[](),.:;': + kind = 'operator' + elif tok_type == tokenize.STRING: + kind = 'string' + if prev_tok_type == tokenize.INDENT or scol==0: + kind = 'docstring' + elif tok_type == tokenize.NAME: + if tok_str in ('def', 'class', 'import', 'from'): + kind = 'definition' + elif prev_tok_str in ('def', 'class'): + kind = 'defname' + elif keyword.iskeyword(tok_str): + kind = 'keyword' + elif is_builtin(tok_str) and prev_tok_str != '.': + kind = 'builtin' + line_upto_token, written = combine_range(lines, written, (srow, scol)) + line_thru_token, written = combine_range(lines, written, (erow, ecol)) + yield kind, line_upto_token, line_thru_token + +default_ansi = { + 'comment': '\033[0;31m', + 'string': '\033[0;32m', + 'docstring': '\033[0;32m', + 'keyword': '\033[0;33m', + 'builtin': '\033[0;35m', + 'definition': '\033[0;33m', + 'defname': '\033[0;34m', + 'operator': '\033[0;33m', +} + +def colorize_ansi(source, colors=default_ansi): + 'Add syntax highlighting to Python source code using ANSI escape sequences' + # http://en.wikipedia.org/wiki/ANSI_escape_code + result = [] + for kind, line_upto_token, line_thru_token in isolate_tokens(source): + if kind: + result += [line_upto_token, colors[kind], line_thru_token, '\033[0m'] + else: + result += [line_upto_token, line_thru_token] + return ''.join(result) + +def colorize_html(source): + 'Convert Python source code to an HTML fragment with colorized markup' + result = ['<pre class="python">\n'] + for kind, line_upto_token, line_thru_token in isolate_tokens(source): + if kind: + result += [cgi.escape(line_upto_token), + '<span class="%s">' % kind, + cgi.escape(line_thru_token), + '</span>'] + else: + result += [cgi.escape(line_upto_token), + cgi.escape(line_thru_token)] + result += ['</pre>\n'] + return ''.join(result) + +default_css = { + '.comment': '{color: crimson;}', + '.string': '{color: forestgreen;}', + '.docstring': '{color: forestgreen; font-style:italic;}', + '.keyword': '{color: darkorange;}', + '.builtin': '{color: purple;}', + '.definition': '{color: darkorange; font-weight:bold;}', + '.defname': '{color: blue;}', + '.operator': '{color: brown;}', +} + +default_html = '''\ +<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01//EN" + "http://www.w3.org/TR/html4/strict.dtd"> +<html> +<head> +<meta http-equiv="Content-type" content="text/html;charset=UTF-8"> +<title> {title} </title> +<style type="text/css"> +{css} +</style> +</head> +<body> +{body} +</body> +</html> +''' + +def build_page(source, title='python', css=default_css, html=default_html): + 'Create a complete HTML page with colorized Python source code' + css_str = '\n'.join(['%s %s' % item for item in css.items()]) + result = colorize_html(source) + title = cgi.escape(title) + return html.format(title=title, css=css_str, body=result) + + +if __name__ == '__main__': + import sys, argparse, webbrowser, os + + parser = argparse.ArgumentParser( + description = 'Add syntax highlighting to Python source') + parser.add_argument('sourcefile', metavar = 'SOURCEFILE', + help = 'File containing Python sourcecode') + parser.add_argument('-b', '--browser', action = 'store_true', + help = 'launch a browser to show results') + parser.add_argument('-c', '--complete', action = 'store_true', + help = 'build a complete html webpage') + parser.add_argument('-s', '--section', action = 'store_true', + help = 'show an HTML section rather than a complete webpage') + args = parser.parse_args() + + if args.section and (args.browser or args.complete): + parser.error('The -s/--section option is incompatible with ' + 'the -b/--browser or -c/--complete options') + + sourcefile = args.sourcefile + with open(sourcefile) as f: + source = f.read() + + if args.complete or args.browser: + encoded = build_page(source, title=sourcefile) + elif args.section: + encoded = colorize_html(source) + else: + encoded = colorize_ansi(source) + + if args.browser: + htmlfile = os.path.splitext(os.path.basename(sourcefile))[0] + '.html' + with open(htmlfile, 'w') as f: + f.write(encoded) + webbrowser.open('file://' + os.path.abspath(htmlfile)) + else: + sys.stdout.write(encoded) diff --git a/Tools/scripts/import_diagnostics.py b/Tools/scripts/import_diagnostics.py new file mode 100755 index 0000000..c907221 --- /dev/null +++ b/Tools/scripts/import_diagnostics.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python3 +"""Miscellaneous diagnostics for the import system""" + +import sys +import argparse +from pprint import pprint + +def _dump_state(args): + print(sys.version) + for name in args.attributes: + print("sys.{}:".format(name)) + pprint(getattr(sys, name)) + +def _add_dump_args(cmd): + cmd.add_argument("attributes", metavar="ATTR", nargs="+", + help="sys module attribute to display") + +COMMANDS = ( + ("dump", "Dump import state", _dump_state, _add_dump_args), +) + +def _make_parser(): + parser = argparse.ArgumentParser() + sub = parser.add_subparsers(title="Commands") + for name, description, implementation, add_args in COMMANDS: + cmd = sub.add_parser(name, help=description) + cmd.set_defaults(command=implementation) + add_args(cmd) + return parser + +def main(args): + parser = _make_parser() + args = parser.parse_args(args) + return args.command(args) + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/Tools/scripts/patchcheck.py b/Tools/scripts/patchcheck.py index 0e18dd9..503c67a 100755 --- a/Tools/scripts/patchcheck.py +++ b/Tools/scripts/patchcheck.py @@ -49,29 +49,15 @@ def mq_patches_applied(): @status("Getting the list of files that have been added/changed", info=lambda x: n_files_str(len(x))) def changed_files(): - """Get the list of changed or added files from the VCS.""" - if os.path.isdir(os.path.join(SRCDIR, '.hg')): - vcs = 'hg' - cmd = 'hg status --added --modified --no-status' - if mq_patches_applied(): - cmd += ' --rev qparent' - elif os.path.isdir('.svn'): - vcs = 'svn' - cmd = 'svn status --quiet --non-interactive --ignore-externals' - else: + """Get the list of changed or added files from Mercurial.""" + if not os.path.isdir(os.path.join(SRCDIR, '.hg')): sys.exit('need a checkout to get modified files') - st = subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) - try: - st.wait() - if vcs == 'hg': - return [x.decode().rstrip() for x in st.stdout] - else: - output = (x.decode().rstrip().rsplit(None, 1)[-1] - for x in st.stdout if x[0] in b'AM') - return set(path for path in output if os.path.isfile(path)) - finally: - st.stdout.close() + cmd = 'hg status --added --modified --no-status' + if mq_patches_applied(): + cmd += ' --rev qparent' + with subprocess.Popen(cmd.split(), stdout=subprocess.PIPE) as st: + return [x.decode().rstrip() for x in st.stdout] def report_modified_files(file_paths): @@ -89,10 +75,8 @@ def report_modified_files(file_paths): def normalize_whitespace(file_paths): """Make sure that the whitespace for .py files have been normalized.""" reindent.makebackup = False # No need to create backups. - fixed = [] - for path in (x for x in file_paths if x.endswith('.py')): - if reindent.check(os.path.join(SRCDIR, path)): - fixed.append(path) + fixed = [path for path in file_paths if path.endswith('.py') and + reindent.check(os.path.join(SRCDIR, path))] return fixed @@ -148,6 +132,21 @@ def reported_news(file_paths): """Check if Misc/NEWS has been changed.""" return 'Misc/NEWS' in file_paths +@status("configure regenerated", modal=True, info=str) +def regenerated_configure(file_paths): + """Check if configure has been regenerated.""" + if 'configure.ac' in file_paths: + return "yes" if 'configure' in file_paths else "no" + else: + return "not needed" + +@status("pyconfig.h.in regenerated", modal=True, info=str) +def regenerated_pyconfig_h_in(file_paths): + """Check if pyconfig.h.in has been regenerated.""" + if 'configure.ac' in file_paths: + return "yes" if 'pyconfig.h.in' in file_paths else "no" + else: + return "not needed" def main(): file_paths = changed_files() @@ -167,6 +166,10 @@ def main(): credit_given(special_files) # Misc/NEWS changed. reported_news(special_files) + # Regenerated configure, if necessary. + regenerated_configure(file_paths) + # Regenerated pyconfig.h.in, if necessary. + regenerated_pyconfig_h_in(file_paths) # Test suite run and passed. if python_files or c_files: diff --git a/Tools/scripts/pysource.py b/Tools/scripts/pysource.py index 048131e..c7dbe60 100755 --- a/Tools/scripts/pysource.py +++ b/Tools/scripts/pysource.py @@ -42,7 +42,7 @@ def _open(fullpath): return None try: - return open(fullpath, 'rU') + return open(fullpath) except IOError as err: # Access denied, or a special file - ignore it print_debug("%s: access denied: %s" % (fullpath, err)) return None diff --git a/Tools/scripts/pyvenv b/Tools/scripts/pyvenv new file mode 100755 index 0000000..978d691 --- /dev/null +++ b/Tools/scripts/pyvenv @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +if __name__ == '__main__': + import sys + rc = 1 + try: + import venv + venv.main() + rc = 0 + except Exception as e: + print('Error: %s' % e, file=sys.stderr) + sys.exit(rc) diff --git a/Tools/scripts/reindent.py b/Tools/scripts/reindent.py index b18993b..4a916ea 100755 --- a/Tools/scripts/reindent.py +++ b/Tools/scripts/reindent.py @@ -8,6 +8,8 @@ -r (--recurse) Recurse. Search for all .py files in subdirectories too. -n (--nobackup) No backup. Does not make a ".bak" file before reindenting. -v (--verbose) Verbose. Print informative msgs; else no output. + (--newline) Newline. Specify the newline character to use (CRLF, LF). + Default is the same as the original file. -h (--help) Help. Print this usage information and exit. Change Python (.py) files to use 4-space indents and no hard tab characters. @@ -50,6 +52,8 @@ verbose = False recurse = False dryrun = False makebackup = True +spec_newline = None +"""A specified newline to be used in the output (set by --newline option)""" def usage(msg=None): @@ -62,13 +66,12 @@ def errprint(*args): sys.stderr.write(" ".join(str(arg) for arg in args)) sys.stderr.write("\n") - def main(): import getopt - global verbose, recurse, dryrun, makebackup + global verbose, recurse, dryrun, makebackup, spec_newline try: opts, args = getopt.getopt(sys.argv[1:], "drnvh", - ["dryrun", "recurse", "nobackup", "verbose", "help"]) + ["dryrun", "recurse", "nobackup", "verbose", "newline=", "help"]) except getopt.error as msg: usage(msg) return @@ -81,6 +84,11 @@ def main(): makebackup = False elif o in ('-v', '--verbose'): verbose = True + elif o in ('--newline',): + if not a.upper() in ('CRLF', 'LF'): + usage() + return + spec_newline = dict(CRLF='\r\n', LF='\n')[a.upper()] elif o in ('-h', '--help'): usage() return @@ -118,9 +126,9 @@ def check(file): errprint("%s: I/O Error: %s" % (file, str(msg))) return - newline = r.newlines + newline = spec_newline if spec_newline else r.newlines if isinstance(newline, tuple): - errprint("%s: mixed newlines detected; cannot process file" % file) + errprint("%s: mixed newlines detected; cannot continue without --newline" % file) return if r.run(): diff --git a/Tools/scripts/run_tests.py b/Tools/scripts/run_tests.py new file mode 100755 index 0000000..e2a2050 --- /dev/null +++ b/Tools/scripts/run_tests.py @@ -0,0 +1,51 @@ +"""Run Python's test suite in a fast, rigorous way. + +The defaults are meant to be reasonably thorough, while skipping certain +tests that can be time-consuming or resource-intensive (e.g. largefile), +or distracting (e.g. audio and gui). These defaults can be overridden by +simply passing a -u option to this script. + +""" + +import os +import sys +import test.support +try: + import threading +except ImportError: + threading = None + + +def is_multiprocess_flag(arg): + return arg.startswith('-j') or arg.startswith('--multiprocess') + + +def is_resource_use_flag(arg): + return arg.startswith('-u') or arg.startswith('--use') + + +def main(regrtest_args): + args = [sys.executable, + '-W', 'default', # Warnings set to 'default' + '-bb', # Warnings about bytes/bytearray + '-E', # Ignore environment variables + ] + # Allow user-specified interpreter options to override our defaults. + args.extend(test.support.args_from_interpreter_flags()) + args.extend(['-m', 'test', # Run the test suite + '-r', # Randomize test order + '-w', # Re-run failed tests in verbose mode + ]) + if sys.platform == 'win32': + args.append('-n') # Silence alerts under Windows + if threading and not any(is_multiprocess_flag(arg) for arg in regrtest_args): + args.extend(['-j', '0']) # Use all CPU cores + if not any(is_resource_use_flag(arg) for arg in regrtest_args): + args.extend(['-u', 'all,-largefile,-audio,-gui']) + args.extend(regrtest_args) + print(' '.join(args)) + os.execv(sys.executable, args) + + +if __name__ == '__main__': + main(sys.argv[1:]) |