diff options
Diffstat (limited to 'Tools')
174 files changed, 7842 insertions, 8194 deletions
diff --git a/Tools/README b/Tools/README index ae9b592..c1f89ba 100644 --- a/Tools/README +++ b/Tools/README @@ -1,33 +1,42 @@ This directory contains a number of Python programs that are useful while building or extending Python. -faqwiz FAQ Wizard. - See http://www.python.org/cgi-bin/faqw.py - for a live example. +buildbot Batchfiles for running on Windows buildslaves. -freeze Create a stand-alone executable from a Python program. +ccbench A Python concurrency benchmark. -i18n Tools for internationalization. pygettext.py - parses Python source code and generates .pot files, - and msgfmt.py generates a binary message catalog - from a catalog in text format. +demo Several Python programming demos. -modulator Interactively generate boiler plate for an extension - module. Works easiest if you have Tk. +freeze Create a stand-alone executable from a Python program. -pynche A Tkinter-based color editor. +gdb Python code to be run inside gdb, to make it easier to + debug Python itself (by David Malcolm). + +i18n Tools for internationalization. pygettext.py + parses Python source code and generates .pot files, + and msgfmt.py generates a binary message catalog + from a catalog in text format. + +iobench Benchmark for the new Python I/O system. + +msi Support for packaging Python as an MSI package on Windows. + +parser Un-parsing tool to generate code from an AST. + +pybench Comprehensive Python benchmarking suite. + +pynche A Tkinter-based color editor. scripts A number of useful single-file programs, e.g. tabnanny.py by Tim Peters, which checks for inconsistent mixing of tabs and spaces, and 2to3, which converts Python 2 code to Python 3 code. -unicode Tools used to generate unicode database files for - Python 2.0 (by Fredrik Lundh). +test2to3 A demonstration of how to use 2to3 transparently in setup.py. -webchecker A link checker for web sites. +unicode Tools for generating unicodedata and codecs from unicode.org + and other mapping files (by Fredrik Lundh, Marc-Andre Lemburg + and Martin von Loewis). -world Script to take a list of Internet addresses and print - out where in the world those addresses originate from, - based on the top-level domain country code found in - the address. +unittestgui A Tkinter based GUI test runner for unittest, with test + discovery. diff --git a/Tools/buildbot/clean-amd64.bat b/Tools/buildbot/clean-amd64.bat index 9fb35e9..715805a 100644 --- a/Tools/buildbot/clean-amd64.bat +++ b/Tools/buildbot/clean-amd64.bat @@ -1,7 +1,10 @@ @rem Used by the buildbot "clean" step. call "%VS90COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64 -cd PCbuild @echo Deleting .pyc/.pyo files ... del /s Lib\*.pyc Lib\*.pyo +@echo Deleting test leftovers ... +rmdir /s /q build +cd PCbuild vcbuild /clean pcbuild.sln "Release|x64" vcbuild /clean pcbuild.sln "Debug|x64" +cd .. diff --git a/Tools/buildbot/clean.bat b/Tools/buildbot/clean.bat index ec71804..5c8f33e 100644 --- a/Tools/buildbot/clean.bat +++ b/Tools/buildbot/clean.bat @@ -2,6 +2,9 @@ call "%VS90COMNTOOLS%vsvars32.bat" @echo Deleting .pyc/.pyo files ... del /s Lib\*.pyc Lib\*.pyo +@echo Deleting test leftovers ... +rmdir /s /q build cd PCbuild vcbuild /clean pcbuild.sln "Release|Win32" vcbuild /clean pcbuild.sln "Debug|Win32" +cd .. diff --git a/Tools/buildbot/external-amd64.bat b/Tools/buildbot/external-amd64.bat index 1cb2ce7..954238e 100644 --- a/Tools/buildbot/external-amd64.bat +++ b/Tools/buildbot/external-amd64.bat @@ -5,17 +5,17 @@ call "Tools\buildbot\external-common.bat" call "%VS90COMNTOOLS%\..\..\VC\vcvarsall.bat" x86_amd64 if not exist tcltk64\bin\tcl85g.dll ( - cd tcl-8.5.2.1\win + cd tcl-8.5.9.0\win nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=1 MACHINE=AMD64 INSTALLDIR=..\..\tcltk64 clean all nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=1 MACHINE=AMD64 INSTALLDIR=..\..\tcltk64 install cd ..\.. ) if not exist tcltk64\bin\tk85g.dll ( - cd tk-8.5.2.0\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 MACHINE=AMD64 INSTALLDIR=..\..\tcltk64 TCLDIR=..\..\tcl-8.5.2.1 clean - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 MACHINE=AMD64 INSTALLDIR=..\..\tcltk64 TCLDIR=..\..\tcl-8.5.2.1 all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 MACHINE=AMD64 INSTALLDIR=..\..\tcltk64 TCLDIR=..\..\tcl-8.5.2.1 install + cd tk-8.5.9.0\win + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 MACHINE=AMD64 INSTALLDIR=..\..\tcltk64 TCLDIR=..\..\tcl-8.5.9.0 clean + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 MACHINE=AMD64 INSTALLDIR=..\..\tcltk64 TCLDIR=..\..\tcl-8.5.9.0 all + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 MACHINE=AMD64 INSTALLDIR=..\..\tcltk64 TCLDIR=..\..\tcl-8.5.9.0 install cd ..\.. ) diff --git a/Tools/buildbot/external-common.bat b/Tools/buildbot/external-common.bat index 8beee14..1ff282e 100644 --- a/Tools/buildbot/external-common.bat +++ b/Tools/buildbot/external-common.bat @@ -14,8 +14,8 @@ cd .. @rem if exist tk8.4.16 rd /s/q tk8.4.16 @rem if exist tk-8.4.18.1 rd /s/q tk-8.4.18.1 @rem if exist db-4.4.20 rd /s/q db-4.4.20 -@rem if exist openssl-0.9.8l rd /s/q openssl-0.9.8l -@rem if exist sqlite-3.5.9 rd /s/q sqlite-3.5.9 +@rem if exist openssl-1.0.0a rd /s/q openssl-1.0.0a +@rem if exist sqlite-3.7.4 rd /s/q sqlite-3.7.4 @rem bzip if not exist bzip2-1.0.5 ( @@ -27,17 +27,17 @@ if not exist bzip2-1.0.5 ( if not exist db-4.4.20 svn export http://svn.python.org/projects/external/db-4.4.20-vs9 db-4.4.20 @rem OpenSSL -if not exist openssl-0.9.8l svn export http://svn.python.org/projects/external/openssl-0.9.8l +if not exist openssl-1.0.0a svn export http://svn.python.org/projects/external/openssl-1.0.0a @rem tcl/tk -if not exist tcl-8.5.2.1 ( +if not exist tcl-8.5.9.0 ( rd /s/q tcltk tcltk64 - svn export http://svn.python.org/projects/external/tcl-8.5.2.1 + svn export http://svn.python.org/projects/external/tcl-8.5.9.0 ) -if not exist tk-8.5.2.0 svn export http://svn.python.org/projects/external/tk-8.5.2.0 +if not exist tk-8.5.9.0 svn export http://svn.python.org/projects/external/tk-8.5.9.0 @rem sqlite3 -if not exist sqlite-3.5.9 ( - rd /s/q sqlite-source-3.3.4 - svn export http://svn.python.org/projects/external/sqlite-3.5.9 +if not exist sqlite-3.7.4 ( + rd /s/q sqlite-source-3.6.21 + svn export http://svn.python.org/projects/external/sqlite-3.7.4 ) diff --git a/Tools/buildbot/external.bat b/Tools/buildbot/external.bat index d90e8ce..e958fd6 100644 --- a/Tools/buildbot/external.bat +++ b/Tools/buildbot/external.bat @@ -6,16 +6,16 @@ call "%VS90COMNTOOLS%\vsvars32.bat" if not exist tcltk\bin\tcl85g.dll ( @rem all and install need to be separate invocations, otherwise nmakehlp is not found on install - cd tcl-8.5.2.1\win + cd tcl-8.5.9.0\win nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 DEBUG=1 INSTALLDIR=..\..\tcltk clean all nmake -f makefile.vc DEBUG=1 INSTALLDIR=..\..\tcltk install cd ..\.. ) if not exist tcltk\bin\tk85g.dll ( - cd tk-8.5.2.0\win - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl-8.5.2.1 clean - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl-8.5.2.1 all - nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl-8.5.2.1 install + cd tk-8.5.9.0\win + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl-8.5.9.0 clean + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl-8.5.9.0 all + nmake -f makefile.vc COMPILERFLAGS=-DWINVER=0x0500 OPTS=noxp DEBUG=1 INSTALLDIR=..\..\tcltk TCLDIR=..\..\tcl-8.5.9.0 install cd ..\.. ) diff --git a/Tools/buildbot/test.bat b/Tools/buildbot/test.bat index 16c07fb..5882def 100644 --- a/Tools/buildbot/test.bat +++ b/Tools/buildbot/test.bat @@ -1,4 +1,4 @@ @rem Used by the buildbot "test" step. cd PCbuild -call rt.bat -d -q -uall -rw -n +call rt.bat -d -q -uall -rwW -n diff --git a/Tools/ccbench/ccbench.py b/Tools/ccbench/ccbench.py new file mode 100644 index 0000000..9f7118f --- /dev/null +++ b/Tools/ccbench/ccbench.py @@ -0,0 +1,612 @@ +# This file should be kept compatible with both Python 2.6 and Python >= 3.0. + +from __future__ import division +from __future__ import print_function + +""" +ccbench, a Python concurrency benchmark. +""" + +import time +import os +import sys +import functools +import itertools +import threading +import subprocess +import socket +from optparse import OptionParser, SUPPRESS_HELP +import platform + +# Compatibility +try: + xrange +except NameError: + xrange = range + +try: + map = itertools.imap +except AttributeError: + pass + + +THROUGHPUT_DURATION = 2.0 + +LATENCY_PING_INTERVAL = 0.1 +LATENCY_DURATION = 2.0 + +BANDWIDTH_PACKET_SIZE = 1024 +BANDWIDTH_DURATION = 2.0 + + +def task_pidigits(): + """Pi calculation (Python)""" + _map = map + _count = itertools.count + _islice = itertools.islice + + def calc_ndigits(n): + # From http://shootout.alioth.debian.org/ + def gen_x(): + return _map(lambda k: (k, 4*k + 2, 0, 2*k + 1), _count(1)) + + def compose(a, b): + aq, ar, as_, at = a + bq, br, bs, bt = b + return (aq * bq, + aq * br + ar * bt, + as_ * bq + at * bs, + as_ * br + at * bt) + + def extract(z, j): + q, r, s, t = z + return (q*j + r) // (s*j + t) + + def pi_digits(): + z = (1, 0, 0, 1) + x = gen_x() + while 1: + y = extract(z, 3) + while y != extract(z, 4): + z = compose(z, next(x)) + y = extract(z, 3) + z = compose((10, -10*y, 0, 1), z) + yield y + + return list(_islice(pi_digits(), n)) + + return calc_ndigits, (50, ) + +def task_regex(): + """regular expression (C)""" + # XXX this task gives horrendous latency results. + import re + # Taken from the `inspect` module + pat = re.compile(r'^(\s*def\s)|(.*(?<!\w)lambda(:|\s))|^(\s*@)', re.MULTILINE) + with open(__file__, "r") as f: + arg = f.read(2000) + + def findall(s): + t = time.time() + try: + return pat.findall(s) + finally: + print(time.time() - t) + return pat.findall, (arg, ) + +def task_sort(): + """list sorting (C)""" + def list_sort(l): + l = l[::-1] + l.sort() + + return list_sort, (list(range(1000)), ) + +def task_compress_zlib(): + """zlib compression (C)""" + import zlib + with open(__file__, "rb") as f: + arg = f.read(5000) * 3 + + def compress(s): + zlib.decompress(zlib.compress(s, 5)) + return compress, (arg, ) + +def task_compress_bz2(): + """bz2 compression (C)""" + import bz2 + with open(__file__, "rb") as f: + arg = f.read(3000) * 2 + + def compress(s): + bz2.compress(s) + return compress, (arg, ) + +def task_hashing(): + """SHA1 hashing (C)""" + import hashlib + with open(__file__, "rb") as f: + arg = f.read(5000) * 30 + + def compute(s): + hashlib.sha1(s).digest() + return compute, (arg, ) + + +throughput_tasks = [task_pidigits, task_regex] +for mod in 'bz2', 'hashlib': + try: + globals()[mod] = __import__(mod) + except ImportError: + globals()[mod] = None + +# For whatever reasons, zlib gives irregular results, so we prefer bz2 or +# hashlib if available. +# (NOTE: hashlib releases the GIL from 2.7 and 3.1 onwards) +if bz2 is not None: + throughput_tasks.append(task_compress_bz2) +elif hashlib is not None: + throughput_tasks.append(task_hashing) +else: + throughput_tasks.append(task_compress_zlib) + +latency_tasks = throughput_tasks +bandwidth_tasks = [task_pidigits] + + +class TimedLoop: + def __init__(self, func, args): + self.func = func + self.args = args + + def __call__(self, start_time, min_duration, end_event, do_yield=False): + step = 20 + niters = 0 + duration = 0.0 + _time = time.time + _sleep = time.sleep + _func = self.func + _args = self.args + t1 = start_time + while True: + for i in range(step): + _func(*_args) + t2 = _time() + # If another thread terminated, the current measurement is invalid + # => return the previous one. + if end_event: + return niters, duration + niters += step + duration = t2 - start_time + if duration >= min_duration: + end_event.append(None) + return niters, duration + if t2 - t1 < 0.01: + # Minimize interference of measurement on overall runtime + step = step * 3 // 2 + elif do_yield: + # OS scheduling of Python threads is sometimes so bad that we + # have to force thread switching ourselves, otherwise we get + # completely useless results. + _sleep(0.0001) + t1 = t2 + + +def run_throughput_test(func, args, nthreads): + assert nthreads >= 1 + + # Warm up + func(*args) + + results = [] + loop = TimedLoop(func, args) + end_event = [] + + if nthreads == 1: + # Pure single-threaded performance, without any switching or + # synchronization overhead. + start_time = time.time() + results.append(loop(start_time, THROUGHPUT_DURATION, + end_event, do_yield=False)) + return results + + started = False + ready_cond = threading.Condition() + start_cond = threading.Condition() + ready = [] + + def run(): + with ready_cond: + ready.append(None) + ready_cond.notify() + with start_cond: + while not started: + start_cond.wait() + results.append(loop(start_time, THROUGHPUT_DURATION, + end_event, do_yield=True)) + + threads = [] + for i in range(nthreads): + threads.append(threading.Thread(target=run)) + for t in threads: + t.setDaemon(True) + t.start() + # We don't want measurements to include thread startup overhead, + # so we arrange for timing to start after all threads are ready. + with ready_cond: + while len(ready) < nthreads: + ready_cond.wait() + with start_cond: + start_time = time.time() + started = True + start_cond.notify(nthreads) + for t in threads: + t.join() + + return results + +def run_throughput_tests(max_threads): + for task in throughput_tasks: + print(task.__doc__) + print() + func, args = task() + nthreads = 1 + baseline_speed = None + while nthreads <= max_threads: + results = run_throughput_test(func, args, nthreads) + # Taking the max duration rather than average gives pessimistic + # results rather than optimistic. + speed = sum(r[0] for r in results) / max(r[1] for r in results) + print("threads=%d: %d" % (nthreads, speed), end="") + if baseline_speed is None: + print(" iterations/s.") + baseline_speed = speed + else: + print(" ( %d %%)" % (speed / baseline_speed * 100)) + nthreads += 1 + print() + + +LAT_END = "END" + +def _sendto(sock, s, addr): + sock.sendto(s.encode('ascii'), addr) + +def _recv(sock, n): + return sock.recv(n).decode('ascii') + +def latency_client(addr, nb_pings, interval): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + _time = time.time + _sleep = time.sleep + def _ping(): + _sendto(sock, "%r\n" % _time(), addr) + # The first ping signals the parent process that we are ready. + _ping() + # We give the parent a bit of time to notice. + _sleep(1.0) + for i in range(nb_pings): + _sleep(interval) + _ping() + _sendto(sock, LAT_END + "\n", addr) + finally: + sock.close() + +def run_latency_client(**kwargs): + cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] + cmd_line.extend(['--latclient', repr(kwargs)]) + return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, + #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + +def run_latency_test(func, args, nthreads): + # Create a listening socket to receive the pings. We use UDP which should + # be painlessly cross-platform. + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 0)) + addr = sock.getsockname() + + interval = LATENCY_PING_INTERVAL + duration = LATENCY_DURATION + nb_pings = int(duration / interval) + + results = [] + threads = [] + end_event = [] + start_cond = threading.Condition() + started = False + if nthreads > 0: + # Warm up + func(*args) + + results = [] + loop = TimedLoop(func, args) + ready = [] + ready_cond = threading.Condition() + + def run(): + with ready_cond: + ready.append(None) + ready_cond.notify() + with start_cond: + while not started: + start_cond.wait() + loop(start_time, duration * 1.5, end_event, do_yield=False) + + for i in range(nthreads): + threads.append(threading.Thread(target=run)) + for t in threads: + t.setDaemon(True) + t.start() + # Wait for threads to be ready + with ready_cond: + while len(ready) < nthreads: + ready_cond.wait() + + # Run the client and wait for the first ping(s) to arrive before + # unblocking the background threads. + chunks = [] + process = run_latency_client(addr=sock.getsockname(), + nb_pings=nb_pings, interval=interval) + s = _recv(sock, 4096) + _time = time.time + + with start_cond: + start_time = _time() + started = True + start_cond.notify(nthreads) + + while LAT_END not in s: + s = _recv(sock, 4096) + t = _time() + chunks.append((t, s)) + + # Tell the background threads to stop. + end_event.append(None) + for t in threads: + t.join() + process.wait() + sock.close() + + for recv_time, chunk in chunks: + # NOTE: it is assumed that a line sent by a client wasn't received + # in two chunks because the lines are very small. + for line in chunk.splitlines(): + line = line.strip() + if line and line != LAT_END: + send_time = eval(line) + assert isinstance(send_time, float) + results.append((send_time, recv_time)) + + return results + +def run_latency_tests(max_threads): + for task in latency_tasks: + print("Background CPU task:", task.__doc__) + print() + func, args = task() + nthreads = 0 + while nthreads <= max_threads: + results = run_latency_test(func, args, nthreads) + n = len(results) + # We print out milliseconds + lats = [1000 * (t2 - t1) for (t1, t2) in results] + #print(list(map(int, lats))) + avg = sum(lats) / n + dev = (sum((x - avg) ** 2 for x in lats) / n) ** 0.5 + print("CPU threads=%d: %d ms. (std dev: %d ms.)" % (nthreads, avg, dev), end="") + print() + #print(" [... from %d samples]" % n) + nthreads += 1 + print() + + +BW_END = "END" + +def bandwidth_client(addr, packet_size, duration): + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 0)) + local_addr = sock.getsockname() + _time = time.time + _sleep = time.sleep + def _send_chunk(msg): + _sendto(sock, ("%r#%s\n" % (local_addr, msg)).rjust(packet_size), addr) + # We give the parent some time to be ready. + _sleep(1.0) + try: + start_time = _time() + end_time = start_time + duration * 2.0 + i = 0 + while _time() < end_time: + _send_chunk(str(i)) + s = _recv(sock, packet_size) + assert len(s) == packet_size + i += 1 + _send_chunk(BW_END) + finally: + sock.close() + +def run_bandwidth_client(**kwargs): + cmd_line = [sys.executable, '-E', os.path.abspath(__file__)] + cmd_line.extend(['--bwclient', repr(kwargs)]) + return subprocess.Popen(cmd_line) #, stdin=subprocess.PIPE, + #stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + +def run_bandwidth_test(func, args, nthreads): + # Create a listening socket to receive the packets. We use UDP which should + # be painlessly cross-platform. + sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + sock.bind(("127.0.0.1", 0)) + addr = sock.getsockname() + + duration = BANDWIDTH_DURATION + packet_size = BANDWIDTH_PACKET_SIZE + + results = [] + threads = [] + end_event = [] + start_cond = threading.Condition() + started = False + if nthreads > 0: + # Warm up + func(*args) + + results = [] + loop = TimedLoop(func, args) + ready = [] + ready_cond = threading.Condition() + + def run(): + with ready_cond: + ready.append(None) + ready_cond.notify() + with start_cond: + while not started: + start_cond.wait() + loop(start_time, duration * 1.5, end_event, do_yield=False) + + for i in range(nthreads): + threads.append(threading.Thread(target=run)) + for t in threads: + t.setDaemon(True) + t.start() + # Wait for threads to be ready + with ready_cond: + while len(ready) < nthreads: + ready_cond.wait() + + # Run the client and wait for the first packet to arrive before + # unblocking the background threads. + process = run_bandwidth_client(addr=addr, + packet_size=packet_size, + duration=duration) + _time = time.time + # This will also wait for the parent to be ready + s = _recv(sock, packet_size) + remote_addr = eval(s.partition('#')[0]) + + with start_cond: + start_time = _time() + started = True + start_cond.notify(nthreads) + + n = 0 + first_time = None + while not end_event and BW_END not in s: + _sendto(sock, s, remote_addr) + s = _recv(sock, packet_size) + if first_time is None: + first_time = _time() + n += 1 + end_time = _time() + + end_event.append(None) + for t in threads: + t.join() + process.kill() + + return (n - 1) / (end_time - first_time) + +def run_bandwidth_tests(max_threads): + for task in bandwidth_tasks: + print("Background CPU task:", task.__doc__) + print() + func, args = task() + nthreads = 0 + baseline_speed = None + while nthreads <= max_threads: + results = run_bandwidth_test(func, args, nthreads) + speed = results + #speed = len(results) * 1.0 / results[-1][0] + print("CPU threads=%d: %.1f" % (nthreads, speed), end="") + if baseline_speed is None: + print(" packets/s.") + baseline_speed = speed + else: + print(" ( %d %%)" % (speed / baseline_speed * 100)) + nthreads += 1 + print() + + +def main(): + usage = "usage: %prog [-h|--help] [options]" + parser = OptionParser(usage=usage) + parser.add_option("-t", "--throughput", + action="store_true", dest="throughput", default=False, + help="run throughput tests") + parser.add_option("-l", "--latency", + action="store_true", dest="latency", default=False, + help="run latency tests") + parser.add_option("-b", "--bandwidth", + action="store_true", dest="bandwidth", default=False, + help="run I/O bandwidth tests") + parser.add_option("-i", "--interval", + action="store", type="int", dest="check_interval", default=None, + help="sys.setcheckinterval() value") + parser.add_option("-I", "--switch-interval", + action="store", type="float", dest="switch_interval", default=None, + help="sys.setswitchinterval() value") + parser.add_option("-n", "--num-threads", + action="store", type="int", dest="nthreads", default=4, + help="max number of threads in tests") + + # Hidden option to run the pinging and bandwidth clients + parser.add_option("", "--latclient", + action="store", dest="latclient", default=None, + help=SUPPRESS_HELP) + parser.add_option("", "--bwclient", + action="store", dest="bwclient", default=None, + help=SUPPRESS_HELP) + + options, args = parser.parse_args() + if args: + parser.error("unexpected arguments") + + if options.latclient: + kwargs = eval(options.latclient) + latency_client(**kwargs) + return + + if options.bwclient: + kwargs = eval(options.bwclient) + bandwidth_client(**kwargs) + return + + if not options.throughput and not options.latency and not options.bandwidth: + options.throughput = options.latency = options.bandwidth = True + if options.check_interval: + sys.setcheckinterval(options.check_interval) + if options.switch_interval: + sys.setswitchinterval(options.switch_interval) + + print("== %s %s (%s) ==" % ( + platform.python_implementation(), + platform.python_version(), + platform.python_build()[0], + )) + # Processor identification often has repeated spaces + cpu = ' '.join(platform.processor().split()) + print("== %s %s on '%s' ==" % ( + platform.machine(), + platform.system(), + cpu, + )) + print() + + if options.throughput: + print("--- Throughput ---") + print() + run_throughput_tests(options.nthreads) + + if options.latency: + print("--- Latency ---") + print() + run_latency_tests(options.nthreads) + + if options.bandwidth: + print("--- I/O bandwidth ---") + print() + run_bandwidth_tests(options.nthreads) + +if __name__ == "__main__": + main() diff --git a/Tools/demo/README b/Tools/demo/README new file mode 100644 index 0000000..e914358 --- /dev/null +++ b/Tools/demo/README @@ -0,0 +1,16 @@ +This directory contains a collection of demonstration scripts for +various aspects of Python programming. + +beer.py Well-known programming example: Bottles of beer. +eiffel.py Python advanced magic: A metaclass for Eiffel post/preconditions. +hanoi.py Well-known programming example: Towers of Hanoi. +life.py Curses programming: Simple game-of-life. +markov.py Algorithms: Markov chain simulation. +mcast.py Network programming: Send and receive UDP multicast packets. +queens.py Well-known programming example: N-Queens problem. +redemo.py Regular Expressions: GUI script to test regexes. +rpython.py Network programming: Small client for remote code execution. +rpythond.py Network programming: Small server for remote code execution. +sortvisu.py GUI programming: Visualization of different sort algorithms. +ss1.py GUI/Application programming: A simple spreadsheet application. +vector.py Python basics: A vector class with demonstrating special methods.
\ No newline at end of file diff --git a/Tools/demo/beer.py b/Tools/demo/beer.py new file mode 100755 index 0000000..af58380 --- /dev/null +++ b/Tools/demo/beer.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +""" +A Python version of the classic "bottles of beer on the wall" programming +example. + +By Guido van Rossum, demystified after a version by Fredrik Lundh. +""" + +import sys + +n = 100 +if sys.argv[1:]: + n = int(sys.argv[1]) + +def bottle(n): + if n == 0: return "no more bottles of beer" + if n == 1: return "one bottle of beer" + return str(n) + " bottles of beer" + +for i in range(n, 0, -1): + print(bottle(i), "on the wall,") + print(bottle(i) + ".") + print("Take one down, pass it around,") + print(bottle(i-1), "on the wall.") diff --git a/Tools/demo/eiffel.py b/Tools/demo/eiffel.py new file mode 100755 index 0000000..3a28224 --- /dev/null +++ b/Tools/demo/eiffel.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 + +""" +Support Eiffel-style preconditions and postconditions for functions. + +An example for Python metaclasses. +""" + +import unittest +from types import FunctionType as function + +class EiffelBaseMetaClass(type): + + def __new__(meta, name, bases, dict): + meta.convert_methods(dict) + return super(EiffelBaseMetaClass, meta).__new__( + meta, name, bases, dict) + + @classmethod + def convert_methods(cls, dict): + """Replace functions in dict with EiffelMethod wrappers. + + The dict is modified in place. + + If a method ends in _pre or _post, it is removed from the dict + regardless of whether there is a corresponding method. + """ + # find methods with pre or post conditions + methods = [] + for k, v in dict.items(): + if k.endswith('_pre') or k.endswith('_post'): + assert isinstance(v, function) + elif isinstance(v, function): + methods.append(k) + for m in methods: + pre = dict.get("%s_pre" % m) + post = dict.get("%s_post" % m) + if pre or post: + dict[k] = cls.make_eiffel_method(dict[m], pre, post) + + +class EiffelMetaClass1(EiffelBaseMetaClass): + # an implementation of the "eiffel" meta class that uses nested functions + + @staticmethod + def make_eiffel_method(func, pre, post): + def method(self, *args, **kwargs): + if pre: + pre(self, *args, **kwargs) + rv = func(self, *args, **kwargs) + if post: + post(self, rv, *args, **kwargs) + return rv + + if func.__doc__: + method.__doc__ = func.__doc__ + + return method + + +class EiffelMethodWrapper: + + def __init__(self, inst, descr): + self._inst = inst + self._descr = descr + + def __call__(self, *args, **kwargs): + return self._descr.callmethod(self._inst, args, kwargs) + + +class EiffelDescriptor: + + def __init__(self, func, pre, post): + self._func = func + self._pre = pre + self._post = post + + self.__name__ = func.__name__ + self.__doc__ = func.__doc__ + + def __get__(self, obj, cls): + return EiffelMethodWrapper(obj, self) + + def callmethod(self, inst, args, kwargs): + if self._pre: + self._pre(inst, *args, **kwargs) + x = self._func(inst, *args, **kwargs) + if self._post: + self._post(inst, x, *args, **kwargs) + return x + + +class EiffelMetaClass2(EiffelBaseMetaClass): + # an implementation of the "eiffel" meta class that uses descriptors + + make_eiffel_method = EiffelDescriptor + + +class Tests(unittest.TestCase): + + def testEiffelMetaClass1(self): + self._test(EiffelMetaClass1) + + def testEiffelMetaClass2(self): + self._test(EiffelMetaClass2) + + def _test(self, metaclass): + class Eiffel(metaclass=metaclass): + pass + + class Test(Eiffel): + def m(self, arg): + """Make it a little larger""" + return arg + 1 + + def m2(self, arg): + """Make it a little larger""" + return arg + 1 + + def m2_pre(self, arg): + assert arg > 0 + + def m2_post(self, result, arg): + assert result > arg + + class Sub(Test): + def m2(self, arg): + return arg**2 + + def m2_post(self, Result, arg): + super(Sub, self).m2_post(Result, arg) + assert Result < 100 + + t = Test() + self.assertEqual(t.m(1), 2) + self.assertEqual(t.m2(1), 2) + self.assertRaises(AssertionError, t.m2, 0) + + s = Sub() + self.assertRaises(AssertionError, s.m2, 1) + self.assertRaises(AssertionError, s.m2, 10) + self.assertEqual(s.m2(5), 25) + + +if __name__ == "__main__": + unittest.main() diff --git a/Tools/demo/hanoi.py b/Tools/demo/hanoi.py new file mode 100755 index 0000000..dad0234 --- /dev/null +++ b/Tools/demo/hanoi.py @@ -0,0 +1,154 @@ +#!/usr/bin/env python3 + +""" +Animated Towers of Hanoi using Tk with optional bitmap file in background. + +Usage: hanoi.py [n [bitmapfile]] + +n is the number of pieces to animate; default is 4, maximum 15. + +The bitmap file can be any X11 bitmap file (look in /usr/include/X11/bitmaps for +samples); it is displayed as the background of the animation. Default is no +bitmap. +""" + +from tkinter import Tk, Canvas + +# Basic Towers-of-Hanoi algorithm: move n pieces from a to b, using c +# as temporary. For each move, call report() +def hanoi(n, a, b, c, report): + if n <= 0: return + hanoi(n-1, a, c, b, report) + report(n, a, b) + hanoi(n-1, c, b, a, report) + + +# The graphical interface +class Tkhanoi: + + # Create our objects + def __init__(self, n, bitmap = None): + self.n = n + self.tk = tk = Tk() + self.canvas = c = Canvas(tk) + c.pack() + width, height = tk.getint(c['width']), tk.getint(c['height']) + + # Add background bitmap + if bitmap: + self.bitmap = c.create_bitmap(width//2, height//2, + bitmap=bitmap, + foreground='blue') + + # Generate pegs + pegwidth = 10 + pegheight = height//2 + pegdist = width//3 + x1, y1 = (pegdist-pegwidth)//2, height*1//3 + x2, y2 = x1+pegwidth, y1+pegheight + self.pegs = [] + p = c.create_rectangle(x1, y1, x2, y2, fill='black') + self.pegs.append(p) + x1, x2 = x1+pegdist, x2+pegdist + p = c.create_rectangle(x1, y1, x2, y2, fill='black') + self.pegs.append(p) + x1, x2 = x1+pegdist, x2+pegdist + p = c.create_rectangle(x1, y1, x2, y2, fill='black') + self.pegs.append(p) + self.tk.update() + + # Generate pieces + pieceheight = pegheight//16 + maxpiecewidth = pegdist*2//3 + minpiecewidth = 2*pegwidth + self.pegstate = [[], [], []] + self.pieces = {} + x1, y1 = (pegdist-maxpiecewidth)//2, y2-pieceheight-2 + x2, y2 = x1+maxpiecewidth, y1+pieceheight + dx = (maxpiecewidth-minpiecewidth) // (2*max(1, n-1)) + for i in range(n, 0, -1): + p = c.create_rectangle(x1, y1, x2, y2, fill='red') + self.pieces[i] = p + self.pegstate[0].append(i) + x1, x2 = x1 + dx, x2-dx + y1, y2 = y1 - pieceheight-2, y2-pieceheight-2 + self.tk.update() + self.tk.after(25) + + # Run -- never returns + def run(self): + while 1: + hanoi(self.n, 0, 1, 2, self.report) + hanoi(self.n, 1, 2, 0, self.report) + hanoi(self.n, 2, 0, 1, self.report) + hanoi(self.n, 0, 2, 1, self.report) + hanoi(self.n, 2, 1, 0, self.report) + hanoi(self.n, 1, 0, 2, self.report) + + # Reporting callback for the actual hanoi function + def report(self, i, a, b): + if self.pegstate[a][-1] != i: raise RuntimeError # Assertion + del self.pegstate[a][-1] + p = self.pieces[i] + c = self.canvas + + # Lift the piece above peg a + ax1, ay1, ax2, ay2 = c.bbox(self.pegs[a]) + while 1: + x1, y1, x2, y2 = c.bbox(p) + if y2 < ay1: break + c.move(p, 0, -1) + self.tk.update() + + # Move it towards peg b + bx1, by1, bx2, by2 = c.bbox(self.pegs[b]) + newcenter = (bx1+bx2)//2 + while 1: + x1, y1, x2, y2 = c.bbox(p) + center = (x1+x2)//2 + if center == newcenter: break + if center > newcenter: c.move(p, -1, 0) + else: c.move(p, 1, 0) + self.tk.update() + + # Move it down on top of the previous piece + pieceheight = y2-y1 + newbottom = by2 - pieceheight*len(self.pegstate[b]) - 2 + while 1: + x1, y1, x2, y2 = c.bbox(p) + if y2 >= newbottom: break + c.move(p, 0, 1) + self.tk.update() + + # Update peg state + self.pegstate[b].append(i) + + +def main(): + import sys + + # First argument is number of pegs, default 4 + if sys.argv[1:]: + n = int(sys.argv[1]) + else: + n = 4 + + # Second argument is bitmap file, default none + if sys.argv[2:]: + bitmap = sys.argv[2] + # Reverse meaning of leading '@' compared to Tk + if bitmap[0] == '@': bitmap = bitmap[1:] + else: bitmap = '@' + bitmap + else: + bitmap = None + + # Create the graphical objects... + h = Tkhanoi(n, bitmap) + + # ...and run! + h.run() + + +# Call main when run as script +if __name__ == '__main__': + main() diff --git a/Tools/demo/life.py b/Tools/demo/life.py new file mode 100755 index 0000000..dfb9ab8 --- /dev/null +++ b/Tools/demo/life.py @@ -0,0 +1,251 @@ +#!/usr/bin/env python3 + +""" +A curses-based version of Conway's Game of Life. + +An empty board will be displayed, and the following commands are available: + E : Erase the board + R : Fill the board randomly + S : Step for a single generation + C : Update continuously until a key is struck + Q : Quit + Cursor keys : Move the cursor around the board + Space or Enter : Toggle the contents of the cursor's position + +Contributed by Andrew Kuchling, Mouse support and color by Dafydd Crosby. +""" + +import curses +import random + + +class LifeBoard: + """Encapsulates a Life board + + Attributes: + X,Y : horizontal and vertical size of the board + state : dictionary mapping (x,y) to 0 or 1 + + Methods: + display(update_board) -- If update_board is true, compute the + next generation. Then display the state + of the board and refresh the screen. + erase() -- clear the entire board + make_random() -- fill the board randomly + set(y,x) -- set the given cell to Live; doesn't refresh the screen + toggle(y,x) -- change the given cell from live to dead, or vice + versa, and refresh the screen display + + """ + def __init__(self, scr, char=ord('*')): + """Create a new LifeBoard instance. + + scr -- curses screen object to use for display + char -- character used to render live cells (default: '*') + """ + self.state = {} + self.scr = scr + Y, X = self.scr.getmaxyx() + self.X, self.Y = X-2, Y-2-1 + self.char = char + self.scr.clear() + + # Draw a border around the board + border_line = '+'+(self.X*'-')+'+' + self.scr.addstr(0, 0, border_line) + self.scr.addstr(self.Y+1, 0, border_line) + for y in range(0, self.Y): + self.scr.addstr(1+y, 0, '|') + self.scr.addstr(1+y, self.X+1, '|') + self.scr.refresh() + + def set(self, y, x): + """Set a cell to the live state""" + if x<0 or self.X<=x or y<0 or self.Y<=y: + raise ValueError("Coordinates out of range %i,%i"% (y, x)) + self.state[x,y] = 1 + + def toggle(self, y, x): + """Toggle a cell's state between live and dead""" + if x < 0 or self.X <= x or y < 0 or self.Y <= y: + raise ValueError("Coordinates out of range %i,%i"% (y, x)) + if (x, y) in self.state: + del self.state[x, y] + self.scr.addch(y+1, x+1, ' ') + else: + self.state[x, y] = 1 + if curses.has_colors(): + # Let's pick a random color! + self.scr.attrset(curses.color_pair(random.randrange(1, 7))) + self.scr.addch(y+1, x+1, self.char) + self.scr.attrset(0) + self.scr.refresh() + + def erase(self): + """Clear the entire board and update the board display""" + self.state = {} + self.display(update_board=False) + + def display(self, update_board=True): + """Display the whole board, optionally computing one generation""" + M,N = self.X, self.Y + if not update_board: + for i in range(0, M): + for j in range(0, N): + if (i,j) in self.state: + self.scr.addch(j+1, i+1, self.char) + else: + self.scr.addch(j+1, i+1, ' ') + self.scr.refresh() + return + + d = {} + self.boring = 1 + for i in range(0, M): + L = range( max(0, i-1), min(M, i+2) ) + for j in range(0, N): + s = 0 + live = (i,j) in self.state + for k in range( max(0, j-1), min(N, j+2) ): + for l in L: + if (l,k) in self.state: + s += 1 + s -= live + if s == 3: + # Birth + d[i,j] = 1 + if curses.has_colors(): + # Let's pick a random color! + self.scr.attrset(curses.color_pair( + random.randrange(1, 7))) + self.scr.addch(j+1, i+1, self.char) + self.scr.attrset(0) + if not live: self.boring = 0 + elif s == 2 and live: d[i,j] = 1 # Survival + elif live: + # Death + self.scr.addch(j+1, i+1, ' ') + self.boring = 0 + self.state = d + self.scr.refresh() + + def make_random(self): + "Fill the board with a random pattern" + self.state = {} + for i in range(0, self.X): + for j in range(0, self.Y): + if random.random() > 0.5: + self.set(j,i) + + +def erase_menu(stdscr, menu_y): + "Clear the space where the menu resides" + stdscr.move(menu_y, 0) + stdscr.clrtoeol() + stdscr.move(menu_y+1, 0) + stdscr.clrtoeol() + +def display_menu(stdscr, menu_y): + "Display the menu of possible keystroke commands" + erase_menu(stdscr, menu_y) + + # If color, then light the menu up :-) + if curses.has_colors(): + stdscr.attrset(curses.color_pair(1)) + stdscr.addstr(menu_y, 4, + 'Use the cursor keys to move, and space or Enter to toggle a cell.') + stdscr.addstr(menu_y+1, 4, + 'E)rase the board, R)andom fill, S)tep once or C)ontinuously, Q)uit') + stdscr.attrset(0) + +def keyloop(stdscr): + # Clear the screen and display the menu of keys + stdscr.clear() + stdscr_y, stdscr_x = stdscr.getmaxyx() + menu_y = (stdscr_y-3)-1 + display_menu(stdscr, menu_y) + + # If color, then initialize the color pairs + if curses.has_colors(): + curses.init_pair(1, curses.COLOR_BLUE, 0) + curses.init_pair(2, curses.COLOR_CYAN, 0) + curses.init_pair(3, curses.COLOR_GREEN, 0) + curses.init_pair(4, curses.COLOR_MAGENTA, 0) + curses.init_pair(5, curses.COLOR_RED, 0) + curses.init_pair(6, curses.COLOR_YELLOW, 0) + curses.init_pair(7, curses.COLOR_WHITE, 0) + + # Set up the mask to listen for mouse events + curses.mousemask(curses.BUTTON1_CLICKED) + + # Allocate a subwindow for the Life board and create the board object + subwin = stdscr.subwin(stdscr_y-3, stdscr_x, 0, 0) + board = LifeBoard(subwin, char=ord('*')) + board.display(update_board=False) + + # xpos, ypos are the cursor's position + xpos, ypos = board.X//2, board.Y//2 + + # Main loop: + while True: + stdscr.move(1+ypos, 1+xpos) # Move the cursor + c = stdscr.getch() # Get a keystroke + if 0 < c < 256: + c = chr(c) + if c in ' \n': + board.toggle(ypos, xpos) + elif c in 'Cc': + erase_menu(stdscr, menu_y) + stdscr.addstr(menu_y, 6, ' Hit any key to stop continuously ' + 'updating the screen.') + stdscr.refresh() + # Activate nodelay mode; getch() will return -1 + # if no keystroke is available, instead of waiting. + stdscr.nodelay(1) + while True: + c = stdscr.getch() + if c != -1: + break + stdscr.addstr(0, 0, '/') + stdscr.refresh() + board.display() + stdscr.addstr(0, 0, '+') + stdscr.refresh() + + stdscr.nodelay(0) # Disable nodelay mode + display_menu(stdscr, menu_y) + + elif c in 'Ee': + board.erase() + elif c in 'Qq': + break + elif c in 'Rr': + board.make_random() + board.display(update_board=False) + elif c in 'Ss': + board.display() + else: pass # Ignore incorrect keys + elif c == curses.KEY_UP and ypos > 0: ypos -= 1 + elif c == curses.KEY_DOWN and ypos < board.Y-1: ypos += 1 + elif c == curses.KEY_LEFT and xpos > 0: xpos -= 1 + elif c == curses.KEY_RIGHT and xpos < board.X-1: xpos += 1 + elif c == curses.KEY_MOUSE: + mouse_id, mouse_x, mouse_y, mouse_z, button_state = curses.getmouse() + if (mouse_x > 0 and mouse_x < board.X+1 and + mouse_y > 0 and mouse_y < board.Y+1): + xpos = mouse_x - 1 + ypos = mouse_y - 1 + board.toggle(ypos, xpos) + else: + # They've clicked outside the board + curses.flash() + else: + # Ignore incorrect keys + pass + + +def main(stdscr): + keyloop(stdscr) # Enter the main loop + +if __name__ == '__main__': + curses.wrapper(main) diff --git a/Tools/demo/markov.py b/Tools/demo/markov.py new file mode 100755 index 0000000..7a0720f --- /dev/null +++ b/Tools/demo/markov.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 + +""" +Markov chain simulation of words or characters. +""" + +class Markov: + def __init__(self, histsize, choice): + self.histsize = histsize + self.choice = choice + self.trans = {} + + def add(self, state, next): + self.trans.setdefault(state, []).append(next) + + def put(self, seq): + n = self.histsize + add = self.add + add(None, seq[:0]) + for i in range(len(seq)): + add(seq[max(0, i-n):i], seq[i:i+1]) + add(seq[len(seq)-n:], None) + + def get(self): + choice = self.choice + trans = self.trans + n = self.histsize + seq = choice(trans[None]) + while True: + subseq = seq[max(0, len(seq)-n):] + options = trans[subseq] + next = choice(options) + if not next: + break + seq += next + return seq + + +def test(): + import sys, random, getopt + args = sys.argv[1:] + try: + opts, args = getopt.getopt(args, '0123456789cdwq') + except getopt.error: + print('Usage: %s [-#] [-cddqw] [file] ...' % sys.argv[0]) + print('Options:') + print('-#: 1-digit history size (default 2)') + print('-c: characters (default)') + print('-w: words') + print('-d: more debugging output') + print('-q: no debugging output') + print('Input files (default stdin) are split in paragraphs') + print('separated blank lines and each paragraph is split') + print('in words by whitespace, then reconcatenated with') + print('exactly one space separating words.') + print('Output consists of paragraphs separated by blank') + print('lines, where lines are no longer than 72 characters.') + sys.exit(2) + histsize = 2 + do_words = False + debug = 1 + for o, a in opts: + if '-0' <= o <= '-9': histsize = int(o[1:]) + if o == '-c': do_words = False + if o == '-d': debug += 1 + if o == '-q': debug = 0 + if o == '-w': do_words = True + if not args: + args = ['-'] + + m = Markov(histsize, random.choice) + try: + for filename in args: + if filename == '-': + f = sys.stdin + if f.isatty(): + print('Sorry, need stdin from file') + continue + else: + f = open(filename, 'r') + if debug: print('processing', filename, '...') + text = f.read() + f.close() + paralist = text.split('\n\n') + for para in paralist: + if debug > 1: print('feeding ...') + words = para.split() + if words: + if do_words: + data = tuple(words) + else: + data = ' '.join(words) + m.put(data) + except KeyboardInterrupt: + print('Interrupted -- continue with data read so far') + if not m.trans: + print('No valid input files') + return + if debug: print('done.') + + if debug > 1: + for key in m.trans.keys(): + if key is None or len(key) < histsize: + print(repr(key), m.trans[key]) + if histsize == 0: print(repr(''), m.trans['']) + print() + while True: + data = m.get() + if do_words: + words = data + else: + words = data.split() + n = 0 + limit = 72 + for w in words: + if n + len(w) > limit: + print() + n = 0 + print(w, end=' ') + n += len(w) + 1 + print() + print() + +if __name__ == "__main__": + test() diff --git a/Tools/demo/mcast.py b/Tools/demo/mcast.py new file mode 100755 index 0000000..924c7c3 --- /dev/null +++ b/Tools/demo/mcast.py @@ -0,0 +1,82 @@ +#!/usr/bin/env python3 + +""" +Send/receive UDP multicast packets. +Requires that your OS kernel supports IP multicast. + +Usage: + mcast -s (sender, IPv4) + mcast -s -6 (sender, IPv6) + mcast (receivers, IPv4) + mcast -6 (receivers, IPv6) +""" + +MYPORT = 8123 +MYGROUP_4 = '225.0.0.250' +MYGROUP_6 = 'ff15:7079:7468:6f6e:6465:6d6f:6d63:6173' +MYTTL = 1 # Increase to reach other networks + +import time +import struct +import socket +import sys + +def main(): + group = MYGROUP_6 if "-6" in sys.argv[1:] else MYGROUP_4 + + if "-s" in sys.argv[1:]: + sender(group) + else: + receiver(group) + + +def sender(group): + addrinfo = socket.getaddrinfo(group, None)[0] + + s = socket.socket(addrinfo[0], socket.SOCK_DGRAM) + + # Set Time-to-live (optional) + ttl_bin = struct.pack('@i', MYTTL) + if addrinfo[0] == socket.AF_INET: # IPv4 + s.setsockopt(socket.IPPROTO_IP, socket.IP_MULTICAST_TTL, ttl_bin) + else: + s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_MULTICAST_HOPS, ttl_bin) + + while True: + data = repr(time.time()).encode('utf-8') + b'\0' + s.sendto(data, (addrinfo[4][0], MYPORT)) + time.sleep(1) + + +def receiver(group): + # Look up multicast group address in name server and find out IP version + addrinfo = socket.getaddrinfo(group, None)[0] + + # Create a socket + s = socket.socket(addrinfo[0], socket.SOCK_DGRAM) + + # Allow multiple copies of this program on one machine + # (not strictly needed) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + + # Bind it to the port + s.bind(('', MYPORT)) + + group_bin = socket.inet_pton(addrinfo[0], addrinfo[4][0]) + # Join group + if addrinfo[0] == socket.AF_INET: # IPv4 + mreq = group_bin + struct.pack('=I', socket.INADDR_ANY) + s.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) + else: + mreq = group_bin + struct.pack('@I', 0) + s.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_JOIN_GROUP, mreq) + + # Loop, printing any data we receive + while True: + data, sender = s.recvfrom(1500) + while data[-1:] == '\0': data = data[:-1] # Strip trailing \0's + print(str(sender) + ' ' + repr(data)) + + +if __name__ == '__main__': + main() diff --git a/Tools/demo/queens.py b/Tools/demo/queens.py new file mode 100755 index 0000000..dcc1bae --- /dev/null +++ b/Tools/demo/queens.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python3 + +""" +N queens problem. + +The (well-known) problem is due to Niklaus Wirth. + +This solution is inspired by Dijkstra (Structured Programming). It is +a classic recursive backtracking approach. +""" + +N = 8 # Default; command line overrides + +class Queens: + + def __init__(self, n=N): + self.n = n + self.reset() + + def reset(self): + n = self.n + self.y = [None] * n # Where is the queen in column x + self.row = [0] * n # Is row[y] safe? + self.up = [0] * (2*n-1) # Is upward diagonal[x-y] safe? + self.down = [0] * (2*n-1) # Is downward diagonal[x+y] safe? + self.nfound = 0 # Instrumentation + + def solve(self, x=0): # Recursive solver + for y in range(self.n): + if self.safe(x, y): + self.place(x, y) + if x+1 == self.n: + self.display() + else: + self.solve(x+1) + self.remove(x, y) + + def safe(self, x, y): + return not self.row[y] and not self.up[x-y] and not self.down[x+y] + + def place(self, x, y): + self.y[x] = y + self.row[y] = 1 + self.up[x-y] = 1 + self.down[x+y] = 1 + + def remove(self, x, y): + self.y[x] = None + self.row[y] = 0 + self.up[x-y] = 0 + self.down[x+y] = 0 + + silent = 0 # If true, count solutions only + + def display(self): + self.nfound = self.nfound + 1 + if self.silent: + return + print('+-' + '--'*self.n + '+') + for y in range(self.n-1, -1, -1): + print('|', end=' ') + for x in range(self.n): + if self.y[x] == y: + print("Q", end=' ') + else: + print(".", end=' ') + print('|') + print('+-' + '--'*self.n + '+') + +def main(): + import sys + silent = 0 + n = N + if sys.argv[1:2] == ['-n']: + silent = 1 + del sys.argv[1] + if sys.argv[1:]: + n = int(sys.argv[1]) + q = Queens(n) + q.silent = silent + q.solve() + print("Found", q.nfound, "solutions.") + +if __name__ == "__main__": + main() diff --git a/Tools/scripts/redemo.py b/Tools/demo/redemo.py index 5286332..5286332 100644 --- a/Tools/scripts/redemo.py +++ b/Tools/demo/redemo.py diff --git a/Tools/demo/rpython.py b/Tools/demo/rpython.py new file mode 100755 index 0000000..5e7bc0a --- /dev/null +++ b/Tools/demo/rpython.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 + +""" +Remote python client. +Execute Python commands remotely and send output back. +""" + +import sys +from socket import socket, AF_INET, SOCK_STREAM, SHUT_WR + +PORT = 4127 +BUFSIZE = 1024 + +def main(): + if len(sys.argv) < 3: + print("usage: rpython host command") + sys.exit(2) + host = sys.argv[1] + port = PORT + i = host.find(':') + if i >= 0: + port = int(port[i+1:]) + host = host[:i] + command = ' '.join(sys.argv[2:]) + s = socket(AF_INET, SOCK_STREAM) + s.connect((host, port)) + s.send(command.encode()) + s.shutdown(SHUT_WR) + reply = b'' + while True: + data = s.recv(BUFSIZE) + if not data: + break + reply += data + print(reply.decode(), end=' ') + s.close() + +main() diff --git a/Tools/demo/rpythond.py b/Tools/demo/rpythond.py new file mode 100755 index 0000000..9ffe13a --- /dev/null +++ b/Tools/demo/rpythond.py @@ -0,0 +1,58 @@ +#!/usr/bin/env python3 + +""" +Remote python server. +Execute Python commands remotely and send output back. + +WARNING: This version has a gaping security hole -- it accepts requests +from any host on the Internet! +""" + +import sys +from socket import socket, AF_INET, SOCK_STREAM +import io +import traceback + +PORT = 4127 +BUFSIZE = 1024 + +def main(): + if len(sys.argv) > 1: + port = int(sys.argv[1]) + else: + port = PORT + s = socket(AF_INET, SOCK_STREAM) + s.bind(('', port)) + s.listen(1) + while True: + conn, (remotehost, remoteport) = s.accept() + print('connection from', remotehost, remoteport) + request = b'' + while 1: + data = conn.recv(BUFSIZE) + if not data: + break + request += data + reply = execute(request.decode()) + conn.send(reply.encode()) + conn.close() + +def execute(request): + stdout = sys.stdout + stderr = sys.stderr + sys.stdout = sys.stderr = fakefile = io.StringIO() + try: + try: + exec(request, {}, {}) + except: + print() + traceback.print_exc(100) + finally: + sys.stderr = stderr + sys.stdout = stdout + return fakefile.getvalue() + +try: + main() +except KeyboardInterrupt: + pass diff --git a/Tools/demo/sortvisu.py b/Tools/demo/sortvisu.py new file mode 100755 index 0000000..8447bc7 --- /dev/null +++ b/Tools/demo/sortvisu.py @@ -0,0 +1,635 @@ +#!/usr/bin/env python3 + +""" +Sorting algorithms visualizer using Tkinter. + +This module is comprised of three ``components'': + +- an array visualizer with methods that implement basic sorting +operations (compare, swap) as well as methods for ``annotating'' the +sorting algorithm (e.g. to show the pivot element); + +- a number of sorting algorithms (currently quicksort, insertion sort, +selection sort and bubble sort, as well as a randomization function), +all using the array visualizer for its basic operations and with calls +to its annotation methods; + +- and a ``driver'' class which can be used as a Grail applet or as a +stand-alone application. +""" + +from tkinter import * +import random + +XGRID = 10 +YGRID = 10 +WIDTH = 6 + + +class Array: + + class Cancelled(BaseException): + pass + + def __init__(self, master, data=None): + self.master = master + self.frame = Frame(self.master) + self.frame.pack(fill=X) + self.label = Label(self.frame) + self.label.pack() + self.canvas = Canvas(self.frame) + self.canvas.pack() + self.report = Label(self.frame) + self.report.pack() + self.left = self.canvas.create_line(0, 0, 0, 0) + self.right = self.canvas.create_line(0, 0, 0, 0) + self.pivot = self.canvas.create_line(0, 0, 0, 0) + self.items = [] + self.size = self.maxvalue = 0 + if data: + self.setdata(data) + + def setdata(self, data): + olditems = self.items + self.items = [] + for item in olditems: + item.delete() + self.size = len(data) + self.maxvalue = max(data) + self.canvas.config(width=(self.size+1)*XGRID, + height=(self.maxvalue+1)*YGRID) + for i in range(self.size): + self.items.append(ArrayItem(self, i, data[i])) + self.reset("Sort demo, size %d" % self.size) + + speed = "normal" + + def setspeed(self, speed): + self.speed = speed + + def destroy(self): + self.frame.destroy() + + in_mainloop = 0 + stop_mainloop = 0 + + def cancel(self): + self.stop_mainloop = 1 + if self.in_mainloop: + self.master.quit() + + def step(self): + if self.in_mainloop: + self.master.quit() + + def wait(self, msecs): + if self.speed == "fastest": + msecs = 0 + elif self.speed == "fast": + msecs = msecs//10 + elif self.speed == "single-step": + msecs = 1000000000 + if not self.stop_mainloop: + self.master.update() + id = self.master.after(msecs, self.master.quit) + self.in_mainloop = 1 + self.master.mainloop() + self.master.after_cancel(id) + self.in_mainloop = 0 + if self.stop_mainloop: + self.stop_mainloop = 0 + self.message("Cancelled") + raise Array.Cancelled + + def getsize(self): + return self.size + + def show_partition(self, first, last): + for i in range(self.size): + item = self.items[i] + if first <= i < last: + self.canvas.itemconfig(item, fill='red') + else: + self.canvas.itemconfig(item, fill='orange') + self.hide_left_right_pivot() + + def hide_partition(self): + for i in range(self.size): + item = self.items[i] + self.canvas.itemconfig(item, fill='red') + self.hide_left_right_pivot() + + def show_left(self, left): + if not 0 <= left < self.size: + self.hide_left() + return + x1, y1, x2, y2 = self.items[left].position() +## top, bot = HIRO + self.canvas.coords(self.left, (x1 - 2, 0, x1 - 2, 9999)) + self.master.update() + + def show_right(self, right): + if not 0 <= right < self.size: + self.hide_right() + return + x1, y1, x2, y2 = self.items[right].position() + self.canvas.coords(self.right, (x2 + 2, 0, x2 + 2, 9999)) + self.master.update() + + def hide_left_right_pivot(self): + self.hide_left() + self.hide_right() + self.hide_pivot() + + def hide_left(self): + self.canvas.coords(self.left, (0, 0, 0, 0)) + + def hide_right(self): + self.canvas.coords(self.right, (0, 0, 0, 0)) + + def show_pivot(self, pivot): + x1, y1, x2, y2 = self.items[pivot].position() + self.canvas.coords(self.pivot, (0, y1 - 2, 9999, y1 - 2)) + + def hide_pivot(self): + self.canvas.coords(self.pivot, (0, 0, 0, 0)) + + def swap(self, i, j): + if i == j: return + self.countswap() + item = self.items[i] + other = self.items[j] + self.items[i], self.items[j] = other, item + item.swapwith(other) + + def compare(self, i, j): + self.countcompare() + item = self.items[i] + other = self.items[j] + return item.compareto(other) + + def reset(self, msg): + self.ncompares = 0 + self.nswaps = 0 + self.message(msg) + self.updatereport() + self.hide_partition() + + def message(self, msg): + self.label.config(text=msg) + + def countswap(self): + self.nswaps = self.nswaps + 1 + self.updatereport() + + def countcompare(self): + self.ncompares = self.ncompares + 1 + self.updatereport() + + def updatereport(self): + text = "%d cmps, %d swaps" % (self.ncompares, self.nswaps) + self.report.config(text=text) + + +class ArrayItem: + + def __init__(self, array, index, value): + self.array = array + self.index = index + self.value = value + self.canvas = array.canvas + x1, y1, x2, y2 = self.position() + self.item_id = array.canvas.create_rectangle(x1, y1, x2, y2, + fill='red', outline='black', width=1) + self.canvas.tag_bind(self.item_id, '<Button-1>', self.mouse_down) + self.canvas.tag_bind(self.item_id, '<Button1-Motion>', self.mouse_move) + self.canvas.tag_bind(self.item_id, '<ButtonRelease-1>', self.mouse_up) + + def delete(self): + item_id = self.item_id + self.array = None + self.item_id = None + self.canvas.delete(item_id) + + def mouse_down(self, event): + self.lastx = event.x + self.lasty = event.y + self.origx = event.x + self.origy = event.y + self.canvas.tag_raise(self.item_id) + + def mouse_move(self, event): + self.canvas.move(self.item_id, + event.x - self.lastx, event.y - self.lasty) + self.lastx = event.x + self.lasty = event.y + + def mouse_up(self, event): + i = self.nearestindex(event.x) + if i >= self.array.getsize(): + i = self.array.getsize() - 1 + if i < 0: + i = 0 + other = self.array.items[i] + here = self.index + self.array.items[here], self.array.items[i] = other, self + self.index = i + x1, y1, x2, y2 = self.position() + self.canvas.coords(self.item_id, (x1, y1, x2, y2)) + other.setindex(here) + + def setindex(self, index): + nsteps = steps(self.index, index) + if not nsteps: return + if self.array.speed == "fastest": + nsteps = 0 + oldpts = self.position() + self.index = index + newpts = self.position() + trajectory = interpolate(oldpts, newpts, nsteps) + self.canvas.tag_raise(self.item_id) + for pts in trajectory: + self.canvas.coords(self.item_id, pts) + self.array.wait(50) + + def swapwith(self, other): + nsteps = steps(self.index, other.index) + if not nsteps: return + if self.array.speed == "fastest": + nsteps = 0 + myoldpts = self.position() + otheroldpts = other.position() + self.index, other.index = other.index, self.index + mynewpts = self.position() + othernewpts = other.position() + myfill = self.canvas.itemcget(self.item_id, 'fill') + otherfill = self.canvas.itemcget(other.item_id, 'fill') + self.canvas.itemconfig(self.item_id, fill='green') + self.canvas.itemconfig(other.item_id, fill='yellow') + self.array.master.update() + if self.array.speed == "single-step": + self.canvas.coords(self.item_id, mynewpts) + self.canvas.coords(other.item_id, othernewpts) + self.array.master.update() + self.canvas.itemconfig(self.item_id, fill=myfill) + self.canvas.itemconfig(other.item_id, fill=otherfill) + self.array.wait(0) + return + mytrajectory = interpolate(myoldpts, mynewpts, nsteps) + othertrajectory = interpolate(otheroldpts, othernewpts, nsteps) + if self.value > other.value: + self.canvas.tag_raise(self.item_id) + self.canvas.tag_raise(other.item_id) + else: + self.canvas.tag_raise(other.item_id) + self.canvas.tag_raise(self.item_id) + try: + for i in range(len(mytrajectory)): + mypts = mytrajectory[i] + otherpts = othertrajectory[i] + self.canvas.coords(self.item_id, mypts) + self.canvas.coords(other.item_id, otherpts) + self.array.wait(50) + finally: + mypts = mytrajectory[-1] + otherpts = othertrajectory[-1] + self.canvas.coords(self.item_id, mypts) + self.canvas.coords(other.item_id, otherpts) + self.canvas.itemconfig(self.item_id, fill=myfill) + self.canvas.itemconfig(other.item_id, fill=otherfill) + + def compareto(self, other): + myfill = self.canvas.itemcget(self.item_id, 'fill') + otherfill = self.canvas.itemcget(other.item_id, 'fill') + if self.value < other.value: + myflash = 'white' + otherflash = 'black' + outcome = -1 + elif self.value > other.value: + myflash = 'black' + otherflash = 'white' + outcome = 1 + else: + myflash = otherflash = 'grey' + outcome = 0 + try: + self.canvas.itemconfig(self.item_id, fill=myflash) + self.canvas.itemconfig(other.item_id, fill=otherflash) + self.array.wait(500) + finally: + self.canvas.itemconfig(self.item_id, fill=myfill) + self.canvas.itemconfig(other.item_id, fill=otherfill) + return outcome + + def position(self): + x1 = (self.index+1)*XGRID - WIDTH//2 + x2 = x1+WIDTH + y2 = (self.array.maxvalue+1)*YGRID + y1 = y2 - (self.value)*YGRID + return x1, y1, x2, y2 + + def nearestindex(self, x): + return int(round(float(x)/XGRID)) - 1 + + +# Subroutines that don't need an object + +def steps(here, there): + nsteps = abs(here - there) + if nsteps <= 3: + nsteps = nsteps * 3 + elif nsteps <= 5: + nsteps = nsteps * 2 + elif nsteps > 10: + nsteps = 10 + return nsteps + +def interpolate(oldpts, newpts, n): + if len(oldpts) != len(newpts): + raise ValueError("can't interpolate arrays of different length") + pts = [0]*len(oldpts) + res = [tuple(oldpts)] + for i in range(1, n): + for k in range(len(pts)): + pts[k] = oldpts[k] + (newpts[k] - oldpts[k])*i//n + res.append(tuple(pts)) + res.append(tuple(newpts)) + return res + + +# Various (un)sorting algorithms + +def uniform(array): + size = array.getsize() + array.setdata([(size+1)//2] * size) + array.reset("Uniform data, size %d" % size) + +def distinct(array): + size = array.getsize() + array.setdata(range(1, size+1)) + array.reset("Distinct data, size %d" % size) + +def randomize(array): + array.reset("Randomizing") + n = array.getsize() + for i in range(n): + j = random.randint(0, n-1) + array.swap(i, j) + array.message("Randomized") + +def insertionsort(array): + size = array.getsize() + array.reset("Insertion sort") + for i in range(1, size): + j = i-1 + while j >= 0: + if array.compare(j, j+1) <= 0: + break + array.swap(j, j+1) + j = j-1 + array.message("Sorted") + +def selectionsort(array): + size = array.getsize() + array.reset("Selection sort") + try: + for i in range(size): + array.show_partition(i, size) + for j in range(i+1, size): + if array.compare(i, j) > 0: + array.swap(i, j) + array.message("Sorted") + finally: + array.hide_partition() + +def bubblesort(array): + size = array.getsize() + array.reset("Bubble sort") + for i in range(size): + for j in range(1, size): + if array.compare(j-1, j) > 0: + array.swap(j-1, j) + array.message("Sorted") + +def quicksort(array): + size = array.getsize() + array.reset("Quicksort") + try: + stack = [(0, size)] + while stack: + first, last = stack[-1] + del stack[-1] + array.show_partition(first, last) + if last-first < 5: + array.message("Insertion sort") + for i in range(first+1, last): + j = i-1 + while j >= first: + if array.compare(j, j+1) <= 0: + break + array.swap(j, j+1) + j = j-1 + continue + array.message("Choosing pivot") + j, i, k = first, (first+last) // 2, last-1 + if array.compare(k, i) < 0: + array.swap(k, i) + if array.compare(k, j) < 0: + array.swap(k, j) + if array.compare(j, i) < 0: + array.swap(j, i) + pivot = j + array.show_pivot(pivot) + array.message("Pivot at left of partition") + array.wait(1000) + left = first + right = last + while 1: + array.message("Sweep right pointer") + right = right-1 + array.show_right(right) + while right > first and array.compare(right, pivot) >= 0: + right = right-1 + array.show_right(right) + array.message("Sweep left pointer") + left = left+1 + array.show_left(left) + while left < last and array.compare(left, pivot) <= 0: + left = left+1 + array.show_left(left) + if left > right: + array.message("End of partition") + break + array.message("Swap items") + array.swap(left, right) + array.message("Swap pivot back") + array.swap(pivot, right) + n1 = right-first + n2 = last-left + if n1 > 1: stack.append((first, right)) + if n2 > 1: stack.append((left, last)) + array.message("Sorted") + finally: + array.hide_partition() + +def demosort(array): + while 1: + for alg in [quicksort, insertionsort, selectionsort, bubblesort]: + randomize(array) + alg(array) + + +# Sort demo class -- usable as a Grail applet + +class SortDemo: + + def __init__(self, master, size=15): + self.master = master + self.size = size + self.busy = 0 + self.array = Array(self.master) + + self.botframe = Frame(master) + self.botframe.pack(side=BOTTOM) + self.botleftframe = Frame(self.botframe) + self.botleftframe.pack(side=LEFT, fill=Y) + self.botrightframe = Frame(self.botframe) + self.botrightframe.pack(side=RIGHT, fill=Y) + + self.b_qsort = Button(self.botleftframe, + text="Quicksort", command=self.c_qsort) + self.b_qsort.pack(fill=X) + self.b_isort = Button(self.botleftframe, + text="Insertion sort", command=self.c_isort) + self.b_isort.pack(fill=X) + self.b_ssort = Button(self.botleftframe, + text="Selection sort", command=self.c_ssort) + self.b_ssort.pack(fill=X) + self.b_bsort = Button(self.botleftframe, + text="Bubble sort", command=self.c_bsort) + self.b_bsort.pack(fill=X) + + # Terrible hack to overcome limitation of OptionMenu... + class MyIntVar(IntVar): + def __init__(self, master, demo): + self.demo = demo + IntVar.__init__(self, master) + def set(self, value): + IntVar.set(self, value) + if str(value) != '0': + self.demo.resize(value) + + self.v_size = MyIntVar(self.master, self) + self.v_size.set(size) + sizes = [1, 2, 3, 4] + list(range(5, 55, 5)) + if self.size not in sizes: + sizes.append(self.size) + sizes.sort() + self.m_size = OptionMenu(self.botleftframe, self.v_size, *sizes) + self.m_size.pack(fill=X) + + self.v_speed = StringVar(self.master) + self.v_speed.set("normal") + self.m_speed = OptionMenu(self.botleftframe, self.v_speed, + "single-step", "normal", "fast", "fastest") + self.m_speed.pack(fill=X) + + self.b_step = Button(self.botleftframe, + text="Step", command=self.c_step) + self.b_step.pack(fill=X) + + self.b_randomize = Button(self.botrightframe, + text="Randomize", command=self.c_randomize) + self.b_randomize.pack(fill=X) + self.b_uniform = Button(self.botrightframe, + text="Uniform", command=self.c_uniform) + self.b_uniform.pack(fill=X) + self.b_distinct = Button(self.botrightframe, + text="Distinct", command=self.c_distinct) + self.b_distinct.pack(fill=X) + self.b_demo = Button(self.botrightframe, + text="Demo", command=self.c_demo) + self.b_demo.pack(fill=X) + self.b_cancel = Button(self.botrightframe, + text="Cancel", command=self.c_cancel) + self.b_cancel.pack(fill=X) + self.b_cancel.config(state=DISABLED) + self.b_quit = Button(self.botrightframe, + text="Quit", command=self.c_quit) + self.b_quit.pack(fill=X) + + def resize(self, newsize): + if self.busy: + self.master.bell() + return + self.size = newsize + self.array.setdata(range(1, self.size+1)) + + def c_qsort(self): + self.run(quicksort) + + def c_isort(self): + self.run(insertionsort) + + def c_ssort(self): + self.run(selectionsort) + + def c_bsort(self): + self.run(bubblesort) + + def c_demo(self): + self.run(demosort) + + def c_randomize(self): + self.run(randomize) + + def c_uniform(self): + self.run(uniform) + + def c_distinct(self): + self.run(distinct) + + def run(self, func): + if self.busy: + self.master.bell() + return + self.busy = 1 + self.array.setspeed(self.v_speed.get()) + self.b_cancel.config(state=NORMAL) + try: + func(self.array) + except Array.Cancelled: + pass + self.b_cancel.config(state=DISABLED) + self.busy = 0 + + def c_cancel(self): + if not self.busy: + self.master.bell() + return + self.array.cancel() + + def c_step(self): + if not self.busy: + self.master.bell() + return + self.v_speed.set("single-step") + self.array.setspeed("single-step") + self.array.step() + + def c_quit(self): + if self.busy: + self.array.cancel() + self.master.after_idle(self.master.quit) + + +# Main program -- for stand-alone operation outside Grail + +def main(): + root = Tk() + demo = SortDemo(root) + root.protocol('WM_DELETE_WINDOW', demo.c_quit) + root.mainloop() + +if __name__ == '__main__': + main() diff --git a/Tools/demo/ss1.py b/Tools/demo/ss1.py new file mode 100755 index 0000000..4cea667 --- /dev/null +++ b/Tools/demo/ss1.py @@ -0,0 +1,846 @@ +#!/usr/bin/env python3 + +""" +SS1 -- a spreadsheet-like application. +""" + +import os +import re +import sys +import html +from xml.parsers import expat + +LEFT, CENTER, RIGHT = "LEFT", "CENTER", "RIGHT" + +def ljust(x, n): + return x.ljust(n) +def center(x, n): + return x.center(n) +def rjust(x, n): + return x.rjust(n) +align2action = {LEFT: ljust, CENTER: center, RIGHT: rjust} + +align2xml = {LEFT: "left", CENTER: "center", RIGHT: "right"} +xml2align = {"left": LEFT, "center": CENTER, "right": RIGHT} + +align2anchor = {LEFT: "w", CENTER: "center", RIGHT: "e"} + +def sum(seq): + total = 0 + for x in seq: + if x is not None: + total += x + return total + +class Sheet: + + def __init__(self): + self.cells = {} # {(x, y): cell, ...} + self.ns = dict( + cell = self.cellvalue, + cells = self.multicellvalue, + sum = sum, + ) + + def cellvalue(self, x, y): + cell = self.getcell(x, y) + if hasattr(cell, 'recalc'): + return cell.recalc(self.ns) + else: + return cell + + def multicellvalue(self, x1, y1, x2, y2): + if x1 > x2: + x1, x2 = x2, x1 + if y1 > y2: + y1, y2 = y2, y1 + seq = [] + for y in range(y1, y2+1): + for x in range(x1, x2+1): + seq.append(self.cellvalue(x, y)) + return seq + + def getcell(self, x, y): + return self.cells.get((x, y)) + + def setcell(self, x, y, cell): + assert x > 0 and y > 0 + assert isinstance(cell, BaseCell) + self.cells[x, y] = cell + + def clearcell(self, x, y): + try: + del self.cells[x, y] + except KeyError: + pass + + def clearcells(self, x1, y1, x2, y2): + for xy in self.selectcells(x1, y1, x2, y2): + del self.cells[xy] + + def clearrows(self, y1, y2): + self.clearcells(0, y1, sys.maxint, y2) + + def clearcolumns(self, x1, x2): + self.clearcells(x1, 0, x2, sys.maxint) + + def selectcells(self, x1, y1, x2, y2): + if x1 > x2: + x1, x2 = x2, x1 + if y1 > y2: + y1, y2 = y2, y1 + return [(x, y) for x, y in self.cells + if x1 <= x <= x2 and y1 <= y <= y2] + + def movecells(self, x1, y1, x2, y2, dx, dy): + if dx == 0 and dy == 0: + return + if x1 > x2: + x1, x2 = x2, x1 + if y1 > y2: + y1, y2 = y2, y1 + assert x1+dx > 0 and y1+dy > 0 + new = {} + for x, y in self.cells: + cell = self.cells[x, y] + if hasattr(cell, 'renumber'): + cell = cell.renumber(x1, y1, x2, y2, dx, dy) + if x1 <= x <= x2 and y1 <= y <= y2: + x += dx + y += dy + new[x, y] = cell + self.cells = new + + def insertrows(self, y, n): + assert n > 0 + self.movecells(0, y, sys.maxint, sys.maxint, 0, n) + + def deleterows(self, y1, y2): + if y1 > y2: + y1, y2 = y2, y1 + self.clearrows(y1, y2) + self.movecells(0, y2+1, sys.maxint, sys.maxint, 0, y1-y2-1) + + def insertcolumns(self, x, n): + assert n > 0 + self.movecells(x, 0, sys.maxint, sys.maxint, n, 0) + + def deletecolumns(self, x1, x2): + if x1 > x2: + x1, x2 = x2, x1 + self.clearcells(x1, x2) + self.movecells(x2+1, 0, sys.maxint, sys.maxint, x1-x2-1, 0) + + def getsize(self): + maxx = maxy = 0 + for x, y in self.cells: + maxx = max(maxx, x) + maxy = max(maxy, y) + return maxx, maxy + + def reset(self): + for cell in self.cells.values(): + if hasattr(cell, 'reset'): + cell.reset() + + def recalc(self): + self.reset() + for cell in self.cells.values(): + if hasattr(cell, 'recalc'): + cell.recalc(self.ns) + + def display(self): + maxx, maxy = self.getsize() + width, height = maxx+1, maxy+1 + colwidth = [1] * width + full = {} + # Add column heading labels in row 0 + for x in range(1, width): + full[x, 0] = text, alignment = colnum2name(x), RIGHT + colwidth[x] = max(colwidth[x], len(text)) + # Add row labels in column 0 + for y in range(1, height): + full[0, y] = text, alignment = str(y), RIGHT + colwidth[0] = max(colwidth[0], len(text)) + # Add sheet cells in columns with x>0 and y>0 + for (x, y), cell in self.cells.items(): + if x <= 0 or y <= 0: + continue + if hasattr(cell, 'recalc'): + cell.recalc(self.ns) + if hasattr(cell, 'format'): + text, alignment = cell.format() + assert isinstance(text, str) + assert alignment in (LEFT, CENTER, RIGHT) + else: + text = str(cell) + if isinstance(cell, str): + alignment = LEFT + else: + alignment = RIGHT + full[x, y] = (text, alignment) + colwidth[x] = max(colwidth[x], len(text)) + # Calculate the horizontal separator line (dashes and dots) + sep = "" + for x in range(width): + if sep: + sep += "+" + sep += "-"*colwidth[x] + # Now print The full grid + for y in range(height): + line = "" + for x in range(width): + text, alignment = full.get((x, y)) or ("", LEFT) + text = align2action[alignment](text, colwidth[x]) + if line: + line += '|' + line += text + print(line) + if y == 0: + print(sep) + + def xml(self): + out = ['<spreadsheet>'] + for (x, y), cell in self.cells.items(): + if hasattr(cell, 'xml'): + cellxml = cell.xml() + else: + cellxml = '<value>%s</value>' % html.escape(cell) + out.append('<cell row="%s" col="%s">\n %s\n</cell>' % + (y, x, cellxml)) + out.append('</spreadsheet>') + return '\n'.join(out) + + def save(self, filename): + text = self.xml() + f = open(filename, "w") + f.write(text) + if text and not text.endswith('\n'): + f.write('\n') + f.close() + + def load(self, filename): + f = open(filename, 'rb') + SheetParser(self).parsefile(f) + f.close() + +class SheetParser: + + def __init__(self, sheet): + self.sheet = sheet + + def parsefile(self, f): + parser = expat.ParserCreate() + parser.StartElementHandler = self.startelement + parser.EndElementHandler = self.endelement + parser.CharacterDataHandler = self.data + parser.ParseFile(f) + + def startelement(self, tag, attrs): + method = getattr(self, 'start_'+tag, None) + if method: + for key, value in attrs.items(): + attrs[key] = str(value) # XXX Convert Unicode to 8-bit + method(attrs) + self.texts = [] + + def data(self, text): + text = str(text) # XXX Convert Unicode to 8-bit + self.texts.append(text) + + def endelement(self, tag): + method = getattr(self, 'end_'+tag, None) + if method: + method("".join(self.texts)) + + def start_cell(self, attrs): + self.y = int(attrs.get("row")) + self.x = int(attrs.get("col")) + + def start_value(self, attrs): + self.fmt = attrs.get('format') + self.alignment = xml2align.get(attrs.get('align')) + + start_formula = start_value + + def end_int(self, text): + try: + self.value = int(text) + except: + self.value = None + + def end_long(self, text): + try: + self.value = int(text) + except: + self.value = None + + def end_double(self, text): + try: + self.value = float(text) + except: + self.value = None + + def end_complex(self, text): + try: + self.value = complex(text) + except: + self.value = None + + def end_string(self, text): + try: + self.value = text + except: + self.value = None + + def end_value(self, text): + if isinstance(self.value, BaseCell): + self.cell = self.value + elif isinstance(self.value, str): + self.cell = StringCell(self.value, + self.fmt or "%s", + self.alignment or LEFT) + else: + self.cell = NumericCell(self.value, + self.fmt or "%s", + self.alignment or RIGHT) + + def end_formula(self, text): + self.cell = FormulaCell(text, + self.fmt or "%s", + self.alignment or RIGHT) + + def end_cell(self, text): + self.sheet.setcell(self.x, self.y, self.cell) + +class BaseCell: + __init__ = None # Must provide + """Abstract base class for sheet cells. + + Subclasses may but needn't provide the following APIs: + + cell.reset() -- prepare for recalculation + cell.recalc(ns) -> value -- recalculate formula + cell.format() -> (value, alignment) -- return formatted value + cell.xml() -> string -- return XML + """ + +class NumericCell(BaseCell): + + def __init__(self, value, fmt="%s", alignment=RIGHT): + assert isinstance(value, (int, int, float, complex)) + assert alignment in (LEFT, CENTER, RIGHT) + self.value = value + self.fmt = fmt + self.alignment = alignment + + def recalc(self, ns): + return self.value + + def format(self): + try: + text = self.fmt % self.value + except: + text = str(self.value) + return text, self.alignment + + def xml(self): + method = getattr(self, '_xml_' + type(self.value).__name__) + return '<value align="%s" format="%s">%s</value>' % ( + align2xml[self.alignment], + self.fmt, + method()) + + def _xml_int(self): + if -2**31 <= self.value < 2**31: + return '<int>%s</int>' % self.value + else: + return self._xml_long() + + def _xml_long(self): + return '<long>%s</long>' % self.value + + def _xml_float(self): + return '<double>%s</double>' % repr(self.value) + + def _xml_complex(self): + return '<complex>%s</double>' % repr(self.value) + +class StringCell(BaseCell): + + def __init__(self, text, fmt="%s", alignment=LEFT): + assert isinstance(text, (str, str)) + assert alignment in (LEFT, CENTER, RIGHT) + self.text = text + self.fmt = fmt + self.alignment = alignment + + def recalc(self, ns): + return self.text + + def format(self): + return self.text, self.alignment + + def xml(self): + s = '<value align="%s" format="%s"><string>%s</string></value>' + return s % ( + align2xml[self.alignment], + self.fmt, + html.escape(self.text)) + +class FormulaCell(BaseCell): + + def __init__(self, formula, fmt="%s", alignment=RIGHT): + assert alignment in (LEFT, CENTER, RIGHT) + self.formula = formula + self.translated = translate(self.formula) + self.fmt = fmt + self.alignment = alignment + self.reset() + + def reset(self): + self.value = None + + def recalc(self, ns): + if self.value is None: + try: + # A hack to evaluate expressions using true division + self.value = eval(self.translated, ns) + except: + exc = sys.exc_info()[0] + if hasattr(exc, "__name__"): + self.value = exc.__name__ + else: + self.value = str(exc) + return self.value + + def format(self): + try: + text = self.fmt % self.value + except: + text = str(self.value) + return text, self.alignment + + def xml(self): + return '<formula align="%s" format="%s">%s</formula>' % ( + align2xml[self.alignment], + self.fmt, + self.formula) + + def renumber(self, x1, y1, x2, y2, dx, dy): + out = [] + for part in re.split('(\w+)', self.formula): + m = re.match('^([A-Z]+)([1-9][0-9]*)$', part) + if m is not None: + sx, sy = m.groups() + x = colname2num(sx) + y = int(sy) + if x1 <= x <= x2 and y1 <= y <= y2: + part = cellname(x+dx, y+dy) + out.append(part) + return FormulaCell("".join(out), self.fmt, self.alignment) + +def translate(formula): + """Translate a formula containing fancy cell names to valid Python code. + + Examples: + B4 -> cell(2, 4) + B4:Z100 -> cells(2, 4, 26, 100) + """ + out = [] + for part in re.split(r"(\w+(?::\w+)?)", formula): + m = re.match(r"^([A-Z]+)([1-9][0-9]*)(?::([A-Z]+)([1-9][0-9]*))?$", part) + if m is None: + out.append(part) + else: + x1, y1, x2, y2 = m.groups() + x1 = colname2num(x1) + if x2 is None: + s = "cell(%s, %s)" % (x1, y1) + else: + x2 = colname2num(x2) + s = "cells(%s, %s, %s, %s)" % (x1, y1, x2, y2) + out.append(s) + return "".join(out) + +def cellname(x, y): + "Translate a cell coordinate to a fancy cell name (e.g. (1, 1)->'A1')." + assert x > 0 # Column 0 has an empty name, so can't use that + return colnum2name(x) + str(y) + +def colname2num(s): + "Translate a column name to number (e.g. 'A'->1, 'Z'->26, 'AA'->27)." + s = s.upper() + n = 0 + for c in s: + assert 'A' <= c <= 'Z' + n = n*26 + ord(c) - ord('A') + 1 + return n + +def colnum2name(n): + "Translate a column number to name (e.g. 1->'A', etc.)." + assert n > 0 + s = "" + while n: + n, m = divmod(n-1, 26) + s = chr(m+ord('A')) + s + return s + +import tkinter as Tk + +class SheetGUI: + + """Beginnings of a GUI for a spreadsheet. + + TO DO: + - clear multiple cells + - Insert, clear, remove rows or columns + - Show new contents while typing + - Scroll bars + - Grow grid when window is grown + - Proper menus + - Undo, redo + - Cut, copy and paste + - Formatting and alignment + """ + + def __init__(self, filename="sheet1.xml", rows=10, columns=5): + """Constructor. + + Load the sheet from the filename argument. + Set up the Tk widget tree. + """ + # Create and load the sheet + self.filename = filename + self.sheet = Sheet() + if os.path.isfile(filename): + self.sheet.load(filename) + # Calculate the needed grid size + maxx, maxy = self.sheet.getsize() + rows = max(rows, maxy) + columns = max(columns, maxx) + # Create the widgets + self.root = Tk.Tk() + self.root.wm_title("Spreadsheet: %s" % self.filename) + self.beacon = Tk.Label(self.root, text="A1", + font=('helvetica', 16, 'bold')) + self.entry = Tk.Entry(self.root) + self.savebutton = Tk.Button(self.root, text="Save", + command=self.save) + self.cellgrid = Tk.Frame(self.root) + # Configure the widget lay-out + self.cellgrid.pack(side="bottom", expand=1, fill="both") + self.beacon.pack(side="left") + self.savebutton.pack(side="right") + self.entry.pack(side="left", expand=1, fill="x") + # Bind some events + self.entry.bind("<Return>", self.return_event) + self.entry.bind("<Shift-Return>", self.shift_return_event) + self.entry.bind("<Tab>", self.tab_event) + self.entry.bind("<Shift-Tab>", self.shift_tab_event) + self.entry.bind("<Delete>", self.delete_event) + self.entry.bind("<Escape>", self.escape_event) + # Now create the cell grid + self.makegrid(rows, columns) + # Select the top-left cell + self.currentxy = None + self.cornerxy = None + self.setcurrent(1, 1) + # Copy the sheet cells to the GUI cells + self.sync() + + def delete_event(self, event): + if self.cornerxy != self.currentxy and self.cornerxy is not None: + self.sheet.clearcells(*(self.currentxy + self.cornerxy)) + else: + self.sheet.clearcell(*self.currentxy) + self.sync() + self.entry.delete(0, 'end') + return "break" + + def escape_event(self, event): + x, y = self.currentxy + self.load_entry(x, y) + + def load_entry(self, x, y): + cell = self.sheet.getcell(x, y) + if cell is None: + text = "" + elif isinstance(cell, FormulaCell): + text = '=' + cell.formula + else: + text, alignment = cell.format() + self.entry.delete(0, 'end') + self.entry.insert(0, text) + self.entry.selection_range(0, 'end') + + def makegrid(self, rows, columns): + """Helper to create the grid of GUI cells. + + The edge (x==0 or y==0) is filled with labels; the rest is real cells. + """ + self.rows = rows + self.columns = columns + self.gridcells = {} + # Create the top left corner cell (which selects all) + cell = Tk.Label(self.cellgrid, relief='raised') + cell.grid_configure(column=0, row=0, sticky='NSWE') + cell.bind("<ButtonPress-1>", self.selectall) + # Create the top row of labels, and configure the grid columns + for x in range(1, columns+1): + self.cellgrid.grid_columnconfigure(x, minsize=64) + cell = Tk.Label(self.cellgrid, text=colnum2name(x), relief='raised') + cell.grid_configure(column=x, row=0, sticky='WE') + self.gridcells[x, 0] = cell + cell.__x = x + cell.__y = 0 + cell.bind("<ButtonPress-1>", self.selectcolumn) + cell.bind("<B1-Motion>", self.extendcolumn) + cell.bind("<ButtonRelease-1>", self.extendcolumn) + cell.bind("<Shift-Button-1>", self.extendcolumn) + # Create the leftmost column of labels + for y in range(1, rows+1): + cell = Tk.Label(self.cellgrid, text=str(y), relief='raised') + cell.grid_configure(column=0, row=y, sticky='WE') + self.gridcells[0, y] = cell + cell.__x = 0 + cell.__y = y + cell.bind("<ButtonPress-1>", self.selectrow) + cell.bind("<B1-Motion>", self.extendrow) + cell.bind("<ButtonRelease-1>", self.extendrow) + cell.bind("<Shift-Button-1>", self.extendrow) + # Create the real cells + for x in range(1, columns+1): + for y in range(1, rows+1): + cell = Tk.Label(self.cellgrid, relief='sunken', + bg='white', fg='black') + cell.grid_configure(column=x, row=y, sticky='NSWE') + self.gridcells[x, y] = cell + cell.__x = x + cell.__y = y + # Bind mouse events + cell.bind("<ButtonPress-1>", self.press) + cell.bind("<B1-Motion>", self.motion) + cell.bind("<ButtonRelease-1>", self.release) + cell.bind("<Shift-Button-1>", self.release) + + def selectall(self, event): + self.setcurrent(1, 1) + self.setcorner(sys.maxint, sys.maxint) + + def selectcolumn(self, event): + x, y = self.whichxy(event) + self.setcurrent(x, 1) + self.setcorner(x, sys.maxint) + + def extendcolumn(self, event): + x, y = self.whichxy(event) + if x > 0: + self.setcurrent(self.currentxy[0], 1) + self.setcorner(x, sys.maxint) + + def selectrow(self, event): + x, y = self.whichxy(event) + self.setcurrent(1, y) + self.setcorner(sys.maxint, y) + + def extendrow(self, event): + x, y = self.whichxy(event) + if y > 0: + self.setcurrent(1, self.currentxy[1]) + self.setcorner(sys.maxint, y) + + def press(self, event): + x, y = self.whichxy(event) + if x > 0 and y > 0: + self.setcurrent(x, y) + + def motion(self, event): + x, y = self.whichxy(event) + if x > 0 and y > 0: + self.setcorner(x, y) + + release = motion + + def whichxy(self, event): + w = self.cellgrid.winfo_containing(event.x_root, event.y_root) + if w is not None and isinstance(w, Tk.Label): + try: + return w.__x, w.__y + except AttributeError: + pass + return 0, 0 + + def save(self): + self.sheet.save(self.filename) + + def setcurrent(self, x, y): + "Make (x, y) the current cell." + if self.currentxy is not None: + self.change_cell() + self.clearfocus() + self.beacon['text'] = cellname(x, y) + self.load_entry(x, y) + self.entry.focus_set() + self.currentxy = x, y + self.cornerxy = None + gridcell = self.gridcells.get(self.currentxy) + if gridcell is not None: + gridcell['bg'] = 'yellow' + + def setcorner(self, x, y): + if self.currentxy is None or self.currentxy == (x, y): + self.setcurrent(x, y) + return + self.clearfocus() + self.cornerxy = x, y + x1, y1 = self.currentxy + x2, y2 = self.cornerxy or self.currentxy + if x1 > x2: + x1, x2 = x2, x1 + if y1 > y2: + y1, y2 = y2, y1 + for (x, y), cell in self.gridcells.items(): + if x1 <= x <= x2 and y1 <= y <= y2: + cell['bg'] = 'lightBlue' + gridcell = self.gridcells.get(self.currentxy) + if gridcell is not None: + gridcell['bg'] = 'yellow' + self.setbeacon(x1, y1, x2, y2) + + def setbeacon(self, x1, y1, x2, y2): + if x1 == y1 == 1 and x2 == y2 == sys.maxint: + name = ":" + elif (x1, x2) == (1, sys.maxint): + if y1 == y2: + name = "%d" % y1 + else: + name = "%d:%d" % (y1, y2) + elif (y1, y2) == (1, sys.maxint): + if x1 == x2: + name = "%s" % colnum2name(x1) + else: + name = "%s:%s" % (colnum2name(x1), colnum2name(x2)) + else: + name1 = cellname(*self.currentxy) + name2 = cellname(*self.cornerxy) + name = "%s:%s" % (name1, name2) + self.beacon['text'] = name + + + def clearfocus(self): + if self.currentxy is not None: + x1, y1 = self.currentxy + x2, y2 = self.cornerxy or self.currentxy + if x1 > x2: + x1, x2 = x2, x1 + if y1 > y2: + y1, y2 = y2, y1 + for (x, y), cell in self.gridcells.items(): + if x1 <= x <= x2 and y1 <= y <= y2: + cell['bg'] = 'white' + + def return_event(self, event): + "Callback for the Return key." + self.change_cell() + x, y = self.currentxy + self.setcurrent(x, y+1) + return "break" + + def shift_return_event(self, event): + "Callback for the Return key with Shift modifier." + self.change_cell() + x, y = self.currentxy + self.setcurrent(x, max(1, y-1)) + return "break" + + def tab_event(self, event): + "Callback for the Tab key." + self.change_cell() + x, y = self.currentxy + self.setcurrent(x+1, y) + return "break" + + def shift_tab_event(self, event): + "Callback for the Tab key with Shift modifier." + self.change_cell() + x, y = self.currentxy + self.setcurrent(max(1, x-1), y) + return "break" + + def change_cell(self): + "Set the current cell from the entry widget." + x, y = self.currentxy + text = self.entry.get() + cell = None + if text.startswith('='): + cell = FormulaCell(text[1:]) + else: + for cls in int, int, float, complex: + try: + value = cls(text) + except: + continue + else: + cell = NumericCell(value) + break + if cell is None and text: + cell = StringCell(text) + if cell is None: + self.sheet.clearcell(x, y) + else: + self.sheet.setcell(x, y, cell) + self.sync() + + def sync(self): + "Fill the GUI cells from the sheet cells." + self.sheet.recalc() + for (x, y), gridcell in self.gridcells.items(): + if x == 0 or y == 0: + continue + cell = self.sheet.getcell(x, y) + if cell is None: + gridcell['text'] = "" + else: + if hasattr(cell, 'format'): + text, alignment = cell.format() + else: + text, alignment = str(cell), LEFT + gridcell['text'] = text + gridcell['anchor'] = align2anchor[alignment] + + +def test_basic(): + "Basic non-gui self-test." + import os + a = Sheet() + for x in range(1, 11): + for y in range(1, 11): + if x == 1: + cell = NumericCell(y) + elif y == 1: + cell = NumericCell(x) + else: + c1 = cellname(x, 1) + c2 = cellname(1, y) + formula = "%s*%s" % (c1, c2) + cell = FormulaCell(formula) + a.setcell(x, y, cell) +## if os.path.isfile("sheet1.xml"): +## print "Loading from sheet1.xml" +## a.load("sheet1.xml") + a.display() + a.save("sheet1.xml") + +def test_gui(): + "GUI test." + if sys.argv[1:]: + filename = sys.argv[1] + else: + filename = "sheet1.xml" + g = SheetGUI(filename) + g.root.mainloop() + +if __name__ == '__main__': + #test_basic() + test_gui() diff --git a/Tools/demo/vector.py b/Tools/demo/vector.py new file mode 100755 index 0000000..da5b389 --- /dev/null +++ b/Tools/demo/vector.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 + +""" +A demonstration of classes and their special methods in Python. +""" + +class Vec: + """A simple vector class. + + Instances of the Vec class can be constructed from numbers + + >>> a = Vec(1, 2, 3) + >>> b = Vec(3, 2, 1) + + added + >>> a + b + Vec(4, 4, 4) + + subtracted + >>> a - b + Vec(-2, 0, 2) + + and multiplied by a scalar on the left + >>> 3.0 * a + Vec(3.0, 6.0, 9.0) + + or on the right + >>> a * 3.0 + Vec(3.0, 6.0, 9.0) + """ + def __init__(self, *v): + self.v = list(v) + + @classmethod + def fromlist(cls, v): + if not isinstance(v, list): + raise TypeError + inst = cls() + inst.v = v + return inst + + def __repr__(self): + args = ', '.join(repr(x) for x in self.v) + return 'Vec({})'.format(args) + + def __len__(self): + return len(self.v) + + def __getitem__(self, i): + return self.v[i] + + def __add__(self, other): + # Element-wise addition + v = [x + y for x, y in zip(self.v, other.v)] + return Vec.fromlist(v) + + def __sub__(self, other): + # Element-wise subtraction + v = [x - y for x, y in zip(self.v, other.v)] + return Vec.fromlist(v) + + def __mul__(self, scalar): + # Multiply by scalar + v = [x * scalar for x in self.v] + return Vec.fromlist(v) + + __rmul__ = __mul__ + + +def test(): + import doctest + doctest.testmod() + +test() diff --git a/Tools/faqwiz/README b/Tools/faqwiz/README deleted file mode 100644 index a1e4d44..0000000 --- a/Tools/faqwiz/README +++ /dev/null @@ -1,114 +0,0 @@ -FAQ Wizard ----------- - -Author: Guido van Rossum <guido@python.org> -Version: 1.0 -Date: 6 April 1998 - - -This is a CGI program that maintains a user-editable FAQ. It uses RCS -to keep track of changes to individual FAQ entries. It is fully -configurable; everything you might want to change when using this -program to maintain some other FAQ than the Python FAQ is contained in -the configuration module, faqconf.py. - -Note that the bulk of the code is not an executable script; it's an -importable module. The actual script in cgi-bin is minimal. - -Files: - -faqw.py executable script to be edited and installed in cgi-bin -faqwiz.py main module, lives in same directory as FAQ entry files -faqconf.py main configuration module -faqcust.py additional local customization module (optional) -move-faqwiz.sh Script to move faqwiz entries. - - -What's New? ------------ - -Version 1.0 corrects some minor bugs and uses tab-agnostic -indentation; it is otherwise unchanged from version 0.9.0. - -Version 0.9.0 uses the re module (Perl style regular expressions) for -all its regular expression needs, instead of the regex and regsub -modules (Emacs style). This affects the syntax for regular -expressions entered by the user as search strings (with "regular -expression" checked), hence the version number jump. - - -Setup Information ------------------ - -This assumes you are familiar with Python, with your http server, and -with running CGI scripts under your http server. You need Python 1.5 -or better. - -Select a place where the Python modules that constitute the FAQ wizard -will live (the directory where you unpacked it is an obvious choice). -This will be called the SRCDIR. This directory should not be writable -by other users of your system (since they would be able to execute -arbitrary code by invoking the FAQ wizard's CGI script). - -Create a dedicated working directory, preferably one that's not -directly reachable from your http server. This will be called the -FAQDIR. Create a subdirectory named RCS. Make both the working -directory and the RCS subdirectory wrld-writable. (This is essential, -since the FAQ wizard runs as use nobody, and needs to create -additional files here!) - -Edit faqconf.py to reflect your setup. You only need to edit the top -part, up till the line of all dashes. The comments should guide you -in your edits. (Actually, you can also choose to add your changes to -faqcust.py and leave faqconf.py alone. This is essential if you are -maintaining multiple FAQs; see below.) - -Don't forget to edit the SECTION_TITLES variables to reflect the set -of section titles for your FAQ! - -Next, edit faqw.py to reflect the pathname of your Python interpreter -and the values for SRCDIR and FAQDIR that you just chose. Then -install faqw.py in your cgi-bin directory. Make sure that it is -world-executable. You should now be able to connect to the FAQ wizard -by entering the following URL in your web client (subsituting the -appropriate host and port for "your.web.server", and perhaps -specifying a different directory for "cgi-bin" if local conventions so -dictate): - - http://your.web.server/cgi-bin/faqw.py - -If you are unable to get this working, check your server's error_log -file. The documentation for Python's cgi module in the Python Library -Reference Manual gives plentyu additional information about installing -and debugging CGI scripts, including setup debugging. This -documentation is repeated in the doc string in the cgi module; try -``import cgi; print cgi.__doc__''. - -Assuming this works, you should now be able to add the first entry to -your FAQ using the FAQ wizard interface. This creates a file -faq01.001.htp in your working directory and an RCS revision history -file faq01.001.htp,v in the RCS subdirectory. You can now exercise -the other FAQ wizard features (search, index, whole FAQ, what's new, -roulette, and so on). - - -Maintaining Multiple FAQs -------------------------- - -If you have multiple FAQs, you need a separate FAQDIR per FAQ, and a -different customization file per FAQ. The easiest thing to do would -be to have the faqcust.py for each FAQ live in the FAQDIR for that -FAQ, but that creates some security concerns, since the FAQDIR must be -world writable: *if* someone who breaks into your system (or a -legitimate user) manages to edit the faqcust.py file they can get -arbitrary code to execute through the FAQ wizard. Therefore, you will -need a more complex setup. - -The best way is probably to have a directory that is only writable by -you for each FAQ, where you place the copy of faqcust.py for that FAQ, -and have a world-writable subdirectory DATA for the data. You then -set FAQDIR to point to the DATA directory and change the faqw.py -bootstrap script to add FAQDIR/.. to sys.path (in front of SRCDIR, so -the dummy faqcust.py from SRCDIR is ignored). - ---Guido van Rossum (home page: http://www.python.org/~guido/) diff --git a/Tools/faqwiz/faqconf.py b/Tools/faqwiz/faqconf.py deleted file mode 100644 index d1acd80..0000000 --- a/Tools/faqwiz/faqconf.py +++ /dev/null @@ -1,577 +0,0 @@ -"""FAQ Wizard customization module. - -Edit this file to customize the FAQ Wizard. For normal purposes, you -should only have to change the FAQ section titles and the small group -of parameters below it. - -""" - -# Titles of FAQ sections - -SECTION_TITLES = { - # SectionNumber : SectionTitle; need at least one entry - 1: "General information and availability", -} - -# Parameters you definitely want to change - -SHORTNAME = "Generic" # FAQ name with "FAQ" omitted -PASSWORD = "" # Password for editing -OWNERNAME = "FAQ owner" # Name for feedback -OWNEREMAIL = "nobody@anywhere.org" # Email for feedback -HOMEURL = "http://www.python.org" # Related home page -HOMENAME = "Python home" # Name of related home page -RCSBINDIR = "/usr/local/bin/" # Directory containing RCS commands - # (must end in a slash) - -# Parameters you can normally leave alone - -MAXHITS = 10 # Max #hits to be shown directly -COOKIE_LIFETIME = 28*24*3600 # Cookie expiration in seconds - # (28*24*3600 = 28 days = 4 weeks) -PROCESS_PREFORMAT = 1 # toggle whether preformatted text - # will replace urls and emails with - # HTML links - -# Markers appended to title to indicate recently change -# (may contain HTML, e.g. <IMG>); and corresponding - -MARK_VERY_RECENT = " **" # Changed very recently -MARK_RECENT = " *" # Changed recently -DT_VERY_RECENT = 24*3600 # 24 hours -DT_RECENT = 7*24*3600 # 7 days - -EXPLAIN_MARKS = """ -<P>(Entries marked with ** were changed within the last 24 hours; -entries marked with * were changed within the last 7 days.) -<P> -""" - -# Version -- don't change unless you edit faqwiz.py - -WIZVERSION = "1.0.4" # FAQ Wizard version - -import os, sys -if os.name in ['nt',]: - # On NT we'll probably be running python from a batch file, - # so sys.argv[0] is not helpful - FAQCGI = 'faq.bat' # Relative URL of the FAQ cgi script - # LOGNAME is not typically set on NT - os.environ[ 'LOGNAME' ] = "FAQWizard" -else: - # This parameter is normally overwritten with a dynamic value - FAQCGI = 'faqw.py' # Relative URL of the FAQ cgi script - FAQCGI = os.path.basename(sys.argv[0]) or FAQCGI -del os, sys - -# Perl (re module) style regular expression to recognize FAQ entry -# files: group(1) should be the section number, group(2) should be the -# question number. Both should be fixed width so simple-minded -# sorting yields the right order. - -OKFILENAME = r"^faq(\d\d)\.(\d\d\d)\.htp$" - -# Format to construct a FAQ entry file name - -NEWFILENAME = "faq%02d.%03d.htp" - -# Load local customizations on top of the previous parameters - -try: - from faqcust import * -except ImportError: - pass - -# Calculated parameter names - -COOKIE_NAME = SHORTNAME + "-FAQ-Wizard" # Name used for Netscape cookie -FAQNAME = SHORTNAME + " FAQ" # Name of the FAQ - -# ---------------------------------------------------------------------- - -# Anything below this point normally needn't be changed; you would -# change this if you were to create e.g. a French translation or if -# you just aren't happy with the text generated by the FAQ Wizard. - -# Most strings here are subject to substitution (string%dictionary) - -# RCS commands - -import os -if os.name in ['nt', ]: - SH_RLOG = RCSBINDIR + "rlog %(file)s < NUL" - SH_RLOG_H = RCSBINDIR + "rlog -h %(file)s < NUL" - SH_RDIFF = RCSBINDIR + "rcsdiff -r%(prev)s -r%(rev)s %(file)s < NUL" - SH_REVISION = RCSBINDIR + "co -p%(rev)s %(file)s < NUL" - ### Have to use co -l, or the file is not marked rw on NT - SH_LOCK = RCSBINDIR + "co -l %(file)s < NUL" - SH_CHECKIN = RCSBINDIR + "ci -u %(file)s < %(tfn)s" -else: - SH_RLOG = RCSBINDIR + "rlog %(file)s </dev/null 2>&1" - SH_RLOG_H = RCSBINDIR + "rlog -h %(file)s </dev/null 2>&1" - SH_RDIFF = RCSBINDIR + "rcsdiff -r%(prev)s -r%(rev)s %(file)s </dev/null 2>&1" - SH_REVISION = RCSBINDIR + "co -p%(rev)s %(file)s </dev/null 2>&1" - SH_LOCK = RCSBINDIR + "rcs -l %(file)s </dev/null 2>&1" - SH_CHECKIN = RCSBINDIR + "ci -u %(file)s <%(tfn)s 2>&1" -del os - -# Titles for various output pages (not subject to substitution) - -T_HOME = FAQNAME + " Wizard " + WIZVERSION -T_ERROR = "Sorry, an error occurred" -T_ROULETTE = FAQNAME + " Roulette" -T_ALL = "The Whole " + FAQNAME -T_INDEX = FAQNAME + " Index" -T_SEARCH = FAQNAME + " Search Results" -T_RECENT = "What's New in the " + FAQNAME -T_SHOW = FAQNAME + " Entry" -T_LOG = "RCS log for %s entry" % FAQNAME -T_REVISION = "RCS revision for %s entry" % FAQNAME -T_DIFF = "RCS diff for %s entry" % FAQNAME -T_ADD = "Add an entry to the " + FAQNAME -T_DELETE = "Deleting an entry from the " + FAQNAME -T_EDIT = FAQNAME + " Edit Wizard" -T_REVIEW = T_EDIT + " - Review Changes" -T_COMMITTED = T_EDIT + " - Changes Committed" -T_COMMITFAILED = T_EDIT + " - Commit Failed" -T_CANTCOMMIT = T_EDIT + " - Commit Rejected" -T_HELP = T_EDIT + " - Help" - -# Generic prologue and epilogue - -PROLOGUE = ''' -<HTML> -<HEAD> -<TITLE>%(title)s</TITLE> -</HEAD> - -<BODY - BGCOLOR="#FFFFFF" - TEXT="#000000" - LINK="#AA0000" - VLINK="#906A6A"> -<H1>%(title)s</H1> -''' - -EPILOGUE = ''' -<HR> -<A HREF="%(HOMEURL)s">%(HOMENAME)s</A> / -<A HREF="%(FAQCGI)s?req=home">%(FAQNAME)s Wizard %(WIZVERSION)s</A> / -Feedback to <A HREF="mailto:%(OWNEREMAIL)s">%(OWNERNAME)s</A> - -</BODY> -</HTML> -''' - -# Home page - -HOME = """ -<H2>Search the %(FAQNAME)s:</H2> - -<BLOCKQUOTE> - -<FORM ACTION="%(FAQCGI)s"> - <INPUT TYPE=text NAME=query> - <INPUT TYPE=submit VALUE="Search"><BR> - <INPUT TYPE=radio NAME=querytype VALUE=simple CHECKED> - Simple string - / - <INPUT TYPE=radio NAME=querytype VALUE=regex> - Regular expression - /<BR> - <INPUT TYPE=radio NAME=querytype VALUE=anykeywords> - Keywords (any) - / - <INPUT TYPE=radio NAME=querytype VALUE=allkeywords> - Keywords (all) - <BR> - <INPUT TYPE=radio NAME=casefold VALUE=yes CHECKED> - Fold case - / - <INPUT TYPE=radio NAME=casefold VALUE=no> - Case sensitive - <BR> - <INPUT TYPE=hidden NAME=req VALUE=search> -</FORM> - -</BLOCKQUOTE> - -<HR> - -<H2>Other forms of %(FAQNAME)s access:</H2> - -<UL> -<LI><A HREF="%(FAQCGI)s?req=index">FAQ index</A> -<LI><A HREF="%(FAQCGI)s?req=all">The whole FAQ</A> -<LI><A HREF="%(FAQCGI)s?req=recent">What's new in the FAQ?</A> -<LI><A HREF="%(FAQCGI)s?req=roulette">FAQ roulette</A> -<LI><A HREF="%(FAQCGI)s?req=add">Add a FAQ entry</A> -<LI><A HREF="%(FAQCGI)s?req=delete">Delete a FAQ entry</A> -</UL> -""" - -# Index formatting - -INDEX_SECTION = """ -<P> -<HR> -<H2>%(sec)s. %(title)s</H2> -<UL> -""" - -INDEX_ADDSECTION = """ -<P> -<LI><A HREF="%(FAQCGI)s?req=new&section=%(sec)s">Add new entry</A> -(at this point) -""" - -INDEX_ENDSECTION = """ -</UL> -""" - -INDEX_ENTRY = """\ -<LI><A HREF="%(FAQCGI)s?req=show&file=%(file)s">%(title)s</A> -""" - -LOCAL_ENTRY = """\ -<LI><A HREF="#%(sec)s.%(num)s">%(title)s</A> -""" - -# Entry formatting - -ENTRY_HEADER1 = """ -<HR> -<H2><A NAME="%(sec)s.%(num)s">%(title)s</A>\ -""" - -ENTRY_HEADER2 = """\ -</H2> -""" - -ENTRY_FOOTER = """ -<A HREF="%(FAQCGI)s?req=edit&file=%(file)s">Edit this entry</A> / -<A HREF="%(FAQCGI)s?req=log&file=%(file)s">Log info</A> -""" - -ENTRY_LOGINFO = """ -/ Last changed on %(last_changed_date)s by -<A HREF="mailto:%(last_changed_email)s">%(last_changed_author)s</A> -""" - -# Search - -NO_HITS = """ -No hits. -""" - -ONE_HIT = """ -Your search matched the following entry: -""" - -FEW_HITS = """ -Your search matched the following %(count)s entries: -""" - -MANY_HITS = """ -Your search matched more than %(MAXHITS)s entries. -The %(count)s matching entries are presented here ordered by section: -""" - -# RCS log and diff - -LOG = """ -Click on a revision line to see the diff between that revision and the -previous one. -""" - -REVISIONLINK = """\ -<A HREF="%(FAQCGI)s?req=revision&file=%(file)s&rev=%(rev)s" ->%(line)s</A>\ -""" -DIFFLINK = """\ - (<A HREF="%(FAQCGI)s?req=diff&file=%(file)s&\ -prev=%(prev)s&rev=%(rev)s" ->diff -r%(prev)s -r%(rev)s</A>)\ -""" - -# Recently changed entries - -NO_RECENT = """ -<HR> -No %(FAQNAME)s entries were changed in the last %(period)s. -""" - -VIEW_MENU = """ -<HR> -View entries changed in the last... -<UL> -<LI><A HREF="%(FAQCGI)s?req=recent&days=1">24 hours</A> -<LI><A HREF="%(FAQCGI)s?req=recent&days=2">2 days</A> -<LI><A HREF="%(FAQCGI)s?req=recent&days=3">3 days</A> -<LI><A HREF="%(FAQCGI)s?req=recent&days=7">week</A> -<LI><A HREF="%(FAQCGI)s?req=recent&days=28">4 weeks</A> -<LI><A HREF="%(FAQCGI)s?req=recent&days=365250">millennium</A> -</UL> -""" - -ONE_RECENT = VIEW_MENU + """ -The following %(FAQNAME)s entry was changed in the last %(period)s: -""" - -SOME_RECENT = VIEW_MENU + """ -The following %(count)s %(FAQNAME)s entries were changed -in the last %(period)s, most recently changed shown first: -""" - -TAIL_RECENT = VIEW_MENU - -# Last changed banner on "all" (strftime format) -LAST_CHANGED = "Last changed on %c %Z" - -# "Compat" command prologue (this has no <BODY> tag) -COMPAT = """ -<H1>The whole %(FAQNAME)s</H1> -See also the <A HREF="%(FAQCGI)s?req=home">%(FAQNAME)s Wizard</A>. -<P> -""" - -# Editing - -EDITHEAD = """ -<A HREF="%(FAQCGI)s?req=help">Click for Help</A> -""" - -REVIEWHEAD = EDITHEAD - - -EDITFORM1 = """ -<FORM ACTION="%(FAQCGI)s" METHOD=POST> -<INPUT TYPE=hidden NAME=req VALUE=review> -<INPUT TYPE=hidden NAME=file VALUE=%(file)s> -<INPUT TYPE=hidden NAME=editversion VALUE=%(editversion)s> -<HR> -""" - -EDITFORM2 = """ -Title: <INPUT TYPE=text SIZE=70 NAME=title VALUE="%(title)s"><BR> -<TEXTAREA COLS=72 ROWS=20 NAME=body>%(body)s -</TEXTAREA><BR> -Log message (reason for the change):<BR> -<TEXTAREA COLS=72 ROWS=5 NAME=log>%(log)s -</TEXTAREA><BR> -Please provide the following information for logging purposes: -<TABLE FRAME=none COLS=2> - <TR> - <TD>Name: - <TD><INPUT TYPE=text SIZE=40 NAME=author VALUE="%(author)s"> - <TR> - <TD>Email: - <TD><INPUT TYPE=text SIZE=40 NAME=email VALUE="%(email)s"> - <TR> - <TD>Password: - <TD><INPUT TYPE=password SIZE=20 NAME=password VALUE="%(password)s"> -</TABLE> - -<INPUT TYPE=submit NAME=review VALUE="Preview Edit"> -Click this button to preview your changes. -""" - -EDITFORM3 = """ -</FORM> -""" - -COMMIT = """ -<INPUT TYPE=submit NAME=commit VALUE="Commit"> -Click this button to commit your changes. -<HR> -""" - -NOCOMMIT_HEAD = """ -To commit your changes, please correct the following errors in the -form below and click the Preview Edit button. -<UL> -""" -NOCOMMIT_TAIL = """ -</UL> -<HR> -""" - -CANTCOMMIT_HEAD = """ -Some required information is missing: -<UL> -""" -NEED_PASSWD = "<LI>You must provide the correct password.\n" -NEED_AUTHOR = "<LI>You must enter your name.\n" -NEED_EMAIL = "<LI>You must enter your email address.\n" -NEED_LOG = "<LI>You must enter a log message.\n" -CANTCOMMIT_TAIL = """ -</UL> -Please use your browser's Back command to correct the form and commit -again. -""" - -NEWCONFLICT = """ -<P> -You are creating a new entry, but the entry number specified is not -correct. -<P> -The two most common causes of this problem are: -<UL> -<LI>After creating the entry yourself, you went back in your browser, - edited the entry some more, and clicked Commit again. -<LI>Someone else started creating a new entry in the same section and - committed before you did. -</UL> -(It is also possible that the last entry in the section was physically -deleted, but this should not happen except through manual intervention -by the FAQ maintainer.) -<P> -<A HREF="%(FAQCGI)s?req=new&section=%(sec)s">Click here to try -again.</A> -<P> -""" - -VERSIONCONFLICT = """ -<P> -You edited version %(editversion)s but the current version is %(version)s. -<P> -The two most common causes of this problem are: -<UL> -<LI>After committing a change, you went back in your browser, - edited the entry some more, and clicked Commit again. -<LI>Someone else started editing the same entry and committed - before you did. -</UL> -<P> -<A HREF="%(FAQCGI)s?req=show&file=%(file)s">Click here to reload -the entry and try again.</A> -<P> -""" - -CANTWRITE = """ -Can't write file %(file)s (%(why)s). -""" - -FILEHEADER = """\ -Title: %(title)s -Last-Changed-Date: %(date)s -Last-Changed-Author: %(author)s -Last-Changed-Email: %(email)s -Last-Changed-Remote-Host: %(REMOTE_HOST)s -Last-Changed-Remote-Address: %(REMOTE_ADDR)s -""" - -LOGHEADER = """\ -Last-Changed-Date: %(date)s -Last-Changed-Author: %(author)s -Last-Changed-Email: %(email)s -Last-Changed-Remote-Host: %(REMOTE_HOST)s -Last-Changed-Remote-Address: %(REMOTE_ADDR)s - -%(log)s -""" - -COMMITTED = """ -Your changes have been committed. -""" - -COMMITFAILED = """ -Exit status %(sts)s. -""" - -# Add/Delete - -ADD_HEAD = """ -At the moment, new entries can only be added at the end of a section. -This is because the entry numbers are also their -unique identifiers -- it's a bad idea to renumber entries. -<P> -Click on the section to which you want to add a new entry: -<UL> -""" - -ADD_SECTION = """\ -<LI><A HREF="%(FAQCGI)s?req=new&section=%(section)s">%(section)s. %(title)s</A> -""" - -ADD_TAIL = """ -</UL> -""" - -ROULETTE = """ -<P>Hit your browser's Reload button to play again.<P> -""" - -DELETE = """ -At the moment, there's no direct way to delete entries. -This is because the entry numbers are also their -unique identifiers -- it's a bad idea to renumber entries. -<P> -If you really think an entry needs to be deleted, -change the title to "(deleted)" and make the body -empty (keep the entry number in the title though). -""" - -# Help file for the FAQ Edit Wizard - -HELP = """ -Using the %(FAQNAME)s Edit Wizard speaks mostly for itself. Here are -some answers to questions you are likely to ask: - -<P><HR> - -<H2>I can review an entry but I can't commit it.</H2> - -The commit button only appears if the following conditions are met: - -<UL> - -<LI>The Name field is not empty. - -<LI>The Email field contains at least an @ character. - -<LI>The Log message box is not empty. - -<LI>The Password field contains the proper password. - -</UL> - -<P><HR> - -<H2>What is the password?</H2> - -At the moment, only PSA members will be told the password. This is a -good time to join the PSA! See <A -HREF="http://www.python.org/psa/">the PSA home page</A>. - -<P><HR> - -<H2>Can I use HTML in the FAQ entry?</H2> - -Yes, if you include it in <HTML&rt; and </HTML> tags. -<P> -Also, if you include a URL or an email address in the text it will -automatigally become an anchor of the right type. Also, *word* -is made italic (but only for single alphabetic words). - -<P><HR> - -<H2>How do I delineate paragraphs?</H2> - -Use blank lines to separate paragraphs. - -<P><HR> - -<H2>How do I enter example text?</H2> - -Any line that begins with a space or tab is assumed to be part of -literal text. Blocks of literal text delineated by blank lines are -placed inside <PRE>...</PRE>. -""" - -# Load local customizations again, in case they set some other variables - -try: - from faqcust import * -except ImportError: - pass diff --git a/Tools/faqwiz/faqcust.py b/Tools/faqwiz/faqcust.py deleted file mode 100644 index 8f16781..0000000 --- a/Tools/faqwiz/faqcust.py +++ /dev/null @@ -1 +0,0 @@ -# Add your customizations here -- modified copies of what's in faqconf.py. diff --git a/Tools/faqwiz/faqw.py b/Tools/faqwiz/faqw.py deleted file mode 100755 index d972f61..0000000 --- a/Tools/faqwiz/faqw.py +++ /dev/null @@ -1,33 +0,0 @@ -#! /usr/local/bin/python - -"""FAQ wizard bootstrap.""" - -# This is a longer version of the bootstrap script given at the end of -# faqwin.py; it prints timing statistics at the end of the regular CGI -# script's output (so you can monitor how it is doing). - -# This script should be placed in your cgi-bin directory and made -# executable. - -# You need to edit the first line and the lines that define FAQDIR and -# SRCDIR, below: change /usr/local/bin/python to where your Python -# interpreter lives, change the value for FAQDIR to where your FAQ -# lives, and change the value for SRCDIR to where your faqwiz.py -# module lives. The faqconf.py and faqcust.py files live there, too. - -import os -t1 = os.times() # If this doesn't work, just get rid of the timing code! -try: - FAQDIR = "/usr/people/guido/python/FAQ" - SRCDIR = "/usr/people/guido/python/src/Tools/faqwiz" - import os, sys - os.chdir(FAQDIR) - sys.path.insert(0, SRCDIR) - import faqwiz -except SystemExit as n: - sys.exit(n) -except: - t, v, tb = sys.exc_info() - print() - import cgi - cgi.print_exception(t, v, tb) diff --git a/Tools/faqwiz/faqwiz.py b/Tools/faqwiz/faqwiz.py deleted file mode 100644 index b9ab65d..0000000 --- a/Tools/faqwiz/faqwiz.py +++ /dev/null @@ -1,840 +0,0 @@ -"""Generic FAQ Wizard. - -This is a CGI program that maintains a user-editable FAQ. It uses RCS -to keep track of changes to individual FAQ entries. It is fully -configurable; everything you might want to change when using this -program to maintain some other FAQ than the Python FAQ is contained in -the configuration module, faqconf.py. - -Note that this is not an executable script; it's an importable module. -The actual script to place in cgi-bin is faqw.py. - -""" - -import sys, time, os, stat, re, cgi, faqconf -from faqconf import * # This imports all uppercase names -now = time.time() - -class FileError: - def __init__(self, file): - self.file = file - -class InvalidFile(FileError): - pass - -class NoSuchSection(FileError): - def __init__(self, section): - FileError.__init__(self, NEWFILENAME %(section, 1)) - self.section = section - -class NoSuchFile(FileError): - def __init__(self, file, why=None): - FileError.__init__(self, file) - self.why = why - -def escape(s): - s = s.replace('&', '&') - s = s.replace('<', '<') - s = s.replace('>', '>') - return s - -def escapeq(s): - s = escape(s) - s = s.replace('"', '"') - return s - -def _interpolate(format, args, kw): - try: - quote = kw['_quote'] - except KeyError: - quote = 1 - d = (kw,) + args + (faqconf.__dict__,) - m = MagicDict(d, quote) - return format % m - -def interpolate(format, *args, **kw): - return _interpolate(format, args, kw) - -def emit(format, *args, **kw): - try: - f = kw['_file'] - except KeyError: - f = sys.stdout - f.write(_interpolate(format, args, kw)) - -translate_prog = None - -def translate(text, pre=0): - global translate_prog - if not translate_prog: - translate_prog = prog = re.compile( - r'\b(http|ftp|https)://\S+(\b|/)|\b[-.\w]+@[-.\w]+') - else: - prog = translate_prog - i = 0 - list = [] - while 1: - m = prog.search(text, i) - if not m: - break - j = m.start() - list.append(escape(text[i:j])) - i = j - url = m.group(0) - while url[-1] in '();:,.?\'"<>': - url = url[:-1] - i = i + len(url) - url = escape(url) - if not pre or (pre and PROCESS_PREFORMAT): - if ':' in url: - repl = '<A HREF="%s">%s</A>' % (url, url) - else: - repl = '<A HREF="mailto:%s">%s</A>' % (url, url) - else: - repl = url - list.append(repl) - j = len(text) - list.append(escape(text[i:j])) - return ''.join(list) - -def emphasize(line): - return re.sub(r'\*([a-zA-Z]+)\*', r'<I>\1</I>', line) - -revparse_prog = None - -def revparse(rev): - global revparse_prog - if not revparse_prog: - revparse_prog = re.compile(r'^(\d{1,3})\.(\d{1,4})$') - m = revparse_prog.match(rev) - if not m: - return None - [major, minor] = map(int, m.group(1, 2)) - return major, minor - -logon = 0 -def log(text): - if logon: - logfile = open("logfile", "a") - logfile.write(text + "\n") - logfile.close() - -def load_cookies(): - if 'HTTP_COOKIE' not in os.environ: - return {} - raw = os.environ['HTTP_COOKIE'] - words = [s.strip() for s in raw.split(';')] - cookies = {} - for word in words: - i = word.find('=') - if i >= 0: - key, value = word[:i], word[i+1:] - cookies[key] = value - return cookies - -def load_my_cookie(): - cookies = load_cookies() - try: - value = cookies[COOKIE_NAME] - except KeyError: - return {} - import urllib.parse - value = urllib.parse.unquote(value) - words = value.split('/') - while len(words) < 3: - words.append('') - author = '/'.join(words[:-2]) - email = words[-2] - password = words[-1] - return {'author': author, - 'email': email, - 'password': password} - -def send_my_cookie(ui): - name = COOKIE_NAME - value = "%s/%s/%s" % (ui.author, ui.email, ui.password) - import urllib.parse - value = urllib.parse.quote(value) - then = now + COOKIE_LIFETIME - gmt = time.gmtime(then) - path = os.environ.get('SCRIPT_NAME', '/cgi-bin/') - print("Set-Cookie: %s=%s; path=%s;" % (name, value, path), end=' ') - print(time.strftime("expires=%a, %d-%b-%y %X GMT", gmt)) - -class MagicDict: - - def __init__(self, d, quote): - self.__d = d - self.__quote = quote - - def __getitem__(self, key): - for d in self.__d: - try: - value = d[key] - if value: - value = str(value) - if self.__quote: - value = escapeq(value) - return value - except KeyError: - pass - return '' - -class UserInput: - - def __init__(self): - self.__form = cgi.FieldStorage() - #log("\n\nbody: " + self.body) - - def __getattr__(self, name): - if name[0] == '_': - raise AttributeError - try: - value = self.__form[name].value - except (TypeError, KeyError): - value = '' - else: - value = value.strip() - setattr(self, name, value) - return value - - def __getitem__(self, key): - return getattr(self, key) - -class FaqEntry: - - def __init__(self, fp, file, sec_num): - self.file = file - self.sec, self.num = sec_num - if fp: - import email - self.__headers = email.message_from_file(fp) - self.body = fp.read().strip() - else: - self.__headers = {'title': "%d.%d. " % sec_num} - self.body = '' - - def __getattr__(self, name): - if name[0] == '_': - raise AttributeError - key = '-'.join(name.split('_')) - try: - value = self.__headers[key] - except KeyError: - value = '' - setattr(self, name, value) - return value - - def __getitem__(self, key): - return getattr(self, key) - - def load_version(self): - command = interpolate(SH_RLOG_H, self) - p = os.popen(command) - version = '' - while 1: - line = p.readline() - if not line: - break - if line[:5] == 'head:': - version = line[5:].strip() - p.close() - self.version = version - - def getmtime(self): - if not self.last_changed_date: - return 0 - try: - return os.stat(self.file)[stat.ST_MTIME] - except os.error: - return 0 - - def emit_marks(self): - mtime = self.getmtime() - if mtime >= now - DT_VERY_RECENT: - emit(MARK_VERY_RECENT, self) - elif mtime >= now - DT_RECENT: - emit(MARK_RECENT, self) - - def show(self, edit=1): - emit(ENTRY_HEADER1, self) - self.emit_marks() - emit(ENTRY_HEADER2, self) - pre = 0 - raw = 0 - for line in self.body.split('\n'): - # Allow the user to insert raw html into a FAQ answer - # (Skip Montanaro, with changes by Guido) - tag = line.rstrip().lower() - if tag == '<html>': - raw = 1 - continue - if tag == '</html>': - raw = 0 - continue - if raw: - print(line) - continue - if not line.strip(): - if pre: - print('</PRE>') - pre = 0 - else: - print('<P>') - else: - if not line[0].isspace(): - if pre: - print('</PRE>') - pre = 0 - else: - if not pre: - print('<PRE>') - pre = 1 - if '/' in line or '@' in line: - line = translate(line, pre) - elif '<' in line or '&' in line: - line = escape(line) - if not pre and '*' in line: - line = emphasize(line) - print(line) - if pre: - print('</PRE>') - pre = 0 - if edit: - print('<P>') - emit(ENTRY_FOOTER, self) - if self.last_changed_date: - emit(ENTRY_LOGINFO, self) - print('<P>') - -class FaqDir: - - entryclass = FaqEntry - - __okprog = re.compile(OKFILENAME) - - def __init__(self, dir=os.curdir): - self.__dir = dir - self.__files = None - - def __fill(self): - if self.__files is not None: - return - self.__files = files = [] - okprog = self.__okprog - for file in os.listdir(self.__dir): - if self.__okprog.match(file): - files.append(file) - files.sort() - - def good(self, file): - return self.__okprog.match(file) - - def parse(self, file): - m = self.good(file) - if not m: - return None - sec, num = m.group(1, 2) - return int(sec), int(num) - - def list(self): - # XXX Caller shouldn't modify result - self.__fill() - return self.__files - - def open(self, file): - sec_num = self.parse(file) - if not sec_num: - raise InvalidFile(file) - try: - fp = open(file) - except IOError as msg: - raise NoSuchFile(file, msg) - try: - return self.entryclass(fp, file, sec_num) - finally: - fp.close() - - def show(self, file, edit=1): - self.open(file).show(edit=edit) - - def new(self, section): - if section not in SECTION_TITLES: - raise NoSuchSection(section) - maxnum = 0 - for file in self.list(): - sec, num = self.parse(file) - if sec == section: - maxnum = max(maxnum, num) - sec_num = (section, maxnum+1) - file = NEWFILENAME % sec_num - return self.entryclass(None, file, sec_num) - -class FaqWizard: - - def __init__(self): - self.ui = UserInput() - self.dir = FaqDir() - - def go(self): - print('Content-type: text/html') - req = self.ui.req or 'home' - mname = 'do_%s' % req - try: - meth = getattr(self, mname) - except AttributeError: - self.error("Bad request type %r." % (req,)) - else: - try: - meth() - except InvalidFile as exc: - self.error("Invalid entry file name %s" % exc.file) - except NoSuchFile as exc: - self.error("No entry with file name %s" % exc.file) - except NoSuchSection as exc: - self.error("No section number %s" % exc.section) - self.epilogue() - - def error(self, message, **kw): - self.prologue(T_ERROR) - emit(message, kw) - - def prologue(self, title, entry=None, **kw): - emit(PROLOGUE, entry, kwdict=kw, title=escape(title)) - - def epilogue(self): - emit(EPILOGUE) - - def do_home(self): - self.prologue(T_HOME) - emit(HOME) - - def do_debug(self): - self.prologue("FAQ Wizard Debugging") - form = cgi.FieldStorage() - cgi.print_form(form) - cgi.print_environ(os.environ) - cgi.print_directory() - cgi.print_arguments() - - def do_search(self): - query = self.ui.query - if not query: - self.error("Empty query string!") - return - if self.ui.querytype == 'simple': - query = re.escape(query) - queries = [query] - elif self.ui.querytype in ('anykeywords', 'allkeywords'): - words = [_f for _f in re.split('\W+', query) if _f] - if not words: - self.error("No keywords specified!") - return - words = [r'\b%s\b' % w for w in words] - if self.ui.querytype[:3] == 'any': - queries = ['|'.join(words)] - else: - # Each of the individual queries must match - queries = words - else: - # Default to regular expression - queries = [query] - self.prologue(T_SEARCH) - progs = [] - for query in queries: - if self.ui.casefold == 'no': - p = re.compile(query) - else: - p = re.compile(query, re.IGNORECASE) - progs.append(p) - hits = [] - for file in self.dir.list(): - try: - entry = self.dir.open(file) - except FileError: - constants - for p in progs: - if not p.search(entry.title) and not p.search(entry.body): - break - else: - hits.append(file) - if not hits: - emit(NO_HITS, self.ui, count=0) - elif len(hits) <= MAXHITS: - if len(hits) == 1: - emit(ONE_HIT, count=1) - else: - emit(FEW_HITS, count=len(hits)) - self.format_all(hits, headers=0) - else: - emit(MANY_HITS, count=len(hits)) - self.format_index(hits) - - def do_all(self): - self.prologue(T_ALL) - files = self.dir.list() - self.last_changed(files) - self.format_index(files, localrefs=1) - self.format_all(files) - - def do_compat(self): - files = self.dir.list() - emit(COMPAT) - self.last_changed(files) - self.format_index(files, localrefs=1) - self.format_all(files, edit=0) - sys.exit(0) # XXX Hack to suppress epilogue - - def last_changed(self, files): - latest = 0 - for file in files: - entry = self.dir.open(file) - if entry: - mtime = mtime = entry.getmtime() - if mtime > latest: - latest = mtime - print(time.strftime(LAST_CHANGED, time.localtime(latest))) - emit(EXPLAIN_MARKS) - - def format_all(self, files, edit=1, headers=1): - sec = 0 - for file in files: - try: - entry = self.dir.open(file) - except NoSuchFile: - continue - if headers and entry.sec != sec: - sec = entry.sec - try: - title = SECTION_TITLES[sec] - except KeyError: - title = "Untitled" - emit("\n<HR>\n<H1>%(sec)s. %(title)s</H1>\n", - sec=sec, title=title) - entry.show(edit=edit) - - def do_index(self): - self.prologue(T_INDEX) - files = self.dir.list() - self.last_changed(files) - self.format_index(files, add=1) - - def format_index(self, files, add=0, localrefs=0): - sec = 0 - for file in files: - try: - entry = self.dir.open(file) - except NoSuchFile: - continue - if entry.sec != sec: - if sec: - if add: - emit(INDEX_ADDSECTION, sec=sec) - emit(INDEX_ENDSECTION, sec=sec) - sec = entry.sec - try: - title = SECTION_TITLES[sec] - except KeyError: - title = "Untitled" - emit(INDEX_SECTION, sec=sec, title=title) - if localrefs: - emit(LOCAL_ENTRY, entry) - else: - emit(INDEX_ENTRY, entry) - entry.emit_marks() - if sec: - if add: - emit(INDEX_ADDSECTION, sec=sec) - emit(INDEX_ENDSECTION, sec=sec) - - def do_recent(self): - if not self.ui.days: - days = 1 - else: - days = float(self.ui.days) - try: - cutoff = now - days * 24 * 3600 - except OverflowError: - cutoff = 0 - list = [] - for file in self.dir.list(): - entry = self.dir.open(file) - if not entry: - continue - mtime = entry.getmtime() - if mtime >= cutoff: - list.append((mtime, file)) - list.sort() - list.reverse() - self.prologue(T_RECENT) - if days <= 1: - period = "%.2g hours" % (days*24) - else: - period = "%.6g days" % days - if not list: - emit(NO_RECENT, period=period) - elif len(list) == 1: - emit(ONE_RECENT, period=period) - else: - emit(SOME_RECENT, period=period, count=len(list)) - self.format_all([mtime_file[1] for mtime_file in list], headers=0) - emit(TAIL_RECENT) - - def do_roulette(self): - import random - files = self.dir.list() - if not files: - self.error("No entries.") - return - file = random.choice(files) - self.prologue(T_ROULETTE) - emit(ROULETTE) - self.dir.show(file) - - def do_help(self): - self.prologue(T_HELP) - emit(HELP) - - def do_show(self): - entry = self.dir.open(self.ui.file) - self.prologue(T_SHOW) - entry.show() - - def do_add(self): - self.prologue(T_ADD) - emit(ADD_HEAD) - sections = sorted(SECTION_TITLES.items()) - for section, title in sections: - emit(ADD_SECTION, section=section, title=title) - emit(ADD_TAIL) - - def do_delete(self): - self.prologue(T_DELETE) - emit(DELETE) - - def do_log(self): - entry = self.dir.open(self.ui.file) - self.prologue(T_LOG, entry) - emit(LOG, entry) - self.rlog(interpolate(SH_RLOG, entry), entry) - - def rlog(self, command, entry=None): - output = os.popen(command).read() - sys.stdout.write('<PRE>') - athead = 0 - lines = output.split('\n') - while lines and not lines[-1]: - del lines[-1] - if lines: - line = lines[-1] - if line[:1] == '=' and len(line) >= 40 and \ - line == line[0]*len(line): - del lines[-1] - headrev = None - for line in lines: - if entry and athead and line[:9] == 'revision ': - rev = line[9:].split() - mami = revparse(rev) - if not mami: - print(line) - else: - emit(REVISIONLINK, entry, rev=rev, line=line) - if mami[1] > 1: - prev = "%d.%d" % (mami[0], mami[1]-1) - emit(DIFFLINK, entry, prev=prev, rev=rev) - if headrev: - emit(DIFFLINK, entry, prev=rev, rev=headrev) - else: - headrev = rev - print() - athead = 0 - else: - athead = 0 - if line[:1] == '-' and len(line) >= 20 and \ - line == len(line) * line[0]: - athead = 1 - sys.stdout.write('<HR>') - else: - print(line) - print('</PRE>') - - def do_revision(self): - entry = self.dir.open(self.ui.file) - rev = self.ui.rev - mami = revparse(rev) - if not mami: - self.error("Invalid revision number: %r." % (rev,)) - self.prologue(T_REVISION, entry) - self.shell(interpolate(SH_REVISION, entry, rev=rev)) - - def do_diff(self): - entry = self.dir.open(self.ui.file) - prev = self.ui.prev - rev = self.ui.rev - mami = revparse(rev) - if not mami: - self.error("Invalid revision number: %r." % (rev,)) - if prev: - if not revparse(prev): - self.error("Invalid previous revision number: %r." % (prev,)) - else: - prev = '%d.%d' % (mami[0], mami[1]) - self.prologue(T_DIFF, entry) - self.shell(interpolate(SH_RDIFF, entry, rev=rev, prev=prev)) - - def shell(self, command): - output = os.popen(command).read() - sys.stdout.write('<PRE>') - print(escape(output)) - print('</PRE>') - - def do_new(self): - entry = self.dir.new(section=int(self.ui.section)) - entry.version = '*new*' - self.prologue(T_EDIT) - emit(EDITHEAD) - emit(EDITFORM1, entry, editversion=entry.version) - emit(EDITFORM2, entry, load_my_cookie()) - emit(EDITFORM3) - entry.show(edit=0) - - def do_edit(self): - entry = self.dir.open(self.ui.file) - entry.load_version() - self.prologue(T_EDIT) - emit(EDITHEAD) - emit(EDITFORM1, entry, editversion=entry.version) - emit(EDITFORM2, entry, load_my_cookie()) - emit(EDITFORM3) - entry.show(edit=0) - - def do_review(self): - send_my_cookie(self.ui) - if self.ui.editversion == '*new*': - sec, num = self.dir.parse(self.ui.file) - entry = self.dir.new(section=sec) - entry.version = "*new*" - if entry.file != self.ui.file: - self.error("Commit version conflict!") - emit(NEWCONFLICT, self.ui, sec=sec, num=num) - return - else: - entry = self.dir.open(self.ui.file) - entry.load_version() - # Check that the FAQ entry number didn't change - if self.ui.title.split()[:1] != entry.title.split()[:1]: - self.error("Don't change the entry number please!") - return - # Check that the edited version is the current version - if entry.version != self.ui.editversion: - self.error("Commit version conflict!") - emit(VERSIONCONFLICT, entry, self.ui) - return - commit_ok = ((not PASSWORD - or self.ui.password == PASSWORD) - and self.ui.author - and '@' in self.ui.email - and self.ui.log) - if self.ui.commit: - if not commit_ok: - self.cantcommit() - else: - self.commit(entry) - return - self.prologue(T_REVIEW) - emit(REVIEWHEAD) - entry.body = self.ui.body - entry.title = self.ui.title - entry.show(edit=0) - emit(EDITFORM1, self.ui, entry) - if commit_ok: - emit(COMMIT) - else: - emit(NOCOMMIT_HEAD) - self.errordetail() - emit(NOCOMMIT_TAIL) - emit(EDITFORM2, self.ui, entry, load_my_cookie()) - emit(EDITFORM3) - - def cantcommit(self): - self.prologue(T_CANTCOMMIT) - print(CANTCOMMIT_HEAD) - self.errordetail() - print(CANTCOMMIT_TAIL) - - def errordetail(self): - if PASSWORD and self.ui.password != PASSWORD: - emit(NEED_PASSWD) - if not self.ui.log: - emit(NEED_LOG) - if not self.ui.author: - emit(NEED_AUTHOR) - if not self.ui.email: - emit(NEED_EMAIL) - - def commit(self, entry): - file = entry.file - # Normalize line endings in body - if '\r' in self.ui.body: - self.ui.body = re.sub('\r\n?', '\n', self.ui.body) - # Normalize whitespace in title - self.ui.title = ' '.join(self.ui.title.split()) - # Check that there were any changes - if self.ui.body == entry.body and self.ui.title == entry.title: - self.error("You didn't make any changes!") - return - - # need to lock here because otherwise the file exists and is not writable (on NT) - command = interpolate(SH_LOCK, file=file) - p = os.popen(command) - output = p.read() - - try: - os.unlink(file) - except os.error: - pass - try: - f = open(file, 'w') - except IOError as why: - self.error(CANTWRITE, file=file, why=why) - return - date = time.ctime(now) - emit(FILEHEADER, self.ui, os.environ, date=date, _file=f, _quote=0) - f.write('\n') - f.write(self.ui.body) - f.write('\n') - f.close() - - import tempfile - tf = tempfile.NamedTemporaryFile() - emit(LOGHEADER, self.ui, os.environ, date=date, _file=tf) - tf.flush() - tf.seek(0) - - command = interpolate(SH_CHECKIN, file=file, tfn=tf.name) - log("\n\n" + command) - p = os.popen(command) - output = p.read() - sts = p.close() - log("output: " + output) - log("done: " + str(sts)) - log("TempFile:\n" + tf.read() + "end") - - if not sts: - self.prologue(T_COMMITTED) - emit(COMMITTED) - else: - self.error(T_COMMITFAILED) - emit(COMMITFAILED, sts=sts) - print('<PRE>%s</PRE>' % escape(output)) - - try: - os.unlink(tf.name) - except os.error: - pass - - entry = self.dir.open(file) - entry.show() - -wiz = FaqWizard() -wiz.go() diff --git a/Tools/faqwiz/move-faqwiz.sh b/Tools/faqwiz/move-faqwiz.sh deleted file mode 100755 index b3bcc92..0000000 --- a/Tools/faqwiz/move-faqwiz.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -# -# Christian Reis <kiko@async.com.br> -# -# Moves -# -# Example: -# -# blackjesus:~> ./move-faqwiz.sh 2\.1 3\.2 -# Moving FAQ question 02.001 to 03.002 - -if [ x$2 = x ]; then - echo "Need 2 args: original_version final_version." - exit 2 -fi - -if [ ! -d data -o ! -d data/RCS ]; then - echo "Run this inside the faqwiz data/ directory's parent dir." - exit 2 -fi - -cut_n_pad() { - t=`echo $1 | cut -d. -f $2` - export $3=`echo $t | awk "{ tmp = \\$0; l = length(tmp); for (i = 0; i < $2-l+1; i++) { tmp = "0".tmp } print tmp }"` -} - -cut_n_pad $1 1 prefix1 -cut_n_pad $1 2 suffix1 -cut_n_pad $2 1 prefix2 -cut_n_pad $2 2 suffix2 -if which tempfile >/dev/null; then - tmpfile=$(tempfile -d .) -elif [ -n "$RANDOM" ]; then - tmpfile=tmp$RANDOM.tmp -else - tmpfile=tmp$$.tmp -fi -file1=faq$prefix1.$suffix1.htp -file2=faq$prefix2.$suffix2.htp - -echo "Moving FAQ question $prefix1.$suffix1 to $prefix2.$suffix2" - -sed -e "s/$1\./$2\./g" data/$file1 > ${tmpfile}1 -sed -e "s/$1\./$2\./g" data/RCS/$file1,v > ${tmpfile}2 - -if [ -f data/$file2 ]; then - echo "Target FAQ exists. Won't clobber." - exit 2 -fi - -mv ${tmpfile}1 data/$file2 -mv ${tmpfile}2 data/RCS/$file2,v -mv data/$file1 data/$file1.orig -mv data/RCS/$file1,v data/RCS/$file1,v.orig - diff --git a/Tools/framer/README.txt b/Tools/framer/README.txt deleted file mode 100644 index 4a93a4d..0000000 --- a/Tools/framer/README.txt +++ /dev/null @@ -1,8 +0,0 @@ -framer is a tool to generate boilerplate code for C extension types. - -The boilerplate is generated from a specification object written in -Python. The specification uses the class statement to describe the -extension module and any extension types it contains. From the -specification, framer can generate all the boilerplate C code, -including function definitions, argument handling code, and type -objects. diff --git a/Tools/framer/TODO.txt b/Tools/framer/TODO.txt deleted file mode 100644 index 8586c8e..0000000 --- a/Tools/framer/TODO.txt +++ /dev/null @@ -1,6 +0,0 @@ -Add spec for getsets. -Generate a distutils setup script. -Handle operator overloading. -Generate traverse and clear methods for GC. -Handle mapping, sequence, buffer protocols. -Finish the todo list. diff --git a/Tools/framer/example.py b/Tools/framer/example.py deleted file mode 100644 index 8a267e9..0000000 --- a/Tools/framer/example.py +++ /dev/null @@ -1,126 +0,0 @@ -"""Generate the skeleton for cStringIO as an example of framer.""" - -from framer.bases import Module, Type -from framer.member import member - -class cStringIO(Module): - """A simple fast partial StringIO replacement. - - This module provides a simple useful replacement for the StringIO - module that is written in C. It does not provide the full - generality of StringIO, but it provides enough for most - applications and is especially useful in conjunction with the - pickle module. - - Usage: - - from io import StringIO - - an_output_stream = StringIO() - an_output_stream.write(some_stuff) - ... - value = an_output_stream.getvalue() - - an_input_stream = StringIO(a_string) - spam = an_input_stream.readline() - spam = an_input_stream.read(5) - an_input_stream.seek(0) # OK, start over - spam = an_input_stream.read() # and read it all - """ - - __file__ = "cStringIO.c" - - def StringIO(o): - """Return a StringIO-like stream for reading or writing""" - StringIO.pyarg = "|O" - - class InputType(Type): - "Simple type for treating strings as input file streams" - - abbrev = "input" - - struct = """\ - typedef struct { - PyObject_HEAD - char *buf; - int pos; - int size; - PyObject *pbuf; - } InputObject; - """ - - def flush(self): - """Does nothing""" - - def getvalue(self): - """Get the string value. - - If use_pos is specified and is a true value, then the - string returned will include only the text up to the - current file position. - """ - - def isatty(self): - """Always returns False""" - - def read(self, s): - """Return s characters or the rest of the string.""" - read.pyarg = "|i" - - def readline(self): - """Read one line.""" - - def readlines(self, hint): - """Read all lines.""" - readlines.pyarg = "|i" - - def reset(self): - """Reset the file position to the beginning.""" - - def tell(self): - """Get the current position.""" - - def truncate(self, pos): - """Truncate the file at the current position.""" - truncate.pyarg = "|i" - - def seek(self, position, mode=0): - """Set the current position. - - The optional mode argument can be 0 for absolute, 1 for relative, - and 2 for relative to EOF. The default is absolute. - """ - seek.pyarg = "i|i" - - def close(self): - pass - - class OutputType(InputType): - "Simple type for output strings." - - abbrev = "output" - - struct = """\ - typedef struct { - PyObject_HEAD - char *buf; - int pos; - int size; - int softspace; - } OutputObject; - """ - - softspace = member() - - def close(self): - """Explicitly release resources.""" - - def write(self, s): - """Write a string to the file.""" - # XXX Hack: writing None resets the buffer - - def writelines(self, lines): - """Write each string in lines.""" - - -cStringIO.gen() diff --git a/Tools/framer/framer/__init__.py b/Tools/framer/framer/__init__.py deleted file mode 100644 index d8f9058..0000000 --- a/Tools/framer/framer/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -"""A tool to generate basic framework for C extension types. - -The basic ideas is the same as modulator, but the code generates code -using many of the new features introduced in Python 2.2. It also -takes a more declarative approach to generating code. -""" diff --git a/Tools/framer/framer/bases.py b/Tools/framer/framer/bases.py deleted file mode 100644 index 08aacb9..0000000 --- a/Tools/framer/framer/bases.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Provides the Module and Type base classes that user code inherits from.""" - -__all__ = ["Module", "Type", "member"] - -from framer import struct, template -from framer.function import Function, Method -from framer.member import member -from framer.slots import * -from framer.util import cstring, unindent - -from types import FunctionType - -# The Module and Type classes are implemented using metaclasses, -# because most of the methods are class methods. It is easier to use -# metaclasses than the cumbersome classmethod() builtin. They have -# class methods because they are exposed to user code as base classes. - -class BaseMetaclass(type): - """Shared infrastructure for generating modules and types.""" - - # just methoddef so far - - def dump_methoddef(self, f, functions, vars): - def p(templ, vars=vars): # helper function to generate output - print(templ % vars, file=f) - - if not functions: - return - p(template.methoddef_start) - for name, func in sorted(functions.items()): - if func.__doc__: - p(template.methoddef_def_doc, func.vars) - else: - p(template.methoddef_def, func.vars) - p(template.methoddef_end) - -class ModuleMetaclass(BaseMetaclass): - """Provides methods for Module class.""" - - def gen(self): - self.analyze() - self.initvars() - f = open(self.__filename, "w") - self.dump(f) - f.close() - - def analyze(self): - self.name = getattr(self, "abbrev", self.__name__) - self.__functions = {} - self.__types = {} - self.__members = False - - for name, obj in self.__dict__.items(): - if isinstance(obj, FunctionType): - self.__functions[name] = Function(obj, self) - elif isinstance(obj, TypeMetaclass): - obj._TypeMetaclass__module = self.name - obj.analyze() - self.__types[name] = obj - if obj.has_members(): - self.__members = True - - def initvars(self): - v = self.__vars = {} - filename = getattr(self, "__file__", None) - if filename is None: - filename = self.__name__ + "module.c" - self.__filename = v["FileName"] = filename - name = v["ModuleName"] = self.__name__ - v["MethodDefName"] = "%s_methods" % name - v["ModuleDocstring"] = cstring(unindent(self.__doc__)) - - def dump(self, f): - def p(templ, vars=self.__vars): # helper function to generate output - print(templ % vars, file=f) - - p(template.module_start) - if self.__members: - p(template.member_include) - print(file=f) - - if self.__doc__: - p(template.module_doc) - - for name, type in sorted(self.__types.items()): - type.dump(f) - - for name, func in sorted(self.__functions.items()): - func.dump(f) - - self.dump_methoddef(f, self.__functions, self.__vars) - - p(template.module_init_start) - for name, type in sorted(self.__types.items()): - type.dump_init(f) - - p("}") - -class Module(metaclass=ModuleMetaclass): - pass - -class TypeMetaclass(BaseMetaclass): - - def dump(self, f): - self.initvars() - - # defined after initvars() so that __vars is defined - def p(templ, vars=self.__vars): - print(templ % vars, file=f) - - if self.struct is not None: - print(unindent(self.struct, False), file=f) - - if self.__doc__: - p(template.docstring) - - for name, func in sorted(self.__methods.items()): - func.dump(f) - - self.dump_methoddef(f, self.__methods, self.__vars) - self.dump_memberdef(f) - self.dump_slots(f) - - def has_members(self): - if self.__members: - return True - else: - return False - - def analyze(self): - # called by ModuleMetaclass analyze() - self.name = getattr(self, "abbrev", self.__name__) - src = getattr(self, "struct", None) - if src is not None: - self.__struct = struct.parse(src) - else: - self.__struct = None - self.__methods = {} - self.__members = {} - for cls in self.__mro__: - for k, v in cls.__dict__.items(): - if isinstance(v, FunctionType): - self.__methods[k] = Method(v, self) - if isinstance(v, member): - self.__members[k] = v - assert self.__struct is not None - v.register(k, self.__struct) - self.analyze_slots() - - def analyze_slots(self): - self.__slots = {} - for s in Slots: - if s.special is not None: - meth = self.__methods.get(s.special) - if meth is not None: - self.__slots[s] = meth - self.__slots[TP_NAME] = '"%s.%s"' % (self.__module, self.__name__) - if self.__doc__: - self.__slots[TP_DOC] = "%s_doc" % self.name - if self.__struct is not None: - self.__slots[TP_BASICSIZE] = "sizeof(%s)" % self.__struct.name - self.__slots[TP_DEALLOC] = "%s_dealloc" % self.name - if self.__methods: - self.__slots[TP_METHODS] = "%s_methods" % self.name - if self.__members: - self.__slots[TP_MEMBERS] = "%s_members" % self.name - - def initvars(self): - v = self.__vars = {} - v["TypeName"] = self.__name__ - v["CTypeName"] = "Py%s_Type" % self.__name__ - v["MethodDefName"] = self.__slots[TP_METHODS] - if self.__doc__: - v["DocstringVar"] = self.__slots[TP_DOC] - v["Docstring"] = cstring(unindent(self.__doc__)) - if self.__struct is not None: - v["StructName"] = self.__struct.name - if self.__members: - v["MemberDefName"] = self.__slots[TP_MEMBERS] - - def dump_memberdef(self, f): - def p(templ, vars=self.__vars): - print(templ % vars, file=f) - - if not self.__members: - return - p(template.memberdef_start) - for name, slot in sorted(self.__members.items()): - slot.dump(f) - p(template.memberdef_end) - - def dump_slots(self, f): - def p(templ, vars=self.__vars): - print(templ % vars, file=f) - - if self.struct: - p(template.dealloc_func, {"name" : self.__slots[TP_DEALLOC]}) - - p(template.type_struct_start) - for s in Slots[:-5]: # XXX - val = self.__slots.get(s, s.default) - ntabs = 4 - (4 + len(val)) / 8 - line = " %s,%s/* %s */" % (val, "\t" * ntabs, s.name) - print(line, file=f) - p(template.type_struct_end) - - def dump_init(self, f): - def p(templ): - print(templ % self.__vars, file=f) - - p(template.type_init_type) - p(template.module_add_type) - -class Type(metaclass=TypeMetaclass): - pass diff --git a/Tools/framer/framer/function.py b/Tools/framer/framer/function.py deleted file mode 100644 index f95ea20..0000000 --- a/Tools/framer/framer/function.py +++ /dev/null @@ -1,173 +0,0 @@ -"""Functions.""" - -from framer import template -from framer.util import cstring, unindent - -METH_O = "METH_O" -METH_NOARGS = "METH_NOARGS" -METH_VARARGS = "METH_VARARGS" - -def parsefmt(fmt): - for c in fmt: - if c == '|': - continue - yield c - -class Argument: - - def __init__(self, name): - self.name = name - self.ctype = "PyObject *" - self.default = None - - def __str__(self): - return "%s%s" % (self.ctype, self.name) - - def setfmt(self, code): - self.ctype = self._codes[code] - if self.ctype[-1] != "*": - self.ctype += " " - - _codes = {"O": "PyObject *", - "i": "int", - } - - def decl(self): - if self.default is None: - return str(self) + ";" - else: - return "%s = %s;" % (self, self.default) - -class _ArgumentList(object): - - # these instance variables should be initialized by subclasses - ml_meth = None - fmt = None - - def __init__(self, args): - self.args = list(map(Argument, args)) - - def __len__(self): - return len(self.args) - - def __getitem__(self, i): - return self.args[i] - - def dump_decls(self, f): - pass - -class NoArgs(_ArgumentList): - - def __init__(self, args): - assert len(args) == 0 - super(NoArgs, self).__init__(args) - self.ml_meth = METH_NOARGS - - def c_args(self): - return "PyObject *self" - -class OneArg(_ArgumentList): - - def __init__(self, args): - assert len(args) == 1 - super(OneArg, self).__init__(args) - self.ml_meth = METH_O - - def c_args(self): - return "PyObject *self, %s" % self.args[0] - -class VarArgs(_ArgumentList): - - def __init__(self, args, fmt=None): - super(VarArgs, self).__init__(args) - self.ml_meth = METH_VARARGS - if fmt is not None: - self.fmt = fmt - i = 0 - for code in parsefmt(fmt): - self.args[i].setfmt(code) - i += 1 - - def c_args(self): - return "PyObject *self, PyObject *args" - - def targets(self): - return ", ".join(["&%s" % a.name for a in self.args]) - - def dump_decls(self, f): - for a in self.args: - print(" %s" % a.decl(), file=f) - -def ArgumentList(func, method): - code = func.__code__ - args = code.co_varnames[:code.co_argcount] - if method: - args = args[1:] - pyarg = getattr(func, "pyarg", None) - if pyarg is not None: - args = VarArgs(args, pyarg) - if func.__defaults__: - L = list(func.__defaults__) - ndefault = len(L) - i = len(args) - ndefault - while L: - args[i].default = L.pop(0) - return args - else: - if len(args) == 0: - return NoArgs(args) - elif len(args) == 1: - return OneArg(args) - else: - return VarArgs(args) - -class Function: - - method = False - - def __init__(self, func, parent): - self._func = func - self._parent = parent - self.analyze() - self.initvars() - - def dump(self, f): - def p(templ, vars=None): # helper function to generate output - if vars is None: - vars = self.vars - print(templ % vars, file=f) - - if self.__doc__: - p(template.docstring) - - d = {"name" : self.vars["CName"], - "args" : self.args.c_args(), - } - p(template.funcdef_start, d) - - self.args.dump_decls(f) - - if self.args.ml_meth == METH_VARARGS: - p(template.varargs) - - p(template.funcdef_end) - - def analyze(self): - self.__doc__ = self._func.__doc__ - self.args = ArgumentList(self._func, self.method) - - def initvars(self): - v = self.vars = {} - v["PythonName"] = self._func.__name__ - s = v["CName"] = "%s_%s" % (self._parent.name, self._func.__name__) - v["DocstringVar"] = s + "_doc" - v["MethType"] = self.args.ml_meth - if self.__doc__: - v["Docstring"] = cstring(unindent(self.__doc__)) - if self.args.fmt is not None: - v["ArgParse"] = self.args.fmt - v["ArgTargets"] = self.args.targets() - -class Method(Function): - - method = True diff --git a/Tools/framer/framer/member.py b/Tools/framer/framer/member.py deleted file mode 100644 index 4838c1f..0000000 --- a/Tools/framer/framer/member.py +++ /dev/null @@ -1,73 +0,0 @@ -from framer import template -from framer.util import cstring, unindent - -T_SHORT = "T_SHORT" -T_INT = "T_INT" -T_LONG = "T_LONG" -T_FLOAT = "T_FLOAT" -T_DOUBLE = "T_DOUBLE" -T_STRING = "T_STRING" -T_OBJECT = "T_OBJECT" -T_CHAR = "T_CHAR" -T_BYTE = "T_BYTE" -T_UBYTE = "T_UBYTE" -T_UINT = "T_UINT" -T_ULONG = "T_ULONG" -T_STRING_INPLACE = "T_STRING_INPLACE" -T_OBJECT_EX = "T_OBJECT_EX" - -RO = READONLY = "READONLY" -READ_RESTRICTED = "READ_RESTRICTED" -WRITE_RESTRICTED = "WRITE_RESTRICTED" -RESTRICT = "RESTRICTED" - -c2t = {"int" : T_INT, - "unsigned int" : T_UINT, - "long" : T_LONG, - "unsigned long" : T_LONG, - "float" : T_FLOAT, - "double" : T_DOUBLE, - "char *" : T_CHAR, - "PyObject *" : T_OBJECT, - } - -class member(object): - - def __init__(self, cname=None, type=None, flags=None, doc=None): - self.type = type - self.flags = flags - self.cname = cname - self.doc = doc - self.name = None - self.struct = None - - def register(self, name, struct): - self.name = name - self.struct = struct - self.initvars() - - def initvars(self): - v = self.vars = {} - v["PythonName"] = self.name - if self.cname is not None: - v["CName"] = self.cname - else: - v["CName"] = self.name - v["Flags"] = self.flags or "0" - v["Type"] = self.get_type() - if self.doc is not None: - v["Docstring"] = cstring(unindent(self.doc)) - v["StructName"] = self.struct.name - - def get_type(self): - """Deduce type code from struct specification if not defined""" - if self.type is not None: - return self.type - ctype = self.struct.get_type(self.name) - return c2t[ctype] - - def dump(self, f): - if self.doc is None: - print(template.memberdef_def % self.vars, file=f) - else: - print(template.memberdef_def_doc % self.vars, file=f) diff --git a/Tools/framer/framer/slots.py b/Tools/framer/framer/slots.py deleted file mode 100644 index 0f6e6fe..0000000 --- a/Tools/framer/framer/slots.py +++ /dev/null @@ -1,64 +0,0 @@ -"""Descriptions of all the slots in Python's type objects.""" - -class Slot(object): - def __init__(self, name, cast=None, special=None, default="0"): - self.name = name - self.cast = cast - self.special = special - self.default = default - -Slots = (Slot("ob_size"), - Slot("tp_name"), - Slot("tp_basicsize"), - Slot("tp_itemsize"), - Slot("tp_dealloc", "destructor"), - Slot("tp_print", "printfunc"), - Slot("tp_getattr", "getattrfunc"), - Slot("tp_setattr", "setattrfunc"), - Slot("tp_reserved", "void*"), - Slot("tp_repr", "reprfunc", "__repr__"), - Slot("tp_as_number"), - Slot("tp_as_sequence"), - Slot("tp_as_mapping"), - Slot("tp_hash", "hashfunc", "__hash__"), - Slot("tp_call", "ternaryfunc", "__call__"), - Slot("tp_str", "reprfunc", "__str__"), - Slot("tp_getattro", "getattrofunc", "__getattr__", # XXX - "PyObject_GenericGetAttr"), - Slot("tp_setattro", "setattrofunc", "__setattr__"), - Slot("tp_as_buffer"), - Slot("tp_flags", default="Py_TPFLAGS_DEFAULT"), - Slot("tp_doc"), - Slot("tp_traverse", "traverseprox"), - Slot("tp_clear", "inquiry"), - Slot("tp_richcompare", "richcmpfunc"), - Slot("tp_weaklistoffset"), - Slot("tp_iter", "getiterfunc", "__iter__"), - Slot("tp_iternext", "iternextfunc", "__next__"), # XXX - Slot("tp_methods"), - Slot("tp_members"), - Slot("tp_getset"), - Slot("tp_base"), - Slot("tp_dict"), - Slot("tp_descr_get", "descrgetfunc"), - Slot("tp_descr_set", "descrsetfunc"), - Slot("tp_dictoffset"), - Slot("tp_init", "initproc", "__init__"), - Slot("tp_alloc", "allocfunc"), - Slot("tp_new", "newfunc"), - Slot("tp_free", "freefunc"), - Slot("tp_is_gc", "inquiry"), - Slot("tp_bases"), - Slot("tp_mro"), - Slot("tp_cache"), - Slot("tp_subclasses"), - Slot("tp_weaklist"), - ) - -# give some slots symbolic names -TP_NAME = Slots[1] -TP_BASICSIZE = Slots[2] -TP_DEALLOC = Slots[4] -TP_DOC = Slots[20] -TP_METHODS = Slots[27] -TP_MEMBERS = Slots[28] diff --git a/Tools/framer/framer/struct.py b/Tools/framer/framer/struct.py deleted file mode 100644 index 8fc2aec..0000000 --- a/Tools/framer/framer/struct.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Rudimentary parser for C struct definitions.""" - -import re - -PyObject_HEAD = "PyObject_HEAD" -PyObject_VAR_HEAD = "PyObject_VAR_HEAD" - -rx_name = re.compile("} (\w+);") - -class Struct: - def __init__(self, name, head, members): - self.name = name - self.head = head - self.members = members - - def get_type(self, name): - for _name, type in self.members: - if name == _name: - return type - raise ValueError("no member named %s" % name) - -def parse(s): - """Parse a C struct definition. - - The parser is very restricted in what it will accept. - """ - - lines = [_f for _f in s.split("\n") if _f] # get non-empty lines - assert lines[0].strip() == "typedef struct {" - pyhead = lines[1].strip() - assert (pyhead.startswith("PyObject") and - pyhead.endswith("HEAD")) - members = [] - for line in lines[2:]: - line = line.strip() - if line.startswith("}"): - break - - assert line.endswith(";") - line = line[:-1] - words = line.split() - name = words[-1] - type = " ".join(words[:-1]) - if name[0] == "*": - name = name[1:] - type += " *" - members.append((name, type)) - name = None - mo = rx_name.search(line) - assert mo is not None - name = mo.group(1) - return Struct(name, pyhead, members) diff --git a/Tools/framer/framer/structparse.py b/Tools/framer/framer/structparse.py deleted file mode 100644 index cf4224b..0000000 --- a/Tools/framer/framer/structparse.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Rudimentary parser for C struct definitions.""" - -import re - -PyObject_HEAD = "PyObject_HEAD" -PyObject_VAR_HEAD = "PyObject_VAR_HEAD" - -rx_name = re.compile("} (\w+);") - -class Struct: - def __init__(self, name, head, members): - self.name = name - self.head = head - self.members = members - -def parse(s): - """Parse a C struct definition. - - The parser is very restricted in what it will accept. - """ - - lines = [_f for _f in s.split("\n") if _f] # get non-empty lines - assert lines[0].strip() == "typedef struct {" - pyhead = lines[1].strip() - assert (pyhead.startswith("PyObject") and - pyhead.endswith("HEAD")) - members = [] - for line in lines[2:]: - line = line.strip() - if line.startswith("}"): - break - - assert line.endswith(";") - line = line[:-1] - words = line.split() - name = words[-1] - type = " ".join(words[:-1]) - if name[0] == "*": - name = name[1:] - type += " *" - members.append((name, type)) - name = None - mo = rx_name.search(line) - assert mo is not None - name = mo.group(1) - return Struct(name, pyhead, members) diff --git a/Tools/framer/framer/template.py b/Tools/framer/framer/template.py deleted file mode 100644 index 41f9537..0000000 --- a/Tools/framer/framer/template.py +++ /dev/null @@ -1,102 +0,0 @@ -"""framer's C code templates. - -Templates use the following variables: - -FileName: name of the file that contains the C source code -ModuleName: name of the module, as in "import ModuleName" -ModuleDocstring: C string containing the module doc string -""" - -module_start = '#include "Python.h"' -member_include = '#include "structmember.h"' - -module_doc = """\ -PyDoc_STRVAR(%(ModuleName)s_doc, -%(ModuleDocstring)s); -""" - -methoddef_start = """\ -static struct PyMethodDef %(MethodDefName)s[] = {""" - -methoddef_def = """\ - {"%(PythonName)s", (PyCFunction)%(CName)s, %(MethType)s},""" - -methoddef_def_doc = """\ - {"%(PythonName)s", (PyCFunction)%(CName)s, %(MethType)s, - %(DocstringVar)s},""" - -methoddef_end = """\ - {NULL, NULL} -}; -""" - -memberdef_start = """\ -#define OFF(X) offsetof(%(StructName)s, X) - -static struct PyMemberDef %(MemberDefName)s[] = {""" - -memberdef_def_doc = """\ - {"%(PythonName)s", %(Type)s, OFF(%(CName)s), %(Flags)s, - %(Docstring)s},""" - -memberdef_def = """\ - {"%(PythonName)s", %(Type)s, OFF(%(CName)s), %(Flags)s},""" - -memberdef_end = """\ - {NULL} -}; - -#undef OFF -""" - -dealloc_func = """static void -%(name)s(PyObject *ob) -{ -} -""" - -docstring = """\ -PyDoc_STRVAR(%(DocstringVar)s, -%(Docstring)s); -""" - -funcdef_start = """\ -static PyObject * -%(name)s(%(args)s) -{""" - -funcdef_end = """\ -} -""" - -varargs = """\ - if (!PyArg_ParseTuple(args, \"%(ArgParse)s:%(PythonName)s\", - %(ArgTargets)s)) - return NULL;""" - -module_init_start = """\ -PyMODINIT_FUNC -init%(ModuleName)s(void) -{ - PyObject *mod; - - mod = Py_InitModule3("%(ModuleName)s", %(MethodDefName)s, - %(ModuleName)s_doc); - if (mod == NULL) - return; -""" - -type_init_type = " %(CTypeName)s.ob_type = &PyType_Type;" -module_add_type = """\ - if (!PyObject_SetAttrString(mod, "%(TypeName)s", - (PyObject *)&%(CTypeName)s)) - return; -""" - -type_struct_start = """\ -static PyTypeObject %(CTypeName)s = { - PyObject_HEAD_INIT(0)""" - -type_struct_end = """\ -}; -""" diff --git a/Tools/framer/framer/util.py b/Tools/framer/framer/util.py deleted file mode 100644 index 73f3309..0000000 --- a/Tools/framer/framer/util.py +++ /dev/null @@ -1,35 +0,0 @@ -def cstring(s, width=70): - """Return C string representation of a Python string. - - width specifies the maximum width of any line of the C string. - """ - L = [] - for l in s.split("\n"): - if len(l) < width: - L.append(r'"%s\n"' % l) - - return "\n".join(L) - -def unindent(s, skipfirst=True): - """Return an unindented version of a docstring. - - Removes indentation on lines following the first one, using the - leading whitespace of the first indented line that is not blank - to determine the indentation. - """ - - lines = s.split("\n") - if skipfirst: - first = lines.pop(0) - L = [first] - else: - L = [] - indent = None - for l in lines: - ls = l.strip() - if ls: - indent = len(l) - len(ls) - break - L += [l[indent:] for l in lines] - - return "\n".join(L) diff --git a/Tools/freeze/freeze.py b/Tools/freeze/freeze.py index a7673c9..a41267a 100755 --- a/Tools/freeze/freeze.py +++ b/Tools/freeze/freeze.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """Freeze a Python script into a binary. @@ -201,7 +201,7 @@ def main(): # modules that are imported by the Python runtime implicits = [] - for module in ('site', 'warnings',): + for module in ('site', 'warnings', 'encodings.utf_8', 'encodings.latin_1'): if module not in exclude: implicits.append(module) diff --git a/Tools/freeze/makeconfig.py b/Tools/freeze/makeconfig.py index 2aab3d9..018992c 100644 --- a/Tools/freeze/makeconfig.py +++ b/Tools/freeze/makeconfig.py @@ -3,14 +3,13 @@ import sys # Write the config.c file -never = ['marshal', '__main__', 'builtins', 'sys', 'exceptions', '_warnings'] +never = ['marshal', 'imp', '_ast', '__main__', 'builtins', + 'sys', 'gc', '_warnings'] def makeconfig(infp, outfp, modules, with_ifdef=0): m1 = re.compile('-- ADDMODULE MARKER 1 --') m2 = re.compile('-- ADDMODULE MARKER 2 --') - while 1: - line = infp.readline() - if not line: break + for line in infp: outfp.write(line) if m1 and m1.search(line): m1 = None @@ -18,8 +17,8 @@ def makeconfig(infp, outfp, modules, with_ifdef=0): if mod in never: continue if with_ifdef: - outfp.write("#ifndef init%s\n"%mod) - outfp.write('extern void init%s(void);\n' % mod) + outfp.write("#ifndef PyInit_%s\n"%mod) + outfp.write('extern PyObject* PyInit_%s(void);\n' % mod) if with_ifdef: outfp.write("#endif\n") elif m2 and m2.search(line): @@ -27,7 +26,7 @@ def makeconfig(infp, outfp, modules, with_ifdef=0): for mod in modules: if mod in never: continue - outfp.write('\t{"%s", init%s},\n' % + outfp.write('\t{"%s", PyInit_%s},\n' % (mod, mod)) if m1: sys.stderr.write('MARKER 1 never found\n') diff --git a/Tools/gdb/libpython.py b/Tools/gdb/libpython.py new file mode 100644 index 0000000..93b0528 --- /dev/null +++ b/Tools/gdb/libpython.py @@ -0,0 +1,1633 @@ +#!/usr/bin/python +''' +From gdb 7 onwards, gdb's build can be configured --with-python, allowing gdb +to be extended with Python code e.g. for library-specific data visualizations, +such as for the C++ STL types. Documentation on this API can be seen at: +http://sourceware.org/gdb/current/onlinedocs/gdb/Python-API.html + + +This python module deals with the case when the process being debugged (the +"inferior process" in gdb parlance) is itself python, or more specifically, +linked against libpython. In this situation, almost every item of data is a +(PyObject*), and having the debugger merely print their addresses is not very +enlightening. + +This module embeds knowledge about the implementation details of libpython so +that we can emit useful visualizations e.g. a string, a list, a dict, a frame +giving file/line information and the state of local variables + +In particular, given a gdb.Value corresponding to a PyObject* in the inferior +process, we can generate a "proxy value" within the gdb process. For example, +given a PyObject* in the inferior process that is in fact a PyListObject* +holding three PyObject* that turn out to be PyBytesObject* instances, we can +generate a proxy value within the gdb process that is a list of bytes +instances: + [b"foo", b"bar", b"baz"] + +Doing so can be expensive for complicated graphs of objects, and could take +some time, so we also have a "write_repr" method that writes a representation +of the data to a file-like object. This allows us to stop the traversal by +having the file-like object raise an exception if it gets too much data. + +With both "proxyval" and "write_repr" we keep track of the set of all addresses +visited so far in the traversal, to avoid infinite recursion due to cycles in +the graph of object references. + +We try to defer gdb.lookup_type() invocations for python types until as late as +possible: for a dynamically linked python binary, when the process starts in +the debugger, the libpython.so hasn't been dynamically loaded yet, so none of +the type names are known to the debugger + +The module also extends gdb with some python-specific commands. +''' +from __future__ import with_statement +import gdb +import locale +import sys + +# Look up the gdb.Type for some standard types: +_type_char_ptr = gdb.lookup_type('char').pointer() # char* +_type_unsigned_char_ptr = gdb.lookup_type('unsigned char').pointer() # unsigned char* +_type_void_ptr = gdb.lookup_type('void').pointer() # void* +_type_size_t = gdb.lookup_type('size_t') + +SIZEOF_VOID_P = _type_void_ptr.sizeof + + +Py_TPFLAGS_HEAPTYPE = (1L << 9) + +Py_TPFLAGS_LONG_SUBCLASS = (1L << 24) +Py_TPFLAGS_LIST_SUBCLASS = (1L << 25) +Py_TPFLAGS_TUPLE_SUBCLASS = (1L << 26) +Py_TPFLAGS_BYTES_SUBCLASS = (1L << 27) +Py_TPFLAGS_UNICODE_SUBCLASS = (1L << 28) +Py_TPFLAGS_DICT_SUBCLASS = (1L << 29) +Py_TPFLAGS_BASE_EXC_SUBCLASS = (1L << 30) +Py_TPFLAGS_TYPE_SUBCLASS = (1L << 31) + + +MAX_OUTPUT_LEN=1024 + +hexdigits = "0123456789abcdef" + +ENCODING = locale.getpreferredencoding() + +class NullPyObjectPtr(RuntimeError): + pass + + +def safety_limit(val): + # Given a integer value from the process being debugged, limit it to some + # safety threshold so that arbitrary breakage within said process doesn't + # break the gdb process too much (e.g. sizes of iterations, sizes of lists) + return min(val, 1000) + + +def safe_range(val): + # As per range, but don't trust the value too much: cap it to a safety + # threshold in case the data was corrupted + return xrange(safety_limit(val)) + +def write_unicode(file, text): + # Write a byte or unicode string to file. Unicode strings are encoded to + # ENCODING encoding with 'backslashreplace' error handler to avoid + # UnicodeEncodeError. + if isinstance(text, unicode): + text = text.encode(ENCODING, 'backslashreplace') + file.write(text) + +def os_fsencode(filename): + if not isinstance(filename, unicode): + return filename + encoding = sys.getfilesystemencoding() + if encoding == 'mbcs': + # mbcs doesn't support surrogateescape + return filename.encode(encoding) + encoded = [] + for char in filename: + # surrogateescape error handler + if 0xDC80 <= ord(char) <= 0xDCFF: + byte = chr(ord(char) - 0xDC00) + else: + byte = char.encode(encoding) + encoded.append(byte) + return ''.join(encoded) + +class StringTruncated(RuntimeError): + pass + +class TruncatedStringIO(object): + '''Similar to cStringIO, but can truncate the output by raising a + StringTruncated exception''' + def __init__(self, maxlen=None): + self._val = '' + self.maxlen = maxlen + + def write(self, data): + if self.maxlen: + if len(data) + len(self._val) > self.maxlen: + # Truncation: + self._val += data[0:self.maxlen - len(self._val)] + raise StringTruncated() + + self._val += data + + def getvalue(self): + return self._val + +class PyObjectPtr(object): + """ + Class wrapping a gdb.Value that's a either a (PyObject*) within the + inferior process, or some subclass pointer e.g. (PyBytesObject*) + + There will be a subclass for every refined PyObject type that we care + about. + + Note that at every stage the underlying pointer could be NULL, point + to corrupt data, etc; this is the debugger, after all. + """ + _typename = 'PyObject' + + def __init__(self, gdbval, cast_to=None): + if cast_to: + self._gdbval = gdbval.cast(cast_to) + else: + self._gdbval = gdbval + + def field(self, name): + ''' + Get the gdb.Value for the given field within the PyObject, coping with + some python 2 versus python 3 differences. + + Various libpython types are defined using the "PyObject_HEAD" and + "PyObject_VAR_HEAD" macros. + + In Python 2, this these are defined so that "ob_type" and (for a var + object) "ob_size" are fields of the type in question. + + In Python 3, this is defined as an embedded PyVarObject type thus: + PyVarObject ob_base; + so that the "ob_size" field is located insize the "ob_base" field, and + the "ob_type" is most easily accessed by casting back to a (PyObject*). + ''' + if self.is_null(): + raise NullPyObjectPtr(self) + + if name == 'ob_type': + pyo_ptr = self._gdbval.cast(PyObjectPtr.get_gdb_type()) + return pyo_ptr.dereference()[name] + + if name == 'ob_size': + pyo_ptr = self._gdbval.cast(PyVarObjectPtr.get_gdb_type()) + return pyo_ptr.dereference()[name] + + # General case: look it up inside the object: + return self._gdbval.dereference()[name] + + def pyop_field(self, name): + ''' + Get a PyObjectPtr for the given PyObject* field within this PyObject, + coping with some python 2 versus python 3 differences. + ''' + return PyObjectPtr.from_pyobject_ptr(self.field(name)) + + def write_field_repr(self, name, out, visited): + ''' + Extract the PyObject* field named "name", and write its representation + to file-like object "out" + ''' + field_obj = self.pyop_field(name) + field_obj.write_repr(out, visited) + + def get_truncated_repr(self, maxlen): + ''' + Get a repr-like string for the data, but truncate it at "maxlen" bytes + (ending the object graph traversal as soon as you do) + ''' + out = TruncatedStringIO(maxlen) + try: + self.write_repr(out, set()) + except StringTruncated: + # Truncation occurred: + return out.getvalue() + '...(truncated)' + + # No truncation occurred: + return out.getvalue() + + def type(self): + return PyTypeObjectPtr(self.field('ob_type')) + + def is_null(self): + return 0 == long(self._gdbval) + + def is_optimized_out(self): + ''' + Is the value of the underlying PyObject* visible to the debugger? + + This can vary with the precise version of the compiler used to build + Python, and the precise version of gdb. + + See e.g. https://bugzilla.redhat.com/show_bug.cgi?id=556975 with + PyEval_EvalFrameEx's "f" + ''' + return self._gdbval.is_optimized_out + + def safe_tp_name(self): + try: + return self.type().field('tp_name').string() + except NullPyObjectPtr: + # NULL tp_name? + return 'unknown' + except RuntimeError: + # Can't even read the object at all? + return 'unknown' + + def proxyval(self, visited): + ''' + Scrape a value from the inferior process, and try to represent it + within the gdb process, whilst (hopefully) avoiding crashes when + the remote data is corrupt. + + Derived classes will override this. + + For example, a PyIntObject* with ob_ival 42 in the inferior process + should result in an int(42) in this process. + + visited: a set of all gdb.Value pyobject pointers already visited + whilst generating this value (to guard against infinite recursion when + visiting object graphs with loops). Analogous to Py_ReprEnter and + Py_ReprLeave + ''' + + class FakeRepr(object): + """ + Class representing a non-descript PyObject* value in the inferior + process for when we don't have a custom scraper, intended to have + a sane repr(). + """ + + def __init__(self, tp_name, address): + self.tp_name = tp_name + self.address = address + + def __repr__(self): + # For the NULL pointer, we have no way of knowing a type, so + # special-case it as per + # http://bugs.python.org/issue8032#msg100882 + if self.address == 0: + return '0x0' + return '<%s at remote 0x%x>' % (self.tp_name, self.address) + + return FakeRepr(self.safe_tp_name(), + long(self._gdbval)) + + def write_repr(self, out, visited): + ''' + Write a string representation of the value scraped from the inferior + process to "out", a file-like object. + ''' + # Default implementation: generate a proxy value and write its repr + # However, this could involve a lot of work for complicated objects, + # so for derived classes we specialize this + return out.write(repr(self.proxyval(visited))) + + @classmethod + def subclass_from_type(cls, t): + ''' + Given a PyTypeObjectPtr instance wrapping a gdb.Value that's a + (PyTypeObject*), determine the corresponding subclass of PyObjectPtr + to use + + Ideally, we would look up the symbols for the global types, but that + isn't working yet: + (gdb) python print gdb.lookup_symbol('PyList_Type')[0].value + Traceback (most recent call last): + File "<string>", line 1, in <module> + NotImplementedError: Symbol type not yet supported in Python scripts. + Error while executing Python code. + + For now, we use tp_flags, after doing some string comparisons on the + tp_name for some special-cases that don't seem to be visible through + flags + ''' + try: + tp_name = t.field('tp_name').string() + tp_flags = int(t.field('tp_flags')) + except RuntimeError: + # Handle any kind of error e.g. NULL ptrs by simply using the base + # class + return cls + + #print 'tp_flags = 0x%08x' % tp_flags + #print 'tp_name = %r' % tp_name + + name_map = {'bool': PyBoolObjectPtr, + 'classobj': PyClassObjectPtr, + 'instance': PyInstanceObjectPtr, + 'NoneType': PyNoneStructPtr, + 'frame': PyFrameObjectPtr, + 'set' : PySetObjectPtr, + 'frozenset' : PySetObjectPtr, + 'builtin_function_or_method' : PyCFunctionObjectPtr, + } + if tp_name in name_map: + return name_map[tp_name] + + if tp_flags & Py_TPFLAGS_HEAPTYPE: + return HeapTypeObjectPtr + + if tp_flags & Py_TPFLAGS_LONG_SUBCLASS: + return PyLongObjectPtr + if tp_flags & Py_TPFLAGS_LIST_SUBCLASS: + return PyListObjectPtr + if tp_flags & Py_TPFLAGS_TUPLE_SUBCLASS: + return PyTupleObjectPtr + if tp_flags & Py_TPFLAGS_BYTES_SUBCLASS: + return PyBytesObjectPtr + if tp_flags & Py_TPFLAGS_UNICODE_SUBCLASS: + return PyUnicodeObjectPtr + if tp_flags & Py_TPFLAGS_DICT_SUBCLASS: + return PyDictObjectPtr + if tp_flags & Py_TPFLAGS_BASE_EXC_SUBCLASS: + return PyBaseExceptionObjectPtr + #if tp_flags & Py_TPFLAGS_TYPE_SUBCLASS: + # return PyTypeObjectPtr + + # Use the base class: + return cls + + @classmethod + def from_pyobject_ptr(cls, gdbval): + ''' + Try to locate the appropriate derived class dynamically, and cast + the pointer accordingly. + ''' + try: + p = PyObjectPtr(gdbval) + cls = cls.subclass_from_type(p.type()) + return cls(gdbval, cast_to=cls.get_gdb_type()) + except RuntimeError: + # Handle any kind of error e.g. NULL ptrs by simply using the base + # class + pass + return cls(gdbval) + + @classmethod + def get_gdb_type(cls): + return gdb.lookup_type(cls._typename).pointer() + + def as_address(self): + return long(self._gdbval) + +class PyVarObjectPtr(PyObjectPtr): + _typename = 'PyVarObject' + +class ProxyAlreadyVisited(object): + ''' + Placeholder proxy to use when protecting against infinite recursion due to + loops in the object graph. + + Analogous to the values emitted by the users of Py_ReprEnter and Py_ReprLeave + ''' + def __init__(self, rep): + self._rep = rep + + def __repr__(self): + return self._rep + + +def _write_instance_repr(out, visited, name, pyop_attrdict, address): + '''Shared code for use by old-style and new-style classes: + write a representation to file-like object "out"''' + out.write('<') + out.write(name) + + # Write dictionary of instance attributes: + if isinstance(pyop_attrdict, PyDictObjectPtr): + out.write('(') + first = True + for pyop_arg, pyop_val in pyop_attrdict.iteritems(): + if not first: + out.write(', ') + first = False + out.write(pyop_arg.proxyval(visited)) + out.write('=') + pyop_val.write_repr(out, visited) + out.write(')') + out.write(' at remote 0x%x>' % address) + + +class InstanceProxy(object): + + def __init__(self, cl_name, attrdict, address): + self.cl_name = cl_name + self.attrdict = attrdict + self.address = address + + def __repr__(self): + if isinstance(self.attrdict, dict): + kwargs = ', '.join(["%s=%r" % (arg, val) + for arg, val in self.attrdict.iteritems()]) + return '<%s(%s) at remote 0x%x>' % (self.cl_name, + kwargs, self.address) + else: + return '<%s at remote 0x%x>' % (self.cl_name, + self.address) + +def _PyObject_VAR_SIZE(typeobj, nitems): + return ( ( typeobj.field('tp_basicsize') + + nitems * typeobj.field('tp_itemsize') + + (SIZEOF_VOID_P - 1) + ) & ~(SIZEOF_VOID_P - 1) + ).cast(_type_size_t) + +class HeapTypeObjectPtr(PyObjectPtr): + _typename = 'PyObject' + + def get_attr_dict(self): + ''' + Get the PyDictObject ptr representing the attribute dictionary + (or None if there's a problem) + ''' + try: + typeobj = self.type() + dictoffset = int_from_int(typeobj.field('tp_dictoffset')) + if dictoffset != 0: + if dictoffset < 0: + type_PyVarObject_ptr = gdb.lookup_type('PyVarObject').pointer() + tsize = int_from_int(self._gdbval.cast(type_PyVarObject_ptr)['ob_size']) + if tsize < 0: + tsize = -tsize + size = _PyObject_VAR_SIZE(typeobj, tsize) + dictoffset += size + assert dictoffset > 0 + assert dictoffset % SIZEOF_VOID_P == 0 + + dictptr = self._gdbval.cast(_type_char_ptr) + dictoffset + PyObjectPtrPtr = PyObjectPtr.get_gdb_type().pointer() + dictptr = dictptr.cast(PyObjectPtrPtr) + return PyObjectPtr.from_pyobject_ptr(dictptr.dereference()) + except RuntimeError: + # Corrupt data somewhere; fail safe + pass + + # Not found, or some kind of error: + return None + + def proxyval(self, visited): + ''' + Support for new-style classes. + + Currently we just locate the dictionary using a transliteration to + python of _PyObject_GetDictPtr, ignoring descriptors + ''' + # Guard against infinite loops: + if self.as_address() in visited: + return ProxyAlreadyVisited('<...>') + visited.add(self.as_address()) + + pyop_attr_dict = self.get_attr_dict() + if pyop_attr_dict: + attr_dict = pyop_attr_dict.proxyval(visited) + else: + attr_dict = {} + tp_name = self.safe_tp_name() + + # New-style class: + return InstanceProxy(tp_name, attr_dict, long(self._gdbval)) + + def write_repr(self, out, visited): + # Guard against infinite loops: + if self.as_address() in visited: + out.write('<...>') + return + visited.add(self.as_address()) + + pyop_attrdict = self.get_attr_dict() + _write_instance_repr(out, visited, + self.safe_tp_name(), pyop_attrdict, self.as_address()) + +class ProxyException(Exception): + def __init__(self, tp_name, args): + self.tp_name = tp_name + self.args = args + + def __repr__(self): + return '%s%r' % (self.tp_name, self.args) + +class PyBaseExceptionObjectPtr(PyObjectPtr): + """ + Class wrapping a gdb.Value that's a PyBaseExceptionObject* i.e. an exception + within the process being debugged. + """ + _typename = 'PyBaseExceptionObject' + + def proxyval(self, visited): + # Guard against infinite loops: + if self.as_address() in visited: + return ProxyAlreadyVisited('(...)') + visited.add(self.as_address()) + arg_proxy = self.pyop_field('args').proxyval(visited) + return ProxyException(self.safe_tp_name(), + arg_proxy) + + def write_repr(self, out, visited): + # Guard against infinite loops: + if self.as_address() in visited: + out.write('(...)') + return + visited.add(self.as_address()) + + out.write(self.safe_tp_name()) + self.write_field_repr('args', out, visited) + +class PyClassObjectPtr(PyObjectPtr): + """ + Class wrapping a gdb.Value that's a PyClassObject* i.e. a <classobj> + instance within the process being debugged. + """ + _typename = 'PyClassObject' + + +class BuiltInFunctionProxy(object): + def __init__(self, ml_name): + self.ml_name = ml_name + + def __repr__(self): + return "<built-in function %s>" % self.ml_name + +class BuiltInMethodProxy(object): + def __init__(self, ml_name, pyop_m_self): + self.ml_name = ml_name + self.pyop_m_self = pyop_m_self + + def __repr__(self): + return ('<built-in method %s of %s object at remote 0x%x>' + % (self.ml_name, + self.pyop_m_self.safe_tp_name(), + self.pyop_m_self.as_address()) + ) + +class PyCFunctionObjectPtr(PyObjectPtr): + """ + Class wrapping a gdb.Value that's a PyCFunctionObject* + (see Include/methodobject.h and Objects/methodobject.c) + """ + _typename = 'PyCFunctionObject' + + def proxyval(self, visited): + m_ml = self.field('m_ml') # m_ml is a (PyMethodDef*) + ml_name = m_ml['ml_name'].string() + + pyop_m_self = self.pyop_field('m_self') + if pyop_m_self.is_null(): + return BuiltInFunctionProxy(ml_name) + else: + return BuiltInMethodProxy(ml_name, pyop_m_self) + + +class PyCodeObjectPtr(PyObjectPtr): + """ + Class wrapping a gdb.Value that's a PyCodeObject* i.e. a <code> instance + within the process being debugged. + """ + _typename = 'PyCodeObject' + + def addr2line(self, addrq): + ''' + Get the line number for a given bytecode offset + + Analogous to PyCode_Addr2Line; translated from pseudocode in + Objects/lnotab_notes.txt + ''' + co_lnotab = self.pyop_field('co_lnotab').proxyval(set()) + + # Initialize lineno to co_firstlineno as per PyCode_Addr2Line + # not 0, as lnotab_notes.txt has it: + lineno = int_from_int(self.field('co_firstlineno')) + + addr = 0 + for addr_incr, line_incr in zip(co_lnotab[::2], co_lnotab[1::2]): + addr += ord(addr_incr) + if addr > addrq: + return lineno + lineno += ord(line_incr) + return lineno + + +class PyDictObjectPtr(PyObjectPtr): + """ + Class wrapping a gdb.Value that's a PyDictObject* i.e. a dict instance + within the process being debugged. + """ + _typename = 'PyDictObject' + + def iteritems(self): + ''' + Yields a sequence of (PyObjectPtr key, PyObjectPtr value) pairs, + analagous to dict.iteritems() + ''' + for i in safe_range(self.field('ma_mask') + 1): + ep = self.field('ma_table') + i + pyop_value = PyObjectPtr.from_pyobject_ptr(ep['me_value']) + if not pyop_value.is_null(): + pyop_key = PyObjectPtr.from_pyobject_ptr(ep['me_key']) + yield (pyop_key, pyop_value) + + def proxyval(self, visited): + # Guard against infinite loops: + if self.as_address() in visited: + return ProxyAlreadyVisited('{...}') + visited.add(self.as_address()) + + result = {} + for pyop_key, pyop_value in self.iteritems(): + proxy_key = pyop_key.proxyval(visited) + proxy_value = pyop_value.proxyval(visited) + result[proxy_key] = proxy_value + return result + + def write_repr(self, out, visited): + # Guard against infinite loops: + if self.as_address() in visited: + out.write('{...}') + return + visited.add(self.as_address()) + + out.write('{') + first = True + for pyop_key, pyop_value in self.iteritems(): + if not first: + out.write(', ') + first = False + pyop_key.write_repr(out, visited) + out.write(': ') + pyop_value.write_repr(out, visited) + out.write('}') + +class PyInstanceObjectPtr(PyObjectPtr): + _typename = 'PyInstanceObject' + + def proxyval(self, visited): + # Guard against infinite loops: + if self.as_address() in visited: + return ProxyAlreadyVisited('<...>') + visited.add(self.as_address()) + + # Get name of class: + in_class = self.pyop_field('in_class') + cl_name = in_class.pyop_field('cl_name').proxyval(visited) + + # Get dictionary of instance attributes: + in_dict = self.pyop_field('in_dict').proxyval(visited) + + # Old-style class: + return InstanceProxy(cl_name, in_dict, long(self._gdbval)) + + def write_repr(self, out, visited): + # Guard against infinite loops: + if self.as_address() in visited: + out.write('<...>') + return + visited.add(self.as_address()) + + # Old-style class: + + # Get name of class: + in_class = self.pyop_field('in_class') + cl_name = in_class.pyop_field('cl_name').proxyval(visited) + + # Get dictionary of instance attributes: + pyop_in_dict = self.pyop_field('in_dict') + + _write_instance_repr(out, visited, + cl_name, pyop_in_dict, self.as_address()) + +class PyListObjectPtr(PyObjectPtr): + _typename = 'PyListObject' + + def __getitem__(self, i): + # Get the gdb.Value for the (PyObject*) with the given index: + field_ob_item = self.field('ob_item') + return field_ob_item[i] + + def proxyval(self, visited): + # Guard against infinite loops: + if self.as_address() in visited: + return ProxyAlreadyVisited('[...]') + visited.add(self.as_address()) + + result = [PyObjectPtr.from_pyobject_ptr(self[i]).proxyval(visited) + for i in safe_range(int_from_int(self.field('ob_size')))] + return result + + def write_repr(self, out, visited): + # Guard against infinite loops: + if self.as_address() in visited: + out.write('[...]') + return + visited.add(self.as_address()) + + out.write('[') + for i in safe_range(int_from_int(self.field('ob_size'))): + if i > 0: + out.write(', ') + element = PyObjectPtr.from_pyobject_ptr(self[i]) + element.write_repr(out, visited) + out.write(']') + +class PyLongObjectPtr(PyObjectPtr): + _typename = 'PyLongObject' + + def proxyval(self, visited): + ''' + Python's Include/longobjrep.h has this declaration: + struct _longobject { + PyObject_VAR_HEAD + digit ob_digit[1]; + }; + + with this description: + The absolute value of a number is equal to + SUM(for i=0 through abs(ob_size)-1) ob_digit[i] * 2**(SHIFT*i) + Negative numbers are represented with ob_size < 0; + zero is represented by ob_size == 0. + + where SHIFT can be either: + #define PyLong_SHIFT 30 + #define PyLong_SHIFT 15 + ''' + ob_size = long(self.field('ob_size')) + if ob_size == 0: + return 0L + + ob_digit = self.field('ob_digit') + + if gdb.lookup_type('digit').sizeof == 2: + SHIFT = 15L + else: + SHIFT = 30L + + digits = [long(ob_digit[i]) * 2**(SHIFT*i) + for i in safe_range(abs(ob_size))] + result = sum(digits) + if ob_size < 0: + result = -result + return result + + def write_repr(self, out, visited): + # Write this out as a Python 3 int literal, i.e. without the "L" suffix + proxy = self.proxyval(visited) + out.write("%s" % proxy) + + +class PyBoolObjectPtr(PyLongObjectPtr): + """ + Class wrapping a gdb.Value that's a PyBoolObject* i.e. one of the two + <bool> instances (Py_True/Py_False) within the process being debugged. + """ + def proxyval(self, visited): + if PyLongObjectPtr.proxyval(self, visited): + return True + else: + return False + +class PyNoneStructPtr(PyObjectPtr): + """ + Class wrapping a gdb.Value that's a PyObject* pointing to the + singleton (we hope) _Py_NoneStruct with ob_type PyNone_Type + """ + _typename = 'PyObject' + + def proxyval(self, visited): + return None + + +class PyFrameObjectPtr(PyObjectPtr): + _typename = 'PyFrameObject' + + def __init__(self, gdbval, cast_to): + PyObjectPtr.__init__(self, gdbval, cast_to) + + if not self.is_optimized_out(): + self.co = PyCodeObjectPtr.from_pyobject_ptr(self.field('f_code')) + self.co_name = self.co.pyop_field('co_name') + self.co_filename = self.co.pyop_field('co_filename') + + self.f_lineno = int_from_int(self.field('f_lineno')) + self.f_lasti = int_from_int(self.field('f_lasti')) + self.co_nlocals = int_from_int(self.co.field('co_nlocals')) + self.co_varnames = PyTupleObjectPtr.from_pyobject_ptr(self.co.field('co_varnames')) + + def iter_locals(self): + ''' + Yield a sequence of (name,value) pairs of PyObjectPtr instances, for + the local variables of this frame + ''' + if self.is_optimized_out(): + return + + f_localsplus = self.field('f_localsplus') + for i in safe_range(self.co_nlocals): + pyop_value = PyObjectPtr.from_pyobject_ptr(f_localsplus[i]) + if not pyop_value.is_null(): + pyop_name = PyObjectPtr.from_pyobject_ptr(self.co_varnames[i]) + yield (pyop_name, pyop_value) + + def iter_globals(self): + ''' + Yield a sequence of (name,value) pairs of PyObjectPtr instances, for + the global variables of this frame + ''' + if self.is_optimized_out(): + return + + pyop_globals = self.pyop_field('f_globals') + return pyop_globals.iteritems() + + def iter_builtins(self): + ''' + Yield a sequence of (name,value) pairs of PyObjectPtr instances, for + the builtin variables + ''' + if self.is_optimized_out(): + return + + pyop_builtins = self.pyop_field('f_builtins') + return pyop_builtins.iteritems() + + def get_var_by_name(self, name): + ''' + Look for the named local variable, returning a (PyObjectPtr, scope) pair + where scope is a string 'local', 'global', 'builtin' + + If not found, return (None, None) + ''' + for pyop_name, pyop_value in self.iter_locals(): + if name == pyop_name.proxyval(set()): + return pyop_value, 'local' + for pyop_name, pyop_value in self.iter_globals(): + if name == pyop_name.proxyval(set()): + return pyop_value, 'global' + for pyop_name, pyop_value in self.iter_builtins(): + if name == pyop_name.proxyval(set()): + return pyop_value, 'builtin' + return None, None + + def filename(self): + '''Get the path of the current Python source file, as a string''' + if self.is_optimized_out(): + return '(frame information optimized out)' + return self.co_filename.proxyval(set()) + + def current_line_num(self): + '''Get current line number as an integer (1-based) + + Translated from PyFrame_GetLineNumber and PyCode_Addr2Line + + See Objects/lnotab_notes.txt + ''' + if self.is_optimized_out(): + return None + f_trace = self.field('f_trace') + if long(f_trace) != 0: + # we have a non-NULL f_trace: + return self.f_lineno + else: + #try: + return self.co.addr2line(self.f_lasti) + #except ValueError: + # return self.f_lineno + + def current_line(self): + '''Get the text of the current source line as a string, with a trailing + newline character''' + if self.is_optimized_out(): + return '(frame information optimized out)' + filename = self.filename() + with open(os_fsencode(filename), 'r') as f: + all_lines = f.readlines() + # Convert from 1-based current_line_num to 0-based list offset: + return all_lines[self.current_line_num()-1] + + def write_repr(self, out, visited): + if self.is_optimized_out(): + out.write('(frame information optimized out)') + return + out.write('Frame 0x%x, for file %s, line %i, in %s (' + % (self.as_address(), + self.co_filename.proxyval(visited), + self.current_line_num(), + self.co_name.proxyval(visited))) + first = True + for pyop_name, pyop_value in self.iter_locals(): + if not first: + out.write(', ') + first = False + + out.write(pyop_name.proxyval(visited)) + out.write('=') + pyop_value.write_repr(out, visited) + + out.write(')') + +class PySetObjectPtr(PyObjectPtr): + _typename = 'PySetObject' + + def proxyval(self, visited): + # Guard against infinite loops: + if self.as_address() in visited: + return ProxyAlreadyVisited('%s(...)' % self.safe_tp_name()) + visited.add(self.as_address()) + + members = [] + table = self.field('table') + for i in safe_range(self.field('mask')+1): + setentry = table[i] + key = setentry['key'] + if key != 0: + key_proxy = PyObjectPtr.from_pyobject_ptr(key).proxyval(visited) + if key_proxy != '<dummy key>': + members.append(key_proxy) + if self.safe_tp_name() == 'frozenset': + return frozenset(members) + else: + return set(members) + + def write_repr(self, out, visited): + # Emulate Python 3's set_repr + tp_name = self.safe_tp_name() + + # Guard against infinite loops: + if self.as_address() in visited: + out.write('(...)') + return + visited.add(self.as_address()) + + # Python 3's set_repr special-cases the empty set: + if not self.field('used'): + out.write(tp_name) + out.write('()') + return + + # Python 3 uses {} for set literals: + if tp_name != 'set': + out.write(tp_name) + out.write('(') + + out.write('{') + first = True + table = self.field('table') + for i in safe_range(self.field('mask')+1): + setentry = table[i] + key = setentry['key'] + if key != 0: + pyop_key = PyObjectPtr.from_pyobject_ptr(key) + key_proxy = pyop_key.proxyval(visited) # FIXME! + if key_proxy != '<dummy key>': + if not first: + out.write(', ') + first = False + pyop_key.write_repr(out, visited) + out.write('}') + + if tp_name != 'set': + out.write(')') + + +class PyBytesObjectPtr(PyObjectPtr): + _typename = 'PyBytesObject' + + def __str__(self): + field_ob_size = self.field('ob_size') + field_ob_sval = self.field('ob_sval') + char_ptr = field_ob_sval.address.cast(_type_unsigned_char_ptr) + return ''.join([chr(char_ptr[i]) for i in safe_range(field_ob_size)]) + + def proxyval(self, visited): + return str(self) + + def write_repr(self, out, visited): + # Write this out as a Python 3 bytes literal, i.e. with a "b" prefix + + # Get a PyStringObject* within the Python 2 gdb process: + proxy = self.proxyval(visited) + + # Transliteration of Python 3's Objects/bytesobject.c:PyBytes_Repr + # to Python 2 code: + quote = "'" + if "'" in proxy and not '"' in proxy: + quote = '"' + out.write('b') + out.write(quote) + for byte in proxy: + if byte == quote or byte == '\\': + out.write('\\') + out.write(byte) + elif byte == '\t': + out.write('\\t') + elif byte == '\n': + out.write('\\n') + elif byte == '\r': + out.write('\\r') + elif byte < ' ' or ord(byte) >= 0x7f: + out.write('\\x') + out.write(hexdigits[(ord(byte) & 0xf0) >> 4]) + out.write(hexdigits[ord(byte) & 0xf]) + else: + out.write(byte) + out.write(quote) + +class PyTupleObjectPtr(PyObjectPtr): + _typename = 'PyTupleObject' + + def __getitem__(self, i): + # Get the gdb.Value for the (PyObject*) with the given index: + field_ob_item = self.field('ob_item') + return field_ob_item[i] + + def proxyval(self, visited): + # Guard against infinite loops: + if self.as_address() in visited: + return ProxyAlreadyVisited('(...)') + visited.add(self.as_address()) + + result = tuple([PyObjectPtr.from_pyobject_ptr(self[i]).proxyval(visited) + for i in safe_range(int_from_int(self.field('ob_size')))]) + return result + + def write_repr(self, out, visited): + # Guard against infinite loops: + if self.as_address() in visited: + out.write('(...)') + return + visited.add(self.as_address()) + + out.write('(') + for i in safe_range(int_from_int(self.field('ob_size'))): + if i > 0: + out.write(', ') + element = PyObjectPtr.from_pyobject_ptr(self[i]) + element.write_repr(out, visited) + if self.field('ob_size') == 1: + out.write(',)') + else: + out.write(')') + +class PyTypeObjectPtr(PyObjectPtr): + _typename = 'PyTypeObject' + + +def _unichr_is_printable(char): + # Logic adapted from Python 3's Tools/unicode/makeunicodedata.py + if char == u" ": + return True + import unicodedata + return unicodedata.category(char) not in ("C", "Z") + +if sys.maxunicode >= 0x10000: + _unichr = unichr +else: + # Needed for proper surrogate support if sizeof(Py_UNICODE) is 2 in gdb + def _unichr(x): + if x < 0x10000: + return unichr(x) + x -= 0x10000 + ch1 = 0xD800 | (x >> 10) + ch2 = 0xDC00 | (x & 0x3FF) + return unichr(ch1) + unichr(ch2) + + +class PyUnicodeObjectPtr(PyObjectPtr): + _typename = 'PyUnicodeObject' + + def char_width(self): + _type_Py_UNICODE = gdb.lookup_type('Py_UNICODE') + return _type_Py_UNICODE.sizeof + + def proxyval(self, visited): + # From unicodeobject.h: + # Py_ssize_t length; /* Length of raw Unicode data in buffer */ + # Py_UNICODE *str; /* Raw Unicode buffer */ + field_length = long(self.field('length')) + field_str = self.field('str') + + # Gather a list of ints from the Py_UNICODE array; these are either + # UCS-2 or UCS-4 code points: + if self.char_width() > 2: + Py_UNICODEs = [int(field_str[i]) for i in safe_range(field_length)] + else: + # A more elaborate routine if sizeof(Py_UNICODE) is 2 in the + # inferior process: we must join surrogate pairs. + Py_UNICODEs = [] + i = 0 + limit = safety_limit(field_length) + while i < limit: + ucs = int(field_str[i]) + i += 1 + if ucs < 0xD800 or ucs >= 0xDC00 or i == field_length: + Py_UNICODEs.append(ucs) + continue + # This could be a surrogate pair. + ucs2 = int(field_str[i]) + if ucs2 < 0xDC00 or ucs2 > 0xDFFF: + continue + code = (ucs & 0x03FF) << 10 + code |= ucs2 & 0x03FF + code += 0x00010000 + Py_UNICODEs.append(code) + i += 1 + + # Convert the int code points to unicode characters, and generate a + # local unicode instance. + # This splits surrogate pairs if sizeof(Py_UNICODE) is 2 here (in gdb). + result = u''.join([_unichr(ucs) for ucs in Py_UNICODEs]) + return result + + def write_repr(self, out, visited): + # Write this out as a Python 3 str literal, i.e. without a "u" prefix + + # Get a PyUnicodeObject* within the Python 2 gdb process: + proxy = self.proxyval(visited) + + # Transliteration of Python 3's Object/unicodeobject.c:unicode_repr + # to Python 2: + if "'" in proxy and '"' not in proxy: + quote = '"' + else: + quote = "'" + out.write(quote) + + i = 0 + while i < len(proxy): + ch = proxy[i] + i += 1 + + # Escape quotes and backslashes + if ch == quote or ch == '\\': + out.write('\\') + out.write(ch) + + # Map special whitespace to '\t', \n', '\r' + elif ch == '\t': + out.write('\\t') + elif ch == '\n': + out.write('\\n') + elif ch == '\r': + out.write('\\r') + + # Map non-printable US ASCII to '\xhh' */ + elif ch < ' ' or ch == 0x7F: + out.write('\\x') + out.write(hexdigits[(ord(ch) >> 4) & 0x000F]) + out.write(hexdigits[ord(ch) & 0x000F]) + + # Copy ASCII characters as-is + elif ord(ch) < 0x7F: + out.write(ch) + + # Non-ASCII characters + else: + ucs = ch + ch2 = None + if sys.maxunicode < 0x10000: + # If sizeof(Py_UNICODE) is 2 here (in gdb), join + # surrogate pairs before calling _unichr_is_printable. + if (i < len(proxy) + and 0xD800 <= ord(ch) < 0xDC00 \ + and 0xDC00 <= ord(proxy[i]) <= 0xDFFF): + ch2 = proxy[i] + ucs = ch + ch2 + i += 1 + + # Unfortuately, Python 2's unicode type doesn't seem + # to expose the "isprintable" method + printable = _unichr_is_printable(ucs) + if printable: + try: + ucs.encode(ENCODING) + except UnicodeEncodeError: + printable = False + + # Map Unicode whitespace and control characters + # (categories Z* and C* except ASCII space) + if not printable: + if ch2 is not None: + # Match Python 3's representation of non-printable + # wide characters. + code = (ord(ch) & 0x03FF) << 10 + code |= ord(ch2) & 0x03FF + code += 0x00010000 + else: + code = ord(ucs) + + # Map 8-bit characters to '\\xhh' + if code <= 0xff: + out.write('\\x') + out.write(hexdigits[(code >> 4) & 0x000F]) + out.write(hexdigits[code & 0x000F]) + # Map 21-bit characters to '\U00xxxxxx' + elif code >= 0x10000: + out.write('\\U') + out.write(hexdigits[(code >> 28) & 0x0000000F]) + out.write(hexdigits[(code >> 24) & 0x0000000F]) + out.write(hexdigits[(code >> 20) & 0x0000000F]) + out.write(hexdigits[(code >> 16) & 0x0000000F]) + out.write(hexdigits[(code >> 12) & 0x0000000F]) + out.write(hexdigits[(code >> 8) & 0x0000000F]) + out.write(hexdigits[(code >> 4) & 0x0000000F]) + out.write(hexdigits[code & 0x0000000F]) + # Map 16-bit characters to '\uxxxx' + else: + out.write('\\u') + out.write(hexdigits[(code >> 12) & 0x000F]) + out.write(hexdigits[(code >> 8) & 0x000F]) + out.write(hexdigits[(code >> 4) & 0x000F]) + out.write(hexdigits[code & 0x000F]) + else: + # Copy characters as-is + out.write(ch) + if ch2 is not None: + out.write(ch2) + + out.write(quote) + + + + +def int_from_int(gdbval): + return int(str(gdbval)) + + +def stringify(val): + # TODO: repr() puts everything on one line; pformat can be nicer, but + # can lead to v.long results; this function isolates the choice + if True: + return repr(val) + else: + from pprint import pformat + return pformat(val) + + +class PyObjectPtrPrinter: + "Prints a (PyObject*)" + + def __init__ (self, gdbval): + self.gdbval = gdbval + + def to_string (self): + pyop = PyObjectPtr.from_pyobject_ptr(self.gdbval) + if True: + return pyop.get_truncated_repr(MAX_OUTPUT_LEN) + else: + # Generate full proxy value then stringify it. + # Doing so could be expensive + proxyval = pyop.proxyval(set()) + return stringify(proxyval) + +def pretty_printer_lookup(gdbval): + type = gdbval.type.unqualified() + if type.code == gdb.TYPE_CODE_PTR: + type = type.target().unqualified() + t = str(type) + if t in ("PyObject", "PyFrameObject", "PyUnicodeObject"): + return PyObjectPtrPrinter(gdbval) + +""" +During development, I've been manually invoking the code in this way: +(gdb) python + +import sys +sys.path.append('/home/david/coding/python-gdb') +import libpython +end + +then reloading it after each edit like this: +(gdb) python reload(libpython) + +The following code should ensure that the prettyprinter is registered +if the code is autoloaded by gdb when visiting libpython.so, provided +that this python file is installed to the same path as the library (or its +.debug file) plus a "-gdb.py" suffix, e.g: + /usr/lib/libpython2.6.so.1.0-gdb.py + /usr/lib/debug/usr/lib/libpython2.6.so.1.0.debug-gdb.py +""" +def register (obj): + if obj == None: + obj = gdb + + # Wire up the pretty-printer + obj.pretty_printers.append(pretty_printer_lookup) + +register (gdb.current_objfile ()) + + + +# Unfortunately, the exact API exposed by the gdb module varies somewhat +# from build to build +# See http://bugs.python.org/issue8279?#msg102276 + +class Frame(object): + ''' + Wrapper for gdb.Frame, adding various methods + ''' + def __init__(self, gdbframe): + self._gdbframe = gdbframe + + def older(self): + older = self._gdbframe.older() + if older: + return Frame(older) + else: + return None + + def newer(self): + newer = self._gdbframe.newer() + if newer: + return Frame(newer) + else: + return None + + def select(self): + '''If supported, select this frame and return True; return False if unsupported + + Not all builds have a gdb.Frame.select method; seems to be present on Fedora 12 + onwards, but absent on Ubuntu buildbot''' + if not hasattr(self._gdbframe, 'select'): + print ('Unable to select frame: ' + 'this build of gdb does not expose a gdb.Frame.select method') + return False + self._gdbframe.select() + return True + + def get_index(self): + '''Calculate index of frame, starting at 0 for the newest frame within + this thread''' + index = 0 + # Go down until you reach the newest frame: + iter_frame = self + while iter_frame.newer(): + index += 1 + iter_frame = iter_frame.newer() + return index + + def is_evalframeex(self): + '''Is this a PyEval_EvalFrameEx frame?''' + if self._gdbframe.name() == 'PyEval_EvalFrameEx': + ''' + I believe we also need to filter on the inline + struct frame_id.inline_depth, only regarding frames with + an inline depth of 0 as actually being this function + + So we reject those with type gdb.INLINE_FRAME + ''' + if self._gdbframe.type() == gdb.NORMAL_FRAME: + # We have a PyEval_EvalFrameEx frame: + return True + + return False + + def get_pyop(self): + try: + f = self._gdbframe.read_var('f') + return PyFrameObjectPtr.from_pyobject_ptr(f) + except ValueError: + return None + + @classmethod + def get_selected_frame(cls): + _gdbframe = gdb.selected_frame() + if _gdbframe: + return Frame(_gdbframe) + return None + + @classmethod + def get_selected_python_frame(cls): + '''Try to obtain the Frame for the python code in the selected frame, + or None''' + frame = cls.get_selected_frame() + + while frame: + if frame.is_evalframeex(): + return frame + frame = frame.older() + + # Not found: + return None + + def print_summary(self): + if self.is_evalframeex(): + pyop = self.get_pyop() + if pyop: + line = pyop.get_truncated_repr(MAX_OUTPUT_LEN) + write_unicode(sys.stdout, '#%i %s\n' % (self.get_index(), line)) + sys.stdout.write(pyop.current_line()) + else: + sys.stdout.write('#%i (unable to read python frame information)\n' % self.get_index()) + else: + sys.stdout.write('#%i\n' % self.get_index()) + +class PyList(gdb.Command): + '''List the current Python source code, if any + + Use + py-list START + to list at a different line number within the python source. + + Use + py-list START, END + to list a specific range of lines within the python source. + ''' + + def __init__(self): + gdb.Command.__init__ (self, + "py-list", + gdb.COMMAND_FILES, + gdb.COMPLETE_NONE) + + + def invoke(self, args, from_tty): + import re + + start = None + end = None + + m = re.match(r'\s*(\d+)\s*', args) + if m: + start = int(m.group(0)) + end = start + 10 + + m = re.match(r'\s*(\d+)\s*,\s*(\d+)\s*', args) + if m: + start, end = map(int, m.groups()) + + frame = Frame.get_selected_python_frame() + if not frame: + print 'Unable to locate python frame' + return + + pyop = frame.get_pyop() + if not pyop: + print 'Unable to read information on python frame' + return + + filename = pyop.filename() + lineno = pyop.current_line_num() + + if start is None: + start = lineno - 5 + end = lineno + 5 + + if start<1: + start = 1 + + with open(os_fsencode(filename), 'r') as f: + all_lines = f.readlines() + # start and end are 1-based, all_lines is 0-based; + # so [start-1:end] as a python slice gives us [start, end] as a + # closed interval + for i, line in enumerate(all_lines[start-1:end]): + linestr = str(i+start) + # Highlight current line: + if i + start == lineno: + linestr = '>' + linestr + sys.stdout.write('%4s %s' % (linestr, line)) + + +# ...and register the command: +PyList() + +def move_in_stack(move_up): + '''Move up or down the stack (for the py-up/py-down command)''' + frame = Frame.get_selected_python_frame() + while frame: + if move_up: + iter_frame = frame.older() + else: + iter_frame = frame.newer() + + if not iter_frame: + break + + if iter_frame.is_evalframeex(): + # Result: + if iter_frame.select(): + iter_frame.print_summary() + return + + frame = iter_frame + + if move_up: + print 'Unable to find an older python frame' + else: + print 'Unable to find a newer python frame' + +class PyUp(gdb.Command): + 'Select and print the python stack frame that called this one (if any)' + def __init__(self): + gdb.Command.__init__ (self, + "py-up", + gdb.COMMAND_STACK, + gdb.COMPLETE_NONE) + + + def invoke(self, args, from_tty): + move_in_stack(move_up=True) + +class PyDown(gdb.Command): + 'Select and print the python stack frame called by this one (if any)' + def __init__(self): + gdb.Command.__init__ (self, + "py-down", + gdb.COMMAND_STACK, + gdb.COMPLETE_NONE) + + + def invoke(self, args, from_tty): + move_in_stack(move_up=False) + +# Not all builds of gdb have gdb.Frame.select +if hasattr(gdb.Frame, 'select'): + PyUp() + PyDown() + +class PyBacktrace(gdb.Command): + 'Display the current python frame and all the frames within its call stack (if any)' + def __init__(self): + gdb.Command.__init__ (self, + "py-bt", + gdb.COMMAND_STACK, + gdb.COMPLETE_NONE) + + + def invoke(self, args, from_tty): + frame = Frame.get_selected_python_frame() + while frame: + if frame.is_evalframeex(): + frame.print_summary() + frame = frame.older() + +PyBacktrace() + +class PyPrint(gdb.Command): + 'Look up the given python variable name, and print it' + def __init__(self): + gdb.Command.__init__ (self, + "py-print", + gdb.COMMAND_DATA, + gdb.COMPLETE_NONE) + + + def invoke(self, args, from_tty): + name = str(args) + + frame = Frame.get_selected_python_frame() + if not frame: + print 'Unable to locate python frame' + return + + pyop_frame = frame.get_pyop() + if not pyop_frame: + print 'Unable to read information on python frame' + return + + pyop_var, scope = pyop_frame.get_var_by_name(name) + + if pyop_var: + print ('%s %r = %s' + % (scope, + name, + pyop_var.get_truncated_repr(MAX_OUTPUT_LEN))) + else: + print '%r not found' % name + +PyPrint() + +class PyLocals(gdb.Command): + 'Look up the given python variable name, and print it' + def __init__(self): + gdb.Command.__init__ (self, + "py-locals", + gdb.COMMAND_DATA, + gdb.COMPLETE_NONE) + + + def invoke(self, args, from_tty): + name = str(args) + + frame = Frame.get_selected_python_frame() + if not frame: + print 'Unable to locate python frame' + return + + pyop_frame = frame.get_pyop() + if not pyop_frame: + print 'Unable to read information on python frame' + return + + for pyop_name, pyop_value in pyop_frame.iter_locals(): + print ('%s = %s' + % (pyop_name.proxyval(set()), + pyop_value.get_truncated_repr(MAX_OUTPUT_LEN))) + +PyLocals() diff --git a/Tools/i18n/makelocalealias.py b/Tools/i18n/makelocalealias.py index 45876c1..68544ac 100644 --- a/Tools/i18n/makelocalealias.py +++ b/Tools/i18n/makelocalealias.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Convert the X11 locale.alias file into a mapping dictionary suitable for locale.py. @@ -9,7 +9,7 @@ import locale # Location of the alias file -LOCALE_ALIAS = '/usr/lib/X11/locale/locale.alias' +LOCALE_ALIAS = '/usr/share/X11/locale/locale.alias' def parse(filename): diff --git a/Tools/i18n/msgfmt.py b/Tools/i18n/msgfmt.py index 860e4ea..a554442 100755 --- a/Tools/i18n/msgfmt.py +++ b/Tools/i18n/msgfmt.py @@ -1,6 +1,5 @@ -#! /usr/bin/env python -# -*- coding: iso-8859-1 -*- -# Written by Martin v. Löwis <loewis@informatik.hu-berlin.de> +#! /usr/bin/env python3 +# Written by Martin v. Löwis <loewis@informatik.hu-berlin.de> """Generate binary message catalog from textual translation description. @@ -30,6 +29,7 @@ import os import getopt import struct import array +from email.parser import HeaderParser __version__ = "1.1" @@ -59,13 +59,13 @@ def generate(): # the keys are sorted in the .mo file keys = sorted(MESSAGES.keys()) offsets = [] - ids = strs = '' + ids = strs = b'' for id in keys: # For each string, we need size and file offset. Each string is NUL # terminated; the NUL does not count into the size. offsets.append((len(ids), len(id), len(strs), len(MESSAGES[id]))) - ids += id + '\0' - strs += MESSAGES[id] + '\0' + ids += id + b'\0' + strs += MESSAGES[id] + b'\0' output = '' # The header is 7 32-bit unsigned integers. We don't use hash tables, so # the keys start right after the index tables. @@ -108,7 +108,7 @@ def make(filename, outfile): outfile = os.path.splitext(infile)[0] + '.mo' try: - lines = open(infile).readlines() + lines = open(infile, 'rb').readlines() except IOError as msg: print(msg, file=sys.stderr) sys.exit(1) @@ -116,9 +116,14 @@ def make(filename, outfile): section = None fuzzy = 0 + # Start off assuming Latin-1, so everything decodes without failure, + # until we know the exact encoding + encoding = 'latin-1' + # Parse the catalog lno = 0 for l in lines: + l = l.decode(encoding) lno += 1 # If we get a comment line after a msgstr, this is a new entry if l[0] == '#' and section == STR: @@ -132,16 +137,45 @@ def make(filename, outfile): if l[0] == '#': continue # Now we are in a msgid section, output previous section - if l.startswith('msgid'): + if l.startswith('msgid') and not l.startswith('msgid_plural'): if section == STR: add(msgid, msgstr, fuzzy) + if not msgid: + # See whether there is an encoding declaration + p = HeaderParser() + charset = p.parsestr(msgstr.decode(encoding)).get_content_charset() + if charset: + encoding = charset section = ID l = l[5:] - msgid = msgstr = '' + msgid = msgstr = b'' + is_plural = False + # This is a message with plural forms + elif l.startswith('msgid_plural'): + if section != ID: + print('msgid_plural not preceeded by msgid on %s:%d' % (infile, lno), + file=sys.stderr) + sys.exit(1) + l = l[12:] + msgid += b'\0' # separator of singular and plural + is_plural = True # Now we are in a msgstr section elif l.startswith('msgstr'): section = STR - l = l[6:] + if l.startswith('msgstr['): + if not is_plural: + print('plural without msgid_plural on %s:%d' % (infile, lno), + file=sys.stderr) + sys.exit(1) + l = l.split(']', 1)[1] + if msgstr: + msgstr += b'\0' # Separator of the various plural forms + else: + if is_plural: + print('indexed msgstr required for plural on %s:%d' % (infile, lno), + file=sys.stderr) + sys.exit(1) + l = l[6:] # Skip empty lines l = l.strip() if not l: @@ -149,9 +183,9 @@ def make(filename, outfile): # XXX: Does this always follow Python escape semantics? l = eval(l) if section == ID: - msgid += l + msgid += l.encode(encoding) elif section == STR: - msgstr += l + msgstr += l.encode(encoding) else: print('Syntax error on %s:%d' % (infile, lno), \ 'before:', file=sys.stderr) diff --git a/Tools/i18n/pygettext.py b/Tools/i18n/pygettext.py index d4ea597..67a960f 100755 --- a/Tools/i18n/pygettext.py +++ b/Tools/i18n/pygettext.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # -*- coding: iso-8859-1 -*- # Originally written by Barry Warsaw <barry@zope.com> # diff --git a/Tools/iobench/iobench.py b/Tools/iobench/iobench.py new file mode 100644 index 0000000..b3bdd6a --- /dev/null +++ b/Tools/iobench/iobench.py @@ -0,0 +1,545 @@ +# -*- coding: utf-8 -*- +# This file should be kept compatible with both Python 2.6 and Python >= 3.0. + +import time +import os +import re +import sys +import hashlib +import functools +import itertools +from optparse import OptionParser + +out = sys.stdout + +TEXT_ENCODING = 'utf8' +NEWLINES = 'lf' + +# Compatibility +try: + xrange +except NameError: + xrange = range + +def text_open(fn, mode, encoding=None): + try: + return open(fn, mode, encoding=encoding or TEXT_ENCODING) + except TypeError: + return open(fn, mode) + +def get_file_sizes(): + for s in ['20 KB', '400 KB', '10 MB']: + size, unit = s.split() + size = int(size) * {'KB': 1024, 'MB': 1024 ** 2}[unit] + yield s.replace(' ', ''), size + +def get_binary_files(): + return ((name + ".bin", size) for name, size in get_file_sizes()) + +def get_text_files(): + return (("%s-%s-%s.txt" % (name, TEXT_ENCODING, NEWLINES), size) + for name, size in get_file_sizes()) + +def with_open_mode(mode): + def decorate(f): + f.file_open_mode = mode + return f + return decorate + +def with_sizes(*sizes): + def decorate(f): + f.file_sizes = sizes + return f + return decorate + + +# Here begin the tests + +@with_open_mode("r") +@with_sizes("medium") +def read_bytewise(f): + """ read one unit at a time """ + f.seek(0) + while f.read(1): + pass + +@with_open_mode("r") +@with_sizes("medium") +def read_small_chunks(f): + """ read 20 units at a time """ + f.seek(0) + while f.read(20): + pass + +@with_open_mode("r") +@with_sizes("medium") +def read_big_chunks(f): + """ read 4096 units at a time """ + f.seek(0) + while f.read(4096): + pass + +@with_open_mode("r") +@with_sizes("small", "medium", "large") +def read_whole_file(f): + """ read whole contents at once """ + f.seek(0) + while f.read(): + pass + +@with_open_mode("rt") +@with_sizes("medium") +def read_lines(f): + """ read one line at a time """ + f.seek(0) + for line in f: + pass + +@with_open_mode("r") +@with_sizes("medium") +def seek_forward_bytewise(f): + """ seek forward one unit at a time """ + f.seek(0, 2) + size = f.tell() + f.seek(0, 0) + for i in xrange(0, size - 1): + f.seek(i, 0) + +@with_open_mode("r") +@with_sizes("medium") +def seek_forward_blockwise(f): + """ seek forward 1000 units at a time """ + f.seek(0, 2) + size = f.tell() + f.seek(0, 0) + for i in xrange(0, size - 1, 1000): + f.seek(i, 0) + +@with_open_mode("rb") +@with_sizes("medium") +def read_seek_bytewise(f): + """ alternate read & seek one unit """ + f.seek(0) + while f.read(1): + f.seek(1, 1) + +@with_open_mode("rb") +@with_sizes("medium") +def read_seek_blockwise(f): + """ alternate read & seek 1000 units """ + f.seek(0) + while f.read(1000): + f.seek(1000, 1) + + +@with_open_mode("w") +@with_sizes("small") +def write_bytewise(f, source): + """ write one unit at a time """ + for i in xrange(0, len(source)): + f.write(source[i:i+1]) + +@with_open_mode("w") +@with_sizes("medium") +def write_small_chunks(f, source): + """ write 20 units at a time """ + for i in xrange(0, len(source), 20): + f.write(source[i:i+20]) + +@with_open_mode("w") +@with_sizes("medium") +def write_medium_chunks(f, source): + """ write 4096 units at a time """ + for i in xrange(0, len(source), 4096): + f.write(source[i:i+4096]) + +@with_open_mode("w") +@with_sizes("large") +def write_large_chunks(f, source): + """ write 1e6 units at a time """ + for i in xrange(0, len(source), 1000000): + f.write(source[i:i+1000000]) + + +@with_open_mode("w+") +@with_sizes("small") +def modify_bytewise(f, source): + """ modify one unit at a time """ + f.seek(0) + for i in xrange(0, len(source)): + f.write(source[i:i+1]) + +@with_open_mode("w+") +@with_sizes("medium") +def modify_small_chunks(f, source): + """ modify 20 units at a time """ + f.seek(0) + for i in xrange(0, len(source), 20): + f.write(source[i:i+20]) + +@with_open_mode("w+") +@with_sizes("medium") +def modify_medium_chunks(f, source): + """ modify 4096 units at a time """ + f.seek(0) + for i in xrange(0, len(source), 4096): + f.write(source[i:i+4096]) + +@with_open_mode("wb+") +@with_sizes("medium") +def modify_seek_forward_bytewise(f, source): + """ alternate write & seek one unit """ + f.seek(0) + for i in xrange(0, len(source), 2): + f.write(source[i:i+1]) + f.seek(i+2) + +@with_open_mode("wb+") +@with_sizes("medium") +def modify_seek_forward_blockwise(f, source): + """ alternate write & seek 1000 units """ + f.seek(0) + for i in xrange(0, len(source), 2000): + f.write(source[i:i+1000]) + f.seek(i+2000) + +# XXX the 2 following tests don't work with py3k's text IO +@with_open_mode("wb+") +@with_sizes("medium") +def read_modify_bytewise(f, source): + """ alternate read & write one unit """ + f.seek(0) + for i in xrange(0, len(source), 2): + f.read(1) + f.write(source[i+1:i+2]) + +@with_open_mode("wb+") +@with_sizes("medium") +def read_modify_blockwise(f, source): + """ alternate read & write 1000 units """ + f.seek(0) + for i in xrange(0, len(source), 2000): + f.read(1000) + f.write(source[i+1000:i+2000]) + + +read_tests = [ + read_bytewise, read_small_chunks, read_lines, read_big_chunks, + None, read_whole_file, None, + seek_forward_bytewise, seek_forward_blockwise, + read_seek_bytewise, read_seek_blockwise, +] + +write_tests = [ + write_bytewise, write_small_chunks, write_medium_chunks, write_large_chunks, +] + +modify_tests = [ + modify_bytewise, modify_small_chunks, modify_medium_chunks, + None, + modify_seek_forward_bytewise, modify_seek_forward_blockwise, + read_modify_bytewise, read_modify_blockwise, +] + +def run_during(duration, func): + _t = time.time + n = 0 + start = os.times() + start_timestamp = _t() + real_start = start[4] or start_timestamp + while True: + func() + n += 1 + if _t() - start_timestamp > duration: + break + end = os.times() + real = (end[4] if start[4] else time.time()) - real_start + return n, real, sum(end[0:2]) - sum(start[0:2]) + +def warm_cache(filename): + with open(filename, "rb") as f: + f.read() + + +def run_all_tests(options): + def print_label(filename, func): + name = re.split(r'[-.]', filename)[0] + out.write( + ("[%s] %s... " + % (name.center(7), func.__doc__.strip()) + ).ljust(52)) + out.flush() + + def print_results(size, n, real, cpu): + bw = n * float(size) / 1024 ** 2 / real + bw = ("%4d MB/s" if bw > 100 else "%.3g MB/s") % bw + out.write(bw.rjust(12) + "\n") + if cpu < 0.90 * real: + out.write(" warning: test above used only %d%% CPU, " + "result may be flawed!\n" % (100.0 * cpu / real)) + + def run_one_test(name, size, open_func, test_func, *args): + mode = test_func.file_open_mode + print_label(name, test_func) + if "w" not in mode or "+" in mode: + warm_cache(name) + with open_func(name) as f: + n, real, cpu = run_during(1.5, lambda: test_func(f, *args)) + print_results(size, n, real, cpu) + + def run_test_family(tests, mode_filter, files, open_func, *make_args): + for test_func in tests: + if test_func is None: + out.write("\n") + continue + if mode_filter in test_func.file_open_mode: + continue + for s in test_func.file_sizes: + name, size = files[size_names[s]] + #name += file_ext + args = tuple(f(name, size) for f in make_args) + run_one_test(name, size, + open_func, test_func, *args) + + size_names = { + "small": 0, + "medium": 1, + "large": 2, + } + + binary_files = list(get_binary_files()) + text_files = list(get_text_files()) + if "b" in options: + print("Binary unit = one byte") + if "t" in options: + print("Text unit = one character (%s-decoded)" % TEXT_ENCODING) + + # Binary reads + if "b" in options and "r" in options: + print("\n** Binary input **\n") + run_test_family(read_tests, "t", binary_files, lambda fn: open(fn, "rb")) + + # Text reads + if "t" in options and "r" in options: + print("\n** Text input **\n") + run_test_family(read_tests, "b", text_files, lambda fn: text_open(fn, "r")) + + # Binary writes + if "b" in options and "w" in options: + print("\n** Binary append **\n") + def make_test_source(name, size): + with open(name, "rb") as f: + return f.read() + run_test_family(write_tests, "t", binary_files, + lambda fn: open(os.devnull, "wb"), make_test_source) + + # Text writes + if "t" in options and "w" in options: + print("\n** Text append **\n") + def make_test_source(name, size): + with text_open(name, "r") as f: + return f.read() + run_test_family(write_tests, "b", text_files, + lambda fn: text_open(os.devnull, "w"), make_test_source) + + # Binary overwrites + if "b" in options and "w" in options: + print("\n** Binary overwrite **\n") + def make_test_source(name, size): + with open(name, "rb") as f: + return f.read() + run_test_family(modify_tests, "t", binary_files, + lambda fn: open(fn, "r+b"), make_test_source) + + # Text overwrites + if "t" in options and "w" in options: + print("\n** Text overwrite **\n") + def make_test_source(name, size): + with text_open(name, "r") as f: + return f.read() + run_test_family(modify_tests, "b", text_files, + lambda fn: open(fn, "r+"), make_test_source) + + +def prepare_files(): + print("Preparing files...") + # Binary files + for name, size in get_binary_files(): + if os.path.isfile(name) and os.path.getsize(name) == size: + continue + with open(name, "wb") as f: + f.write(os.urandom(size)) + # Text files + chunk = [] + with text_open(__file__, "rU", encoding='utf8') as f: + for line in f: + if line.startswith("# <iobench text chunk marker>"): + break + else: + raise RuntimeError( + "Couldn't find chunk marker in %s !" % __file__) + if NEWLINES == "all": + it = itertools.cycle(["\n", "\r", "\r\n"]) + else: + it = itertools.repeat( + {"cr": "\r", "lf": "\n", "crlf": "\r\n"}[NEWLINES]) + chunk = "".join(line.replace("\n", next(it)) for line in f) + if isinstance(chunk, bytes): + chunk = chunk.decode('utf8') + chunk = chunk.encode(TEXT_ENCODING) + for name, size in get_text_files(): + if os.path.isfile(name) and os.path.getsize(name) == size: + continue + head = chunk * (size // len(chunk)) + tail = chunk[:size % len(chunk)] + # Adjust tail to end on a character boundary + while True: + try: + tail.decode(TEXT_ENCODING) + break + except UnicodeDecodeError: + tail = tail[:-1] + with open(name, "wb") as f: + f.write(head) + f.write(tail) + +def main(): + global TEXT_ENCODING, NEWLINES + + usage = "usage: %prog [-h|--help] [options]" + parser = OptionParser(usage=usage) + parser.add_option("-b", "--binary", + action="store_true", dest="binary", default=False, + help="run binary I/O tests") + parser.add_option("-t", "--text", + action="store_true", dest="text", default=False, + help="run text I/O tests") + parser.add_option("-r", "--read", + action="store_true", dest="read", default=False, + help="run read tests") + parser.add_option("-w", "--write", + action="store_true", dest="write", default=False, + help="run write & modify tests") + parser.add_option("-E", "--encoding", + action="store", dest="encoding", default=None, + help="encoding for text tests (default: %s)" % TEXT_ENCODING) + parser.add_option("-N", "--newlines", + action="store", dest="newlines", default='lf', + help="line endings for text tests " + "(one of: {lf (default), cr, crlf, all})") + parser.add_option("-m", "--io-module", + action="store", dest="io_module", default=None, + help="io module to test (default: builtin open())") + options, args = parser.parse_args() + if args: + parser.error("unexpected arguments") + NEWLINES = options.newlines.lower() + if NEWLINES not in ('lf', 'cr', 'crlf', 'all'): + parser.error("invalid 'newlines' option: %r" % NEWLINES) + + test_options = "" + if options.read: + test_options += "r" + if options.write: + test_options += "w" + elif not options.read: + test_options += "rw" + if options.text: + test_options += "t" + if options.binary: + test_options += "b" + elif not options.text: + test_options += "tb" + + if options.encoding: + TEXT_ENCODING = options.encoding + + if options.io_module: + globals()['open'] = __import__(options.io_module, {}, {}, ['open']).open + + prepare_files() + run_all_tests(test_options) + +if __name__ == "__main__": + main() + + +# -- This part to exercise text reading. Don't change anything! -- +# <iobench text chunk marker> + +""" +1. +Gáttir allar, +áðr gangi fram, +um skoðask skyli, +um skyggnast skyli, +þvà at óvÃst er at vita, +hvar óvinir +sitja á fleti fyrir. + +2. +Gefendr heilir! +Gestr er inn kominn, +hvar skal sitja sjá? +Mjök er bráðr, +sá er á bröndum skal +sÃns of freista frama. + +3. +Elds er þörf, +þeims inn er kominn +ok á kné kalinn; +matar ok váða +er manni þörf, +þeim er hefr um fjall farit. + +4. +Vatns er þörf, +þeim er til verðar kemr, +þerru ok þjóðlaðar, +góðs of æðis, +ef sér geta mætti, +orðs ok endrþögu. + +5. +Vits er þörf, +þeim er vÃða ratar; +dælt er heima hvat; +at augabragði verðr, +sá er ekki kann +ok með snotrum sitr. + +6. +At hyggjandi sinni +skyli-t maðr hræsinn vera, +heldr gætinn at geði; +þá er horskr ok þögull +kemr heimisgarða til, +sjaldan verðr vÃti vörum, +þvà at óbrigðra vin +fær maðr aldregi +en mannvit mikit. + +7. +Inn vari gestr, +er til verðar kemr, +þunnu hljóði þegir, +eyrum hlýðir, +en augum skoðar; +svá nýsisk fróðra hverr fyrir. + +8. +Hinn er sæll, +er sér of getr +lof ok lÃknstafi; +ódælla er við þat, +er maðr eiga skal +annars brjóstum Ã. +""" + +""" +C'est revenir tard, je le sens, sur un sujet trop rebattu et déjà presque oublié. Mon état, qui ne me permet plus aucun travail suivi, mon aversion pour le genre polémique, ont causé ma lenteur à écrire et ma répugnance à publier. J'aurais même tout à fait supprimé ces Lettres, ou plutôt je lie les aurais point écrites, s'il n'eût été question que de moi : Mais ma patrie ne m'est pas tellement devenue étrangère que je puisse voir tranquillement opprimer ses citoyens, surtout lorsqu'ils n'ont compromis leurs droits qu'en défendant ma cause. Je serais le dernier des hommes si dans une telle occasion j'écoutais un sentiment qui n'est plus ni douceur ni patience, mais faiblesse et lâcheté, dans celui qu'il empêche de remplir son devoir. +Rien de moins important pour le public, j'en conviens, que la matière de ces lettres. La constitution d'une petite République, le sort d'un petit particulier, l'exposé de quelques injustices, la réfutation de quelques sophismes ; tout cela n'a rien en soi d'assez considérable pour mériter beaucoup de lecteurs : mais si mes sujets sont petits mes objets sont grands, et dignes de l'attention de tout honnête homme. Laissons Genève à sa place, et Rousseau dans sa dépression ; mais la religion, mais la liberté, la justice ! voilà , qui que vous soyez, ce qui n'est pas au-dessous de vous. +Qu'on ne cherche pas même ici dans le style le dédommagement de l'aridité de la matière. Ceux que quelques traits heureux de ma plume ont si fort irrités trouveront de quoi s'apaiser dans ces lettres, L'honneur de défendre un opprimé eût enflammé mon coeur si j'avais parlé pour un autre. Réduit au triste emploi de me défendre moi-même, j'ai dû me borner à raisonner ; m'échauffer eût été m'avilir. J'aurai donc trouvé grâce en ce point devant ceux qui s'imaginent qu'il est essentiel à la vérité d'être dite froidement ; opinion que pourtant j'ai peine à comprendre. Lorsqu'une vive persuasion nous anime, le moyen d'employer un langage glacé ? Quand Archimède tout transporté courait nu dans les rues de Syracuse, en avait-il moins trouvé la vérité parce qu'il se passionnait pour elle ? Tout au contraire, celui qui la sent ne peut s'abstenir de l'adorer ; celui qui demeure froid ne l'a pas vue. +Quoi qu'il en soit, je prie les lecteurs de vouloir bien mettre à part mon beau style, et d'examiner seulement si je raisonne bien ou mal ; car enfin, de cela seul qu'un auteur s'exprime en bons termes, je ne vois pas comment il peut s'ensuivre que cet auteur ne sait ce qu'il dit. +""" diff --git a/Tools/modulator/EXAMPLE.py b/Tools/modulator/EXAMPLE.py deleted file mode 100644 index b36a5a7..0000000 --- a/Tools/modulator/EXAMPLE.py +++ /dev/null @@ -1,53 +0,0 @@ -# -# Example input file for modulator if you don't have tk. -# -# You may also have to strip some imports out of modulator to make -# it work. - -import genmodule - -# -# Generate code for a simple object with a method called sample - -o = genmodule.object() -o.name = 'simple object' -o.abbrev = 'simp' -o.methodlist = ['sample'] -o.funclist = ['new'] - -# -# Generate code for an object that looks numberish -# -o2 = genmodule.object() -o2.name = 'number-like object' -o2.abbrev = 'nl' -o2.typelist = ['tp_as_number'] -o2.funclist = ['new', 'tp_repr', 'tp_compare'] - -# -# Generate code for a method with a full complement of functions, -# some methods, accessible as sequence and allowing structmember.c type -# structure access as well. -# -o3 = genmodule.object() -o3.name = 'over-the-top object' -o3.abbrev = 'ot' -o3.methodlist = ['method1', 'method2'] -o3.funclist = ['new', 'tp_dealloc', 'tp_print', 'tp_getattr', 'tp_setattr', - 'tp_compare', 'tp_repr', 'tp_hash'] -o3.typelist = ['tp_as_sequence', 'structure'] - -# -# Now generate code for a module that incorporates these object types. -# Also add the boilerplates for functions to create instances of each -# type. -# -m = genmodule.module() -m.name = 'sample' -m.abbrev = 'sample' -m.methodlist = ['newsimple', 'newnumberish', 'newott'] -m.objects = [o, o2, o3] - -fp = open('EXAMPLEmodule.c', 'w') -genmodule.write(fp, m) -fp.close() diff --git a/Tools/modulator/README b/Tools/modulator/README deleted file mode 100644 index aae86b4..0000000 --- a/Tools/modulator/README +++ /dev/null @@ -1,25 +0,0 @@ -This is release 1.2 of modulator, a generator of boilerplate code for -modules to be written in C. - -Difference between 1.2 and 1.1: __doc__ templates are now generated -(thanks to Jim Fulton). - -Difference between 1.1 and 1.0: the templates now use "new-style" -naming conventions. Many thanks to Chak Tan <tan@ee.rochester.edu> for -supplying them. - -Usage when you have tk is *really* simple: start modulator, fill out -the forms specifying all the objects and methods, tell modulator -whether objects should also be accessible as sequences, etc and press -'generate code'. It will write a complete skeleton module for you. - -Usage when you don't have tk is slightly more difficult. Look at -EXAMPLE.py for some details (to run, use "python EXAMPLE.py"). Don't -bother with EXAMPLE.py if you have Tkinter!!! - -Oh yeah: you'll probably want to change Templates/copyright, or all -your code ends up as being copyrighted to CWI:-) - -Let me know what you think, - Jack Jansen, jack@cwi.nl - diff --git a/Tools/modulator/ScrolledListbox.py b/Tools/modulator/ScrolledListbox.py deleted file mode 100644 index 2ec646d..0000000 --- a/Tools/modulator/ScrolledListbox.py +++ /dev/null @@ -1,37 +0,0 @@ -# A ScrolledList widget feels like a list widget but also has a -# vertical scroll bar on its right. (Later, options may be added to -# add a horizontal bar as well, to make the bars disappear -# automatically when not needed, to move them to the other side of the -# window, etc.) -# -# Configuration options are passed to the List widget. -# A Frame widget is inserted between the master and the list, to hold -# the Scrollbar widget. -# Most methods calls are inherited from the List widget; Pack methods -# are redirected to the Frame widget however. - -from Tkinter import * -from Tkinter import _cnfmerge - -class ScrolledListbox(Listbox): - def __init__(self, master=None, cnf={}): - cnf = _cnfmerge(cnf) - fcnf = {} - vcnf = {'name': 'vbar', - Pack: {'side': 'right', 'fill': 'y'},} - for k in list(cnf.keys()): - if type(k) == ClassType or k == 'name': - fcnf[k] = cnf[k] - del cnf[k] - self.frame = Frame(master, fcnf) - self.vbar = Scrollbar(self.frame, vcnf) - cnf[Pack] = {'side': 'left', 'fill': 'both', 'expand': 'yes'} - cnf['name'] = 'list' - Listbox.__init__(self, self.frame, cnf) - self['yscrollcommand'] = (self.vbar, 'set') - self.vbar['command'] = (self, 'yview') - - # Copy Pack methods of self.frame -- hack! - for m in Pack.__dict__: - if m[0] != '_' and m != 'config': - setattr(self, m, getattr(self.frame, m)) diff --git a/Tools/modulator/Templates/copyright b/Tools/modulator/Templates/copyright deleted file mode 100644 index e69de29..0000000 --- a/Tools/modulator/Templates/copyright +++ /dev/null diff --git a/Tools/modulator/Templates/module_head b/Tools/modulator/Templates/module_head deleted file mode 100644 index be00109..0000000 --- a/Tools/modulator/Templates/module_head +++ /dev/null @@ -1,6 +0,0 @@ - -#include "Python.h" - -static PyObject *ErrorObject; - -/* ----------------------------------------------------- */ diff --git a/Tools/modulator/Templates/module_method b/Tools/modulator/Templates/module_method deleted file mode 100644 index 3048b1f..0000000 --- a/Tools/modulator/Templates/module_method +++ /dev/null @@ -1,14 +0,0 @@ - -static char $abbrev$_$method$__doc__[] = -"" -; - -static PyObject * -$abbrev$_$method$(PyObject *self /* Not used */, PyObject *args) -{ - - if (!PyArg_ParseTuple(args, "")) - return NULL; - Py_INCREF(Py_None); - return Py_None; -} diff --git a/Tools/modulator/Templates/module_tail b/Tools/modulator/Templates/module_tail deleted file mode 100644 index 59cc50b..0000000 --- a/Tools/modulator/Templates/module_tail +++ /dev/null @@ -1,37 +0,0 @@ - -/* List of methods defined in the module */ - -static struct PyMethodDef $abbrev$_methods[] = { - $methodlist$ - {NULL, (PyCFunction)NULL, 0, NULL} /* sentinel */ -}; - - -/* Initialization function for the module (*must* be called init$name$) */ - -static char $name$_module_documentation[] = -"" -; - -void -init$name$() -{ - PyObject *m, *d; - - /* Create the module and add the functions */ - m = Py_InitModule4("$name$", $abbrev$_methods, - $name$_module_documentation, - (PyObject*)NULL,PYTHON_API_VERSION); - - /* Add some symbolic constants to the module */ - d = PyModule_GetDict(m); - ErrorObject = PyString_FromString("$name$.error"); - PyDict_SetItemString(d, "error", ErrorObject); - - /* XXXX Add constants here */ - - /* Check for errors */ - if (PyErr_Occurred()) - Py_FatalError("can't initialize module $name$"); -} - diff --git a/Tools/modulator/Templates/object_head b/Tools/modulator/Templates/object_head deleted file mode 100644 index 07d1f6a..0000000 --- a/Tools/modulator/Templates/object_head +++ /dev/null @@ -1,13 +0,0 @@ - -/* Declarations for objects of type $name$ */ - -typedef struct { - PyObject_HEAD - /* XXXX Add your own stuff here */ -} $abbrev$object; - -static PyTypeObject $Abbrev$type; - - - -/* ---------------------------------------------------------------- */ diff --git a/Tools/modulator/Templates/object_method b/Tools/modulator/Templates/object_method deleted file mode 100644 index b15162c..0000000 --- a/Tools/modulator/Templates/object_method +++ /dev/null @@ -1,14 +0,0 @@ - -static char $abbrev$_$method$__doc__[] = -"" -; - -static PyObject * -$abbrev$_$method$($abbrev$object *self, PyObject *args) -{ - if (!PyArg_ParseTuple(args, "")) - return NULL; - Py_INCREF(Py_None); - return Py_None; -} - diff --git a/Tools/modulator/Templates/object_mlist b/Tools/modulator/Templates/object_mlist deleted file mode 100644 index a12a9e1..0000000 --- a/Tools/modulator/Templates/object_mlist +++ /dev/null @@ -1,8 +0,0 @@ - -static struct PyMethodDef $abbrev$_methods[] = { - $methodlist$ - {NULL, NULL} /* sentinel */ -}; - -/* ---------- */ - diff --git a/Tools/modulator/Templates/object_new b/Tools/modulator/Templates/object_new deleted file mode 100644 index 30c5e36..0000000 --- a/Tools/modulator/Templates/object_new +++ /dev/null @@ -1,13 +0,0 @@ - -static $abbrev$object * -new$abbrev$object() -{ - $abbrev$object *self; - - self = PyObject_NEW($abbrev$object, &$Abbrev$type); - if (self == NULL) - return NULL; - /* XXXX Add your own initializers here */ - return self; -} - diff --git a/Tools/modulator/Templates/object_structure b/Tools/modulator/Templates/object_structure deleted file mode 100644 index 78daa62..0000000 --- a/Tools/modulator/Templates/object_structure +++ /dev/null @@ -1,37 +0,0 @@ - -/* Code to access structure members by accessing attributes */ - -#include "structmember.h" - -#define OFF(x) offsetof(XXXXobject, x) - -static PyMemberDef $abbrev$_memberlist[] = { - /* XXXX Add lines like { "foo", T_INT, OFF(foo), READONLY } */ - - {NULL} /* Sentinel */ -}; - -static PyObject * -$abbrev$_getattr($abbrev$object *self, char *name) -{ - PyObject *rv; - - /* XXXX Add your own getattr code here */ - rv = PyMember_GetOne((char *)/*XXXX*/0, &$abbrev$_memberlist[i]); - if (rv) - return rv; - PyErr_Clear(); - return Py_FindMethod($abbrev$_methods, (PyObject *)self, name); -} - - -static int -$abbrev$_setattr($abbrev$object *self, char *name, PyObject *v) -{ - /* XXXX Add your own setattr code here */ - if ( v == NULL ) { - PyErr_SetString(PyExc_AttributeError, "Cannot delete attribute"); - return -1; - } - return PyMember_SetOne((char *)/*XXXX*/0, &$abbrev$_memberlist[i], v); -} diff --git a/Tools/modulator/Templates/object_tail b/Tools/modulator/Templates/object_tail deleted file mode 100644 index 1d1334c..0000000 --- a/Tools/modulator/Templates/object_tail +++ /dev/null @@ -1,33 +0,0 @@ - -static char $Abbrev$type__doc__[] = -"" -; - -static PyTypeObject $Abbrev$type = { - PyObject_HEAD_INIT(&PyType_Type) - 0, /*ob_size*/ - "$name$", /*tp_name*/ - sizeof($abbrev$object), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - /* methods */ - (destructor)$tp_dealloc$, /*tp_dealloc*/ - (printfunc)$tp_print$, /*tp_print*/ - (getattrfunc)$tp_getattr$, /*tp_getattr*/ - (setattrfunc)$tp_setattr$, /*tp_setattr*/ - (cmpfunc)$tp_compare$, /*tp_compare*/ - (reprfunc)$tp_repr$, /*tp_repr*/ - $tp_as_number$, /*tp_as_number*/ - $tp_as_sequence$, /*tp_as_sequence*/ - $tp_as_mapping$, /*tp_as_mapping*/ - (hashfunc)$tp_hash$, /*tp_hash*/ - (ternaryfunc)$tp_call$, /*tp_call*/ - (reprfunc)$tp_str$, /*tp_str*/ - - /* Space for future expansion */ - 0L,0L,0L,0L, - $Abbrev$type__doc__ /* Documentation string */ -}; - -/* End of code for $name$ objects */ -/* -------------------------------------------------------- */ - diff --git a/Tools/modulator/Templates/object_tp_as_mapping b/Tools/modulator/Templates/object_tp_as_mapping deleted file mode 100644 index f9213b7..0000000 --- a/Tools/modulator/Templates/object_tp_as_mapping +++ /dev/null @@ -1,29 +0,0 @@ - -/* Code to access $name$ objects as mappings */ - -static int -$abbrev$_length($abbrev$object *self) -{ - /* XXXX Return the size of the mapping */ -} - -static PyObject * -$abbrev$_subscript($abbrev$object *self, PyObject *key) -{ - /* XXXX Return the item of self indexed by key */ -} - -static int -$abbrev$_ass_sub($abbrev$object *self, PyObject *v, PyObject *w) -{ - /* XXXX Put w in self under key v */ - return 0; -} - -static PyMappingMethods $abbrev$_as_mapping = { - (inquiry)$abbrev$_length, /*mp_length*/ - (binaryfunc)$abbrev$_subscript, /*mp_subscript*/ - (objobjargproc)$abbrev$_ass_sub, /*mp_ass_subscript*/ -}; - -/* -------------------------------------------------------- */ diff --git a/Tools/modulator/Templates/object_tp_as_number b/Tools/modulator/Templates/object_tp_as_number deleted file mode 100644 index a3426c2..0000000 --- a/Tools/modulator/Templates/object_tp_as_number +++ /dev/null @@ -1,162 +0,0 @@ - -/* Code to access $name$ objects as numbers */ - -static PyObject * -$abbrev$_add($abbrev$object *v, $abbrev$object *w) -{ - /* XXXX Add them */ -} - -static PyObject * -$abbrev$_sub($abbrev$object *v, $abbrev$object *w) -{ - /* XXXX Subtract them */ -} - -static PyObject * -$abbrev$_mul($abbrev$object *v, $abbrev$object *w) -{ - /* XXXX Multiply them */ -} - -static PyObject * -$abbrev$_div($abbrev$object *x, $abbrev$object *y) -{ - /* XXXX Divide them */ -} - -static PyObject * -$abbrev$_mod($abbrev$object *x, $abbrev$object *y) -{ - /* XXXX Modulo them */ -} - -static PyObject * -$abbrev$_divmod($abbrev$object *x, $abbrev$object *y) -{ - /* XXXX Return 2-tuple with div and mod */ -} - -static PyObject * -$abbrev$_pow($abbrev$object *v, $abbrev$object *w, $abbrev$object *z) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_neg($abbrev$object *v) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_pos($abbrev$object *v) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_abs($abbrev$object *v) -{ - /* XXXX */ -} - -static int -$abbrev$_nonzero($abbrev$object *v) -{ - /* XXXX Return 1 if non-zero */ -} - -static PyObject * -$abbrev$_invert($abbrev$object *v) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_lshift($abbrev$object *v, $abbrev$object *w) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_rshift($abbrev$object *v, $abbrev$object *w) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_and($abbrev$object *v, $abbrev$object *w) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_xor($abbrev$object *v, $abbrev$object *w) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_or($abbrev$object *v, $abbrev$object *w) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_int($abbrev$object *v) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_long($abbrev$object *v) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_float($abbrev$object *v) -{ - /* XXXX */ -} - -static PyObject * -$abbrev$_oct($abbrev$object *v) -{ - /* XXXX Return object as octal stringobject */ -} - -static PyObject * -$abbrev$_hex($abbrev$object *v) -{ - /* XXXX Return object as hex stringobject */ -} - -static PyNumberMethods $abbrev$_as_number = { - (binaryfunc)$abbrev$_add, /*nb_add*/ - (binaryfunc)$abbrev$_sub, /*nb_subtract*/ - (binaryfunc)$abbrev$_mul, /*nb_multiply*/ - (binaryfunc)$abbrev$_div, /*nb_divide*/ - (binaryfunc)$abbrev$_mod, /*nb_remainder*/ - (binaryfunc)$abbrev$_divmod, /*nb_divmod*/ - (ternaryfunc)$abbrev$_pow, /*nb_power*/ - (unaryfunc)$abbrev$_neg, /*nb_negative*/ - (unaryfunc)$abbrev$_pos, /*nb_positive*/ - (unaryfunc)$abbrev$_abs, /*nb_absolute*/ - (inquiry)$abbrev$_nonzero, /*nb_nonzero*/ - (unaryfunc)$abbrev$_invert, /*nb_invert*/ - (binaryfunc)$abbrev$_lshift, /*nb_lshift*/ - (binaryfunc)$abbrev$_rshift, /*nb_rshift*/ - (binaryfunc)$abbrev$_and, /*nb_and*/ - (binaryfunc)$abbrev$_xor, /*nb_xor*/ - (binaryfunc)$abbrev$_or, /*nb_or*/ - (coercion)$abbrev$_coerce, /*nb_coerce*/ - (unaryfunc)$abbrev$_int, /*nb_int*/ - (unaryfunc)$abbrev$_long, /*nb_long*/ - (unaryfunc)$abbrev$_float, /*nb_float*/ - (unaryfunc)$abbrev$_oct, /*nb_oct*/ - (unaryfunc)$abbrev$_hex, /*nb_hex*/ -}; - -/* ------------------------------------------------------- */ diff --git a/Tools/modulator/Templates/object_tp_as_sequence b/Tools/modulator/Templates/object_tp_as_sequence deleted file mode 100644 index 54c0b92..0000000 --- a/Tools/modulator/Templates/object_tp_as_sequence +++ /dev/null @@ -1,58 +0,0 @@ - -/* Code to handle accessing $name$ objects as sequence objects */ - -static int -$abbrev$_length($abbrev$object *self) -{ - /* XXXX Return the size of the object */ -} - -static PyObject * -$abbrev$_concat($abbrev$object *self, PyObject *bb) -{ - /* XXXX Return the concatenation of self and bb */ -} - -static PyObject * -$abbrev$_repeat($abbrev$object *self, int n) -{ - /* XXXX Return a new object that is n times self */ -} - -static PyObject * -$abbrev$_item($abbrev$object *self, int i) -{ - /* XXXX Return the i-th object of self */ -} - -static PyObject * -$abbrev$_slice($abbrev$object *self, int ilow, int ihigh) -{ - /* XXXX Return the ilow..ihigh slice of self in a new object */ -} - -static int -$abbrev$_ass_item($abbrev$object *self, int i, PyObject *v) -{ - /* XXXX Assign to the i-th element of self */ - return 0; -} - -static int -$abbrev$_ass_slice(PyListObject *self, int ilow, int ihigh, PyObject *v) -{ - /* XXXX Replace ilow..ihigh slice of self with v */ - return 0; -} - -static PySequenceMethods $abbrev$_as_sequence = { - (inquiry)$abbrev$_length, /*sq_length*/ - (binaryfunc)$abbrev$_concat, /*sq_concat*/ - (intargfunc)$abbrev$_repeat, /*sq_repeat*/ - (intargfunc)$abbrev$_item, /*sq_item*/ - (intintargfunc)$abbrev$_slice, /*sq_slice*/ - (intobjargproc)$abbrev$_ass_item, /*sq_ass_item*/ - (intintobjargproc)$abbrev$_ass_slice, /*sq_ass_slice*/ -}; - -/* -------------------------------------------------------------- */ diff --git a/Tools/modulator/Templates/object_tp_call b/Tools/modulator/Templates/object_tp_call deleted file mode 100644 index a93f17f..0000000 --- a/Tools/modulator/Templates/object_tp_call +++ /dev/null @@ -1,7 +0,0 @@ - -static PyObject * -$abbrev$_call($abbrev$object *self, PyObject *args, PyObject *kwargs) -{ - /* XXXX Return the result of calling self with argument args */ -} - diff --git a/Tools/modulator/Templates/object_tp_compare b/Tools/modulator/Templates/object_tp_compare deleted file mode 100644 index 153bae0..0000000 --- a/Tools/modulator/Templates/object_tp_compare +++ /dev/null @@ -1,6 +0,0 @@ - -static int -$abbrev$_compare($abbrev$object *v, $abbrev$object *w) -{ - /* XXXX Compare objects and return -1, 0 or 1 */ -} diff --git a/Tools/modulator/Templates/object_tp_dealloc b/Tools/modulator/Templates/object_tp_dealloc deleted file mode 100644 index 440419a..0000000 --- a/Tools/modulator/Templates/object_tp_dealloc +++ /dev/null @@ -1,7 +0,0 @@ - -static void -$abbrev$_dealloc($abbrev$object *self) -{ - /* XXXX Add your own cleanup code here */ - PyMem_DEL(self); -} diff --git a/Tools/modulator/Templates/object_tp_getattr b/Tools/modulator/Templates/object_tp_getattr deleted file mode 100644 index 6a1b2e8..0000000 --- a/Tools/modulator/Templates/object_tp_getattr +++ /dev/null @@ -1,7 +0,0 @@ - -static PyObject * -$abbrev$_getattr($abbrev$object *self, char *name) -{ - /* XXXX Add your own getattr code here */ - return Py_FindMethod($abbrev$_methods, (PyObject *)self, name); -} diff --git a/Tools/modulator/Templates/object_tp_hash b/Tools/modulator/Templates/object_tp_hash deleted file mode 100644 index 2d63f6a..0000000 --- a/Tools/modulator/Templates/object_tp_hash +++ /dev/null @@ -1,6 +0,0 @@ - -static long -$abbrev$_hash($abbrev$object *self) -{ - /* XXXX Return a hash of self (or -1) */ -} diff --git a/Tools/modulator/Templates/object_tp_print b/Tools/modulator/Templates/object_tp_print deleted file mode 100644 index 76408d2..0000000 --- a/Tools/modulator/Templates/object_tp_print +++ /dev/null @@ -1,7 +0,0 @@ - -static int -$abbrev$_print($abbrev$object *self, FILE *fp, int flags) -{ - /* XXXX Add code here to print self to fp */ - return 0; -} diff --git a/Tools/modulator/Templates/object_tp_repr b/Tools/modulator/Templates/object_tp_repr deleted file mode 100644 index f122225..0000000 --- a/Tools/modulator/Templates/object_tp_repr +++ /dev/null @@ -1,9 +0,0 @@ - -static PyObject * -$abbrev$_repr($abbrev$object *self) -{ - PyObject *s; - - /* XXXX Add code here to put self into s */ - return s; -} diff --git a/Tools/modulator/Templates/object_tp_setattr b/Tools/modulator/Templates/object_tp_setattr deleted file mode 100644 index dfe4bc8..0000000 --- a/Tools/modulator/Templates/object_tp_setattr +++ /dev/null @@ -1,9 +0,0 @@ - -static int -$abbrev$_setattr($abbrev$object *self, char *name, PyObject *v) -{ - /* Set attribute 'name' to value 'v'. v==NULL means delete */ - - /* XXXX Add your own setattr code here */ - return -1; -} diff --git a/Tools/modulator/Templates/object_tp_str b/Tools/modulator/Templates/object_tp_str deleted file mode 100644 index 2e3648e..0000000 --- a/Tools/modulator/Templates/object_tp_str +++ /dev/null @@ -1,10 +0,0 @@ - -static PyObject * -$abbrev$_str($abbrev$object *self) -{ - PyObject *s; - - /* XXXX Add code here to put self into s */ - return s; -} - diff --git a/Tools/modulator/Tkextra.py b/Tools/modulator/Tkextra.py deleted file mode 100755 index b1051aa..0000000 --- a/Tools/modulator/Tkextra.py +++ /dev/null @@ -1,234 +0,0 @@ -#! /usr/bin/env python - -# A Python function that generates dialog boxes with a text message, -# optional bitmap, and any number of buttons. -# Cf. Ousterhout, Tcl and the Tk Toolkit, Figs. 27.2-3, pp. 269-270. - -from Tkinter import * - -mainWidget = None - -def dialog(master, title, text, bitmap, default, *args): - - # 1. Create the top-level window and divide it into top - # and bottom parts. - - w = Toplevel(master, {'class': 'Dialog'}) - w.title(title) - w.iconname('Dialog') - - top = Frame(w, {'relief': 'raised', 'bd': 1, - Pack: {'side': 'top', 'fill': 'both'}}) - bot = Frame(w, {'relief': 'raised', 'bd': 1, - Pack: {'side': 'bottom', 'fill': 'both'}}) - - # 2. Fill the top part with the bitmap and message. - - msg = Message(top, - {'width': '3i', - 'text': text, - 'font': '-Adobe-Times-Medium-R-Normal-*-180-*', - Pack: {'side': 'right', 'expand': 1, - 'fill': 'both', - 'padx': '3m', 'pady': '3m'}}) - if bitmap: - bm = Label(top, {'bitmap': bitmap, - Pack: {'side': 'left', - 'padx': '3m', 'pady': '3m'}}) - - # 3. Create a row of buttons at the bottom of the dialog. - - buttons = [] - i = 0 - for but in args: - b = Button(bot, {'text': but, - 'command': ('set', 'button', i)}) - buttons.append(b) - if i == default: - bd = Frame(bot, {'relief': 'sunken', 'bd': 1, - Pack: {'side': 'left', 'expand': 1, - 'padx': '3m', 'pady': '2m'}}) - b.lift() - b.pack ({'in': bd, 'side': 'left', - 'padx': '2m', 'pady': '2m', - 'ipadx': '2m', 'ipady': '1m'}) - else: - b.pack ({'side': 'left', 'expand': 1, - 'padx': '3m', 'pady': '3m', - 'ipady': '2m', 'ipady': '1m'}) - i = i+1 - - # 4. Set up a binding for <Return>, if there's a default, - # set a grab, and claim the focus too. - - if default >= 0: - w.bind('<Return>', - lambda e, b=buttons[default], i=default: - (b.flash(), - b.setvar('button', i))) - - oldFocus = w.tk.call('focus') # XXX - w.grab_set() - w.focus() - - # 5. Wait for the user to respond, then restore the focus - # and return the index of the selected button. - - w.waitvar('button') - w.destroy() - w.tk.call('focus', oldFocus) # XXX - return w.getint(w.getvar('button')) - -def strdialog(master, title, text, bitmap, default, *args): - - # 1. Create the top-level window and divide it into top - # and bottom parts. - - w = Toplevel(master, {'class': 'Dialog'}) - w.title(title) - w.iconname('Dialog') - - top = Frame(w, {'relief': 'raised', 'bd': 1, - Pack: {'side': 'top', 'fill': 'both'}}) - if args: - bot = Frame(w, {'relief': 'raised', 'bd': 1, - Pack: {'side': 'bottom', 'fill': 'both'}}) - - # 2. Fill the top part with the bitmap, message and input field. - - if bitmap: - bm = Label(top, {'bitmap': bitmap, - Pack: {'side': 'left', - 'padx': '3m', 'pady': '3m'}}) - - msg = Message(top, - {'width': '3i', - 'text': text, - 'font': '-Adobe-Times-Medium-R-Normal-*-180-*', - Pack: {'side': 'left', - 'fill': 'both', - 'padx': '3m', 'pady': '3m'}}) - - field = Entry(top, - {'relief':'sunken', - Pack:{'side':'left', - 'fill':'x', - 'expand':1, - 'padx':'3m', 'pady':'3m'}}) - # 3. Create a row of buttons at the bottom of the dialog. - - buttons = [] - i = 0 - for but in args: - b = Button(bot, {'text': but, - 'command': ('set', 'button', i)}) - buttons.append(b) - if i == default: - bd = Frame(bot, {'relief': 'sunken', 'bd': 1, - Pack: {'side': 'left', 'expand': 1, - 'padx': '3m', 'pady': '2m'}}) - b.lift() - b.pack ({'in': bd, 'side': 'left', - 'padx': '2m', 'pady': '2m', - 'ipadx': '2m', 'ipady': '1m'}) - else: - b.pack ({'side': 'left', 'expand': 1, - 'padx': '3m', 'pady': '3m', - 'ipady': '2m', 'ipady': '1m'}) - i = i+1 - - # 4. Set up a binding for <Return>, if there's a default, - # set a grab, and claim the focus too. - - if not args: - w.bind('<Return>', lambda arg, top=top: top.setvar('button', 0)) - field.bind('<Return>', lambda arg, top=top: top.setvar('button', 0)) - elif default >= 0: - w.bind('<Return>', - lambda e, b=buttons[default], i=default: - (b.flash(), - b.setvar('button', i))) - field.bind('<Return>', - lambda e, b=buttons[default], i=default: - (b.flash(), - b.setvar('button', i))) - - oldFocus = w.tk.call('focus') # XXX - w.grab_set() - field.focus() - - # 5. Wait for the user to respond, then restore the focus - # and return the index of the selected button. - - w.waitvar('button') - v = field.get() - w.destroy() - w.tk.call('focus', oldFocus) # XXX - if args: - return v, w.getint(w.getvar('button')) - else: - return v - -def message(str): - i = dialog(mainWidget, 'Message', str, '', 0, 'OK') - -def askyn(str): - i = dialog(mainWidget, 'Question', str, '', 0, 'No', 'Yes') - return i - -def askync(str): - i = dialog(mainWidget, 'Question', str, '', 0, 'Cancel', 'No', 'Yes') - return i-1 - -def askstr(str): - i = strdialog(mainWidget, 'Question', str, '', 0) - return i - -def askfile(str): # XXXX For now... - i = strdialog(mainWidget, 'Question', str, '', 0) - return i - -# The rest is the test program. - -def _go(): - i = dialog(mainWidget, - 'Not Responding', - "The file server isn't responding right now; " - "I'll keep trying.", - '', - -1, - 'OK') - print('pressed button', i) - i = dialog(mainWidget, - 'File Modified', - 'File "tcl.h" has been modified since ' - 'the last time it was saved. ' - 'Do you want to save it before exiting the application?', - 'warning', - 0, - 'Save File', - 'Discard Changes', - 'Return To Editor') - print('pressed button', i) - print(message('Test of message')) - print(askyn('Test of yes/no')) - print(askync('Test of yes/no/cancel')) - print(askstr('Type a string:')) - print(strdialog(mainWidget, 'Question', 'Another string:', '', - 0, 'Save', 'Save as text')) - -def _test(): - global mainWidget - mainWidget = Frame() - Pack.config(mainWidget) - start = Button(mainWidget, - {'text': 'Press Here To Start', 'command': _go}) - start.pack() - endit = Button(mainWidget, - {'text': 'Exit', - 'command': 'exit', - Pack: {'fill' : 'both'}}) - mainWidget.mainloop() - -if __name__ == '__main__': - _test() diff --git a/Tools/modulator/genmodule.py b/Tools/modulator/genmodule.py deleted file mode 100755 index f8703ef..0000000 --- a/Tools/modulator/genmodule.py +++ /dev/null @@ -1,160 +0,0 @@ -# -# Genmodule - A python program to help you build (template) modules. -# -# Usage: -# -# o = genmodule.object() -# o.name = 'dwarve object' -# o.abbrev = 'dw' -# o.funclist = ['new', 'dealloc', 'getattr', 'setattr'] -# o.methodlist = ['dig'] -# -# m = genmodule.module() -# m.name = 'beings' -# m.abbrev = 'be' -# m.methodlist = ['newdwarve'] -# m.objects = [o] -# -# genmodule.write(sys.stdout, m) -# -import sys -import os -import varsubst - -error = 'genmodule.error' - -# -# Names of functions in the object-description struct. -# -FUNCLIST = ['new', 'tp_dealloc', 'tp_print', 'tp_getattr', 'tp_setattr', - 'tp_compare', 'tp_repr', 'tp_hash', 'tp_call', 'tp_str'] -TYPELIST = ['tp_as_number', 'tp_as_sequence', 'tp_as_mapping', 'structure'] - -# -# writer is a base class for the object and module classes -# it contains code common to both. -# -class writer: - def __init__(self): - self._subst = None - - def makesubst(self): - if not self._subst: - if 'abbrev' not in self.__dict__: - self.abbrev = self.name - self.Abbrev = self.abbrev[0].upper()+self.abbrev[1:] - subst = varsubst.Varsubst(self.__dict__) - subst.useindent(1) - self._subst = subst.subst - - def addcode(self, name, fp): - ifp = self.opentemplate(name) - self.makesubst() - d = ifp.read() - d = self._subst(d) - fp.write(d) - - def opentemplate(self, name): - for p in sys.path: - fn = os.path.join(p, name) - if os.path.exists(fn): - return open(fn, 'r') - fn = os.path.join(p, 'Templates') - fn = os.path.join(fn, name) - if os.path.exists(fn): - return open(fn, 'r') - raise error('Template '+name+' not found for '+self._type+' '+ \ - self.name) - -class module(writer): - _type = 'module' - - def writecode(self, fp): - self.addcode('copyright', fp) - self.addcode('module_head', fp) - for o in self.objects: - o.writehead(fp) - for o in self.objects: - o.writebody(fp) - new_ml = '' - for fn in self.methodlist: - self.method = fn - self.addcode('module_method', fp) - new_ml = new_ml + ( - '{"%s",\t(PyCFunction)%s_%s,\tMETH_VARARGS,\t%s_%s__doc__},\n' - %(fn, self.abbrev, fn, self.abbrev, fn)) - self.methodlist = new_ml - self.addcode('module_tail', fp) - -class object(writer): - _type = 'object' - def __init__(self): - self.typelist = [] - self.methodlist = [] - self.funclist = ['new'] - writer.__init__(self) - - def writecode(self, fp): - self.addcode('copyright', fp) - self.writehead(fp) - self.writebody(fp) - - def writehead(self, fp): - self.addcode('object_head', fp) - - def writebody(self, fp): - new_ml = '' - for fn in self.methodlist: - self.method = fn - self.addcode('object_method', fp) - new_ml = new_ml + ( - '{"%s",\t(PyCFunction)%s_%s,\tMETH_VARARGS,\t%s_%s__doc__},\n' - %(fn, self.abbrev, fn, self.abbrev, fn)) - self.methodlist = new_ml - self.addcode('object_mlist', fp) - - # Add getattr if we have methods - if self.methodlist and not 'tp_getattr' in self.funclist: - self.funclist.insert(0, 'tp_getattr') - - for fn in FUNCLIST: - setattr(self, fn, '0') - - # - # Special case for structure-access objects: put getattr in the - # list of functions but don't generate code for it directly, - # the code is obtained from the object_structure template. - # The same goes for setattr. - # - if 'structure' in self.typelist: - if 'tp_getattr' in self.funclist: - self.funclist.remove('tp_getattr') - if 'tp_setattr' in self.funclist: - self.funclist.remove('tp_setattr') - self.tp_getattr = self.abbrev + '_getattr' - self.tp_setattr = self.abbrev + '_setattr' - for fn in self.funclist: - self.addcode('object_'+fn, fp) - setattr(self, fn, '%s_%s'%(self.abbrev, fn[3:])) - for tn in TYPELIST: - setattr(self, tn, '0') - for tn in self.typelist: - self.addcode('object_'+tn, fp) - setattr(self, tn, '&%s_%s'%(self.abbrev, tn[3:])) - self.addcode('object_tail', fp) - -def write(fp, obj): - obj.writecode(fp) - -if __name__ == '__main__': - o = object() - o.name = 'dwarve object' - o.abbrev = 'dw' - o.funclist = ['new', 'tp_dealloc'] - o.methodlist = ['dig'] - m = module() - m.name = 'beings' - m.abbrev = 'be' - m.methodlist = ['newdwarve'] - m.objects = [o] - write(sys.stdout, m) diff --git a/Tools/modulator/modulator.py b/Tools/modulator/modulator.py deleted file mode 100755 index d719649..0000000 --- a/Tools/modulator/modulator.py +++ /dev/null @@ -1,383 +0,0 @@ -#! /usr/bin/env python -# -# Modulator - Generate skeleton modules. -# -# The user fills out some forms with information about what the module -# should support (methods, objects), names of these things, prefixes to -# use for C code, whether the objects should also support access as numbers, -# etc etc etc. -# When the user presses 'Generate code' we generate a complete skeleton -# module in C. -# -# Alternatively, the selections made can be save to a python sourcefile and -# this sourcefile can be passed on the command line (resulting in the same -# skeleton C code). -# -# Jack Jansen, CWI, October 1994. -# - -import sys, os -if os.name != 'mac': - sys.path.append(os.path.join(os.environ['HOME'], - 'src/python/Tools/modulator')) - -from Tkinter import * -from Tkextra import * -from ScrolledListbox import ScrolledListbox -import sys -import genmodule -import string - -oops = 'oops' - -IDENTSTARTCHARS = string.ascii_letters + '_' -IDENTCHARS = string.ascii_letters + string.digits + '_' - -# Check that string is a legal C identifier -def checkid(str): - if not str: return 0 - if not str[0] in IDENTSTARTCHARS: - return 0 - for c in str[1:]: - if not c in IDENTCHARS: - return 0 - return 1 - -def getlistlist(list): - rv = [] - n = list.size() - for i in range(n): - rv.append(list.get(i)) - return rv - -class UI: - def __init__(self): - self.main = Frame() - self.main.pack() - self.main.master.title('Modulator: Module view') - self.cmdframe = Frame(self.main, {'relief':'raised', 'bd':'0.5m', - Pack:{'side':'top', - 'fill':'x'}}) - self.objframe = Frame(self.main, {Pack:{'side':'top', 'fill':'x', - 'expand':1}}) - - - self.check_button = Button(self.cmdframe, - {'text':'Check', 'command':self.cb_check, - Pack:{'side':'left', 'padx':'0.5m'}}) - self.save_button = Button(self.cmdframe, - {'text':'Save...', 'command':self.cb_save, - Pack:{'side':'left', 'padx':'0.5m'}}) - self.code_button = Button(self.cmdframe, - {'text':'Generate code...', - 'command':self.cb_gencode, - Pack:{'side':'left', 'padx':'0.5m'}}) - self.quit_button = Button(self.cmdframe, - {'text':'Quit', - 'command':self.cb_quit, - Pack:{'side':'right', 'padx':'0.5m'}}) - - self.module = UI_module(self) - self.objects = [] - self.modified = 0 - - def run(self): - self.main.mainloop() - - def cb_quit(self, *args): - if self.modified: - if not askyn('You have not saved\nAre you sure you want to quit?'): - return - sys.exit(0) - - def cb_check(self, *args): - try: - self.module.synchronize() - for o in self.objects: - o.synchronize() - except oops: - pass - - def cb_save(self, *args): - try: - pycode = self.module.gencode('m', self.objects) - except oops: - return - - fn = askfile('Python file name: ') - if not fn: - return - - fp = open(fn, 'w') - - fp.write(pycode) - fp.close() - - def cb_gencode(self, *args): - try: - pycode = self.module.gencode('m', self.objects) - except oops: - pass - - fn = askfile('C file name: ') - if not fn: - return - - fp = open(fn, 'w') - - try: - exec(pycode) - except: - message('An error occurred:-)') - return - genmodule.write(fp, m) - fp.close() - -class UI_module: - def __init__(self, parent): - self.parent = parent - self.frame = Frame(parent.objframe, {'relief':'raised', 'bd':'0.2m', - Pack:{'side':'top', - 'fill':'x'}}) - self.f1 = Frame(self.frame, {Pack:{'side':'top', 'pady':'0.5m', - 'fill':'x'}}) - self.f2 = Frame(self.frame, {Pack:{'side':'top', 'pady':'0.5m', - 'fill':'x'}}) - self.f3 = Frame(self.frame, {Pack:{'side':'top', 'pady':'0.5m', - 'fill':'x'}}) - self.f4 = Frame(self.frame, {Pack:{'side':'top', 'pady':'0.5m', - 'fill':'x'}}) - - self.l1 = Label(self.f1, {'text':'Module:', Pack:{'side':'left', - 'padx':'0.5m'}}) - self.name_entry = Entry(self.f1, {'relief':'sunken', - Pack:{'side':'left', 'padx':'0.5m', 'expand':1}}) - self.l2 = Label(self.f1, {'text':'Abbrev:', Pack:{'side':'left', - 'padx':'0.5m'}}) - self.abbrev_entry = Entry(self.f1, {'relief':'sunken', 'width':5, - Pack:{'side':'left', 'padx':'0.5m'}}) - - self.l3 = Label(self.f2, {'text':'Methods:', Pack:{'side':'left', - 'padx':'0.5m'}}) - self.method_list = ScrolledListbox(self.f2, {'relief':'sunken','bd':2, - Pack:{'side':'left', 'expand':1, - 'padx':'0.5m', 'fill':'both'}}) - - self.l4 = Label(self.f3, {'text':'Add method:', Pack:{'side':'left', - 'padx':'0.5m'}}) - self.method_entry = Entry(self.f3, {'relief':'sunken', - Pack:{'side':'left', 'padx':'0.5m', 'expand':1}}) - self.method_entry.bind('<Return>', self.cb_method) - self.delete_button = Button(self.f3, {'text':'Delete method', - 'command':self.cb_delmethod, - Pack:{'side':'left', - 'padx':'0.5m'}}) - - self.newobj_button = Button(self.f4, {'text':'new object', - 'command':self.cb_newobj, - Pack:{'side':'left', - 'padx':'0.5m'}}) - - def cb_delmethod(self, *args): - list = self.method_list.curselection() - for i in list: - self.method_list.delete(i) - - def cb_newobj(self, *arg): - self.parent.objects.append(UI_object(self.parent)) - - def cb_method(self, *arg): - name = self.method_entry.get() - if not name: - return - self.method_entry.delete('0', 'end') - self.method_list.insert('end', name) - - def synchronize(self): - n = self.name_entry.get() - if not n: - message('Module name not set') - raise oops - if not checkid(n): - message('Module name not an identifier:\n'+n) - raise oops - if not self.abbrev_entry.get(): - self.abbrev_entry.insert('end', n) - m = getlistlist(self.method_list) - for n in m: - if not checkid(n): - message('Method name not an identifier:\n'+n) - raise oops - - def gencode(self, name, objects): - rv = '' - self.synchronize() - for o in objects: - o.synchronize() - onames = [] - for i in range(len(objects)): - oname = 'o%d' % (i+1) - rv = rv + objects[i].gencode(oname) - onames.append(oname) - rv = rv + '%s = genmodule.module()\n' % (name,) - rv = rv + '%s.name = %r\n' % (name, self.name_entry.get()) - rv = rv + '%s.abbrev = %r\n' % (name, self.abbrev_entry.get()) - rv = rv + '%s.methodlist = %r\n' % (name, getlistlist(self.method_list)) - rv = rv + '%s.objects = [%s]\n' % (name, ','.join(onames)) - rv = rv + '\n' - return rv - -object_number = 0 - -class UI_object: - def __init__(self, parent): - global object_number - - object_number = object_number + 1 - self.num = object_number - self.vpref = 'o%r_' % self.num - self.frame = Toplevel(parent.objframe) -# self.frame.pack() - self.frame.title('Modulator: object view') -# self.frame = Frame(parent.objframe, {'relief':'raised', 'bd':'0.2m', -# Pack:{'side':'top', -# 'fill':'x'}}) - self.f1 = Frame(self.frame, {Pack:{'side':'top', 'pady':'0.5m', - 'fill':'x'}}) - self.f2 = Frame(self.frame, {Pack:{'side':'top', 'pady':'0.5m', - 'fill':'x'}}) - self.f3 = Frame(self.frame, {Pack:{'side':'top', 'pady':'0.5m', - 'fill':'x'}}) - self.f4 = Frame(self.frame, {Pack:{'side':'top', 'pady':'0.5m', - 'fill':'x'}}) - - - self.l1 = Label(self.f1, {'text':'Object:', Pack:{'side':'left', - 'padx':'0.5m'}}) - self.name_entry = Entry(self.f1, {'relief':'sunken', - Pack:{'side':'left', 'padx':'0.5m', 'expand':1}}) - self.l2 = Label(self.f1, {'text':'Abbrev:', Pack:{'side':'left', - 'padx':'0.5m'}}) - self.abbrev_entry = Entry(self.f1, {'relief':'sunken', 'width':5, - Pack:{'side':'left', 'padx':'0.5m'}}) - - self.l3 = Label(self.f2, {'text':'Methods:', Pack:{'side':'left', - 'padx':'0.5m'}}) - self.method_list = ScrolledListbox(self.f2, {'relief':'sunken','bd':2, - Pack:{'side':'left', 'expand':1, - 'padx':'0.5m', 'fill':'both'}}) - - self.l4 = Label(self.f3, {'text':'Add method:', Pack:{'side':'left', - 'padx':'0.5m'}}) - self.method_entry = Entry(self.f3, {'relief':'sunken', - Pack:{'side':'left', 'padx':'0.5m', 'expand':1}}) - self.method_entry.bind('<Return>', self.cb_method) - self.delete_button = Button(self.f3, {'text':'Delete method', - 'command':self.cb_delmethod, - Pack:{'side':'left', - 'padx':'0.5m'}}) - - - self.l5 = Label(self.f4, {'text':'functions:', - Pack:{'side':'left', - 'padx':'0.5m'}}) - self.f5 = Frame(self.f4, {Pack:{'side':'left', 'pady':'0.5m', - 'fill':'both'}}) - self.l6 = Label(self.f4, {'text':'Types:', - Pack:{'side':'left', 'padx':'0.5m'}}) - self.f6 = Frame(self.f4, {Pack:{'side':'left', 'pady':'0.5m', - 'fill':'x'}}) - self.funcs = {} - for i in genmodule.FUNCLIST: - vname = self.vpref+i - self.f5.setvar(vname, 0) - b = Checkbutton(self.f5, {'variable':vname, 'text':i, - Pack:{'side':'top', 'pady':'0.5m', - 'anchor':'w','expand':1}}) - self.funcs[i] = b - self.f5.setvar(self.vpref+'new', 1) - - self.types = {} - for i in genmodule.TYPELIST: - vname = self.vpref + i - self.f6.setvar(vname, 0) - b = Checkbutton(self.f6, {'variable':vname, 'text':i, - Pack:{'side':'top', 'pady':'0.5m', - 'anchor':'w'}}) - self.types[i] = b - - def cb_method(self, *arg): - name = self.method_entry.get() - if not name: - return - self.method_entry.delete('0', 'end') - self.method_list.insert('end', name) - - def cb_delmethod(self, *args): - list = self.method_list.curselection() - for i in list: - self.method_list.delete(i) - - def synchronize(self): - n = self.name_entry.get() - if not n: - message('Object name not set') - raise oops - if not self.abbrev_entry.get(): - self.abbrev_entry.insert('end', n) - n = self.abbrev_entry.get() - if not checkid(n): - message('Abbreviation not an identifier:\n'+n) - raise oops - m = getlistlist(self.method_list) - for n in m: - if not checkid(n): - message('Method name not an identifier:\n'+n) - raise oops - if m: - self.f5.setvar(self.vpref+'tp_getattr', 1) - pass - - def gencode(self, name): - rv = '' - rv = rv + '%s = genmodule.object()\n' % (name,) - rv = rv + '%s.name = %r\n' % (name, self.name_entry.get()) - rv = rv + '%s.abbrev = %r\n' % (name, self.abbrev_entry.get()) - rv = rv + '%s.methodlist = %r\n' % (name, getlistlist(self.method_list)) - fl = [] - for fn in genmodule.FUNCLIST: - vname = self.vpref + fn - if self.f5.getvar(vname) == '1': - fl.append(fn) - rv = rv + '%s.funclist = %r\n' % (name, fl) - - fl = [] - for fn in genmodule.TYPELIST: - vname = self.vpref + fn - if self.f5.getvar(vname) == '1': - fl.append(fn) - - rv = rv + '%s.typelist = %r\n' % (name, fl) - - rv = rv + '\n' - return rv - - -def main(): - if len(sys.argv) < 2: - ui = UI() - ui.run() - elif len(sys.argv) == 2: - fp = open(sys.argv[1]) - pycode = fp.read() - try: - exec(pycode) - except: - sys.stderr.write('An error occurred:-)\n') - sys.exit(1) - ##genmodule.write(sys.stdout, m) - else: - sys.stderr.write('Usage: modulator [file]\n') - sys.exit(1) - -main() diff --git a/Tools/modulator/varsubst.py b/Tools/modulator/varsubst.py deleted file mode 100644 index 4b68512..0000000 --- a/Tools/modulator/varsubst.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Variable substitution. Variables are $delimited$ -# -import re - -error = 'varsubst.error' - -class Varsubst: - def __init__(self, dict): - self.dict = dict - self.prog = re.compile('\$([a-zA-Z0-9_]*)\$') - self.do_useindent = 0 - - def useindent(self, onoff): - self.do_useindent = onoff - - def subst(self, s): - rv = '' - while 1: - m = self.prog.search(s) - if not m: - return rv + s - rv = rv + s[:m.start()] - s = s[m.end():] - if m.end() - m.start() == 2: - # Escaped dollar - rv = rv + '$' - s = s[2:] - continue - name = m.group(1) - if name not in self.dict: - raise error('No such variable: '+name) - value = self.dict[name] - if self.do_useindent and '\n' in value: - value = self._modindent(value, rv) - rv = rv + value - - def _modindent(self, value, old): - lastnl = old.rfind('\n', 0) + 1 - lastnl = len(old) - lastnl - sub = '\n' + (' '*lastnl) - return re.sub('\n', sub, value) - -def _test(): - import sys - import os - - sys.stderr.write('-- Copying stdin to stdout with environment map --\n') - c = Varsubst(os.environ) - c.useindent(1) - d = sys.stdin.read() - sys.stdout.write(c.subst(d)) - sys.exit(1) - -if __name__ == '__main__': - _test() diff --git a/Tools/msi/merge.py b/Tools/msi/merge.py deleted file mode 100644 index 85de209..0000000 --- a/Tools/msi/merge.py +++ /dev/null @@ -1,84 +0,0 @@ -import msilib,os,win32com,tempfile,sys -PCBUILD="PCBuild" -certname = None -from config import * - -Win64 = "amd64" in PCBUILD - -mod_dir = os.path.join(os.environ["ProgramFiles"], "Common Files", "Merge Modules") -msi = None -if len(sys.argv)==2: - msi = sys.argv[1] -if Win64: - modules = ["Microsoft_VC90_CRT_x86_x64.msm", "policy_9_0_Microsoft_VC90_CRT_x86_x64.msm"] - if not msi: msi = "python-%s.amd64.msi" % full_current_version -else: - modules = ["Microsoft_VC90_CRT_x86.msm","policy_9_0_Microsoft_VC90_CRT_x86.msm"] - if not msi: msi = "python-%s.msi" % full_current_version -for i, n in enumerate(modules): - modules[i] = os.path.join(mod_dir, n) - -def merge(msi, feature, rootdir, modules): - cab_and_filecount = [] - # Step 1: Merge databases, extract cabfiles - m = msilib.MakeMerge2() - m.OpenLog("merge.log") - print "Opened Log" - m.OpenDatabase(msi) - print "Opened DB" - for module in modules: - print module - m.OpenModule(module,0) - print "Opened Module",module - m.Merge(feature, rootdir) - print "Errors:" - for e in m.Errors: - print e.Type, e.ModuleTable, e.DatabaseTable - print " Modkeys:", - for s in e.ModuleKeys: print s, - print - print " DBKeys:", - for s in e.DatabaseKeys: print s, - print - cabname = tempfile.mktemp(suffix=".cab") - m.ExtractCAB(cabname) - cab_and_filecount.append((cabname, len(m.ModuleFiles))) - m.CloseModule() - m.CloseDatabase(True) - m.CloseLog() - - # Step 2: Add CAB files - i = msilib.MakeInstaller() - db = i.OpenDatabase(msi, win32com.client.constants.msiOpenDatabaseModeTransact) - - v = db.OpenView("SELECT LastSequence FROM Media") - v.Execute(None) - maxmedia = -1 - while 1: - r = v.Fetch() - if not r: break - seq = r.IntegerData(1) - if seq > maxmedia: - maxmedia = seq - print "Start of Media", maxmedia - - for cabname, count in cab_and_filecount: - stream = "merged%d" % maxmedia - msilib.add_data(db, "Media", - [(maxmedia+1, maxmedia+count, None, "#"+stream, None, None)]) - msilib.add_stream(db, stream, cabname) - os.unlink(cabname) - maxmedia += count - # The merge module sets ALLUSERS to 1 in the property table. - # This is undesired; delete that - v = db.OpenView("DELETE FROM Property WHERE Property='ALLUSERS'") - v.Execute(None) - v.Close() - db.Commit() - -merge(msi, "SharedCRT", "TARGETDIR", modules) - -# certname (from config.py) should be (a substring of) -# the certificate subject, e.g. "Python Software Foundation" -if certname: - os.system('signtool sign /n "%s" /t http://timestamp.verisign.com/scripts/timestamp.dll %s' % (certname, msi)) diff --git a/Tools/msi/msi.py b/Tools/msi/msi.py index b668c7a..53e652d 100644 --- a/Tools/msi/msi.py +++ b/Tools/msi/msi.py @@ -7,6 +7,7 @@ import uisample from win32com.client import constants from distutils.spawn import find_executable from uuids import product_codes +import tempfile # Settings can be overridden in config.py below # 0 for official python.org releases @@ -28,6 +29,8 @@ have_tcl = True PCBUILD="PCbuild" # msvcrt version MSVCR = "90" +# Name of certificate in default store to sign MSI with +certname = None # Make a zip file containing the PDB files for this build? pdbzip = True @@ -115,6 +118,7 @@ pythondll_uuid = { "27":"{4fe21c76-1760-437b-a2f2-99909130a175}", "30":"{6953bc3b-6768-4291-8410-7914ce6e2ca8}", "31":"{4afcba0b-13e4-47c3-bebe-477428b46913}", + "32":"{3ff95315-1096-4d31-bd86-601d5438ad5e}", } [major+minor] # Compute the name that Sphinx gives to the docfile @@ -221,7 +225,8 @@ def build_database(): # schema represents the installer 2.0 database schema. # sequence is the set of standard sequences # (ui/execute, admin/advt/install) - db = msilib.init_database("python-%s%s.msi" % (full_current_version, msilib.arch_ext), + msiname = "python-%s%s.msi" % (full_current_version, msilib.arch_ext) + db = msilib.init_database(msiname, schema, ProductName="Python "+full_current_version+productsuffix, ProductCode=product_code, ProductVersion=current_version, @@ -244,7 +249,7 @@ def build_database(): ("ProductLine", "Python%s%s" % (major, minor)), ]) db.Commit() - return db + return db, msiname def remove_old_versions(db): "Fill the upgrade table." @@ -876,7 +881,6 @@ def generate_license(): shutil.copyfileobj(open(os.path.join(srcdir, "LICENSE")), out) shutil.copyfileobj(open("crtlicense.txt"), out) for name, pat, file in (("bzip2","bzip2-*", "LICENSE"), - ("Berkeley DB", "db-*", "LICENSE"), ("openssl", "openssl-*", "LICENSE"), ("Tcl", "tcl8*", "license.terms"), ("Tk", "tk8*", "license.terms"), @@ -900,6 +904,13 @@ class PyDirectory(Directory): kw['componentflags'] = 2 #msidbComponentAttributesOptional Directory.__init__(self, *args, **kw) + def check_unpackaged(self): + self.unpackaged_files.discard('__pycache__') + self.unpackaged_files.discard('.svn') + if self.unpackaged_files: + print "Warning: Unpackaged files in %s" % self.absolute + print self.unpackaged_files + # See "File Table", "Component Table", "Directory Table", # "FeatureComponents Table" def add_files(db): @@ -963,13 +974,13 @@ def add_files(db): extensions.remove("_ctypes.pyd") # Add all .py files in Lib, except tkinter, test - dirs={} + dirs = [] pydirs = [(root,"Lib")] while pydirs: # Commit every now and then, or else installer will complain db.Commit() parent, dir = pydirs.pop() - if dir == ".svn" or dir.startswith("plat-"): + if dir == ".svn" or dir == '__pycache__' or dir.startswith("plat-"): continue elif dir in ["tkinter", "idlelib", "Icons"]: if not have_tcl: @@ -987,7 +998,7 @@ def add_files(db): default_feature.set_current() lib = PyDirectory(db, cab, parent, dir, dir, "%s|%s" % (parent.make_short(dir), dir)) # Add additional files - dirs[dir]=lib + dirs.append(lib) lib.glob("*.txt") if dir=='site-packages': lib.add_file("README.txt", src="README") @@ -997,19 +1008,14 @@ def add_files(db): if files: # Add an entry to the RemoveFile table to remove bytecode files. lib.remove_pyc() - if dir.endswith('.egg-info'): - lib.add_file('entry_points.txt') - lib.add_file('PKG-INFO') - lib.add_file('top_level.txt') - lib.add_file('zip-safe') - continue + # package READMEs if present + lib.glob("README") + if dir=='Lib': + lib.add_file('wsgiref.egg-info') if dir=='test' and parent.physical=='Lib': lib.add_file("185test.db") lib.add_file("audiotest.au") - lib.add_file("cfgparser.1") lib.add_file("sgml_input.html") - lib.add_file("test.xml") - lib.add_file("test.xml.out") lib.add_file("testtar.tar") lib.add_file("test_difflib_expect.html") lib.add_file("check_soundcard.vbs") @@ -1018,26 +1024,41 @@ def add_files(db): lib.glob("*.uue") lib.glob("*.pem") lib.glob("*.pck") + lib.glob("cfgparser.*") + lib.add_file("zip_cp437_header.zip") lib.add_file("zipdir.zip") + if dir=='capath': + lib.glob("*.0") if dir=='tests' and parent.physical=='distutils': lib.add_file("Setup.sample") if dir=='decimaltestdata': lib.glob("*.decTest") + if dir=='xmltestdata': + lib.glob("*.xml") + lib.add_file("test.xml.out") if dir=='output': lib.glob("test_*") + if dir=='sndhdrdata': + lib.glob("sndhdr.*") if dir=='idlelib': lib.glob("*.def") lib.add_file("idle.bat") + lib.add_file("ChangeLog") if dir=="Icons": lib.glob("*.gif") lib.add_file("idle.icns") if dir=="command" and parent.physical=="distutils": lib.glob("wininst*.exe") - if dir=="setuptools": - lib.add_file("cli.exe") - lib.add_file("gui.exe") + lib.add_file("command_template") if dir=="lib2to3": lib.removefile("pickle", "*.pickle") + if dir=="macholib": + lib.add_file("README.ctypes") + lib.glob("fetch_macholib*") + if dir=='turtledemo': + lib.add_file("turtle.cfg") + if dir=="pydoc_data": + lib.add_file("_pydoc.css") if dir=="data" and parent.physical=="test" and parent.basedir.physical=="email": # This should contain all non-.svn files listed in subversion for f in os.listdir(lib.absolute): @@ -1049,6 +1070,8 @@ def add_files(db): for f in os.listdir(lib.absolute): if os.path.isdir(os.path.join(lib.absolute, f)): pydirs.append((lib, f)) + for d in dirs: + d.check_unpackaged() # Add DLLs default_feature.set_current() lib = DLLs @@ -1064,6 +1087,7 @@ def add_files(db): continue dlls.append(f) lib.add_file(f) + lib.add_file('python3.dll') # Add sqlite if msilib.msi_type=="Intel64;1033": sqlite_arch = "/ia64" @@ -1100,6 +1124,7 @@ def add_files(db): for f in dlls: lib.add_file(f.replace('pyd','lib')) lib.add_file('python%s%s.lib' % (major, minor)) + lib.add_file('python3.lib') # Add the mingw-format library if have_mingw: lib.add_file('libpython%s%s.a' % (major, minor)) @@ -1120,7 +1145,7 @@ def add_files(db): # Add tools tools.set_current() tooldir = PyDirectory(db, cab, root, "Tools", "Tools", "TOOLS|Tools") - for f in ['i18n', 'pynche', 'Scripts', 'webchecker']: + for f in ['i18n', 'pynche', 'Scripts']: lib = PyDirectory(db, cab, tooldir, f, f, "%s|%s" % (tooldir.make_short(f), f)) lib.glob("*.py") lib.glob("*.pyw", exclude=['pydocgui.pyw']) @@ -1307,7 +1332,7 @@ def build_pdbzip(): pdbzip.write(os.path.join(srcdir, PCBUILD, f), f) pdbzip.close() -db = build_database() +db,msiname = build_database() try: add_features(db) add_ui(db) @@ -1318,5 +1343,77 @@ try: finally: del db +# Merge CRT into MSI file. This requires the database to be closed. +mod_dir = os.path.join(os.environ["ProgramFiles"], "Common Files", "Merge Modules") +if msilib.Win64: + modules = ["Microsoft_VC90_CRT_x86_x64.msm", "policy_9_0_Microsoft_VC90_CRT_x86_x64.msm"] +else: + modules = ["Microsoft_VC90_CRT_x86.msm","policy_9_0_Microsoft_VC90_CRT_x86.msm"] + +for i, n in enumerate(modules): + modules[i] = os.path.join(mod_dir, n) + +def merge(msi, feature, rootdir, modules): + cab_and_filecount = [] + # Step 1: Merge databases, extract cabfiles + m = msilib.MakeMerge2() + m.OpenLog("merge.log") + m.OpenDatabase(msi) + for module in modules: + print module + m.OpenModule(module,0) + m.Merge(feature, rootdir) + print "Errors:" + for e in m.Errors: + print e.Type, e.ModuleTable, e.DatabaseTable + print " Modkeys:", + for s in e.ModuleKeys: print s, + print + print " DBKeys:", + for s in e.DatabaseKeys: print s, + print + cabname = tempfile.mktemp(suffix=".cab") + m.ExtractCAB(cabname) + cab_and_filecount.append((cabname, len(m.ModuleFiles))) + m.CloseModule() + m.CloseDatabase(True) + m.CloseLog() + + # Step 2: Add CAB files + i = msilib.MakeInstaller() + db = i.OpenDatabase(msi, constants.msiOpenDatabaseModeTransact) + + v = db.OpenView("SELECT LastSequence FROM Media") + v.Execute(None) + maxmedia = -1 + while 1: + r = v.Fetch() + if not r: break + seq = r.IntegerData(1) + if seq > maxmedia: + maxmedia = seq + print "Start of Media", maxmedia + + for cabname, count in cab_and_filecount: + stream = "merged%d" % maxmedia + msilib.add_data(db, "Media", + [(maxmedia+1, maxmedia+count, None, "#"+stream, None, None)]) + msilib.add_stream(db, stream, cabname) + os.unlink(cabname) + maxmedia += count + # The merge module sets ALLUSERS to 1 in the property table. + # This is undesired; delete that + v = db.OpenView("DELETE FROM Property WHERE Property='ALLUSERS'") + v.Execute(None) + v.Close() + db.Commit() + +merge(msiname, "SharedCRT", "TARGETDIR", modules) + +# certname (from config.py) should be (a substring of) +# the certificate subject, e.g. "Python Software Foundation" +if certname: + os.system('signtool sign /n "%s" /t http://timestamp.verisign.com/scripts/timestamp.dll %s' % (certname, msiname)) + if pdbzip: build_pdbzip() diff --git a/Tools/msi/msilib.py b/Tools/msi/msilib.py index 6f49b4c..5795d0e 100644 --- a/Tools/msi/msilib.py +++ b/Tools/msi/msilib.py @@ -5,7 +5,7 @@ import win32com.client.gencache import win32com.client import pythoncom, pywintypes from win32com.client import constants -import re, string, os, sets, glob, subprocess, sys, _winreg, struct +import re, string, os, sets, glob, subprocess, sys, _winreg, struct, _msi try: basestring @@ -350,7 +350,7 @@ def gen_uuid(): class CAB: def __init__(self, name): self.name = name - self.file = open(name+".txt", "wt") + self.files = [] self.filenames = sets.Set() self.index = 0 @@ -369,51 +369,18 @@ class CAB: if not logical: logical = self.gen_id(dir, file) self.index += 1 - if full.find(" ")!=-1: - print >>self.file, '"%s" %s' % (full, logical) - else: - print >>self.file, '%s %s' % (full, logical) + self.files.append((full, logical)) return self.index, logical def commit(self, db): - self.file.close() try: os.unlink(self.name+".cab") except OSError: pass - for k, v in [(r"Software\Microsoft\VisualStudio\7.1\Setup\VS", "VS7CommonBinDir"), - (r"Software\Microsoft\VisualStudio\8.0\Setup\VS", "VS7CommonBinDir"), - (r"Software\Microsoft\VisualStudio\9.0\Setup\VS", "VS7CommonBinDir"), - (r"Software\Microsoft\Win32SDK\Directories", "Install Dir"), - ]: - try: - key = _winreg.OpenKey(_winreg.HKEY_LOCAL_MACHINE, k) - dir = _winreg.QueryValueEx(key, v)[0] - _winreg.CloseKey(key) - except (WindowsError, IndexError): - continue - cabarc = os.path.join(dir, r"Bin", "cabarc.exe") - if not os.path.exists(cabarc): - continue - break - else: - print "WARNING: cabarc.exe not found in registry" - cabarc = "cabarc.exe" - cmd = r'"%s" -m lzx:21 n %s.cab @%s.txt' % (cabarc, self.name, self.name) - p = subprocess.Popen(cmd, shell=True, stdin=subprocess.PIPE, - stdout=subprocess.PIPE, stderr=subprocess.STDOUT) - for line in p.stdout: - if line.startswith(" -- adding "): - sys.stdout.write(".") - else: - sys.stdout.write(line) - sys.stdout.flush() - if not os.path.exists(self.name+".cab"): - raise IOError, "cabarc failed" + _msi.FCICreate(self.name+".cab", self.files) add_data(db, "Media", [(1, self.index, None, "#"+self.name, None, None)]) add_stream(db, self.name, self.name+".cab") - os.unlink(self.name+".txt") os.unlink(self.name+".cab") db.Commit() @@ -451,6 +418,12 @@ class Directory: else: self.absolute = physical blogical = None + # initially assume that all files in this directory are unpackaged + # as files from self.absolute get added, this set is reduced + self.unpackaged_files = set() + for f in os.listdir(self.absolute): + if os.path.isfile(os.path.join(self.absolute, f)): + self.unpackaged_files.add(f) add_data(db, "Directory", [(logical, blogical, default)]) def start_component(self, component = None, feature = None, flags = None, keyfile = None, uuid=None): @@ -527,6 +500,11 @@ class Directory: src = file file = os.path.basename(file) absolute = os.path.join(self.absolute, src) + if absolute.startswith(self.absolute): + # mark file as packaged + relative = absolute[len(self.absolute)+1:] + if relative in self.unpackaged_files: + self.unpackaged_files.remove(relative) assert not re.search(r'[\?|><:/*]"', file) # restrictions on long names if self.keyfiles.has_key(file): logical = self.keyfiles[file] @@ -572,10 +550,17 @@ class Directory: return files def remove_pyc(self): - "Remove .pyc/.pyo files on uninstall" + "Remove .pyc/.pyo files from __pycache__ on uninstall" + directory = self.logical + "_pycache" + add_data(self.db, "Directory", [(directory, self.logical, "__PYCA~1|__pycache__")]) + flags = 256 if Win64 else 0 + add_data(self.db, "Component", + [(directory, gen_uuid(), directory, flags, None, None)]) + add_data(self.db, "FeatureComponents", [(current_feature.id, directory)]) + add_data(self.db, "CreateFolder", [(directory, directory)]) add_data(self.db, "RemoveFile", - [(self.component+"c", self.component, "*.pyc", self.logical, 2), - (self.component+"o", self.component, "*.pyo", self.logical, 2)]) + [(self.component, self.component, "*.*", directory, 2), + ]) def removefile(self, key, pattern): "Add a RemoveFile entry" diff --git a/Tools/msi/uuids.py b/Tools/msi/uuids.py index a5e5cd2..09da40a 100644 --- a/Tools/msi/uuids.py +++ b/Tools/msi/uuids.py @@ -7,23 +7,6 @@ # generated. For official releases, we record the product codes, # so people can refer to them. product_codes = { - '2.4.101': '{0e9b4d8e-6cda-446e-a208-7b92f3ddffa0}', # 2.4a1, released as a snapshot - '2.4.102': '{1b998745-4901-4edb-bc52-213689e1b922}', # 2.4a2 - '2.4.103': '{33fc8bd2-1e8f-4add-a40a-ade2728d5942}', # 2.4a3 - '2.4.111': '{51a7e2a8-2025-4ef0-86ff-e6aab742d1fa}', # 2.4b1 - '2.4.112': '{4a5e7c1d-c659-4fe3-b8c9-7c65bd9c95a5}', # 2.4b2 - '2.4.121': '{75508821-a8e9-40a8-95bd-dbe6033ddbea}', # 2.4c1 - '2.4.122': '{83a9118b-4bdd-473b-afc3-bcb142feca9e}', # 2.4c2 - '2.4.150': '{82d9302e-f209-4805-b548-52087047483a}', # 2.4.0 - '2.4.1121':'{be027411-8e6b-4440-a29b-b07df0690230}', # 2.4.1c1 - '2.4.1122':'{02818752-48bf-4074-a281-7a4114c4f1b1}', # 2.4.1c2 - '2.4.1150':'{4d4f5346-7e4a-40b5-9387-fdb6181357fc}', # 2.4.1 - '2.4.2121':'{5ef9d6b6-df78-45d2-ab09-14786a3c5a99}', # 2.4.2c1 - '2.4.2150':'{b191e49c-ea23-43b2-b28a-14e0784069b8}', # 2.4.2 - '2.4.3121':'{f669ed4d-1dce-41c4-9617-d985397187a1}', # 2.4.3c1 - '2.4.3150':'{75e71add-042c-4f30-bfac-a9ec42351313}', # 2.4.3 - '2.4.4121':'{cd2862db-22a4-4688-8772-85407ea21550}', # 2.4.4c1 - '2.4.4150':'{60e2c8c9-6cf3-4b1a-9618-e304946c94e6}', # 2.4.4 '2.5.101': '{bc14ce3e-5e72-4a64-ac1f-bf59a571898c}', # 2.5a1 '2.5.102': '{5eed51c1-8e9d-4071-94c5-b40de5d49ba5}', # 2.5a2 '2.5.103': '{73dcd966-ffec-415f-bb39-8342c1f47017}', # 2.5a3 @@ -50,6 +33,23 @@ product_codes = { '2.6.1150':'{9cc89170-000b-457d-91f1-53691f85b223}', # 2.6.1 '2.6.2121':'{adac412b-b209-4c15-b6ab-dca1b6e47144}', # 2.6.2c1 '2.6.2150':'{24aab420-4e30-4496-9739-3e216f3de6ae}', # 2.6.2 + '2.6.3121':'{a73e0254-dcda-4fe4-bf37-c7e1c4f4ebb6}', # 2.6.3c1 + '2.6.3150':'{3d9ac095-e115-4e94-bdef-7f7edf17697d}', # 2.6.3 + '2.6.4121':'{727de605-0359-4606-a94b-c2033652379b}', # 2.6.4c1 + '2.6.4122':'{4f7603c6-6352-4299-a398-150a31b19acc}', # 2.6.4c2 + '2.6.4150':'{e7394a0f-3f80-45b1-87fc-abcd51893246}', # 2.6.4 + '2.6.5121':'{e0e273d7-7598-4701-8325-c90c069fd5ff}', # 2.6.5c1 + '2.6.5122':'{fa227b76-0671-4dc6-b826-c2ff2a70dfd5}', # 2.6.5c2 + '2.6.5150':'{4723f199-fa64-4233-8e6e-9fccc95a18ee}', # 2.6.5 + '2.7.101': '{eca1bbef-432c-49ae-a667-c213cc7bbf22}', # 2.7a1 + '2.7.102': '{21ce16ed-73c4-460d-9b11-522f417b2090}', # 2.7a2 + '2.7.103': '{6e7dbd55-ba4a-48ac-a688-6c75db4d7500}', # 2.7a3 + '2.7.104': '{ee774ba3-74a5-48d9-b425-b35a287260c8}', # 2.7a4 + '2.7.111': '{9cfd9ec7-a9c7-4980-a1c6-054fc6493eb3}', # 2.7b1 + '2.7.112': '{9a72faf6-c304-4165-8595-9291ff30cac6}', # 2.7b2 + '2.7.121': '{f530c94a-dd53-4de9-948e-b632b9cb48d2}', # 2.7c1 + '2.7.122': '{f80905d2-dd8d-4b8e-8a40-c23c93dca07d}', # 2.7c2 + '2.7.150': '{20c31435-2a0a-4580-be8b-ac06fc243ca4}', # 2.7.0 '3.0.101': '{8554263a-3242-4857-9359-aa87bc2c58c2}', # 3.0a1 '3.0.102': '{692d6e2c-f0ac-40b8-a133-7191aeeb67f9}', # 3.0a2 '3.0.103': '{49cb2995-751a-4753-be7a-d0b1bb585e06}', # 3.0a3 @@ -77,7 +77,14 @@ product_codes = { '3.1.1150':'{7ff90460-89b7-435b-b583-b37b2815ccc7}', # 3.1.1 '3.1.2121':'{ec45624a-378c-43be-91f3-3f7a59b0d90c}', # 3.1.2c1 '3.1.2150':'{d40af016-506c-43fb-a738-bd54fa8c1e85}', # 3.1.2 - '3.1.3121':'{a1e436f8-92fc-4ddb-af18-a12529c57aaf}', # 3.1.3rc1 - '3.1.3122':'{2fc19026-a3e5-493e-92a0-c1f3b4a272ae}', # 3.1.3rc2 - '3.1.3150':'{f719d8a6-46fc-4d71-94c6-ffd17a8c9f35}', # 3.1.3 + '3.2.101' :'{b411f168-7a36-4fff-902c-a554d1c78a4f}', # 3.2a1 + '3.2.102' :'{79ff73b7-8359-410f-b9c5-152d2026f8c8}', # 3.2a2 + '3.2.103' :'{e7635c65-c221-4b9b-b70a-5611b8369d77}', # 3.2a3 + '3.2.104' :'{748cd139-75b8-4ca8-98a7-58262298181e}', # 3.2a4 + '3.2.111' :'{20bfc16f-c7cd-4fc0-8f96-9914614a3c50}', # 3.2b1 + '3.2.112' :'{0e350c98-8d73-4993-b686-cfe87160046e}', # 3.2b2 + '3.2.121' :'{2094968d-7583-47f6-a7fd-22304532e09f}', # 3.2rc1 + '3.2.122' :'{4f3edfa6-cf70-469a-825f-e1206aa7f412}', # 3.2rc2 + '3.2.123' :'{90c673d7-8cfd-4969-9816-f7d70bad87f3}', # 3.2rc3 + '3.2.150' :'{b2042d5e-986d-44ec-aee3-afe4108ccc93}', # 3.2.0 } diff --git a/Tools/parser/test_unparse.py b/Tools/parser/test_unparse.py new file mode 100644 index 0000000..d457523 --- /dev/null +++ b/Tools/parser/test_unparse.py @@ -0,0 +1,240 @@ +import unittest +import test.support +import io +import os +import tokenize +import ast +import unparse + +def read_pyfile(filename): + """Read and return the contents of a Python source file (as a + string), taking into account the file encoding.""" + with open(filename, "rb") as pyfile: + encoding = tokenize.detect_encoding(pyfile.readline)[0] + with open(filename, "r", encoding=encoding) as pyfile: + source = pyfile.read() + return source + +for_else = """\ +def f(): + for x in range(10): + break + else: + y = 2 + z = 3 +""" + +while_else = """\ +def g(): + while True: + break + else: + y = 2 + z = 3 +""" + +relative_import = """\ +from . import fred +from .. import barney +from .australia import shrimp as prawns +""" + +nonlocal_ex = """\ +def f(): + x = 1 + def g(): + nonlocal x + x = 2 + y = 7 + def h(): + nonlocal x, y +""" + +# also acts as test for 'except ... as ...' +raise_from = """\ +try: + 1 / 0 +except ZeroDivisionError as e: + raise ArithmeticError from e +""" + +class_decorator = """\ +@f1(arg) +@f2 +class Foo: pass +""" + +elif1 = """\ +if cond1: + suite1 +elif cond2: + suite2 +else: + suite3 +""" + +elif2 = """\ +if cond1: + suite1 +elif cond2: + suite2 +""" + +try_except_finally = """\ +try: + suite1 +except ex1: + suite2 +except ex2: + suite3 +else: + suite4 +finally: + suite5 +""" + +class ASTTestCase(unittest.TestCase): + def assertASTEqual(self, ast1, ast2): + self.assertEqual(ast.dump(ast1), ast.dump(ast2)) + + def check_roundtrip(self, code1, filename="internal"): + ast1 = compile(code1, filename, "exec", ast.PyCF_ONLY_AST) + unparse_buffer = io.StringIO() + unparse.Unparser(ast1, unparse_buffer) + code2 = unparse_buffer.getvalue() + ast2 = compile(code2, filename, "exec", ast.PyCF_ONLY_AST) + self.assertASTEqual(ast1, ast2) + +class UnparseTestCase(ASTTestCase): + # Tests for specific bugs found in earlier versions of unparse + + def test_del_statement(self): + self.check_roundtrip("del x, y, z") + + def test_shifts(self): + self.check_roundtrip("45 << 2") + self.check_roundtrip("13 >> 7") + + def test_for_else(self): + self.check_roundtrip(for_else) + + def test_while_else(self): + self.check_roundtrip(while_else) + + def test_unary_parens(self): + self.check_roundtrip("(-1)**7") + self.check_roundtrip("(-1.)**8") + self.check_roundtrip("(-1j)**6") + self.check_roundtrip("not True or False") + self.check_roundtrip("True or not False") + + def test_integer_parens(self): + self.check_roundtrip("3 .__abs__()") + + def test_huge_float(self): + self.check_roundtrip("1e1000") + self.check_roundtrip("-1e1000") + self.check_roundtrip("1e1000j") + self.check_roundtrip("-1e1000j") + + def test_min_int(self): + self.check_roundtrip(str(-2**31)) + self.check_roundtrip(str(-2**63)) + + def test_imaginary_literals(self): + self.check_roundtrip("7j") + self.check_roundtrip("-7j") + self.check_roundtrip("0j") + self.check_roundtrip("-0j") + + def test_lambda_parentheses(self): + self.check_roundtrip("(lambda: int)()") + + def test_chained_comparisons(self): + self.check_roundtrip("1 < 4 <= 5") + self.check_roundtrip("a is b is c is not d") + + def test_function_arguments(self): + self.check_roundtrip("def f(): pass") + self.check_roundtrip("def f(a): pass") + self.check_roundtrip("def f(b = 2): pass") + self.check_roundtrip("def f(a, b): pass") + self.check_roundtrip("def f(a, b = 2): pass") + self.check_roundtrip("def f(a = 5, b = 2): pass") + self.check_roundtrip("def f(*, a = 1, b = 2): pass") + self.check_roundtrip("def f(*, a = 1, b): pass") + self.check_roundtrip("def f(*, a, b = 2): pass") + self.check_roundtrip("def f(a, b = None, *, c, **kwds): pass") + self.check_roundtrip("def f(a=2, *args, c=5, d, **kwds): pass") + self.check_roundtrip("def f(*args, **kwargs): pass") + + def test_relative_import(self): + self.check_roundtrip(relative_import) + + def test_nonlocal(self): + self.check_roundtrip(nonlocal_ex) + + def test_raise_from(self): + self.check_roundtrip(raise_from) + + def test_bytes(self): + self.check_roundtrip("b'123'") + + def test_annotations(self): + self.check_roundtrip("def f(a : int): pass") + self.check_roundtrip("def f(a: int = 5): pass") + self.check_roundtrip("def f(*args: [int]): pass") + self.check_roundtrip("def f(**kwargs: dict): pass") + self.check_roundtrip("def f() -> None: pass") + + def test_set_literal(self): + self.check_roundtrip("{'a', 'b', 'c'}") + + def test_set_comprehension(self): + self.check_roundtrip("{x for x in range(5)}") + + def test_dict_comprehension(self): + self.check_roundtrip("{x: x*x for x in range(10)}") + + def test_class_decorators(self): + self.check_roundtrip(class_decorator) + + def test_class_definition(self): + self.check_roundtrip("class A(metaclass=type, *[], **{}): pass") + + def test_elifs(self): + self.check_roundtrip(elif1) + self.check_roundtrip(elif2) + + def test_try_except_finally(self): + self.check_roundtrip(try_except_finally) + +class DirectoryTestCase(ASTTestCase): + """Test roundtrip behaviour on all files in Lib and Lib/test.""" + + # test directories, relative to the root of the distribution + test_directories = 'Lib', os.path.join('Lib', 'test') + + def test_files(self): + # get names of files to test + dist_dir = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir) + + names = [] + for d in self.test_directories: + test_dir = os.path.join(dist_dir, d) + for n in os.listdir(test_dir): + if n.endswith('.py') and not n.startswith('bad'): + names.append(os.path.join(test_dir, n)) + + for filename in names: + if test.support.verbose: + print('Testing %s' % filename) + source = read_pyfile(filename) + self.check_roundtrip(source) + + +def test_main(): + test.support.run_unittest(UnparseTestCase, DirectoryTestCase) + +if __name__ == '__main__': + test_main() diff --git a/Tools/parser/unparse.py b/Tools/parser/unparse.py new file mode 100644 index 0000000..e96ef54 --- /dev/null +++ b/Tools/parser/unparse.py @@ -0,0 +1,600 @@ +"Usage: unparse.py <path to source file>" +import sys +import math +import ast +import tokenize +import io +import os + +# Large float and imaginary literals get turned into infinities in the AST. +# We unparse those infinities to INFSTR. +INFSTR = "1e" + repr(sys.float_info.max_10_exp + 1) + +def interleave(inter, f, seq): + """Call f on each item in seq, calling inter() in between. + """ + seq = iter(seq) + try: + f(next(seq)) + except StopIteration: + pass + else: + for x in seq: + inter() + f(x) + +class Unparser: + """Methods in this class recursively traverse an AST and + output source code for the abstract syntax; original formatting + is disregarded. """ + + def __init__(self, tree, file = sys.stdout): + """Unparser(tree, file=sys.stdout) -> None. + Print the source for tree to file.""" + self.f = file + self._indent = 0 + self.dispatch(tree) + print("", file=self.f) + self.f.flush() + + def fill(self, text = ""): + "Indent a piece of text, according to the current indentation level" + self.f.write("\n"+" "*self._indent + text) + + def write(self, text): + "Append a piece of text to the current line." + self.f.write(text) + + def enter(self): + "Print ':', and increase the indentation." + self.write(":") + self._indent += 1 + + def leave(self): + "Decrease the indentation level." + self._indent -= 1 + + def dispatch(self, tree): + "Dispatcher function, dispatching tree type T to method _T." + if isinstance(tree, list): + for t in tree: + self.dispatch(t) + return + meth = getattr(self, "_"+tree.__class__.__name__) + meth(tree) + + + ############### Unparsing methods ###################### + # There should be one method per concrete grammar type # + # Constructors should be grouped by sum type. Ideally, # + # this would follow the order in the grammar, but # + # currently doesn't. # + ######################################################## + + def _Module(self, tree): + for stmt in tree.body: + self.dispatch(stmt) + + # stmt + def _Expr(self, tree): + self.fill() + self.dispatch(tree.value) + + def _Import(self, t): + self.fill("import ") + interleave(lambda: self.write(", "), self.dispatch, t.names) + + def _ImportFrom(self, t): + self.fill("from ") + self.write("." * t.level) + if t.module: + self.write(t.module) + self.write(" import ") + interleave(lambda: self.write(", "), self.dispatch, t.names) + + def _Assign(self, t): + self.fill() + for target in t.targets: + self.dispatch(target) + self.write(" = ") + self.dispatch(t.value) + + def _AugAssign(self, t): + self.fill() + self.dispatch(t.target) + self.write(" "+self.binop[t.op.__class__.__name__]+"= ") + self.dispatch(t.value) + + def _Return(self, t): + self.fill("return") + if t.value: + self.write(" ") + self.dispatch(t.value) + + def _Pass(self, t): + self.fill("pass") + + def _Break(self, t): + self.fill("break") + + def _Continue(self, t): + self.fill("continue") + + def _Delete(self, t): + self.fill("del ") + interleave(lambda: self.write(", "), self.dispatch, t.targets) + + def _Assert(self, t): + self.fill("assert ") + self.dispatch(t.test) + if t.msg: + self.write(", ") + self.dispatch(t.msg) + + def _Global(self, t): + self.fill("global ") + interleave(lambda: self.write(", "), self.write, t.names) + + def _Nonlocal(self, t): + self.fill("nonlocal ") + interleave(lambda: self.write(", "), self.write, t.names) + + def _Yield(self, t): + self.write("(") + self.write("yield") + if t.value: + self.write(" ") + self.dispatch(t.value) + self.write(")") + + def _Raise(self, t): + self.fill("raise") + if not t.exc: + assert not t.cause + return + self.write(" ") + self.dispatch(t.exc) + if t.cause: + self.write(" from ") + self.dispatch(t.cause) + + def _TryExcept(self, t): + self.fill("try") + self.enter() + self.dispatch(t.body) + self.leave() + + for ex in t.handlers: + self.dispatch(ex) + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _TryFinally(self, t): + if len(t.body) == 1 and isinstance(t.body[0], ast.TryExcept): + # try-except-finally + self.dispatch(t.body) + else: + self.fill("try") + self.enter() + self.dispatch(t.body) + self.leave() + + self.fill("finally") + self.enter() + self.dispatch(t.finalbody) + self.leave() + + def _ExceptHandler(self, t): + self.fill("except") + if t.type: + self.write(" ") + self.dispatch(t.type) + if t.name: + self.write(" as ") + self.write(t.name) + self.enter() + self.dispatch(t.body) + self.leave() + + def _ClassDef(self, t): + self.write("\n") + for deco in t.decorator_list: + self.fill("@") + self.dispatch(deco) + self.fill("class "+t.name) + self.write("(") + comma = False + for e in t.bases: + if comma: self.write(", ") + else: comma = True + self.dispatch(e) + for e in t.keywords: + if comma: self.write(", ") + else: comma = True + self.dispatch(e) + if t.starargs: + if comma: self.write(", ") + else: comma = True + self.write("*") + self.dispatch(t.starargs) + if t.kwargs: + if comma: self.write(", ") + else: comma = True + self.write("**") + self.dispatch(t.kwargs) + self.write(")") + + self.enter() + self.dispatch(t.body) + self.leave() + + def _FunctionDef(self, t): + self.write("\n") + for deco in t.decorator_list: + self.fill("@") + self.dispatch(deco) + self.fill("def "+t.name + "(") + self.dispatch(t.args) + self.write(")") + if t.returns: + self.write(" -> ") + self.dispatch(t.returns) + self.enter() + self.dispatch(t.body) + self.leave() + + def _For(self, t): + self.fill("for ") + self.dispatch(t.target) + self.write(" in ") + self.dispatch(t.iter) + self.enter() + self.dispatch(t.body) + self.leave() + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _If(self, t): + self.fill("if ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + # collapse nested ifs into equivalent elifs. + while (t.orelse and len(t.orelse) == 1 and + isinstance(t.orelse[0], ast.If)): + t = t.orelse[0] + self.fill("elif ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + # final else + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _While(self, t): + self.fill("while ") + self.dispatch(t.test) + self.enter() + self.dispatch(t.body) + self.leave() + if t.orelse: + self.fill("else") + self.enter() + self.dispatch(t.orelse) + self.leave() + + def _With(self, t): + self.fill("with ") + self.dispatch(t.context_expr) + if t.optional_vars: + self.write(" as ") + self.dispatch(t.optional_vars) + self.enter() + self.dispatch(t.body) + self.leave() + + # expr + def _Bytes(self, t): + self.write(repr(t.s)) + + def _Str(self, tree): + self.write(repr(tree.s)) + + def _Name(self, t): + self.write(t.id) + + def _Num(self, t): + # Substitute overflowing decimal literal for AST infinities. + self.write(repr(t.n).replace("inf", INFSTR)) + + def _List(self, t): + self.write("[") + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write("]") + + def _ListComp(self, t): + self.write("[") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write("]") + + def _GeneratorExp(self, t): + self.write("(") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write(")") + + def _SetComp(self, t): + self.write("{") + self.dispatch(t.elt) + for gen in t.generators: + self.dispatch(gen) + self.write("}") + + def _DictComp(self, t): + self.write("{") + self.dispatch(t.key) + self.write(": ") + self.dispatch(t.value) + for gen in t.generators: + self.dispatch(gen) + self.write("}") + + def _comprehension(self, t): + self.write(" for ") + self.dispatch(t.target) + self.write(" in ") + self.dispatch(t.iter) + for if_clause in t.ifs: + self.write(" if ") + self.dispatch(if_clause) + + def _IfExp(self, t): + self.write("(") + self.dispatch(t.body) + self.write(" if ") + self.dispatch(t.test) + self.write(" else ") + self.dispatch(t.orelse) + self.write(")") + + def _Set(self, t): + assert(t.elts) # should be at least one element + self.write("{") + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write("}") + + def _Dict(self, t): + self.write("{") + def write_pair(pair): + (k, v) = pair + self.dispatch(k) + self.write(": ") + self.dispatch(v) + interleave(lambda: self.write(", "), write_pair, zip(t.keys, t.values)) + self.write("}") + + def _Tuple(self, t): + self.write("(") + if len(t.elts) == 1: + (elt,) = t.elts + self.dispatch(elt) + self.write(",") + else: + interleave(lambda: self.write(", "), self.dispatch, t.elts) + self.write(")") + + unop = {"Invert":"~", "Not": "not", "UAdd":"+", "USub":"-"} + def _UnaryOp(self, t): + self.write("(") + self.write(self.unop[t.op.__class__.__name__]) + self.write(" ") + self.dispatch(t.operand) + self.write(")") + + binop = { "Add":"+", "Sub":"-", "Mult":"*", "Div":"/", "Mod":"%", + "LShift":"<<", "RShift":">>", "BitOr":"|", "BitXor":"^", "BitAnd":"&", + "FloorDiv":"//", "Pow": "**"} + def _BinOp(self, t): + self.write("(") + self.dispatch(t.left) + self.write(" " + self.binop[t.op.__class__.__name__] + " ") + self.dispatch(t.right) + self.write(")") + + cmpops = {"Eq":"==", "NotEq":"!=", "Lt":"<", "LtE":"<=", "Gt":">", "GtE":">=", + "Is":"is", "IsNot":"is not", "In":"in", "NotIn":"not in"} + def _Compare(self, t): + self.write("(") + self.dispatch(t.left) + for o, e in zip(t.ops, t.comparators): + self.write(" " + self.cmpops[o.__class__.__name__] + " ") + self.dispatch(e) + self.write(")") + + boolops = {ast.And: 'and', ast.Or: 'or'} + def _BoolOp(self, t): + self.write("(") + s = " %s " % self.boolops[t.op.__class__] + interleave(lambda: self.write(s), self.dispatch, t.values) + self.write(")") + + def _Attribute(self,t): + self.dispatch(t.value) + # Special case: 3.__abs__() is a syntax error, so if t.value + # is an integer literal then we need to either parenthesize + # it or add an extra space to get 3 .__abs__(). + if isinstance(t.value, ast.Num) and isinstance(t.value.n, int): + self.write(" ") + self.write(".") + self.write(t.attr) + + def _Call(self, t): + self.dispatch(t.func) + self.write("(") + comma = False + for e in t.args: + if comma: self.write(", ") + else: comma = True + self.dispatch(e) + for e in t.keywords: + if comma: self.write(", ") + else: comma = True + self.dispatch(e) + if t.starargs: + if comma: self.write(", ") + else: comma = True + self.write("*") + self.dispatch(t.starargs) + if t.kwargs: + if comma: self.write(", ") + else: comma = True + self.write("**") + self.dispatch(t.kwargs) + self.write(")") + + def _Subscript(self, t): + self.dispatch(t.value) + self.write("[") + self.dispatch(t.slice) + self.write("]") + + # slice + def _Ellipsis(self, t): + self.write("...") + + def _Index(self, t): + self.dispatch(t.value) + + def _Slice(self, t): + if t.lower: + self.dispatch(t.lower) + self.write(":") + if t.upper: + self.dispatch(t.upper) + if t.step: + self.write(":") + self.dispatch(t.step) + + def _ExtSlice(self, t): + interleave(lambda: self.write(', '), self.dispatch, t.dims) + + # argument + def _arg(self, t): + self.write(t.arg) + if t.annotation: + self.write(": ") + self.dispatch(t.annotation) + + # others + def _arguments(self, t): + first = True + # normal arguments + defaults = [None] * (len(t.args) - len(t.defaults)) + t.defaults + for a, d in zip(t.args, defaults): + if first:first = False + else: self.write(", ") + self.dispatch(a) + if d: + self.write("=") + self.dispatch(d) + + # varargs, or bare '*' if no varargs but keyword-only arguments present + if t.vararg or t.kwonlyargs: + if first:first = False + else: self.write(", ") + self.write("*") + if t.vararg: + self.write(t.vararg) + if t.varargannotation: + self.write(": ") + self.dispatch(t.varargannotation) + + # keyword-only arguments + if t.kwonlyargs: + for a, d in zip(t.kwonlyargs, t.kw_defaults): + if first:first = False + else: self.write(", ") + self.dispatch(a), + if d: + self.write("=") + self.dispatch(d) + + # kwargs + if t.kwarg: + if first:first = False + else: self.write(", ") + self.write("**"+t.kwarg) + if t.kwargannotation: + self.write(": ") + self.dispatch(t.kwargannotation) + + def _keyword(self, t): + self.write(t.arg) + self.write("=") + self.dispatch(t.value) + + def _Lambda(self, t): + self.write("(") + self.write("lambda ") + self.dispatch(t.args) + self.write(": ") + self.dispatch(t.body) + self.write(")") + + def _alias(self, t): + self.write(t.name) + if t.asname: + self.write(" as "+t.asname) + +def roundtrip(filename, output=sys.stdout): + with open(filename, "rb") as pyfile: + encoding = tokenize.detect_encoding(pyfile.readline)[0] + with open(filename, "r", encoding=encoding) as pyfile: + source = pyfile.read() + tree = compile(source, filename, "exec", ast.PyCF_ONLY_AST) + Unparser(tree, output) + + + +def testdir(a): + try: + names = [n for n in os.listdir(a) if n.endswith('.py')] + except OSError: + print("Directory not readable: %s" % a, file=sys.stderr) + else: + for n in names: + fullname = os.path.join(a, n) + if os.path.isfile(fullname): + output = io.StringIO() + print('Testing %s' % fullname) + try: + roundtrip(fullname, output) + except Exception as e: + print(' Failed to compile, exception is %s' % repr(e)) + elif os.path.isdir(fullname): + testdir(fullname) + +def main(args): + if args[0] == '--testdir': + for a in args[1:]: + testdir(a) + else: + for a in args: + roundtrip(a) + +if __name__=='__main__': + main(sys.argv[1:]) diff --git a/Tools/pybench/pybench.py b/Tools/pybench/pybench.py index bee0e56..811ef96 100755 --- a/Tools/pybench/pybench.py +++ b/Tools/pybench/pybench.py @@ -228,7 +228,7 @@ class Test: raise ValueError('at least one calibration run is required') self.calibration_runs = calibration_runs if timer is not None: - timer = timer + self.timer = timer # Init variables self.times = [] diff --git a/Tools/pynche/ChipViewer.py b/Tools/pynche/ChipViewer.py index f59aa28..78139f8 100644 --- a/Tools/pynche/ChipViewer.py +++ b/Tools/pynche/ChipViewer.py @@ -13,7 +13,7 @@ The ChipViewer class includes the entire lower left quandrant; i.e. both the selected and nearest ChipWidgets. """ -from Tkinter import * +from tkinter import * import ColorDB diff --git a/Tools/pynche/DetailsViewer.py b/Tools/pynche/DetailsViewer.py index 11a99a6..fdc79b7 100644 --- a/Tools/pynche/DetailsViewer.py +++ b/Tools/pynche/DetailsViewer.py @@ -52,7 +52,7 @@ Shift + Left == -25 Shift + Right == +25 """ -from Tkinter import * +from tkinter import * STOP = 'Stop' WRAP = 'Wrap Around' diff --git a/Tools/pynche/ListViewer.py b/Tools/pynche/ListViewer.py index 213cd08..b187844 100644 --- a/Tools/pynche/ListViewer.py +++ b/Tools/pynche/ListViewer.py @@ -15,7 +15,7 @@ You can turn off Update On Click if all you want to see is the alias for a given name, without selecting the color. """ -from Tkinter import * +from tkinter import * import ColorDB ADDTOVIEW = 'Color %List Window...' diff --git a/Tools/pynche/PyncheWidget.py b/Tools/pynche/PyncheWidget.py index fcfe7ce..364f22b 100644 --- a/Tools/pynche/PyncheWidget.py +++ b/Tools/pynche/PyncheWidget.py @@ -6,9 +6,8 @@ It is used to bring up other windows. import sys import os -from Tkinter import * -import tkMessageBox -import tkFileDialog +from tkinter import * +from tkinter import messagebox, filedialog import ColorDB # Milliseconds between interrupt checks @@ -150,7 +149,7 @@ class PyncheWidget: def __popup_about(self, event=None): from Main import __version__ - tkMessageBox.showinfo('About Pynche ' + __version__, + messagebox.showinfo('About Pynche ' + __version__, '''\ Pynche %s The PYthonically Natural @@ -168,7 +167,7 @@ email: bwarsaw@python.org''' % __version__) def __load(self, event=None): while 1: idir, ifile = os.path.split(self.__sb.colordb().filename()) - file = tkFileDialog.askopenfilename( + file = filedialog.askopenfilename( filetypes=[('Text files', '*.txt'), ('All files', '*'), ], @@ -180,12 +179,12 @@ email: bwarsaw@python.org''' % __version__) try: colordb = ColorDB.get_colordb(file) except IOError: - tkMessageBox.showerror('Read error', '''\ + messagebox.showerror('Read error', '''\ Could not open file for reading: %s''' % file) continue if colordb is None: - tkMessageBox.showerror('Unrecognized color file type', '''\ + messagebox.showerror('Unrecognized color file type', '''\ Unrecognized color file type in file: %s''' % file) continue @@ -249,6 +248,8 @@ class Helpwin: +import functools +@functools.total_ordering class PopupViewer: def __init__(self, module, name, switchboard, root): self.__m = module @@ -279,8 +280,11 @@ class PopupViewer: self.__sb.add_view(self.__window) self.__window.deiconify() - def __cmp__(self, other): - return cmp(self.__menutext, other.__menutext) + def __eq__(self, other): + return self.__menutext == other.__menutext + + def __lt__(self, other): + return self.__menutext < other.__menutext def make_view_popups(switchboard, root, extrapath): diff --git a/Tools/pynche/StripViewer.py b/Tools/pynche/StripViewer.py index a7de398..2b0ede3 100644 --- a/Tools/pynche/StripViewer.py +++ b/Tools/pynche/StripViewer.py @@ -24,7 +24,7 @@ select the color under the cursor while you drag it, but be forewarned that this can be slow. """ -from Tkinter import * +from tkinter import * import ColorDB # Load this script into the Tcl interpreter and call it in @@ -62,32 +62,32 @@ def constant(numchips): # red variations, green+blue = cyan constant def constant_red_generator(numchips, red, green, blue): seq = constant(numchips) - return list(map(None, [red] * numchips, seq, seq)) + return list(zip([red] * numchips, seq, seq)) # green variations, red+blue = magenta constant def constant_green_generator(numchips, red, green, blue): seq = constant(numchips) - return list(map(None, seq, [green] * numchips, seq)) + return list(zip(seq, [green] * numchips, seq)) # blue variations, red+green = yellow constant def constant_blue_generator(numchips, red, green, blue): seq = constant(numchips) - return list(map(None, seq, seq, [blue] * numchips)) + return list(zip(seq, seq, [blue] * numchips)) # red variations, green+blue = cyan constant def constant_cyan_generator(numchips, red, green, blue): seq = constant(numchips) - return list(map(None, seq, [green] * numchips, [blue] * numchips)) + return list(zip(seq, [green] * numchips, [blue] * numchips)) # green variations, red+blue = magenta constant def constant_magenta_generator(numchips, red, green, blue): seq = constant(numchips) - return list(map(None, [red] * numchips, seq, [blue] * numchips)) + return list(zip([red] * numchips, seq, [blue] * numchips)) # blue variations, red+green = yellow constant def constant_yellow_generator(numchips, red, green, blue): seq = constant(numchips) - return list(map(None, [red] * numchips, [green] * numchips, seq)) + return list(zip([red] * numchips, [green] * numchips, seq)) @@ -119,7 +119,7 @@ class LeftArrow: return arrow, text def _x(self): - coords = self._canvas.coords(self._TAG) + coords = list(self._canvas.coords(self._TAG)) assert coords return coords[0] @@ -151,7 +151,7 @@ class RightArrow(LeftArrow): return arrow, text def _x(self): - coords = self._canvas.coords(self._TAG) + coords = list(self._canvas.coords(self._TAG)) assert coords return coords[0] + self._ARROWWIDTH diff --git a/Tools/pynche/Switchboard.py b/Tools/pynche/Switchboard.py index 29f99af..013bb01 100644 --- a/Tools/pynche/Switchboard.py +++ b/Tools/pynche/Switchboard.py @@ -42,7 +42,6 @@ master window. """ import sys -from types import DictType import marshal @@ -62,10 +61,11 @@ class Switchboard: if initfile: try: try: - fp = open(initfile) + fp = open(initfile, 'rb') self.__optiondb = marshal.load(fp) - if not isinstance(self.__optiondb, DictType): - print('Problem reading options from file:', initfile, file=sys.stderr) + if not isinstance(self.__optiondb, dict): + print('Problem reading options from file:', initfile, + file=sys.stderr) self.__optiondb = {} except (IOError, EOFError, ValueError): pass @@ -116,7 +116,7 @@ class Switchboard: fp = None try: try: - fp = open(self.__initfile, 'w') + fp = open(self.__initfile, 'wb') except IOError: print('Cannot write options to file:', \ self.__initfile, file=sys.stderr) diff --git a/Tools/pynche/TextViewer.py b/Tools/pynche/TextViewer.py index 456bd96..baa1e62 100644 --- a/Tools/pynche/TextViewer.py +++ b/Tools/pynche/TextViewer.py @@ -15,7 +15,7 @@ button and drag it through some text. The Insertion is the insertion cursor in the text window (which only has a background). """ -from Tkinter import * +from tkinter import * import ColorDB ADDTOVIEW = 'Text Window...' diff --git a/Tools/pynche/TypeinViewer.py b/Tools/pynche/TypeinViewer.py index d56c1b3..2f93e6b 100644 --- a/Tools/pynche/TypeinViewer.py +++ b/Tools/pynche/TypeinViewer.py @@ -12,7 +12,7 @@ color selection will be made on every change to the text field. Otherwise, you must hit Return or Tab to select the color. """ -from Tkinter import * +from tkinter import * diff --git a/Tools/scripts/README b/Tools/scripts/README index b1c167e..8c02529 100644 --- a/Tools/scripts/README +++ b/Tools/scripts/README @@ -1,67 +1,63 @@ -This directory contains a collection of executable Python scripts that -are useful while building, extending or managing Python. Some (e.g., -dutree or lll) are also generally useful UNIX tools. +This directory contains a collection of executable Python scripts that are +useful while building, extending or managing Python. Some (e.g., dutree or lll) +are also generally useful UNIX tools. -See also the Demo/scripts directory! - -analyze_dxp.py Analyzes the result of sys.getdxp() -byext.py Print lines/words/chars stats of files by extension -byteyears.py Print product of a file's size and age -checkappend.py Search for multi-argument .append() calls -checkpyc.py Check presence and validity of ".pyc" files -classfix.py Convert old class syntax to new -cleanfuture.py Fix reduntant Python __future__ statements -combinerefs.py A helper for analyzing PYTHONDUMPREFS output. -copytime.py Copy one file's atime and mtime to another -crlf.py Change CRLF line endings to LF (Windows to Unix) -cvsfiles.py Print a list of files that are under CVS -db2pickle.py Dump a database file to a pickle -diff.py Print file diffs in context, unified, or ndiff formats -dutree.py Format du(1) output as a tree sorted by size -eptags.py Create Emacs TAGS file for Python modules +2to3 Main script for running the 2to3 conversion tool +analyze_dxp.py Analyzes the result of sys.getdxp() +byext.py Print lines/words/chars stats of files by extension +byteyears.py Print product of a file's size and age +checkpyc.py Check presence and validity of ".pyc" files +cleanfuture.py Fix redundant Python __future__ statements +combinerefs.py A helper for analyzing PYTHONDUMPREFS output +copytime.py Copy one file's atime and mtime to another +crlf.py Change CRLF line endings to LF (Windows to Unix) +db2pickle.py Dump a database file to a pickle +diff.py Print file diffs in context, unified, or ndiff formats +dutree.py Format du(1) output as a tree sorted by size +eptags.py Create Emacs TAGS file for Python modules find_recursionlimit.py Find the maximum recursion limit on this machine -finddiv.py A grep-like tool that looks for division operators -findlinksto.py Recursively find symbolic links to a given path prefix -findnocoding.py Find source files which need an encoding declaration -fixcid.py Massive identifier substitution on C source files -fixdiv.py Tool to fix division operators. -fixheader.py Add some cpp magic to a C include file -fixnotice.py Fix the copyright notice in source files -fixps.py Fix Python scripts' first line (if #!) -ftpmirror.py FTP mirror script -google.py Open a webbrowser with Google -gprof2html.py Transform gprof(1) output into useful HTML -h2py.py Translate #define's into Python assignments -idle Main program to start IDLE -ifdef.py Remove #if(n)def groups from C sources -lfcr.py Change LF line endings to CRLF (Unix to Windows) -linktree.py Make a copy of a tree with links to original files -lll.py Find and list symbolic links in current directory -logmerge.py Consolidate CVS/RCS logs read from stdin -mailerdaemon.py parse error messages from mailer daemons (Sjoerd&Jack) -md5sum.py Print MD5 checksums of argument files. -methfix.py Fix old method syntax def f(self, (a1, ..., aN)): -mkreal.py Turn a symbolic link into a real file or directory -ndiff.py Intelligent diff between text files (Tim Peters) -nm2def.py Create a template for PC/python_nt.def (Marc Lemburg) -objgraph.py Print object graph from nm output on a library -parseentities.py Utility for parsing HTML entity definitions -pathfix.py Change #!/usr/local/bin/python into something else -pdeps.py Print dependencies between Python modules -pickle2db.py Load a pickle generated by db2pickle.py to a database -pindent.py Indent Python code, giving block-closing comments -ptags.py Create vi tags file for Python modules -pydoc Python documentation browser. -pysource.py Find Python source files -redemo.py Basic regular expression demonstration facility -reindent.py Change .py files to use 4-space indents. -rgrep.py Reverse grep through a file (useful for big logfiles) -setup.py Install all scripts listed here -suff.py Sort a list of files by suffix -svneol.py Sets svn:eol-style on all files in directory -texcheck.py Validate Python LaTeX formatting (Raymond Hettinger) -texi2html.py Convert GNU texinfo files into HTML -treesync.py Synchronize source trees (very ideosyncratic) -untabify.py Replace tabs with spaces in argument files -which.py Find a program in $PATH -xxci.py Wrapper for rcsdiff and ci +finddiv.py A grep-like tool that looks for division operators +findlinksto.py Recursively find symbolic links to a given path prefix +findnocoding.py Find source files which need an encoding declaration +fixcid.py Massive identifier substitution on C source files +fixdiv.py Tool to fix division operators. +fixheader.py Add some cpp magic to a C include file +fixnotice.py Fix the copyright notice in source files +fixps.py Fix Python scripts' first line (if #!) +ftpmirror.py FTP mirror script +google.py Open a webbrowser with Google +gprof2html.py Transform gprof(1) output into useful HTML +h2py.py Translate #define's into Python assignments +idle3 Main program to start IDLE +ifdef.py Remove #if(n)def groups from C sources +lfcr.py Change LF line endings to CRLF (Unix to Windows) +linktree.py Make a copy of a tree with links to original files +lll.py Find and list symbolic links in current directory +mailerdaemon.py Parse error messages from mailer daemons (Sjoerd&Jack) +make_ctype.py Generate ctype.h replacement in stringobject.c +md5sum.py Print MD5 checksums of argument files +mkreal.py Turn a symbolic link into a real file or directory +ndiff.py Intelligent diff between text files (Tim Peters) +nm2def.py Create a template for PC/python_nt.def (Marc Lemburg) +objgraph.py Print object graph from nm output on a library +parseentities.py Utility for parsing HTML entity definitions +patchcheck.py Perform common checks and cleanup before committing +pathfix.py Change #!/usr/local/bin/python into something else +pdeps.py Print dependencies between Python modules +pickle2db.py Load a pickle generated by db2pickle.py to a database +pindent.py Indent Python code, giving block-closing comments +ptags.py Create vi tags file for Python modules +pydoc3 Python documentation browser +pysource.py Find Python source files +redemo.py Basic regular expression demonstration facility +reindent.py Change .py files to use 4-space indents +reindent-rst.py Fix-up reStructuredText file whitespace +rgrep.py Reverse grep through a file (useful for big logfiles) +serve.py Small wsgiref-based web server, used in make serve in Doc +suff.py Sort a list of files by suffix +svneol.py Set svn:eol-style on all files in directory +texi2html.py Convert GNU texinfo files into HTML +treesync.py Synchronize source trees (very idiosyncratic) +untabify.py Replace tabs with spaces in argument files +win_add2path.py Add Python to the search path on Windows +which.py Find a program in $PATH diff --git a/Tools/scripts/abitype.py b/Tools/scripts/abitype.py new file mode 100644 index 0000000..e35ef6a --- /dev/null +++ b/Tools/scripts/abitype.py @@ -0,0 +1,199 @@ +# This script converts a C file to use the PEP 384 type definition API +# Usage: abitype.py < old_code > new_code +import re, sys + +############ Simplistic C scanner ################################## +tokenizer = re.compile( + r"(?P<preproc>#.*\n)" + r"|(?P<comment>/\*.*?\*/)" + r"|(?P<ident>[a-zA-Z_][a-zA-Z0-9_]*)" + r"|(?P<ws>[ \t\n]+)" + r"|(?P<other>.)", + re.MULTILINE) + +tokens = [] +source = sys.stdin.read() +pos = 0 +while pos != len(source): + m = tokenizer.match(source, pos) + tokens.append([m.lastgroup, m.group()]) + pos += len(tokens[-1][1]) + if tokens[-1][0] == 'preproc': + # continuation lines are considered + # only in preprocess statements + while tokens[-1][1].endswith('\\\n'): + nl = source.find('\n', pos) + if nl == -1: + line = source[pos:] + else: + line = source[pos:nl+1] + tokens[-1][1] += line + pos += len(line) + +###### Replacement of PyTypeObject static instances ############## + +# classify each token, giving it a one-letter code: +# S: static +# T: PyTypeObject +# I: ident +# W: whitespace +# =, {, }, ; : themselves +def classify(): + res = [] + for t,v in tokens: + if t == 'other' and v in "={};": + res.append(v) + elif t == 'ident': + if v == 'PyTypeObject': + res.append('T') + elif v == 'static': + res.append('S') + else: + res.append('I') + elif t == 'ws': + res.append('W') + else: + res.append('.') + return ''.join(res) + +# Obtain a list of fields of a PyTypeObject, in declaration order, +# skipping ob_base +# All comments are dropped from the variable (which are typically +# just the slot names, anyway), and information is discarded whether +# the original type was static. +def get_fields(start, real_end): + pos = start + # static? + if tokens[pos][1] == 'static': + pos += 2 + # PyTypeObject + pos += 2 + # name + name = tokens[pos][1] + pos += 1 + while tokens[pos][1] != '{': + pos += 1 + pos += 1 + # PyVarObject_HEAD_INIT + while tokens[pos][0] in ('ws', 'comment'): + pos += 1 + if tokens[pos][1] != 'PyVarObject_HEAD_INIT': + raise Exception, '%s has no PyVarObject_HEAD_INIT' % name + while tokens[pos][1] != ')': + pos += 1 + pos += 1 + # field definitions: various tokens, comma-separated + fields = [] + while True: + while tokens[pos][0] in ('ws', 'comment'): + pos += 1 + end = pos + while tokens[end][1] not in ',}': + if tokens[end][1] == '(': + nesting = 1 + while nesting: + end += 1 + if tokens[end][1] == '(': nesting+=1 + if tokens[end][1] == ')': nesting-=1 + end += 1 + assert end < real_end + # join field, excluding separator and trailing ws + end1 = end-1 + while tokens[end1][0] in ('ws', 'comment'): + end1 -= 1 + fields.append(''.join(t[1] for t in tokens[pos:end1+1])) + if tokens[end][1] == '}': + break + pos = end+1 + return name, fields + +# List of type slots as of Python 3.2, omitting ob_base +typeslots = [ + 'tp_name', + 'tp_basicsize', + 'tp_itemsize', + 'tp_dealloc', + 'tp_print', + 'tp_getattr', + 'tp_setattr', + 'tp_reserved', + 'tp_repr', + 'tp_as_number', + 'tp_as_sequence', + 'tp_as_mapping', + 'tp_hash', + 'tp_call', + 'tp_str', + 'tp_getattro', + 'tp_setattro', + 'tp_as_buffer', + 'tp_flags', + 'tp_doc', + 'tp_traverse', + 'tp_clear', + 'tp_richcompare', + 'tp_weaklistoffset', + 'tp_iter', + 'iternextfunc', + 'tp_methods', + 'tp_members', + 'tp_getset', + 'tp_base', + 'tp_dict', + 'tp_descr_get', + 'tp_descr_set', + 'tp_dictoffset', + 'tp_init', + 'tp_alloc', + 'tp_new', + 'tp_free', + 'tp_is_gc', + 'tp_bases', + 'tp_mro', + 'tp_cache', + 'tp_subclasses', + 'tp_weaklist', + 'tp_del' + 'tp_version_tag' +] + +# Generate a PyType_Spec definition +def make_slots(name, fields): + res = [] + res.append('static PyType_Slot %s_slots[] = {' % name) + # defaults for spec + spec = { 'tp_itemsize':'0' } + for i, val in enumerate(fields): + if val.endswith('0'): + continue + if typeslots[i] in ('tp_name', 'tp_doc', 'tp_basicsize', + 'tp_itemsize', 'tp_flags'): + spec[typeslots[i]] = val + continue + res.append(' {Py_%s, %s},' % (typeslots[i], val)) + res.append('};') + res.append('static PyType_Spec %s_spec = {' % name) + res.append(' %s,' % spec['tp_name']) + res.append(' %s,' % spec['tp_basicsize']) + res.append(' %s,' % spec['tp_itemsize']) + res.append(' %s,' % spec['tp_flags']) + res.append(' %s_slots,' % name) + res.append('};\n') + return '\n'.join(res) + + +# Main loop: replace all static PyTypeObjects until +# there are none left. +while 1: + c = classify() + m = re.search('(SW)?TWIW?=W?{.*?};', c) + if not m: + break + start = m.start() + end = m.end() + name, fields = get_fields(start, m) + tokens[start:end] = [('',make_slots(name, fields))] + +# Output result to stdout +for t, v in tokens: + sys.stdout.write(v) diff --git a/Tools/scripts/byext.py b/Tools/scripts/byext.py index e5b090c..b79ff37 100755 --- a/Tools/scripts/byext.py +++ b/Tools/scripts/byext.py @@ -1,10 +1,11 @@ -#! /usr/bin/env python3.0 +#! /usr/bin/env python3 """Show file statistics by extension.""" import os import sys + class Stats: def __init__(self): @@ -28,12 +29,11 @@ class Stats: sys.stderr.write("Can't list %s: %s\n" % (dir, err)) self.addstats("<dir>", "unlistable", 1) return - names.sort() - for name in names: + for name in sorted(names): if name.startswith(".#"): - continue # Skip CVS temp files + continue # Skip CVS temp files if name.endswith("~"): - continue# Skip Emacs backup files + continue # Skip Emacs backup files full = os.path.join(dir, name) if os.path.islink(full): self.addstats("<lnk>", "links", 1) @@ -46,26 +46,25 @@ class Stats: head, ext = os.path.splitext(filename) head, base = os.path.split(filename) if ext == base: - ext = "" # E.g. .cvsignore is deemed not to have an extension + ext = "" # E.g. .cvsignore is deemed not to have an extension ext = os.path.normcase(ext) if not ext: ext = "<none>" self.addstats(ext, "files", 1) try: - f = open(filename, "rb") + with open(filename, "rb") as f: + data = f.read() except IOError as err: sys.stderr.write("Can't open %s: %s\n" % (filename, err)) self.addstats(ext, "unopenable", 1) return - data = f.read() - f.close() self.addstats(ext, "bytes", len(data)) if b'\0' in data: self.addstats(ext, "binary", 1) return if not data: self.addstats(ext, "empty", 1) - #self.addstats(ext, "chars", len(data)) + # self.addstats(ext, "chars", len(data)) lines = str(data, "latin-1").splitlines() self.addstats(ext, "lines", len(lines)) del lines @@ -105,17 +104,20 @@ class Stats: for ext in exts: self.stats[ext]["ext"] = ext cols.insert(0, "ext") + def printheader(): for col in cols: print("%*s" % (colwidth[col], col), end=' ') print() + printheader() for ext in exts: for col in cols: value = self.stats[ext].get(col, "") print("%*s" % (colwidth[col], value), end=' ') print() - printheader() # Another header at the bottom + printheader() # Another header at the bottom + def main(): args = sys.argv[1:] @@ -125,5 +127,6 @@ def main(): s.statargs(args) s.report() + if __name__ == "__main__": main() diff --git a/Tools/scripts/byteyears.py b/Tools/scripts/byteyears.py index f486d26..490b37f 100755 --- a/Tools/scripts/byteyears.py +++ b/Tools/scripts/byteyears.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Print the product of age and size of each file, in suitable units. # diff --git a/Tools/scripts/checkappend.py b/Tools/scripts/checkappend.py deleted file mode 100755 index 4c74ee5..0000000 --- a/Tools/scripts/checkappend.py +++ /dev/null @@ -1,169 +0,0 @@ -#! /usr/bin/env python - -# Released to the public domain, by Tim Peters, 28 February 2000. - -"""checkappend.py -- search for multi-argument .append() calls. - -Usage: specify one or more file or directory paths: - checkappend [-v] file_or_dir [file_or_dir] ... - -Each file_or_dir is checked for multi-argument .append() calls. When -a directory, all .py files in the directory, and recursively in its -subdirectories, are checked. - -Use -v for status msgs. Use -vv for more status msgs. - -In the absence of -v, the only output is pairs of the form - - filename(linenumber): - line containing the suspicious append - -Note that this finds multi-argument append calls regardless of whether -they're attached to list objects. If a module defines a class with an -append method that takes more than one argument, calls to that method -will be listed. - -Note that this will not find multi-argument list.append calls made via a -bound method object. For example, this is not caught: - - somelist = [] - push = somelist.append - push(1, 2, 3) -""" - -__version__ = 1, 0, 0 - -import os -import sys -import getopt -import tokenize - -verbose = 0 - -def errprint(*args): - msg = ' '.join(args) - sys.stderr.write(msg) - sys.stderr.write("\n") - -def main(): - args = sys.argv[1:] - global verbose - try: - opts, args = getopt.getopt(sys.argv[1:], "v") - except getopt.error as msg: - errprint(str(msg) + "\n\n" + __doc__) - return - for opt, optarg in opts: - if opt == '-v': - verbose = verbose + 1 - if not args: - errprint(__doc__) - return - for arg in args: - check(arg) - -def check(file): - if os.path.isdir(file) and not os.path.islink(file): - if verbose: - print("%r: listing directory" % (file,)) - names = os.listdir(file) - for name in names: - fullname = os.path.join(file, name) - if ((os.path.isdir(fullname) and - not os.path.islink(fullname)) - or os.path.normcase(name[-3:]) == ".py"): - check(fullname) - return - - try: - f = open(file) - except IOError as msg: - errprint("%r: I/O Error: %s" % (file, msg)) - return - - if verbose > 1: - print("checking %r ..." % (file,)) - - ok = AppendChecker(file, f).run() - if verbose and ok: - print("%r: Clean bill of health." % (file,)) - -[FIND_DOT, - FIND_APPEND, - FIND_LPAREN, - FIND_COMMA, - FIND_STMT] = range(5) - -class AppendChecker: - def __init__(self, fname, file): - self.fname = fname - self.file = file - self.state = FIND_DOT - self.nerrors = 0 - - def run(self): - try: - tokens = tokenize.generate_tokens(self.file.readline) - for _token in tokens: - self.tokeneater(*_token) - except tokenize.TokenError as msg: - errprint("%r: Token Error: %s" % (self.fname, msg)) - self.nerrors = self.nerrors + 1 - return self.nerrors == 0 - - def tokeneater(self, type, token, start, end, line, - NEWLINE=tokenize.NEWLINE, - JUNK=(tokenize.COMMENT, tokenize.NL), - OP=tokenize.OP, - NAME=tokenize.NAME): - - state = self.state - - if type in JUNK: - pass - - elif state is FIND_DOT: - if type is OP and token == ".": - state = FIND_APPEND - - elif state is FIND_APPEND: - if type is NAME and token == "append": - self.line = line - self.lineno = start[0] - state = FIND_LPAREN - else: - state = FIND_DOT - - elif state is FIND_LPAREN: - if type is OP and token == "(": - self.level = 1 - state = FIND_COMMA - else: - state = FIND_DOT - - elif state is FIND_COMMA: - if type is OP: - if token in ("(", "{", "["): - self.level = self.level + 1 - elif token in (")", "}", "]"): - self.level = self.level - 1 - if self.level == 0: - state = FIND_DOT - elif token == "," and self.level == 1: - self.nerrors = self.nerrors + 1 - print("%s(%d):\n%s" % (self.fname, self.lineno, - self.line)) - # don't gripe about this stmt again - state = FIND_STMT - - elif state is FIND_STMT: - if type is NEWLINE: - state = FIND_DOT - - else: - raise SystemError("unknown internal state '%r'" % (state,)) - - self.state = state - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/checkpyc.py b/Tools/scripts/checkpyc.py index 2e8fd5a..d4fdce2 100755 --- a/Tools/scripts/checkpyc.py +++ b/Tools/scripts/checkpyc.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Check that all ".pyc" files exist and are up-to-date # Uses module 'os' @@ -7,14 +7,17 @@ import os from stat import ST_MTIME import imp +# PEP 3147 compatibility (PYC Repository Directories) +cache_from_source = (imp.cache_from_source if hasattr(imp, 'get_tag') else + lambda path: path + 'c') + + def main(): - silent = 0 - verbose = 0 - if sys.argv[1:]: - if sys.argv[1] == '-v': - verbose = 1 - elif sys.argv[1] == '-s': - silent = 1 + if len(sys.argv) > 1: + verbose = (sys.argv[1] == '-v') + silent = (sys.argv[1] == '-s') + else: + verbose = silent = False MAGIC = imp.get_magic() if not silent: print('Using MAGIC word', repr(MAGIC)) @@ -26,9 +29,8 @@ def main(): continue if not silent: print('Checking ', repr(dirname), '...') - names.sort() - for name in names: - if name[-3:] == '.py': + for name in sorted(names): + if name.endswith('.py'): name = os.path.join(dirname, name) try: st = os.stat(name) @@ -37,30 +39,31 @@ def main(): continue if verbose: print('Check', repr(name), '...') - name_c = name + 'c' + name_c = cache_from_source(name) try: - f = open(name_c, 'r') + with open(name_c, 'rb') as f: + magic_str = f.read(4) + mtime_str = f.read(4) except IOError: print('Cannot open', repr(name_c)) continue - magic_str = f.read(4) - mtime_str = f.read(4) - f.close() if magic_str != MAGIC: print('Bad MAGIC word in ".pyc" file', end=' ') print(repr(name_c)) continue mtime = get_long(mtime_str) - if mtime == 0 or mtime == -1: + if mtime in {0, -1}: print('Bad ".pyc" file', repr(name_c)) elif mtime != st[ST_MTIME]: print('Out-of-date ".pyc" file', end=' ') print(repr(name_c)) + def get_long(s): if len(s) != 4: return -1 - return ord(s[0]) + (ord(s[1])<<8) + (ord(s[2])<<16) + (ord(s[3])<<24) + return s[0] + (s[1] << 8) + (s[2] << 16) + (s[3] << 24) + if __name__ == '__main__': main() diff --git a/Tools/scripts/classfix.py b/Tools/scripts/classfix.py deleted file mode 100755 index 0cd1e49..0000000 --- a/Tools/scripts/classfix.py +++ /dev/null @@ -1,190 +0,0 @@ -#! /usr/bin/env python - -# This script is obsolete -- it is kept for historical purposes only. -# -# Fix Python source files to use the new class definition syntax, i.e., -# the syntax used in Python versions before 0.9.8: -# class C() = base(), base(), ...: ... -# is changed to the current syntax: -# class C(base, base, ...): ... -# -# The script uses heuristics to find class definitions that usually -# work but occasionally can fail; carefully check the output! -# -# Command line arguments are files or directories to be processed. -# Directories are searched recursively for files whose name looks -# like a python module. -# Symbolic links are always ignored (except as explicit directory -# arguments). Of course, the original file is kept as a back-up -# (with a "~" attached to its name). -# -# Changes made are reported to stdout in a diff-like format. -# -# Undoubtedly you can do this using find and sed or perl, but this is -# a nice example of Python code that recurses down a directory tree -# and uses regular expressions. Also note several subtleties like -# preserving the file's mode and avoiding to even write a temp file -# when no changes are needed for a file. -# -# NB: by changing only the function fixline() you can turn this -# into a program for a different change to Python programs... - -import sys -import re -import os -from stat import * - -err = sys.stderr.write -dbg = err -rep = sys.stdout.write - -def main(): - bad = 0 - if not sys.argv[1:]: # No arguments - err('usage: ' + sys.argv[0] + ' file-or-directory ...\n') - sys.exit(2) - for arg in sys.argv[1:]: - if os.path.isdir(arg): - if recursedown(arg): bad = 1 - elif os.path.islink(arg): - err(arg + ': will not process symbolic links\n') - bad = 1 - else: - if fix(arg): bad = 1 - sys.exit(bad) - -ispythonprog = re.compile('^[a-zA-Z0-9_]+\.py$') -def ispython(name): - return ispythonprog.match(name) >= 0 - -def recursedown(dirname): - dbg('recursedown(%r)\n' % (dirname,)) - bad = 0 - try: - names = os.listdir(dirname) - except os.error as msg: - err('%s: cannot list directory: %r\n' % (dirname, msg)) - return 1 - names.sort() - subdirs = [] - for name in names: - if name in (os.curdir, os.pardir): continue - fullname = os.path.join(dirname, name) - if os.path.islink(fullname): pass - elif os.path.isdir(fullname): - subdirs.append(fullname) - elif ispython(name): - if fix(fullname): bad = 1 - for fullname in subdirs: - if recursedown(fullname): bad = 1 - return bad - -def fix(filename): -## dbg('fix(%r)\n' % (filename,)) - try: - f = open(filename, 'r') - except IOError as msg: - err('%s: cannot open: %r\n' % (filename, msg)) - return 1 - head, tail = os.path.split(filename) - tempname = os.path.join(head, '@' + tail) - g = None - # If we find a match, we rewind the file and start over but - # now copy everything to a temp file. - lineno = 0 - while 1: - line = f.readline() - if not line: break - lineno = lineno + 1 - while line[-2:] == '\\\n': - nextline = f.readline() - if not nextline: break - line = line + nextline - lineno = lineno + 1 - newline = fixline(line) - if newline != line: - if g is None: - try: - g = open(tempname, 'w') - except IOError as msg: - f.close() - err('%s: cannot create: %r\n' % (tempname, msg)) - return 1 - f.seek(0) - lineno = 0 - rep(filename + ':\n') - continue # restart from the beginning - rep(repr(lineno) + '\n') - rep('< ' + line) - rep('> ' + newline) - if g is not None: - g.write(newline) - - # End of file - f.close() - if not g: return 0 # No changes - - # Finishing touch -- move files - - # First copy the file's mode to the temp file - try: - statbuf = os.stat(filename) - os.chmod(tempname, statbuf[ST_MODE] & 0o7777) - except os.error as msg: - err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) - # Then make a backup of the original file as filename~ - try: - os.rename(filename, filename + '~') - except os.error as msg: - err('%s: warning: backup failed (%r)\n' % (filename, msg)) - # Now move the temp file to the original file - try: - os.rename(tempname, filename) - except os.error as msg: - err('%s: rename failed (%r)\n' % (filename, msg)) - return 1 - # Return succes - return 0 - -# This expression doesn't catch *all* class definition headers, -# but it's pretty darn close. -classexpr = '^([ \t]*class +[a-zA-Z0-9_]+) *( *) *((=.*)?):' -classprog = re.compile(classexpr) - -# Expressions for finding base class expressions. -baseexpr = '^ *(.*) *( *) *$' -baseprog = re.compile(baseexpr) - -def fixline(line): - if classprog.match(line) < 0: # No 'class' keyword -- no change - return line - - (a0, b0), (a1, b1), (a2, b2) = classprog.regs[:3] - # a0, b0 = Whole match (up to ':') - # a1, b1 = First subexpression (up to classname) - # a2, b2 = Second subexpression (=.*) - head = line[:b1] - tail = line[b0:] # Unmatched rest of line - - if a2 == b2: # No base classes -- easy case - return head + ':' + tail - - # Get rid of leading '=' - basepart = line[a2+1:b2] - - # Extract list of base expressions - bases = basepart.split(',') - - # Strip trailing '()' from each base expression - for i in range(len(bases)): - if baseprog.match(bases[i]) >= 0: - x1, y1 = baseprog.regs[1] - bases[i] = bases[i][x1:y1] - - # Join the bases back again and build the new line - basepart = ', '.join(bases) - - return head + '(' + basepart + '):' + tail - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/cleanfuture.py b/Tools/scripts/cleanfuture.py index e6c8c8c..b48ab60 100644 --- a/Tools/scripts/cleanfuture.py +++ b/Tools/scripts/cleanfuture.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """cleanfuture [-d][-r][-v] path ... diff --git a/Tools/scripts/combinerefs.py b/Tools/scripts/combinerefs.py index 68704dd..e10e49a 100644 --- a/Tools/scripts/combinerefs.py +++ b/Tools/scripts/combinerefs.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """ combinerefs path diff --git a/Tools/scripts/copytime.py b/Tools/scripts/copytime.py index ba4a267..e0220b5 100755 --- a/Tools/scripts/copytime.py +++ b/Tools/scripts/copytime.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Copy one file's atime and mtime to another diff --git a/Tools/scripts/crlf.py b/Tools/scripts/crlf.py index 3dfa131..0622282 100755 --- a/Tools/scripts/crlf.py +++ b/Tools/scripts/crlf.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 "Replace CRLF with LF in argument files. Print names of changed files." import sys, os diff --git a/Tools/scripts/cvsfiles.py b/Tools/scripts/cvsfiles.py deleted file mode 100755 index 9e65dc8..0000000 --- a/Tools/scripts/cvsfiles.py +++ /dev/null @@ -1,72 +0,0 @@ -#! /usr/bin/env python - -"""Print a list of files that are mentioned in CVS directories. - -Usage: cvsfiles.py [-n file] [directory] ... - -If the '-n file' option is given, only files under CVS that are newer -than the given file are printed; by default, all files under CVS are -printed. As a special case, if a file does not exist, it is always -printed. -""" - -import os -import sys -import stat -import getopt - -cutofftime = 0 - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], "n:") - except getopt.error as msg: - print(msg) - print(__doc__, end=' ') - return 1 - global cutofftime - newerfile = None - for o, a in opts: - if o == '-n': - cutofftime = getmtime(a) - if args: - for arg in args: - process(arg) - else: - process(".") - -def process(dir): - cvsdir = 0 - subdirs = [] - names = os.listdir(dir) - for name in names: - fullname = os.path.join(dir, name) - if name == "CVS": - cvsdir = fullname - else: - if os.path.isdir(fullname): - if not os.path.islink(fullname): - subdirs.append(fullname) - if cvsdir: - entries = os.path.join(cvsdir, "Entries") - for e in open(entries).readlines(): - words = e.split('/') - if words[0] == '' and words[1:]: - name = words[1] - fullname = os.path.join(dir, name) - if cutofftime and getmtime(fullname) <= cutofftime: - pass - else: - print(fullname) - for sub in subdirs: - process(sub) - -def getmtime(filename): - try: - st = os.stat(filename) - except os.error: - return 0 - return st[stat.ST_MTIME] - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/db2pickle.py b/Tools/scripts/db2pickle.py index 9dd8bd3..a5532a8 100644 --- a/Tools/scripts/db2pickle.py +++ b/Tools/scripts/db2pickle.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Synopsis: %(prog)s [-h|-g|-b|-r|-a] dbfile [ picklefile ] diff --git a/Tools/scripts/dutree.py b/Tools/scripts/dutree.py index dbf4f1a..6b4361a 100755 --- a/Tools/scripts/dutree.py +++ b/Tools/scripts/dutree.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Format du output in a tree shape import os, sys, errno diff --git a/Tools/scripts/eptags.py b/Tools/scripts/eptags.py index 8d35dfb..671ff11 100755 --- a/Tools/scripts/eptags.py +++ b/Tools/scripts/eptags.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """Create a TAGS file for Python programs, usable with GNU Emacs. usage: eptags pyfiles... diff --git a/Tools/scripts/find-uname.py b/Tools/scripts/find-uname.py new file mode 100755 index 0000000..b6ec1b6 --- /dev/null +++ b/Tools/scripts/find-uname.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python3 + +""" +For each argument on the command line, look for it in the set of all Unicode +names. Arguments are treated as case-insensitive regular expressions, e.g.: + + % find-uname 'small letter a$' 'horizontal line' + *** small letter a$ matches *** + LATIN SMALL LETTER A (97) + COMBINING LATIN SMALL LETTER A (867) + CYRILLIC SMALL LETTER A (1072) + PARENTHESIZED LATIN SMALL LETTER A (9372) + CIRCLED LATIN SMALL LETTER A (9424) + FULLWIDTH LATIN SMALL LETTER A (65345) + *** horizontal line matches *** + HORIZONTAL LINE EXTENSION (9135) +""" + +import unicodedata +import sys +import re + +def main(args): + unicode_names = [] + for ix in range(sys.maxunicode+1): + try: + unicode_names.append((ix, unicodedata.name(chr(ix)))) + except ValueError: # no name for the character + pass + for arg in args: + pat = re.compile(arg, re.I) + matches = [(y,x) for (x,y) in unicode_names + if pat.search(y) is not None] + if matches: + print("***", arg, "matches", "***") + for match in matches: + print("%s (%d)" % match) + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/Tools/scripts/find_recursionlimit.py b/Tools/scripts/find_recursionlimit.py index 6f75d6d..443f052 100644 --- a/Tools/scripts/find_recursionlimit.py +++ b/Tools/scripts/find_recursionlimit.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """Find the maximum recursion limit that prevents interpreter termination. This script finds the maximum safe recursion limit on a particular diff --git a/Tools/scripts/finddiv.py b/Tools/scripts/finddiv.py index 558791f..f24a702 100755 --- a/Tools/scripts/finddiv.py +++ b/Tools/scripts/finddiv.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """finddiv - a grep-like tool that looks for division operators. diff --git a/Tools/scripts/findlinksto.py b/Tools/scripts/findlinksto.py index d3da7e4..b4c09ef 100755 --- a/Tools/scripts/findlinksto.py +++ b/Tools/scripts/findlinksto.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # findlinksto # diff --git a/Tools/scripts/findnocoding.py b/Tools/scripts/findnocoding.py index 78fc8ef..77607ce 100755 --- a/Tools/scripts/findnocoding.py +++ b/Tools/scripts/findnocoding.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """List all those Python files that require a coding directive diff --git a/Tools/scripts/fixcid.py b/Tools/scripts/fixcid.py index b21a836..2d4cd1a 100755 --- a/Tools/scripts/fixcid.py +++ b/Tools/scripts/fixcid.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Perform massive identifier substitution on C source files. # This actually tokenizes the files (to some extent) so it can diff --git a/Tools/scripts/fixdiv.py b/Tools/scripts/fixdiv.py index 8b15cc6..4ecbea1 100755 --- a/Tools/scripts/fixdiv.py +++ b/Tools/scripts/fixdiv.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """fixdiv - tool to fix division operators. diff --git a/Tools/scripts/fixheader.py b/Tools/scripts/fixheader.py index 1208031..ec84057 100755 --- a/Tools/scripts/fixheader.py +++ b/Tools/scripts/fixheader.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Add some standard cpp magic to a header file diff --git a/Tools/scripts/fixnotice.py b/Tools/scripts/fixnotice.py index d35a339..aac8697 100755 --- a/Tools/scripts/fixnotice.py +++ b/Tools/scripts/fixnotice.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """(Ostensibly) fix copyright notices in files. diff --git a/Tools/scripts/fixps.py b/Tools/scripts/fixps.py index fd2ca71..b002261 100755 --- a/Tools/scripts/fixps.py +++ b/Tools/scripts/fixps.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Fix Python script(s) to reference the interpreter via /usr/bin/env python. # Warning: this overwrites the file without making a backup. diff --git a/Tools/scripts/ftpmirror.py b/Tools/scripts/ftpmirror.py index b79db1a..9e8be1d 100755 --- a/Tools/scripts/ftpmirror.py +++ b/Tools/scripts/ftpmirror.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """Mirror a remote ftp subtree into a local directory tree. diff --git a/Tools/ssl/get-remote-certificate.py b/Tools/scripts/get-remote-certificate.py index ab851ef..5811f20 100644 --- a/Tools/ssl/get-remote-certificate.py +++ b/Tools/scripts/get-remote-certificate.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # fetch the certificate that the server(s) are providing in PEM form # @@ -8,7 +8,6 @@ import re import os -import ssl import sys import tempfile @@ -38,7 +37,8 @@ def fetch_server_certificate (host, port): status, output = subproc(r'openssl x509 -in "%s" -out "%s"' % (tn, tn2)) if status != 0: - raise OperationError(status, tsig, output) + raise RuntimeError('OpenSSL x509 failed with status %s and ' + 'output: %r' % (status, output)) fp = open(tn2, 'rb') data = fp.read() fp.close() @@ -63,7 +63,8 @@ def fetch_server_certificate (host, port): 'openssl s_client -connect "%s:%s" -showcerts < /dev/null' % (host, port)) if status != 0: - raise OSError(status) + raise RuntimeError('OpenSSL connect failed with status %s and ' + 'output: %r' % (status, output)) certtext = strip_to_x509_cert(output) if not certtext: raise ValueError("Invalid response received from server at %s:%s" % diff --git a/Tools/scripts/google.py b/Tools/scripts/google.py index 6219c2d..12152bb 100755 --- a/Tools/scripts/google.py +++ b/Tools/scripts/google.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 import sys, webbrowser diff --git a/Tools/scripts/gprof2html.py b/Tools/scripts/gprof2html.py index cb01c2c..f3c7202 100755 --- a/Tools/scripts/gprof2html.py +++ b/Tools/scripts/gprof2html.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python2.3 +#! /usr/bin/env python32.3 """Transform gprof(1) output into useful HTML.""" diff --git a/Tools/scripts/h2py.py b/Tools/scripts/h2py.py index 9b5b697..45aa439 100755 --- a/Tools/scripts/h2py.py +++ b/Tools/scripts/h2py.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Read #define's and translate to Python code. # Handle #include statements. @@ -49,13 +49,7 @@ except KeyError: try: searchdirs=os.environ['INCLUDE'].split(';') except KeyError: - try: - if sys.platform.startswith("atheos"): - searchdirs=os.environ['C_INCLUDE_PATH'].split(':') - else: - raise KeyError - except KeyError: - searchdirs=['/usr/include'] + searchdirs=['/usr/include'] def main(): global filedict diff --git a/Tools/scripts/ifdef.py b/Tools/scripts/ifdef.py index 2ed7a66..46167ad 100755 --- a/Tools/scripts/ifdef.py +++ b/Tools/scripts/ifdef.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Selectively preprocess #ifdef / #ifndef statements. # Usage: diff --git a/Tools/scripts/lfcr.py b/Tools/scripts/lfcr.py index 1b9a5b7..d094022 100755 --- a/Tools/scripts/lfcr.py +++ b/Tools/scripts/lfcr.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 "Replace LF with CRLF in argument files. Print names of changed files." diff --git a/Tools/scripts/linktree.py b/Tools/scripts/linktree.py index 748b042..982f480 100755 --- a/Tools/scripts/linktree.py +++ b/Tools/scripts/linktree.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # linktree # diff --git a/Tools/scripts/lll.py b/Tools/scripts/lll.py index 5ee1504..aa4e550 100755 --- a/Tools/scripts/lll.py +++ b/Tools/scripts/lll.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Find symbolic links and show where they point to. # Arguments are directories to search; default is current directory. diff --git a/Tools/scripts/logmerge.py b/Tools/scripts/logmerge.py deleted file mode 100755 index 06750b0..0000000 --- a/Tools/scripts/logmerge.py +++ /dev/null @@ -1,185 +0,0 @@ -#! /usr/bin/env python - -"""Consolidate a bunch of CVS or RCS logs read from stdin. - -Input should be the output of a CVS or RCS logging command, e.g. - - cvs log -rrelease14: - -which dumps all log messages from release1.4 upwards (assuming that -release 1.4 was tagged with tag 'release14'). Note the trailing -colon! - -This collects all the revision records and outputs them sorted by date -rather than by file, collapsing duplicate revision record, i.e., -records with the same message for different files. - -The -t option causes it to truncate (discard) the last revision log -entry; this is useful when using something like the above cvs log -command, which shows the revisions including the given tag, while you -probably want everything *since* that tag. - -The -r option reverses the output (oldest first; the default is oldest -last). - -The -b tag option restricts the output to *only* checkin messages -belonging to the given branch tag. The form -b HEAD restricts the -output to checkin messages belonging to the CVS head (trunk). (It -produces some output if tag is a non-branch tag, but this output is -not very useful.) - --h prints this message and exits. - -XXX This code was created by reverse engineering CVS 1.9 and RCS 5.7 -from their output. -""" - -import sys, errno, getopt, re - -sep1 = '='*77 + '\n' # file separator -sep2 = '-'*28 + '\n' # revision separator - -def main(): - """Main program""" - truncate_last = 0 - reverse = 0 - branch = None - opts, args = getopt.getopt(sys.argv[1:], "trb:h") - for o, a in opts: - if o == '-t': - truncate_last = 1 - elif o == '-r': - reverse = 1 - elif o == '-b': - branch = a - elif o == '-h': - print(__doc__) - sys.exit(0) - database = [] - while 1: - chunk = read_chunk(sys.stdin) - if not chunk: - break - records = digest_chunk(chunk, branch) - if truncate_last: - del records[-1] - database[len(database):] = records - database.sort() - if not reverse: - database.reverse() - format_output(database) - -def read_chunk(fp): - """Read a chunk -- data for one file, ending with sep1. - - Split the chunk in parts separated by sep2. - - """ - chunk = [] - lines = [] - while 1: - line = fp.readline() - if not line: - break - if line == sep1: - if lines: - chunk.append(lines) - break - if line == sep2: - if lines: - chunk.append(lines) - lines = [] - else: - lines.append(line) - return chunk - -def digest_chunk(chunk, branch=None): - """Digest a chunk -- extract working file name and revisions""" - lines = chunk[0] - key = 'Working file:' - keylen = len(key) - for line in lines: - if line[:keylen] == key: - working_file = line[keylen:].strip() - break - else: - working_file = None - if branch is None: - pass - elif branch == "HEAD": - branch = re.compile(r"^\d+\.\d+$") - else: - revisions = {} - key = 'symbolic names:\n' - found = 0 - for line in lines: - if line == key: - found = 1 - elif found: - if line[0] in '\t ': - tag, rev = line.split() - if tag[-1] == ':': - tag = tag[:-1] - revisions[tag] = rev - else: - found = 0 - rev = revisions.get(branch) - branch = re.compile(r"^<>$") # <> to force a mismatch by default - if rev: - if rev.find('.0.') >= 0: - rev = rev.replace('.0.', '.') - branch = re.compile(r"^" + re.escape(rev) + r"\.\d+$") - records = [] - for lines in chunk[1:]: - revline = lines[0] - dateline = lines[1] - text = lines[2:] - words = dateline.split() - author = None - if len(words) >= 3 and words[0] == 'date:': - dateword = words[1] - timeword = words[2] - if timeword[-1:] == ';': - timeword = timeword[:-1] - date = dateword + ' ' + timeword - if len(words) >= 5 and words[3] == 'author:': - author = words[4] - if author[-1:] == ';': - author = author[:-1] - else: - date = None - text.insert(0, revline) - words = revline.split() - if len(words) >= 2 and words[0] == 'revision': - rev = words[1] - else: - # No 'revision' line -- weird... - rev = None - text.insert(0, revline) - if branch: - if rev is None or not branch.match(rev): - continue - records.append((date, working_file, rev, author, text)) - return records - -def format_output(database): - prevtext = None - prev = [] - database.append((None, None, None, None, None)) # Sentinel - for (date, working_file, rev, author, text) in database: - if text != prevtext: - if prev: - print(sep2, end=' ') - for (p_date, p_working_file, p_rev, p_author) in prev: - print(p_date, p_author, p_working_file, p_rev) - sys.stdout.writelines(prevtext) - prev = [] - prev.append((date, working_file, rev, author)) - prevtext = text - -if __name__ == '__main__': - try: - main() - except IOError as e: - if e.errno != errno.EPIPE: - raise diff --git a/Tools/scripts/mailerdaemon.py b/Tools/scripts/mailerdaemon.py index 4934b92..4934b92 100755..100644 --- a/Tools/scripts/mailerdaemon.py +++ b/Tools/scripts/mailerdaemon.py diff --git a/Tools/scripts/md5sum.py b/Tools/scripts/md5sum.py index 140c0b3..743da72 100644 --- a/Tools/scripts/md5sum.py +++ b/Tools/scripts/md5sum.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """Python utility to print MD5 checksums of argument files. """ diff --git a/Tools/scripts/methfix.py b/Tools/scripts/methfix.py deleted file mode 100755 index f5fe7cd..0000000 --- a/Tools/scripts/methfix.py +++ /dev/null @@ -1,171 +0,0 @@ -#! /usr/bin/env python - -# Fix Python source files to avoid using -# def method(self, (arg1, ..., argn)): -# instead of the more rational -# def method(self, arg1, ..., argn): -# -# Command line arguments are files or directories to be processed. -# Directories are searched recursively for files whose name looks -# like a python module. -# Symbolic links are always ignored (except as explicit directory -# arguments). Of course, the original file is kept as a back-up -# (with a "~" attached to its name). -# It complains about binaries (files containing null bytes) -# and about files that are ostensibly not Python files: if the first -# line starts with '#!' and does not contain the string 'python'. -# -# Changes made are reported to stdout in a diff-like format. -# -# Undoubtedly you can do this using find and sed or perl, but this is -# a nice example of Python code that recurses down a directory tree -# and uses regular expressions. Also note several subtleties like -# preserving the file's mode and avoiding to even write a temp file -# when no changes are needed for a file. -# -# NB: by changing only the function fixline() you can turn this -# into a program for a different change to Python programs... - -import sys -import re -import os -from stat import * - -err = sys.stderr.write -dbg = err -rep = sys.stdout.write - -def main(): - bad = 0 - if not sys.argv[1:]: # No arguments - err('usage: ' + sys.argv[0] + ' file-or-directory ...\n') - sys.exit(2) - for arg in sys.argv[1:]: - if os.path.isdir(arg): - if recursedown(arg): bad = 1 - elif os.path.islink(arg): - err(arg + ': will not process symbolic links\n') - bad = 1 - else: - if fix(arg): bad = 1 - sys.exit(bad) - -ispythonprog = re.compile('^[a-zA-Z0-9_]+\.py$') -def ispython(name): - return ispythonprog.match(name) >= 0 - -def recursedown(dirname): - dbg('recursedown(%r)\n' % (dirname,)) - bad = 0 - try: - names = os.listdir(dirname) - except os.error as msg: - err('%s: cannot list directory: %r\n' % (dirname, msg)) - return 1 - names.sort() - subdirs = [] - for name in names: - if name in (os.curdir, os.pardir): continue - fullname = os.path.join(dirname, name) - if os.path.islink(fullname): pass - elif os.path.isdir(fullname): - subdirs.append(fullname) - elif ispython(name): - if fix(fullname): bad = 1 - for fullname in subdirs: - if recursedown(fullname): bad = 1 - return bad - -def fix(filename): -## dbg('fix(%r)\n' % (filename,)) - try: - f = open(filename, 'r') - except IOError as msg: - err('%s: cannot open: %r\n' % (filename, msg)) - return 1 - head, tail = os.path.split(filename) - tempname = os.path.join(head, '@' + tail) - g = None - # If we find a match, we rewind the file and start over but - # now copy everything to a temp file. - lineno = 0 - while 1: - line = f.readline() - if not line: break - lineno = lineno + 1 - if g is None and '\0' in line: - # Check for binary files - err(filename + ': contains null bytes; not fixed\n') - f.close() - return 1 - if lineno == 1 and g is None and line[:2] == '#!': - # Check for non-Python scripts - words = line[2:].split() - if words and re.search('[pP]ython', words[0]) < 0: - msg = filename + ': ' + words[0] - msg = msg + ' script; not fixed\n' - err(msg) - f.close() - return 1 - while line[-2:] == '\\\n': - nextline = f.readline() - if not nextline: break - line = line + nextline - lineno = lineno + 1 - newline = fixline(line) - if newline != line: - if g is None: - try: - g = open(tempname, 'w') - except IOError as msg: - f.close() - err('%s: cannot create: %r\n' % (tempname, msg)) - return 1 - f.seek(0) - lineno = 0 - rep(filename + ':\n') - continue # restart from the beginning - rep(repr(lineno) + '\n') - rep('< ' + line) - rep('> ' + newline) - if g is not None: - g.write(newline) - - # End of file - f.close() - if not g: return 0 # No changes - - # Finishing touch -- move files - - # First copy the file's mode to the temp file - try: - statbuf = os.stat(filename) - os.chmod(tempname, statbuf[ST_MODE] & 0o7777) - except os.error as msg: - err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) - # Then make a backup of the original file as filename~ - try: - os.rename(filename, filename + '~') - except os.error as msg: - err('%s: warning: backup failed (%r)\n' % (filename, msg)) - # Now move the temp file to the original file - try: - os.rename(tempname, filename) - except os.error as msg: - err('%s: rename failed (%r)\n' % (filename, msg)) - return 1 - # Return succes - return 0 - - -fixpat = '^[ \t]+def +[a-zA-Z0-9_]+ *( *self *, *(( *(.*) *)) *) *:' -fixprog = re.compile(fixpat) - -def fixline(line): - if fixprog.match(line) >= 0: - (a, b), (c, d) = fixprog.regs[1:3] - line = line[:a] + line[c:d] + line[b:] - return line - -if __name__ == '__main__': - main() diff --git a/Tools/scripts/mkreal.py b/Tools/scripts/mkreal.py index 8bc2ec1..b21909e 100755 --- a/Tools/scripts/mkreal.py +++ b/Tools/scripts/mkreal.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # mkreal # diff --git a/Tools/scripts/ndiff.py b/Tools/scripts/ndiff.py index c60c8a8..2422091 100755 --- a/Tools/scripts/ndiff.py +++ b/Tools/scripts/ndiff.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Module ndiff version 1.7.0 # Released to the public domain 08-Dec-2000, diff --git a/Tools/scripts/nm2def.py b/Tools/scripts/nm2def.py index 9dfb991..8f07559 100755 --- a/Tools/scripts/nm2def.py +++ b/Tools/scripts/nm2def.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """nm2def.py Helpers to extract symbols from Unix libs and auto-generate diff --git a/Tools/scripts/objgraph.py b/Tools/scripts/objgraph.py index 0975a3b..1e1fce0 100755 --- a/Tools/scripts/objgraph.py +++ b/Tools/scripts/objgraph.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # objgraph # diff --git a/Tools/scripts/parseentities.py b/Tools/scripts/parseentities.py index 8d93167..5b0f1c6 100755 --- a/Tools/scripts/parseentities.py +++ b/Tools/scripts/parseentities.py @@ -1,4 +1,4 @@ -#!/usr/local/bin/python +#!/usr/bin/env python3 """ Utility for parsing HTML entity definitions available from: http://www.w3.org/ as e.g. diff --git a/Tools/scripts/pathfix.py b/Tools/scripts/pathfix.py index 6cbac10..dd08e0a 100755 --- a/Tools/scripts/pathfix.py +++ b/Tools/scripts/pathfix.py @@ -30,20 +30,24 @@ dbg = err rep = sys.stdout.write new_interpreter = None +preserve_timestamps = False def main(): global new_interpreter - usage = ('usage: %s -i /interpreter file-or-directory ...\n' % + global preserve_timestamps + usage = ('usage: %s -i /interpreter -p file-or-directory ...\n' % sys.argv[0]) try: - opts, args = getopt.getopt(sys.argv[1:], 'i:') + opts, args = getopt.getopt(sys.argv[1:], 'i:p') except getopt.error as msg: - err(msg + '\n') + err(str(msg) + '\n') err(usage) sys.exit(2) for o, a in opts: if o == '-i': new_interpreter = a.encode() + if o == '-p': + preserve_timestamps = True if not new_interpreter or not new_interpreter.startswith(b'/') or \ not args: err('-i option or file-or-directory missing\n') @@ -119,9 +123,13 @@ def fix(filename): # Finishing touch -- move files + mtime = None + atime = None # First copy the file's mode to the temp file try: statbuf = os.stat(filename) + mtime = statbuf.st_mtime + atime = statbuf.st_atime os.chmod(tempname, statbuf[ST_MODE] & 0o7777) except os.error as msg: err('%s: warning: chmod failed (%r)\n' % (tempname, msg)) @@ -136,6 +144,13 @@ def fix(filename): except os.error as msg: err('%s: rename failed (%r)\n' % (filename, msg)) return 1 + if preserve_timestamps: + if atime and mtime: + try: + os.utime(filename, (atime, mtime)) + except os.error as msg: + err('%s: reset of timestamp failed (%r)\n' % (filename, msg)) + return 1 # Return succes return 0 diff --git a/Tools/scripts/pdeps.py b/Tools/scripts/pdeps.py index 5c5a05b..938f31c 100755 --- a/Tools/scripts/pdeps.py +++ b/Tools/scripts/pdeps.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # pdeps # diff --git a/Tools/scripts/pickle2db.py b/Tools/scripts/pickle2db.py index a43ffae..b5b6571 100644 --- a/Tools/scripts/pickle2db.py +++ b/Tools/scripts/pickle2db.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Synopsis: %(prog)s [-h|-b|-g|-r|-a|-d] [ picklefile ] dbfile diff --git a/Tools/scripts/pindent.py b/Tools/scripts/pindent.py index 3f3000d..15b6399 100755 --- a/Tools/scripts/pindent.py +++ b/Tools/scripts/pindent.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # This file contains a class and a main program that perform three # related (though complimentary) formatting operations on Python @@ -88,10 +88,10 @@ next = {} next['if'] = next['elif'] = 'elif', 'else', 'end' next['while'] = next['for'] = 'else', 'end' next['try'] = 'except', 'finally' -next['except'] = 'except', 'else', 'end' +next['except'] = 'except', 'else', 'finally', 'end' next['else'] = next['finally'] = next['def'] = next['class'] = 'end' next['end'] = () -start = 'if', 'while', 'for', 'try', 'def', 'class' +start = 'if', 'while', 'for', 'try', 'with', 'def', 'class' class PythonIndenter: diff --git a/Tools/scripts/ptags.py b/Tools/scripts/ptags.py index ac01356..ca643b3 100755 --- a/Tools/scripts/ptags.py +++ b/Tools/scripts/ptags.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # ptags # diff --git a/Tools/scripts/pysource.py b/Tools/scripts/pysource.py index 05c2b86..048131e 100644 --- a/Tools/scripts/pysource.py +++ b/Tools/scripts/pysource.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """\ List python source files. diff --git a/Tools/scripts/reindent.py b/Tools/scripts/reindent.py index 8557b5d..bb41520 100755 --- a/Tools/scripts/reindent.py +++ b/Tools/scripts/reindent.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Released to the public domain, by Tim Peters, 03 October 2000. @@ -42,26 +42,27 @@ you'd prefer. You can always use the --nobackup option to prevent this. __version__ = "1" import tokenize -import os, shutil +import os +import shutil import sys -verbose = 0 -recurse = 0 -dryrun = 0 +verbose = False +recurse = False +dryrun = False makebackup = True + def usage(msg=None): - if msg is not None: - print(msg, file=sys.stderr) - print(__doc__, file=sys.stderr) + if msg is None: + msg = __doc__ + print(msg, file=sys.stderr) + def errprint(*args): - sep = "" - for arg in args: - sys.stderr.write(sep + str(arg)) - sep = " " + sys.stderr.write(" ".join(str(arg) for arg in args)) sys.stderr.write("\n") + def main(): import getopt global verbose, recurse, dryrun, makebackup @@ -73,13 +74,13 @@ def main(): return for o, a in opts: if o in ('-d', '--dryrun'): - dryrun += 1 + dryrun = True elif o in ('-r', '--recurse'): - recurse += 1 + recurse = True elif o in ('-n', '--nobackup'): makebackup = False elif o in ('-v', '--verbose'): - verbose += 1 + verbose = True elif o in ('-h', '--help'): usage() return @@ -91,6 +92,7 @@ def main(): for arg in args: check(arg) + def check(file): if os.path.isdir(file) and not os.path.islink(file): if verbose: @@ -107,14 +109,15 @@ def check(file): if verbose: print("checking", file, "...", end=' ') + with open(file, 'rb') as f: + encoding, _ = tokenize.detect_encoding(f.readline) try: - f = open(file) + with open(file, encoding=encoding) as f: + r = Reindenter(f) except IOError as msg: errprint("%s: I/O Error: %s" % (file, str(msg))) return - r = Reindenter(f) - f.close() if r.run(): if verbose: print("changed.") @@ -126,9 +129,8 @@ def check(file): shutil.copyfile(file, bak) if verbose: print("backed up", file, "to", bak) - f = open(file, "w") - r.write(f) - f.close() + with open(file, "w", encoding=encoding) as f: + r.write(f) if verbose: print("wrote new", file) return True @@ -137,6 +139,7 @@ def check(file): print("unchanged.") return False + def _rstrip(line, JUNK='\n \t'): """Return line stripped of trailing spaces, tabs, newlines. @@ -146,10 +149,11 @@ def _rstrip(line, JUNK='\n \t'): """ i = len(line) - while i > 0 and line[i-1] in JUNK: + while i > 0 and line[i - 1] in JUNK: i -= 1 return line[:i] + class Reindenter: def __init__(self, f): @@ -192,9 +196,9 @@ class Reindenter: # we see a line with *something* on it. i = stats[0][0] after.extend(lines[1:i]) - for i in range(len(stats)-1): + for i in range(len(stats) - 1): thisstmt, thislevel = stats[i] - nextstmt = stats[i+1][0] + nextstmt = stats[i + 1][0] have = getlspace(lines[thisstmt]) want = thislevel * 4 if want < 0: @@ -206,7 +210,7 @@ class Reindenter: want = have2want.get(have, -1) if want < 0: # Then it probably belongs to the next real stmt. - for j in range(i+1, len(stats)-1): + for j in range(i + 1, len(stats) - 1): jline, jlevel = stats[j] if jlevel >= 0: if have == getlspace(lines[jline]): @@ -216,11 +220,11 @@ class Reindenter: # comment like this one, # in which case we should shift it like its base # line got shifted. - for j in range(i-1, -1, -1): + for j in range(i - 1, -1, -1): jline, jlevel = stats[j] if jlevel >= 0: - want = have + getlspace(after[jline-1]) - \ - getlspace(lines[jline]) + want = have + (getlspace(after[jline - 1]) - + getlspace(lines[jline])) break if want < 0: # Still no luck -- leave it alone. @@ -295,6 +299,7 @@ class Reindenter: if line: # not endmarker self.stats.append((slinecol[0], self.level)) + # Count number of leading blanks. def getlspace(line): i, n = 0, len(line) @@ -302,5 +307,6 @@ def getlspace(line): i += 1 return i + if __name__ == '__main__': main() diff --git a/Tools/scripts/rgrep.py b/Tools/scripts/rgrep.py index 12d736e..1917e05 100755 --- a/Tools/scripts/rgrep.py +++ b/Tools/scripts/rgrep.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """Reverse grep. @@ -9,8 +9,9 @@ import sys import re import getopt + def main(): - bufsize = 64*1024 + bufsize = 64 * 1024 reflags = 0 opts, args = getopt.getopt(sys.argv[1:], "i") for o, a in opts: @@ -24,11 +25,11 @@ def main(): try: prog = re.compile(pattern, reflags) except re.error as msg: - usage("error in regular expression: %s" % str(msg)) + usage("error in regular expression: %s" % msg) try: f = open(filename) except IOError as msg: - usage("can't open %s: %s" % (repr(filename), str(msg)), 1) + usage("can't open %r: %s" % (filename, msg), 1) f.seek(0, 2) pos = f.tell() leftover = None @@ -49,16 +50,17 @@ def main(): del lines[0] else: leftover = None - lines.reverse() - for line in lines: + for line in reversed(lines): if prog.search(line): print(line) + def usage(msg, code=2): sys.stdout = sys.stderr print(msg) print(__doc__) sys.exit(code) + if __name__ == '__main__': main() diff --git a/Tools/scripts/serve.py b/Tools/scripts/serve.py new file mode 100755 index 0000000..89b3d62 --- /dev/null +++ b/Tools/scripts/serve.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +''' +Small wsgiref based web server. Takes a path to serve from and an +optional port number (defaults to 8000), then tries to serve files. +Mime types are guessed from the file names, 404 errors are thrown +if the file is not found. Used for the make serve target in Doc. +''' +import sys +import os +import mimetypes +from wsgiref import simple_server, util + +def app(environ, respond): + + fn = os.path.join(path, environ['PATH_INFO'][1:]) + if '.' not in fn.split(os.path.sep)[-1]: + fn = os.path.join(fn, 'index.html') + type = mimetypes.guess_type(fn)[0] + + if os.path.exists(fn): + respond('200 OK', [('Content-Type', type)]) + return util.FileWrapper(open(fn, "rb")) + else: + respond('404 Not Found', [('Content-Type', 'text/plain')]) + return ['not found'] + +if __name__ == '__main__': + path = sys.argv[1] + port = int(sys.argv[2]) if len(sys.argv) > 2 else 8000 + httpd = simple_server.make_server('', port, app) + print("Serving {} on port {}, control-C to stop".format(path, port)) + try: + httpd.serve_forever() + except KeyboardInterrupt: + print("\b\bShutting down.") diff --git a/Tools/scripts/setup.py b/Tools/scripts/setup.py deleted file mode 100644 index 7a50368..0000000 --- a/Tools/scripts/setup.py +++ /dev/null @@ -1,20 +0,0 @@ -from distutils.core import setup - -if __name__ == '__main__': - setup( - scripts=[ - 'byteyears.py', - 'checkpyc.py', - 'copytime.py', - 'crlf.py', - 'dutree.py', - 'ftpmirror.py', - 'h2py.py', - 'lfcr.py', - '../i18n/pygettext.py', - 'logmerge.py', - '../../Lib/tabnanny.py', - '../../Lib/timeit.py', - 'untabify.py', - ], - ) diff --git a/Tools/scripts/suff.py b/Tools/scripts/suff.py index 462ec32..0eea0d7 100755 --- a/Tools/scripts/suff.py +++ b/Tools/scripts/suff.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # suff # @@ -6,24 +6,21 @@ import sys + def main(): files = sys.argv[1:] suffixes = {} for filename in files: suff = getsuffix(filename) - if suff not in suffixes: - suffixes[suff] = [] - suffixes[suff].append(filename) - keys = sorted(suffixes.keys()) - for suff in keys: - print(repr(suff), len(suffixes[suff])) + suffixes.setdefault(suff, []).append(filename) + for suff, filenames in sorted(suffixes.items()): + print(repr(suff), len(filenames)) + def getsuffix(filename): - suff = '' - for i in range(len(filename)): - if filename[i] == '.': - suff = filename[i:] - return suff + name, sep, suff = filename.rpartition('.') + return sep + suff if sep else '' + if __name__ == '__main__': main() diff --git a/Tools/scripts/svneol.py b/Tools/scripts/svneol.py index 9357c7e..8abdd01 100644 --- a/Tools/scripts/svneol.py +++ b/Tools/scripts/svneol.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """ SVN helper script. @@ -32,9 +32,12 @@ and for a file with a binary mime-type property: import re import os +import sys +import subprocess + def propfiles(root, fn): - default = os.path.join(root, ".svn", "props", fn+".svn-work") + default = os.path.join(root, ".svn", "props", fn + ".svn-work") try: format = int(open(os.path.join(root, ".svn", "format")).read().strip()) except IOError: @@ -42,12 +45,13 @@ def propfiles(root, fn): if format in (8, 9): # In version 8 and 9, committed props are stored in prop-base, local # modifications in props - return [os.path.join(root, ".svn", "prop-base", fn+".svn-base"), - os.path.join(root, ".svn", "props", fn+".svn-work")] - raise ValueError, "Unknown repository format" + return [os.path.join(root, ".svn", "prop-base", fn + ".svn-base"), + os.path.join(root, ".svn", "props", fn + ".svn-work")] + raise ValueError("Unknown repository format") + def proplist(root, fn): - "Return a list of property names for file fn in directory root" + """Return a list of property names for file fn in directory root.""" result = [] for path in propfiles(root, fn): try: @@ -56,7 +60,7 @@ def proplist(root, fn): # no properties file: not under version control, # or no properties set continue - while 1: + while True: # key-value pairs, of the form # K <length> # <keyname>NL @@ -79,13 +83,32 @@ def proplist(root, fn): f.close() return result + +def set_eol_native(path): + cmd = 'svn propset svn:eol-style native "{}"'.format(path) + propset = subprocess.Popen(cmd, shell=True) + propset.wait() + + possible_text_file = re.compile(r"\.([hc]|py|txt|sln|vcproj)$").search -for root, dirs, files in os.walk('.'): - if '.svn' in dirs: - dirs.remove('.svn') - for fn in files: - if possible_text_file(fn): + +def main(): + for arg in sys.argv[1:] or [os.curdir]: + if os.path.isfile(arg): + root, fn = os.path.split(arg) if 'svn:eol-style' not in proplist(root, fn): - path = os.path.join(root, fn) - os.system('svn propset svn:eol-style native "%s"' % path) + set_eol_native(arg) + elif os.path.isdir(arg): + for root, dirs, files in os.walk(arg): + if '.svn' in dirs: + dirs.remove('.svn') + for fn in files: + if possible_text_file(fn): + if 'svn:eol-style' not in proplist(root, fn): + path = os.path.join(root, fn) + set_eol_native(path) + + +if __name__ == '__main__': + main() diff --git a/Tools/scripts/texi2html.py b/Tools/scripts/texi2html.py index 86229f2..af2147a 100755 --- a/Tools/scripts/texi2html.py +++ b/Tools/scripts/texi2html.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Convert GNU texinfo files into HTML, one file per node. # Based on Texinfo 2.14. diff --git a/Tools/scripts/treesync.py b/Tools/scripts/treesync.py index 8643ee7..b2649c4 100755 --- a/Tools/scripts/treesync.py +++ b/Tools/scripts/treesync.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 """Script to synchronize two source trees. diff --git a/Tools/scripts/untabify.py b/Tools/scripts/untabify.py index 1f45520..4b67c15 100755 --- a/Tools/scripts/untabify.py +++ b/Tools/scripts/untabify.py @@ -1,10 +1,11 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 "Replace tabs with spaces in argument files. Print names of changed files." import os import sys import getopt +import tokenize def main(): tabsize = 8 @@ -23,11 +24,12 @@ def main(): for filename in args: process(filename, tabsize) + def process(filename, tabsize, verbose=True): try: - f = open(filename) - text = f.read() - f.close() + with tokenize.open(filename) as f: + text = f.read() + encoding = f.encoding except IOError as msg: print("%r: I/O error: %s" % (filename, msg)) return @@ -43,11 +45,11 @@ def process(filename, tabsize, verbose=True): os.rename(filename, backup) except os.error: pass - f = open(filename, "w") - f.write(newtext) - f.close() + with open(filename, "w", encoding=encoding) as f: + f.write(newtext) if verbose: print(filename) + if __name__ == '__main__': main() diff --git a/Tools/scripts/which.py b/Tools/scripts/which.py index a9f4907..4fc37a0 100755 --- a/Tools/scripts/which.py +++ b/Tools/scripts/which.py @@ -1,4 +1,4 @@ -#! /usr/bin/env python +#! /usr/bin/env python3 # Variant of "which". # On stderr, near and total misses are reported. diff --git a/Tools/scripts/xxci.py b/Tools/scripts/xxci.py deleted file mode 100755 index 8cffc9e..0000000 --- a/Tools/scripts/xxci.py +++ /dev/null @@ -1,121 +0,0 @@ -#! /usr/bin/env python - -# xxci -# -# check in files for which rcsdiff returns nonzero exit status - -import sys -import os -from stat import * -import fnmatch - -EXECMAGIC = '\001\140\000\010' - -MAXSIZE = 200*1024 # Files this big must be binaries and are skipped. - -def getargs(): - args = sys.argv[1:] - if args: - return args - print('No arguments, checking almost *, in "ls -t" order') - list = [] - for file in os.listdir(os.curdir): - if not skipfile(file): - list.append((getmtime(file), file)) - list.sort() - if not list: - print('Nothing to do -- exit 1') - sys.exit(1) - list.sort() - list.reverse() - for mtime, file in list: args.append(file) - return args - -def getmtime(file): - try: - st = os.stat(file) - return st[ST_MTIME] - except os.error: - return -1 - -badnames = ['tags', 'TAGS', 'xyzzy', 'nohup.out', 'core'] -badprefixes = ['.', ',', '@', '#', 'o.'] -badsuffixes = \ - ['~', '.a', '.o', '.old', '.bak', '.orig', '.new', '.prev', '.not', \ - '.pyc', '.fdc', '.rgb', '.elc', ',v'] -ignore = [] - -def setup(): - ignore[:] = badnames - for p in badprefixes: - ignore.append(p + '*') - for p in badsuffixes: - ignore.append('*' + p) - try: - f = open('.xxcign', 'r') - except IOError: - return - ignore[:] = ignore + f.read().split() - -def skipfile(file): - for p in ignore: - if fnmatch.fnmatch(file, p): return 1 - try: - st = os.lstat(file) - except os.error: - return 1 # Doesn't exist -- skip it - # Skip non-plain files. - if not S_ISREG(st[ST_MODE]): return 1 - # Skip huge files -- probably binaries. - if st[ST_SIZE] >= MAXSIZE: return 1 - # Skip executables - try: - data = open(file, 'r').read(len(EXECMAGIC)) - if data == EXECMAGIC: return 1 - except: - pass - return 0 - -def badprefix(file): - for bad in badprefixes: - if file[:len(bad)] == bad: return 1 - return 0 - -def badsuffix(file): - for bad in badsuffixes: - if file[-len(bad):] == bad: return 1 - return 0 - -def go(args): - for file in args: - print(file + ':') - if differing(file): - showdiffs(file) - if askyesno('Check in ' + file + ' ? '): - sts = os.system('rcs -l ' + file) # ignored - sts = os.system('ci -l ' + file) - -def differing(file): - cmd = 'co -p ' + file + ' 2>/dev/null | cmp -s - ' + file - sts = os.system(cmd) - return sts != 0 - -def showdiffs(file): - cmd = 'rcsdiff ' + file + ' 2>&1 | ${PAGER-more}' - sts = os.system(cmd) - -def raw_input(prompt): - sys.stdout.write(prompt) - sys.stdout.flush() - return sys.stdin.readline() - -def askyesno(prompt): - s = input(prompt) - return s in ['y', 'yes'] - -if __name__ == '__main__': - try: - setup() - go(getargs()) - except KeyboardInterrupt: - print('[Intr]') diff --git a/Tools/test2to3/README b/Tools/test2to3/README new file mode 100644 index 0000000..9365593 --- /dev/null +++ b/Tools/test2to3/README @@ -0,0 +1,3 @@ +This project demonstrates how a distutils package +can support Python 2.x and Python 3.x from a single +source, using lib2to3.
\ No newline at end of file diff --git a/Tools/test2to3/maintest.py b/Tools/test2to3/maintest.py new file mode 100644 index 0000000..036dd4f --- /dev/null +++ b/Tools/test2to3/maintest.py @@ -0,0 +1,10 @@ +#!/usr/bin/env python3 + +# The above line should get replaced with the path to the Python +# interpreter; the block below should get 2to3-converted. + +try: + from test2to3.hello import hello +except ImportError, e: + print "Import failed", e +hello() diff --git a/Tools/test2to3/setup.py b/Tools/test2to3/setup.py new file mode 100644 index 0000000..a0f9024 --- /dev/null +++ b/Tools/test2to3/setup.py @@ -0,0 +1,26 @@ +# -*- coding: iso-8859-1 -*- +from distutils.core import setup + +try: + from distutils.command.build_py import build_py_2to3 as build_py +except ImportError: + from distutils.command.build_py import build_py + +try: + from distutils.command.build_scripts import build_scripts_2to3 as build_scripts +except ImportError: + from distutils.command.build_scripts import build_scripts + +setup( + name = "test2to3", + version = "1.0", + description = "2to3 distutils test package", + author = "Martin v. Löwis", + author_email = "python-dev@python.org", + license = "PSF license", + packages = ["test2to3"], + scripts = ["maintest.py"], + cmdclass = {'build_py': build_py, + 'build_scripts': build_scripts, + } +) diff --git a/Tools/test2to3/test/runtests.py b/Tools/test2to3/test/runtests.py new file mode 100644 index 0000000..1730f0d --- /dev/null +++ b/Tools/test2to3/test/runtests.py @@ -0,0 +1,19 @@ +# Fictitious test runner for the project + +import sys, os + +if sys.version_info > (3,): + # copy test suite over to "build/lib" and convert it + from distutils.util import copydir_run_2to3 + testroot = os.path.dirname(__file__) + newroot = os.path.join(testroot, '..', 'build/lib/test') + copydir_run_2to3(testroot, newroot) + # in the following imports, pick up the converted modules + sys.path[0] = newroot + +# run the tests here... + +from test_foo import FooTest + +import unittest +unittest.main() diff --git a/Tools/test2to3/test/test_foo.py b/Tools/test2to3/test/test_foo.py new file mode 100644 index 0000000..ec8f26a --- /dev/null +++ b/Tools/test2to3/test/test_foo.py @@ -0,0 +1,8 @@ +import sys +import unittest + +class FooTest(unittest.TestCase): + def test_foo(self): + # use 2.6 syntax to demonstrate conversion + print 'In test_foo, using Python %s...' % (sys.version_info,) + self.assertTrue(False) diff --git a/Tools/test2to3/test2to3/__init__.py b/Tools/test2to3/test2to3/__init__.py new file mode 100644 index 0000000..1bb8bf6 --- /dev/null +++ b/Tools/test2to3/test2to3/__init__.py @@ -0,0 +1 @@ +# empty diff --git a/Tools/test2to3/test2to3/hello.py b/Tools/test2to3/test2to3/hello.py new file mode 100644 index 0000000..f52926b --- /dev/null +++ b/Tools/test2to3/test2to3/hello.py @@ -0,0 +1,5 @@ +def hello(): + try: + print "Hello, world" + except IOError, e: + print e.errno diff --git a/Tools/unicode/comparecodecs.py b/Tools/unicode/comparecodecs.py index dade1ce..0f5c1e2 100644 --- a/Tools/unicode/comparecodecs.py +++ b/Tools/unicode/comparecodecs.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """ Compare the output of two codecs. diff --git a/Tools/unicode/gencodec.py b/Tools/unicode/gencodec.py index c3846e9..7e7d6d0 100644 --- a/Tools/unicode/gencodec.py +++ b/Tools/unicode/gencodec.py @@ -34,14 +34,16 @@ MAX_TABLE_SIZE = 8192 # Standard undefined Unicode code point UNI_UNDEFINED = chr(0xFFFE) +# Placeholder for a missing codepoint +MISSING_CODE = -1 + mapRE = re.compile('((?:0x[0-9a-fA-F]+\+?)+)' '\s+' '((?:(?:0x[0-9a-fA-Z]+|<[A-Za-z]+>)\+?)*)' '\s*' '(#.+)?') -def parsecodes(codes, - len=len, filter=filter,range=range): +def parsecodes(codes, len=len, range=range): """ Converts code combinations to either a single code integer or a tuple of integers. @@ -53,7 +55,7 @@ def parsecodes(codes, """ if not codes: - return None + return MISSING_CODE l = codes.split('+') if len(l) == 1: return int(l[0],16) @@ -61,8 +63,8 @@ def parsecodes(codes, try: l[i] = int(l[i],16) except ValueError: - l[i] = None - l = [x for x in l if x is not None] + l[i] = MISSING_CODE + l = [x for x in l if x != MISSING_CODE] if len(l) == 1: return l[0] else: @@ -114,7 +116,7 @@ def readmap(filename): # mappings to None for the rest if len(identity) >= len(unmapped): for enc in unmapped: - enc2uni[enc] = (None, "") + enc2uni[enc] = (MISSING_CODE, "") enc2uni['IDENTITY'] = 256 return enc2uni @@ -212,7 +214,7 @@ def python_tabledef_code(varname, map, comments=1, key_precision=2): (mapkey, mapcomment) = mapkey if isinstance(mapvalue, tuple): (mapvalue, mapcomment) = mapvalue - if mapkey is None: + if mapkey == MISSING_CODE: continue table[mapkey] = (mapvalue, mapcomment) if mapkey > maxkey: @@ -224,11 +226,11 @@ def python_tabledef_code(varname, map, comments=1, key_precision=2): # Create table code for key in range(maxkey + 1): if key not in table: - mapvalue = None + mapvalue = MISSING_CODE mapcomment = 'UNDEFINED' else: mapvalue, mapcomment = table[key] - if mapvalue is None: + if mapvalue == MISSING_CODE: mapchar = UNI_UNDEFINED else: if isinstance(mapvalue, tuple): @@ -237,11 +239,11 @@ def python_tabledef_code(varname, map, comments=1, key_precision=2): else: mapchar = chr(mapvalue) if mapcomment and comments: - append(' %r\t# %s -> %s' % (mapchar, + append(' %a \t# %s -> %s' % (mapchar, hexrepr(key, key_precision), mapcomment)) else: - append(' %r' % mapchar) + append(' %a' % mapchar) append(')') return l diff --git a/Tools/unicode/genwincodec.py b/Tools/unicode/genwincodec.py new file mode 100644 index 0000000..7a1ef5e --- /dev/null +++ b/Tools/unicode/genwincodec.py @@ -0,0 +1,61 @@ +"""This script generates a Python codec module from a Windows Code Page. + +It uses the function MultiByteToWideChar to generate a decoding table. +""" + +import ctypes +from ctypes import wintypes +from gencodec import codegen +import unicodedata + +def genwinmap(codepage): + MultiByteToWideChar = ctypes.windll.kernel32.MultiByteToWideChar + MultiByteToWideChar.argtypes = [wintypes.UINT, wintypes.DWORD, + wintypes.LPCSTR, ctypes.c_int, + wintypes.LPWSTR, ctypes.c_int] + MultiByteToWideChar.restype = ctypes.c_int + + enc2uni = {} + + for i in list(range(32)) + [127]: + enc2uni[i] = (i, 'CONTROL CHARACTER') + + for i in range(256): + buf = ctypes.create_unicode_buffer(2) + ret = MultiByteToWideChar( + codepage, 0, + bytes([i]), 1, + buf, 2) + assert ret == 1, "invalid code page" + assert buf[1] == '\x00' + try: + name = unicodedata.name(buf[0]) + except ValueError: + try: + name = enc2uni[i][1] + except KeyError: + name = '' + + enc2uni[i] = (ord(buf[0]), name) + + return enc2uni + +def genwincodec(codepage): + import platform + map = genwinmap(codepage) + encodingname = 'cp%d' % codepage + code = codegen("", map, encodingname) + # Replace first lines with our own docstring + code = '''\ +"""Python Character Mapping Codec %s generated on Windows: +%s with the command: + python Tools/unicode/genwincodec.py %s +"""#" +''' % (encodingname, ' '.join(platform.win32_ver()), codepage + ) + code.split('"""#"', 1)[1] + + print(code) + +if __name__ == '__main__': + import sys + genwincodec(int(sys.argv[1])) diff --git a/Tools/unicode/genwincodecs.bat b/Tools/unicode/genwincodecs.bat new file mode 100644 index 0000000..43cab0d --- /dev/null +++ b/Tools/unicode/genwincodecs.bat @@ -0,0 +1,7 @@ +@rem Recreate some python charmap codecs from the Windows function +@rem MultiByteToWideChar. + +@cd /d %~dp0 +@mkdir build +@rem Arabic DOS code page +c:\python30\python genwincodec.py 720 > build/cp720.py diff --git a/Tools/unicode/makeunicodedata.py b/Tools/unicode/makeunicodedata.py index c35170c..d503190 100644 --- a/Tools/unicode/makeunicodedata.py +++ b/Tools/unicode/makeunicodedata.py @@ -25,18 +25,20 @@ # written by Fredrik Lundh (fredrik@pythonware.com) # -import sys +import sys, os, zipfile SCRIPT = sys.argv[0] -VERSION = "2.6" +VERSION = "3.2" # The Unicode Database -UNIDATA_VERSION = "5.1.0" +UNIDATA_VERSION = "6.0.0" UNICODE_DATA = "UnicodeData%s.txt" COMPOSITION_EXCLUSIONS = "CompositionExclusions%s.txt" EASTASIAN_WIDTH = "EastAsianWidth%s.txt" +UNIHAN = "Unihan%s.zip" DERIVED_CORE_PROPERTIES = "DerivedCoreProperties%s.txt" DERIVEDNORMALIZATION_PROPS = "DerivedNormalizationProps%s.txt" +LINE_BREAK = "LineBreak%s.txt" old_versions = ["3.2.0"] @@ -51,6 +53,8 @@ BIDIRECTIONAL_NAMES = [ "", "L", "LRE", "LRO", "R", "AL", "RLE", "RLO", EASTASIANWIDTH_NAMES = [ "F", "H", "W", "Na", "A", "N" ] +MANDATORY_LINE_BREAKS = [ "BK", "CR", "LF", "NL" ] + # note: should match definitions in Objects/unicodectype.c ALPHA_MASK = 0x01 DECIMAL_MASK = 0x02 @@ -64,26 +68,29 @@ XID_START_MASK = 0x100 XID_CONTINUE_MASK = 0x200 PRINTABLE_MASK = 0x400 NODELTA_MASK = 0x800 +NUMERIC_MASK = 0x1000 + +# these ranges need to match unicodedata.c:is_unified_ideograph +cjk_ranges = [ + ('3400', '4DB5'), + ('4E00', '9FCB'), + ('20000', '2A6D6'), + ('2A700', '2B734'), + ('2B740', '2B81D') +] def maketables(trace=0): print("--- Reading", UNICODE_DATA % "", "...") version = "" - unicode = UnicodeData(UNICODE_DATA % version, - COMPOSITION_EXCLUSIONS % version, - EASTASIAN_WIDTH % version, - DERIVED_CORE_PROPERTIES % version, - DERIVEDNORMALIZATION_PROPS % version) + unicode = UnicodeData(UNIDATA_VERSION) print(len(list(filter(None, unicode.table))), "characters") for version in old_versions: print("--- Reading", UNICODE_DATA % ("-"+version), "...") - old_unicode = UnicodeData(UNICODE_DATA % ("-"+version), - COMPOSITION_EXCLUSIONS % ("-"+version), - EASTASIAN_WIDTH % ("-"+version), - DERIVED_CORE_PROPERTIES % ("-"+version)) + old_unicode = UnicodeData(version, cjk_check=False) print(len(list(filter(None, old_unicode.table))), "characters") merge_old_version(version, unicode, old_unicode) @@ -357,6 +364,9 @@ def makeunicodetype(unicode, trace): table = [dummy] cache = {0: dummy} index = [0] * len(unicode.chars) + numeric = {} + spaces = [] + linebreaks = [] for char in unicode.chars: record = unicode.table[char] @@ -371,10 +381,12 @@ def makeunicodetype(unicode, trace): flags |= ALPHA_MASK if category == "Ll": flags |= LOWER_MASK - if category == "Zl" or bidirectional == "B": + if 'Line_Break' in properties or bidirectional == "B": flags |= LINEBREAK_MASK + linebreaks.append(char) if category == "Zs" or bidirectional in ("WS", "B", "S"): flags |= SPACE_MASK + spaces.append(char) if category == "Lt": flags |= TITLE_MASK if category == "Lu": @@ -423,6 +435,9 @@ def makeunicodetype(unicode, trace): if record[7]: flags |= DIGIT_MASK digit = int(record[7]) + if record[8]: + flags |= NUMERIC_MASK + numeric.setdefault(record[8], []).append(char) item = ( upper, lower, title, decimal, digit, flags ) @@ -434,6 +449,9 @@ def makeunicodetype(unicode, trace): index[char] = i print(len(table), "unique character type entries") + print(sum(map(len, numeric.values())), "numeric code points") + print(len(spaces), "whitespace code points") + print(len(linebreaks), "linebreak code points") print("--- Writing", FILE, "...") @@ -455,6 +473,63 @@ def makeunicodetype(unicode, trace): Array("index1", index1).dump(fp, trace) Array("index2", index2).dump(fp, trace) + # Generate code for _PyUnicode_ToNumeric() + numeric_items = sorted(numeric.items()) + print('/* Returns the numeric value as double for Unicode characters', file=fp) + print(' * having this property, -1.0 otherwise.', file=fp) + print(' */', file=fp) + print('double _PyUnicode_ToNumeric(Py_UCS4 ch)', file=fp) + print('{', file=fp) + print(' switch (ch) {', file=fp) + for value, codepoints in numeric_items: + # Turn text into float literals + parts = value.split('/') + parts = [repr(float(part)) for part in parts] + value = '/'.join(parts) + + codepoints.sort() + for codepoint in codepoints: + print(' case 0x%04X:' % (codepoint,), file=fp) + print(' return (double) %s;' % (value,), file=fp) + print(' }', file=fp) + print(' return -1.0;', file=fp) + print('}', file=fp) + print(file=fp) + + # Generate code for _PyUnicode_IsWhitespace() + print("/* Returns 1 for Unicode characters having the bidirectional", file=fp) + print(" * type 'WS', 'B' or 'S' or the category 'Zs', 0 otherwise.", file=fp) + print(" */", file=fp) + print('int _PyUnicode_IsWhitespace(register const Py_UCS4 ch)', file=fp) + print('{', file=fp) + print(' switch (ch) {', file=fp) + + for codepoint in sorted(spaces): + print(' case 0x%04X:' % (codepoint,), file=fp) + print(' return 1;', file=fp) + + print(' }', file=fp) + print(' return 0;', file=fp) + print('}', file=fp) + print(file=fp) + + # Generate code for _PyUnicode_IsLinebreak() + print("/* Returns 1 for Unicode characters having the line break", file=fp) + print(" * property 'BK', 'CR', 'LF' or 'NL' or having bidirectional", file=fp) + print(" * type 'B', 0 otherwise.", file=fp) + print(" */", file=fp) + print('int _PyUnicode_IsLinebreak(register const Py_UCS4 ch)', file=fp) + print('{', file=fp) + print(' switch (ch) {', file=fp) + for codepoint in sorted(linebreaks): + print(' case 0x%04X:' % (codepoint,), file=fp) + print(' return 1;', file=fp) + + print(' }', file=fp) + print(' return 0;', file=fp) + print('}', file=fp) + print(file=fp) + fp.close() # -------------------------------------------------------------------- @@ -670,12 +745,11 @@ def merge_old_version(version, new, old): elif k == 8: # print "NUMERIC",hex(i), `old.table[i][k]`, new.table[i][k] # Since 0 encodes "no change", the old value is better not 0 - assert value != "0" and value != "-1" if not value: numeric_changes[i] = -1 else: - assert re.match("^[0-9]+$", value) - numeric_changes[i] = int(value) + numeric_changes[i] = float(value) + assert numeric_changes[i] not in (0, -1) elif k == 9: if value == 'Y': mirrored_changes[i] = '1' @@ -696,6 +770,10 @@ def merge_old_version(version, new, old): elif k == 16: # derived property changes; not yet pass + elif k == 17: + # normalization quickchecks are not performed + # for older versions + pass else: class Difference(Exception):pass raise Difference(hex(i), k, old.table[i], new.table[i]) @@ -704,6 +782,21 @@ def merge_old_version(version, new, old): numeric_changes)), normalization_changes)) +def open_data(template, version): + local = template % ('-'+version,) + if not os.path.exists(local): + import urllib.request + if version == '3.2.0': + # irregular url structure + url = 'http://www.unicode.org/Public/3.2-Update/' + local + else: + url = ('http://www.unicode.org/Public/%s/ucd/'+template) % (version, '') + urllib.request.urlretrieve(url, filename=local) + if local.endswith('.txt'): + return open(local, encoding='utf-8') + else: + # Unihan.zip + return open(local, 'rb') # -------------------------------------------------------------------- # the following support code is taken from the unidb utilities @@ -711,8 +804,6 @@ def merge_old_version(version, new, old): # load a unicode-data file from disk -import sys - class UnicodeData: # Record structure: # [ID, name, category, combining, bidi, decomp, (6) @@ -720,10 +811,12 @@ class UnicodeData: # ISO-comment, uppercase, lowercase, titlecase, ea-width, (16) # derived-props] (17) - def __init__(self, filename, exclusions, eastasianwidth, - derivedprops, derivednormalizationprops=None, expand=1): + def __init__(self, version, + linebreakprops=False, + expand=1, + cjk_check=True): self.changed = [] - file = open(filename) + file = open_data(UNICODE_DATA, version) table = [None] * 0x110000 while 1: s = file.readline() @@ -733,6 +826,8 @@ class UnicodeData: char = int(s[0], 16) table[char] = s + cjk_ranges_found = [] + # expand first-last ranges if expand: field = None @@ -743,19 +838,24 @@ class UnicodeData: s[1] = "" field = s elif s[1][-5:] == "Last>": + if s[1].startswith("<CJK Ideograph"): + cjk_ranges_found.append((field[0], + s[0])) s[1] = "" field = None elif field: f2 = field[:] f2[0] = "%X" % i table[i] = f2 + if cjk_check and cjk_ranges != cjk_ranges_found: + raise ValueError("CJK ranges deviate: have %r" % cjk_ranges_found) # public attributes - self.filename = filename + self.filename = UNICODE_DATA % '' self.table = table self.chars = list(range(0x110000)) # unicode 3.2 - file = open(exclusions) + file = open_data(COMPOSITION_EXCLUSIONS, version) self.exclusions = {} for s in file: s = s.strip() @@ -767,7 +867,7 @@ class UnicodeData: self.exclusions[char] = 1 widths = [None] * 0x110000 - for s in open(eastasianwidth): + for s in open_data(EASTASIAN_WIDTH, version): s = s.strip() if not s: continue @@ -788,7 +888,7 @@ class UnicodeData: for i in range(0, 0x110000): if table[i] is not None: table[i].append(set()) - for s in open(derivedprops): + for s in open_data(DERIVED_CORE_PROPERTIES, version): s = s.split('#', 1)[0].strip() if not s: continue @@ -807,28 +907,64 @@ class UnicodeData: # apply to unassigned code points; ignore them table[char][-1].add(p) - if derivednormalizationprops: - quickchecks = [0] * 0x110000 # default is Yes - qc_order = 'NFD_QC NFKD_QC NFC_QC NFKC_QC'.split() - for s in open(derivednormalizationprops): - if '#' in s: - s = s[:s.index('#')] - s = [i.strip() for i in s.split(';')] - if len(s) < 2 or s[1] not in qc_order: - continue - quickcheck = 'MN'.index(s[2]) + 1 # Maybe or No - quickcheck_shift = qc_order.index(s[1])*2 - quickcheck <<= quickcheck_shift - if '..' not in s[0]: - first = last = int(s[0], 16) - else: - first, last = [int(c, 16) for c in s[0].split('..')] - for char in range(first, last+1): - assert not (quickchecks[char]>>quickcheck_shift)&3 - quickchecks[char] |= quickcheck - for i in range(0, 0x110000): - if table[i] is not None: - table[i].append(quickchecks[i]) + for s in open_data(LINE_BREAK, version): + s = s.partition('#')[0] + s = [i.strip() for i in s.split(';')] + if len(s) < 2 or s[1] not in MANDATORY_LINE_BREAKS: + continue + if '..' not in s[0]: + first = last = int(s[0], 16) + else: + first, last = [int(c, 16) for c in s[0].split('..')] + for char in range(first, last+1): + table[char][-1].add('Line_Break') + + # We only want the quickcheck properties + # Format: NF?_QC; Y(es)/N(o)/M(aybe) + # Yes is the default, hence only N and M occur + # In 3.2.0, the format was different (NF?_NO) + # The parsing will incorrectly determine these as + # "yes", however, unicodedata.c will not perform quickchecks + # for older versions, and no delta records will be created. + quickchecks = [0] * 0x110000 + qc_order = 'NFD_QC NFKD_QC NFC_QC NFKC_QC'.split() + for s in open_data(DERIVEDNORMALIZATION_PROPS, version): + if '#' in s: + s = s[:s.index('#')] + s = [i.strip() for i in s.split(';')] + if len(s) < 2 or s[1] not in qc_order: + continue + quickcheck = 'MN'.index(s[2]) + 1 # Maybe or No + quickcheck_shift = qc_order.index(s[1])*2 + quickcheck <<= quickcheck_shift + if '..' not in s[0]: + first = last = int(s[0], 16) + else: + first, last = [int(c, 16) for c in s[0].split('..')] + for char in range(first, last+1): + assert not (quickchecks[char]>>quickcheck_shift)&3 + quickchecks[char] |= quickcheck + for i in range(0, 0x110000): + if table[i] is not None: + table[i].append(quickchecks[i]) + + zip = zipfile.ZipFile(open_data(UNIHAN, version)) + if version == '3.2.0': + data = zip.open('Unihan-3.2.0.txt').read() + else: + data = zip.open('Unihan_NumericValues.txt').read() + for line in data.decode("utf-8").splitlines(): + if not line.startswith('U+'): + continue + code, tag, value = line.split(None, 3)[:3] + if tag not in ('kAccountingNumeric', 'kPrimaryNumeric', + 'kOtherNumeric'): + continue + value = value.strip().replace(',', '') + i = int(code[2:], 16) + # Patch the numeric field + if table[i] is not None: + table[i][8] = value def uselatin1(self): # restrict character range to ISO Latin 1 @@ -979,7 +1115,6 @@ def splitbins(t, trace=0): you'll get. """ - import sys if trace: def dump(t1, t2, shift, bytes): print("%d+%d bins at shift %d; %d bytes" % ( diff --git a/Tools/unittestgui/README.txt b/Tools/unittestgui/README.txt new file mode 100644 index 0000000..4d809df --- /dev/null +++ b/Tools/unittestgui/README.txt @@ -0,0 +1,16 @@ +unittestgui.py is GUI framework and application for use with Python unit +testing framework. It executes tests written using the framework provided +by the 'unittest' module. + +Based on the original by Steve Purcell, from: + + http://pyunit.sourceforge.net/ + +Updated for unittest test discovery by Mark Roddy and Python 3 +support by Brian Curtin. + +For details on how to make your tests work with test discovery, +and for explanations of the configuration options, see the unittest +documentation: + + http://docs.python.org/library/unittest.html#test-discovery diff --git a/Tools/unittestgui/unittestgui.py b/Tools/unittestgui/unittestgui.py new file mode 100644 index 0000000..b526646 --- /dev/null +++ b/Tools/unittestgui/unittestgui.py @@ -0,0 +1,479 @@ +#!/usr/bin/env python +""" +GUI framework and application for use with Python unit testing framework. +Execute tests written using the framework provided by the 'unittest' module. + +Updated for unittest test discovery by Mark Roddy and Python 3 +support by Brian Curtin. + +Based on the original by Steve Purcell, from: + + http://pyunit.sourceforge.net/ + +Copyright (c) 1999, 2000, 2001 Steve Purcell +This module is free software, and you may redistribute it and/or modify +it under the same terms as Python itself, so long as this copyright message +and disclaimer are retained in their original form. + +IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF +THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +DAMAGE. + +THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, +AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, +SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. +""" + +__author__ = "Steve Purcell (stephen_purcell@yahoo.com)" +__version__ = "$Revision: 1.7 $"[11:-2] + +import sys +import traceback +import unittest + +import tkinter as tk +from tkinter import messagebox +from tkinter import filedialog +from tkinter import simpledialog + + + + +############################################################################## +# GUI framework classes +############################################################################## + +class BaseGUITestRunner(object): + """Subclass this class to create a GUI TestRunner that uses a specific + windowing toolkit. The class takes care of running tests in the correct + manner, and making callbacks to the derived class to obtain information + or signal that events have occurred. + """ + def __init__(self, *args, **kwargs): + self.currentResult = None + self.running = 0 + self.__rollbackImporter = None + self.__rollbackImporter = RollbackImporter() + self.test_suite = None + + #test discovery variables + self.directory_to_read = '' + self.top_level_dir = '' + self.test_file_glob_pattern = 'test*.py' + + self.initGUI(*args, **kwargs) + + def errorDialog(self, title, message): + "Override to display an error arising from GUI usage" + pass + + def getDirectoryToDiscover(self): + "Override to prompt user for directory to perform test discovery" + pass + + def runClicked(self): + "To be called in response to user choosing to run a test" + if self.running: return + if not self.test_suite: + self.errorDialog("Test Discovery", "You discover some tests first!") + return + self.currentResult = GUITestResult(self) + self.totalTests = self.test_suite.countTestCases() + self.running = 1 + self.notifyRunning() + self.test_suite.run(self.currentResult) + self.running = 0 + self.notifyStopped() + + def stopClicked(self): + "To be called in response to user stopping the running of a test" + if self.currentResult: + self.currentResult.stop() + + def discoverClicked(self): + self.__rollbackImporter.rollbackImports() + directory = self.getDirectoryToDiscover() + if not directory: + return + self.directory_to_read = directory + try: + # Explicitly use 'None' value if no top level directory is + # specified (indicated by empty string) as discover() explicitly + # checks for a 'None' to determine if no tld has been specified + top_level_dir = self.top_level_dir or None + tests = unittest.defaultTestLoader.discover(directory, self.test_file_glob_pattern, top_level_dir) + self.test_suite = tests + except: + exc_type, exc_value, exc_tb = sys.exc_info() + traceback.print_exception(*sys.exc_info()) + self.errorDialog("Unable to run test '%s'" % directory, + "Error loading specified test: %s, %s" % (exc_type, exc_value)) + return + self.notifyTestsDiscovered(self.test_suite) + + # Required callbacks + + def notifyTestsDiscovered(self, test_suite): + "Override to display information about the suite of discovered tests" + pass + + def notifyRunning(self): + "Override to set GUI in 'running' mode, enabling 'stop' button etc." + pass + + def notifyStopped(self): + "Override to set GUI in 'stopped' mode, enabling 'run' button etc." + pass + + def notifyTestFailed(self, test, err): + "Override to indicate that a test has just failed" + pass + + def notifyTestErrored(self, test, err): + "Override to indicate that a test has just errored" + pass + + def notifyTestSkipped(self, test, reason): + "Override to indicate that test was skipped" + pass + + def notifyTestFailedExpectedly(self, test, err): + "Override to indicate that test has just failed expectedly" + pass + + def notifyTestStarted(self, test): + "Override to indicate that a test is about to run" + pass + + def notifyTestFinished(self, test): + """Override to indicate that a test has finished (it may already have + failed or errored)""" + pass + + +class GUITestResult(unittest.TestResult): + """A TestResult that makes callbacks to its associated GUI TestRunner. + Used by BaseGUITestRunner. Need not be created directly. + """ + def __init__(self, callback): + unittest.TestResult.__init__(self) + self.callback = callback + + def addError(self, test, err): + unittest.TestResult.addError(self, test, err) + self.callback.notifyTestErrored(test, err) + + def addFailure(self, test, err): + unittest.TestResult.addFailure(self, test, err) + self.callback.notifyTestFailed(test, err) + + def addSkip(self, test, reason): + super(GUITestResult,self).addSkip(test, reason) + self.callback.notifyTestSkipped(test, reason) + + def addExpectedFailure(self, test, err): + super(GUITestResult,self).addExpectedFailure(test, err) + self.callback.notifyTestFailedExpectedly(test, err) + + def stopTest(self, test): + unittest.TestResult.stopTest(self, test) + self.callback.notifyTestFinished(test) + + def startTest(self, test): + unittest.TestResult.startTest(self, test) + self.callback.notifyTestStarted(test) + + +class RollbackImporter: + """This tricky little class is used to make sure that modules under test + will be reloaded the next time they are imported. + """ + def __init__(self): + self.previousModules = sys.modules.copy() + + def rollbackImports(self): + for modname in sys.modules.copy().keys(): + if not modname in self.previousModules: + # Force reload when modname next imported + del(sys.modules[modname]) + + +############################################################################## +# Tkinter GUI +############################################################################## + +class DiscoverSettingsDialog(simpledialog.Dialog): + """ + Dialog box for prompting test discovery settings + """ + + def __init__(self, master, top_level_dir, test_file_glob_pattern, *args, **kwargs): + self.top_level_dir = top_level_dir + self.dirVar = tk.StringVar() + self.dirVar.set(top_level_dir) + + self.test_file_glob_pattern = test_file_glob_pattern + self.testPatternVar = tk.StringVar() + self.testPatternVar.set(test_file_glob_pattern) + + simpledialog.Dialog.__init__(self, master, title="Discover Settings", + *args, **kwargs) + + def body(self, master): + tk.Label(master, text="Top Level Directory").grid(row=0) + self.e1 = tk.Entry(master, textvariable=self.dirVar) + self.e1.grid(row = 0, column=1) + tk.Button(master, text="...", + command=lambda: self.selectDirClicked(master)).grid(row=0,column=3) + + tk.Label(master, text="Test File Pattern").grid(row=1) + self.e2 = tk.Entry(master, textvariable = self.testPatternVar) + self.e2.grid(row = 1, column=1) + return None + + def selectDirClicked(self, master): + dir_path = filedialog.askdirectory(parent=master) + if dir_path: + self.dirVar.set(dir_path) + + def apply(self): + self.top_level_dir = self.dirVar.get() + self.test_file_glob_pattern = self.testPatternVar.get() + +class TkTestRunner(BaseGUITestRunner): + """An implementation of BaseGUITestRunner using Tkinter. + """ + def initGUI(self, root, initialTestName): + """Set up the GUI inside the given root window. The test name entry + field will be pre-filled with the given initialTestName. + """ + self.root = root + + self.statusVar = tk.StringVar() + self.statusVar.set("Idle") + + #tk vars for tracking counts of test result types + self.runCountVar = tk.IntVar() + self.failCountVar = tk.IntVar() + self.errorCountVar = tk.IntVar() + self.skipCountVar = tk.IntVar() + self.expectFailCountVar = tk.IntVar() + self.remainingCountVar = tk.IntVar() + + self.top = tk.Frame() + self.top.pack(fill=tk.BOTH, expand=1) + self.createWidgets() + + def getDirectoryToDiscover(self): + return filedialog.askdirectory() + + def settingsClicked(self): + d = DiscoverSettingsDialog(self.top, self.top_level_dir, self.test_file_glob_pattern) + self.top_level_dir = d.top_level_dir + self.test_file_glob_pattern = d.test_file_glob_pattern + + def notifyTestsDiscovered(self, test_suite): + discovered = test_suite.countTestCases() + self.runCountVar.set(0) + self.failCountVar.set(0) + self.errorCountVar.set(0) + self.remainingCountVar.set(discovered) + self.progressBar.setProgressFraction(0.0) + self.errorListbox.delete(0, tk.END) + self.statusVar.set("Discovering tests from %s. Found: %s" % + (self.directory_to_read, discovered)) + self.stopGoButton['state'] = tk.NORMAL + + def createWidgets(self): + """Creates and packs the various widgets. + + Why is it that GUI code always ends up looking a mess, despite all the + best intentions to keep it tidy? Answers on a postcard, please. + """ + # Status bar + statusFrame = tk.Frame(self.top, relief=tk.SUNKEN, borderwidth=2) + statusFrame.pack(anchor=tk.SW, fill=tk.X, side=tk.BOTTOM) + tk.Label(statusFrame, width=1, textvariable=self.statusVar).pack(side=tk.TOP, fill=tk.X) + + # Area to enter name of test to run + leftFrame = tk.Frame(self.top, borderwidth=3) + leftFrame.pack(fill=tk.BOTH, side=tk.LEFT, anchor=tk.NW, expand=1) + suiteNameFrame = tk.Frame(leftFrame, borderwidth=3) + suiteNameFrame.pack(fill=tk.X) + + # Progress bar + progressFrame = tk.Frame(leftFrame, relief=tk.GROOVE, borderwidth=2) + progressFrame.pack(fill=tk.X, expand=0, anchor=tk.NW) + tk.Label(progressFrame, text="Progress:").pack(anchor=tk.W) + self.progressBar = ProgressBar(progressFrame, relief=tk.SUNKEN, + borderwidth=2) + self.progressBar.pack(fill=tk.X, expand=1) + + + # Area with buttons to start/stop tests and quit + buttonFrame = tk.Frame(self.top, borderwidth=3) + buttonFrame.pack(side=tk.LEFT, anchor=tk.NW, fill=tk.Y) + + tk.Button(buttonFrame, text="Discover Tests", + command=self.discoverClicked).pack(fill=tk.X) + + + self.stopGoButton = tk.Button(buttonFrame, text="Start", + command=self.runClicked, state=tk.DISABLED) + self.stopGoButton.pack(fill=tk.X) + + tk.Button(buttonFrame, text="Close", + command=self.top.quit).pack(side=tk.BOTTOM, fill=tk.X) + tk.Button(buttonFrame, text="Settings", + command=self.settingsClicked).pack(side=tk.BOTTOM, fill=tk.X) + + # Area with labels reporting results + for label, var in (('Run:', self.runCountVar), + ('Failures:', self.failCountVar), + ('Errors:', self.errorCountVar), + ('Skipped:', self.skipCountVar), + ('Expected Failures:', self.expectFailCountVar), + ('Remaining:', self.remainingCountVar), + ): + tk.Label(progressFrame, text=label).pack(side=tk.LEFT) + tk.Label(progressFrame, textvariable=var, + foreground="blue").pack(side=tk.LEFT, fill=tk.X, + expand=1, anchor=tk.W) + + # List box showing errors and failures + tk.Label(leftFrame, text="Failures and errors:").pack(anchor=tk.W) + listFrame = tk.Frame(leftFrame, relief=tk.SUNKEN, borderwidth=2) + listFrame.pack(fill=tk.BOTH, anchor=tk.NW, expand=1) + self.errorListbox = tk.Listbox(listFrame, foreground='red', + selectmode=tk.SINGLE, + selectborderwidth=0) + self.errorListbox.pack(side=tk.LEFT, fill=tk.BOTH, expand=1, + anchor=tk.NW) + listScroll = tk.Scrollbar(listFrame, command=self.errorListbox.yview) + listScroll.pack(side=tk.LEFT, fill=tk.Y, anchor=tk.N) + self.errorListbox.bind("<Double-1>", + lambda e, self=self: self.showSelectedError()) + self.errorListbox.configure(yscrollcommand=listScroll.set) + + def errorDialog(self, title, message): + messagebox.showerror(parent=self.root, title=title, + message=message) + + def notifyRunning(self): + self.runCountVar.set(0) + self.failCountVar.set(0) + self.errorCountVar.set(0) + self.remainingCountVar.set(self.totalTests) + self.errorInfo = [] + while self.errorListbox.size(): + self.errorListbox.delete(0) + #Stopping seems not to work, so simply disable the start button + #self.stopGoButton.config(command=self.stopClicked, text="Stop") + self.stopGoButton.config(state=tk.DISABLED) + self.progressBar.setProgressFraction(0.0) + self.top.update_idletasks() + + def notifyStopped(self): + self.stopGoButton.config(state=tk.DISABLED) + #self.stopGoButton.config(command=self.runClicked, text="Start") + self.statusVar.set("Idle") + + def notifyTestStarted(self, test): + self.statusVar.set(str(test)) + self.top.update_idletasks() + + def notifyTestFailed(self, test, err): + self.failCountVar.set(1 + self.failCountVar.get()) + self.errorListbox.insert(tk.END, "Failure: %s" % test) + self.errorInfo.append((test,err)) + + def notifyTestErrored(self, test, err): + self.errorCountVar.set(1 + self.errorCountVar.get()) + self.errorListbox.insert(tk.END, "Error: %s" % test) + self.errorInfo.append((test,err)) + + def notifyTestSkipped(self, test, reason): + super(TkTestRunner, self).notifyTestSkipped(test, reason) + self.skipCountVar.set(1 + self.skipCountVar.get()) + + def notifyTestFailedExpectedly(self, test, err): + super(TkTestRunner, self).notifyTestFailedExpectedly(test, err) + self.expectFailCountVar.set(1 + self.expectFailCountVar.get()) + + + def notifyTestFinished(self, test): + self.remainingCountVar.set(self.remainingCountVar.get() - 1) + self.runCountVar.set(1 + self.runCountVar.get()) + fractionDone = float(self.runCountVar.get())/float(self.totalTests) + fillColor = len(self.errorInfo) and "red" or "green" + self.progressBar.setProgressFraction(fractionDone, fillColor) + + def showSelectedError(self): + selection = self.errorListbox.curselection() + if not selection: return + selected = int(selection[0]) + txt = self.errorListbox.get(selected) + window = tk.Toplevel(self.root) + window.title(txt) + window.protocol('WM_DELETE_WINDOW', window.quit) + test, error = self.errorInfo[selected] + tk.Label(window, text=str(test), + foreground="red", justify=tk.LEFT).pack(anchor=tk.W) + tracebackLines = traceback.format_exception(*error) + tracebackText = "".join(tracebackLines) + tk.Label(window, text=tracebackText, justify=tk.LEFT).pack() + tk.Button(window, text="Close", + command=window.quit).pack(side=tk.BOTTOM) + window.bind('<Key-Return>', lambda e, w=window: w.quit()) + window.mainloop() + window.destroy() + + +class ProgressBar(tk.Frame): + """A simple progress bar that shows a percentage progress in + the given colour.""" + + def __init__(self, *args, **kwargs): + tk.Frame.__init__(self, *args, **kwargs) + self.canvas = tk.Canvas(self, height='20', width='60', + background='white', borderwidth=3) + self.canvas.pack(fill=tk.X, expand=1) + self.rect = self.text = None + self.canvas.bind('<Configure>', self.paint) + self.setProgressFraction(0.0) + + def setProgressFraction(self, fraction, color='blue'): + self.fraction = fraction + self.color = color + self.paint() + self.canvas.update_idletasks() + + def paint(self, *args): + totalWidth = self.canvas.winfo_width() + width = int(self.fraction * float(totalWidth)) + height = self.canvas.winfo_height() + if self.rect is not None: self.canvas.delete(self.rect) + if self.text is not None: self.canvas.delete(self.text) + self.rect = self.canvas.create_rectangle(0, 0, width, height, + fill=self.color) + percentString = "%3.0f%%" % (100.0 * self.fraction) + self.text = self.canvas.create_text(totalWidth/2, height/2, + anchor=tk.CENTER, + text=percentString) + +def main(initialTestName=""): + root = tk.Tk() + root.title("PyUnit") + runner = TkTestRunner(root, initialTestName) + root.protocol('WM_DELETE_WINDOW', root.quit) + root.mainloop() + + +if __name__ == '__main__': + if len(sys.argv) == 2: + main(sys.argv[1]) + else: + main() diff --git a/Tools/webchecker/README b/Tools/webchecker/README deleted file mode 100644 index a51bb3d..0000000 --- a/Tools/webchecker/README +++ /dev/null @@ -1,23 +0,0 @@ -Webchecker ----------- - -This is a simple web tree checker, useful to find bad links in a web -tree. It currently checks links pointing within the same subweb for -validity. The main program is "webchecker.py". See its doc string -(or invoke it with the option "-?") for more defails. - -History: - -- Jan 1997. First release. The module robotparser.py was written by -Skip Montanaro; the rest is original work by Guido van Rossum. - -- May 1999. Sam Bayer contributed a new version, wcnew.py, which -supports checking internal links (#spam fragments in URLs) and some -other options. - -- Nov 1999. Sam Bayer contributed patches to reintegrate wcnew.py -into webchecker.py, and corresponding mods to wcgui.py and -websucker.py. - -- Mar 2004. Chris Herborth contributed a patch to let webchecker.py -handle XHTML's 'id' attribute. diff --git a/Tools/webchecker/tktools.py b/Tools/webchecker/tktools.py deleted file mode 100644 index 3a68f9a..0000000 --- a/Tools/webchecker/tktools.py +++ /dev/null @@ -1,366 +0,0 @@ -"""Assorted Tk-related subroutines used in Grail.""" - - -from types import * -from Tkinter import * - -def _clear_entry_widget(event): - try: - widget = event.widget - widget.delete(0, INSERT) - except: pass -def install_keybindings(root): - root.bind_class('Entry', '<Control-u>', _clear_entry_widget) - - -def make_toplevel(master, title=None, class_=None): - """Create a Toplevel widget. - - This is a shortcut for a Toplevel() instantiation plus calls to - set the title and icon name of the widget. - - """ - - if class_: - widget = Toplevel(master, class_=class_) - else: - widget = Toplevel(master) - if title: - widget.title(title) - widget.iconname(title) - return widget - -def set_transient(widget, master, relx=0.5, rely=0.3, expose=1): - """Make an existing toplevel widget transient for a master. - - The widget must exist but should not yet have been placed; in - other words, this should be called after creating all the - subwidget but before letting the user interact. - """ - - widget.withdraw() # Remain invisible while we figure out the geometry - widget.transient(master) - widget.update_idletasks() # Actualize geometry information - if master.winfo_ismapped(): - m_width = master.winfo_width() - m_height = master.winfo_height() - m_x = master.winfo_rootx() - m_y = master.winfo_rooty() - else: - m_width = master.winfo_screenwidth() - m_height = master.winfo_screenheight() - m_x = m_y = 0 - w_width = widget.winfo_reqwidth() - w_height = widget.winfo_reqheight() - x = m_x + (m_width - w_width) * relx - y = m_y + (m_height - w_height) * rely - widget.geometry("+%d+%d" % (x, y)) - if expose: - widget.deiconify() # Become visible at the desired location - return widget - - -def make_scrollbars(parent, hbar, vbar, pack=1, class_=None, name=None, - takefocus=0): - - """Subroutine to create a frame with scrollbars. - - This is used by make_text_box and similar routines. - - Note: the caller is responsible for setting the x/y scroll command - properties (e.g. by calling set_scroll_commands()). - - Return a tuple containing the hbar, the vbar, and the frame, where - hbar and vbar are None if not requested. - - """ - if class_: - if name: frame = Frame(parent, class_=class_, name=name) - else: frame = Frame(parent, class_=class_) - else: - if name: frame = Frame(parent, name=name) - else: frame = Frame(parent) - - if pack: - frame.pack(fill=BOTH, expand=1) - - corner = None - if vbar: - if not hbar: - vbar = Scrollbar(frame, takefocus=takefocus) - vbar.pack(fill=Y, side=RIGHT) - else: - vbarframe = Frame(frame, borderwidth=0) - vbarframe.pack(fill=Y, side=RIGHT) - vbar = Scrollbar(frame, name="vbar", takefocus=takefocus) - vbar.pack(in_=vbarframe, expand=1, fill=Y, side=TOP) - sbwidth = vbar.winfo_reqwidth() - corner = Frame(vbarframe, width=sbwidth, height=sbwidth) - corner.propagate(0) - corner.pack(side=BOTTOM) - else: - vbar = None - - if hbar: - hbar = Scrollbar(frame, orient=HORIZONTAL, name="hbar", - takefocus=takefocus) - hbar.pack(fill=X, side=BOTTOM) - else: - hbar = None - - return hbar, vbar, frame - - -def set_scroll_commands(widget, hbar, vbar): - - """Link a scrollable widget to its scroll bars. - - The scroll bars may be empty. - - """ - - if vbar: - widget['yscrollcommand'] = (vbar, 'set') - vbar['command'] = (widget, 'yview') - - if hbar: - widget['xscrollcommand'] = (hbar, 'set') - hbar['command'] = (widget, 'xview') - - widget.vbar = vbar - widget.hbar = hbar - - -def make_text_box(parent, width=0, height=0, hbar=0, vbar=1, - fill=BOTH, expand=1, wrap=WORD, pack=1, - class_=None, name=None, takefocus=None): - - """Subroutine to create a text box. - - Create: - - a both-ways filling and expanding frame, containing: - - a text widget on the left, and - - possibly a vertical scroll bar on the right. - - possibly a horizonta; scroll bar at the bottom. - - Return the text widget and the frame widget. - - """ - hbar, vbar, frame = make_scrollbars(parent, hbar, vbar, pack, - class_=class_, name=name, - takefocus=takefocus) - - widget = Text(frame, wrap=wrap, name="text") - if width: widget.config(width=width) - if height: widget.config(height=height) - widget.pack(expand=expand, fill=fill, side=LEFT) - - set_scroll_commands(widget, hbar, vbar) - - return widget, frame - - -def make_list_box(parent, width=0, height=0, hbar=0, vbar=1, - fill=BOTH, expand=1, pack=1, class_=None, name=None, - takefocus=None): - - """Subroutine to create a list box. - - Like make_text_box(). - """ - hbar, vbar, frame = make_scrollbars(parent, hbar, vbar, pack, - class_=class_, name=name, - takefocus=takefocus) - - widget = Listbox(frame, name="listbox") - if width: widget.config(width=width) - if height: widget.config(height=height) - widget.pack(expand=expand, fill=fill, side=LEFT) - - set_scroll_commands(widget, hbar, vbar) - - return widget, frame - - -def make_canvas(parent, width=0, height=0, hbar=1, vbar=1, - fill=BOTH, expand=1, pack=1, class_=None, name=None, - takefocus=None): - - """Subroutine to create a canvas. - - Like make_text_box(). - - """ - - hbar, vbar, frame = make_scrollbars(parent, hbar, vbar, pack, - class_=class_, name=name, - takefocus=takefocus) - - widget = Canvas(frame, scrollregion=(0, 0, width, height), name="canvas") - if width: widget.config(width=width) - if height: widget.config(height=height) - widget.pack(expand=expand, fill=fill, side=LEFT) - - set_scroll_commands(widget, hbar, vbar) - - return widget, frame - - - -def make_form_entry(parent, label, borderwidth=None): - - """Subroutine to create a form entry. - - Create: - - a horizontally filling and expanding frame, containing: - - a label on the left, and - - a text entry on the right. - - Return the entry widget and the frame widget. - - """ - - frame = Frame(parent) - frame.pack(fill=X) - - label = Label(frame, text=label) - label.pack(side=LEFT) - - if borderwidth is None: - entry = Entry(frame, relief=SUNKEN) - else: - entry = Entry(frame, relief=SUNKEN, borderwidth=borderwidth) - entry.pack(side=LEFT, fill=X, expand=1) - - return entry, frame - -# This is a slightly modified version of the function above. This -# version does the proper alighnment of labels with their fields. It -# should probably eventually replace make_form_entry altogether. -# -# The one annoying bug is that the text entry field should be -# expandable while still aligning the colons. This doesn't work yet. -# -def make_labeled_form_entry(parent, label, entrywidth=20, entryheight=1, - labelwidth=0, borderwidth=None, - takefocus=None): - """Subroutine to create a form entry. - - Create: - - a horizontally filling and expanding frame, containing: - - a label on the left, and - - a text entry on the right. - - Return the entry widget and the frame widget. - """ - if label and label[-1] != ':': label = label + ':' - - frame = Frame(parent) - - label = Label(frame, text=label, width=labelwidth, anchor=E) - label.pack(side=LEFT) - if entryheight == 1: - if borderwidth is None: - entry = Entry(frame, relief=SUNKEN, width=entrywidth) - else: - entry = Entry(frame, relief=SUNKEN, width=entrywidth, - borderwidth=borderwidth) - entry.pack(side=RIGHT, expand=1, fill=X) - frame.pack(fill=X) - else: - entry = make_text_box(frame, entrywidth, entryheight, 1, 1, - takefocus=takefocus) - frame.pack(fill=BOTH, expand=1) - - return entry, frame, label - - -def make_double_frame(master=None, class_=None, name=None, relief=RAISED, - borderwidth=1): - """Create a pair of frames suitable for 'hosting' a dialog.""" - if name: - if class_: frame = Frame(master, class_=class_, name=name) - else: frame = Frame(master, name=name) - else: - if class_: frame = Frame(master, class_=class_) - else: frame = Frame(master) - top = Frame(frame, name="topframe", relief=relief, - borderwidth=borderwidth) - bottom = Frame(frame, name="bottomframe") - bottom.pack(fill=X, padx='1m', pady='1m', side=BOTTOM) - top.pack(expand=1, fill=BOTH, padx='1m', pady='1m') - frame.pack(expand=1, fill=BOTH) - top = Frame(top) - top.pack(expand=1, fill=BOTH, padx='2m', pady='2m') - - return frame, top, bottom - - -def make_group_frame(master, name=None, label=None, fill=Y, - side=None, expand=None, font=None): - """Create nested frames with a border and optional label. - - The outer frame is only used to provide the decorative border, to - control packing, and to host the label. The inner frame is packed - to fill the outer frame and should be used as the parent of all - sub-widgets. Only the inner frame is returned. - - """ - font = font or "-*-helvetica-medium-r-normal-*-*-100-*-*-*-*-*-*" - outer = Frame(master, borderwidth=2, relief=GROOVE) - outer.pack(expand=expand, fill=fill, side=side) - if label: - Label(outer, text=label, font=font, anchor=W).pack(fill=X) - inner = Frame(master, borderwidth='1m', name=name) - inner.pack(expand=1, fill=BOTH, in_=outer) - inner.forget = outer.forget - return inner - - -def unify_button_widths(*buttons): - """Make buttons passed in all have the same width. - - Works for labels and other widgets with the 'text' option. - - """ - wid = 0 - for btn in buttons: - wid = max(wid, len(btn["text"])) - for btn in buttons: - btn["width"] = wid - - -def flatten(msg): - """Turn a list or tuple into a single string -- recursively.""" - t = type(msg) - if t in (ListType, TupleType): - msg = ' '.join(map(flatten, msg)) - elif t is ClassType: - msg = msg.__name__ - else: - msg = str(msg) - return msg - - -def boolean(s): - """Test whether a string is a Tk boolean, without error checking.""" - if s.lower() in ('', '0', 'no', 'off', 'false'): return 0 - else: return 1 - - -def test(): - """Test make_text_box(), make_form_entry(), flatten(), boolean().""" - import sys - root = Tk() - entry, eframe = make_form_entry(root, 'Boolean:') - text, tframe = make_text_box(root) - def enter(event, entry=entry, text=text): - s = boolean(entry.get()) and '\nyes' or '\nno' - text.insert('end', s) - entry.bind('<Return>', enter) - entry.insert(END, flatten(sys.argv)) - root.mainloop() - - -if __name__ == '__main__': - test() diff --git a/Tools/webchecker/wcgui.py b/Tools/webchecker/wcgui.py deleted file mode 100755 index 4ce613a..0000000 --- a/Tools/webchecker/wcgui.py +++ /dev/null @@ -1,462 +0,0 @@ -#! /usr/bin/env python - -"""GUI interface to webchecker. - -This works as a Grail applet too! E.g. - - <APPLET CODE=wcgui.py NAME=CheckerWindow></APPLET> - -Checkpoints are not (yet??? ever???) supported. - -User interface: - -Enter a root to check in the text entry box. To enter more than one root, -enter them one at a time and press <Return> for each one. - -Command buttons Start, Stop and "Check one" govern the checking process in -the obvious way. Start and "Check one" also enter the root from the text -entry box if one is present. There's also a check box (enabled by default) -to decide whether actually to follow external links (since this can slow -the checking down considerably). Finally there's a Quit button. - -A series of checkbuttons determines whether the corresponding output panel -is shown. List panels are also automatically shown or hidden when their -status changes between empty to non-empty. There are six panels: - -Log -- raw output from the checker (-v, -q affect this) -To check -- links discovered but not yet checked -Checked -- links that have been checked -Bad links -- links that failed upon checking -Errors -- pages containing at least one bad link -Details -- details about one URL; double click on a URL in any of - the above list panels (not in Log) will show details - for that URL - -Use your window manager's Close command to quit. - -Command line options: - --m bytes -- skip HTML pages larger than this size (default %(MAXPAGE)d) --q -- quiet operation (also suppresses external links report) --v -- verbose operation; repeating -v will increase verbosity --t root -- specify root dir which should be treated as internal (can repeat) --a -- don't check name anchors - -Command line arguments: - -rooturl -- URL to start checking - (default %(DEFROOT)s) - -XXX The command line options (-m, -q, -v) should be GUI accessible. - -XXX The roots should be visible as a list (?). - -XXX The multipanel user interface is clumsy. - -""" - -# ' Emacs bait - - -import sys -import getopt -from Tkinter import * -import tktools -import webchecker - -# Override some for a weaker platform -if sys.platform == 'mac': - webchecker.DEFROOT = "http://grail.cnri.reston.va.us/" - webchecker.MAXPAGE = 50000 - webchecker.verbose = 4 - -def main(): - try: - opts, args = getopt.getopt(sys.argv[1:], 't:m:qva') - except getopt.error as msg: - sys.stdout = sys.stderr - print(msg) - print(__doc__%vars(webchecker)) - sys.exit(2) - webchecker.verbose = webchecker.VERBOSE - webchecker.nonames = webchecker.NONAMES - webchecker.maxpage = webchecker.MAXPAGE - extra_roots = [] - for o, a in opts: - if o == '-m': - webchecker.maxpage = int(a) - if o == '-q': - webchecker.verbose = 0 - if o == '-v': - webchecker.verbose = webchecker.verbose + 1 - if o == '-t': - extra_roots.append(a) - if o == '-a': - webchecker.nonames = not webchecker.nonames - root = Tk(className='Webchecker') - root.protocol("WM_DELETE_WINDOW", root.quit) - c = CheckerWindow(root) - c.setflags(verbose=webchecker.verbose, maxpage=webchecker.maxpage, - nonames=webchecker.nonames) - if args: - for arg in args[:-1]: - c.addroot(arg) - c.suggestroot(args[-1]) - # Usually conditioned on whether external links - # will be checked, but since that's not a command - # line option, just toss them in. - for url_root in extra_roots: - # Make sure it's terminated by a slash, - # so that addroot doesn't discard the last - # directory component. - if url_root[-1] != "/": - url_root = url_root + "/" - c.addroot(url_root, add_to_do = 0) - root.mainloop() - - -class CheckerWindow(webchecker.Checker): - - def __init__(self, parent, root=webchecker.DEFROOT): - self.__parent = parent - - self.__topcontrols = Frame(parent) - self.__topcontrols.pack(side=TOP, fill=X) - self.__label = Label(self.__topcontrols, text="Root URL:") - self.__label.pack(side=LEFT) - self.__rootentry = Entry(self.__topcontrols, width=60) - self.__rootentry.pack(side=LEFT) - self.__rootentry.bind('<Return>', self.enterroot) - self.__rootentry.focus_set() - - self.__controls = Frame(parent) - self.__controls.pack(side=TOP, fill=X) - self.__running = 0 - self.__start = Button(self.__controls, text="Run", command=self.start) - self.__start.pack(side=LEFT) - self.__stop = Button(self.__controls, text="Stop", command=self.stop, - state=DISABLED) - self.__stop.pack(side=LEFT) - self.__step = Button(self.__controls, text="Check one", - command=self.step) - self.__step.pack(side=LEFT) - self.__cv = BooleanVar(parent) - self.__cv.set(self.checkext) - self.__checkext = Checkbutton(self.__controls, variable=self.__cv, - command=self.update_checkext, - text="Check nonlocal links",) - self.__checkext.pack(side=LEFT) - self.__reset = Button(self.__controls, text="Start over", command=self.reset) - self.__reset.pack(side=LEFT) - if __name__ == '__main__': # No Quit button under Grail! - self.__quit = Button(self.__controls, text="Quit", - command=self.__parent.quit) - self.__quit.pack(side=RIGHT) - - self.__status = Label(parent, text="Status: initial", anchor=W) - self.__status.pack(side=TOP, fill=X) - self.__checking = Label(parent, text="Idle", anchor=W) - self.__checking.pack(side=TOP, fill=X) - self.__mp = mp = MultiPanel(parent) - sys.stdout = self.__log = LogPanel(mp, "Log") - self.__todo = ListPanel(mp, "To check", self, self.showinfo) - self.__done = ListPanel(mp, "Checked", self, self.showinfo) - self.__bad = ListPanel(mp, "Bad links", self, self.showinfo) - self.__errors = ListPanel(mp, "Pages w/ bad links", self, self.showinfo) - self.__details = LogPanel(mp, "Details") - self.root_seed = None - webchecker.Checker.__init__(self) - if root: - root = str(root).strip() - if root: - self.suggestroot(root) - self.newstatus() - - def reset(self): - webchecker.Checker.reset(self) - for p in self.__todo, self.__done, self.__bad, self.__errors: - p.clear() - if self.root_seed: - self.suggestroot(self.root_seed) - - def suggestroot(self, root): - self.__rootentry.delete(0, END) - self.__rootentry.insert(END, root) - self.__rootentry.select_range(0, END) - self.root_seed = root - - def enterroot(self, event=None): - root = self.__rootentry.get() - root = root.strip() - if root: - self.__checking.config(text="Adding root "+root) - self.__checking.update_idletasks() - self.addroot(root) - self.__checking.config(text="Idle") - try: - i = self.__todo.items.index(root) - except (ValueError, IndexError): - pass - else: - self.__todo.list.select_clear(0, END) - self.__todo.list.select_set(i) - self.__todo.list.yview(i) - self.__rootentry.delete(0, END) - - def start(self): - self.__start.config(state=DISABLED, relief=SUNKEN) - self.__stop.config(state=NORMAL) - self.__step.config(state=DISABLED) - self.enterroot() - self.__running = 1 - self.go() - - def stop(self): - self.__stop.config(state=DISABLED, relief=SUNKEN) - self.__running = 0 - - def step(self): - self.__start.config(state=DISABLED) - self.__step.config(state=DISABLED, relief=SUNKEN) - self.enterroot() - self.__running = 0 - self.dosomething() - - def go(self): - if self.__running: - self.__parent.after_idle(self.dosomething) - else: - self.__checking.config(text="Idle") - self.__start.config(state=NORMAL, relief=RAISED) - self.__stop.config(state=DISABLED, relief=RAISED) - self.__step.config(state=NORMAL, relief=RAISED) - - __busy = 0 - - def dosomething(self): - if self.__busy: return - self.__busy = 1 - if self.todo: - l = self.__todo.selectedindices() - if l: - i = l[0] - else: - i = 0 - self.__todo.list.select_set(i) - self.__todo.list.yview(i) - url = self.__todo.items[i] - self.__checking.config(text="Checking "+self.format_url(url)) - self.__parent.update() - self.dopage(url) - else: - self.stop() - self.__busy = 0 - self.go() - - def showinfo(self, url): - d = self.__details - d.clear() - d.put("URL: %s\n" % self.format_url(url)) - if url in self.bad: - d.put("Error: %s\n" % str(self.bad[url])) - if url in self.roots: - d.put("Note: This is a root URL\n") - if url in self.done: - d.put("Status: checked\n") - o = self.done[url] - elif url in self.todo: - d.put("Status: to check\n") - o = self.todo[url] - else: - d.put("Status: unknown (!)\n") - o = [] - if (not url[1]) and url[0] in self.errors: - d.put("Bad links from this page:\n") - for triple in self.errors[url[0]]: - link, rawlink, msg = triple - d.put(" HREF %s" % self.format_url(link)) - if self.format_url(link) != rawlink: d.put(" (%s)" %rawlink) - d.put("\n") - d.put(" error %s\n" % str(msg)) - self.__mp.showpanel("Details") - for source, rawlink in o: - d.put("Origin: %s" % source) - if rawlink != self.format_url(url): - d.put(" (%s)" % rawlink) - d.put("\n") - d.text.yview("1.0") - - def setbad(self, url, msg): - webchecker.Checker.setbad(self, url, msg) - self.__bad.insert(url) - self.newstatus() - - def setgood(self, url): - webchecker.Checker.setgood(self, url) - self.__bad.remove(url) - self.newstatus() - - def newlink(self, url, origin): - webchecker.Checker.newlink(self, url, origin) - if url in self.done: - self.__done.insert(url) - elif url in self.todo: - self.__todo.insert(url) - self.newstatus() - - def markdone(self, url): - webchecker.Checker.markdone(self, url) - self.__done.insert(url) - self.__todo.remove(url) - self.newstatus() - - def seterror(self, url, triple): - webchecker.Checker.seterror(self, url, triple) - self.__errors.insert((url, '')) - self.newstatus() - - def newstatus(self): - self.__status.config(text="Status: "+self.status()) - self.__parent.update() - - def update_checkext(self): - self.checkext = self.__cv.get() - - -class ListPanel: - - def __init__(self, mp, name, checker, showinfo=None): - self.mp = mp - self.name = name - self.showinfo = showinfo - self.checker = checker - self.panel = mp.addpanel(name) - self.list, self.frame = tktools.make_list_box( - self.panel, width=60, height=5) - self.list.config(exportselection=0) - if showinfo: - self.list.bind('<Double-Button-1>', self.doubleclick) - self.items = [] - - def clear(self): - self.items = [] - self.list.delete(0, END) - self.mp.hidepanel(self.name) - - def doubleclick(self, event): - l = self.selectedindices() - if l: - self.showinfo(self.items[l[0]]) - - def selectedindices(self): - l = self.list.curselection() - if not l: return [] - return list(map(int, l)) - - def insert(self, url): - if url not in self.items: - if not self.items: - self.mp.showpanel(self.name) - # (I tried sorting alphabetically, but the display is too jumpy) - i = len(self.items) - self.list.insert(i, self.checker.format_url(url)) - self.list.yview(i) - self.items.insert(i, url) - - def remove(self, url): - try: - i = self.items.index(url) - except (ValueError, IndexError): - pass - else: - was_selected = i in self.selectedindices() - self.list.delete(i) - del self.items[i] - if not self.items: - self.mp.hidepanel(self.name) - elif was_selected: - if i >= len(self.items): - i = len(self.items) - 1 - self.list.select_set(i) - - -class LogPanel: - - def __init__(self, mp, name): - self.mp = mp - self.name = name - self.panel = mp.addpanel(name) - self.text, self.frame = tktools.make_text_box(self.panel, height=10) - self.text.config(wrap=NONE) - - def clear(self): - self.text.delete("1.0", END) - self.text.yview("1.0") - - def put(self, s): - self.text.insert(END, s) - if '\n' in s: - self.text.yview(END) - - def write(self, s): - self.text.insert(END, s) - if '\n' in s: - self.text.yview(END) - self.panel.update() - - -class MultiPanel: - - def __init__(self, parent): - self.parent = parent - self.frame = Frame(self.parent) - self.frame.pack(expand=1, fill=BOTH) - self.topframe = Frame(self.frame, borderwidth=2, relief=RAISED) - self.topframe.pack(fill=X) - self.botframe = Frame(self.frame) - self.botframe.pack(expand=1, fill=BOTH) - self.panelnames = [] - self.panels = {} - - def addpanel(self, name, on=0): - v = StringVar(self.parent) - if on: - v.set(name) - else: - v.set("") - check = Checkbutton(self.topframe, text=name, - offvalue="", onvalue=name, variable=v, - command=self.checkpanel) - check.pack(side=LEFT) - panel = Frame(self.botframe) - label = Label(panel, text=name, borderwidth=2, relief=RAISED, anchor=W) - label.pack(side=TOP, fill=X) - t = v, check, panel - self.panelnames.append(name) - self.panels[name] = t - if on: - panel.pack(expand=1, fill=BOTH) - return panel - - def showpanel(self, name): - v, check, panel = self.panels[name] - v.set(name) - panel.pack(expand=1, fill=BOTH) - - def hidepanel(self, name): - v, check, panel = self.panels[name] - v.set("") - panel.pack_forget() - - def checkpanel(self): - for name in self.panelnames: - v, check, panel = self.panels[name] - panel.pack_forget() - for name in self.panelnames: - v, check, panel = self.panels[name] - if v.get(): - panel.pack(expand=1, fill=BOTH) - - -if __name__ == '__main__': - main() diff --git a/Tools/webchecker/wcmac.py b/Tools/webchecker/wcmac.py deleted file mode 100644 index 9edcd5d..0000000 --- a/Tools/webchecker/wcmac.py +++ /dev/null @@ -1,9 +0,0 @@ -import webchecker, sys -webchecker.DEFROOT = "http://www.python.org/python/" -webchecker.MAXPAGE = 50000 -webchecker.verbose = 2 -sys.argv.append('-x') -webchecker.main() -sys.stdout.write("\nCR to exit: ") -sys.stdout.flush() -sys.stdin.readline() diff --git a/Tools/webchecker/webchecker.py b/Tools/webchecker/webchecker.py deleted file mode 100755 index 651cf85..0000000 --- a/Tools/webchecker/webchecker.py +++ /dev/null @@ -1,890 +0,0 @@ -#! /usr/bin/env python - -# Original code by Guido van Rossum; extensive changes by Sam Bayer, -# including code to check URL fragments. - -"""Web tree checker. - -This utility is handy to check a subweb of the world-wide web for -errors. A subweb is specified by giving one or more ``root URLs''; a -page belongs to the subweb if one of the root URLs is an initial -prefix of it. - -File URL extension: - -In order to easy the checking of subwebs via the local file system, -the interpretation of ``file:'' URLs is extended to mimic the behavior -of your average HTTP daemon: if a directory pathname is given, the -file index.html in that directory is returned if it exists, otherwise -a directory listing is returned. Now, you can point webchecker to the -document tree in the local file system of your HTTP daemon, and have -most of it checked. In fact the default works this way if your local -web tree is located at /usr/local/etc/httpd/htdpcs (the default for -the NCSA HTTP daemon and probably others). - -Report printed: - -When done, it reports pages with bad links within the subweb. When -interrupted, it reports for the pages that it has checked already. - -In verbose mode, additional messages are printed during the -information gathering phase. By default, it prints a summary of its -work status every 50 URLs (adjustable with the -r option), and it -reports errors as they are encountered. Use the -q option to disable -this output. - -Checkpoint feature: - -Whether interrupted or not, it dumps its state (a Python pickle) to a -checkpoint file and the -R option allows it to restart from the -checkpoint (assuming that the pages on the subweb that were already -processed haven't changed). Even when it has run till completion, -R -can still be useful -- it will print the reports again, and -Rq prints -the errors only. In this case, the checkpoint file is not written -again. The checkpoint file can be set with the -d option. - -The checkpoint file is written as a Python pickle. Remember that -Python's pickle module is currently quite slow. Give it the time it -needs to load and save the checkpoint file. When interrupted while -writing the checkpoint file, the old checkpoint file is not -overwritten, but all work done in the current run is lost. - -Miscellaneous: - -- You may find the (Tk-based) GUI version easier to use. See wcgui.py. - -- Webchecker honors the "robots.txt" convention. Thanks to Skip -Montanaro for his robotparser.py module (included in this directory)! -The agent name is hardwired to "webchecker". URLs that are disallowed -by the robots.txt file are reported as external URLs. - -- Because the SGML parser is a bit slow, very large SGML files are -skipped. The size limit can be set with the -m option. - -- When the server or protocol does not tell us a file's type, we guess -it based on the URL's suffix. The mimetypes.py module (also in this -directory) has a built-in table mapping most currently known suffixes, -and in addition attempts to read the mime.types configuration files in -the default locations of Netscape and the NCSA HTTP daemon. - -- We follow links indicated by <A>, <FRAME> and <IMG> tags. We also -honor the <BASE> tag. - -- We now check internal NAME anchor links, as well as toplevel links. - -- Checking external links is now done by default; use -x to *disable* -this feature. External links are now checked during normal -processing. (XXX The status of a checked link could be categorized -better. Later...) - -- If external links are not checked, you can use the -t flag to -provide specific overrides to -x. - -Usage: webchecker.py [option] ... [rooturl] ... - -Options: - --R -- restart from checkpoint file --d file -- checkpoint filename (default %(DUMPFILE)s) --m bytes -- skip HTML pages larger than this size (default %(MAXPAGE)d) --n -- reports only, no checking (use with -R) --q -- quiet operation (also suppresses external links report) --r number -- number of links processed per round (default %(ROUNDSIZE)d) --t root -- specify root dir which should be treated as internal (can repeat) --v -- verbose operation; repeating -v will increase verbosity --x -- don't check external links (these are often slow to check) --a -- don't check name anchors - -Arguments: - -rooturl -- URL to start checking - (default %(DEFROOT)s) - -""" - - -__version__ = "$Revision$" - - -import sys -import os -from types import * -import io -import getopt -import pickle - -import urllib.request -import urllib.parse as urlparse -import sgmllib -import cgi - -import mimetypes -from urllib import robotparser - -# Extract real version number if necessary -if __version__[0] == '$': - _v = __version__.split() - if len(_v) == 3: - __version__ = _v[1] - - -# Tunable parameters -DEFROOT = "file:/usr/local/etc/httpd/htdocs/" # Default root URL -CHECKEXT = 1 # Check external references (1 deep) -VERBOSE = 1 # Verbosity level (0-3) -MAXPAGE = 150000 # Ignore files bigger than this -ROUNDSIZE = 50 # Number of links processed per round -DUMPFILE = "@webchecker.pickle" # Pickled checkpoint -AGENTNAME = "webchecker" # Agent name for robots.txt parser -NONAMES = 0 # Force name anchor checking - - -# Global variables - - -def main(): - checkext = CHECKEXT - verbose = VERBOSE - maxpage = MAXPAGE - roundsize = ROUNDSIZE - dumpfile = DUMPFILE - restart = 0 - norun = 0 - - try: - opts, args = getopt.getopt(sys.argv[1:], 'Rd:m:nqr:t:vxa') - except getopt.error as msg: - sys.stdout = sys.stderr - print(msg) - print(__doc__%globals()) - sys.exit(2) - - # The extra_roots variable collects extra roots. - extra_roots = [] - nonames = NONAMES - - for o, a in opts: - if o == '-R': - restart = 1 - if o == '-d': - dumpfile = a - if o == '-m': - maxpage = int(a) - if o == '-n': - norun = 1 - if o == '-q': - verbose = 0 - if o == '-r': - roundsize = int(a) - if o == '-t': - extra_roots.append(a) - if o == '-a': - nonames = not nonames - if o == '-v': - verbose = verbose + 1 - if o == '-x': - checkext = not checkext - - if verbose > 0: - print(AGENTNAME, "version", __version__) - - if restart: - c = load_pickle(dumpfile=dumpfile, verbose=verbose) - else: - c = Checker() - - c.setflags(checkext=checkext, verbose=verbose, - maxpage=maxpage, roundsize=roundsize, - nonames=nonames - ) - - if not restart and not args: - args.append(DEFROOT) - - for arg in args: - c.addroot(arg) - - # The -t flag is only needed if external links are not to be - # checked. So -t values are ignored unless -x was specified. - if not checkext: - for root in extra_roots: - # Make sure it's terminated by a slash, - # so that addroot doesn't discard the last - # directory component. - if root[-1] != "/": - root = root + "/" - c.addroot(root, add_to_do = 0) - - try: - - if not norun: - try: - c.run() - except KeyboardInterrupt: - if verbose > 0: - print("[run interrupted]") - - try: - c.report() - except KeyboardInterrupt: - if verbose > 0: - print("[report interrupted]") - - finally: - if c.save_pickle(dumpfile): - if dumpfile == DUMPFILE: - print("Use ``%s -R'' to restart." % sys.argv[0]) - else: - print("Use ``%s -R -d %s'' to restart." % (sys.argv[0], - dumpfile)) - - -def load_pickle(dumpfile=DUMPFILE, verbose=VERBOSE): - if verbose > 0: - print("Loading checkpoint from %s ..." % dumpfile) - f = open(dumpfile, "rb") - c = pickle.load(f) - f.close() - if verbose > 0: - print("Done.") - print("Root:", "\n ".join(c.roots)) - return c - - -class Checker: - - checkext = CHECKEXT - verbose = VERBOSE - maxpage = MAXPAGE - roundsize = ROUNDSIZE - nonames = NONAMES - - validflags = tuple(dir()) - - def __init__(self): - self.reset() - - def setflags(self, **kw): - for key in kw: - if key not in self.validflags: - raise NameError("invalid keyword argument: %s" % str(key)) - for key, value in kw.items(): - setattr(self, key, value) - - def reset(self): - self.roots = [] - self.todo = {} - self.done = {} - self.bad = {} - - # Add a name table, so that the name URLs can be checked. Also - # serves as an implicit cache for which URLs are done. - self.name_table = {} - - self.round = 0 - # The following are not pickled: - self.robots = {} - self.errors = {} - self.urlopener = MyURLopener() - self.changed = 0 - - def note(self, level, format, *args): - if self.verbose > level: - if args: - format = format%args - self.message(format) - - def message(self, format, *args): - if args: - format = format%args - print(format) - - def __getstate__(self): - return (self.roots, self.todo, self.done, self.bad, self.round) - - def __setstate__(self, state): - self.reset() - (self.roots, self.todo, self.done, self.bad, self.round) = state - for root in self.roots: - self.addrobot(root) - for url in self.bad: - self.markerror(url) - - def addroot(self, root, add_to_do = 1): - if root not in self.roots: - troot = root - scheme, netloc, path, params, query, fragment = \ - urlparse.urlparse(root) - i = path.rfind("/") + 1 - if 0 < i < len(path): - path = path[:i] - troot = urlparse.urlunparse((scheme, netloc, path, - params, query, fragment)) - self.roots.append(troot) - self.addrobot(root) - if add_to_do: - self.newlink((root, ""), ("<root>", root)) - - def addrobot(self, root): - root = urlparse.urljoin(root, "/") - if root in self.robots: return - url = urlparse.urljoin(root, "/robots.txt") - self.robots[root] = rp = robotparser.RobotFileParser() - self.note(2, "Parsing %s", url) - rp.debug = self.verbose > 3 - rp.set_url(url) - try: - rp.read() - except (OSError, IOError) as msg: - self.note(1, "I/O error parsing %s: %s", url, msg) - - def run(self): - while self.todo: - self.round = self.round + 1 - self.note(0, "\nRound %d (%s)\n", self.round, self.status()) - urls = sorted(self.todo.keys()) - del urls[self.roundsize:] - for url in urls: - self.dopage(url) - - def status(self): - return "%d total, %d to do, %d done, %d bad" % ( - len(self.todo)+len(self.done), - len(self.todo), len(self.done), - len(self.bad)) - - def report(self): - self.message("") - if not self.todo: s = "Final" - else: s = "Interim" - self.message("%s Report (%s)", s, self.status()) - self.report_errors() - - def report_errors(self): - if not self.bad: - self.message("\nNo errors") - return - self.message("\nError Report:") - sources = sorted(self.errors.keys()) - for source in sources: - triples = self.errors[source] - self.message("") - if len(triples) > 1: - self.message("%d Errors in %s", len(triples), source) - else: - self.message("Error in %s", source) - # Call self.format_url() instead of referring - # to the URL directly, since the URLs in these - # triples is now a (URL, fragment) pair. The value - # of the "source" variable comes from the list of - # origins, and is a URL, not a pair. - for url, rawlink, msg in triples: - if rawlink != self.format_url(url): s = " (%s)" % rawlink - else: s = "" - self.message(" HREF %s%s\n msg %s", - self.format_url(url), s, msg) - - def dopage(self, url_pair): - - # All printing of URLs uses format_url(); argument changed to - # url_pair for clarity. - if self.verbose > 1: - if self.verbose > 2: - self.show("Check ", self.format_url(url_pair), - " from", self.todo[url_pair]) - else: - self.message("Check %s", self.format_url(url_pair)) - url, local_fragment = url_pair - if local_fragment and self.nonames: - self.markdone(url_pair) - return - try: - page = self.getpage(url_pair) - except sgmllib.SGMLParseError as msg: - msg = self.sanitize(msg) - self.note(0, "Error parsing %s: %s", - self.format_url(url_pair), msg) - # Dont actually mark the URL as bad - it exists, just - # we can't parse it! - page = None - if page: - # Store the page which corresponds to this URL. - self.name_table[url] = page - # If there is a fragment in this url_pair, and it's not - # in the list of names for the page, call setbad(), since - # it's a missing anchor. - if local_fragment and local_fragment not in page.getnames(): - self.setbad(url_pair, ("Missing name anchor `%s'" % local_fragment)) - for info in page.getlinkinfos(): - # getlinkinfos() now returns the fragment as well, - # and we store that fragment here in the "todo" dictionary. - link, rawlink, fragment = info - # However, we don't want the fragment as the origin, since - # the origin is logically a page. - origin = url, rawlink - self.newlink((link, fragment), origin) - else: - # If no page has been created yet, we want to - # record that fact. - self.name_table[url_pair[0]] = None - self.markdone(url_pair) - - def newlink(self, url, origin): - if url in self.done: - self.newdonelink(url, origin) - else: - self.newtodolink(url, origin) - - def newdonelink(self, url, origin): - if origin not in self.done[url]: - self.done[url].append(origin) - - # Call self.format_url(), since the URL here - # is now a (URL, fragment) pair. - self.note(3, " Done link %s", self.format_url(url)) - - # Make sure that if it's bad, that the origin gets added. - if url in self.bad: - source, rawlink = origin - triple = url, rawlink, self.bad[url] - self.seterror(source, triple) - - def newtodolink(self, url, origin): - # Call self.format_url(), since the URL here - # is now a (URL, fragment) pair. - if url in self.todo: - if origin not in self.todo[url]: - self.todo[url].append(origin) - self.note(3, " Seen todo link %s", self.format_url(url)) - else: - self.todo[url] = [origin] - self.note(3, " New todo link %s", self.format_url(url)) - - def format_url(self, url): - link, fragment = url - if fragment: return link + "#" + fragment - else: return link - - def markdone(self, url): - self.done[url] = self.todo[url] - del self.todo[url] - self.changed = 1 - - def inroots(self, url): - for root in self.roots: - if url[:len(root)] == root: - return self.isallowed(root, url) - return 0 - - def isallowed(self, root, url): - root = urlparse.urljoin(root, "/") - return self.robots[root].can_fetch(AGENTNAME, url) - - def getpage(self, url_pair): - # Incoming argument name is a (URL, fragment) pair. - # The page may have been cached in the name_table variable. - url, fragment = url_pair - if url in self.name_table: - return self.name_table[url] - - scheme, path = urllib.request.splittype(url) - if scheme in ('mailto', 'news', 'javascript', 'telnet'): - self.note(1, " Not checking %s URL" % scheme) - return None - isint = self.inroots(url) - - # Ensure that openpage gets the URL pair to - # print out its error message and record the error pair - # correctly. - if not isint: - if not self.checkext: - self.note(1, " Not checking ext link") - return None - f = self.openpage(url_pair) - if f: - self.safeclose(f) - return None - text, nurl = self.readhtml(url_pair) - - if nurl != url: - self.note(1, " Redirected to %s", nurl) - url = nurl - if text: - return Page(text, url, maxpage=self.maxpage, checker=self) - - # These next three functions take (URL, fragment) pairs as - # arguments, so that openpage() receives the appropriate tuple to - # record error messages. - def readhtml(self, url_pair): - url, fragment = url_pair - text = None - f, url = self.openhtml(url_pair) - if f: - text = f.read() - f.close() - return text, url - - def openhtml(self, url_pair): - url, fragment = url_pair - f = self.openpage(url_pair) - if f: - url = f.geturl() - info = f.info() - if not self.checkforhtml(info, url): - self.safeclose(f) - f = None - return f, url - - def openpage(self, url_pair): - url, fragment = url_pair - try: - return self.urlopener.open(url) - except (OSError, IOError) as msg: - msg = self.sanitize(msg) - self.note(0, "Error %s", msg) - if self.verbose > 0: - self.show(" HREF ", url, " from", self.todo[url_pair]) - self.setbad(url_pair, msg) - return None - - def checkforhtml(self, info, url): - if 'content-type' in info: - ctype = cgi.parse_header(info['content-type'])[0].lower() - if ';' in ctype: - # handle content-type: text/html; charset=iso8859-1 : - ctype = ctype.split(';', 1)[0].strip() - else: - if url[-1:] == "/": - return 1 - ctype, encoding = mimetypes.guess_type(url) - if ctype == 'text/html': - return 1 - else: - self.note(1, " Not HTML, mime type %s", ctype) - return 0 - - def setgood(self, url): - if url in self.bad: - del self.bad[url] - self.changed = 1 - self.note(0, "(Clear previously seen error)") - - def setbad(self, url, msg): - if url in self.bad and self.bad[url] == msg: - self.note(0, "(Seen this error before)") - return - self.bad[url] = msg - self.changed = 1 - self.markerror(url) - - def markerror(self, url): - try: - origins = self.todo[url] - except KeyError: - origins = self.done[url] - for source, rawlink in origins: - triple = url, rawlink, self.bad[url] - self.seterror(source, triple) - - def seterror(self, url, triple): - try: - # Because of the way the URLs are now processed, I need to - # check to make sure the URL hasn't been entered in the - # error list. The first element of the triple here is a - # (URL, fragment) pair, but the URL key is not, since it's - # from the list of origins. - if triple not in self.errors[url]: - self.errors[url].append(triple) - except KeyError: - self.errors[url] = [triple] - - # The following used to be toplevel functions; they have been - # changed into methods so they can be overridden in subclasses. - - def show(self, p1, link, p2, origins): - self.message("%s %s", p1, link) - i = 0 - for source, rawlink in origins: - i = i+1 - if i == 2: - p2 = ' '*len(p2) - if rawlink != link: s = " (%s)" % rawlink - else: s = "" - self.message("%s %s%s", p2, source, s) - - def sanitize(self, msg): - if isinstance(IOError, ClassType) and isinstance(msg, IOError): - # Do the other branch recursively - msg.args = self.sanitize(msg.args) - elif isinstance(msg, TupleType): - if len(msg) >= 4 and msg[0] == 'http error' and \ - isinstance(msg[3], InstanceType): - # Remove the Message instance -- it may contain - # a file object which prevents pickling. - msg = msg[:3] + msg[4:] - return msg - - def safeclose(self, f): - try: - url = f.geturl() - except AttributeError: - pass - else: - if url[:4] == 'ftp:' or url[:7] == 'file://': - # Apparently ftp connections don't like to be closed - # prematurely... - text = f.read() - f.close() - - def save_pickle(self, dumpfile=DUMPFILE): - if not self.changed: - self.note(0, "\nNo need to save checkpoint") - elif not dumpfile: - self.note(0, "No dumpfile, won't save checkpoint") - else: - self.note(0, "\nSaving checkpoint to %s ...", dumpfile) - newfile = dumpfile + ".new" - f = open(newfile, "wb") - pickle.dump(self, f) - f.close() - try: - os.unlink(dumpfile) - except os.error: - pass - os.rename(newfile, dumpfile) - self.note(0, "Done.") - return 1 - - -class Page: - - def __init__(self, text, url, verbose=VERBOSE, maxpage=MAXPAGE, checker=None): - self.text = text - self.url = url - self.verbose = verbose - self.maxpage = maxpage - self.checker = checker - - # The parsing of the page is done in the __init__() routine in - # order to initialize the list of names the file - # contains. Stored the parser in an instance variable. Passed - # the URL to MyHTMLParser(). - size = len(self.text) - if size > self.maxpage: - self.note(0, "Skip huge file %s (%.0f Kbytes)", self.url, (size*0.001)) - self.parser = None - return - self.checker.note(2, " Parsing %s (%d bytes)", self.url, size) - self.parser = MyHTMLParser(url, verbose=self.verbose, - checker=self.checker) - self.parser.feed(self.text) - self.parser.close() - - def note(self, level, msg, *args): - if self.checker: - self.checker.note(level, msg, *args) - else: - if self.verbose >= level: - if args: - msg = msg%args - print(msg) - - # Method to retrieve names. - def getnames(self): - if self.parser: - return self.parser.names - else: - return [] - - def getlinkinfos(self): - # File reading is done in __init__() routine. Store parser in - # local variable to indicate success of parsing. - - # If no parser was stored, fail. - if not self.parser: return [] - - rawlinks = self.parser.getlinks() - base = urlparse.urljoin(self.url, self.parser.getbase() or "") - infos = [] - for rawlink in rawlinks: - t = urlparse.urlparse(rawlink) - # DON'T DISCARD THE FRAGMENT! Instead, include - # it in the tuples which are returned. See Checker.dopage(). - fragment = t[-1] - t = t[:-1] + ('',) - rawlink = urlparse.urlunparse(t) - link = urlparse.urljoin(base, rawlink) - infos.append((link, rawlink, fragment)) - - return infos - - -class MyStringIO(io.StringIO): - - def __init__(self, url, info): - self.__url = url - self.__info = info - super(MyStringIO, self).__init__(self) - - def info(self): - return self.__info - - def geturl(self): - return self.__url - - -class MyURLopener(urllib.request.FancyURLopener): - - http_error_default = urllib.request.URLopener.http_error_default - - def __init__(*args): - self = args[0] - urllib.request.FancyURLopener.__init__(*args) - self.addheaders = [ - ('User-agent', 'Python-webchecker/%s' % __version__), - ] - - def http_error_401(self, url, fp, errcode, errmsg, headers): - return None - - def open_file(self, url): - path = urllib.url2pathname(urllib.unquote(url)) - if os.path.isdir(path): - if path[-1] != os.sep: - url = url + '/' - indexpath = os.path.join(path, "index.html") - if os.path.exists(indexpath): - return self.open_file(url + "index.html") - try: - names = os.listdir(path) - except os.error as msg: - exc_type, exc_value, exc_tb = sys.exc_info() - raise IOError(msg).with_traceback(exc_tb) - names.sort() - s = MyStringIO("file:"+url, {'content-type': 'text/html'}) - s.write('<BASE HREF="file:%s">\n' % - urllib.quote(os.path.join(path, ""))) - for name in names: - q = urllib.quote(name) - s.write('<A HREF="%s">%s</A>\n' % (q, q)) - s.seek(0) - return s - return urllib.request.FancyURLopener.open_file(self, url) - - -class MyHTMLParser(sgmllib.SGMLParser): - - def __init__(self, url, verbose=VERBOSE, checker=None): - self.myverbose = verbose # now unused - self.checker = checker - self.base = None - self.links = {} - self.names = [] - self.url = url - sgmllib.SGMLParser.__init__(self) - - def check_name_id(self, attributes): - """ Check the name or id attributes on an element. - """ - # We must rescue the NAME or id (name is deprecated in XHTML) - # attributes from the anchor, in order to - # cache the internal anchors which are made - # available in the page. - for name, value in attributes: - if name == "name" or name == "id": - if value in self.names: - self.checker.message("WARNING: duplicate ID name %s in %s", - value, self.url) - else: self.names.append(value) - break - - def unknown_starttag(self, tag, attributes): - """ In XHTML, you can have id attributes on any element. - """ - self.check_name_id(attributes) - - def start_a(self, attributes): - self.link_attr(attributes, 'href') - self.check_name_id(attributes) - - def end_a(self): pass - - def do_area(self, attributes): - self.link_attr(attributes, 'href') - self.check_name_id(attributes) - - def do_body(self, attributes): - self.link_attr(attributes, 'background', 'bgsound') - self.check_name_id(attributes) - - def do_img(self, attributes): - self.link_attr(attributes, 'src', 'lowsrc') - self.check_name_id(attributes) - - def do_frame(self, attributes): - self.link_attr(attributes, 'src', 'longdesc') - self.check_name_id(attributes) - - def do_iframe(self, attributes): - self.link_attr(attributes, 'src', 'longdesc') - self.check_name_id(attributes) - - def do_link(self, attributes): - for name, value in attributes: - if name == "rel": - parts = value.lower().split() - if ( parts == ["stylesheet"] - or parts == ["alternate", "stylesheet"]): - self.link_attr(attributes, "href") - break - self.check_name_id(attributes) - - def do_object(self, attributes): - self.link_attr(attributes, 'data', 'usemap') - self.check_name_id(attributes) - - def do_script(self, attributes): - self.link_attr(attributes, 'src') - self.check_name_id(attributes) - - def do_table(self, attributes): - self.link_attr(attributes, 'background') - self.check_name_id(attributes) - - def do_td(self, attributes): - self.link_attr(attributes, 'background') - self.check_name_id(attributes) - - def do_th(self, attributes): - self.link_attr(attributes, 'background') - self.check_name_id(attributes) - - def do_tr(self, attributes): - self.link_attr(attributes, 'background') - self.check_name_id(attributes) - - def link_attr(self, attributes, *args): - for name, value in attributes: - if name in args: - if value: value = value.strip() - if value: self.links[value] = None - - def do_base(self, attributes): - for name, value in attributes: - if name == 'href': - if value: value = value.strip() - if value: - if self.checker: - self.checker.note(1, " Base %s", value) - self.base = value - self.check_name_id(attributes) - - def getlinks(self): - return list(self.links.keys()) - - def getbase(self): - return self.base - - -if __name__ == '__main__': - main() diff --git a/Tools/webchecker/websucker.py b/Tools/webchecker/websucker.py deleted file mode 100755 index 3bbdec3..0000000 --- a/Tools/webchecker/websucker.py +++ /dev/null @@ -1,125 +0,0 @@ -#! /usr/bin/env python - -"""A variant on webchecker that creates a mirror copy of a remote site.""" - -__version__ = "$Revision$" - -import os -import sys -import getopt -import urllib.parse - -import webchecker - -# Extract real version number if necessary -if __version__[0] == '$': - _v = __version__.split() - if len(_v) == 3: - __version__ = _v[1] - -def main(): - verbose = webchecker.VERBOSE - try: - opts, args = getopt.getopt(sys.argv[1:], "qv") - except getopt.error as msg: - print(msg) - print("usage:", sys.argv[0], "[-qv] ... [rooturl] ...") - return 2 - for o, a in opts: - if o == "-q": - verbose = 0 - if o == "-v": - verbose = verbose + 1 - c = Sucker() - c.setflags(verbose=verbose) - c.urlopener.addheaders = [ - ('User-agent', 'websucker/%s' % __version__), - ] - for arg in args: - print("Adding root", arg) - c.addroot(arg) - print("Run...") - c.run() - -class Sucker(webchecker.Checker): - - checkext = 0 - nonames = 1 - - # SAM 11/13/99: in general, URLs are now URL pairs. - # Since we've suppressed name anchor checking, - # we can ignore the second dimension. - - def readhtml(self, url_pair): - url = url_pair[0] - text = None - path = self.savefilename(url) - try: - f = open(path, "rb") - except IOError: - f = self.openpage(url_pair) - if f: - info = f.info() - nurl = f.geturl() - if nurl != url: - url = nurl - path = self.savefilename(url) - text = f.read() - f.close() - self.savefile(text, path) - if not self.checkforhtml(info, url): - text = None - else: - if self.checkforhtml({}, url): - text = f.read() - f.close() - return text, url - - def savefile(self, text, path): - dir, base = os.path.split(path) - makedirs(dir) - try: - f = open(path, "wb") - f.write(text) - f.close() - self.message("saved %s", path) - except IOError as msg: - self.message("didn't save %s: %s", path, str(msg)) - - def savefilename(self, url): - type, rest = urllib.parse.splittype(url) - host, path = urllib.parse.splithost(rest) - path = path.lstrip("/") - user, host = urllib.parse.splituser(host) - host, port = urllib.parse.splitnport(host) - host = host.lower() - if not path or path[-1] == "/": - path = path + "index.html" - if os.sep != "/": - path = os.sep.join(path.split("/")) - if os.name == "mac": - path = os.sep + path - path = os.path.join(host, path) - return path - -def makedirs(dir): - if not dir: - return - if os.path.exists(dir): - if not os.path.isdir(dir): - try: - os.rename(dir, dir + ".bak") - os.mkdir(dir) - os.rename(dir + ".bak", os.path.join(dir, "index.html")) - except os.error: - pass - return - head, tail = os.path.split(dir) - if not tail: - print("Huh? Don't know how to make dir", dir) - return - makedirs(head) - os.mkdir(dir, 0o777) - -if __name__ == '__main__': - sys.exit(main() or 0) diff --git a/Tools/webchecker/wsgui.py b/Tools/webchecker/wsgui.py deleted file mode 100755 index b2223c4..0000000 --- a/Tools/webchecker/wsgui.py +++ /dev/null @@ -1,240 +0,0 @@ -#! /usr/bin/env python - -"""Tkinter-based GUI for websucker. - -Easy use: type or paste source URL and destination directory in -their respective text boxes, click GO or hit return, and presto. -""" - -from Tkinter import * -import websucker -import os -import threading -import queue -import time - -VERBOSE = 2 - - -try: - class Canceled(Exception): - "Exception used to cancel run()." -except (NameError, TypeError): - Canceled = __name__ + ".Canceled" - - -class SuckerThread(websucker.Sucker): - - stopit = 0 - savedir = None - rootdir = None - - def __init__(self, msgq): - self.msgq = msgq - websucker.Sucker.__init__(self) - self.setflags(verbose=VERBOSE) - self.urlopener.addheaders = [ - ('User-agent', 'websucker/%s' % websucker.__version__), - ] - - def message(self, format, *args): - if args: - format = format%args - ##print format - self.msgq.put(format) - - def run1(self, url): - try: - try: - self.reset() - self.addroot(url) - self.run() - except Canceled: - self.message("[canceled]") - else: - self.message("[done]") - finally: - self.msgq.put(None) - - def savefile(self, text, path): - if self.stopit: - raise Canceled - websucker.Sucker.savefile(self, text, path) - - def getpage(self, url): - if self.stopit: - raise Canceled - return websucker.Sucker.getpage(self, url) - - def savefilename(self, url): - path = websucker.Sucker.savefilename(self, url) - if self.savedir: - n = len(self.rootdir) - if path[:n] == self.rootdir: - path = path[n:] - while path[:1] == os.sep: - path = path[1:] - path = os.path.join(self.savedir, path) - return path - - def XXXaddrobot(self, *args): - pass - - def XXXisallowed(self, *args): - return 1 - - -class App: - - sucker = None - msgq = None - - def __init__(self, top): - self.top = top - top.columnconfigure(99, weight=1) - self.url_label = Label(top, text="URL:") - self.url_label.grid(row=0, column=0, sticky='e') - self.url_entry = Entry(top, width=60, exportselection=0) - self.url_entry.grid(row=0, column=1, sticky='we', - columnspan=99) - self.url_entry.focus_set() - self.url_entry.bind("<Key-Return>", self.go) - self.dir_label = Label(top, text="Directory:") - self.dir_label.grid(row=1, column=0, sticky='e') - self.dir_entry = Entry(top) - self.dir_entry.grid(row=1, column=1, sticky='we', - columnspan=99) - self.go_button = Button(top, text="Go", command=self.go) - self.go_button.grid(row=2, column=1, sticky='w') - self.cancel_button = Button(top, text="Cancel", - command=self.cancel, - state=DISABLED) - self.cancel_button.grid(row=2, column=2, sticky='w') - self.auto_button = Button(top, text="Paste+Go", - command=self.auto) - self.auto_button.grid(row=2, column=3, sticky='w') - self.status_label = Label(top, text="[idle]") - self.status_label.grid(row=2, column=4, sticky='w') - self.top.update_idletasks() - self.top.grid_propagate(0) - - def message(self, text, *args): - if args: - text = text % args - self.status_label.config(text=text) - - def check_msgq(self): - while not self.msgq.empty(): - msg = self.msgq.get() - if msg is None: - self.go_button.configure(state=NORMAL) - self.auto_button.configure(state=NORMAL) - self.cancel_button.configure(state=DISABLED) - if self.sucker: - self.sucker.stopit = 0 - self.top.bell() - else: - self.message(msg) - self.top.after(100, self.check_msgq) - - def go(self, event=None): - if not self.msgq: - self.msgq = queue.Queue(0) - self.check_msgq() - if not self.sucker: - self.sucker = SuckerThread(self.msgq) - if self.sucker.stopit: - return - self.url_entry.selection_range(0, END) - url = self.url_entry.get() - url = url.strip() - if not url: - self.top.bell() - self.message("[Error: No URL entered]") - return - self.rooturl = url - dir = self.dir_entry.get().strip() - if not dir: - self.sucker.savedir = None - else: - self.sucker.savedir = dir - self.sucker.rootdir = os.path.dirname( - websucker.Sucker.savefilename(self.sucker, url)) - self.go_button.configure(state=DISABLED) - self.auto_button.configure(state=DISABLED) - self.cancel_button.configure(state=NORMAL) - self.message( '[running...]') - self.sucker.stopit = 0 - t = threading.Thread(target=self.sucker.run1, args=(url,)) - t.start() - - def cancel(self): - if self.sucker: - self.sucker.stopit = 1 - self.message("[canceling...]") - - def auto(self): - tries = ['PRIMARY', 'CLIPBOARD'] - text = "" - for t in tries: - try: - text = self.top.selection_get(selection=t) - except TclError: - continue - text = text.strip() - if text: - break - if not text: - self.top.bell() - self.message("[Error: clipboard is empty]") - return - self.url_entry.delete(0, END) - self.url_entry.insert(0, text) - self.go() - - -class AppArray: - - def __init__(self, top=None): - if not top: - top = Tk() - top.title("websucker GUI") - top.iconname("wsgui") - top.wm_protocol('WM_DELETE_WINDOW', self.exit) - self.top = top - self.appframe = Frame(self.top) - self.appframe.pack(fill='both') - self.applist = [] - self.exit_button = Button(top, text="Exit", command=self.exit) - self.exit_button.pack(side=RIGHT) - self.new_button = Button(top, text="New", command=self.addsucker) - self.new_button.pack(side=LEFT) - self.addsucker() - ##self.applist[0].url_entry.insert(END, "http://www.python.org/doc/essays/") - - def addsucker(self): - self.top.geometry("") - frame = Frame(self.appframe, borderwidth=2, relief=GROOVE) - frame.pack(fill='x') - app = App(frame) - self.applist.append(app) - - done = 0 - - def mainloop(self): - while not self.done: - time.sleep(0.1) - self.top.update() - - def exit(self): - for app in self.applist: - app.cancel() - app.message("[exiting...]") - self.done = 1 - - -def main(): - AppArray().mainloop() - -if __name__ == '__main__': - main() diff --git a/Tools/world/README b/Tools/world/README deleted file mode 100644 index f7bb610..0000000 --- a/Tools/world/README +++ /dev/null @@ -1,85 +0,0 @@ -world -- Print mappings between country names and DNS country codes. - -Contact: Barry Warsaw -Email: bwarsaw@python.org - -This script will take a list of Internet addresses and print out where in the -world those addresses originate from, based on the top-level domain country -code found in the address. Addresses can be in any of the following forms: - - xx -- just the country code or top-level domain identifier - host.domain.xx -- any Internet host or network name - somebody@where.xx -- an Internet email address - -If no match is found, the address is interpreted as a regular expression [*] -and a reverse lookup is attempted. This script will search the country names -and print a list of matching entries. You can force reverse mappings with the -`-r' flag (see below). - -For example: - - %% world tz us - tz originated from Tanzania, United Republic of - us originated from United States - - %% world united - united matches 6 countries: - ae: United Arab Emirates - uk: United Kingdom (common practice) - um: United States Minor Outlying Islands - us: United States - tz: Tanzania, United Republic of - gb: United Kingdom - - - [*] Note that regular expressions must conform to Python 1.5's re.py module - syntax. The comparison is done with the search() method. - -Country codes are maintained by the RIPE Network Coordination Centre, -in coordination with the ISO 3166 Maintenance Agency at DIN Berlin. The -authoritative source of counry code mappings is: - - <url:ftp://info.ripe.net/iso3166-countrycodes> - -The latest known change to this information was: - - Thu Aug 7 17:59:51 MET DST 1997 - -This script also knows about non-geographic top-level domains. - -Usage: world [-d] [-p file] [-o] [-h] addr [addr ...] - - --dump - -d - Print mapping of all top-level domains. - - --parse file - -p file - Parse an iso3166-countrycodes file extracting the two letter country - code followed by the country name. Note that the three letter country - codes and numbers, which are also provided in the standard format - file, are ignored. - - --outputdict - -o - When used in conjunction with the `-p' option, output is in the form - of a Python dictionary, and country names are normalized - w.r.t. capitalization. This makes it appropriate for cutting and - pasting back into this file. - - --reverse - -r - Force reverse lookup. In this mode the address can be any Python - regular expression; this is matched against all country names and a - list of matching mappings is printed. In normal mode (e.g. without - this flag), reverse lookup is performed on addresses if no matching - country code is found. - - -h - --help - Print this message. - - -Local Variables: -indent-tabs-mode: nil -End: diff --git a/Tools/world/world b/Tools/world/world deleted file mode 100755 index 16452f3..0000000 --- a/Tools/world/world +++ /dev/null @@ -1,568 +0,0 @@ -#! /usr/bin/env python - -"""world -- Print mappings between country names and DNS country codes. - -Contact: Barry Warsaw -Email: barry@python.org -Version: %(__version__)s - -This script will take a list of Internet addresses and print out where in the -world those addresses originate from, based on the top-level domain country -code found in the address. Addresses can be in any of the following forms: - - xx -- just the country code or top-level domain identifier - host.domain.xx -- any Internet host or network name - somebody@where.xx -- an Internet email address - -If no match is found, the address is interpreted as a regular expression and a -reverse lookup is attempted. This script will search the country names and -print a list of matching entries. You can force reverse mappings with the -`-r' flag (see below). - -For example: - - %% world tz us - tz originated from Tanzania, United Republic of - us originated from United States - - %% world united - united matches 6 countries: - ae: United Arab Emirates - uk: United Kingdom (common practice) - um: United States Minor Outlying Islands - us: United States - tz: Tanzania, United Republic of - gb: United Kingdom - -Country codes are maintained by the RIPE Network Coordination Centre, -in coordination with the ISO 3166 Maintenance Agency at DIN Berlin. The -authoritative source of country code mappings is: - - <url:ftp://ftp.ripe.net/iso3166-countrycodes.txt> - -The latest known change to this information was: - - Monday, 10 October 2006, 17:59:51 UTC 2006 - -This script also knows about non-geographic top-level domains, and the -additional ccTLDs reserved by IANA. - -Usage: %(PROGRAM)s [-d] [-p file] [-o] [-h] addr [addr ...] - - --dump - -d - Print mapping of all top-level domains. - - --parse file - -p file - Parse an iso3166-countrycodes file extracting the two letter country - code followed by the country name. Note that the three letter country - codes and numbers, which are also provided in the standard format - file, are ignored. - - --outputdict - -o - When used in conjunction with the `-p' option, output is in the form - of a Python dictionary, and country names are normalized - w.r.t. capitalization. This makes it appropriate for cutting and - pasting back into this file. Output is always to standard out. - - --reverse - -r - Force reverse lookup. In this mode the address can be any Python - regular expression; this is matched against all country names and a - list of matching mappings is printed. In normal mode (e.g. without - this flag), reverse lookup is performed on addresses if no matching - country code is found. - - -h - --help - Print this message. -""" -__version__ = '$Revision$' - - -import sys -import getopt -import re - -PROGRAM = sys.argv[0] - - - -def usage(code, msg=''): - print(__doc__ % globals()) - if msg: - print(msg) - sys.exit(code) - - - -def resolve(rawaddr): - parts = rawaddr.split('.') - if not len(parts): - # no top level domain found, bounce it to the next step - return rawaddr - addr = parts[-1] - if addr in nameorgs: - print(rawaddr, 'is in the', nameorgs[addr], 'top level domain') - return None - elif addr in countries: - print(rawaddr, 'originated from', countries[addr]) - return None - else: - # Not resolved, bounce it to the next step - return rawaddr - - - -def reverse(regexp): - matches = [] - cre = re.compile(regexp, re.IGNORECASE) - for code, country in all.items(): - mo = cre.search(country) - if mo: - matches.append(code) - # print results - if not matches: - # not resolved, bounce it to the next step - return regexp - if len(matches) == 1: - code = matches[0] - print(regexp, "matches code `%s', %s" % (code, all[code])) - else: - print(regexp, 'matches %d countries:' % len(matches)) - for code in matches: - print(" %s: %s" % (code, all[code])) - return None - - - -def parse(file, normalize): - try: - fp = open(file) - except IOError as err: - errno, msg = err.args - print(msg, ':', file) - return - - cre = re.compile('(.*?)[ \t]+([A-Z]{2})[ \t]+[A-Z]{3}[ \t]+[0-9]{3}') - scanning = 0 - - if normalize: - print('countries = {') - - while 1: - line = fp.readline() - if line == '': - break # EOF - if scanning: - mo = cre.match(line) - if not mo: - line = line.strip() - if not line: - continue - elif line[0] == '-': - break - else: - print('Could not parse line:', line) - continue - country, code = mo.group(1, 2) - if normalize: - words = country.split() - for i in range(len(words)): - w = words[i] - # XXX special cases - if w in ('AND', 'OF', 'OF)', 'name:', 'METROPOLITAN'): - words[i] = w.lower() - elif w == 'THE' and i != 1: - words[i] = w.lower() - elif len(w) > 3 and w[1] == "'": - words[i] = w[0:3].upper() + w[3:].lower() - elif w in ('(U.S.)', 'U.S.'): - pass - elif w[0] == '(' and w != '(local': - words[i] = '(' + w[1:].capitalize() - elif w.find('-') != -1: - words[i] = '-'.join( - [s.capitalize() for s in w.split('-')]) - else: - words[i] = w.capitalize() - code = code.lower() - country = ' '.join(words) - print(' "%s": "%s",' % (code, country)) - else: - print(code, country) - - elif line[0] == '-': - scanning = 1 - - if normalize: - print(' }') - - -def main(): - help = 0 - status = 0 - dump = 0 - parsefile = None - normalize = 0 - forcerev = 0 - - try: - opts, args = getopt.getopt( - sys.argv[1:], - 'p:rohd', - ['parse=', 'reverse', 'outputdict', 'help', 'dump']) - except getopt.error as msg: - usage(1, msg) - - for opt, arg in opts: - if opt in ('-h', '--help'): - help = 1 - elif opt in ('-d', '--dump'): - dump = 1 - elif opt in ('-p', '--parse'): - parsefile = arg - elif opt in ('-o', '--outputdict'): - normalize = 1 - elif opt in ('-r', '--reverse'): - forcerev = 1 - - if help: - usage(status) - - if dump: - print('Official country coded domains:') - codes = sorted(countries) - for code in codes: - print(' %2s:' % code, countries[code]) - - print('\nOther top-level domains:') - codes = sorted(nameorgs) - for code in codes: - print(' %6s:' % code, nameorgs[code]) - elif parsefile: - parse(parsefile, normalize) - else: - if not forcerev: - args = filter(None, map(resolve, args)) - args = filter(None, map(reverse, args)) - for arg in args: - print('Where in the world is %s?' % arg) - - - -# The mappings -nameorgs = { - # New top level domains as described by ICANN - # http://www.icann.org/tlds/ - "aero": "air-transport industry", - "asia": "from Asia/for Asia", - "arpa": "Arpanet", - "biz": "business", - "cat": "Catalan community", - "com": "commercial", - "coop": "cooperatives", - "edu": "educational", - "gov": "government", - "info": "unrestricted `info'", - "int": "international", - "jobs": "employment-related", - "mil": "military", - "mobi": "mobile specific", - "museum": "museums", - "name": "`name' (for registration by individuals)", - "net": "networking", - "org": "non-commercial", - "pro": "professionals", - "tel": "business telecommunications", - "travel": "travel and tourism", - # These additional ccTLDs are included here even though they are not part - # of ISO 3166. IANA has a decoding table listing all reserved ccTLDs: - # - # http://www.iso.org/iso/iso-3166-1_decoding_table - # - # Note that `uk' is the common practice country code for the United - # Kingdom. AFAICT, the official `gb' code is routinely ignored! - # - # <D.M.Pick@qmw.ac.uk> tells me that `uk' was long in use before ISO3166 - # was adopted for top-level DNS zone names (although in the reverse order - # like uk.ac.qmw) and was carried forward (with the reversal) to avoid a - # large-scale renaming process as the UK switched from their old `Coloured - # Book' protocols over X.25 to Internet protocols over IP. - # - # See <url:ftp://ftp.ripe.net/ripe/docs/ripe-159.txt> - # - # Also, `su', while obsolete is still in limited use. - "ac": "Ascension Island", - "cp": "Clipperton Island", - "dg": "Diego Garcia", - "ea": "Ceuta, Melilla", - "eu": "European Union", - "fx": "Metropolitan France", - "ic": "Canary Islands", - "ta": "Tristan da Cunha", - "uk": "United Kingdom (common practice)", - "su": "Soviet Union (still in limited use)", - } - - - -countries = { - "af": "Afghanistan", - "ax": "Aland Islands", - "al": "Albania", - "dz": "Algeria", - "as": "American Samoa", - "ad": "Andorra", - "ao": "Angola", - "ai": "Anguilla", - "aq": "Antarctica", - "ag": "Antigua and Barbuda", - "ar": "Argentina", - "am": "Armenia", - "aw": "Aruba", - "au": "Australia", - "at": "Austria", - "az": "Azerbaijan", - "bs": "Bahamas", - "bh": "Bahrain", - "bd": "Bangladesh", - "bb": "Barbados", - "by": "Belarus", - "be": "Belgium", - "bz": "Belize", - "bj": "Benin", - "bm": "Bermuda", - "bt": "Bhutan", - "bo": "Bolivia", - "ba": "Bosnia and Herzegovina", - "bw": "Botswana", - "bv": "Bouvet Island", - "br": "Brazil", - "io": "British Indian Ocean Territory", - "bn": "Brunei Darussalam", - "bg": "Bulgaria", - "bf": "Burkina Faso", - "bi": "Burundi", - "kh": "Cambodia", - "cm": "Cameroon", - "ca": "Canada", - "cv": "Cape Verde", - "ky": "Cayman Islands", - "cf": "Central African Republic", - "td": "Chad", - "cl": "Chile", - "cn": "China", - "cx": "Christmas Island", - "cc": "Cocos (Keeling) Islands", - "co": "Colombia", - "km": "Comoros", - "cg": "Congo", - "cd": "Congo, The Democratic Republic of the", - "ck": "Cook Islands", - "cr": "Costa Rica", - "ci": "Cote D'Ivoire", - "hr": "Croatia", - "cu": "Cuba", - "cy": "Cyprus", - "cz": "Czech Republic", - "dk": "Denmark", - "dj": "Djibouti", - "dm": "Dominica", - "do": "Dominican Republic", - "ec": "Ecuador", - "eg": "Egypt", - "sv": "El Salvador", - "gq": "Equatorial Guinea", - "er": "Eritrea", - "ee": "Estonia", - "et": "Ethiopia", - "fk": "Falkland Islands (Malvinas)", - "fo": "Faroe Islands", - "fj": "Fiji", - "fi": "Finland", - "fr": "France", - "gf": "French Guiana", - "pf": "French Polynesia", - "tf": "French Southern Territories", - "ga": "Gabon", - "gm": "Gambia", - "ge": "Georgia", - "de": "Germany", - "gh": "Ghana", - "gi": "Gibraltar", - "gr": "Greece", - "gl": "Greenland", - "gd": "Grenada", - "gp": "Guadeloupe", - "gu": "Guam", - "gt": "Guatemala", - "gg": "Guernsey", - "gn": "Guinea", - "gw": "Guinea-Bissau", - "gy": "Guyana", - "ht": "Haiti", - "hm": "Heard Island and Mcdonald Islands", - "va": "Holy See (Vatican City State)", - "hn": "Honduras", - "hk": "Hong Kong", - "hu": "Hungary", - "is": "Iceland", - "in": "India", - "id": "Indonesia", - "ir": "Iran (Islamic Republic of)", - "iq": "Iraq", - "ie": "Ireland", - "im": "Isle of Man", - "il": "Israel", - "it": "Italy", - "jm": "Jamaica", - "jp": "Japan", - "je": "Jersey", - "jo": "Jordan", - "kz": "Kazakhstan", - "ke": "Kenya", - "ki": "Kiribati", - "kp": "Korea, Democratic People's Republic of", - "kr": "Korea, Republic of", - "kw": "Kuwait", - "kg": "Kyrgyzstan", - "la": "Lao People's Democratic Republic", - "lv": "Latvia", - "lb": "Lebanon", - "ls": "Lesotho", - "lr": "Liberia", - "ly": "Libyan Arab Jamahiriya", - "li": "Liechtenstein", - "lt": "Lithuania", - "lu": "Luxembourg", - "mo": "Macao", - "mk": "Macedonia, The Former Yugoslav Republic of", - "mg": "Madagascar", - "mw": "Malawi", - "my": "Malaysia", - "mv": "Maldives", - "ml": "Mali", - "mt": "Malta", - "mh": "Marshall Islands", - "mq": "Martinique", - "mr": "Mauritania", - "mu": "Mauritius", - "yt": "Mayotte", - "mx": "Mexico", - "fm": "Micronesia, Federated States of", - "md": "Moldova, Republic of", - "mc": "Monaco", - "mn": "Mongolia", - "me": "Montenegro", - "ms": "Montserrat", - "ma": "Morocco", - "mz": "Mozambique", - "mm": "Myanmar", - "na": "Namibia", - "nr": "Nauru", - "np": "Nepal", - "nl": "Netherlands", - "an": "Netherlands Antilles", - "nc": "New Caledonia", - "nz": "New Zealand", - "ni": "Nicaragua", - "ne": "Niger", - "ng": "Nigeria", - "nu": "Niue", - "nf": "Norfolk Island", - "mp": "Northern Mariana Islands", - "no": "Norway", - "om": "Oman", - "pk": "Pakistan", - "pw": "Palau", - "ps": "Palestinian Territory, Occupied", - "pa": "Panama", - "pg": "Papua New Guinea", - "py": "Paraguay", - "pe": "Peru", - "ph": "Philippines", - "pn": "Pitcairn", - "pl": "Poland", - "pt": "Portugal", - "pr": "Puerto Rico", - "qa": "Qatar", - "re": "Reunion", - "ro": "Romania", - "ru": "Russian Federation", - "rw": "Rwanda", - "sh": "Saint Helena", - "kn": "Saint Kitts and Nevis", - "lc": "Saint Lucia", - "pm": "Saint Pierre and Miquelon", - "vc": "Saint Vincent and the Grenadines", - "ws": "Samoa", - "sm": "San Marino", - "st": "Sao Tome and Principe", - "sa": "Saudi Arabia", - "sn": "Senegal", - "rs": "Serbia", - "sc": "Seychelles", - "sl": "Sierra Leone", - "sg": "Singapore", - "sk": "Slovakia", - "si": "Slovenia", - "sb": "Solomon Islands", - "so": "Somalia", - "za": "South Africa", - "gs": "South Georgia and the South Sandwich Islands", - "es": "Spain", - "lk": "Sri Lanka", - "sd": "Sudan", - "sr": "Suriname", - "sj": "Svalbard and Jan Mayen", - "sh": "St. Helena", - "pm": "St. Pierre and Miquelon", - "sz": "Swaziland", - "se": "Sweden", - "ch": "Switzerland", - "sy": "Syrian Arab Republic", - "tw": "Taiwan, Province of China", - "tj": "Tajikistan", - "tz": "Tanzania, United Republic of", - "th": "Thailand", - "tl": "Timor-Leste", - "tg": "Togo", - "tk": "Tokelau", - "to": "Tonga", - "tt": "Trinidad and Tobago", - "tn": "Tunisia", - "tr": "Turkey", - "tm": "Turkmenistan", - "tc": "Turks and Caicos Islands", - "tv": "Tuvalu", - "ug": "Uganda", - "ua": "Ukraine", - "ae": "United Arab Emirates", - "gb": "United Kingdom", - "us": "United States", - "um": "United States Minor Outlying Islands", - "uy": "Uruguay", - "uz": "Uzbekistan", - "vu": "Vanuatu", - "va": "Vatican City State (Holy See)", - "ve": "Venezuela", - "vn": "Viet Nam", - "vg": "Virgin Islands (British)", - "vi": "Virgin Islands (U.S.)", - "wf": "Wallis and Futuna", - "eh": "Western Sahara", - "ye": "Yemen", - "yu": "Yugoslavia", - "zm": "Zambia", - "zw": "Zimbabwe", - } - -all = nameorgs.copy() -all.update(countries) - - -if __name__ == '__main__': - main() |