diff options
author | Victor Stinner <vstinner@python.org> | 2021-11-15 23:29:17 (GMT) |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-11-15 23:29:17 (GMT) |
commit | 9bf2cbc4c498812e14f20d86acb61c53928a5a57 (patch) | |
tree | 3967d6f8ac254b8ed51ee8ab26eb94cb593223f9 /Lib/test/support | |
parent | 3352834f59531dfa42dbef00ada4fb95ded2ae3a (diff) | |
download | cpython-9bf2cbc4c498812e14f20d86acb61c53928a5a57.zip cpython-9bf2cbc4c498812e14f20d86acb61c53928a5a57.tar.gz cpython-9bf2cbc4c498812e14f20d86acb61c53928a5a57.tar.bz2 |
bpo-28533: Remove asyncore, asynchat, smtpd modules (GH-29521)
Remove the asyncore and asynchat modules, deprecated in Python
3.6: use the asyncio module instead.
Remove the smtpd module, deprecated in Python 3.6: the aiosmtpd
module can be used instead, it is based on asyncio.
* Remove asyncore, asynchat and smtpd documentation
* Remove test_asyncore, test_asynchat and test_smtpd
* Rename Lib/asynchat.py to Lib/test/support/_asynchat.py
* Rename Lib/asyncore.py to Lib/test/support/_asyncore.py
* Rename Lib/smtpd.py to Lib/test/support/_smtpd.py
* Remove DeprecationWarning from private _asyncore, _asynchat and
_smtpd modules
* _smtpd: remove deprecated properties
Diffstat (limited to 'Lib/test/support')
-rw-r--r-- | Lib/test/support/_asynchat.py | 307 | ||||
-rw-r--r-- | Lib/test/support/_asyncore.py | 643 | ||||
-rwxr-xr-x | Lib/test/support/_smtpd.py | 754 |
3 files changed, 1704 insertions, 0 deletions
diff --git a/Lib/test/support/_asynchat.py b/Lib/test/support/_asynchat.py new file mode 100644 index 0000000..941cc1d --- /dev/null +++ b/Lib/test/support/_asynchat.py @@ -0,0 +1,307 @@ +# -*- Mode: Python; tab-width: 4 -*- +# Id: asynchat.py,v 2.26 2000/09/07 22:29:26 rushing Exp +# Author: Sam Rushing <rushing@nightmare.com> + +# ====================================================================== +# Copyright 1996 by Sam Rushing +# +# All Rights Reserved +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose and without fee is hereby +# granted, provided that the above copyright notice appear in all +# copies and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of Sam +# Rushing not be used in advertising or publicity pertaining to +# distribution of the software without specific, written prior +# permission. +# +# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN +# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# ====================================================================== + +r"""A class supporting chat-style (command/response) protocols. + +This class adds support for 'chat' style protocols - where one side +sends a 'command', and the other sends a response (examples would be +the common internet protocols - smtp, nntp, ftp, etc..). + +The handle_read() method looks at the input stream for the current +'terminator' (usually '\r\n' for single-line responses, '\r\n.\r\n' +for multi-line output), calling self.found_terminator() on its +receipt. + +for example: +Say you build an async nntp client using this class. At the start +of the connection, you'll have self.terminator set to '\r\n', in +order to process the single-line greeting. Just before issuing a +'LIST' command you'll set it to '\r\n.\r\n'. The output of the LIST +command will be accumulated (using your own 'collect_incoming_data' +method) up to the terminator, and then control will be returned to +you - by calling your self.found_terminator() method. +""" +from test.support import _asyncore as asyncore +from collections import deque + + +class async_chat(asyncore.dispatcher): + """This is an abstract class. You must derive from this class, and add + the two methods collect_incoming_data() and found_terminator()""" + + # these are overridable defaults + + ac_in_buffer_size = 65536 + ac_out_buffer_size = 65536 + + # we don't want to enable the use of encoding by default, because that is a + # sign of an application bug that we don't want to pass silently + + use_encoding = 0 + encoding = 'latin-1' + + def __init__(self, sock=None, map=None): + # for string terminator matching + self.ac_in_buffer = b'' + + # we use a list here rather than io.BytesIO for a few reasons... + # del lst[:] is faster than bio.truncate(0) + # lst = [] is faster than bio.truncate(0) + self.incoming = [] + + # we toss the use of the "simple producer" and replace it with + # a pure deque, which the original fifo was a wrapping of + self.producer_fifo = deque() + asyncore.dispatcher.__init__(self, sock, map) + + def collect_incoming_data(self, data): + raise NotImplementedError("must be implemented in subclass") + + def _collect_incoming_data(self, data): + self.incoming.append(data) + + def _get_data(self): + d = b''.join(self.incoming) + del self.incoming[:] + return d + + def found_terminator(self): + raise NotImplementedError("must be implemented in subclass") + + def set_terminator(self, term): + """Set the input delimiter. + + Can be a fixed string of any length, an integer, or None. + """ + if isinstance(term, str) and self.use_encoding: + term = bytes(term, self.encoding) + elif isinstance(term, int) and term < 0: + raise ValueError('the number of received bytes must be positive') + self.terminator = term + + def get_terminator(self): + return self.terminator + + # grab some more data from the socket, + # throw it to the collector method, + # check for the terminator, + # if found, transition to the next state. + + def handle_read(self): + + try: + data = self.recv(self.ac_in_buffer_size) + except BlockingIOError: + return + except OSError: + self.handle_error() + return + + if isinstance(data, str) and self.use_encoding: + data = bytes(str, self.encoding) + self.ac_in_buffer = self.ac_in_buffer + data + + # Continue to search for self.terminator in self.ac_in_buffer, + # while calling self.collect_incoming_data. The while loop + # is necessary because we might read several data+terminator + # combos with a single recv(4096). + + while self.ac_in_buffer: + lb = len(self.ac_in_buffer) + terminator = self.get_terminator() + if not terminator: + # no terminator, collect it all + self.collect_incoming_data(self.ac_in_buffer) + self.ac_in_buffer = b'' + elif isinstance(terminator, int): + # numeric terminator + n = terminator + if lb < n: + self.collect_incoming_data(self.ac_in_buffer) + self.ac_in_buffer = b'' + self.terminator = self.terminator - lb + else: + self.collect_incoming_data(self.ac_in_buffer[:n]) + self.ac_in_buffer = self.ac_in_buffer[n:] + self.terminator = 0 + self.found_terminator() + else: + # 3 cases: + # 1) end of buffer matches terminator exactly: + # collect data, transition + # 2) end of buffer matches some prefix: + # collect data to the prefix + # 3) end of buffer does not match any prefix: + # collect data + terminator_len = len(terminator) + index = self.ac_in_buffer.find(terminator) + if index != -1: + # we found the terminator + if index > 0: + # don't bother reporting the empty string + # (source of subtle bugs) + self.collect_incoming_data(self.ac_in_buffer[:index]) + self.ac_in_buffer = self.ac_in_buffer[index+terminator_len:] + # This does the Right Thing if the terminator + # is changed here. + self.found_terminator() + else: + # check for a prefix of the terminator + index = find_prefix_at_end(self.ac_in_buffer, terminator) + if index: + if index != lb: + # we found a prefix, collect up to the prefix + self.collect_incoming_data(self.ac_in_buffer[:-index]) + self.ac_in_buffer = self.ac_in_buffer[-index:] + break + else: + # no prefix, collect it all + self.collect_incoming_data(self.ac_in_buffer) + self.ac_in_buffer = b'' + + def handle_write(self): + self.initiate_send() + + def handle_close(self): + self.close() + + def push(self, data): + if not isinstance(data, (bytes, bytearray, memoryview)): + raise TypeError('data argument must be byte-ish (%r)', + type(data)) + sabs = self.ac_out_buffer_size + if len(data) > sabs: + for i in range(0, len(data), sabs): + self.producer_fifo.append(data[i:i+sabs]) + else: + self.producer_fifo.append(data) + self.initiate_send() + + def push_with_producer(self, producer): + self.producer_fifo.append(producer) + self.initiate_send() + + def readable(self): + "predicate for inclusion in the readable for select()" + # cannot use the old predicate, it violates the claim of the + # set_terminator method. + + # return (len(self.ac_in_buffer) <= self.ac_in_buffer_size) + return 1 + + def writable(self): + "predicate for inclusion in the writable for select()" + return self.producer_fifo or (not self.connected) + + def close_when_done(self): + "automatically close this channel once the outgoing queue is empty" + self.producer_fifo.append(None) + + def initiate_send(self): + while self.producer_fifo and self.connected: + first = self.producer_fifo[0] + # handle empty string/buffer or None entry + if not first: + del self.producer_fifo[0] + if first is None: + self.handle_close() + return + + # handle classic producer behavior + obs = self.ac_out_buffer_size + try: + data = first[:obs] + except TypeError: + data = first.more() + if data: + self.producer_fifo.appendleft(data) + else: + del self.producer_fifo[0] + continue + + if isinstance(data, str) and self.use_encoding: + data = bytes(data, self.encoding) + + # send the data + try: + num_sent = self.send(data) + except OSError: + self.handle_error() + return + + if num_sent: + if num_sent < len(data) or obs < len(first): + self.producer_fifo[0] = first[num_sent:] + else: + del self.producer_fifo[0] + # we tried to send some actual data + return + + def discard_buffers(self): + # Emergencies only! + self.ac_in_buffer = b'' + del self.incoming[:] + self.producer_fifo.clear() + + +class simple_producer: + + def __init__(self, data, buffer_size=512): + self.data = data + self.buffer_size = buffer_size + + def more(self): + if len(self.data) > self.buffer_size: + result = self.data[:self.buffer_size] + self.data = self.data[self.buffer_size:] + return result + else: + result = self.data + self.data = b'' + return result + + +# Given 'haystack', see if any prefix of 'needle' is at its end. This +# assumes an exact match has already been checked. Return the number of +# characters matched. +# for example: +# f_p_a_e("qwerty\r", "\r\n") => 1 +# f_p_a_e("qwertydkjf", "\r\n") => 0 +# f_p_a_e("qwerty\r\n", "\r\n") => <undefined> + +# this could maybe be made faster with a computed regex? +# [answer: no; circa Python-2.0, Jan 2001] +# new python: 28961/s +# old python: 18307/s +# re: 12820/s +# regex: 14035/s + +def find_prefix_at_end(haystack, needle): + l = len(needle) - 1 + while l and not haystack.endswith(needle[:l]): + l -= 1 + return l diff --git a/Lib/test/support/_asyncore.py b/Lib/test/support/_asyncore.py new file mode 100644 index 0000000..7863efa --- /dev/null +++ b/Lib/test/support/_asyncore.py @@ -0,0 +1,643 @@ +# -*- Mode: Python -*- +# Id: asyncore.py,v 2.51 2000/09/07 22:29:26 rushing Exp +# Author: Sam Rushing <rushing@nightmare.com> + +# ====================================================================== +# Copyright 1996 by Sam Rushing +# +# All Rights Reserved +# +# Permission to use, copy, modify, and distribute this software and +# its documentation for any purpose and without fee is hereby +# granted, provided that the above copyright notice appear in all +# copies and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of Sam +# Rushing not be used in advertising or publicity pertaining to +# distribution of the software without specific, written prior +# permission. +# +# SAM RUSHING DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, +# INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN +# NO EVENT SHALL SAM RUSHING BE LIABLE FOR ANY SPECIAL, INDIRECT OR +# CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS +# OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, +# NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN +# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. +# ====================================================================== + +"""Basic infrastructure for asynchronous socket service clients and servers. + +There are only two ways to have a program on a single processor do "more +than one thing at a time". Multi-threaded programming is the simplest and +most popular way to do it, but there is another very different technique, +that lets you have nearly all the advantages of multi-threading, without +actually using multiple threads. it's really only practical if your program +is largely I/O bound. If your program is CPU bound, then pre-emptive +scheduled threads are probably what you really need. Network servers are +rarely CPU-bound, however. + +If your operating system supports the select() system call in its I/O +library (and nearly all do), then you can use it to juggle multiple +communication channels at once; doing other work while your I/O is taking +place in the "background." Although this strategy can seem strange and +complex, especially at first, it is in many ways easier to understand and +control than multi-threaded programming. The module documented here solves +many of the difficult problems for you, making the task of building +sophisticated high-performance network servers and clients a snap. +""" + +import select +import socket +import sys +import time +import warnings + +import os +from errno import EALREADY, EINPROGRESS, EWOULDBLOCK, ECONNRESET, EINVAL, \ + ENOTCONN, ESHUTDOWN, EISCONN, EBADF, ECONNABORTED, EPIPE, EAGAIN, \ + errorcode + + +_DISCONNECTED = frozenset({ECONNRESET, ENOTCONN, ESHUTDOWN, ECONNABORTED, EPIPE, + EBADF}) + +try: + socket_map +except NameError: + socket_map = {} + +def _strerror(err): + try: + return os.strerror(err) + except (ValueError, OverflowError, NameError): + if err in errorcode: + return errorcode[err] + return "Unknown error %s" %err + +class ExitNow(Exception): + pass + +_reraised_exceptions = (ExitNow, KeyboardInterrupt, SystemExit) + +def read(obj): + try: + obj.handle_read_event() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def write(obj): + try: + obj.handle_write_event() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def _exception(obj): + try: + obj.handle_expt_event() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def readwrite(obj, flags): + try: + if flags & select.POLLIN: + obj.handle_read_event() + if flags & select.POLLOUT: + obj.handle_write_event() + if flags & select.POLLPRI: + obj.handle_expt_event() + if flags & (select.POLLHUP | select.POLLERR | select.POLLNVAL): + obj.handle_close() + except OSError as e: + if e.errno not in _DISCONNECTED: + obj.handle_error() + else: + obj.handle_close() + except _reraised_exceptions: + raise + except: + obj.handle_error() + +def poll(timeout=0.0, map=None): + if map is None: + map = socket_map + if map: + r = []; w = []; e = [] + for fd, obj in list(map.items()): + is_r = obj.readable() + is_w = obj.writable() + if is_r: + r.append(fd) + # accepting sockets should not be writable + if is_w and not obj.accepting: + w.append(fd) + if is_r or is_w: + e.append(fd) + if [] == r == w == e: + time.sleep(timeout) + return + + r, w, e = select.select(r, w, e, timeout) + + for fd in r: + obj = map.get(fd) + if obj is None: + continue + read(obj) + + for fd in w: + obj = map.get(fd) + if obj is None: + continue + write(obj) + + for fd in e: + obj = map.get(fd) + if obj is None: + continue + _exception(obj) + +def poll2(timeout=0.0, map=None): + # Use the poll() support added to the select module in Python 2.0 + if map is None: + map = socket_map + if timeout is not None: + # timeout is in milliseconds + timeout = int(timeout*1000) + pollster = select.poll() + if map: + for fd, obj in list(map.items()): + flags = 0 + if obj.readable(): + flags |= select.POLLIN | select.POLLPRI + # accepting sockets should not be writable + if obj.writable() and not obj.accepting: + flags |= select.POLLOUT + if flags: + pollster.register(fd, flags) + + r = pollster.poll(timeout) + for fd, flags in r: + obj = map.get(fd) + if obj is None: + continue + readwrite(obj, flags) + +poll3 = poll2 # Alias for backward compatibility + +def loop(timeout=30.0, use_poll=False, map=None, count=None): + if map is None: + map = socket_map + + if use_poll and hasattr(select, 'poll'): + poll_fun = poll2 + else: + poll_fun = poll + + if count is None: + while map: + poll_fun(timeout, map) + + else: + while map and count > 0: + poll_fun(timeout, map) + count = count - 1 + +class dispatcher: + + debug = False + connected = False + accepting = False + connecting = False + closing = False + addr = None + ignore_log_types = frozenset({'warning'}) + + def __init__(self, sock=None, map=None): + if map is None: + self._map = socket_map + else: + self._map = map + + self._fileno = None + + if sock: + # Set to nonblocking just to make sure for cases where we + # get a socket from a blocking source. + sock.setblocking(False) + self.set_socket(sock, map) + self.connected = True + # The constructor no longer requires that the socket + # passed be connected. + try: + self.addr = sock.getpeername() + except OSError as err: + if err.errno in (ENOTCONN, EINVAL): + # To handle the case where we got an unconnected + # socket. + self.connected = False + else: + # The socket is broken in some unknown way, alert + # the user and remove it from the map (to prevent + # polling of broken sockets). + self.del_channel(map) + raise + else: + self.socket = None + + def __repr__(self): + status = [self.__class__.__module__+"."+self.__class__.__qualname__] + if self.accepting and self.addr: + status.append('listening') + elif self.connected: + status.append('connected') + if self.addr is not None: + try: + status.append('%s:%d' % self.addr) + except TypeError: + status.append(repr(self.addr)) + return '<%s at %#x>' % (' '.join(status), id(self)) + + def add_channel(self, map=None): + #self.log_info('adding channel %s' % self) + if map is None: + map = self._map + map[self._fileno] = self + + def del_channel(self, map=None): + fd = self._fileno + if map is None: + map = self._map + if fd in map: + #self.log_info('closing channel %d:%s' % (fd, self)) + del map[fd] + self._fileno = None + + def create_socket(self, family=socket.AF_INET, type=socket.SOCK_STREAM): + self.family_and_type = family, type + sock = socket.socket(family, type) + sock.setblocking(False) + self.set_socket(sock) + + def set_socket(self, sock, map=None): + self.socket = sock + self._fileno = sock.fileno() + self.add_channel(map) + + def set_reuse_addr(self): + # try to re-use a server port if possible + try: + self.socket.setsockopt( + socket.SOL_SOCKET, socket.SO_REUSEADDR, + self.socket.getsockopt(socket.SOL_SOCKET, + socket.SO_REUSEADDR) | 1 + ) + except OSError: + pass + + # ================================================== + # predicates for select() + # these are used as filters for the lists of sockets + # to pass to select(). + # ================================================== + + def readable(self): + return True + + def writable(self): + return True + + # ================================================== + # socket object methods. + # ================================================== + + def listen(self, num): + self.accepting = True + if os.name == 'nt' and num > 5: + num = 5 + return self.socket.listen(num) + + def bind(self, addr): + self.addr = addr + return self.socket.bind(addr) + + def connect(self, address): + self.connected = False + self.connecting = True + err = self.socket.connect_ex(address) + if err in (EINPROGRESS, EALREADY, EWOULDBLOCK) \ + or err == EINVAL and os.name == 'nt': + self.addr = address + return + if err in (0, EISCONN): + self.addr = address + self.handle_connect_event() + else: + raise OSError(err, errorcode[err]) + + def accept(self): + # XXX can return either an address pair or None + try: + conn, addr = self.socket.accept() + except TypeError: + return None + except OSError as why: + if why.errno in (EWOULDBLOCK, ECONNABORTED, EAGAIN): + return None + else: + raise + else: + return conn, addr + + def send(self, data): + try: + result = self.socket.send(data) + return result + except OSError as why: + if why.errno == EWOULDBLOCK: + return 0 + elif why.errno in _DISCONNECTED: + self.handle_close() + return 0 + else: + raise + + def recv(self, buffer_size): + try: + data = self.socket.recv(buffer_size) + if not data: + # a closed connection is indicated by signaling + # a read condition, and having recv() return 0. + self.handle_close() + return b'' + else: + return data + except OSError as why: + # winsock sometimes raises ENOTCONN + if why.errno in _DISCONNECTED: + self.handle_close() + return b'' + else: + raise + + def close(self): + self.connected = False + self.accepting = False + self.connecting = False + self.del_channel() + if self.socket is not None: + try: + self.socket.close() + except OSError as why: + if why.errno not in (ENOTCONN, EBADF): + raise + + # log and log_info may be overridden to provide more sophisticated + # logging and warning methods. In general, log is for 'hit' logging + # and 'log_info' is for informational, warning and error logging. + + def log(self, message): + sys.stderr.write('log: %s\n' % str(message)) + + def log_info(self, message, type='info'): + if type not in self.ignore_log_types: + print('%s: %s' % (type, message)) + + def handle_read_event(self): + if self.accepting: + # accepting sockets are never connected, they "spawn" new + # sockets that are connected + self.handle_accept() + elif not self.connected: + if self.connecting: + self.handle_connect_event() + self.handle_read() + else: + self.handle_read() + + def handle_connect_event(self): + err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if err != 0: + raise OSError(err, _strerror(err)) + self.handle_connect() + self.connected = True + self.connecting = False + + def handle_write_event(self): + if self.accepting: + # Accepting sockets shouldn't get a write event. + # We will pretend it didn't happen. + return + + if not self.connected: + if self.connecting: + self.handle_connect_event() + self.handle_write() + + def handle_expt_event(self): + # handle_expt_event() is called if there might be an error on the + # socket, or if there is OOB data + # check for the error condition first + err = self.socket.getsockopt(socket.SOL_SOCKET, socket.SO_ERROR) + if err != 0: + # we can get here when select.select() says that there is an + # exceptional condition on the socket + # since there is an error, we'll go ahead and close the socket + # like we would in a subclassed handle_read() that received no + # data + self.handle_close() + else: + self.handle_expt() + + def handle_error(self): + nil, t, v, tbinfo = compact_traceback() + + # sometimes a user repr method will crash. + try: + self_repr = repr(self) + except: + self_repr = '<__repr__(self) failed for object at %0x>' % id(self) + + self.log_info( + 'uncaptured python exception, closing channel %s (%s:%s %s)' % ( + self_repr, + t, + v, + tbinfo + ), + 'error' + ) + self.handle_close() + + def handle_expt(self): + self.log_info('unhandled incoming priority event', 'warning') + + def handle_read(self): + self.log_info('unhandled read event', 'warning') + + def handle_write(self): + self.log_info('unhandled write event', 'warning') + + def handle_connect(self): + self.log_info('unhandled connect event', 'warning') + + def handle_accept(self): + pair = self.accept() + if pair is not None: + self.handle_accepted(*pair) + + def handle_accepted(self, sock, addr): + sock.close() + self.log_info('unhandled accepted event', 'warning') + + def handle_close(self): + self.log_info('unhandled close event', 'warning') + self.close() + +# --------------------------------------------------------------------------- +# adds simple buffered output capability, useful for simple clients. +# [for more sophisticated usage use asynchat.async_chat] +# --------------------------------------------------------------------------- + +class dispatcher_with_send(dispatcher): + + def __init__(self, sock=None, map=None): + dispatcher.__init__(self, sock, map) + self.out_buffer = b'' + + def initiate_send(self): + num_sent = 0 + num_sent = dispatcher.send(self, self.out_buffer[:65536]) + self.out_buffer = self.out_buffer[num_sent:] + + def handle_write(self): + self.initiate_send() + + def writable(self): + return (not self.connected) or len(self.out_buffer) + + def send(self, data): + if self.debug: + self.log_info('sending %s' % repr(data)) + self.out_buffer = self.out_buffer + data + self.initiate_send() + +# --------------------------------------------------------------------------- +# used for debugging. +# --------------------------------------------------------------------------- + +def compact_traceback(): + t, v, tb = sys.exc_info() + tbinfo = [] + if not tb: # Must have a traceback + raise AssertionError("traceback does not exist") + while tb: + tbinfo.append(( + tb.tb_frame.f_code.co_filename, + tb.tb_frame.f_code.co_name, + str(tb.tb_lineno) + )) + tb = tb.tb_next + + # just to be safe + del tb + + file, function, line = tbinfo[-1] + info = ' '.join(['[%s|%s|%s]' % x for x in tbinfo]) + return (file, function, line), t, v, info + +def close_all(map=None, ignore_all=False): + if map is None: + map = socket_map + for x in list(map.values()): + try: + x.close() + except OSError as x: + if x.errno == EBADF: + pass + elif not ignore_all: + raise + except _reraised_exceptions: + raise + except: + if not ignore_all: + raise + map.clear() + +# Asynchronous File I/O: +# +# After a little research (reading man pages on various unixen, and +# digging through the linux kernel), I've determined that select() +# isn't meant for doing asynchronous file i/o. +# Heartening, though - reading linux/mm/filemap.c shows that linux +# supports asynchronous read-ahead. So _MOST_ of the time, the data +# will be sitting in memory for us already when we go to read it. +# +# What other OS's (besides NT) support async file i/o? [VMS?] +# +# Regardless, this is useful for pipes, and stdin/stdout... + +if os.name == 'posix': + class file_wrapper: + # Here we override just enough to make a file + # look like a socket for the purposes of asyncore. + # The passed fd is automatically os.dup()'d + + def __init__(self, fd): + self.fd = os.dup(fd) + + def __del__(self): + if self.fd >= 0: + warnings.warn("unclosed file %r" % self, ResourceWarning, + source=self) + self.close() + + def recv(self, *args): + return os.read(self.fd, *args) + + def send(self, *args): + return os.write(self.fd, *args) + + def getsockopt(self, level, optname, buflen=None): + if (level == socket.SOL_SOCKET and + optname == socket.SO_ERROR and + not buflen): + return 0 + raise NotImplementedError("Only asyncore specific behaviour " + "implemented.") + + read = recv + write = send + + def close(self): + if self.fd < 0: + return + fd = self.fd + self.fd = -1 + os.close(fd) + + def fileno(self): + return self.fd + + class file_dispatcher(dispatcher): + + def __init__(self, fd, map=None): + dispatcher.__init__(self, None, map) + self.connected = True + try: + fd = fd.fileno() + except AttributeError: + pass + self.set_file(fd) + # set it to non-blocking mode + os.set_blocking(fd, False) + + def set_file(self, fd): + self.socket = file_wrapper(fd) + self._fileno = self.socket.fileno() + self.add_channel() diff --git a/Lib/test/support/_smtpd.py b/Lib/test/support/_smtpd.py new file mode 100755 index 0000000..0e37d08 --- /dev/null +++ b/Lib/test/support/_smtpd.py @@ -0,0 +1,754 @@ +#! /usr/bin/env python3 +"""An RFC 5321 smtp proxy with optional RFC 1870 and RFC 6531 extensions. + +Usage: %(program)s [options] [localhost:localport [remotehost:remoteport]] + +Options: + + --nosetuid + -n + This program generally tries to setuid `nobody', unless this flag is + set. The setuid call will fail if this program is not run as root (in + which case, use this flag). + + --version + -V + Print the version number and exit. + + --class classname + -c classname + Use `classname' as the concrete SMTP proxy class. Uses `PureProxy' by + default. + + --size limit + -s limit + Restrict the total size of the incoming message to "limit" number of + bytes via the RFC 1870 SIZE extension. Defaults to 33554432 bytes. + + --smtputf8 + -u + Enable the SMTPUTF8 extension and behave as an RFC 6531 smtp proxy. + + --debug + -d + Turn on debugging prints. + + --help + -h + Print this message and exit. + +Version: %(__version__)s + +If localhost is not given then `localhost' is used, and if localport is not +given then 8025 is used. If remotehost is not given then `localhost' is used, +and if remoteport is not given, then 25 is used. +""" + +# Overview: +# +# This file implements the minimal SMTP protocol as defined in RFC 5321. It +# has a hierarchy of classes which implement the backend functionality for the +# smtpd. A number of classes are provided: +# +# SMTPServer - the base class for the backend. Raises NotImplementedError +# if you try to use it. +# +# DebuggingServer - simply prints each message it receives on stdout. +# +# PureProxy - Proxies all messages to a real smtpd which does final +# delivery. One known problem with this class is that it doesn't handle +# SMTP errors from the backend server at all. This should be fixed +# (contributions are welcome!). +# +# +# Author: Barry Warsaw <barry@python.org> +# +# TODO: +# +# - support mailbox delivery +# - alias files +# - Handle more ESMTP extensions +# - handle error codes from the backend smtpd + +import sys +import os +import errno +import getopt +import time +import socket +import collections +from warnings import warn +from email._header_value_parser import get_addr_spec, get_angle_addr + +from test.support import _asyncore as asyncore +from test.support import _asynchat as asynchat + +__all__ = [ + "SMTPChannel", "SMTPServer", "DebuggingServer", "PureProxy", +] + + +program = sys.argv[0] +__version__ = 'Python SMTP proxy version 0.3' + + +class Devnull: + def write(self, msg): pass + def flush(self): pass + + +DEBUGSTREAM = Devnull() +NEWLINE = '\n' +COMMASPACE = ', ' +DATA_SIZE_DEFAULT = 33554432 + + +def usage(code, msg=''): + print(__doc__ % globals(), file=sys.stderr) + if msg: + print(msg, file=sys.stderr) + sys.exit(code) + + +class SMTPChannel(asynchat.async_chat): + COMMAND = 0 + DATA = 1 + + command_size_limit = 512 + command_size_limits = collections.defaultdict(lambda x=command_size_limit: x) + + @property + def max_command_size_limit(self): + try: + return max(self.command_size_limits.values()) + except ValueError: + return self.command_size_limit + + def __init__(self, server, conn, addr, data_size_limit=DATA_SIZE_DEFAULT, + map=None, enable_SMTPUTF8=False, decode_data=False): + asynchat.async_chat.__init__(self, conn, map=map) + self.smtp_server = server + self.conn = conn + self.addr = addr + self.data_size_limit = data_size_limit + self.enable_SMTPUTF8 = enable_SMTPUTF8 + self._decode_data = decode_data + if enable_SMTPUTF8 and decode_data: + raise ValueError("decode_data and enable_SMTPUTF8 cannot" + " be set to True at the same time") + if decode_data: + self._emptystring = '' + self._linesep = '\r\n' + self._dotsep = '.' + self._newline = NEWLINE + else: + self._emptystring = b'' + self._linesep = b'\r\n' + self._dotsep = ord(b'.') + self._newline = b'\n' + self._set_rset_state() + self.seen_greeting = '' + self.extended_smtp = False + self.command_size_limits.clear() + self.fqdn = socket.getfqdn() + try: + self.peer = conn.getpeername() + except OSError as err: + # a race condition may occur if the other end is closing + # before we can get the peername + self.close() + if err.errno != errno.ENOTCONN: + raise + return + print('Peer:', repr(self.peer), file=DEBUGSTREAM) + self.push('220 %s %s' % (self.fqdn, __version__)) + + def _set_post_data_state(self): + """Reset state variables to their post-DATA state.""" + self.smtp_state = self.COMMAND + self.mailfrom = None + self.rcpttos = [] + self.require_SMTPUTF8 = False + self.num_bytes = 0 + self.set_terminator(b'\r\n') + + def _set_rset_state(self): + """Reset all state variables except the greeting.""" + self._set_post_data_state() + self.received_data = '' + self.received_lines = [] + + + # Overrides base class for convenience. + def push(self, msg): + asynchat.async_chat.push(self, bytes( + msg + '\r\n', 'utf-8' if self.require_SMTPUTF8 else 'ascii')) + + # Implementation of base class abstract method + def collect_incoming_data(self, data): + limit = None + if self.smtp_state == self.COMMAND: + limit = self.max_command_size_limit + elif self.smtp_state == self.DATA: + limit = self.data_size_limit + if limit and self.num_bytes > limit: + return + elif limit: + self.num_bytes += len(data) + if self._decode_data: + self.received_lines.append(str(data, 'utf-8')) + else: + self.received_lines.append(data) + + # Implementation of base class abstract method + def found_terminator(self): + line = self._emptystring.join(self.received_lines) + print('Data:', repr(line), file=DEBUGSTREAM) + self.received_lines = [] + if self.smtp_state == self.COMMAND: + sz, self.num_bytes = self.num_bytes, 0 + if not line: + self.push('500 Error: bad syntax') + return + if not self._decode_data: + line = str(line, 'utf-8') + i = line.find(' ') + if i < 0: + command = line.upper() + arg = None + else: + command = line[:i].upper() + arg = line[i+1:].strip() + max_sz = (self.command_size_limits[command] + if self.extended_smtp else self.command_size_limit) + if sz > max_sz: + self.push('500 Error: line too long') + return + method = getattr(self, 'smtp_' + command, None) + if not method: + self.push('500 Error: command "%s" not recognized' % command) + return + method(arg) + return + else: + if self.smtp_state != self.DATA: + self.push('451 Internal confusion') + self.num_bytes = 0 + return + if self.data_size_limit and self.num_bytes > self.data_size_limit: + self.push('552 Error: Too much mail data') + self.num_bytes = 0 + return + # Remove extraneous carriage returns and de-transparency according + # to RFC 5321, Section 4.5.2. + data = [] + for text in line.split(self._linesep): + if text and text[0] == self._dotsep: + data.append(text[1:]) + else: + data.append(text) + self.received_data = self._newline.join(data) + args = (self.peer, self.mailfrom, self.rcpttos, self.received_data) + kwargs = {} + if not self._decode_data: + kwargs = { + 'mail_options': self.mail_options, + 'rcpt_options': self.rcpt_options, + } + status = self.smtp_server.process_message(*args, **kwargs) + self._set_post_data_state() + if not status: + self.push('250 OK') + else: + self.push(status) + + # SMTP and ESMTP commands + def smtp_HELO(self, arg): + if not arg: + self.push('501 Syntax: HELO hostname') + return + # See issue #21783 for a discussion of this behavior. + if self.seen_greeting: + self.push('503 Duplicate HELO/EHLO') + return + self._set_rset_state() + self.seen_greeting = arg + self.push('250 %s' % self.fqdn) + + def smtp_EHLO(self, arg): + if not arg: + self.push('501 Syntax: EHLO hostname') + return + # See issue #21783 for a discussion of this behavior. + if self.seen_greeting: + self.push('503 Duplicate HELO/EHLO') + return + self._set_rset_state() + self.seen_greeting = arg + self.extended_smtp = True + self.push('250-%s' % self.fqdn) + if self.data_size_limit: + self.push('250-SIZE %s' % self.data_size_limit) + self.command_size_limits['MAIL'] += 26 + if not self._decode_data: + self.push('250-8BITMIME') + if self.enable_SMTPUTF8: + self.push('250-SMTPUTF8') + self.command_size_limits['MAIL'] += 10 + self.push('250 HELP') + + def smtp_NOOP(self, arg): + if arg: + self.push('501 Syntax: NOOP') + else: + self.push('250 OK') + + def smtp_QUIT(self, arg): + # args is ignored + self.push('221 Bye') + self.close_when_done() + + def _strip_command_keyword(self, keyword, arg): + keylen = len(keyword) + if arg[:keylen].upper() == keyword: + return arg[keylen:].strip() + return '' + + def _getaddr(self, arg): + if not arg: + return '', '' + if arg.lstrip().startswith('<'): + address, rest = get_angle_addr(arg) + else: + address, rest = get_addr_spec(arg) + if not address: + return address, rest + return address.addr_spec, rest + + def _getparams(self, params): + # Return params as dictionary. Return None if not all parameters + # appear to be syntactically valid according to RFC 1869. + result = {} + for param in params: + param, eq, value = param.partition('=') + if not param.isalnum() or eq and not value: + return None + result[param] = value if eq else True + return result + + def smtp_HELP(self, arg): + if arg: + extended = ' [SP <mail-parameters>]' + lc_arg = arg.upper() + if lc_arg == 'EHLO': + self.push('250 Syntax: EHLO hostname') + elif lc_arg == 'HELO': + self.push('250 Syntax: HELO hostname') + elif lc_arg == 'MAIL': + msg = '250 Syntax: MAIL FROM: <address>' + if self.extended_smtp: + msg += extended + self.push(msg) + elif lc_arg == 'RCPT': + msg = '250 Syntax: RCPT TO: <address>' + if self.extended_smtp: + msg += extended + self.push(msg) + elif lc_arg == 'DATA': + self.push('250 Syntax: DATA') + elif lc_arg == 'RSET': + self.push('250 Syntax: RSET') + elif lc_arg == 'NOOP': + self.push('250 Syntax: NOOP') + elif lc_arg == 'QUIT': + self.push('250 Syntax: QUIT') + elif lc_arg == 'VRFY': + self.push('250 Syntax: VRFY <address>') + else: + self.push('501 Supported commands: EHLO HELO MAIL RCPT ' + 'DATA RSET NOOP QUIT VRFY') + else: + self.push('250 Supported commands: EHLO HELO MAIL RCPT DATA ' + 'RSET NOOP QUIT VRFY') + + def smtp_VRFY(self, arg): + if arg: + address, params = self._getaddr(arg) + if address: + self.push('252 Cannot VRFY user, but will accept message ' + 'and attempt delivery') + else: + self.push('502 Could not VRFY %s' % arg) + else: + self.push('501 Syntax: VRFY <address>') + + def smtp_MAIL(self, arg): + if not self.seen_greeting: + self.push('503 Error: send HELO first') + return + print('===> MAIL', arg, file=DEBUGSTREAM) + syntaxerr = '501 Syntax: MAIL FROM: <address>' + if self.extended_smtp: + syntaxerr += ' [SP <mail-parameters>]' + if arg is None: + self.push(syntaxerr) + return + arg = self._strip_command_keyword('FROM:', arg) + address, params = self._getaddr(arg) + if not address: + self.push(syntaxerr) + return + if not self.extended_smtp and params: + self.push(syntaxerr) + return + if self.mailfrom: + self.push('503 Error: nested MAIL command') + return + self.mail_options = params.upper().split() + params = self._getparams(self.mail_options) + if params is None: + self.push(syntaxerr) + return + if not self._decode_data: + body = params.pop('BODY', '7BIT') + if body not in ['7BIT', '8BITMIME']: + self.push('501 Error: BODY can only be one of 7BIT, 8BITMIME') + return + if self.enable_SMTPUTF8: + smtputf8 = params.pop('SMTPUTF8', False) + if smtputf8 is True: + self.require_SMTPUTF8 = True + elif smtputf8 is not False: + self.push('501 Error: SMTPUTF8 takes no arguments') + return + size = params.pop('SIZE', None) + if size: + if not size.isdigit(): + self.push(syntaxerr) + return + elif self.data_size_limit and int(size) > self.data_size_limit: + self.push('552 Error: message size exceeds fixed maximum message size') + return + if len(params.keys()) > 0: + self.push('555 MAIL FROM parameters not recognized or not implemented') + return + self.mailfrom = address + print('sender:', self.mailfrom, file=DEBUGSTREAM) + self.push('250 OK') + + def smtp_RCPT(self, arg): + if not self.seen_greeting: + self.push('503 Error: send HELO first'); + return + print('===> RCPT', arg, file=DEBUGSTREAM) + if not self.mailfrom: + self.push('503 Error: need MAIL command') + return + syntaxerr = '501 Syntax: RCPT TO: <address>' + if self.extended_smtp: + syntaxerr += ' [SP <mail-parameters>]' + if arg is None: + self.push(syntaxerr) + return + arg = self._strip_command_keyword('TO:', arg) + address, params = self._getaddr(arg) + if not address: + self.push(syntaxerr) + return + if not self.extended_smtp and params: + self.push(syntaxerr) + return + self.rcpt_options = params.upper().split() + params = self._getparams(self.rcpt_options) + if params is None: + self.push(syntaxerr) + return + # XXX currently there are no options we recognize. + if len(params.keys()) > 0: + self.push('555 RCPT TO parameters not recognized or not implemented') + return + self.rcpttos.append(address) + print('recips:', self.rcpttos, file=DEBUGSTREAM) + self.push('250 OK') + + def smtp_RSET(self, arg): + if arg: + self.push('501 Syntax: RSET') + return + self._set_rset_state() + self.push('250 OK') + + def smtp_DATA(self, arg): + if not self.seen_greeting: + self.push('503 Error: send HELO first'); + return + if not self.rcpttos: + self.push('503 Error: need RCPT command') + return + if arg: + self.push('501 Syntax: DATA') + return + self.smtp_state = self.DATA + self.set_terminator(b'\r\n.\r\n') + self.push('354 End data with <CR><LF>.<CR><LF>') + + # Commands that have not been implemented + def smtp_EXPN(self, arg): + self.push('502 EXPN not implemented') + + +class SMTPServer(asyncore.dispatcher): + # SMTPChannel class to use for managing client connections + channel_class = SMTPChannel + + def __init__(self, localaddr, remoteaddr, + data_size_limit=DATA_SIZE_DEFAULT, map=None, + enable_SMTPUTF8=False, decode_data=False): + self._localaddr = localaddr + self._remoteaddr = remoteaddr + self.data_size_limit = data_size_limit + self.enable_SMTPUTF8 = enable_SMTPUTF8 + self._decode_data = decode_data + if enable_SMTPUTF8 and decode_data: + raise ValueError("decode_data and enable_SMTPUTF8 cannot" + " be set to True at the same time") + asyncore.dispatcher.__init__(self, map=map) + try: + gai_results = socket.getaddrinfo(*localaddr, + type=socket.SOCK_STREAM) + self.create_socket(gai_results[0][0], gai_results[0][1]) + # try to re-use a server port if possible + self.set_reuse_addr() + self.bind(localaddr) + self.listen(5) + except: + self.close() + raise + else: + print('%s started at %s\n\tLocal addr: %s\n\tRemote addr:%s' % ( + self.__class__.__name__, time.ctime(time.time()), + localaddr, remoteaddr), file=DEBUGSTREAM) + + def handle_accepted(self, conn, addr): + print('Incoming connection from %s' % repr(addr), file=DEBUGSTREAM) + channel = self.channel_class(self, + conn, + addr, + self.data_size_limit, + self._map, + self.enable_SMTPUTF8, + self._decode_data) + + # API for "doing something useful with the message" + def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): + """Override this abstract method to handle messages from the client. + + peer is a tuple containing (ipaddr, port) of the client that made the + socket connection to our smtp port. + + mailfrom is the raw address the client claims the message is coming + from. + + rcpttos is a list of raw addresses the client wishes to deliver the + message to. + + data is a string containing the entire full text of the message, + headers (if supplied) and all. It has been `de-transparencied' + according to RFC 821, Section 4.5.2. In other words, a line + containing a `.' followed by other text has had the leading dot + removed. + + kwargs is a dictionary containing additional information. It is + empty if decode_data=True was given as init parameter, otherwise + it will contain the following keys: + 'mail_options': list of parameters to the mail command. All + elements are uppercase strings. Example: + ['BODY=8BITMIME', 'SMTPUTF8']. + 'rcpt_options': same, for the rcpt command. + + This function should return None for a normal `250 Ok' response; + otherwise, it should return the desired response string in RFC 821 + format. + + """ + raise NotImplementedError + + +class DebuggingServer(SMTPServer): + + def _print_message_content(self, peer, data): + inheaders = 1 + lines = data.splitlines() + for line in lines: + # headers first + if inheaders and not line: + peerheader = 'X-Peer: ' + peer[0] + if not isinstance(data, str): + # decoded_data=false; make header match other binary output + peerheader = repr(peerheader.encode('utf-8')) + print(peerheader) + inheaders = 0 + if not isinstance(data, str): + # Avoid spurious 'str on bytes instance' warning. + line = repr(line) + print(line) + + def process_message(self, peer, mailfrom, rcpttos, data, **kwargs): + print('---------- MESSAGE FOLLOWS ----------') + if kwargs: + if kwargs.get('mail_options'): + print('mail options: %s' % kwargs['mail_options']) + if kwargs.get('rcpt_options'): + print('rcpt options: %s\n' % kwargs['rcpt_options']) + self._print_message_content(peer, data) + print('------------ END MESSAGE ------------') + + +class PureProxy(SMTPServer): + def __init__(self, *args, **kwargs): + if 'enable_SMTPUTF8' in kwargs and kwargs['enable_SMTPUTF8']: + raise ValueError("PureProxy does not support SMTPUTF8.") + super(PureProxy, self).__init__(*args, **kwargs) + + def process_message(self, peer, mailfrom, rcpttos, data): + lines = data.split('\n') + # Look for the last header + i = 0 + for line in lines: + if not line: + break + i += 1 + lines.insert(i, 'X-Peer: %s' % peer[0]) + data = NEWLINE.join(lines) + refused = self._deliver(mailfrom, rcpttos, data) + # TBD: what to do with refused addresses? + print('we got some refusals:', refused, file=DEBUGSTREAM) + + def _deliver(self, mailfrom, rcpttos, data): + import smtplib + refused = {} + try: + s = smtplib.SMTP() + s.connect(self._remoteaddr[0], self._remoteaddr[1]) + try: + refused = s.sendmail(mailfrom, rcpttos, data) + finally: + s.quit() + except smtplib.SMTPRecipientsRefused as e: + print('got SMTPRecipientsRefused', file=DEBUGSTREAM) + refused = e.recipients + except (OSError, smtplib.SMTPException) as e: + print('got', e.__class__, file=DEBUGSTREAM) + # All recipients were refused. If the exception had an associated + # error code, use it. Otherwise,fake it with a non-triggering + # exception code. + errcode = getattr(e, 'smtp_code', -1) + errmsg = getattr(e, 'smtp_error', 'ignore') + for r in rcpttos: + refused[r] = (errcode, errmsg) + return refused + + +class Options: + setuid = True + classname = 'PureProxy' + size_limit = None + enable_SMTPUTF8 = False + + +def parseargs(): + global DEBUGSTREAM + try: + opts, args = getopt.getopt( + sys.argv[1:], 'nVhc:s:du', + ['class=', 'nosetuid', 'version', 'help', 'size=', 'debug', + 'smtputf8']) + except getopt.error as e: + usage(1, e) + + options = Options() + for opt, arg in opts: + if opt in ('-h', '--help'): + usage(0) + elif opt in ('-V', '--version'): + print(__version__) + sys.exit(0) + elif opt in ('-n', '--nosetuid'): + options.setuid = False + elif opt in ('-c', '--class'): + options.classname = arg + elif opt in ('-d', '--debug'): + DEBUGSTREAM = sys.stderr + elif opt in ('-u', '--smtputf8'): + options.enable_SMTPUTF8 = True + elif opt in ('-s', '--size'): + try: + int_size = int(arg) + options.size_limit = int_size + except: + print('Invalid size: ' + arg, file=sys.stderr) + sys.exit(1) + + # parse the rest of the arguments + if len(args) < 1: + localspec = 'localhost:8025' + remotespec = 'localhost:25' + elif len(args) < 2: + localspec = args[0] + remotespec = 'localhost:25' + elif len(args) < 3: + localspec = args[0] + remotespec = args[1] + else: + usage(1, 'Invalid arguments: %s' % COMMASPACE.join(args)) + + # split into host/port pairs + i = localspec.find(':') + if i < 0: + usage(1, 'Bad local spec: %s' % localspec) + options.localhost = localspec[:i] + try: + options.localport = int(localspec[i+1:]) + except ValueError: + usage(1, 'Bad local port: %s' % localspec) + i = remotespec.find(':') + if i < 0: + usage(1, 'Bad remote spec: %s' % remotespec) + options.remotehost = remotespec[:i] + try: + options.remoteport = int(remotespec[i+1:]) + except ValueError: + usage(1, 'Bad remote port: %s' % remotespec) + return options + + +if __name__ == '__main__': + options = parseargs() + # Become nobody + classname = options.classname + if "." in classname: + lastdot = classname.rfind(".") + mod = __import__(classname[:lastdot], globals(), locals(), [""]) + classname = classname[lastdot+1:] + else: + import __main__ as mod + class_ = getattr(mod, classname) + proxy = class_((options.localhost, options.localport), + (options.remotehost, options.remoteport), + options.size_limit, enable_SMTPUTF8=options.enable_SMTPUTF8) + if options.setuid: + try: + import pwd + except ImportError: + print('Cannot import module "pwd"; try running with -n option.', file=sys.stderr) + sys.exit(1) + nobody = pwd.getpwnam('nobody')[2] + try: + os.setuid(nobody) + except PermissionError: + print('Cannot setuid "nobody"; try running with -n option.', file=sys.stderr) + sys.exit(1) + try: + asyncore.loop() + except KeyboardInterrupt: + pass |